diff --git a/arch/x86/events/amd/core.c b/arch/x86/events/amd/core.c index 305774b..8ad771f 100644 --- a/arch/x86/events/amd/core.c +++ b/arch/x86/events/amd/core.c @@ -1412,12 +1412,12 @@ static int __init amd_core_pmu_init(void) u64 even_ctr_mask = 0ULL; int i; - if (!boot_cpu_has(X86_FEATURE_PERFCTR_CORE)) - return 0; - /* Avoid calculating the value each time in the NMI handler */ perf_nmi_window = msecs_to_jiffies(100); + if (!boot_cpu_has(X86_FEATURE_PERFCTR_CORE)) + return 0; + /* * If core performance counter extensions exists, we must use * MSR_F15H_PERF_CTL/MSR_F15H_PERF_CTR msrs. See also diff --git a/arch/x86/events/amd/uncore.c b/arch/x86/events/amd/uncore.c index dd956cf..a0364ca 100644 --- a/arch/x86/events/amd/uncore.c +++ b/arch/x86/events/amd/uncore.c @@ -700,7 +700,7 @@ void amd_uncore_df_ctx_scan(struct amd_uncore *uncore, unsigned int cpu) info.split.aux_data = 0; info.split.num_pmcs = NUM_COUNTERS_NB; info.split.gid = 0; - info.split.cid = topology_logical_package_id(cpu); + info.split.cid = topology_amd_node_id(cpu); if (pmu_version >= 2) { ebx.full = cpuid_ebx(EXT_PERFMON_DEBUG_FEATURES); @@ -999,8 +999,8 @@ void amd_uncore_umc_ctx_scan(struct amd_uncore *uncore, unsigned int cpu) cpuid(EXT_PERFMON_DEBUG_FEATURES, &eax, &ebx.full, &ecx, &edx); info.split.aux_data = ecx; /* stash active mask */ info.split.num_pmcs = ebx.split.num_umc_pmc; - info.split.gid = topology_logical_package_id(cpu); - info.split.cid = topology_logical_package_id(cpu); + info.split.gid = topology_amd_node_id(cpu); + info.split.cid = topology_amd_node_id(cpu); *per_cpu_ptr(uncore->info, cpu) = info; } diff --git a/arch/x86/events/intel/core.c b/arch/x86/events/intel/core.c index 1b5ec78..6f4e0e5 100644 --- a/arch/x86/events/intel/core.c +++ b/arch/x86/events/intel/core.c @@ -215,39 +215,47 @@ static struct event_constraint intel_slm_event_constraints[] __read_mostly = static struct event_constraint intel_grt_event_constraints[] __read_mostly = { FIXED_EVENT_CONSTRAINT(0x00c0, 0), /* INST_RETIRED.ANY */ + FIXED_EVENT_CONSTRAINT(0x0100, 0), /* pseudo INST_RETIRED.ANY */ FIXED_EVENT_CONSTRAINT(0x003c, 1), /* CPU_CLK_UNHALTED.CORE */ - FIXED_EVENT_CONSTRAINT(0x0300, 2), /* pseudo CPU_CLK_UNHALTED.REF */ + FIXED_EVENT_CONSTRAINT(0x0200, 1), /* pseudo CPU_CLK_UNHALTED.THREAD */ + FIXED_EVENT_CONSTRAINT(0x0300, 2), /* pseudo CPU_CLK_UNHALTED.REF_TSC */ FIXED_EVENT_CONSTRAINT(0x013c, 2), /* CPU_CLK_UNHALTED.REF_TSC_P */ EVENT_CONSTRAINT_END }; static struct event_constraint intel_skt_event_constraints[] __read_mostly = { FIXED_EVENT_CONSTRAINT(0x00c0, 0), /* INST_RETIRED.ANY */ + FIXED_EVENT_CONSTRAINT(0x0100, 0), /* pseudo INST_RETIRED.ANY */ FIXED_EVENT_CONSTRAINT(0x003c, 1), /* CPU_CLK_UNHALTED.CORE */ - FIXED_EVENT_CONSTRAINT(0x0300, 2), /* pseudo CPU_CLK_UNHALTED.REF */ + FIXED_EVENT_CONSTRAINT(0x0200, 1), /* pseudo CPU_CLK_UNHALTED.THREAD */ + FIXED_EVENT_CONSTRAINT(0x0300, 2), /* pseudo CPU_CLK_UNHALTED.REF_TSC */ FIXED_EVENT_CONSTRAINT(0x013c, 2), /* CPU_CLK_UNHALTED.REF_TSC_P */ FIXED_EVENT_CONSTRAINT(0x0073, 4), /* TOPDOWN_BAD_SPECULATION.ALL */ + FIXED_EVENT_CONSTRAINT(0x0500, 4), /* pseudo TOPDOWN_BAD_SPECULATION.ALL */ FIXED_EVENT_CONSTRAINT(0x019c, 5), /* TOPDOWN_FE_BOUND.ALL */ + FIXED_EVENT_CONSTRAINT(0x0600, 5), /* pseudo TOPDOWN_FE_BOUND.ALL */ FIXED_EVENT_CONSTRAINT(0x02c2, 6), /* TOPDOWN_RETIRING.ALL */ + FIXED_EVENT_CONSTRAINT(0x0700, 6), /* pseudo TOPDOWN_RETIRING.ALL */ EVENT_CONSTRAINT_END }; static struct event_constraint intel_arw_event_constraints[] __read_mostly = { FIXED_EVENT_CONSTRAINT(0x00c0, 0), /* INST_RETIRED.ANY */ + FIXED_EVENT_CONSTRAINT(0x0100, 0), /* pseudo INST_RETIRED.ANY */ FIXED_EVENT_CONSTRAINT(0x003c, 1), /* CPU_CLK_UNHALTED.CORE */ - FIXED_EVENT_CONSTRAINT(0x0300, 2), /* pseudo CPU_CLK_UNHALTED.REF */ + FIXED_EVENT_CONSTRAINT(0x0200, 1), /* pseudo CPU_CLK_UNHALTED.THREAD */ + FIXED_EVENT_CONSTRAINT(0x0300, 2), /* pseudo CPU_CLK_UNHALTED.REF_TSC */ FIXED_EVENT_CONSTRAINT(0x013c, 2), /* CPU_CLK_UNHALTED.REF_TSC_P */ FIXED_EVENT_CONSTRAINT(0x0073, 4), /* TOPDOWN_BAD_SPECULATION.ALL */ + FIXED_EVENT_CONSTRAINT(0x0500, 4), /* pseudo TOPDOWN_BAD_SPECULATION.ALL */ FIXED_EVENT_CONSTRAINT(0x019c, 5), /* TOPDOWN_FE_BOUND.ALL */ + FIXED_EVENT_CONSTRAINT(0x0600, 5), /* pseudo TOPDOWN_FE_BOUND.ALL */ FIXED_EVENT_CONSTRAINT(0x02c2, 6), /* TOPDOWN_RETIRING.ALL */ + FIXED_EVENT_CONSTRAINT(0x0700, 6), /* pseudo TOPDOWN_RETIRING.ALL */ INTEL_UEVENT_CONSTRAINT(0x01b7, 0x1), INTEL_UEVENT_CONSTRAINT(0x02b7, 0x2), INTEL_UEVENT_CONSTRAINT(0x04b7, 0x4), INTEL_UEVENT_CONSTRAINT(0x08b7, 0x8), - INTEL_UEVENT_CONSTRAINT(0x01d4, 0x1), - INTEL_UEVENT_CONSTRAINT(0x02d4, 0x2), - INTEL_UEVENT_CONSTRAINT(0x04d4, 0x4), - INTEL_UEVENT_CONSTRAINT(0x08d4, 0x8), INTEL_UEVENT_CONSTRAINT(0x0175, 0x1), INTEL_UEVENT_CONSTRAINT(0x0275, 0x2), INTEL_UEVENT_CONSTRAINT(0x21d3, 0x1), @@ -310,10 +318,11 @@ static struct extra_reg intel_skl_extra_regs[] __read_mostly = { static struct event_constraint intel_icl_event_constraints[] = { FIXED_EVENT_CONSTRAINT(0x00c0, 0), /* INST_RETIRED.ANY */ FIXED_EVENT_CONSTRAINT(0x01c0, 0), /* old INST_RETIRED.PREC_DIST */ - FIXED_EVENT_CONSTRAINT(0x0100, 0), /* INST_RETIRED.PREC_DIST */ + FIXED_EVENT_CONSTRAINT(0x0100, 0), /* pseudo INST_RETIRED.ANY */ FIXED_EVENT_CONSTRAINT(0x003c, 1), /* CPU_CLK_UNHALTED.CORE */ - FIXED_EVENT_CONSTRAINT(0x0300, 2), /* CPU_CLK_UNHALTED.REF */ - FIXED_EVENT_CONSTRAINT(0x0400, 3), /* SLOTS */ + FIXED_EVENT_CONSTRAINT(0x0200, 1), /* pseudo CPU_CLK_UNHALTED.THREAD */ + FIXED_EVENT_CONSTRAINT(0x0300, 2), /* pseudo CPU_CLK_UNHALTED.REF_TSC */ + FIXED_EVENT_CONSTRAINT(0x0400, 3), /* pseudo TOPDOWN.SLOTS */ METRIC_EVENT_CONSTRAINT(INTEL_TD_METRIC_RETIRING, 0), METRIC_EVENT_CONSTRAINT(INTEL_TD_METRIC_BAD_SPEC, 1), METRIC_EVENT_CONSTRAINT(INTEL_TD_METRIC_FE_BOUND, 2), @@ -355,11 +364,12 @@ static struct extra_reg intel_glc_extra_regs[] __read_mostly = { static struct event_constraint intel_glc_event_constraints[] = { FIXED_EVENT_CONSTRAINT(0x00c0, 0), /* INST_RETIRED.ANY */ - FIXED_EVENT_CONSTRAINT(0x0100, 0), /* INST_RETIRED.PREC_DIST */ + FIXED_EVENT_CONSTRAINT(0x0100, 0), /* pseudo INST_RETIRED.ANY */ FIXED_EVENT_CONSTRAINT(0x003c, 1), /* CPU_CLK_UNHALTED.CORE */ - FIXED_EVENT_CONSTRAINT(0x0300, 2), /* CPU_CLK_UNHALTED.REF */ + FIXED_EVENT_CONSTRAINT(0x0200, 1), /* pseudo CPU_CLK_UNHALTED.THREAD */ + FIXED_EVENT_CONSTRAINT(0x0300, 2), /* pseudo CPU_CLK_UNHALTED.REF_TSC */ FIXED_EVENT_CONSTRAINT(0x013c, 2), /* CPU_CLK_UNHALTED.REF_TSC_P */ - FIXED_EVENT_CONSTRAINT(0x0400, 3), /* SLOTS */ + FIXED_EVENT_CONSTRAINT(0x0400, 3), /* pseudo TOPDOWN.SLOTS */ METRIC_EVENT_CONSTRAINT(INTEL_TD_METRIC_RETIRING, 0), METRIC_EVENT_CONSTRAINT(INTEL_TD_METRIC_BAD_SPEC, 1), METRIC_EVENT_CONSTRAINT(INTEL_TD_METRIC_FE_BOUND, 2), @@ -379,9 +389,13 @@ static struct event_constraint intel_glc_event_constraints[] = { INTEL_UEVENT_CONSTRAINT(0x01a3, 0xf), INTEL_UEVENT_CONSTRAINT(0x02a3, 0xf), + INTEL_UEVENT_CONSTRAINT(0x05a3, 0xf), + INTEL_UEVENT_CONSTRAINT(0x06a3, 0xf), INTEL_UEVENT_CONSTRAINT(0x08a3, 0xf), + INTEL_UEVENT_CONSTRAINT(0x0ca3, 0xf), INTEL_UEVENT_CONSTRAINT(0x04a4, 0x1), INTEL_UEVENT_CONSTRAINT(0x08a4, 0x1), + INTEL_UEVENT_CONSTRAINT(0x01cd, 0xfe), INTEL_UEVENT_CONSTRAINT(0x02cd, 0x1), INTEL_EVENT_CONSTRAINT(0xce, 0x1), INTEL_EVENT_CONSTRAINT_RANGE(0xd0, 0xdf, 0xf), @@ -407,11 +421,12 @@ static struct extra_reg intel_rwc_extra_regs[] __read_mostly = { static struct event_constraint intel_lnc_event_constraints[] = { FIXED_EVENT_CONSTRAINT(0x00c0, 0), /* INST_RETIRED.ANY */ - FIXED_EVENT_CONSTRAINT(0x0100, 0), /* INST_RETIRED.PREC_DIST */ + FIXED_EVENT_CONSTRAINT(0x0100, 0), /* pseudo INST_RETIRED.ANY */ FIXED_EVENT_CONSTRAINT(0x003c, 1), /* CPU_CLK_UNHALTED.CORE */ - FIXED_EVENT_CONSTRAINT(0x0300, 2), /* CPU_CLK_UNHALTED.REF */ + FIXED_EVENT_CONSTRAINT(0x0200, 1), /* pseudo CPU_CLK_UNHALTED.THREAD */ + FIXED_EVENT_CONSTRAINT(0x0300, 2), /* pseudo CPU_CLK_UNHALTED.REF_TSC */ FIXED_EVENT_CONSTRAINT(0x013c, 2), /* CPU_CLK_UNHALTED.REF_TSC_P */ - FIXED_EVENT_CONSTRAINT(0x0400, 3), /* SLOTS */ + FIXED_EVENT_CONSTRAINT(0x0400, 3), /* pseudo TOPDOWN.SLOTS */ METRIC_EVENT_CONSTRAINT(INTEL_TD_METRIC_RETIRING, 0), METRIC_EVENT_CONSTRAINT(INTEL_TD_METRIC_BAD_SPEC, 1), METRIC_EVENT_CONSTRAINT(INTEL_TD_METRIC_FE_BOUND, 2), @@ -423,8 +438,6 @@ static struct event_constraint intel_lnc_event_constraints[] = { INTEL_EVENT_CONSTRAINT(0x20, 0xf), - INTEL_UEVENT_CONSTRAINT(0x012a, 0xf), - INTEL_UEVENT_CONSTRAINT(0x012b, 0xf), INTEL_UEVENT_CONSTRAINT(0x0148, 0x4), INTEL_UEVENT_CONSTRAINT(0x0175, 0x4), @@ -435,15 +448,14 @@ static struct event_constraint intel_lnc_event_constraints[] = { INTEL_UEVENT_CONSTRAINT(0x0ca3, 0x4), INTEL_UEVENT_CONSTRAINT(0x04a4, 0x1), INTEL_UEVENT_CONSTRAINT(0x08a4, 0x1), - INTEL_UEVENT_CONSTRAINT(0x10a4, 0x1), + INTEL_UEVENT_CONSTRAINT(0x10a4, 0x8), INTEL_UEVENT_CONSTRAINT(0x01b1, 0x8), INTEL_UEVENT_CONSTRAINT(0x01cd, 0x3fc), INTEL_UEVENT_CONSTRAINT(0x02cd, 0x3), + INTEL_UEVENT_CONSTRAINT(0x87d0, 0x3ff), INTEL_EVENT_CONSTRAINT_RANGE(0xd0, 0xdf, 0xf), - INTEL_UEVENT_CONSTRAINT(0x00e0, 0xf), - EVENT_CONSTRAINT_END }; @@ -460,11 +472,12 @@ static struct extra_reg intel_lnc_extra_regs[] __read_mostly = { static struct event_constraint intel_pnc_event_constraints[] = { FIXED_EVENT_CONSTRAINT(0x00c0, 0), /* INST_RETIRED.ANY */ - FIXED_EVENT_CONSTRAINT(0x0100, 0), /* INST_RETIRED.PREC_DIST */ + FIXED_EVENT_CONSTRAINT(0x0100, 0), /* pseudo INST_RETIRED.ANY */ FIXED_EVENT_CONSTRAINT(0x003c, 1), /* CPU_CLK_UNHALTED.CORE */ - FIXED_EVENT_CONSTRAINT(0x0300, 2), /* CPU_CLK_UNHALTED.REF */ + FIXED_EVENT_CONSTRAINT(0x0200, 1), /* pseudo CPU_CLK_UNHALTED.THREAD */ + FIXED_EVENT_CONSTRAINT(0x0300, 2), /* pseudo CPU_CLK_UNHALTED.REF_TSC */ FIXED_EVENT_CONSTRAINT(0x013c, 2), /* CPU_CLK_UNHALTED.REF_TSC_P */ - FIXED_EVENT_CONSTRAINT(0x0400, 3), /* SLOTS */ + FIXED_EVENT_CONSTRAINT(0x0400, 3), /* pseudo TOPDOWN.SLOTS */ METRIC_EVENT_CONSTRAINT(INTEL_TD_METRIC_RETIRING, 0), METRIC_EVENT_CONSTRAINT(INTEL_TD_METRIC_BAD_SPEC, 1), METRIC_EVENT_CONSTRAINT(INTEL_TD_METRIC_FE_BOUND, 2), @@ -706,6 +719,80 @@ static __initconst const u64 glc_hw_cache_event_ids }, }; +/* ADL P-core (Golden cove) specific event code. */ +static __initconst const u64 adl_glc_hw_cache_event_ids + [PERF_COUNT_HW_CACHE_MAX] + [PERF_COUNT_HW_CACHE_OP_MAX] + [PERF_COUNT_HW_CACHE_RESULT_MAX] = +{ + [ C(L1D ) ] = { + [ C(OP_READ) ] = { + [ C(RESULT_ACCESS) ] = 0x81d0, + [ C(RESULT_MISS) ] = 0xe124, + }, + [ C(OP_WRITE) ] = { + [ C(RESULT_ACCESS) ] = 0x82d0, + }, + }, + [ C(L1I ) ] = { + [ C(OP_READ) ] = { + [ C(RESULT_MISS) ] = 0xe424, + }, + [ C(OP_WRITE) ] = { + [ C(RESULT_ACCESS) ] = -1, + [ C(RESULT_MISS) ] = -1, + }, + }, + [ C(LL ) ] = { + [ C(OP_READ) ] = { + [ C(RESULT_ACCESS) ] = 0x12a, + [ C(RESULT_MISS) ] = 0x12a, + }, + [ C(OP_WRITE) ] = { + [ C(RESULT_ACCESS) ] = 0x12a, + [ C(RESULT_MISS) ] = 0x12a, + }, + }, + [ C(DTLB) ] = { + [ C(OP_READ) ] = { + [ C(RESULT_ACCESS) ] = 0x81d0, + [ C(RESULT_MISS) ] = 0xe12, + }, + [ C(OP_WRITE) ] = { + [ C(RESULT_ACCESS) ] = 0x82d0, + [ C(RESULT_MISS) ] = 0xe13, + }, + }, + [ C(ITLB) ] = { + [ C(OP_READ) ] = { + [ C(RESULT_ACCESS) ] = -1, + [ C(RESULT_MISS) ] = 0xe11, + }, + [ C(OP_WRITE) ] = { + [ C(RESULT_ACCESS) ] = -1, + [ C(RESULT_MISS) ] = -1, + }, + [ C(OP_PREFETCH) ] = { + [ C(RESULT_ACCESS) ] = -1, + [ C(RESULT_MISS) ] = -1, + }, + }, + [ C(BPU ) ] = { + [ C(OP_READ) ] = { + [ C(RESULT_ACCESS) ] = 0x4c4, + [ C(RESULT_MISS) ] = 0x4c5, + }, + [ C(OP_WRITE) ] = { + [ C(RESULT_ACCESS) ] = -1, + [ C(RESULT_MISS) ] = -1, + }, + [ C(OP_PREFETCH) ] = { + [ C(RESULT_ACCESS) ] = -1, + [ C(RESULT_MISS) ] = -1, + }, + }, +}; + static __initconst const u64 glc_hw_cache_extra_regs [PERF_COUNT_HW_CACHE_MAX] [PERF_COUNT_HW_CACHE_OP_MAX] @@ -713,18 +800,71 @@ static __initconst const u64 glc_hw_cache_extra_regs { [ C(LL ) ] = { [ C(OP_READ) ] = { - [ C(RESULT_ACCESS) ] = 0x10001, - [ C(RESULT_MISS) ] = 0x3fbfc00001, + [ C(RESULT_ACCESS) ] = 0x10001, /* OCR.DEMAND_DATA_RD.ANY_RESPONSE */ + [ C(RESULT_MISS) ] = 0x3fbfc00001, /* OCR.DEMAND_DATA_RD.L3_MISS */ }, [ C(OP_WRITE) ] = { - [ C(RESULT_ACCESS) ] = 0x3f3ffc0002, - [ C(RESULT_MISS) ] = 0x3f3fc00002, + [ C(RESULT_ACCESS) ] = 0x3f3ffc0002, /* OCR.DEMAND_RFO.ANY_RESPONSE */ + [ C(RESULT_MISS) ] = 0x3f3fc00002, /* OCR.DEMAND_RFO.L3_MISS */ }, }, [ C(NODE) ] = { [ C(OP_READ) ] = { - [ C(RESULT_ACCESS) ] = 0x10c000001, - [ C(RESULT_MISS) ] = 0x3fb3000001, + [ C(RESULT_ACCESS) ] = 0x104000001, /* OCR.DEMAND_DATA_RD.LOCAL_DRAM */ + [ C(RESULT_MISS) ] = 0x730000001, /* OCR.DEMAND_DATA_RD.REMOTE_DRAM */ + }, + }, +}; + +/* ADL P-core (Golden cove) specific extra regs value. */ +static __initconst const u64 adl_glc_hw_cache_extra_regs + [PERF_COUNT_HW_CACHE_MAX] + [PERF_COUNT_HW_CACHE_OP_MAX] + [PERF_COUNT_HW_CACHE_RESULT_MAX] = +{ + [ C(LL ) ] = { + [ C(OP_READ) ] = { + [ C(RESULT_ACCESS) ] = 0x10001, /* OCR.DEMAND_DATA_RD.ANY_RESPONSE */ + [ C(RESULT_MISS) ] = 0x3fbfc00001, /* OCR.DEMAND_DATA_RD.L3_MISS */ + }, + [ C(OP_WRITE) ] = { + [ C(RESULT_ACCESS) ] = 0x10002, /* OCR.DEMAND_RFO.ANY_RESPONSE */ + [ C(RESULT_MISS) ] = 0x3fbfc00002, /* OCR.DEMAND_RFO.L3_MISS */ + }, + }, +}; + +static __initconst const u64 lnc_hw_cache_extra_regs + [PERF_COUNT_HW_CACHE_MAX] + [PERF_COUNT_HW_CACHE_OP_MAX] + [PERF_COUNT_HW_CACHE_RESULT_MAX] = +{ + [ C(LL ) ] = { + [ C(OP_READ) ] = { + [ C(RESULT_ACCESS) ] = 0x10001, /* OCR.DEMAND_DATA_RD.ANY_RESPONSE */ + [ C(RESULT_MISS) ] = 0x9E7FA000001, /* OCR.DEMAND_DATA_RD.L3_MISS */ + }, + [ C(OP_WRITE) ] = { + [ C(RESULT_ACCESS) ] = 0x10002, /* OCR.DEMAND_RFO.ANY_RESPONSE */ + [ C(RESULT_MISS) ] = 0x9E7FA000002, /* OCR.DEMAND_RFO.L3_MISS */ + }, + }, +}; + +/* ARL specific lioncove hw_cache_extra_regs[] variant. */ +static __initconst const u64 arl_lnc_hw_cache_extra_regs + [PERF_COUNT_HW_CACHE_MAX] + [PERF_COUNT_HW_CACHE_OP_MAX] + [PERF_COUNT_HW_CACHE_RESULT_MAX] = +{ + [ C(LL ) ] = { + [ C(OP_READ) ] = { + [ C(RESULT_ACCESS) ] = 0x10001, /* OCR.DEMAND_DATA_RD.ANY_RESPONSE */ + [ C(RESULT_MISS) ] = 0xFE7F8000001, /* OCR.DEMAND_DATA_RD.L3_MISS */ + }, + [ C(OP_WRITE) ] = { + [ C(RESULT_ACCESS) ] = 0x10002, /* OCR.DEMAND_RFO.ANY_RESPONSE */ + [ C(RESULT_MISS) ] = 0xFE7F8000002, /* OCR.DEMAND_RFO.L3_MISS */ }, }, }; @@ -815,12 +955,29 @@ static __initconst const u64 pnc_hw_cache_extra_regs { [ C(LL ) ] = { [ C(OP_READ) ] = { - [ C(RESULT_ACCESS) ] = 0x4000000000000001, - [ C(RESULT_MISS) ] = 0xFFFFF000000001, + [ C(RESULT_ACCESS) ] = 0x4000000000000001, /* OMR.DEMAND_DATA_RD.ANY_RESPONSE */ + [ C(RESULT_MISS) ] = 0xFFFFF000000001, /* OMR.DEMAND_DATA_RD.L3_MISS */ }, [ C(OP_WRITE) ] = { - [ C(RESULT_ACCESS) ] = 0x4000000000000002, - [ C(RESULT_MISS) ] = 0xFFFFF000000002, + [ C(RESULT_ACCESS) ] = 0x4000000000000002, /* OMR.DEMAND_RFO.ANY_RESPONSE */ + [ C(RESULT_MISS) ] = 0xFFFFF000000002, /* OMR.DEMAND_RFO.L3_MISS */ + }, + }, +}; + +static __initconst const u64 cyc_hw_cache_extra_regs + [PERF_COUNT_HW_CACHE_MAX] + [PERF_COUNT_HW_CACHE_OP_MAX] + [PERF_COUNT_HW_CACHE_RESULT_MAX] = +{ + [ C(LL ) ] = { + [ C(OP_READ) ] = { + [ C(RESULT_ACCESS) ] = 0x4000000000000001, /* OMR.DEMAND_DATA_RD.ANY_RESPONSE */ + [ C(RESULT_MISS) ] = 0xFF03F000000001, /* OMR.DEMAND_DATA_RD.L3_MISS */ + }, + [ C(OP_WRITE) ] = { + [ C(RESULT_ACCESS) ] = 0x4000000000000002, /* OMR.DEMAND_RFO.ANY_RESPONSE */ + [ C(RESULT_MISS) ] = 0xFF03F000000002, /* OMR.DEMAND_RFO.L3_MISS */ }, }, }; @@ -1019,6 +1176,41 @@ static __initconst const u64 skl_hw_cache_extra_regs }, }; +static __initconst const u64 snc_hw_cache_extra_regs + [PERF_COUNT_HW_CACHE_MAX] + [PERF_COUNT_HW_CACHE_OP_MAX] + [PERF_COUNT_HW_CACHE_RESULT_MAX] = +{ + [ C(LL ) ] = { + [ C(OP_READ) ] = { + [ C(RESULT_ACCESS) ] = 0x10001, /* OCR.DEMAND_DATA_RD.ANY_RESPONSE */ + [ C(RESULT_MISS) ] = 0x3FBFC00001, /* OCR.DEMAND_DATA_RD.L3_MISS */ + }, + [ C(OP_WRITE) ] = { + [ C(RESULT_ACCESS) ] = 0x3F3FFC0002, /* OCR.DEMAND_RFO.ANY_RESPONSE */ + [ C(RESULT_MISS) ] = 0x3F3FC00002, /* OCR.DEMAND_RFO.L3_MISS */ + }, + [ C(OP_PREFETCH) ] = { + [ C(RESULT_ACCESS) ] = 0x0, + [ C(RESULT_MISS) ] = 0x0, + }, + }, + [ C(NODE) ] = { + [ C(OP_READ) ] = { + [ C(RESULT_ACCESS) ] = 0x104000001, /* OCR.DEMAND_DATA_RD.LOCAL_DRAM */ + [ C(RESULT_MISS) ] = 0x730000001, /* OCR.DEMAND_DATA_RD.REMOTE_DRAM */ + }, + [ C(OP_WRITE) ] = { + [ C(RESULT_ACCESS) ] = 0x104000002, /* OCR.DEMAND_RFO.LOCAL_DRAM */ + [ C(RESULT_MISS) ] = 0x730000002, /* OCR.DEMAND_RFO.REMOTE_DRAM */ + }, + [ C(OP_PREFETCH) ] = { + [ C(RESULT_ACCESS) ] = 0x0, + [ C(RESULT_MISS) ] = 0x0, + }, + }, +}; + #define SNB_DMND_DATA_RD (1ULL << 0) #define SNB_DMND_RFO (1ULL << 1) #define SNB_DMND_IFETCH (1ULL << 2) @@ -2342,22 +2534,82 @@ static __initconst const u64 tnt_hw_cache_extra_regs }, }; -static __initconst const u64 arw_hw_cache_extra_regs +static __initconst const u64 grt_hw_cache_extra_regs [PERF_COUNT_HW_CACHE_MAX] [PERF_COUNT_HW_CACHE_OP_MAX] [PERF_COUNT_HW_CACHE_RESULT_MAX] = { [C(LL)] = { [C(OP_READ)] = { - [C(RESULT_ACCESS)] = 0x4000000000000001, - [C(RESULT_MISS)] = 0xFFFFF000000001, + [C(RESULT_ACCESS)] = 0x10001, /* OCR.DEMAND_DATA_RD.ANY_RESPONSE */ + [C(RESULT_MISS)] = 0x3F84400001, /* OCR.DEMAND_DATA_RD.L3_MISS */ }, [C(OP_WRITE)] = { - [C(RESULT_ACCESS)] = 0x4000000000000002, - [C(RESULT_MISS)] = 0xFFFFF000000002, + [C(RESULT_ACCESS)] = 0x10002, /* OCR.DEMAND_RFO.ANY_RESPONSE */ + [C(RESULT_MISS)] = 0x3F84400002, /* OCR.DEMAND_RFO.L3_MISS */ }, - [C(OP_PREFETCH)] = { - [C(RESULT_ACCESS)] = 0x0, - [C(RESULT_MISS)] = 0x0, + }, +}; + +static __initconst const u64 cmt_hw_cache_extra_regs + [PERF_COUNT_HW_CACHE_MAX] + [PERF_COUNT_HW_CACHE_OP_MAX] + [PERF_COUNT_HW_CACHE_RESULT_MAX] = { + [C(LL)] = { + [C(OP_READ)] = { + [C(RESULT_ACCESS)] = 0x10001, /* OCR.DEMAND_DATA_RD.ANY_RESPONSE */ + [C(RESULT_MISS)] = 0x3fbfc00001, /* OCR.DEMAND_DATA_RD.L3_MISS */ + }, + [C(OP_WRITE)] = { + [C(RESULT_ACCESS)] = 0x10002, /* OCR.DEMAND_RFO.ANY_RESPONSE */ + [C(RESULT_MISS)] = 0x3fbfc00002, /* OCR.DEMAND_RFO.L3_MISS */ + }, + }, +}; + +static __initconst const u64 skt_hw_cache_extra_regs + [PERF_COUNT_HW_CACHE_MAX] + [PERF_COUNT_HW_CACHE_OP_MAX] + [PERF_COUNT_HW_CACHE_RESULT_MAX] = { + [C(LL)] = { + [C(OP_READ)] = { + [C(RESULT_ACCESS)] = 0x10001, /* OCR.DEMAND_DATA_RD.ANY_RESPONSE */ + [C(RESULT_MISS)] = 0x13FBFC00001, /* OCR.DEMAND_DATA_RD.L3_MISS */ + }, + [C(OP_WRITE)] = { + [C(RESULT_ACCESS)] = 0x10002, /* OCR.DEMAND_RFO.ANY_RESPONSE */ + [C(RESULT_MISS)] = 0x13FBFC00002, /* OCR.DEMAND_RFO.L3_MISS */ + }, + }, +}; + +static __initconst const u64 dkt_hw_cache_extra_regs + [PERF_COUNT_HW_CACHE_MAX] + [PERF_COUNT_HW_CACHE_OP_MAX] + [PERF_COUNT_HW_CACHE_RESULT_MAX] = { + [C(LL)] = { + [C(OP_READ)] = { + [C(RESULT_ACCESS)] = 0x10001, /* OCR.DEMAND_DATA_RD.ANY_RESPONSE */ + [C(RESULT_MISS)] = 0x33FBFC00001, /* OCR.DEMAND_DATA_RD.L3_MISS */ + }, + [C(OP_WRITE)] = { + [C(RESULT_ACCESS)] = 0x10002, /* OCR.DEMAND_RFO.ANY_RESPONSE */ + [C(RESULT_MISS)] = 0x33FBFC00002, /* OCR.DEMAND_RFO.L3_MISS */ + }, + }, +}; + +static __initconst const u64 arw_hw_cache_extra_regs + [PERF_COUNT_HW_CACHE_MAX] + [PERF_COUNT_HW_CACHE_OP_MAX] + [PERF_COUNT_HW_CACHE_RESULT_MAX] = { + [C(LL)] = { + [C(OP_READ)] = { + [C(RESULT_ACCESS)] = 0x4000000000000009, /* OMR.DEMAND_DATA_RD.ANY_RESPONSE */ + [C(RESULT_MISS)] = 0xFF03F000000009, /* OMR.DEMAND_DATA_RD.L3_MISS */ + }, + [C(OP_WRITE)] = { + [C(RESULT_ACCESS)] = 0x400000000000000A, /* OMR.DEMAND_RFO.ANY_RESPONSE */ + [C(RESULT_MISS)] = 0xFF03F00000000A, /* OMR.DEMAND_RFO.L3_MISS */ }, }, }; @@ -2392,9 +2644,12 @@ static struct attribute *grt_mem_attrs[] = { }; static struct extra_reg intel_grt_extra_regs[] __read_mostly = { - /* must define OFFCORE_RSP_X first, see intel_fixup_er() */ - INTEL_UEVENT_EXTRA_REG(0x01b7, MSR_OFFCORE_RSP_0, 0x3fffffffffull, RSP_0), - INTEL_UEVENT_EXTRA_REG(0x02b7, MSR_OFFCORE_RSP_1, 0x3fffffffffull, RSP_1), + /* + * Must define OFFCORE_RSP_X first, see intel_fixup_er(). + * Bit 63 only valid on OFFCORE_RSP_0 MSR. + */ + INTEL_UEVENT_EXTRA_REG(0x01b7, MSR_OFFCORE_RSP_0, 0x8003f03fffffffffull, RSP_0), + INTEL_UEVENT_EXTRA_REG(0x02b7, MSR_OFFCORE_RSP_1, 0x3f03fffffffffull, RSP_1), INTEL_UEVENT_PEBS_LDLAT_EXTRA_REG(0x5d0), EVENT_EXTRA_END }; @@ -2426,10 +2681,6 @@ static struct extra_reg intel_arw_extra_regs[] __read_mostly = { INTEL_UEVENT_EXTRA_REG(0x02b7, MSR_OMR_1, 0xc0ffffffffffffffull, OMR_1), INTEL_UEVENT_EXTRA_REG(0x04b7, MSR_OMR_2, 0xc0ffffffffffffffull, OMR_2), INTEL_UEVENT_EXTRA_REG(0x08b7, MSR_OMR_3, 0xc0ffffffffffffffull, OMR_3), - INTEL_UEVENT_EXTRA_REG(0x01d4, MSR_OMR_0, 0xc0ffffffffffffffull, OMR_0), - INTEL_UEVENT_EXTRA_REG(0x02d4, MSR_OMR_1, 0xc0ffffffffffffffull, OMR_1), - INTEL_UEVENT_EXTRA_REG(0x04d4, MSR_OMR_2, 0xc0ffffffffffffffull, OMR_2), - INTEL_UEVENT_EXTRA_REG(0x08d4, MSR_OMR_3, 0xc0ffffffffffffffull, OMR_3), INTEL_UEVENT_PEBS_LDLAT_EXTRA_REG(0x5d0), INTEL_UEVENT_EXTRA_REG(0x0127, MSR_SNOOP_RSP_0, 0xffffffffffffffffull, SNOOP_0), INTEL_UEVENT_EXTRA_REG(0x0227, MSR_SNOOP_RSP_1, 0xffffffffffffffffull, SNOOP_1), @@ -3169,10 +3420,10 @@ static void intel_pmu_config_acr(int idx, u64 mask, u32 reload) wrmsrl(msr_b + msr_offset, mask); cpuc->acr_cfg_b[idx] = mask; } - /* Only need to update the reload value when there is a valid config value. */ - if (mask && cpuc->acr_cfg_c[idx] != reload) { + /* Only update CFG_C reload when ACR is actively enabled (mask != 0) */ + if (mask && ((cpuc->cfg_c_val[idx] & ARCH_PEBS_RELOAD) != reload)) { wrmsrl(msr_c + msr_offset, reload); - cpuc->acr_cfg_c[idx] = reload; + cpuc->cfg_c_val[idx] = reload; } } @@ -3198,14 +3449,15 @@ static void intel_pmu_enable_event_ext(struct perf_event *event) { struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events); struct hw_perf_event *hwc = &event->hw; - union arch_pebs_index old, new; - struct arch_pebs_cap cap; u64 ext = 0; - cap = hybrid(cpuc->pmu, arch_pebs_cap); + if (is_acr_event_group(event)) + ext |= (-hwc->sample_period) & ARCH_PEBS_RELOAD; if (event->attr.precise_ip) { u64 pebs_data_cfg = intel_get_arch_pebs_data_config(event); + struct arch_pebs_cap cap = hybrid(cpuc->pmu, arch_pebs_cap); + union arch_pebs_index old, new; ext |= ARCH_PEBS_EN; if (hwc->flags & PERF_X86_EVENT_AUTO_RELOAD) @@ -7490,6 +7742,15 @@ static __always_inline void intel_pmu_init_glc(struct pmu *pmu) intel_pmu_ref_cycles_ext(); } +static __always_inline void intel_pmu_init_glc_hybrid(struct pmu *pmu) +{ + intel_pmu_init_glc(pmu); + + /* ADL has different extra MSR values from Server for the L3 or node OCR/OMR events. */ + memcpy(hybrid_var(pmu, hw_cache_event_ids), adl_glc_hw_cache_event_ids, sizeof(hw_cache_event_ids)); + memcpy(hybrid_var(pmu, hw_cache_extra_regs), adl_glc_hw_cache_extra_regs, sizeof(hw_cache_extra_regs)); +} + static __always_inline void intel_pmu_init_grt(struct pmu *pmu) { x86_pmu.mid_ack = true; @@ -7502,7 +7763,7 @@ static __always_inline void intel_pmu_init_grt(struct pmu *pmu) x86_pmu.flags |= PMU_FL_INSTR_LATENCY; memcpy(hybrid_var(pmu, hw_cache_event_ids), glp_hw_cache_event_ids, sizeof(hw_cache_event_ids)); - memcpy(hybrid_var(pmu, hw_cache_extra_regs), tnt_hw_cache_extra_regs, sizeof(hw_cache_extra_regs)); + memcpy(hybrid_var(pmu, hw_cache_extra_regs), grt_hw_cache_extra_regs, sizeof(hw_cache_extra_regs)); hybrid_var(pmu, hw_cache_event_ids)[C(ITLB)][C(OP_READ)][C(RESULT_ACCESS)] = -1; hybrid(pmu, event_constraints) = intel_grt_event_constraints; hybrid(pmu, pebs_constraints) = intel_grt_pebs_event_constraints; @@ -7511,12 +7772,24 @@ static __always_inline void intel_pmu_init_grt(struct pmu *pmu) intel_pmu_ref_cycles_ext(); } +static __always_inline void intel_pmu_init_cmt(struct pmu *pmu) +{ + intel_pmu_init_grt(pmu); + memcpy(hybrid_var(pmu, hw_cache_extra_regs), + cmt_hw_cache_extra_regs, sizeof(hw_cache_extra_regs)); + hybrid(pmu, pebs_constraints) = intel_cmt_pebs_event_constraints; + hybrid(pmu, extra_regs) = intel_cmt_extra_regs; +} + static __always_inline void intel_pmu_init_lnc(struct pmu *pmu) { intel_pmu_init_glc(pmu); hybrid(pmu, event_constraints) = intel_lnc_event_constraints; hybrid(pmu, pebs_constraints) = intel_lnc_pebs_event_constraints; hybrid(pmu, extra_regs) = intel_lnc_extra_regs; + + memcpy(hybrid_var(pmu, hw_cache_event_ids), adl_glc_hw_cache_event_ids, sizeof(hw_cache_event_ids)); + memcpy(hybrid_var(pmu, hw_cache_extra_regs), lnc_hw_cache_extra_regs, sizeof(hw_cache_extra_regs)); } static __always_inline void intel_pmu_init_pnc(struct pmu *pmu) @@ -7534,14 +7807,41 @@ static __always_inline void intel_pmu_init_pnc(struct pmu *pmu) static_call_update(intel_pmu_enable_acr_event, intel_pmu_enable_acr); } +static __always_inline void intel_pmu_init_cyc(struct pmu *pmu) +{ + intel_pmu_init_pnc(pmu); + memcpy(hybrid_var(pmu, hw_cache_extra_regs), + cyc_hw_cache_extra_regs, sizeof(hw_cache_extra_regs)); +} + static __always_inline void intel_pmu_init_skt(struct pmu *pmu) { - intel_pmu_init_grt(pmu); + intel_pmu_init_cmt(pmu); hybrid(pmu, event_constraints) = intel_skt_event_constraints; - hybrid(pmu, extra_regs) = intel_cmt_extra_regs; + memcpy(hybrid_var(pmu, hw_cache_extra_regs), + skt_hw_cache_extra_regs, sizeof(hw_cache_extra_regs)); static_call_update(intel_pmu_enable_acr_event, intel_pmu_enable_acr); } +/* Hybrid client variant. */ +static __always_inline void intel_pmu_init_dkt_hybrid(struct pmu *pmu) +{ + intel_pmu_init_skt(pmu); + hybrid(pmu, pebs_constraints) = intel_dkt_pebs_event_constraints; +} + +/* + * Darkmont is used by the CWF and PTL E-cores, but their L3 OCR + * events require different extra MSR values. Keep a separate init + * function for the non-hybrid server variant. + */ +static __always_inline void intel_pmu_init_dkt(struct pmu *pmu) +{ + intel_pmu_init_dkt_hybrid(pmu); + memcpy(hybrid_var(pmu, hw_cache_extra_regs), + dkt_hw_cache_extra_regs, sizeof(hw_cache_extra_regs)); +} + static __always_inline void intel_pmu_init_arw(struct pmu *pmu) { intel_pmu_init_grt(pmu); @@ -7550,7 +7850,7 @@ static __always_inline void intel_pmu_init_arw(struct pmu *pmu) memcpy(hybrid_var(pmu, hw_cache_extra_regs), arw_hw_cache_extra_regs, sizeof(hw_cache_extra_regs)); hybrid(pmu, event_constraints) = intel_arw_event_constraints; - hybrid(pmu, pebs_constraints) = intel_arw_pebs_event_constraints; + hybrid(pmu, pebs_constraints) = intel_dkt_pebs_event_constraints; hybrid(pmu, extra_regs) = intel_arw_extra_regs; static_call_update(intel_pmu_enable_acr_event, intel_pmu_enable_acr); } @@ -7864,8 +8164,7 @@ __init int intel_pmu_init(void) case INTEL_ATOM_CRESTMONT: case INTEL_ATOM_CRESTMONT_X: - intel_pmu_init_grt(NULL); - x86_pmu.extra_regs = intel_cmt_extra_regs; + intel_pmu_init_cmt(NULL); intel_pmu_pebs_data_source_cmt(); x86_pmu.pebs_latency_data = cmt_latency_data; x86_pmu.get_event_constraints = cmt_get_event_constraints; @@ -7877,7 +8176,7 @@ __init int intel_pmu_init(void) break; case INTEL_ATOM_DARKMONT_X: - intel_pmu_init_skt(NULL); + intel_pmu_init_dkt(NULL); intel_pmu_pebs_data_source_cmt(); x86_pmu.pebs_latency_data = cmt_latency_data; x86_pmu.get_event_constraints = cmt_get_event_constraints; @@ -8153,17 +8452,19 @@ __init int intel_pmu_init(void) case INTEL_ICELAKE_X: case INTEL_ICELAKE_D: + memcpy(hw_cache_extra_regs, snc_hw_cache_extra_regs, sizeof(hw_cache_extra_regs)); x86_pmu.pebs_ept = 1; pmem = true; - fallthrough; + goto snc_common; case INTEL_ICELAKE_L: case INTEL_ICELAKE: case INTEL_TIGERLAKE_L: case INTEL_TIGERLAKE: case INTEL_ROCKETLAKE: + memcpy(hw_cache_extra_regs, skl_hw_cache_extra_regs, sizeof(hw_cache_extra_regs)); + snc_common: x86_pmu.late_ack = true; memcpy(hw_cache_event_ids, skl_hw_cache_event_ids, sizeof(hw_cache_event_ids)); - memcpy(hw_cache_extra_regs, skl_hw_cache_extra_regs, sizeof(hw_cache_extra_regs)); hw_cache_event_ids[C(ITLB)][C(OP_READ)][C(RESULT_ACCESS)] = -1; intel_pmu_lbr_init_skl(); @@ -8259,7 +8560,7 @@ __init int intel_pmu_init(void) /* Initialize big core specific PerfMon capabilities.*/ pmu = &x86_pmu.hybrid_pmu[X86_HYBRID_PMU_CORE_IDX]; - intel_pmu_init_glc(&pmu->pmu); + intel_pmu_init_glc_hybrid(&pmu->pmu); if (cpu_feature_enabled(X86_FEATURE_HYBRID_CPU)) { pmu->cntr_mask64 <<= 2; pmu->cntr_mask64 |= 0x3; @@ -8316,13 +8617,12 @@ __init int intel_pmu_init(void) /* Initialize big core specific PerfMon capabilities.*/ pmu = &x86_pmu.hybrid_pmu[X86_HYBRID_PMU_CORE_IDX]; - intel_pmu_init_glc(&pmu->pmu); + intel_pmu_init_glc_hybrid(&pmu->pmu); pmu->extra_regs = intel_rwc_extra_regs; /* Initialize Atom core specific PerfMon capabilities.*/ pmu = &x86_pmu.hybrid_pmu[X86_HYBRID_PMU_ATOM_IDX]; - intel_pmu_init_grt(&pmu->pmu); - pmu->extra_regs = intel_cmt_extra_regs; + intel_pmu_init_cmt(&pmu->pmu); intel_pmu_pebs_data_source_mtl(); pr_cont("Meteorlake Hybrid events, "); @@ -8333,16 +8633,50 @@ __init int intel_pmu_init(void) case INTEL_WILDCATLAKE_L: pr_cont("Pantherlake Hybrid events, "); name = "pantherlake_hybrid"; + + intel_pmu_init_hybrid(hybrid_big_small); + + /* Initialize big core specific PerfMon capabilities.*/ + pmu = &x86_pmu.hybrid_pmu[X86_HYBRID_PMU_CORE_IDX]; + intel_pmu_init_lnc(&pmu->pmu); + /* Initialize Atom core specific PerfMon capabilities.*/ + pmu = &x86_pmu.hybrid_pmu[X86_HYBRID_PMU_ATOM_IDX]; + intel_pmu_init_dkt_hybrid(&pmu->pmu); + goto lnl_common; - case INTEL_LUNARLAKE_M: case INTEL_ARROWLAKE: + pr_cont("Arrowlake Hybrid events, "); + name = "arrowlake_hybrid"; + + intel_pmu_init_hybrid(hybrid_big_small); + + /* Initialize big core specific PerfMon capabilities.*/ + pmu = &x86_pmu.hybrid_pmu[X86_HYBRID_PMU_CORE_IDX]; + intel_pmu_init_lnc(&pmu->pmu); + memcpy(hybrid_var(&pmu->pmu, hw_cache_extra_regs), + arl_lnc_hw_cache_extra_regs, sizeof(hw_cache_extra_regs)); + /* Initialize Atom core specific PerfMon capabilities.*/ + pmu = &x86_pmu.hybrid_pmu[X86_HYBRID_PMU_ATOM_IDX]; + intel_pmu_init_skt(&pmu->pmu); + + goto lnl_common; + + case INTEL_LUNARLAKE_M: pr_cont("Lunarlake Hybrid events, "); name = "lunarlake_hybrid"; - lnl_common: intel_pmu_init_hybrid(hybrid_big_small); + /* Initialize big core specific PerfMon capabilities.*/ + pmu = &x86_pmu.hybrid_pmu[X86_HYBRID_PMU_CORE_IDX]; + intel_pmu_init_lnc(&pmu->pmu); + /* Initialize Atom core specific PerfMon capabilities.*/ + pmu = &x86_pmu.hybrid_pmu[X86_HYBRID_PMU_ATOM_IDX]; + intel_pmu_init_skt(&pmu->pmu); + + lnl_common: + x86_pmu.pebs_latency_data = lnl_latency_data; x86_pmu.get_event_constraints = mtl_get_event_constraints; x86_pmu.hw_config = adl_hw_config; @@ -8353,14 +8687,6 @@ __init int intel_pmu_init(void) extra_attr = boot_cpu_has(X86_FEATURE_RTM) ? mtl_hybrid_extra_attr_rtm : mtl_hybrid_extra_attr; - /* Initialize big core specific PerfMon capabilities.*/ - pmu = &x86_pmu.hybrid_pmu[X86_HYBRID_PMU_CORE_IDX]; - intel_pmu_init_lnc(&pmu->pmu); - - /* Initialize Atom core specific PerfMon capabilities.*/ - pmu = &x86_pmu.hybrid_pmu[X86_HYBRID_PMU_ATOM_IDX]; - intel_pmu_init_skt(&pmu->pmu); - intel_pmu_pebs_data_source_lnl(); break; @@ -8380,6 +8706,8 @@ __init int intel_pmu_init(void) /* Initialize big core specific PerfMon capabilities. */ pmu = &x86_pmu.hybrid_pmu[X86_HYBRID_PMU_CORE_IDX]; intel_pmu_init_lnc(&pmu->pmu); + memcpy(hybrid_var(&pmu->pmu, hw_cache_extra_regs), + arl_lnc_hw_cache_extra_regs, sizeof(hw_cache_extra_regs)); /* Initialize Atom core specific PerfMon capabilities. */ pmu = &x86_pmu.hybrid_pmu[X86_HYBRID_PMU_ATOM_IDX]; @@ -8387,8 +8715,7 @@ __init int intel_pmu_init(void) /* Initialize Lower Power Atom specific PerfMon capabilities. */ pmu = &x86_pmu.hybrid_pmu[X86_HYBRID_PMU_TINY_IDX]; - intel_pmu_init_grt(&pmu->pmu); - pmu->extra_regs = intel_cmt_extra_regs; + intel_pmu_init_cmt(&pmu->pmu); intel_pmu_pebs_data_source_arl_h(); pr_cont("ArrowLake-H Hybrid events, "); @@ -8413,7 +8740,7 @@ __init int intel_pmu_init(void) /* Initialize big core specific PerfMon capabilities.*/ pmu = &x86_pmu.hybrid_pmu[X86_HYBRID_PMU_CORE_IDX]; - intel_pmu_init_pnc(&pmu->pmu); + intel_pmu_init_cyc(&pmu->pmu); /* Initialize Atom core specific PerfMon capabilities.*/ pmu = &x86_pmu.hybrid_pmu[X86_HYBRID_PMU_ATOM_IDX]; diff --git a/arch/x86/events/intel/ds.c b/arch/x86/events/intel/ds.c index 7f0d515..cb72af9 100644 --- a/arch/x86/events/intel/ds.c +++ b/arch/x86/events/intel/ds.c @@ -1292,18 +1292,21 @@ struct event_constraint intel_glm_pebs_event_constraints[] = { struct event_constraint intel_grt_pebs_event_constraints[] = { /* Allow all events as PEBS with no flags */ INTEL_HYBRID_LAT_CONSTRAINT(0x5d0, 0x3), - INTEL_HYBRID_LAT_CONSTRAINT(0x6d0, 0xf), + INTEL_HYBRID_LAT_CONSTRAINT(0x6d0, 0x3f), EVENT_CONSTRAINT_END }; -struct event_constraint intel_arw_pebs_event_constraints[] = { +struct event_constraint intel_cmt_pebs_event_constraints[] = { + /* Allow all events as PEBS with no flags */ + INTEL_HYBRID_LAT_CONSTRAINT(0x5d0, 0x3), + INTEL_HYBRID_LAT_CONSTRAINT(0x6d0, 0xff), + EVENT_CONSTRAINT_END +}; + +struct event_constraint intel_dkt_pebs_event_constraints[] = { /* Allow all events as PEBS with no flags */ INTEL_HYBRID_LAT_CONSTRAINT(0x5d0, 0xff), INTEL_HYBRID_LAT_CONSTRAINT(0x6d0, 0xff), - INTEL_FLAGS_UEVENT_CONSTRAINT(0x01d4, 0x1), - INTEL_FLAGS_UEVENT_CONSTRAINT(0x02d4, 0x2), - INTEL_FLAGS_UEVENT_CONSTRAINT(0x04d4, 0x4), - INTEL_FLAGS_UEVENT_CONSTRAINT(0x08d4, 0x8), EVENT_CONSTRAINT_END }; @@ -1500,6 +1503,13 @@ struct event_constraint intel_lnc_pebs_event_constraints[] = { INTEL_FLAGS_UEVENT_CONSTRAINT(0x100, 0x100000000ULL), /* INST_RETIRED.PREC_DIST */ INTEL_FLAGS_UEVENT_CONSTRAINT(0x0400, 0x800000000ULL), + INTEL_FLAGS_UEVENT_CONSTRAINT(0x012a, 0x1), /* OCR.* events */ + INTEL_FLAGS_UEVENT_CONSTRAINT(0x012b, 0x1), /* OCR.* events */ + + INTEL_FLAGS_UEVENT_CONSTRAINT(0x04a4, 0x1), /* TOPDOWN.BAD_SPEC_SLOTS */ + INTEL_FLAGS_UEVENT_CONSTRAINT(0x08a4, 0x1), /* TOPDOWN.BR_MISPREDICT_SLOTS */ + INTEL_FLAGS_UEVENT_CONSTRAINT(0x10a4, 0x8), /* TOPDOWN.MEMORY_BOUND_SLOTS */ + INTEL_HYBRID_LDLAT_CONSTRAINT(0x1cd, 0x3fc), INTEL_HYBRID_STLAT_CONSTRAINT(0x2cd, 0x3), INTEL_FLAGS_UEVENT_CONSTRAINT_DATALA_LD(0x11d0, 0xf), /* MEM_INST_RETIRED.STLB_MISS_LOADS */ @@ -1509,6 +1519,7 @@ struct event_constraint intel_lnc_pebs_event_constraints[] = { INTEL_FLAGS_UEVENT_CONSTRAINT_DATALA_ST(0x42d0, 0xf), /* MEM_INST_RETIRED.SPLIT_STORES */ INTEL_FLAGS_UEVENT_CONSTRAINT_DATALA_LD(0x81d0, 0xf), /* MEM_INST_RETIRED.ALL_LOADS */ INTEL_FLAGS_UEVENT_CONSTRAINT_DATALA_ST(0x82d0, 0xf), /* MEM_INST_RETIRED.ALL_STORES */ + INTEL_FLAGS_UEVENT_CONSTRAINT(0x87d0, 0x3ff), /* MEM_INST_RETIRED.ANY */ INTEL_FLAGS_EVENT_CONSTRAINT_DATALA_LD_RANGE(0xd1, 0xd4, 0xf), diff --git a/arch/x86/events/intel/uncore.c b/arch/x86/events/intel/uncore.c index e9cc1ba..7857959 100644 --- a/arch/x86/events/intel/uncore.c +++ b/arch/x86/events/intel/uncore.c @@ -83,6 +83,25 @@ int uncore_device_to_die(struct pci_dev *dev) return -1; } +/* + * Using cpus_read_lock() to ensure cpu is not going down between + * looking at cpu_online_mask. + * + * The lock must be held by the caller. + */ +int uncore_die_to_cpu(int die) +{ + int res = -1, cpu; + + for_each_online_cpu(cpu) { + if (topology_logical_die_id(cpu) == die) { + res = cpu; + break; + } + } + return res; +} + static void uncore_free_pcibus_map(void) { struct pci2phy_map *map, *tmp; @@ -1697,7 +1716,7 @@ err: return ret; } -static int uncore_mmio_global_init(u64 ctl) +static int uncore_mmio_global_init(int die, u64 ctl) { void __iomem *io_addr; @@ -1712,6 +1731,16 @@ static int uncore_mmio_global_init(u64 ctl) return 0; } +static int uncore_msr_global_init(int die, u64 msr) +{ + int cpu = uncore_die_to_cpu(die); + + if (cpu == -1) + return -ENODEV; + + return wrmsrq_on_cpu(cpu, msr, 0); +} + static const struct uncore_plat_init nhm_uncore_init __initconst = { .cpu_init = nhm_uncore_cpu_init, }; @@ -1852,6 +1881,7 @@ static const struct uncore_plat_init gnr_uncore_init __initconst = { .domain[0].base_is_pci = true, .domain[0].discovery_base = UNCORE_DISCOVERY_TABLE_DEVICE, .domain[0].units_ignore = gnr_uncore_units_ignore, + .domain[0].global_init = uncore_msr_global_init, }; static const struct uncore_plat_init dmr_uncore_init __initconst = { diff --git a/arch/x86/events/intel/uncore.h b/arch/x86/events/intel/uncore.h index c35918c..c2e5ccb 100644 --- a/arch/x86/events/intel/uncore.h +++ b/arch/x86/events/intel/uncore.h @@ -53,7 +53,7 @@ struct uncore_discovery_domain { /* MSR address or PCI device used as the discovery base */ u32 discovery_base; bool base_is_pci; - int (*global_init)(u64 ctl); + int (*global_init)(int die, u64 ctl); /* The units in the discovery table should be ignored. */ int *units_ignore; @@ -235,6 +235,7 @@ struct pci2phy_map *__find_pci2phy_map(int segment); int uncore_pcibus_to_dieid(struct pci_bus *bus); int uncore_die_to_segment(int die); int uncore_device_to_die(struct pci_dev *dev); +int uncore_die_to_cpu(int die); ssize_t uncore_event_show(struct device *dev, struct device_attribute *attr, char *buf); diff --git a/arch/x86/events/intel/uncore_discovery.c b/arch/x86/events/intel/uncore_discovery.c index 583cbd0..e507762 100644 --- a/arch/x86/events/intel/uncore_discovery.c +++ b/arch/x86/events/intel/uncore_discovery.c @@ -287,7 +287,7 @@ static int __parse_discovery_table(struct uncore_discovery_domain *domain, if (!io_addr) return -ENOMEM; - if (domain->global_init && domain->global_init(global.ctl)) { + if (domain->global_init && domain->global_init(die, global.ctl)) { ret = -ENODEV; goto out; } @@ -399,7 +399,6 @@ static bool uncore_discovery_msr(struct uncore_discovery_domain *domain) if (!die_mask) return false; - cpus_read_lock(); for_each_online_cpu(cpu) { die = topology_logical_die_id(cpu); if (__test_and_set_bit(die, die_mask)) @@ -414,8 +413,6 @@ static bool uncore_discovery_msr(struct uncore_discovery_domain *domain) __parse_discovery_table(domain, base, die, &parsed); } - cpus_read_unlock(); - kfree(die_mask); return parsed; } @@ -429,10 +426,14 @@ bool uncore_discovery(struct uncore_plat_init *init) for (i = 0; i < UNCORE_DISCOVERY_DOMAINS; i++) { domain = &init->domain[i]; if (domain->discovery_base) { + cpus_read_lock(); + if (!domain->base_is_pci) ret |= uncore_discovery_msr(domain); else ret |= uncore_discovery_pci(domain); + + cpus_read_unlock(); } } @@ -481,8 +482,8 @@ static u64 intel_generic_uncore_box_ctl(struct intel_uncore_box *box) struct intel_uncore_discovery_unit *unit; unit = intel_uncore_find_discovery_unit(box->pmu->type->boxes, - -1, box->pmu->pmu_idx); - if (WARN_ON_ONCE(!unit)) + box->dieid, box->pmu->pmu_idx); + if (!unit) return 0; return unit->addr; @@ -490,17 +491,28 @@ static u64 intel_generic_uncore_box_ctl(struct intel_uncore_box *box) void intel_generic_uncore_msr_init_box(struct intel_uncore_box *box) { - wrmsrq(intel_generic_uncore_box_ctl(box), GENERIC_PMON_BOX_CTL_INT); + u64 box_ctl = intel_generic_uncore_box_ctl(box); + + if (!box_ctl) + return; + + wrmsrq(box_ctl, GENERIC_PMON_BOX_CTL_INT); } void intel_generic_uncore_msr_disable_box(struct intel_uncore_box *box) { - wrmsrq(intel_generic_uncore_box_ctl(box), GENERIC_PMON_BOX_CTL_FRZ); + u64 box_ctl = intel_generic_uncore_box_ctl(box); + + if (box_ctl) + wrmsrq(box_ctl, GENERIC_PMON_BOX_CTL_FRZ); } void intel_generic_uncore_msr_enable_box(struct intel_uncore_box *box) { - wrmsrq(intel_generic_uncore_box_ctl(box), 0); + u64 box_ctl = intel_generic_uncore_box_ctl(box); + + if (box_ctl) + wrmsrq(box_ctl, 0); } static void intel_generic_uncore_msr_enable_event(struct intel_uncore_box *box, @@ -549,6 +561,7 @@ bool intel_generic_uncore_assign_hw_event(struct perf_event *event, if (box->pci_dev) { box_ctl = UNCORE_DISCOVERY_PCI_BOX_CTRL(box_ctl); + hwc->config_base = box_ctl + uncore_pci_event_ctl(box, hwc->idx); hwc->event_base = box_ctl + uncore_pci_perf_ctr(box, hwc->idx); return true; @@ -567,27 +580,30 @@ static inline int intel_pci_uncore_box_ctl(struct intel_uncore_box *box) void intel_generic_uncore_pci_init_box(struct intel_uncore_box *box) { - struct pci_dev *pdev = box->pci_dev; int box_ctl = intel_pci_uncore_box_ctl(box); + if (!box_ctl) + return; + __set_bit(UNCORE_BOX_FLAG_CTL_OFFS8, &box->flags); - pci_write_config_dword(pdev, box_ctl, GENERIC_PMON_BOX_CTL_INT); + pci_write_config_dword(box->pci_dev, box_ctl, GENERIC_PMON_BOX_CTL_INT); } void intel_generic_uncore_pci_disable_box(struct intel_uncore_box *box) { - struct pci_dev *pdev = box->pci_dev; int box_ctl = intel_pci_uncore_box_ctl(box); - pci_write_config_dword(pdev, box_ctl, GENERIC_PMON_BOX_CTL_FRZ); + if (box_ctl) + pci_write_config_dword(box->pci_dev, box_ctl, + GENERIC_PMON_BOX_CTL_FRZ); } void intel_generic_uncore_pci_enable_box(struct intel_uncore_box *box) { - struct pci_dev *pdev = box->pci_dev; int box_ctl = intel_pci_uncore_box_ctl(box); - pci_write_config_dword(pdev, box_ctl, 0); + if (box_ctl) + pci_write_config_dword(box->pci_dev, box_ctl, 0); } static void intel_generic_uncore_pci_enable_event(struct intel_uncore_box *box, diff --git a/arch/x86/events/intel/uncore_snb.c b/arch/x86/events/intel/uncore_snb.c index 3dbc6ba..edddd4f 100644 --- a/arch/x86/events/intel/uncore_snb.c +++ b/arch/x86/events/intel/uncore_snb.c @@ -563,12 +563,6 @@ void tgl_uncore_cpu_init(void) skl_uncore_msr_ops.init_box = rkl_uncore_msr_init_box; } -static void adl_uncore_msr_init_box(struct intel_uncore_box *box) -{ - if (box->pmu->pmu_idx == 0) - wrmsrq(ADL_UNC_PERF_GLOBAL_CTL, SNB_UNC_GLOBAL_CTL_EN); -} - static void adl_uncore_msr_enable_box(struct intel_uncore_box *box) { wrmsrq(ADL_UNC_PERF_GLOBAL_CTL, SNB_UNC_GLOBAL_CTL_EN); @@ -587,7 +581,6 @@ static void adl_uncore_msr_exit_box(struct intel_uncore_box *box) } static struct intel_uncore_ops adl_uncore_msr_ops = { - .init_box = adl_uncore_msr_init_box, .enable_box = adl_uncore_msr_enable_box, .disable_box = adl_uncore_msr_disable_box, .exit_box = adl_uncore_msr_exit_box, diff --git a/arch/x86/events/intel/uncore_snbep.c b/arch/x86/events/intel/uncore_snbep.c index 215d33e..334dc38 100644 --- a/arch/x86/events/intel/uncore_snbep.c +++ b/arch/x86/events/intel/uncore_snbep.c @@ -3704,25 +3704,6 @@ static int skx_msr_cpu_bus_read(int cpu, u64 *topology) return 0; } -static int die_to_cpu(int die) -{ - int res = 0, cpu, current_die; - /* - * Using cpus_read_lock() to ensure cpu is not going down between - * looking at cpu_online_mask. - */ - cpus_read_lock(); - for_each_online_cpu(cpu) { - current_die = topology_logical_die_id(cpu); - if (current_die == die) { - res = cpu; - break; - } - } - cpus_read_unlock(); - return res; -} - enum { IIO_TOPOLOGY_TYPE, UPI_TOPOLOGY_TYPE, @@ -3791,11 +3772,17 @@ static void pmu_free_topology(struct intel_uncore_type *type) static int skx_pmu_get_topology(struct intel_uncore_type *type, int (*topology_cb)(struct intel_uncore_type*, int, int, u64)) { - int die, ret = -EPERM; + int die, ret = -ENODEV; u64 cpu_bus_msr; + int cpu; + cpus_read_lock(); for (die = 0; die < uncore_max_dies(); die++) { - ret = skx_msr_cpu_bus_read(die_to_cpu(die), &cpu_bus_msr); + cpu = uncore_die_to_cpu(die); + if (cpu == -1) + continue; + + ret = skx_msr_cpu_bus_read(cpu, &cpu_bus_msr); if (ret) break; @@ -3807,6 +3794,7 @@ static int skx_pmu_get_topology(struct intel_uncore_type *type, if (ret) break; } + cpus_read_unlock(); return ret; } @@ -4261,7 +4249,7 @@ err: static int skx_upi_topology_cb(struct intel_uncore_type *type, int segment, int die, u64 cpu_bus_msr) { - int idx, ret; + int idx, ret = 0; struct intel_uncore_topology *upi; unsigned int devfn; struct pci_dev *dev = NULL; @@ -4274,12 +4262,12 @@ static int skx_upi_topology_cb(struct intel_uncore_type *type, int segment, dev = pci_get_domain_bus_and_slot(segment, bus, devfn); if (dev) { ret = upi_fill_topology(dev, upi, idx); + pci_dev_put(dev); if (ret) break; } } - pci_dev_put(dev); return ret; } @@ -5499,6 +5487,7 @@ static int discover_upi_topology(struct intel_uncore_type *type, int ubox_did, i devfn); if (dev) { ret = upi_fill_topology(dev, upi, idx); + pci_dev_put(dev); if (ret) goto err; } @@ -5506,7 +5495,6 @@ static int discover_upi_topology(struct intel_uncore_type *type, int ubox_did, i } err: pci_dev_put(ubox); - pci_dev_put(dev); return ret; } diff --git a/arch/x86/events/perf_event.h b/arch/x86/events/perf_event.h index 524668d..eae24bb 100644 --- a/arch/x86/events/perf_event.h +++ b/arch/x86/events/perf_event.h @@ -322,10 +322,8 @@ struct cpu_hw_events { u64 fixed_ctrl_val; u64 active_fixed_ctrl_val; - /* Intel ACR configuration */ + /* Intel ACR/arch-PEBS configuration */ u64 acr_cfg_b[X86_PMC_IDX_MAX]; - u64 acr_cfg_c[X86_PMC_IDX_MAX]; - /* Cached CFG_C values */ u64 cfg_c_val[X86_PMC_IDX_MAX]; /* @@ -1712,7 +1710,9 @@ extern struct event_constraint intel_glp_pebs_event_constraints[]; extern struct event_constraint intel_grt_pebs_event_constraints[]; -extern struct event_constraint intel_arw_pebs_event_constraints[]; +extern struct event_constraint intel_cmt_pebs_event_constraints[]; + +extern struct event_constraint intel_dkt_pebs_event_constraints[]; extern struct event_constraint intel_nehalem_pebs_event_constraints[]; diff --git a/kernel/events/core.c b/kernel/events/core.c index 7935d56..95d806b 100644 --- a/kernel/events/core.c +++ b/kernel/events/core.c @@ -58,6 +58,7 @@ #include #include #include +#include #include "internal.h" @@ -7546,6 +7547,33 @@ static int perf_fasync(int fd, struct file *filp, int on) return 0; } +static void perf_show_fdinfo(struct seq_file *m, struct file *f) +{ + struct perf_event *event = f->private_data; + struct perf_event_context *ctx; + struct mutex *child_mutex; + + ctx = perf_event_ctx_lock(event); + child_mutex = event->parent ? &event->parent->child_mutex : &event->child_mutex; + mutex_lock(child_mutex); + + seq_printf(m, "perf_event_attr.type:\t%u\n", event->orig_type); + if (event->pmu) + seq_printf(m, "pmu_type:\t%u\n", event->pmu->type); + seq_printf(m, "perf_event_attr.config:\t0x%llx\n", (unsigned long long)event->attr.config); + seq_printf(m, "perf_event_attr.config1:\t0x%llx\n", + (unsigned long long)event->attr.config1); + seq_printf(m, "perf_event_attr.config2:\t0x%llx\n", + (unsigned long long)event->attr.config2); + seq_printf(m, "perf_event_attr.config3:\t0x%llx\n", + (unsigned long long)event->attr.config3); + seq_printf(m, "perf_event_attr.config4:\t0x%llx\n", + (unsigned long long)event->attr.config4); + + mutex_unlock(child_mutex); + perf_event_ctx_unlock(event, ctx); +} + static const struct file_operations perf_fops = { .release = perf_release, .read = perf_read, @@ -7554,6 +7582,7 @@ static const struct file_operations perf_fops = { .compat_ioctl = perf_compat_ioctl, .mmap = perf_mmap, .fasync = perf_fasync, + .show_fdinfo = perf_show_fdinfo, }; /*