From cec38d5c098a350dcf084d345025136ade7e6d1e Mon Sep 17 00:00:00 2001 From: Sean Christopherson Date: Mon, 21 Sep 2026 12:14:09 -0700 Subject: [PATCH 01/20] perf/x86/intel: Ensure KVM guest PEBS path doesn't set unwanted PERF_GLOBAL_CTRL bits When reinstating PEBS counters into PERF_GLOBAL_CTRL for a KVM guest, mask the value with perf's desired/original PERF_GLOBAL_CTRL value to ensure KVM doesn't unintentionally set reserved bits in PERF_GLOBAL_CTRL. E.g. if the guest's PEBS_ENABLE value had bit 63, "Enable Precise Store", set, then using the raw guest PEBS value would propagate bit 63 to the guest's PERF_GLOBAL_CTRL value (which thankfully would be a failed VM-Entry, not a VMX Abort). The only reason this bug isn't reachable is because KVM doesn't support "Enable Precise Store" (which is probably a KVM bug?), i.e. bit 63 can't be set in kvm_pmu->pebs_enable and thus not in arr[pebs_enable].guest. In other words, this _should_ be a glorified NOP in the current code base. Fixes: c59a1f106f5c ("KVM: x86/pmu: Add IA32_PEBS_ENABLE MSR emulation for extended PEBS") Signed-off-by: Sean Christopherson Signed-off-by: Peter Zijlstra (Intel) Signed-off-by: Ingo Molnar Reviewed-by: Dapeng Mi Link: https://patch.msgid.link/20260921191418.950933-2-seanjc@google.com --- arch/x86/events/intel/core.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/arch/x86/events/intel/core.c b/arch/x86/events/intel/core.c index 1ac2ca35db53..0400b5111db9 100644 --- a/arch/x86/events/intel/core.c +++ b/arch/x86/events/intel/core.c @@ -5359,7 +5359,7 @@ static struct perf_guest_switch_msr *intel_guest_get_msrs(int *nr, void *data) arr[pebs_enable].guest &= ~kvm_pmu->host_cross_mapped_mask; arr[global_ctrl].guest &= ~kvm_pmu->host_cross_mapped_mask; /* Set hw GLOBAL_CTRL bits for PEBS counter when it runs for guest */ - arr[global_ctrl].guest |= arr[pebs_enable].guest; + arr[global_ctrl].guest |= intel_ctrl & arr[pebs_enable].guest; } return arr; From 4b64dbdc5861477f148e13d1ed127e7fe7182e4f Mon Sep 17 00:00:00 2001 From: Sean Christopherson Date: Mon, 21 Sep 2026 12:14:10 -0700 Subject: [PATCH 02/20] perf/x86/intel: Don't write PEBS_ENABLED on host<=>guest xfers if CPU has PEBS isolation, to fix stuck PEBS_ENABLED When filling the list of MSRs to be loaded by KVM on VM-Enter and VM-Exit, *never* insert an entry for PEBS_ENABLED if the CPU properly isolates PEBS events, in which case disabling counters via PERF_GLOBAL_CTRL is sufficient to prevent unwanted PEBS events in the guest (or host). Because perf loads PEBS_ENABLE with the unfiltered cpu_hw_events.pebs_enabled, i.e. with both host and guest masks, there is no need to load different values for the guest versus host, perf+KVM can and should simply control which counters are enabled/disabled via PERF_GLOBAL_CTRL. Avoiding touching PEBS_ENABLED "fixes" a bug where PEBS_ENABLED can end up with "stuck" bits if a PEBS event is throttled between generating the list and actually entering the guest (Intel CPUs can't arbtitrarily block NMIs). Fixes in quotes because leaving PEBS_ENABLED as-is doesn't fix the underlying problem of perf (via PMIs) being able to modify state after the perf<=>KVM handoff. But not writing PEBS_ENABLED is desirable no matter what, as stating the obvious, leaving PEBS_ENABLED as-is avoids three MSR writes on every VMX transition: one each on entry/exit, and one more explicit WRMSR to zero PEBS_ENABLED before VM-Entry (KVM assumes the only reason PEBS_ENABLED is in the load list is if the CPU lacks PEBS isolation and thus needs a quiescent period). Opportunistically add comments to (better) explain the rules for generating the set of PEBS counters that will be active while the guest is running, along with a FIXME for the suspected hack-a-fix where perf disables guest PEBS if _any_ PEBS event is configured to count in the host (commit 854250329c02 ("KVM: x86/pmu: Disable guest PEBS temporarily in two rare situations") doesn't explain the motivation, at all). Fixes: c59a1f106f5c ("KVM: x86/pmu: Add IA32_PEBS_ENABLE MSR emulation for extended PEBS") Signed-off-by: Sean Christopherson Signed-off-by: Peter Zijlstra (Intel) Signed-off-by: Ingo Molnar Reviewed-by: Dapeng Mi Link: https://patch.msgid.link/20260921191418.950933-3-seanjc@google.com --- arch/x86/events/intel/core.c | 55 ++++++++++++++++++++++++------------ 1 file changed, 37 insertions(+), 18 deletions(-) diff --git a/arch/x86/events/intel/core.c b/arch/x86/events/intel/core.c index 0400b5111db9..5ac8fe31871b 100644 --- a/arch/x86/events/intel/core.c +++ b/arch/x86/events/intel/core.c @@ -5292,12 +5292,15 @@ static struct perf_guest_switch_msr *intel_guest_get_msrs(int *nr, void *data) struct kvm_pmu *kvm_pmu = (struct kvm_pmu *)data; u64 intel_ctrl = hybrid(cpuc->pmu, intel_ctrl); u64 pebs_mask = cpuc->pebs_enabled & x86_pmu.pebs_capable; - int global_ctrl, pebs_enable; + u64 guest_pebs_mask; + int global_ctrl; /* * In addition to obeying exclude_guest/exclude_host, remove bits being * used for PEBS when running a guest, because PEBS writes to virtual - * addresses (not physical addresses). + * addresses (not physical addresses). If the guest wants to utilize + * PEBS, and PEBS can be safely enabled in the guest, bits for the guest's + * PEBS-enabled counters will be OR'd back in as appropriate. */ *nr = 0; global_ctrl = (*nr)++; @@ -5344,24 +5347,40 @@ static struct perf_guest_switch_msr *intel_guest_get_msrs(int *nr, void *data) }; } - pebs_enable = (*nr)++; - arr[pebs_enable] = (struct perf_guest_switch_msr){ - .msr = MSR_IA32_PEBS_ENABLE, - .host = cpuc->pebs_enabled & ~cpuc->intel_ctrl_guest_mask, - .guest = pebs_mask & ~cpuc->intel_ctrl_host_mask & kvm_pmu->pebs_enable, - }; + /* + * Restrict guest PEBS events to counters that (a) perf supports, (b) + * the guest wants to use for PEBS, (c) are not excluded from counting + * in the guest, and (d) _are_ excluded from counting in the host. + */ + guest_pebs_mask = pebs_mask & intel_ctrl & kvm_pmu->pebs_enable & + ~cpuc->intel_ctrl_host_mask & + cpuc->intel_ctrl_guest_mask; - if (arr[pebs_enable].host) { - /* Disable guest PEBS if host PEBS is enabled. */ - arr[pebs_enable].guest = 0; - } else { - /* Disable guest PEBS thoroughly for cross-mapped PEBS counters. */ - arr[pebs_enable].guest &= ~kvm_pmu->host_cross_mapped_mask; - arr[global_ctrl].guest &= ~kvm_pmu->host_cross_mapped_mask; - /* Set hw GLOBAL_CTRL bits for PEBS counter when it runs for guest */ - arr[global_ctrl].guest |= intel_ctrl & arr[pebs_enable].guest; - } + /* + * Disable counters where the guest PMC is different than the host PMC + * being used on behalf of the guest, as the PEBS record includes + * PERF_GLOBAL_STATUS, i.e. the guest will see overflow status for the + * wrong counter(s). + */ + guest_pebs_mask &= ~kvm_pmu->host_cross_mapped_mask; + /* + * FIXME: Allow guest and host usage of PEBS events to co-exist instead + * of disabling guest PEBS entirely if the host is using PEBS. + * What exactly goes wrong if guest and host are using PEBS is + * unknown. + */ + if (pebs_mask & ~cpuc->intel_ctrl_guest_mask) + guest_pebs_mask = 0; + + /* + * Do NOT mess with PEBS_ENABLED. As above, disabling counters via + * PERF_GLOBAL_CTRL is sufficient, and loading a stale PEBS_ENABLED, + * e.g. on VM-Exit, can put the system in a bad state. Simply enable + * counters in PERF_GLOBAL_CTRL, as perf load PEBS_ENABLED with the + * full value, i.e. perf *also* relies on PERF_GLOBAL_CTRL. + */ + arr[global_ctrl].guest |= guest_pebs_mask; return arr; } From d06260e99eb93d2942b7af4ccd789eb8a6c829d3 Mon Sep 17 00:00:00 2001 From: Sean Christopherson Date: Mon, 21 Sep 2026 12:14:11 -0700 Subject: [PATCH 03/20] perf/x86/intel: Don't pointlessly context switch DS_AREA (and PEBS config) if PEBS is unused When filling the list of MSRs to be loaded by KVM on VM-Enter and VM-Exit, load the guest values for DS_AREA and (conditionally) MSR_PEBS_DATA_CFG if and only if PEBS will be active in the guest, i.e. only if a PEBS record may be generated while running the guest. As shown by the !pebs_ept path, it's perfectly safe to run with the host's DS_AREA, so long as PEBS-enabled counters are disabled via PERF_GLOBAL_CTRL. Omitting DS_AREA and MSR_PEBS_DATA_CFG when PEBS is unused saves two MSR writes per MSR on each VMX transition, i.e. eliminates two/four pointless MSR writes on each VMX roundtrip when PEBS isn't being used by the guest. Fixes: c59a1f106f5c ("KVM: x86/pmu: Add IA32_PEBS_ENABLE MSR emulation for extended PEBS") Signed-off-by: Sean Christopherson Signed-off-by: Peter Zijlstra (Intel) Signed-off-by: Ingo Molnar Reviewed-by: Jim Mattson Reviewed-by: Dapeng Mi Link: https://patch.msgid.link/20260921191418.950933-4-seanjc@google.com --- arch/x86/events/intel/core.c | 39 +++++++++++++++++++++++------------- 1 file changed, 25 insertions(+), 14 deletions(-) diff --git a/arch/x86/events/intel/core.c b/arch/x86/events/intel/core.c index 5ac8fe31871b..75c74e8fe2f4 100644 --- a/arch/x86/events/intel/core.c +++ b/arch/x86/events/intel/core.c @@ -5330,23 +5330,14 @@ static struct perf_guest_switch_msr *intel_guest_get_msrs(int *nr, void *data) return arr; } + /* + * If the guest won't use PEBS or the CPU doesn't support PEBS in the + * guest, then there's nothing more to do as disabling PMCs via + * PERF_GLOBAL_CTRL is sufficient on CPUs with guest/host isolation. + */ if (!kvm_pmu || !x86_pmu.pebs_ept) return arr; - arr[(*nr)++] = (struct perf_guest_switch_msr){ - .msr = MSR_IA32_DS_AREA, - .host = (unsigned long)cpuc->ds, - .guest = kvm_pmu->ds_area, - }; - - if (x86_pmu.intel_cap.pebs_baseline) { - arr[(*nr)++] = (struct perf_guest_switch_msr){ - .msr = MSR_PEBS_DATA_CFG, - .host = cpuc->active_pebs_data_cfg, - .guest = kvm_pmu->pebs_data_cfg, - }; - } - /* * Restrict guest PEBS events to counters that (a) perf supports, (b) * the guest wants to use for PEBS, (c) are not excluded from counting @@ -5373,6 +5364,26 @@ static struct perf_guest_switch_msr *intel_guest_get_msrs(int *nr, void *data) if (pebs_mask & ~cpuc->intel_ctrl_guest_mask) guest_pebs_mask = 0; + /* + * Context switch DS_AREA and PEBS_DATA_CFG if and only if PEBS will be + * active in the guest; if no records will be generated while the guest + * is running, then simply keep the host values resident in hardware. + */ + arr[(*nr)++] = (struct perf_guest_switch_msr){ + .msr = MSR_IA32_DS_AREA, + .host = (unsigned long)cpuc->ds, + .guest = guest_pebs_mask ? kvm_pmu->ds_area : (unsigned long)cpuc->ds, + }; + + if (x86_pmu.intel_cap.pebs_baseline) { + arr[(*nr)++] = (struct perf_guest_switch_msr){ + .msr = MSR_PEBS_DATA_CFG, + .host = cpuc->active_pebs_data_cfg, + .guest = guest_pebs_mask ? kvm_pmu->pebs_data_cfg : + cpuc->active_pebs_data_cfg, + }; + } + /* * Do NOT mess with PEBS_ENABLED. As above, disabling counters via * PERF_GLOBAL_CTRL is sufficient, and loading a stale PEBS_ENABLED, From a391618e1d563f099e4c2a704f45d08329ccdf7c Mon Sep 17 00:00:00 2001 From: Sean Christopherson Date: Mon, 21 Sep 2026 12:14:12 -0700 Subject: [PATCH 04/20] perf/x86/intel: Make @data a mandatory param for intel_guest_get_msrs() Drop "support" for passing a NULL @data/@kvm_pmu param when getting guest MSRs. KVM, the only in-tree user, unconditionally passes a non-NULL pointer, and carrying code that suggests @data may be NULL is confusing, e.g. incorrectly implies that there are scenarios where KVM doesn't pass a PMU context. Fixes: 8183a538cd95 ("KVM: x86/pmu: Add IA32_DS_AREA MSR emulation to support guest DS") Signed-off-by: Sean Christopherson Signed-off-by: Peter Zijlstra (Intel) Signed-off-by: Ingo Molnar Reviewed-by: Jim Mattson Reviewed-by: Dapeng Mi Link: https://patch.msgid.link/20260921191418.950933-5-seanjc@google.com --- arch/x86/events/intel/core.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/arch/x86/events/intel/core.c b/arch/x86/events/intel/core.c index 75c74e8fe2f4..757d4082de60 100644 --- a/arch/x86/events/intel/core.c +++ b/arch/x86/events/intel/core.c @@ -5331,11 +5331,11 @@ static struct perf_guest_switch_msr *intel_guest_get_msrs(int *nr, void *data) } /* - * If the guest won't use PEBS or the CPU doesn't support PEBS in the - * guest, then there's nothing more to do as disabling PMCs via - * PERF_GLOBAL_CTRL is sufficient on CPUs with guest/host isolation. + * If the CPU doesn't support PEBS in the guest, then there's nothing + * more to do as disabling PMCs via PERF_GLOBAL_CTRL is sufficient on + * CPUs with guest/host isolation. */ - if (!kvm_pmu || !x86_pmu.pebs_ept) + if (!x86_pmu.pebs_ept) return arr; /* From 89dc568e8c0be60e05e5fcd0b528c79077d7b84b Mon Sep 17 00:00:00 2001 From: Dapeng Mi Date: Thu, 17 Sep 2026 09:52:23 +0800 Subject: [PATCH 05/20] perf/x86/intel: Fix GRT PEBS load/store direction for latency events, to fix sample classification On Gracemont, intel_grt_pebs_event_constraints[] applies LAT_CONSTRAINT constraints to MEM_UOPS_RETIRED.{LOAD,STORE}_LATENCY, but does not set explicit LOAD/STORE flags for those events. The PEBS latency path (pebs_latency_data(), via __grt_latency_data()) uses the event flags to determine memory operation direction. Without an explicit STORE flag, samples from MEM_UOPS_RETIRED.STORE_LATENCY can be misclassified as LOADs. Set explicit LOAD/STORE flags in intel_grt_pebs_event_constraints[] for: - MEM_UOPS_RETIRED.LOAD_LATENCY - MEM_UOPS_RETIRED.STORE_LATENCY Also update __grt_latency_data() to explicitly interpret these flags when assigning the sampled memory operation direction. This fixes incorrect STORE sample classification. Fixes: 39a41278f041 ("perf/x86/intel: Fix PEBS memory access info encoding for ADL") Signed-off-by: Dapeng Mi Signed-off-by: Peter Zijlstra (Intel) Signed-off-by: Ingo Molnar Cc: # v7.2+ Link: https://patch.msgid.link/20260917015234.981153-2-dapeng1.mi@linux.intel.com --- arch/x86/events/intel/ds.c | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/arch/x86/events/intel/ds.c b/arch/x86/events/intel/ds.c index b98029b44052..fff98f457576 100644 --- a/arch/x86/events/intel/ds.c +++ b/arch/x86/events/intel/ds.c @@ -455,6 +455,7 @@ static inline void pebs_set_tlb_lock(u64 *val, bool tlb, bool lock) static u64 __grt_latency_data(struct perf_event *event, u64 status, u8 dse, bool tlb, bool lock, bool blk) { + union perf_mem_data_src src; u64 val; WARN_ON_ONCE(is_hybrid() && @@ -470,7 +471,16 @@ static u64 __grt_latency_data(struct perf_event *event, u64 status, else val |= P(BLK, NA); - return val; + src.val = val; + + if (event->hw.flags & + (PERF_X86_EVENT_PEBS_LDLAT | PERF_X86_EVENT_PEBS_LD_HSW)) + src.mem_op = P(OP, LOAD); + if (event->hw.flags & + (PERF_X86_EVENT_PEBS_STLAT | PERF_X86_EVENT_PEBS_ST_HSW)) + src.mem_op = P(OP, STORE); + + return src.val; } u64 grt_latency_data(struct perf_event *event, u64 status) @@ -1291,8 +1301,8 @@ struct event_constraint intel_glm_pebs_event_constraints[] = { struct event_constraint intel_grt_pebs_event_constraints[] = { /* Allow all events as PEBS with no flags */ - INTEL_HYBRID_LAT_CONSTRAINT(0x5d0, 0x3), - INTEL_HYBRID_LAT_CONSTRAINT(0x6d0, 0x3f), + INTEL_HYBRID_LDLAT_CONSTRAINT(0x5d0, 0x3), + INTEL_HYBRID_STLAT_CONSTRAINT(0x6d0, 0x3f), EVENT_CONSTRAINT_END }; From e961d6db42d1f1b66c81438969a648f08573bd9c Mon Sep 17 00:00:00 2001 From: Dapeng Mi Date: Thu, 17 Sep 2026 09:52:24 +0800 Subject: [PATCH 06/20] perf/x86/intel: Fix CMT PEBS load/store direction for latency events, to fix sample classification The same bug exists on Crestmont as on Gracemont: intel_cmt_pebs_event_constraints[] applies LAT_CONSTRAINT constraints to MEM_UOPS_RETIRED.{LOAD,STORE}_LATENCY, but does not set explicit LOAD/STORE flags for those events. The PEBS latency path (pebs_latency_data(), via cmt_latency_data) uses the event flags to determine memory operation direction. Without an explicit STORE flag, samples from MEM_UOPS_RETIRED.STORE_LATENCY can be misclassified as LOADs. Set explicit LOAD/STORE flags in intel_cmt_pebs_event_constraints[] for: - MEM_UOPS_RETIRED.LOAD_LATENCY - MEM_UOPS_RETIRED.STORE_LATENCY This fixes incorrect STORE sample classification. Fixes: e99fb45436ea ("perf/x86/intel: Update event constraints and cache_extra_regsfor MTL") Signed-off-by: Dapeng Mi Signed-off-by: Peter Zijlstra (Intel) Signed-off-by: Ingo Molnar Cc: # v7.2+ Link: https://patch.msgid.link/20260917015234.981153-3-dapeng1.mi@linux.intel.com --- arch/x86/events/intel/ds.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/arch/x86/events/intel/ds.c b/arch/x86/events/intel/ds.c index fff98f457576..fa7e0268b4f2 100644 --- a/arch/x86/events/intel/ds.c +++ b/arch/x86/events/intel/ds.c @@ -1308,8 +1308,8 @@ struct event_constraint intel_grt_pebs_event_constraints[] = { struct event_constraint intel_cmt_pebs_event_constraints[] = { /* Allow all events as PEBS with no flags */ - INTEL_HYBRID_LAT_CONSTRAINT(0x5d0, 0x3), - INTEL_HYBRID_LAT_CONSTRAINT(0x6d0, 0xff), + INTEL_HYBRID_LDLAT_CONSTRAINT(0x5d0, 0x3), + INTEL_HYBRID_STLAT_CONSTRAINT(0x6d0, 0xff), EVENT_CONSTRAINT_END }; From 8302c5f475fa5a4ed36a7d32c7b965023d5d7066 Mon Sep 17 00:00:00 2001 From: Dapeng Mi Date: Thu, 17 Sep 2026 09:52:25 +0800 Subject: [PATCH 07/20] perf/x86/intel: Fix DKT PEBS load/store direction for latency events, to fix sample classification Same bug exists on Darkmont as on Gracemont: intel_dkt_pebs_event_constraints[] applies LAT_CONSTRAINT constraints to MEM_UOPS_RETIRED.{LOAD,STORE}_LATENCY, but does not set explicit LOAD/STORE flags for those events. The PEBS latency path (pebs_latency_data(), via cmt_latency_data) uses the event flags to determine memory operation direction. Without an explicit STORE flag, samples from MEM_UOPS_RETIRED.STORE_LATENCY can be misclassified as LOADs. Set explicit LOAD/STORE flags in intel_dkt_pebs_event_constraints[] for: - MEM_UOPS_RETIRED.LOAD_LATENCY - MEM_UOPS_RETIRED.STORE_LATENCY This fixes incorrect STORE sample classification. Additionally remove INTEL_HYBRID_LAT_CONSTRAINT() since no one uses it anymore. Fixes: 65fd435095bb ("perf/x86/intel: Update event constraints for PTL") Signed-off-by: Dapeng Mi Signed-off-by: Peter Zijlstra (Intel) Signed-off-by: Ingo Molnar Cc: # v7.2+ Link: https://patch.msgid.link/20260917015234.981153-4-dapeng1.mi@linux.intel.com --- arch/x86/events/intel/ds.c | 4 ++-- arch/x86/events/perf_event.h | 4 ---- 2 files changed, 2 insertions(+), 6 deletions(-) diff --git a/arch/x86/events/intel/ds.c b/arch/x86/events/intel/ds.c index fa7e0268b4f2..ea5b331c58d1 100644 --- a/arch/x86/events/intel/ds.c +++ b/arch/x86/events/intel/ds.c @@ -1315,8 +1315,8 @@ struct event_constraint intel_cmt_pebs_event_constraints[] = { struct event_constraint intel_dkt_pebs_event_constraints[] = { /* Allow all events as PEBS with no flags */ - INTEL_HYBRID_LAT_CONSTRAINT(0x5d0, 0xff), - INTEL_HYBRID_LAT_CONSTRAINT(0x6d0, 0xff), + INTEL_HYBRID_LDLAT_CONSTRAINT(0x5d0, 0xff), + INTEL_HYBRID_STLAT_CONSTRAINT(0x6d0, 0xff), EVENT_CONSTRAINT_END }; diff --git a/arch/x86/events/perf_event.h b/arch/x86/events/perf_event.h index 4680cba91340..fab9da78a5c7 100644 --- a/arch/x86/events/perf_event.h +++ b/arch/x86/events/perf_event.h @@ -517,10 +517,6 @@ struct cpu_hw_events { __EVENT_CONSTRAINT(c, n, INTEL_ARCH_EVENT_MASK|X86_ALL_EVENT_FLAGS, \ HWEIGHT(n), 0, PERF_X86_EVENT_PEBS_ST) -#define INTEL_HYBRID_LAT_CONSTRAINT(c, n) \ - __EVENT_CONSTRAINT(c, n, INTEL_ARCH_EVENT_MASK|X86_ALL_EVENT_FLAGS, \ - HWEIGHT(n), 0, PERF_X86_EVENT_PEBS_LAT_HYBRID) - #define INTEL_HYBRID_LDLAT_CONSTRAINT(c, n) \ __EVENT_CONSTRAINT(c, n, INTEL_ARCH_EVENT_MASK|X86_ALL_EVENT_FLAGS, \ HWEIGHT(n), 0, PERF_X86_EVENT_PEBS_LAT_HYBRID|PERF_X86_EVENT_PEBS_LD_HSW) From 335b0642812ee0f2b7798b634ce507b8b7c9fe0c Mon Sep 17 00:00:00 2001 From: Dapeng Mi Date: Thu, 17 Sep 2026 09:52:26 +0800 Subject: [PATCH 08/20] perf/x86/intel: Update arw_latency_data() mem-op direction handling Align arw_latency_data() with other *_latency_data() helpers by explicitly decoding LOAD/STORE event flags when setting the sampled memory operation direction. This keeps the latency data path behavior consistent across platforms and avoids relying on implicit direction inference. Signed-off-by: Dapeng Mi Signed-off-by: Peter Zijlstra (Intel) Signed-off-by: Ingo Molnar Link: https://patch.msgid.link/20260917015234.981153-5-dapeng1.mi@linux.intel.com --- arch/x86/events/intel/ds.c | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/arch/x86/events/intel/ds.c b/arch/x86/events/intel/ds.c index ea5b331c58d1..c0004b08b087 100644 --- a/arch/x86/events/intel/ds.c +++ b/arch/x86/events/intel/ds.c @@ -538,7 +538,11 @@ static u64 arw_latency_data(struct perf_event *event, u64 status) val |= P(BLK, NA); src.val = val; - if (event->hw.flags & PERF_X86_EVENT_PEBS_ST_HSW) + if (event->hw.flags & + (PERF_X86_EVENT_PEBS_LDLAT | PERF_X86_EVENT_PEBS_LD_HSW)) + src.mem_op = P(OP, LOAD); + if (event->hw.flags & + (PERF_X86_EVENT_PEBS_STLAT | PERF_X86_EVENT_PEBS_ST_HSW)) src.mem_op = P(OP, STORE); return src.val; From 7c944595cc43a664019edc22505b6d6f6039be5e Mon Sep 17 00:00:00 2001 From: Dapeng Mi Date: Thu, 17 Sep 2026 09:52:27 +0800 Subject: [PATCH 09/20] perf/x86/intel: Remove incorrect LionCove PEBS data-source constraints On Lion Cove, PEBS data source is valid only for these events: - MEM_TRANS_RETIRED.LOAD_LATENCY (0x1cd) - MEM_TRANS_RETIRED.STORE_SAMPLE (0x2cd) The perfmon database (https://github.com/intel/perfmon) previously tagged additional memory events such as MEM_INST_RETIRED.STLB_MISS_LOADS with L1_Hit_Indication, implying PEBS data-source support, which is incorrect. The database has since been fixed, but intel_lnc_pebs_event_constraints[] still follows the old definition and marks those events as data-source capable. As a result, get_data_src() may decode data-source information for events that do not provide valid PEBS data-source data and mislead users. Remove those non-data-source memory events from the Lion Cove PEBS constraint table so matching falls back to the regular non-PEBS constraints, which already provide the same counter constraints. Also update lnc_latency_data() to decode LOAD/STORE flags explicitly when setting memory operation direction, for consistency with other *_latency_data() helpers. Fixes: a932aa0e868f ("perf/x86: Add Lunar Lake and Arrow Lake support") Signed-off-by: Dapeng Mi Signed-off-by: Peter Zijlstra (Intel) Signed-off-by: Ingo Molnar Cc: Link: https://patch.msgid.link/20260917015234.981153-6-dapeng1.mi@linux.intel.com --- arch/x86/events/intel/ds.c | 22 +++++----------------- 1 file changed, 5 insertions(+), 17 deletions(-) diff --git a/arch/x86/events/intel/ds.c b/arch/x86/events/intel/ds.c index c0004b08b087..cbcc0b36536b 100644 --- a/arch/x86/events/intel/ds.c +++ b/arch/x86/events/intel/ds.c @@ -577,7 +577,11 @@ static u64 lnc_latency_data(struct perf_event *event, u64 status) val |= P(BLK, NA); src.val = val; - if (event->hw.flags & PERF_X86_EVENT_PEBS_ST_HSW) + if (event->hw.flags & + (PERF_X86_EVENT_PEBS_LDLAT | PERF_X86_EVENT_PEBS_LD_HSW)) + src.mem_op = P(OP, LOAD); + if (event->hw.flags & + (PERF_X86_EVENT_PEBS_STLAT | PERF_X86_EVENT_PEBS_ST_HSW)) src.mem_op = P(OP, STORE); return src.val; @@ -1510,24 +1514,8 @@ struct event_constraint intel_lnc_pebs_event_constraints[] = { INTEL_FLAGS_UEVENT_CONSTRAINT(0x012a, 0x1), /* OCR.* events */ INTEL_FLAGS_UEVENT_CONSTRAINT(0x012b, 0x1), /* OCR.* events */ - INTEL_FLAGS_UEVENT_CONSTRAINT(0x04a4, 0x1), /* TOPDOWN.BAD_SPEC_SLOTS */ - INTEL_FLAGS_UEVENT_CONSTRAINT(0x08a4, 0x1), /* TOPDOWN.BR_MISPREDICT_SLOTS */ - INTEL_FLAGS_UEVENT_CONSTRAINT(0x10a4, 0x8), /* TOPDOWN.MEMORY_BOUND_SLOTS */ - INTEL_HYBRID_LDLAT_CONSTRAINT(0x1cd, 0x3fc), INTEL_HYBRID_STLAT_CONSTRAINT(0x2cd, 0x3), - INTEL_FLAGS_UEVENT_CONSTRAINT_DATALA_LD(0x11d0, 0xf), /* MEM_INST_RETIRED.STLB_MISS_LOADS */ - INTEL_FLAGS_UEVENT_CONSTRAINT_DATALA_ST(0x12d0, 0xf), /* MEM_INST_RETIRED.STLB_MISS_STORES */ - INTEL_FLAGS_UEVENT_CONSTRAINT_DATALA_LD(0x21d0, 0xf), /* MEM_INST_RETIRED.LOCK_LOADS */ - INTEL_FLAGS_UEVENT_CONSTRAINT_DATALA_LD(0x41d0, 0xf), /* MEM_INST_RETIRED.SPLIT_LOADS */ - INTEL_FLAGS_UEVENT_CONSTRAINT_DATALA_ST(0x42d0, 0xf), /* MEM_INST_RETIRED.SPLIT_STORES */ - INTEL_FLAGS_UEVENT_CONSTRAINT_DATALA_LD(0x81d0, 0xf), /* MEM_INST_RETIRED.ALL_LOADS */ - INTEL_FLAGS_UEVENT_CONSTRAINT_DATALA_ST(0x82d0, 0xf), /* MEM_INST_RETIRED.ALL_STORES */ - INTEL_FLAGS_UEVENT_CONSTRAINT(0x87d0, 0x3ff), /* MEM_INST_RETIRED.ANY */ - - INTEL_FLAGS_EVENT_CONSTRAINT_DATALA_LD_RANGE(0xd1, 0xd4, 0xf), - - INTEL_FLAGS_EVENT_CONSTRAINT(0xd0, 0xf), /* * Everything else is handled by PMU_FL_PEBS_ALL, because we From 9f93d33ad65af9853974c1ec4ba1f937e8ba1376 Mon Sep 17 00:00:00 2001 From: Dapeng Mi Date: Thu, 17 Sep 2026 09:52:28 +0800 Subject: [PATCH 10/20] perf/x86/intel: Remove incorrect Panther Cove PEBS data-source constraints Same issue exists on Panther Cove, PEBS data source is valid only for these events: - MEM_TRANS_RETIRED.LOAD_LATENCY (0x1cd) - MEM_TRANS_RETIRED.STORE_SAMPLE (0x2cd) The perfmon database (https://github.com/intel/perfmon) previously tagged additional memory events such as MEM_INST_RETIRED.STLB_MISS_LOADS with L1_Hit_Indication, implying PEBS data-source support, which is incorrect. The database has since been fixed, but intel_pnc_pebs_event_constraints[] still follows the old definition and marks those events as data-source capable. As a result, get_data_src() may decode data-source information for events that do not provide valid PEBS data-source data and mislead users. Remove those non-data-source memory events from the Pather Cove PEBS constraint table so matching falls back to the regular non-PEBS constraints, which already provide the same counter constraints. Also update pnc_latency_data() to decode LOAD/STORE flags explicitly when setting memory operation direction, for consistency with other *_latency_data() helpers. Fixes: d345b6bb8860 ("perf/x86/intel: Add core PMU support for DMR") Signed-off-by: Dapeng Mi Signed-off-by: Peter Zijlstra (Intel) Signed-off-by: Ingo Molnar Cc: # v7.0+ Link: https://patch.msgid.link/20260917015234.981153-7-dapeng1.mi@linux.intel.com --- arch/x86/events/intel/ds.c | 18 +++++------------- 1 file changed, 5 insertions(+), 13 deletions(-) diff --git a/arch/x86/events/intel/ds.c b/arch/x86/events/intel/ds.c index cbcc0b36536b..72179bcf6d0d 100644 --- a/arch/x86/events/intel/ds.c +++ b/arch/x86/events/intel/ds.c @@ -639,7 +639,11 @@ u64 pnc_latency_data(struct perf_event *event, u64 status) val |= P(BLK, NA); src.val = val; - if (event->hw.flags & PERF_X86_EVENT_PEBS_ST_HSW) + if (event->hw.flags & + (PERF_X86_EVENT_PEBS_LDLAT | PERF_X86_EVENT_PEBS_LD_HSW)) + src.mem_op = P(OP, LOAD); + if (event->hw.flags & + (PERF_X86_EVENT_PEBS_STLAT | PERF_X86_EVENT_PEBS_ST_HSW)) src.mem_op = P(OP, STORE); return src.val; @@ -1528,18 +1532,6 @@ struct event_constraint intel_lnc_pebs_event_constraints[] = { struct event_constraint intel_pnc_pebs_event_constraints[] = { INTEL_HYBRID_LDLAT_CONSTRAINT(0x1cd, 0xfc), INTEL_HYBRID_STLAT_CONSTRAINT(0x2cd, 0x3), - INTEL_FLAGS_UEVENT_CONSTRAINT_DATALA_LD(0x11d0, 0xf), /* MEM_INST_RETIRED.STLB_MISS_LOADS */ - INTEL_FLAGS_UEVENT_CONSTRAINT_DATALA_ST(0x12d0, 0xf), /* MEM_INST_RETIRED.STLB_MISS_STORES */ - INTEL_FLAGS_UEVENT_CONSTRAINT_DATALA_LD(0x21d0, 0xf), /* MEM_INST_RETIRED.LOCK_LOADS */ - INTEL_FLAGS_UEVENT_CONSTRAINT_DATALA_LD(0x41d0, 0xf), /* MEM_INST_RETIRED.SPLIT_LOADS */ - INTEL_FLAGS_UEVENT_CONSTRAINT_DATALA_ST(0x42d0, 0xf), /* MEM_INST_RETIRED.SPLIT_STORES */ - INTEL_FLAGS_UEVENT_CONSTRAINT_DATALA_LD(0x81d0, 0xf), /* MEM_INST_RETIRED.ALL_LOADS */ - INTEL_FLAGS_UEVENT_CONSTRAINT_DATALA_ST(0x82d0, 0xf), /* MEM_INST_RETIRED.ALL_STORES */ - - INTEL_FLAGS_EVENT_CONSTRAINT_DATALA_LD_RANGE(0xd1, 0xd4, 0xf), - - INTEL_FLAGS_EVENT_CONSTRAINT(0xd0, 0xf), - INTEL_FLAGS_EVENT_CONSTRAINT(0xd6, 0xf), /* * Everything else is handled by PMU_FL_PEBS_ALL, because we From 0ac5d6ca2c3b7d047963b67dccef17902dc6c017 Mon Sep 17 00:00:00 2001 From: Dapeng Mi Date: Thu, 17 Sep 2026 09:52:29 +0800 Subject: [PATCH 11/20] perf/x86/intel: Fix Panther Cove PEBS data-source snoop states For Panther Cove, the snoop states for the data source encodings "Prefetch Promotion" and "Cross Core Prefetch Promotion" should be SNOOP_NONE instead of SNOOP_MISS. Fix the incorrect snooping states for Panther Cove. Fixes: d2bdcde9626c ("perf/x86/intel: Add support for PEBS memory auxiliary info field in DMR") Signed-off-by: Dapeng Mi Signed-off-by: Peter Zijlstra (Intel) Signed-off-by: Ingo Molnar Cc: # v7.0+ Link: https://patch.msgid.link/20260917015234.981153-8-dapeng1.mi@linux.intel.com --- arch/x86/events/intel/ds.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/arch/x86/events/intel/ds.c b/arch/x86/events/intel/ds.c index 72179bcf6d0d..d0329a7eb8a5 100644 --- a/arch/x86/events/intel/ds.c +++ b/arch/x86/events/intel/ds.c @@ -277,8 +277,8 @@ static u64 pnc_pebs_l2_hit_data_source[PNC_PEBS_DATA_SOURCE_MAX] = { 0, /* 0x06: Reserved */ OP_LH | P(LVL, L2) | LEVEL(L2) | P(SNOOP, HIT), /* 0x07: L2 Hit Snoop HIT */ OP_LH | P(LVL, L2) | LEVEL(L2) | P(SNOOP, HITM), /* 0x08: L2 Hit Snoop Hit Modified */ - OP_LH | P(LVL, L2) | LEVEL(L2) | P(SNOOP, MISS), /* 0x09: Prefetch Promotion */ - OP_LH | P(LVL, L2) | LEVEL(L2) | P(SNOOP, MISS), /* 0x0a: Cross Core Prefetch Promotion */ + OP_LH | P(LVL, L2) | LEVEL(L2) | P(SNOOP, NONE), /* 0x09: Prefetch Promotion */ + OP_LH | P(LVL, L2) | LEVEL(L2) | P(SNOOP, NONE), /* 0x0a: Cross Core Prefetch Promotion */ 0, /* 0x0b: Reserved */ 0, /* 0x0c: Reserved */ 0, /* 0x0d: Reserved */ From 858b37ca19d3f695f7fa94cd14be5a856fd8e7d7 Mon Sep 17 00:00:00 2001 From: Dapeng Mi Date: Thu, 17 Sep 2026 09:52:30 +0800 Subject: [PATCH 12/20] perf/x86/intel: Delete dead NVL PEBS data-source initcall Nova Lake now uses the OMR data-source table for PEBS data-source decoding and no longer depends on the legacy static pebs_data_source[] mapping. Remove the dead intel_pmu_pebs_data_source_lnl() initialization call for NVL. Fixes: c847a208f43b ("perf/x86/intel: Add core PMU support for Novalake") Signed-off-by: Dapeng Mi Signed-off-by: Peter Zijlstra (Intel) Signed-off-by: Ingo Molnar Link: https://patch.msgid.link/20260917015234.981153-9-dapeng1.mi@linux.intel.com --- arch/x86/events/intel/core.c | 2 -- 1 file changed, 2 deletions(-) diff --git a/arch/x86/events/intel/core.c b/arch/x86/events/intel/core.c index 757d4082de60..a7dbbed8f94e 100644 --- a/arch/x86/events/intel/core.c +++ b/arch/x86/events/intel/core.c @@ -8824,8 +8824,6 @@ __init int intel_pmu_init(void) /* Initialize Atom core specific PerfMon capabilities.*/ pmu = &x86_pmu.hybrid_pmu[X86_HYBRID_PMU_ATOM_IDX]; intel_pmu_init_arw(&pmu->pmu); - - intel_pmu_pebs_data_source_lnl(); break; default: From 04a7ef3b7aa3af34202285043e3423a2e3983595 Mon Sep 17 00:00:00 2001 From: Dapeng Mi Date: Thu, 17 Sep 2026 09:52:31 +0800 Subject: [PATCH 13/20] perf/x86/intel: Constrain Panther Cove UOPS_DISPATCHED events to PMCs 0-3 Per the latest Panther Cove event definitions, the following events are only supported on PMCs 0-3: - UOPS_DISPATCHED.INT_EU_ALL (0x1b2) - UOPS_DISPATCHED.ALU (0x2b2) Add explicit event constraints for these two events so scheduling does not place them on unsupported counters. Fixes: d345b6bb8860 ("perf/x86/intel: Add core PMU support for DMR") Signed-off-by: Dapeng Mi Signed-off-by: Peter Zijlstra (Intel) Signed-off-by: Ingo Molnar Cc: # v7.0+ Link: https://patch.msgid.link/20260917015234.981153-10-dapeng1.mi@linux.intel.com --- arch/x86/events/intel/core.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/arch/x86/events/intel/core.c b/arch/x86/events/intel/core.c index a7dbbed8f94e..ecf6d6325dfe 100644 --- a/arch/x86/events/intel/core.c +++ b/arch/x86/events/intel/core.c @@ -506,6 +506,8 @@ static struct event_constraint intel_pnc_event_constraints[] = { INTEL_EVENT_CONSTRAINT(0xce, 0x1), INTEL_UEVENT_CONSTRAINT(0x01b1, 0x8), + INTEL_UEVENT_CONSTRAINT(0x01b2, 0xf), + INTEL_UEVENT_CONSTRAINT(0x02b2, 0xf), INTEL_UEVENT_CONSTRAINT(0x0847, 0xf), INTEL_UEVENT_CONSTRAINT(0x0446, 0xf), INTEL_UEVENT_CONSTRAINT(0x0846, 0xf), From ff1621adfdd7cfb73f2ccdd3856cd3462ac98ac5 Mon Sep 17 00:00:00 2001 From: Dapeng Mi Date: Thu, 17 Sep 2026 09:52:32 +0800 Subject: [PATCH 14/20] perf/x86/intel: Fix precise OMR event scheduling for DMR/NVL The latest perfmon event database introduces below precise OMR event support for DMR/NVL: - MEM_LOAD_L2_MISS_RETIRED.* (event 0xd6) - MEM_STORE_L2_MISS_RETIRED.* (event 0x4f) These events use the same OMR MSRs as the existing OMR events, but they are not listed in intel_pnc_extra_regs[]. As a result, perf cannot assign the required OMR extra registers when scheduling them. Add the new precise OMR events to intel_pnc_extra_regs[] so they can be scheduled with the correct OMR MSRs. MEM_LOAD_L2_MISS_RETIRED.* remains limited to GP counters 0-3, while MEM_STORE_L2_MISS_RETIRED.* is available on all GP counters. Signed-off-by: Dapeng Mi Signed-off-by: Peter Zijlstra (Intel) Signed-off-by: Ingo Molnar Link: https://patch.msgid.link/20260917015234.981153-11-dapeng1.mi@linux.intel.com --- arch/x86/events/intel/core.c | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/arch/x86/events/intel/core.c b/arch/x86/events/intel/core.c index ecf6d6325dfe..22715602f7f8 100644 --- a/arch/x86/events/intel/core.c +++ b/arch/x86/events/intel/core.c @@ -522,6 +522,14 @@ static struct extra_reg intel_pnc_extra_regs[] __read_mostly = { INTEL_UEVENT_EXTRA_REG(0x022a, MSR_OMR_1, 0x40ffffff0000ffffull, OMR_1), INTEL_UEVENT_EXTRA_REG(0x042a, MSR_OMR_2, 0x40ffffff0000ffffull, OMR_2), INTEL_UEVENT_EXTRA_REG(0x082a, MSR_OMR_3, 0x40ffffff0000ffffull, OMR_3), + INTEL_UEVENT_EXTRA_REG(0x014f, MSR_OMR_0, 0x40ffffff0000ffffull, OMR_0), + INTEL_UEVENT_EXTRA_REG(0x024f, MSR_OMR_1, 0x40ffffff0000ffffull, OMR_1), + INTEL_UEVENT_EXTRA_REG(0x044f, MSR_OMR_2, 0x40ffffff0000ffffull, OMR_2), + INTEL_UEVENT_EXTRA_REG(0x084f, MSR_OMR_3, 0x40ffffff0000ffffull, OMR_3), + INTEL_UEVENT_EXTRA_REG(0x01d6, MSR_OMR_0, 0x40ffffff0000ffffull, OMR_0), + INTEL_UEVENT_EXTRA_REG(0x02d6, MSR_OMR_1, 0x40ffffff0000ffffull, OMR_1), + INTEL_UEVENT_EXTRA_REG(0x04d6, MSR_OMR_2, 0x40ffffff0000ffffull, OMR_2), + INTEL_UEVENT_EXTRA_REG(0x08d6, MSR_OMR_3, 0x40ffffff0000ffffull, OMR_3), INTEL_UEVENT_PEBS_LDLAT_EXTRA_REG(0x01cd), INTEL_UEVENT_EXTRA_REG(0x02c6, MSR_PEBS_FRONTEND, 0x9, FE), INTEL_UEVENT_EXTRA_REG(0x03c6, MSR_PEBS_FRONTEND, 0x7fff1f, FE), From 0102c8c7fdfc4bd700971daff2b94470f4eafae4 Mon Sep 17 00:00:00 2001 From: Dapeng Mi Date: Thu, 17 Sep 2026 09:52:33 +0800 Subject: [PATCH 15/20] perf/x86/intel: Rename DMR offcore_rsp attribute to offmodule_rsp DMR introduces Offmodule Response events in place of the legacy Offcore Response events, but it still exposes the inherited offcore_rsp PMU attribute for programming the corresponding MSR data. Rename the DMR PMU attribute to offmodule_rsp so the sysfs interface matches the underlying event name and avoids user & tooling confusion. Signed-off-by: Dapeng Mi Signed-off-by: Peter Zijlstra (Intel) Signed-off-by: Ingo Molnar Link: https://patch.msgid.link/20260917015234.981153-12-dapeng1.mi@linux.intel.com --- arch/x86/events/intel/core.c | 23 ++++++++++++++++++++--- 1 file changed, 20 insertions(+), 3 deletions(-) diff --git a/arch/x86/events/intel/core.c b/arch/x86/events/intel/core.c index 22715602f7f8..3843dbe530a5 100644 --- a/arch/x86/events/intel/core.c +++ b/arch/x86/events/intel/core.c @@ -6601,6 +6601,8 @@ static void intel_pmu_filter(struct pmu *pmu, int cpu, bool *ret) PMU_FORMAT_ATTR(offcore_rsp, "config1:0-63"); +PMU_FORMAT_ATTR(offmodule_rsp, "config1:0-63"); + PMU_FORMAT_ATTR(ldlat, "config1:0-15"); PMU_FORMAT_ATTR(frontend, "config1:0-23"); @@ -6649,6 +6651,20 @@ static struct attribute *skl_format_attr[] = { NULL, }; +static struct attribute *pnc_format_attr_rtm[] = { + &format_attr_in_tx.attr, + &format_attr_in_tx_cp.attr, + &format_attr_offmodule_rsp.attr, + &format_attr_ldlat.attr, + NULL +}; + +static struct attribute *pnc_format_attr[] = { + &format_attr_offmodule_rsp.attr, + &format_attr_ldlat.attr, + NULL +}; + static __initconst const struct x86_pmu core_pmu = { .name = "core", .handle_irq = x86_pmu_handle_irq, @@ -8589,6 +8605,8 @@ __init int intel_pmu_init(void) case INTEL_DIAMONDRAPIDS_X: intel_pmu_init_pnc(NULL); x86_pmu.pebs_latency_data = pnc_latency_data; + extra_attr = boot_cpu_has(X86_FEATURE_RTM) ? + pnc_format_attr_rtm : pnc_format_attr; pr_cont("Panthercove events, "); name = "panthercove"; @@ -8597,13 +8615,12 @@ __init int intel_pmu_init(void) glc_common: intel_pmu_init_glc(NULL); intel_pmu_pebs_data_source_skl(true); - + extra_attr = boot_cpu_has(X86_FEATURE_RTM) ? + hsw_format_attr : nhm_format_attr; glc_base: x86_pmu.pebs_ept = 1; x86_pmu.hw_config = hsw_hw_config; x86_pmu.get_event_constraints = glc_get_event_constraints; - extra_attr = boot_cpu_has(X86_FEATURE_RTM) ? - hsw_format_attr : nhm_format_attr; extra_skl_attr = skl_format_attr; mem_attr = glc_events_attrs; td_attr = glc_td_events_attrs; From d4d9ccbad527af115b8e83a7a2b74785c0e8dfea Mon Sep 17 00:00:00 2001 From: Dapeng Mi Date: Thu, 17 Sep 2026 09:52:34 +0800 Subject: [PATCH 16/20] perf/x86/intel: Rename NVL offcore_rsp attribute to offmodule_rsp NVL introduces Offmodule Response events in place of the legacy Offcore Response events, but it still exposes the inherited offcore_rsp PMU attribute for programming the corresponding MSR data. Rename the NVL PMU attribute to offmodule_rsp so the sysfs interface matches the underlying event name and avoids user & tooling confusion. Signed-off-by: Dapeng Mi Signed-off-by: Peter Zijlstra (Intel) Signed-off-by: Ingo Molnar Link: https://patch.msgid.link/20260917015234.981153-13-dapeng1.mi@linux.intel.com --- arch/x86/events/intel/core.c | 20 +++++++++++++++++++- 1 file changed, 19 insertions(+), 1 deletion(-) diff --git a/arch/x86/events/intel/core.c b/arch/x86/events/intel/core.c index 3843dbe530a5..3ef80882843e 100644 --- a/arch/x86/events/intel/core.c +++ b/arch/x86/events/intel/core.c @@ -7530,6 +7530,7 @@ static struct attribute *adl_hybrid_tsx_attrs[] = { FORMAT_ATTR_HYBRID(in_tx, hybrid_big); FORMAT_ATTR_HYBRID(in_tx_cp, hybrid_big); FORMAT_ATTR_HYBRID(offcore_rsp, hybrid_big_small_tiny); +FORMAT_ATTR_HYBRID(offmodule_rsp, hybrid_big_small_tiny); FORMAT_ATTR_HYBRID(ldlat, hybrid_big_small_tiny); FORMAT_ATTR_HYBRID(frontend, hybrid_big); @@ -7568,6 +7569,23 @@ static struct attribute *mtl_hybrid_extra_attr[] = { NULL }; +static struct attribute *nvl_hybrid_extra_attr_rtm[] = { + ADL_HYBRID_RTM_FORMAT_ATTR, + FORMAT_HYBRID_PTR(offmodule_rsp), + FORMAT_HYBRID_PTR(ldlat), + FORMAT_HYBRID_PTR(frontend), + FORMAT_HYBRID_PTR(snoop_rsp), + NULL +}; + +static struct attribute *nvl_hybrid_extra_attr[] = { + FORMAT_HYBRID_PTR(offmodule_rsp), + FORMAT_HYBRID_PTR(ldlat), + FORMAT_HYBRID_PTR(frontend), + FORMAT_HYBRID_PTR(snoop_rsp), + NULL +}; + static bool is_attr_for_this_pmu(struct kobject *kobj, struct attribute *attr) { struct device *dev = kobj_to_dev(kobj); @@ -8842,7 +8860,7 @@ __init int intel_pmu_init(void) mem_attr = mtl_hybrid_mem_attrs; tsx_attr = adl_hybrid_tsx_attrs; extra_attr = boot_cpu_has(X86_FEATURE_RTM) ? - mtl_hybrid_extra_attr_rtm : mtl_hybrid_extra_attr; + nvl_hybrid_extra_attr_rtm : nvl_hybrid_extra_attr; /* Initialize big core specific PerfMon capabilities.*/ pmu = &x86_pmu.hybrid_pmu[X86_HYBRID_PMU_CORE_IDX]; From cca4980630b3c7a85f53cb43c6018184ce5d4e37 Mon Sep 17 00:00:00 2001 From: Namhyung Kim Date: Sun, 20 Sep 2026 16:16:39 -0700 Subject: [PATCH 17/20] perf/core: Fix a refcount leak in attach_perf_ctx_data() The attach_perf_ctx_data() can race on global and !global cases. The global case is protected by global_ctx_data_rwsem and shares a single reference count using perf_ctx_data.global field. But when it races with !global case, it may miss to set the global field and result in a reference count leak. CPU1 CPU2 ---------------------------------------------------------------- attach_task_ctx_data(.global=1) attach_task_ctx_data(.global=0) cd1 = alloc_perf_ctx_data(); cd2 = alloc_perf_ctx_data(); // { .global = 0, .refcount = 1 }; try_cmpxchg(); // success, // task->perf_ctx_data = cd2 try_cmpxhg(); // fail; old = cd2 refcount_inc_not_zero(&old->refcount); // success // old.refcount = 2 free_perf_ctx_data(cd1); Then later detach_global_ctx_data() will see the data but it's not marked as global, so it won't call detach_task_ctx_data(). Fixes: 506e64e710ff ("perf: attach/detach PMU specific data") Assisted-by: Sashiko.dev:Gemini-3.1-pro Signed-off-by: Namhyung Kim Signed-off-by: Peter Zijlstra (Intel) Link: https://patch.msgid.link/20260920231639.11910-1-namhyung@kernel.org --- kernel/events/core.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/kernel/events/core.c b/kernel/events/core.c index db7b76d6b68a..e180134bad5e 100644 --- a/kernel/events/core.c +++ b/kernel/events/core.c @@ -5454,6 +5454,8 @@ attach_task_ctx_data(struct task_struct *task, struct kmem_cache *ctx_cache, } if (refcount_inc_not_zero(&old->refcount)) { + if (global) + old->global = true; free_perf_ctx_data(cd); /* unused */ return 0; } From 36bb85cf36cab15fb611cb44b78a5df06e4e69a2 Mon Sep 17 00:00:00 2001 From: Puranjay Mohan Date: Mon, 10 Aug 2026 06:35:34 -0700 Subject: [PATCH 18/20] perf/core: Fix NULL pmu_ctx passed to pmu->sched_task() perf_pmu_sched_task() returns early when cpuctx->task_ctx is set, and cpc->task_epc is only non-NULL while a task context is scheduled in on this CPU. __perf_pmu_sched_task() therefore always passes NULL: Unable to handle kernel NULL pointer dereference at virtual address 00 pc : armv8pmu_sched_task+0x14/0x50 Call trace: armv8pmu_sched_task+0x14/0x50 (P) perf_pmu_sched_task+0xac/0x108 __perf_event_task_sched_out+0x6c/0xe0 Pass &cpc->epc instead, the CPU-wide context for this PMU, which the function already dereferences a few lines up to find pmu. armv8pmu_sched_task() is the only in-tree implementation that dereferences the argument, and it only reads ->pmu, so the oops needs BRBE, added in v6.17. Fixes: bd2756811766 ("perf: Rewrite core context handling") Signed-off-by: Puranjay Mohan Signed-off-by: Peter Zijlstra (Intel) Tested-by: Yifan Wu Cc: stable@vger.kernel.org Link: https://patch.msgid.link/20260810133540.1947118-2-puranjay@kernel.org --- kernel/events/core.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/kernel/events/core.c b/kernel/events/core.c index e180134bad5e..7ce72f366f6c 100644 --- a/kernel/events/core.c +++ b/kernel/events/core.c @@ -3914,7 +3914,7 @@ static void __perf_pmu_sched_task(struct perf_cpu_pmu_context *cpc, perf_ctx_lock(cpuctx, cpuctx->task_ctx); perf_pmu_disable(pmu); - pmu->sched_task(cpc->task_epc, task, sched_in); + pmu->sched_task(&cpc->epc, task, sched_in); perf_pmu_enable(pmu); perf_ctx_unlock(cpuctx, cpuctx->task_ctx); From 3d8d74100954a3b17e5c5e37adfe14e16b1db103 Mon Sep 17 00:00:00 2001 From: Puranjay Mohan Date: Mon, 10 Aug 2026 06:35:35 -0700 Subject: [PATCH 19/20] perf/core: Run sched_task() for PMUs with only CPU-wide events perf_pmu_sched_task() returns early when cpuctx->task_ctx is set and leaves the work to perf_ctx_sched_task_cb(), which only walks ctx->pmu_ctx_list. A PMU whose events are all CPU-wide is not on that list, so nothing calls its sched_task(). With perf record -b -e cycles -a -- ls armv8pmu_sched_task() is skipped on every switch to a task that has a perf context but no event on that PMU, and BRBE records leak across the task boundary. intel_pmu_lbr_add() calls perf_sched_cb_inc() unconditionally too, so LBR records leak the same way on x86. Drop the early return and skip only the CPCs that perf_ctx_sched_task_cb() handles. That one needs a gate of its own to make the split exact: it tests cpc->sched_cb_usage, which perf_sched_cb_inc() sets per CPU for every branch stack user, so a task with an event for that PMU pinned to another CPU would be handled twice. On x86 the second __intel_pmu_lbr_restore() finds lbr_stack_state == LBR_NONE and calls intel_pmu_lbr_reset(), throwing away the callstack the first one restored. cpc->task_epc is set only while a task context is scheduled in, and there is one epc per PMU on ctx->pmu_ctx_list, so the two gates are inverses. For the CPCs perf_pmu_sched_task() picks up, the callback now runs outside the perf_ctx_disable() and perf_ctx_enable() pair in perf_event_context_sched_in(). __perf_pmu_sched_task() disables the PMU around the call itself. Fixes: bd2756811766 ("perf: Rewrite core context handling") Signed-off-by: Puranjay Mohan Signed-off-by: Peter Zijlstra (Intel) Tested-by: Yifan Wu Link: https://patch.msgid.link/20260810133540.1947118-3-puranjay@kernel.org Cc: stable@vger.kernel.org --- kernel/events/core.c | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/kernel/events/core.c b/kernel/events/core.c index 7ce72f366f6c..634d2ccbab82 100644 --- a/kernel/events/core.c +++ b/kernel/events/core.c @@ -3764,6 +3764,9 @@ static void perf_ctx_sched_task_cb(struct perf_event_context *ctx, list_for_each_entry(pmu_ctx, &ctx->pmu_ctx_list, pmu_ctx_entry) { cpc = this_cpc(pmu_ctx->pmu); + if (cpc->task_epc != pmu_ctx) + continue; + if (cpc->sched_cb_usage && pmu_ctx->pmu->sched_task) pmu_ctx->pmu->sched_task(pmu_ctx, task, sched_in); } @@ -3924,15 +3927,17 @@ static void perf_pmu_sched_task(struct task_struct *prev, struct task_struct *next, bool sched_in) { - struct perf_cpu_context *cpuctx = this_cpu_ptr(&perf_cpu_context); struct perf_cpu_pmu_context *cpc, *cpc2; - /* cpuctx->task_ctx will be handled in perf_event_context_sched_in/out */ - if (prev == next || cpuctx->task_ctx) + if (prev == next) return; - list_for_each_entry_safe(cpc, cpc2, this_cpu_ptr(&sched_cb_list), sched_cb_entry) + list_for_each_entry_safe(cpc, cpc2, this_cpu_ptr(&sched_cb_list), sched_cb_entry) { + if (cpc->task_epc) + continue; + __perf_pmu_sched_task(cpc, sched_in ? next : prev, sched_in); + } } static void perf_event_switch(struct task_struct *task, From 24b620729e53d978b3e425f55bc66efd3bab1f59 Mon Sep 17 00:00:00 2001 From: Puranjay Mohan Date: Mon, 10 Aug 2026 06:35:36 -0700 Subject: [PATCH 20/20] perf/core: Fill branch entries with a single assignment perf_clear_branch_entry_bitfields() clears the bitfields of struct perf_branch_entry one by one and leaves from/to alone, since callers overwrite those straight away. The list has to be kept in sync with the struct by hand and has already fallen behind: new_type and priv were added to perf_branch_entry and never added here. Only BRBE writes those two, and neither for every record. brbe_set_perf_entry_type() leaves new_type alone for a branch type it does not recognise, and priv is not set for source-only records. arm_pmuv3.c allocates the per-CPU branch stack with kmalloc(), so such a record reaches userspace with whatever the slot held: uninitialised kmalloc() data on the first pass over the buffer, the previous record's values after that. Nothing under arch/x86/events/ writes either field, so only arm64 is affected. Assign the whole entry at each site instead. Everything not named is then zero, and there is no list to keep in sync. The bitfields add up to exactly 64 bits, so the struct has no padding to leave undefined. perf_clear_branch_entry_bitfields() has no callers left, so remove it. perf_entry_from_brbe_regset() assigns an empty literal instead, since it fills from/to conditionally. PERF_BR_SPEC_NA is 0, so dropping the explicit spec assignment changes nothing. Fixes: b190bc4ac9e6 ("perf: Extend branch type classification") Fixes: 5402d25aa571 ("perf: Capture branch privilege information") Suggested-by: Peter Zijlstra Signed-off-by: Puranjay Mohan Signed-off-by: Peter Zijlstra (Intel) Tested-by: Yifan Wu Link: https://patch.msgid.link/20260810133540.1947118-4-puranjay@kernel.org --- arch/x86/events/amd/brs.c | 9 +++--- arch/x86/events/amd/lbr.c | 16 +++++----- arch/x86/events/intel/lbr.c | 63 ++++++++++++++++++++----------------- drivers/perf/arm_brbe.c | 2 +- include/linux/perf_event.h | 17 ---------- 5 files changed, 47 insertions(+), 60 deletions(-) diff --git a/arch/x86/events/amd/brs.c b/arch/x86/events/amd/brs.c index dc564688f3d7..54b13faba116 100644 --- a/arch/x86/events/amd/brs.c +++ b/arch/x86/events/amd/brs.c @@ -343,11 +343,10 @@ void amd_brs_drain(void) if (!amd_brs_match_plm(event, from, to)) continue; - perf_clear_branch_entry_bitfields(br+nr); - - br[nr].from = from; - br[nr].to = to; - + br[nr] = (struct perf_branch_entry){ + .from = from, + .to = to, + }; nr++; } empty: diff --git a/arch/x86/events/amd/lbr.c b/arch/x86/events/amd/lbr.c index 9d9c961989d5..a55646fcb846 100644 --- a/arch/x86/events/amd/lbr.c +++ b/arch/x86/events/amd/lbr.c @@ -184,13 +184,6 @@ void amd_pmu_lbr_read(void) entry.to.split.reserved) continue; - perf_clear_branch_entry_bitfields(br + out); - - br[out].from = sign_ext_branch_ip(entry.from.split.ip); - br[out].to = sign_ext_branch_ip(entry.to.split.ip); - br[out].mispred = entry.from.split.mispredict; - br[out].predicted = !br[out].mispred; - /* * Set branch speculation information using the status of * the valid and spec bits. @@ -208,7 +201,14 @@ void amd_pmu_lbr_read(void) * speculative and took the correct path */ idx = (entry.to.split.valid << 1) | entry.to.split.spec; - br[out].spec = lbr_spec_map[idx]; + + br[out] = (struct perf_branch_entry){ + .from = sign_ext_branch_ip(entry.from.split.ip), + .to = sign_ext_branch_ip(entry.to.split.ip), + .mispred = entry.from.split.mispredict, + .predicted = !entry.from.split.mispredict, + .spec = lbr_spec_map[idx], + }; out++; } diff --git a/arch/x86/events/intel/lbr.c b/arch/x86/events/intel/lbr.c index cbe5c762008d..22e2a06d5786 100644 --- a/arch/x86/events/intel/lbr.c +++ b/arch/x86/events/intel/lbr.c @@ -756,10 +756,10 @@ void intel_pmu_lbr_read_32(struct cpu_hw_events *cpuc) rdmsrq(x86_pmu.lbr_from + lbr_idx, msr_lastbranch.lbr); - perf_clear_branch_entry_bitfields(br); - - br->from = msr_lastbranch.from; - br->to = msr_lastbranch.to; + *br = (struct perf_branch_entry){ + .from = msr_lastbranch.from, + .to = msr_lastbranch.to, + }; br++; } cpuc->lbr_stack.nr = i; @@ -847,14 +847,15 @@ void intel_pmu_lbr_read_64(struct cpu_hw_events *cpuc) if (abort && x86_pmu.lbr_double_abort && out > 0) out--; - perf_clear_branch_entry_bitfields(br+out); - br[out].from = from; - br[out].to = to; - br[out].mispred = mis; - br[out].predicted = pred; - br[out].in_tx = in_tx; - br[out].abort = abort; - br[out].cycles = cycles; + br[out] = (struct perf_branch_entry){ + .from = from, + .to = to, + .mispred = mis, + .predicted = pred, + .in_tx = in_tx, + .abort = abort, + .cycles = cycles, + }; out++; } cpuc->lbr_stack.nr = out; @@ -905,6 +906,7 @@ static void intel_pmu_store_lbr(struct cpu_hw_events *cpuc, struct perf_branch_entry *e; struct lbr_entry *lbr; u64 from, to, info; + bool mispred; int i; for (i = 0; i < x86_pmu.lbr_nr; i++) { @@ -921,24 +923,27 @@ static void intel_pmu_store_lbr(struct cpu_hw_events *cpuc, to = rdlbr_to(i, lbr); info = rdlbr_info(i, lbr); - perf_clear_branch_entry_bitfields(e); + mispred = get_lbr_mispred(info); - e->from = from; - e->to = to; - e->mispred = get_lbr_mispred(info); - e->predicted = !e->mispred; - e->in_tx = !!(info & LBR_INFO_IN_TX); - e->abort = !!(info & LBR_INFO_ABORT); - e->cycles = get_lbr_cycles(info); - e->type = get_lbr_br_type(info); - - /* - * Leverage the reserved field of cpuc->lbr_entries[i] to - * temporarily store the branch counters information. - * The later code will decide what content can be disclosed - * to the perf tool. Pleae see intel_pmu_lbr_counters_reorder(). - */ - e->reserved = (info >> LBR_INFO_BR_CNTR_OFFSET) & LBR_INFO_BR_CNTR_FULL_MASK; + *e = (struct perf_branch_entry){ + .from = from, + .to = to, + .mispred = mispred, + .predicted = !mispred, + .in_tx = !!(info & LBR_INFO_IN_TX), + .abort = !!(info & LBR_INFO_ABORT), + .cycles = get_lbr_cycles(info), + .type = get_lbr_br_type(info), + /* + * Leverage the reserved field of + * cpuc->lbr_entries[i] to temporarily store the + * branch counters information. The later code will + * decide what content can be disclosed to the perf + * tool. Pleae see intel_pmu_lbr_counters_reorder(). + */ + .reserved = (info >> LBR_INFO_BR_CNTR_OFFSET) & + LBR_INFO_BR_CNTR_FULL_MASK, + }; } cpuc->lbr_stack.nr = i; diff --git a/drivers/perf/arm_brbe.c b/drivers/perf/arm_brbe.c index ba554e0c846c..254be4da8ae2 100644 --- a/drivers/perf/arm_brbe.c +++ b/drivers/perf/arm_brbe.c @@ -604,7 +604,7 @@ static bool perf_entry_from_brbe_regset(int index, struct perf_branch_entry *ent return false; brbinf = bregs.brbinf; - perf_clear_branch_entry_bitfields(entry); + *entry = (struct perf_branch_entry){ }; if (brbe_record_is_complete(brbinf)) { entry->from = bregs.brbsrc; entry->to = bregs.brbtgt; diff --git a/include/linux/perf_event.h b/include/linux/perf_event.h index 5842552294c1..915c6fd3f084 100644 --- a/include/linux/perf_event.h +++ b/include/linux/perf_event.h @@ -1467,23 +1467,6 @@ static inline u32 perf_sample_data_size(struct perf_sample_data *data, return size; } -/* - * Clear all bitfields in the perf_branch_entry. - * The to and from fields are not cleared because they are - * systematically modified by caller. - */ -static inline void perf_clear_branch_entry_bitfields(struct perf_branch_entry *br) -{ - br->mispred = 0; - br->predicted = 0; - br->in_tx = 0; - br->abort = 0; - br->cycles = 0; - br->type = 0; - br->spec = PERF_BR_SPEC_NA; - br->reserved = 0; -} - extern void perf_output_sample(struct perf_output_handle *handle, struct perf_event_header *header, struct perf_sample_data *data,