From 59e63416f5153e7d58652c616fbdcb7d5e01fff7 Mon Sep 17 00:00:00 2001 From: Thomas Richter Date: Tue, 8 Sep 2026 12:56:37 +0200 Subject: [PATCH 1/3] perf/core: Allow list_del during perf_event_overflow() A PMU might use perf_sched_cb_inc() and perf_sched_cb_dec() interface to get the PMU call back function pmu::sched_task invoked at schedule in and schedule out. This is achieved by walking along the list anchored by sched_cb_list. The following scenario might lead to a list corruption. perf_pmu_sched_task() for_each_list_entry(..., &sched_cb_list) +--> __perf_pmu_sched_task() +--> event->pmu->sched_task()) +--> PMU_push_sample() +--> perf_event_overflow() +--> __perf_event_overflow() +--> pmu->stop() +--> perf_sched_cb_dec() remove entry from sched_cb_list while list node in use. This happens when ioctl(fd, PERF_EVENT_IOC_REFRESH, xxx) has been invoked and perf_event::event_limit hits zero. Prevent the list corruption and convert for_each_list_entry() to for_each_list_entry_safe(). Fixes: bd2756811766 ("perf: Rewrite core context handling") Signed-off-by: Thomas Richter Signed-off-by: Peter Zijlstra (Intel) Link: https://patch.msgid.link/20260908105637.627004-1-tmricht@linux.ibm.com --- kernel/events/core.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/kernel/events/core.c b/kernel/events/core.c index 33210aff3ee6..fe33fe15689d 100644 --- a/kernel/events/core.c +++ b/kernel/events/core.c @@ -3925,13 +3925,13 @@ static void perf_pmu_sched_task(struct task_struct *prev, bool sched_in) { struct perf_cpu_context *cpuctx = this_cpu_ptr(&perf_cpu_context); - struct perf_cpu_pmu_context *cpc; + struct perf_cpu_pmu_context *cpc, *cpc2; /* cpuctx->task_ctx will be handled in perf_event_context_sched_in/out */ if (prev == next || cpuctx->task_ctx) return; - list_for_each_entry(cpc, this_cpu_ptr(&sched_cb_list), sched_cb_entry) + list_for_each_entry_safe(cpc, cpc2, this_cpu_ptr(&sched_cb_list), sched_cb_entry) __perf_pmu_sched_task(cpc, sched_in ? next : prev, sched_in); } From 88aa1223bfffb1a0a98c639e9e1f71058f0d9178 Mon Sep 17 00:00:00 2001 From: Dapeng Mi Date: Tue, 8 Sep 2026 15:51:01 +0800 Subject: [PATCH 2/3] perf/x86/intel: Correct pt_regs->flags update for PEBS path pt_regs->flags holds the saved CPU FLAGS register. In the PEBS path, it was incorrectly set to PERF_EFLAGS_EXACT instead of being populated from the PEBS flags snapshot. Update pt_regs->flags from PEBS GPR flags if GPRs group is present. Fixes: c22497f5838c ("perf/x86/intel: Support adaptive PEBS v4") Signed-off-by: Dapeng Mi Signed-off-by: Peter Zijlstra (Intel) Link: https://patch.msgid.link/20260908075102.540715-1-dapeng1.mi@linux.intel.com --- arch/x86/events/intel/ds.c | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/arch/x86/events/intel/ds.c b/arch/x86/events/intel/ds.c index 8940f0292229..d0bb767a0fef 100644 --- a/arch/x86/events/intel/ds.c +++ b/arch/x86/events/intel/ds.c @@ -2432,7 +2432,7 @@ static inline void __setup_pebs_basic_group(struct perf_event *event, { /* The ip in basic is EventingIP */ set_linear_ip(regs, ip); - regs->flags = PERF_EFLAGS_EXACT; + regs->flags |= PERF_EFLAGS_EXACT; setup_pebs_time(event, data, tsc); if (sample_type & PERF_SAMPLE_WEIGHT_STRUCT) @@ -2444,9 +2444,17 @@ static inline void __setup_pebs_gpr_group(struct perf_event *event, struct pebs_gprs *gprs, u64 sample_type) { + /* + * Update flags with PEBS data. PERF_EFLAGS_EXACT must be set + * in previous basic group handling. + */ + regs->flags = gprs->flags | PERF_EFLAGS_EXACT; + if (event->attr.precise_ip < 2) { set_linear_ip(regs, gprs->ip); regs->flags &= ~PERF_EFLAGS_EXACT; + } else if (regs->flags & X86_VM_MASK) { + regs->flags ^= (PERF_EFLAGS_VM | X86_VM_MASK); } if (sample_type & (PERF_SAMPLE_REGS_INTR | PERF_SAMPLE_REGS_USER)) From a56c03a397e2cd0c4cf8da96dcd6214f7d0e7d8c Mon Sep 17 00:00:00 2001 From: Dapeng Mi Date: Thu, 13 Aug 2026 14:43:46 +0800 Subject: [PATCH 3/3] perf/x86/intel: Prevent drain_pebs() reentry The PEBS buffer is shared by all events on a CPU, so drain_pebs() must not be reentered. If so, one instance may observe stale buffer state and potentially access out-of-bound memory. Most invocations happen in NMI context, which naturally prevents reentry. However, drain_pebs() is also reachable from process context via intel_pmu_drain_pebs_buffer(). In those paths, the PMU is often already disabled, but not guaranteed. For example, __intel_pmu_pebs_disable() only disables the target counter, so other active counters can still raise a PMI and interrupt an in-flight drain_pebs(). Here is an example, __perf_addr_filters_adjust() perf_event_stop() __perf_event_stop() x86_pmu_stop() (event->pmu->stop) intel_pmu_disable_event() intel_pmu_pebs_disable() __intel_pmu_pebs_disable() intel_pmu_drain_large_pebs() intel_pmu_drain_pebs_buffer() Introduce __intel_pmu_quiesce() and __intel_pmu_resume() helpers and use them in intel_pmu_drain_large_pebs() to disable the full PMU around the intel_pmu_drain_pebs_buffer() call, preventing reentry. Also add a warning in intel_pmu_drain_pebs_buffer() when the full PMU is not disabled. Fixes: b752ea0c28e3 ("perf/x86/intel/ds: Flush PEBS DS when changing PEBS_DATA_CFG") Signed-off-by: Dapeng Mi Signed-off-by: Peter Zijlstra (Intel) Link: https://patch.msgid.link/20260813064346.335458-1-dapeng1.mi@linux.intel.com --- arch/x86/events/intel/core.c | 33 ++++++++++++++++++++++++--------- arch/x86/events/intel/ds.c | 8 +++++++- arch/x86/events/perf_event.h | 3 +++ 3 files changed, 34 insertions(+), 10 deletions(-) diff --git a/arch/x86/events/intel/core.c b/arch/x86/events/intel/core.c index cc13164d948f..1ac2ca35db53 100644 --- a/arch/x86/events/intel/core.c +++ b/arch/x86/events/intel/core.c @@ -3125,6 +3125,27 @@ static void intel_pmu_del_event(struct perf_event *event) this_cpu_ptr(&cpu_hw_events)->n_late_setup--; } +int __intel_pmu_quiesce(void) +{ + struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events); + int pmu_enabled = cpuc->enabled; + + cpuc->enabled = 0; + if (pmu_enabled) + intel_pmu_disable_all(); + + return pmu_enabled; +} + +void __intel_pmu_resume(int pmu_enabled) +{ + struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events); + + cpuc->enabled = pmu_enabled; + if (pmu_enabled) + intel_pmu_enable_all(0); +} + static int icl_set_topdown_event_period(struct perf_event *event) { struct hw_perf_event *hwc = &event->hw; @@ -3316,16 +3337,13 @@ static void intel_pmu_read_event(struct perf_event *event) if (event->hw.flags & (PERF_X86_EVENT_AUTO_RELOAD | PERF_X86_EVENT_TOPDOWN) || is_pebs_counter_event_group(event)) { struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events); - bool pmu_enabled = cpuc->enabled; + int pmu_enabled; /* Only need to call update_topdown_event() once for group read. */ if (is_metric_event(event) && (cpuc->txn_flags & PERF_PMU_TXN_READ)) return; - cpuc->enabled = 0; - if (pmu_enabled) - intel_pmu_disable_all(); - + pmu_enabled = __intel_pmu_quiesce(); /* * If the PEBS counters snapshotting is enabled, * the topdown event is available in PEBS records. @@ -3334,10 +3352,7 @@ static void intel_pmu_read_event(struct perf_event *event) static_call(intel_pmu_update_topdown_event)(event, NULL); else intel_pmu_drain_pebs_buffer(); - - cpuc->enabled = pmu_enabled; - if (pmu_enabled) - intel_pmu_enable_all(0); + __intel_pmu_resume(pmu_enabled); return; } diff --git a/arch/x86/events/intel/ds.c b/arch/x86/events/intel/ds.c index d0bb767a0fef..b98029b44052 100644 --- a/arch/x86/events/intel/ds.c +++ b/arch/x86/events/intel/ds.c @@ -1242,8 +1242,11 @@ unlock: void intel_pmu_drain_pebs_buffer(void) { + struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events); struct perf_sample_data data; + WARN_ON_ONCE(cpuc->enabled); + static_call(x86_pmu_drain_pebs)(NULL, &data); } @@ -1864,8 +1867,11 @@ static void intel_pmu_pebs_via_pt_enable(struct perf_event *event) static inline void intel_pmu_drain_large_pebs(struct cpu_hw_events *cpuc) { if (cpuc->n_pebs == cpuc->n_large_pebs && - cpuc->n_pebs != cpuc->n_pebs_via_pt) + cpuc->n_pebs != cpuc->n_pebs_via_pt) { + int enabled = __intel_pmu_quiesce(); intel_pmu_drain_pebs_buffer(); + __intel_pmu_resume(enabled); + } } static void __intel_pmu_pebs_enable(struct perf_event *event) diff --git a/arch/x86/events/perf_event.h b/arch/x86/events/perf_event.h index 71ed5b2acea2..4680cba91340 100644 --- a/arch/x86/events/perf_event.h +++ b/arch/x86/events/perf_event.h @@ -1638,6 +1638,9 @@ static __always_inline void __intel_pmu_lbr_disable(void) wrmsrq(MSR_IA32_DEBUGCTLMSR, debugctl); } +extern int __intel_pmu_quiesce(void); +extern void __intel_pmu_resume(int pmu_enabled); + int intel_pmu_save_and_restart(struct perf_event *event); struct event_constraint *