diff --git a/kernel/sched/ext/idle.c b/kernel/sched/ext/idle.c index aa9fb6de0ad6..28ed9e327085 100644 --- a/kernel/sched/ext/idle.c +++ b/kernel/sched/ext/idle.c @@ -952,12 +952,43 @@ static bool is_bpf_migration_disabled(const struct task_struct *p) return p->migration_disabled; } +static s32 select_cpu_restricted(struct task_struct *p, s32 prev_cpu, + const struct cpumask *allowed, u64 flags, + bool migration_disabled) +{ + s32 cpu; + + if (migration_disabled) + cpu = task_cpu(p); + else + cpu = cpumask_first(p->cpus_ptr); + + if (!cpumask_test_cpu(cpu, p->cpus_ptr)) + return -EBUSY; + + if (allowed && !cpumask_test_cpu(cpu, allowed)) + return -EBUSY; + + if (flags & SCX_PICK_IDLE_IN_NODE) { + int node = scx_cpu_node_if_enabled(prev_cpu); + + if (scx_cpu_node_if_enabled(cpu) != node) + return -EBUSY; + } + + if (!scx_idle_test_and_clear_cpu(cpu)) + return -EBUSY; + + return cpu; +} + static s32 select_cpu_from_kfunc(struct scx_sched *sch, struct task_struct *p, s32 prev_cpu, u64 wake_flags, const struct cpumask *allowed, u64 flags) { unsigned long irq_flags; bool we_locked = false; + bool migration_disabled; s32 cpu; if (!scx_cpu_valid(sch, prev_cpu, NULL)) @@ -994,20 +1025,17 @@ static s32 select_cpu_from_kfunc(struct scx_sched *sch, struct task_struct *p, /* * This may also be called from ops.enqueue(), so we need to handle - * per-CPU tasks as well. For these tasks, we can skip all idle CPU - * selection optimizations and simply check whether the previously - * used CPU is idle and within the allowed cpumask. + * per-CPU tasks as well. For these tasks, skip general idle CPU + * selection and validate the task's only usable CPU directly. */ - if (p->nr_cpus_allowed == 1 || is_bpf_migration_disabled(p)) { - if (cpumask_test_cpu(prev_cpu, allowed ?: p->cpus_ptr) && - scx_idle_test_and_clear_cpu(prev_cpu)) - cpu = prev_cpu; - else - cpu = -EBUSY; - } else { + migration_disabled = is_bpf_migration_disabled(p); + + if (p->nr_cpus_allowed == 1 || migration_disabled) + cpu = select_cpu_restricted(p, prev_cpu, allowed, flags, + migration_disabled); + else cpu = scx_select_cpu_dfl(p, prev_cpu, wake_flags, allowed ?: p->cpus_ptr, flags); - } if (we_locked) raw_spin_unlock_irqrestore(&p->pi_lock, irq_flags);