projects/linux·commit 9b87fdc9

Merge tag 'sched_ext-for-7.3-rc3-fixes' of git://git.kernel.org/pub/scm/linux/kernel/git/tj/sched_ext

9b87fdc9 · Linus Torvalds <[email protected]> · 2026-09-15 11:57:51 -07:00

Pull sched_ext fixes from Tejun Heo:

 - An error raised by a BPF program before the scheduler finished
   enabling was consumed by the disable path's pre-enable shortcut,
   leaving a running scheduler that couldn't be disabled and was later
   freed while in use.

 - Two compat kfuncs dereferenced a NULL scheduler when handed an exited
   or idle task, oopsing the kernel.

 - Keep-running decisions in the dispatch path used the root scheduler's
   flags for tasks belonging to a sub-scheduler, causing warnings and
   stalls.

 - Schedulers with their own CPU ID mapping had no way to learn which
   IDs are online. Add a kernel-maintained online mask to plug the hole.

 - Cgroup idle state: the initial cpu.idle state wasn't passed on cgroup
   init and same-value rewrites delivered spurious callbacks.

 - Example scheduler fixes for a reenqueue loop on attach, placements on
   CPUs without effective grants, stalled partition work and stale idle
   tracking.

* tag 'sched_ext-for-7.3-rc3-fixes' of git://git.kernel.org/pub/scm/linux/kernel/git/tj/sched_ext:
  sched_ext: Maintain an online cid mask in the scheduler arena
  sched_ext: scx_qmap: Restore unused idle claims from ops.dispatch()
  sched_ext: Close the pre-enable ops error claim window
  sched_ext: scx_qmap: Fix pending partition work handoff
  sched_ext: scx_qmap: Place only on cids whose caps are in effect
  sched_ext: scx_qmap: Do not add IMMED to rescue inserts
  sched_ext: Use @prev's scheduler for the keep decisions in dispatch_one()
  sched_ext: Rename sch to root_sch in dispatch_one()
  sched_ext: Fix NULL sched deref in kfunc sub-sched error paths
  sched_ext: Don't deliver duplicate ops.cgroup_set_idle() for same value
  sched_ext: Pass the initial cpu.idle state in scx_cgroup_init_args
Commit
9b87fdc9af2fbfcdb5c24a64139685ef80f6573f
Parents
6fb20c02 · a9e3760b

Merge commit. Changes below are compared with the first parent: 6fb20c02.

Changes

7 files changed · +254 −89

Only the first 600 diff lines are shown. Download the complete patch.

kernel/sched/ext/ext.c

View file
diff --git a/kernel/sched/ext/ext.c b/kernel/sched/ext/ext.c
index 51de1d8..3219f0d 100644
--- a/kernel/sched/ext/ext.c
+++ b/kernel/sched/ext/ext.c
@@ -2919,7 +2919,7 @@ static inline void maybe_queue_balance_callback(struct rq *rq)
static enum scx_dsp_verdict dispatch_one(struct rq *rq, struct task_struct *prev)
{
- struct scx_sched *sch = scx_root_protected_live();
+ struct scx_sched *root_sch = scx_root_protected_live();
enum scx_dsp_verdict verdict;
s32 cpu = cpu_of(rq);
@@ -2928,7 +2928,7 @@ static enum scx_dsp_verdict dispatch_one(struct rq *rq, struct task_struct *prev
scx_process_sync_ecaps(rq, prev);
- if ((sch->ops.flags & SCX_OPS_HAS_CPU_PREEMPT) &&
+ if ((root_sch->ops.flags & SCX_OPS_HAS_CPU_PREEMPT) &&
unlikely(rq->scx.cpu_released)) {
/*
* If the previous sched_class for the current CPU was not SCX,
@@ -2936,8 +2936,8 @@ static enum scx_dsp_verdict dispatch_one(struct rq *rq, struct task_struct *prev
* core. This callback complements ->cpu_release(), which is
* emitted in switch_class().
*/
- if (sch->ops.cpu_acquire)
- SCX_CALL_OP(sch, cpu_acquire, rq, cpu, NULL);
+ if (root_sch->ops.cpu_acquire)
+ SCX_CALL_OP(root_sch, cpu_acquire, rq, cpu, NULL);
rq->scx.cpu_released = false;
}
@@ -2955,7 +2955,7 @@ static enum scx_dsp_verdict dispatch_one(struct rq *rq, struct task_struct *prev
* test.
*/
if ((prev->scx.flags & SCX_TASK_QUEUED) && prev->scx.slice &&
- !scx_bypassing(sch, cpu)) {
+ !scx_bypassing(scx_task_sched(prev), cpu)) {
verdict = SCX_DSP_PREV;
goto has_tasks;
}
@@ -2967,20 +2967,25 @@ static enum scx_dsp_verdict dispatch_one(struct rq *rq, struct task_struct *prev
goto has_tasks;
}
- verdict = scx_dispatch_sched(sch, rq, prev, false);
+ verdict = scx_dispatch_sched(root_sch, rq, prev, false);
if (verdict != SCX_DSP_NONE)
goto has_tasks;
/*
- * Didn't find another task to run. Keep running @prev unless
- * %SCX_OPS_ENQ_LAST is in effect.
+ * Didn't find another task to run. Keep running @prev unless its own
+ * scheduler set %SCX_OPS_ENQ_LAST and takes the enqueue instead, see
+ * put_prev_task_scx(). Read the scheduler here as the dispatch above
+ * may have dropped the rq lock while @prev changed class or scheduler.
*/
- if ((prev->scx.flags & SCX_TASK_QUEUED) &&
- (!(sch->ops.flags & SCX_OPS_ENQ_LAST) || scx_bypassing(sch, cpu)) &&
- scx_task_can_stay_on_cpu(rq, prev)) {
- __scx_add_event(sch, SCX_EV_DISPATCH_KEEP_LAST, 1);
- verdict = SCX_DSP_PREV;
- goto has_tasks;
+ if (prev->scx.flags & SCX_TASK_QUEUED) {
+ struct scx_sched *prev_sch = scx_task_sched(prev);
+
+ if ((!(prev_sch->ops.flags & SCX_OPS_ENQ_LAST) ||
+ scx_bypassing(prev_sch, cpu)) && scx_task_can_stay_on_cpu(rq, prev)) {
+ __scx_add_event(prev_sch, SCX_EV_DISPATCH_KEEP_LAST, 1);
+ verdict = SCX_DSP_PREV;
+ goto has_tasks;
+ }
}
rq->scx.flags &= ~SCX_RQ_IN_DISPATCH;
return SCX_DSP_NONE;
@@ -3665,8 +3670,20 @@ static void handle_hotplug(struct rq *rq, bool online)
s16 *tbl = rcu_dereference_check(scx_cpu_to_cid_tbl,
lockdep_is_cpus_held());
- if (tbl)
+ if (tbl) {
+ struct scx_sched *pos;
+
cpu_or_cid = tbl[cpu];
+
+ guard(raw_spinlock_irqsave)(&scx_sched_lock);
+ list_for_each_entry(pos, &scx_sched_all, all) {
+ struct scx_cmask *mask = pos->online_cmask;
+
+ if (mask)
+ __assign_bit(cpu_or_cid, (unsigned long *)mask->bits,
+ online);
+ }
+ }
}
if (online && SCX_HAS_OP(sch, cpu_online))
@@ -4766,7 +4783,8 @@ int scx_tg_online(struct task_group *tg)
{ .weight = tg->scx.weight,
.bw_period_us = tg->scx.bw_period_us,
.bw_quota_us = tg->scx.bw_quota_us,
- .bw_burst_us = tg->scx.bw_burst_us };
+ .bw_burst_us = tg->scx.bw_burst_us,
+ .sched_idle = tg->scx.idle };
ret = SCX_CALL_OP_RET(sch, cgroup_init,
NULL, tg->css.cgroup, &args);
@@ -4932,7 +4950,8 @@ void scx_group_set_idle(struct task_group *tg, bool idle)
percpu_down_read(&scx_cgroup_ops_rwsem);
sch = scx_tg_knob_sched(tg);
- if (scx_cgroup_enabled && sch && SCX_HAS_OP(sch, cgroup_set_idle))
+ if (scx_cgroup_enabled && sch && SCX_HAS_OP(sch, cgroup_set_idle) &&
+ tg->scx.idle != idle)
SCX_CALL_OP(sch, cgroup_set_idle, NULL, tg_cgrp(tg), idle);
/* Update the task group's idle state */
@@ -5187,6 +5206,7 @@ static int scx_cgroup_init(struct scx_sched *sch)
.bw_period_us = tg->scx.bw_period_us,
.bw_quota_us = tg->scx.bw_quota_us,
.bw_burst_us = tg->scx.bw_burst_us,
+ .sched_idle = tg->scx.idle,
};
ret = SCX_CALL_OP_RET(sch, cgroup_init, NULL, css->cgroup, &args);
@@ -5272,12 +5292,17 @@ static void free_exit_info(struct scx_exit_info *ei);
static const char *scx_exit_reason(enum scx_exit_kind kind);
static bool scx_claim_exit(struct scx_sched *sch, enum scx_exit_kind kind);
-s32 scx_set_cmask_scratch_alloc(struct scx_sched *sch)
+s32 scx_alloc_kern_arena_objs(struct scx_sched *sch)
{
size_t size = struct_size_t(struct scx_cmask, bits,
SCX_CMASK_NR_WORDS(num_possible_cpus()));
+ struct scx_cmask *online;
+ struct scx_cmask_ref ref;
int cpu;
+ /* hotplug stays excluded until the online mask is published */
+ lockdep_assert_cpus_held();
+
if (!sch->is_cid_type || !sch->arena_pool)
return 0;
@@ -5293,15 +5318,28 @@ s32 scx_set_cmask_scratch_alloc(struct scx_sched *sch)
return -ENOMEM;
scx_cmask_init(*slot, 0, num_possible_cpus());
}
+
+ /* pack the online mask alongside the scratch masks */
+ online = scx_arena_alloc(sch, size);
+ if (!online)
+ return -ENOMEM;
+
+ scoped_guard(rcu) {
+ scx_cmask_ref_init_kern(sch, online, 0, num_possible_cpus(), &ref);
+ scx_cmask_ref_from_cpumask(&ref, cpu_active_mask);
+ }
+ sch->online_cmask = online;
+
return 0;
}
-static void scx_set_cmask_scratch_free(struct scx_sched *sch)
+static void scx_free_kern_arena_objs(struct scx_sched *sch)
{
size_t size = struct_size_t(struct scx_cmask, bits,
SCX_CMASK_NR_WORDS(num_possible_cpus()));
int cpu;
+ scx_arena_free(sch, sch->online_cmask, size);
if (!sch->set_cmask_scratch)
return;
@@ -5388,7 +5426,7 @@ static void scx_sched_free_rcu_work(struct work_struct *work)
rhashtable_free_and_destroy(&sch->dsq_hash, NULL, NULL);
free_exit_info(sch->exit_info);
- scx_set_cmask_scratch_free(sch);
+ scx_free_kern_arena_objs(sch);
scx_arena_pool_destroy(sch);
if (sch->arena_map)
bpf_map_put(sch->arena_map);
@@ -7508,22 +7546,24 @@ static void scx_root_enable_workfn(struct kthread_work *work)
#ifdef CONFIG_EXT_SUB_SCHED
cgroup_get(cgrp);
#endif
+ /*
+ * Transition to ENABLING to arm the disable path. Allocation failure
+ * still unwinds locally. Full disabling on failure applies only after
+ * scx_alloc_and_add_sched() succeeds.
+ */
+ WARN_ON_ONCE(scx_set_enable_state(SCX_ENABLING) != SCX_DISABLED);
+ WARN_ON_ONCE(scx_root);
+
sch = scx_alloc_and_add_sched(cmd, cgrp, NULL);
if (IS_ERR(sch)) {
ret = PTR_ERR(sch);
+ WARN_ON_ONCE(scx_set_enable_state(SCX_DISABLED) != SCX_ENABLING);
goto err_free_tid_hash;
}
if (sch->is_cid_type)
static_branch_enable(&__scx_is_cid_type);
- /*
- * Transition to ENABLING and clear exit info to arm the disable path.
- * Failure triggers full disabling from here on.
- */
- WARN_ON_ONCE(scx_set_enable_state(SCX_ENABLING) != SCX_DISABLED);
- WARN_ON_ONCE(scx_root);
-
atomic_long_set(&scx_nr_rejected, 0);
for_each_possible_cpu(cpu) {
@@ -7591,7 +7631,7 @@ static void scx_root_enable_workfn(struct kthread_work *work)
goto err_disable;
}
- ret = scx_set_cmask_scratch_alloc(sch);
+ ret = scx_alloc_kern_arena_objs(sch);
if (ret) {
cpus_read_unlock();
goto err_disable;
@@ -8946,10 +8986,17 @@ __bpf_kfunc void scx_bpf_dsq_insert_vtime(struct task_struct *p, u64 dsq_id,
#ifdef CONFIG_EXT_SUB_SCHED
/*
* Disallow if any sub-scheds are attached. There is no way to tell
- * which scheduler called us, just error out @p's scheduler.
+ * which scheduler called us, so error out @p's scheduler -- read it
+ * under RCU as @p's locks aren't necessarily held here. @p may be a
+ * task past sched_ext_dead() or an idle task, in which case its
+ * scheduler can't be determined and there is nothing obviously wrong
+ * to report; just refuse the call.
*/
if (unlikely(!list_empty(&sch->children))) {
- scx_error(scx_task_sched(p), "__scx_bpf_dsq_insert_vtime() must be used");
+ struct scx_sched *tsch = scx_task_sched_rcu(p);
+
+ if (tsch)
+ scx_error(tsch, "__scx_bpf_dsq_insert_vtime() must be used");
return;
}
#endif
@@ -10321,7 +10368,8 @@ __bpf_kfunc u32 scx_bpf_nr_cids(void)
* hotplug, which lets schedulers treat [0, nr_online_cids) as the online
* range. Schedulers that prefer to handle hotplug without a restart should
* install a custom mapping via scx_bpf_cid_override() and track onlining
- * through the ops.cid_online / ops.cid_offline callbacks.
+ * through the ops.cid_online / ops.cid_offline callbacks, starting from the
+ * mask scx_bpf_online_cmask() returns.
*/
__bpf_kfunc u32 scx_bpf_nr_online_cids(void)
{
@@ -10329,6 +10377,37 @@ __bpf_kfunc u32 scx_bpf_nr_online_cids(void)
}
/**
+ * scx_bpf_online_cmask - Return the online cid mask in the scheduler arena
+ * @aux: implicit BPF argument to access bpf_prog_aux hidden from BPF progs
+ *
+ * Return a kernel-maintained cmask covering [0, scx_bpf_nr_cids()), or NULL if
+ * the calling program is not associated with a live cid-form scheduler or the
+ * mask is not allocated yet, as in ops.init_cids(). Treat the mask as read-only
+ * even though arena memory stays writable by the BPF scheduler. The mask
+ * follows the SCX hotplug notifications: a cid's bit is updated before
+ * ops.cid_online/offline() runs for it. The pointer is valid from ops.init()
+ * through ops.exit(). Root ops.init() runs with hotplug excluded. Other
+ * contexts can observe concurrent updates.
+ */
+__bpf_kfunc const void *scx_bpf_online_cmask(const struct bpf_prog_aux *aux)
+{
+ struct scx_sched *sch;
+ struct scx_cmask *online;
+
+ guard(rcu)();
+
+ sch = scx_prog_sched(aux);
+ if (unlikely(!sch))
+ return NULL;
+ online = sch->online_cmask;
+ if (unlikely(!online))
+ return NULL;
+
+ /* BPF rebases by the low 32 bits, like __arena callback args */
+ return (void *)((unsigned long)online - sch->arena_kern_base);
+}
+
+/**
* scx_bpf_this_cid - Return the cid of the CPU this program is running on
*
* cid-addressed equivalent of bpf_get_smp_processor_id() for scx programs.
@@ -10691,6 +10770,7 @@ BTF_ID_FLAGS(func, scx_bpf_nr_node_ids)
BTF_ID_FLAGS(func, scx_bpf_nr_cpu_ids)
BTF_ID_FLAGS(func, scx_bpf_nr_cids)
BTF_ID_FLAGS(func, scx_bpf_nr_online_cids)
+BTF_ID_FLAGS(func, scx_bpf_online_cmask, KF_IMPLICIT_ARGS | KF_ARENA_RET)
BTF_ID_FLAGS(func, scx_bpf_this_cid)
BTF_ID_FLAGS(func, scx_bpf_get_possible_cpumask, KF_ACQUIRE)
BTF_ID_FLAGS(func, scx_bpf_get_online_cpumask, KF_ACQUIRE)

kernel/sched/ext/idle.c

View file
diff --git a/kernel/sched/ext/idle.c b/kernel/sched/ext/idle.c
index d2973fb..aa9fb6d 100644
--- a/kernel/sched/ext/idle.c
+++ b/kernel/sched/ext/idle.c
@@ -1142,10 +1142,17 @@ __bpf_kfunc s32 scx_bpf_select_cpu_and(struct task_struct *p, s32 prev_cpu, u64
#ifdef CONFIG_EXT_SUB_SCHED
/*
* Disallow if any sub-scheds are attached. There is no way to tell
- * which scheduler called us, just error out @p's scheduler.
+ * which scheduler called us, so error out @p's scheduler -- read it
+ * under RCU as @p's locks aren't necessarily held here. @p may be a
+ * task past sched_ext_dead() or an idle task, in which case its
+ * scheduler can't be determined and there is nothing obviously wrong
+ * to report; just refuse the call.
*/
if (unlikely(!list_empty(&sch->children))) {
- scx_error(scx_task_sched(p), "__scx_bpf_select_cpu_and() must be used");
+ struct scx_sched *tsch = scx_task_sched_rcu(p);
+
+ if (tsch)
+ scx_error(tsch, "__scx_bpf_select_cpu_and() must be used");
return -EINVAL;
}
#endif

kernel/sched/ext/internal.h

View file
diff --git a/kernel/sched/ext/internal.h b/kernel/sched/ext/internal.h
index 0967b99..3464e0f 100644
--- a/kernel/sched/ext/internal.h
+++ b/kernel/sched/ext/internal.h
@@ -259,6 +259,9 @@ struct scx_cgroup_init_args {
u64 bw_period_us;
u64 bw_quota_us;
u64 bw_burst_us;
+
+ /* whether the cgroup is configured SCHED_IDLE via cpu.idle */
+ bool sched_idle;
};
enum scx_cpu_preempt_reason {
@@ -569,6 +572,12 @@ struct sched_ext_ops {
*
* Specify the %SCX_OPS_KEEP_BUILTIN_IDLE flag to keep the built-in idle
* tracking.
+ *
+ * Only actual transitions are reported. A CPU that is claimed with an
+ * idle pick and kicked but dispatches no task returns to idle without a
+ * transition. A scheduler tracking idle CPUs itself must restore the
+ * idle state from ops.dispatch() when it returns without the next task
+ * to run.
*/
void (*update_idle)(s32 cpu, bool idle);
@@ -1552,6 +1561,7 @@ struct scx_sched {
* and passes it to the callback's __arena argument.
*/
struct scx_cmask * __percpu *set_cmask_scratch;
+ struct scx_cmask *online_cmask;
DECLARE_BITMAP(has_op, SCX_OPI_END);
@@ -2078,7 +2088,7 @@ void scx_disable_and_exit_task(struct scx_sched *sch, struct task_struct *p);
void scx_cgroup_lock(void);
void scx_cgroup_unlock(void);
#endif
-s32 scx_set_cmask_scratch_alloc(struct scx_sched *sch);
+s32 scx_alloc_kern_arena_objs(struct scx_sched *sch);
void scx_disable_bypass_dsp(struct scx_sched *sch);
void scx_bypass(struct scx_sched *sch, bool bypass);
s32 scx_link_sched(struct scx_sched *sch);

kernel/sched/ext/sub.c

View file
diff --git a/kernel/sched/ext/sub.c b/kernel/sched/ext/sub.c
index 9e70404..34e642a 100644
--- a/kernel/sched/ext/sub.c
+++ b/kernel/sched/ext/sub.c
@@ -1361,6 +1361,7 @@ static s32 scx_cgroup_claim_subtree(struct scx_sched *sch)
.bw_period_us = tg->scx.bw_period_us,
.bw_quota_us = tg->scx.bw_quota_us,
.bw_burst_us = tg->scx.bw_burst_us,
+ .sched_idle = tg->scx.idle,
};
if (tg->scx.sched != parent ||
@@ -1464,6 +1465,7 @@ static void scx_cgroup_return_subtree(struct scx_sched *sch)
.bw_period_us = tg->scx.bw_period_us,
.bw_quota_us = tg->scx.bw_quota_us,
.bw_burst_us = tg->scx.bw_burst_us,
+ .sched_idle = tg->scx.idle,
};
/* the first pass must have transferred everything */
@@ -1803,6 +1805,12 @@ void scx_sub_enable_workfn(struct kthread_work *work)
goto err_disable;
}
+ scoped_guard(cpus_read_lock) {
+ ret = scx_alloc_kern_arena_objs(sch);
+ if (ret)
+ goto err_disable;
+ }
+
if (sch->ops.init) {
ret = SCX_CALL_OP_RET(sch, init, NULL);
if (ret) {
@@ -1813,10 +1821,6 @@ void scx_sub_enable_workfn(struct kthread_work *work)
sch->exit_info->flags |= SCX_EFLAG_INITIALIZED;
}
- ret = scx_set_cmask_scratch_alloc(sch);
- if (ret)
- goto err_disable;
-
struct scx_sub_attach_args sub_attach_args = {
.ops = &sch->ops,
.cgroup_path = sch->cgrp_path,

tools/sched_ext/include/scx/common.bpf.h

View file
diff --git a/tools/sched_ext/include/scx/common.bpf.h b/tools/sched_ext/include/scx/common.bpf.h
index 76f5e02..2ddb01a 100644
--- a/tools/sched_ext/include/scx/common.bpf.h
+++ b/tools/sched_ext/include/scx/common.bpf.h
@@ -113,6 +113,7 @@ s32 scx_bpf_this_cid(void) __ksym __weak;
struct task_struct *scx_bpf_cid_curr(s32 cid) __ksym __weak;
u32 scx_bpf_nr_cids(void) __ksym __weak;
u32 scx_bpf_nr_online_cids(void) __ksym __weak;
+const void __arena *scx_bpf_online_cmask(void) __ksym __weak;
u32 scx_bpf_cidperf_cap(s32 cid) __ksym __weak;
u32 scx_bpf_cidperf_cur(s32 cid) __ksym __weak;
s32 scx_bpf_cidperf_set(s32 cid, u32 perf) __ksym __weak;

tools/sched_ext/scx_qmap.bpf.c

View file
diff --git a/tools/sched_ext/scx_qmap.bpf.c b/tools/sched_ext/scx_qmap.bpf.c
index 9f6e61d..67b7c01 100644
--- a/tools/sched_ext/scx_qmap.bpf.c
+++ b/tools/sched_ext/scx_qmap.bpf.c
@@ -24,6 +24,9 @@
* time-share that stays self-local.
* self - The excl cpus the node kept for itself, plus all of held_shared.
* owner - Who holds a cid - a child slot, CID_SELF, or CID_NONE.
+ * avail - Cpus whose caps are in effect, per ops.sub_ecaps_updated().
+ * usable - self AND avail. Placement decisions use this: self is the
+ * delegation split and can run ahead of what the cpus honor.
*
* The scheduler splits its held-excl cpus among self and the children in
* proportion to each node's cpu.weight, handing each the floor of its share as
@@ -208,8 +211,8 @@ static int qmap_spin_lock(struct bpf_res_spin_lock *lock)
}
/*
- * Try prev_cid, then scan cpus_allowed AND idle_cids AND self_cids round-robin
- * from prev_cid + 1. Atomic claim retries on race; bounded by
+ * Try prev_cid, then scan cpus_allowed AND idle_cids AND usable_cids
+ * round-robin from prev_cid + 1. Atomic claim retries on race; bounded by
* IDLE_PICK_RETRIES to keep the verifier's insn budget in check.
*/
#define IDLE_PICK_RETRIES 16
@@ -221,7 +224,7 @@ static s32 pick_direct_dispatch_cid(struct task_struct *p, s32 prev_cid,
s32 cid;
u32 i;
- if (cmask_test(prev_cid, &qa.self_cids.mask) &&
+ if (cmask_test(prev_cid, &qa.usable_cids.mask) &&
cmask_test_and_clear(prev_cid, &qa.idle_cids.mask))
return prev_cid;
@@ -229,7 +232,7 @@ static s32 pick_direct_dispatch_cid(struct task_struct *p, s32 prev_cid,
bpf_for(i, 0, IDLE_PICK_RETRIES) {
cid = cmask_next_and2_set_wrap(&taskc->cpus_allowed,
&qa.idle_cids.mask,
- &qa.self_cids.mask, cid + 1);
+ &qa.usable_cids.mask, cid + 1);
barrier_var(cid);
if (cid >= nr_cids)
return -1;
@@ -358,8 +361,8 @@ s32 BPF_STRUCT_OPS(qmap_select_cid, struct task_struct *p,
}
/*
- * A received time-shared cid is held ENQ_IMMED-only, so inserts must set
- * SCX_ENQ_IMMED.
+ * A received time-shared cid is held ENQ_IMMED-only, so inserts meant to run
+ * there must set SCX_ENQ_IMMED.
*/
static u64 needs_immed(s32 cid)
{
@@ -444,9 +447,11 @@ void BPF_STRUCT_OPS(qmap_enqueue, struct task_struct *p, u64 enq_flags)
* didn't grant them or we delegated them to children - would starve in
* SHARED/FIFO since we only pull from those on self cids.
*
- * Force it onto its first allowed cid's local DSQ. If we hold that cid
- * it runs. Otherwise the insert carries SCX_ENQ_RESCUE and the kernel
- * diverts the task to its rescue path.
+ * Force it onto its first allowed cid's local DSQ with SCX_ENQ_RESCUE.
+ * If we hold ENQ on that cid it runs. Otherwise the kernel diverts the
+ * task to its rescue path. IMMED would turn the insert into a legal
+ * placement on a time-shared cid and the kernel would bounce it back
+ * here instead of rescuing it.
*/
if (!cmask_intersects(&taskc->cpus_allowed, &qa.self_cids.mask)) {
s32 c = cmask_next_set_wrap(&taskc->cpus_allowed, 0);
@@ -455,7 +460,7 @@ void BPF_STRUCT_OPS(qmap_enqueue, struct task_struct *p, u64 enq_flags)
taskc->force_local = false;
__sync_fetch_and_add(&qa.nr_rescue_dsp, 1);
scx_bpf_dsq_insert(p, SCX_DSQ_LOCAL_ON | c, slice_ns,
- enq_flags | needs_immed(c) | SCX_ENQ_RESCUE);
+ enq_flags | SCX_ENQ_RESCUE);
return;
}
}
@@ -540,7 +545,7 @@ void BPF_STRUCT_OPS(qmap_enqueue, struct task_struct *p, u64 enq_flags)
scx_bpf_dsq_insert(p, SHARED_DSQ, 0, enq_flags);
cid = cmask_next_and2_set_wrap(&taskc->cpus_allowed,
&qa.idle_cids.mask,
- &qa.self_cids.mask, 0);
+ &qa.usable_cids.mask, 0);
if (cid < scx_bpf_nr_cids())
scx_bpf_kick_cid(cid, SCX_KICK_IDLE);
return;
@@ -618,7 +623,7 @@ static bool scan_shared_dsq(bool from_timer)
if (c >= 0 && c < scx_bpf_nr_cids()) {
__sync_fetch_and_add(&qa.nr_rescue_dsp, 1);
scx_bpf_dsq_move(BPF_FOR_EACH_ITER, p, SCX_DSQ_LOCAL_ON | c,
- needs_immed(c) | SCX_ENQ_RESCUE);
+ SCX_ENQ_RESCUE);
}
continue;
}
@@ -644,22 +649,27 @@ static bool scan_shared_dsq(bool from_timer)
if (!(taskc = lookup_task_ctx(p)))
return false;
- /* only run highpri tasks on cids this node holds, not delegated ones */
+ /* only run highpri tasks on cids this node can use right now */
if (cmask_test(this_cid, &taskc->cpus_allowed) &&
- cmask_test(this_cid, &qa.self_cids.mask))
+ cmask_test(this_cid, &qa.usable_cids.mask))
cid = this_cid;
else
cid = cmask_next_and_set_wrap(&taskc->cpus_allowed,
- &qa.self_cids.mask,
+ &qa.usable_cids.mask,
this_cid + 1);
if (cid >= nr_cids) {
- /* stranded after the cull - rescue it from here */
- s32 c = cmask_next_set_wrap(&taskc->cpus_allowed, 0);
+ s32 c;
+
+ /* self cids lack caps in effect yet, leave it queued */
+ if (cmask_intersects(&taskc->cpus_allowed, &qa.self_cids.mask))
+ continue;
+ /* stranded after the cull - rescue it from here */
+ c = cmask_next_set_wrap(&taskc->cpus_allowed, 0);
if (c >= 0 && c < nr_cids) {
__sync_fetch_and_add(&qa.nr_rescue_dsp, 1);
scx_bpf_dsq_move(BPF_FOR_EACH_ITER, p, SCX_DSQ_LOCAL_ON | c,
- needs_immed(c) | SCX_ENQ_RESCUE);
+ SCX_ENQ_RESCUE);
}
continue;
}
@@ -808,10 +818,10 @@ void BPF_STRUCT_OPS(qmap_dispatch, s32 cid, struct task_struct *prev)
batch--;
cpuc->dsp_cnt--;
if (!batch || !scx_bpf_dispatch_nr_slots()) {
- if (scan_shared_dsq(false))
+ if (scan_shared_dsq(false) ||
+ scx_bpf_dsq_move_to_local(SHARED_DSQ, needs_immed(cid)))
return;
- scx_bpf_dsq_move_to_local(SHARED_DSQ, needs_immed(cid));
- return;
+ goto prev;
}
if (!cpuc->dsp_cnt)
break;
@@ -822,10 +832,14 @@ void BPF_STRUCT_OPS(qmap_dispatch, s32 cid, struct task_struct *prev)
if (scan_shared_dsq(false))
return;
-
+prev:
/*
* No other tasks. @prev will keep running. Update its core_sched_seq as
* if the task were enqueued and dispatched immediately.
+ *
+ * No @prev to keep running means the CPU goes idle. If its claim was
+ * never used, that is not a transition and ops.update_idle() stays
+ * silent. Restore the claim here.
*/
if (prev) {
taskc = lookup_task_ctx(prev);
@@ -834,6 +848,8 @@ void BPF_STRUCT_OPS(qmap_dispatch, s32 cid, struct task_struct *prev)
taskc->core_sched_seq =
qa.core_sched_tail_seqs[weight_to_idx(prev->scx.weight)]++;
+ } else {
+ cmask_set(cid, &qa.idle_cids.mask);
}
}
@@ -1113,7 +1129,7 @@ void BPF_STRUCT_OPS(qmap_update_idle, s32 cid, bool idle)
/*
* The kernel delivers update_idle() for every cid this node holds
* SCX_CAP_BASE on. Track every cid's idle state regardless of
- * delegation: the direct-dispatch pick masks idle_cids with self_cids

This diff is incomplete. Download the complete patch.

tools/sched_ext/scx_qmap.h

View file

This diff is incomplete. Download the complete patch.