sched_ext: Generate qseq from a per-task counter
Changes
4 files changed · +16 −5
| File | Status | Add | Del | Content |
|---|---|---|---|---|
| include/linux/sched/ext.h | modified | +1 | −0 | View file |
| kernel/sched/ext/ext.c | modified | +10 | −4 | View file |
| kernel/sched/ext/internal.h | modified | +5 | −0 | View file |
| kernel/sched/sched.h | modified | +0 | −1 | View file |
include/linux/sched/ext.h
View filediff --git a/include/linux/sched/ext.h b/include/linux/sched/ext.h
index 582d7cd..36c0479 100644
--- a/include/linux/sched/ext.h
+++ b/include/linux/sched/ext.h
@@ -207,6 +207,7 @@ struct sched_ext_entity {
s32 holding_cpu;
s32 selected_cpu;
s32 runnable_cpu; /* cpu @p is runnable on, -1 if not */
+ u32 ops_qseq; /* protected by rq lock */
struct task_struct *kf_tasks[2]; /* see SCX_CALL_OP_TASK() */
struct list_head runnable_node; /* rq->scx.runnable_list */
kernel/sched/ext/ext.c
View filediff --git a/kernel/sched/ext/ext.c b/kernel/sched/ext/ext.c
index 5209510..ac8fe70 100644
--- a/kernel/sched/ext/ext.c
+++ b/kernel/sched/ext/ext.c
@@ -2057,8 +2057,14 @@ void scx_do_enqueue_task(struct rq *rq, struct task_struct *p, u64 enq_flags,
if (unlikely(!SCX_HAS_OP(sch, enqueue)))
goto global;
- /* DSQ bypass didn't trigger, enqueue on the BPF scheduler */
- qseq = rq->scx.ops_qseq++ << SCX_OPSS_QSEQ_SHIFT;
+ /*
+ * DSQ bypass didn't trigger, enqueue on the BPF scheduler. Wrap the
+ * per-task qseq counter where the QSEQ field wraps and skip 0, which is
+ * what scx_bpf_dsq_insert() records for a task in NONE or DISPATCHING.
+ */
+ p->scx.ops_qseq = ((p->scx.ops_qseq + 1) &
+ (SCX_OPSS_QSEQ_MASK >> SCX_OPSS_QSEQ_SHIFT)) ?: 1;
+ qseq = (unsigned long)p->scx.ops_qseq << SCX_OPSS_QSEQ_SHIFT;
WARN_ON_ONCE(atomic_long_read(&p->scx.ops_state) != SCX_OPSS_NONE);
atomic_long_set(&p->scx.ops_state, SCX_OPSS_QUEUEING | qseq);
@@ -6969,9 +6975,9 @@ static void scx_dump_cpu(struct scx_sched *sch, struct seq_buf *s,
seq_buf_init(&ns, buf, avail);
dump_newline(&ns);
- scx_dump_line(&ns, "CPU %-4d: nr_run=%u flags=0x%x cpu_rel=%d ops_qseq=%lu ksync=%lu",
+ scx_dump_line(&ns, "CPU %-4d: nr_run=%u flags=0x%x cpu_rel=%d ksync=%lu",
cpu, rq->scx.nr_running, rq->scx.flags, rq->scx.cpu_released,
- rq->scx.ops_qseq, rq->scx.kick_sync);
+ rq->scx.kick_sync);
scx_rescue_dump(&ns, rq);
scx_dump_line(&ns, " curr=%s[%d] class=%ps",
rq->curr->comm, rq->curr->pid, rq->curr->sched_class);
kernel/sched/ext/internal.h
View filediff --git a/kernel/sched/ext/internal.h b/kernel/sched/ext/internal.h
index 1df8f58..5b37faa 100644
--- a/kernel/sched/ext/internal.h
+++ b/kernel/sched/ext/internal.h
@@ -1998,6 +1998,11 @@ enum scx_ops_state {
* dequeue/requeue, the dispatcher can tell whether it still has a claim
* on the task being dispatched.
*
+ * QSEQ is generated from the per-task p->scx.ops_qseq counter so that
+ * it doesn't repeat across QUEUED instances of the same task even if
+ * the task moves between rqs. 0 is never used as a valid QSEQ since
+ * NONE and DISPATCHING map to this value.
+ *
* As some 32bit archs can't do 64bit store_release/load_acquire,
* p->scx.ops_state is atomic_long_t which leaves 30 bits for QSEQ on
* 32bit machines. The dispatch race window QSEQ protects is very narrow
kernel/sched/sched.h
View filediff --git a/kernel/sched/sched.h b/kernel/sched/sched.h
index 742cce9..9ed0cb6 100644
--- a/kernel/sched/sched.h
+++ b/kernel/sched/sched.h
@@ -813,7 +813,6 @@ struct scx_rq {
#endif
struct list_head runnable_list; /* runnable tasks on this rq */
struct list_head ddsp_deferred_locals; /* deferred ddsps from enq */
- unsigned long ops_qseq;
/* both stashed across the activate_task() in move_remote_task_to_local_dsq() */
u64 remote_activate_enq_flags;
struct scx_sched *remote_activate_sch;