| 1 |
|
| 2 | |
| 3 | * BPF extensible scheduler class: Documentation/scheduler/sched-ext.rst
|
| 4 | *
|
| 5 | * Inline definitions layered on top of internal.h and cid.h.
|
| 6 | *
|
| 7 | * Copyright (c) 2026 Meta Platforms, Inc. and affiliates.
|
| 8 | * Copyright (c) 2026 Tejun Heo <[email protected]>
|
| 9 |
|
| 10 | #ifndef _KERNEL_SCHED_EXT_INLINES_H
|
| 11 | #define _KERNEL_SCHED_EXT_INLINES_H
|
| 12 |
|
| 13 | #include "internal.h"
|
| 14 | #include "cid.h"
|
| 15 |
|
| 16 |
|
| 17 | enum scx_dsp_verdict {
|
| 18 | SCX_DSP_NONE,
|
| 19 | SCX_DSP_LOCAL,
|
| 20 | SCX_DSP_PREV,
|
| 21 | SCX_DSP_RETRY,
|
| 22 | };
|
| 23 |
|
| 24 | |
| 25 | * One user of this function is scx_bpf_sub_dispatch() which can be called
|
| 26 | * recursively as sub-sched dispatches nest. Always inline to reduce stack usage
|
| 27 | * from the call frame.
|
| 28 |
|
| 29 | static __always_inline enum scx_dsp_verdict
|
| 30 | scx_dispatch_sched(struct scx_sched *sch, struct rq *rq,
|
| 31 | struct task_struct *prev, bool nested)
|
| 32 | {
|
| 33 | struct scx_dsp_ctx *dspc = &this_cpu_ptr(sch->pcpu)->dsp_ctx;
|
| 34 | int nr_loops = SCX_DSP_MAX_LOOPS;
|
| 35 | s32 cpu = cpu_of(rq);
|
| 36 | bool prev_on_sch = (prev->sched_class == &ext_sched_class) &&
|
| 37 | scx_task_on_sched(sch, prev);
|
| 38 |
|
| 39 | if (scx_consume_global_dsq(sch, rq))
|
| 40 | return SCX_DSP_LOCAL;
|
| 41 |
|
| 42 | if (scx_bypass_dsp_enabled(sch)) {
|
| 43 |
|
| 44 | if (scx_bypassing(sch, cpu)) {
|
| 45 | if (scx_consume_dispatch_q(sch, rq, scx_bypass_dsq(sch, cpu), 0))
|
| 46 | return SCX_DSP_LOCAL;
|
| 47 | return SCX_DSP_NONE;
|
| 48 | }
|
| 49 |
|
| 50 | #ifdef CONFIG_EXT_SUB_SCHED
|
| 51 | |
| 52 | * If @sch isn't bypassing but its children are, @sch is
|
| 53 | * responsible for making forward progress for both its own
|
| 54 | * tasks that aren't bypassing and the bypassing descendants'
|
| 55 | * tasks. The following implements a simple built-in behavior -
|
| 56 | * let each CPU try to run the bypass DSQ every Nth time.
|
| 57 | *
|
| 58 | * Later, if necessary, we can add an ops flag to suppress the
|
| 59 | * auto-consumption and a kfunc to consume the bypass DSQ and,
|
| 60 | * so that the BPF scheduler can fully control scheduling of
|
| 61 | * bypassed tasks.
|
| 62 |
|
| 63 | struct scx_sched_pcpu *pcpu = per_cpu_ptr(sch->pcpu, cpu);
|
| 64 |
|
| 65 | if (!(pcpu->bypass_host_seq++ % SCX_BYPASS_HOST_NTH) &&
|
| 66 | scx_consume_dispatch_q(sch, rq, scx_bypass_dsq(sch, cpu), 0)) {
|
| 67 | __scx_add_event(sch, SCX_EV_SUB_BYPASS_DISPATCH, 1);
|
| 68 | return SCX_DSP_LOCAL;
|
| 69 | }
|
| 70 | #endif
|
| 71 | }
|
| 72 |
|
| 73 | |
| 74 | * scx_rq_online() can't be used. Its cpu_active() test goes false
|
| 75 | * before CPU hotplug waits for an RCU grace period, and
|
| 76 | * rq_offline_scx() moves this CPU's tasks to the local DSQ only after
|
| 77 | * the wait. The grace period can depend on those tasks running.
|
| 78 |
|
| 79 | if (unlikely(!SCX_HAS_OP(sch, dispatch)) || !(rq->scx.flags & SCX_RQ_ONLINE))
|
| 80 | return SCX_DSP_NONE;
|
| 81 |
|
| 82 | dspc->rq = rq;
|
| 83 |
|
| 84 | |
| 85 | * The dispatch loop. Because scx_flush_dispatch_buf() may drop the rq
|
| 86 | * lock, the local DSQ might still end up empty after a successful
|
| 87 | * ops.dispatch(). If the local DSQ is empty even after ops.dispatch()
|
| 88 | * produced some tasks, retry. The BPF scheduler may depend on this
|
| 89 | * looping behavior to simplify its implementation.
|
| 90 |
|
| 91 | do {
|
| 92 | dspc->nr_tasks = 0;
|
| 93 |
|
| 94 | #ifdef CONFIG_EXT_SUB_SCHED
|
| 95 |
|
| 96 | if (!nested)
|
| 97 | rq->scx.sub_dispatch_prev = prev;
|
| 98 | #endif
|
| 99 |
|
| 100 | SCX_CALL_OP(sch, dispatch, rq, scx_cpu_arg(cpu),
|
| 101 | prev_on_sch ? prev : NULL);
|
| 102 |
|
| 103 | #ifdef CONFIG_EXT_SUB_SCHED
|
| 104 | if (!nested)
|
| 105 | rq->scx.sub_dispatch_prev = NULL;
|
| 106 | #endif
|
| 107 |
|
| 108 | scx_flush_dispatch_buf(sch, rq);
|
| 109 |
|
| 110 | if ((prev->scx.flags & SCX_TASK_QUEUED) && prev->scx.slice)
|
| 111 | return SCX_DSP_PREV;
|
| 112 | if (rq->scx.local_dsq.nr)
|
| 113 | return SCX_DSP_LOCAL;
|
| 114 | if (scx_consume_global_dsq(sch, rq))
|
| 115 | return SCX_DSP_LOCAL;
|
| 116 |
|
| 117 | |
| 118 | * ops.dispatch() can trap us in this loop by repeatedly
|
| 119 | * dispatching ineligible tasks. Break out once in a while to
|
| 120 | * allow the watchdog to run. As IRQ can't be enabled in
|
| 121 | * dispatch, we want to complete this scheduling cycle and then
|
| 122 | * start a new one. IOW, we want to call resched_curr() on the
|
| 123 | * next, most likely idle, task, not the current one. Use
|
| 124 | * __scx_bpf_kick_cpu() for deferred kicking.
|
| 125 |
|
| 126 | if (unlikely(!--nr_loops)) {
|
| 127 | scx_kick_cpu(sch, cpu, 0);
|
| 128 | break;
|
| 129 | }
|
| 130 | } while (dspc->nr_tasks);
|
| 131 |
|
| 132 | |
| 133 | * Prevent the CPU from going idle while bypassed descendants have tasks
|
| 134 | * queued. Without this fallback, bypassed tasks could stall if the host
|
| 135 | * scheduler's ops.dispatch() doesn't yield any tasks.
|
| 136 |
|
| 137 | if (scx_bypass_dsp_enabled(sch) &&
|
| 138 | scx_consume_dispatch_q(sch, rq, scx_bypass_dsq(sch, cpu), 0)) {
|
| 139 | __scx_add_event(sch, SCX_EV_SUB_BYPASS_DISPATCH, 1);
|
| 140 | return SCX_DSP_LOCAL;
|
| 141 | }
|
| 142 |
|
| 143 | return SCX_DSP_NONE;
|
| 144 | }
|
| 145 |
|
| 146 | #endif
|