linux - kernel/sched/ext/ext.c

11291 lines · 334.5 KiB

1	/* SPDX-License-Identifier: GPL-2.0 */
2	/*
3	* BPF extensible scheduler class: Documentation/scheduler/sched-ext.rst
4	*
5	* Copyright (c) 2022 Meta Platforms, Inc. and affiliates.
6	* Copyright (c) 2022 Tejun Heo <[email protected]>
7	* Copyright (c) 2022 David Vernet <[email protected]>
8	*/
9	#include <linux/bitmap.h>
10	#include <linux/btf_ids.h>
11	#include <linux/rhashtable.h>
12	#include <linux/sched/clock.h>
13	#include <linux/sched/isolation.h>
14	#include <linux/suspend.h>
15	#include <linux/sysrq.h>
16
17	#include "../pelt.h"
18	#include "internal.h"
19	#include "cid.h"
20	#include "arena.h"
21	#include "idle.h"
22	#include "sub.h"
23	#include "inlines.h"
24
25	DEFINE_RAW_SPINLOCK(scx_sched_lock);
26
27	bool scx_allow_proxy_exec(const struct task_struct *p)
28	{
29	return true;
30	}
31
32	/*
33	* NOTE: sched_ext is in the process of growing multiple scheduler support and
34	* scx_root usage is in a transitional state. Naked dereferences are safe if the
35	* caller is one of the tasks attached to SCX and explicit RCU dereference is
36	* necessary otherwise. Naked scx_root dereferences trigger sparse warnings but
37	* are used as temporary markers to indicate that the dereferences need to be
38	* updated to point to the associated scheduler instances rather than scx_root.
39	*/
40	struct scx_sched __rcu *scx_root;
41
42	/*
43	* All scheds, writers must hold both scx_enable_mutex and scx_sched_lock.
44	* Readers can hold either or rcu_read_lock().
45	*/
46	LIST_HEAD(scx_sched_all);
47
48	#ifdef CONFIG_EXT_SUB_SCHED
49	const struct rhashtable_params scx_sched_hash_params = {
50	.key_len = sizeof_field(struct scx_sched, ops.sub_cgroup_id),
51	.key_offset = offsetof(struct scx_sched, ops.sub_cgroup_id),
52	.head_offset = offsetof(struct scx_sched, hash_node),
53	.insecure_elasticity = true, /* inserted under scx_sched_lock */
54	};
55
56	struct rhashtable scx_sched_hash;
57	#endif
58
59	/* see SCX_OPS_TID_TO_TASK */
60	static const struct rhashtable_params scx_tid_hash_params = {
61	.key_len = sizeof_field(struct sched_ext_entity, tid),
62	.key_offset = offsetof(struct sched_ext_entity, tid),
63	.head_offset = offsetof(struct sched_ext_entity, tid_hash_node),
64	.insecure_elasticity = true, /* inserted/removed under scx_tasks_lock */
65	};
66	static struct rhashtable scx_tid_hash;
67
68	/*
69	* During exit, a task may schedule after losing its PIDs. When disabling the
70	* BPF scheduler, we need to be able to iterate tasks in every state to
71	* guarantee system safety. Maintain a dedicated task list which contains every
72	* task between its fork and eventual free.
73	*/
74	static DEFINE_RAW_SPINLOCK(scx_tasks_lock);
75	static LIST_HEAD(scx_tasks);
76
77	/* ops enable/disable */
78	DEFINE_MUTEX(scx_enable_mutex);
79	DEFINE_STATIC_KEY_FALSE(__scx_enabled);
80	DEFINE_PERCPU_RWSEM(scx_fork_rwsem);
81	static atomic_t scx_enable_state_var = ATOMIC_INIT(SCX_DISABLED);
82	static DEFINE_RAW_SPINLOCK(scx_bypass_lock);
83	static bool scx_init_task_enabled;
84	static bool scx_switching_all;
85	DEFINE_STATIC_KEY_FALSE(__scx_switched_all);
86	static DEFINE_STATIC_KEY_FALSE(__scx_tid_to_task_enabled);
87
88	/*
89	* Gates cgroup ops delivery. Set at the end of the cgroup init phase of root
90	* enable and cleared before root disable starts tearing down tasks, both under
91	* scx_cgroup_lock(). Holding cgroup_lock() and seeing %true guarantees no race
92	* against root tearing down tasks.
93	*/
94	bool scx_cgroup_enabled;
95
96	/*
97	* True once SCX_OPS_TID_TO_TASK has been negotiated with the root scheduler
98	* and the tid->task table is live. Wraps the static key so callers don't
99	* take the address, and hints "likely enabled" for the common case where
100	* the feature is in use.
101	*/
102	static inline bool scx_tid_to_task_enabled(void)
103	{
104	return static_branch_likely(&__scx_tid_to_task_enabled);
105	}
106
107	static atomic_long_t scx_nr_rejected = ATOMIC_LONG_INIT(0);
108	static atomic_long_t scx_hotplug_seq = ATOMIC_LONG_INIT(0);
109
110	/* Global cursor for the per-CPU tid allocator. Starts at 1; tid 0 is reserved. */
111	static atomic64_t scx_tid_cursor = ATOMIC64_INIT(1);
112
113	/* is @dsq synchronized by the containing rq lock instead of dsq->lock? */
114	static bool dsq_is_rq_owned(struct scx_dispatch_q *dsq)
115	{
116	switch (dsq->id) {
117	case SCX_DSQ_LOCAL:
118	case SCX_DSQ_REJECT:
119	case SCX_DSQ_RESCUE:
120	return true;
121	default:
122	return false;
123	}
124	}
125
126	/* Cursor for unique scx_sched instance ids. id 0 is reserved. */
127	static atomic64_t scx_sched_id_cursor = ATOMIC64_INIT(0);
128
129	#ifdef CONFIG_EXT_SUB_SCHED
130	/*
131	* The sub sched being enabled. Used by scx_disable_and_exit_task() to exit
132	* tasks for the sub-sched being enabled. Use a global variable instead of a
133	* per-task field as all enables are serialized.
134	*/
135	struct scx_sched *scx_enabling_sub_sched;
136	#else
137	#define scx_enabling_sub_sched (struct scx_sched *)NULL
138	#endif /* CONFIG_EXT_SUB_SCHED */
139
140	/*
141	* A monotonically increasing sequence number that is incremented every time a
142	* scheduler is enabled. This can be used to check if any custom sched_ext
143	* scheduler has ever been used in the system.
144	*/
145	static atomic_long_t scx_enable_seq = ATOMIC_LONG_INIT(0);
146
147	/*
148	* Watchdog interval. All scx_sched's share a single watchdog timer and the
149	* interval is half of the shortest sch->watchdog_timeout.
150	*/
151	static unsigned long scx_watchdog_interval;
152
153	/*
154	* The last time the delayed work was run. This delayed work relies on
155	* ksoftirqd being able to run to service timer interrupts, so it's possible
156	* that this work itself could get wedged. To account for this, we check that
157	* it's not stalled in the timer tick, and trigger an error if it is.
158	*/
159	static unsigned long scx_watchdog_timestamp = INITIAL_JIFFIES;
160
161	static struct delayed_work scx_watchdog_work;
162
163	/*
164	* For %SCX_KICK_WAIT: Each CPU has a pointer to an array of kick_sync sequence
165	* numbers. The arrays are allocated with kvzalloc() as size can exceed percpu
166	* allocator limits on large machines. O(nr_cpu_ids^2) allocation, allocated
167	* lazily when enabling and freed when disabling to avoid waste when sched_ext
168	* isn't active.
169	*/
170	struct scx_kick_syncs {
171	struct rcu_head rcu;
172	unsigned long syncs[];
173	};
174
175	static DEFINE_PER_CPU(struct scx_kick_syncs __rcu *, scx_kick_syncs);
176
177	/*
178	* Per-CPU buffered allocator state for p->scx.tid. Each CPU pulls a chunk of
179	* SCX_TID_CHUNK ids from scx_tid_cursor and hands them out locally without
180	* further synchronization. See scx_alloc_tid().
181	*/
182	struct scx_tid_alloc {
183	u64 next;
184	u64 end;
185	};
186	static DEFINE_PER_CPU(struct scx_tid_alloc, scx_tid_alloc);
187
188	/*
189	* Direct dispatch marker.
190	*
191	* Non-NULL values are used for direct dispatch from enqueue path. A valid
192	* pointer points to the task currently being enqueued. An ERR_PTR value is used
193	* to indicate that direct dispatch has already happened.
194	*/
195	static DEFINE_PER_CPU(struct task_struct *, direct_dispatch_task);
196
197	static const struct rhashtable_params dsq_hash_params = {
198	.key_len = sizeof_field(struct scx_dispatch_q, id),
199	.key_offset = offsetof(struct scx_dispatch_q, id),
200	.head_offset = offsetof(struct scx_dispatch_q, hash_node),
201	};
202
203	static LLIST_HEAD(dsqs_to_free);
204
205	/* ops debug dump */
206	static DEFINE_RAW_SPINLOCK(scx_dump_lock);
207
208	struct scx_dump_data {
209	s32 cpu;
210	bool first;
211	s32 cursor;
212	struct seq_buf *s;
213	const char *prefix;
214	struct scx_bstr_buf buf;
215	};
216
217	static struct scx_dump_data scx_dump_data = {
218	.cpu = -1,
219	};
220
221	/* /sys/kernel/sched_ext interface */
222	static struct kset *scx_kset;
223
224	/*
225	* Parameters that can be adjusted through /sys/module/sched_ext/parameters.
226	* There usually is no reason to modify these as normal scheduler operation
227	* shouldn't be affected by them. The knobs are primarily for debugging.
228	*/
229	static unsigned int scx_slice_bypass_us = SCX_SLICE_BYPASS / NSEC_PER_USEC;
230	static unsigned int scx_bypass_lb_intv_us = SCX_BYPASS_LB_DFL_INTV_US;
231
232	static int set_slice_us(const char val, const struct kernel_param kp)
233	{
234	return param_set_uint_minmax(val, kp, 100, 100 * USEC_PER_MSEC);
235	}
236
237	static const struct kernel_param_ops slice_us_param_ops = {
238	.set = set_slice_us,
239	.get = param_get_uint,
240	};
241
242	static int set_bypass_lb_intv_us(const char val, const struct kernel_param kp)
243	{
244	return param_set_uint_minmax(val, kp, 0, 10 * USEC_PER_SEC);
245	}
246
247	static const struct kernel_param_ops bypass_lb_intv_us_param_ops = {
248	.set = set_bypass_lb_intv_us,
249	.get = param_get_uint,
250	};
251
252	#undef MODULE_PARAM_PREFIX
253	#define MODULE_PARAM_PREFIX "sched_ext."
254
255	module_param_cb(slice_bypass_us, &slice_us_param_ops, &scx_slice_bypass_us, 0600);
256	MODULE_PARM_DESC(slice_bypass_us, "bypass slice in microseconds, applied on [un]load (100us to 100ms)");
257	module_param_cb(bypass_lb_intv_us, &bypass_lb_intv_us_param_ops, &scx_bypass_lb_intv_us, 0600);
258	MODULE_PARM_DESC(bypass_lb_intv_us, "bypass load balance interval in microseconds (0 (disable) to 10s)");
259
260	#undef MODULE_PARAM_PREFIX
261
262	#define CREATE_TRACE_POINTS
263	#include <trace/events/sched_ext.h>
264
265	static void run_deferred(struct rq *rq);
266	static bool task_dead_and_done(struct task_struct *p);
267	static void scx_disable(struct scx_sched *sch, enum scx_exit_kind kind);
268
269	__printf(5, 6) bool __scx_exit(struct scx_sched *sch,
270	enum scx_exit_kind kind, s64 exit_code,
271	s32 exit_cpu, const char *fmt, ...)
272	{
273	va_list args;
274	bool ret;
275
276	va_start(args, fmt);
277	ret = scx_vexit(sch, kind, exit_code, exit_cpu, fmt, args);
278	va_end(args);
279
280	return ret;
281	}
282
283	static long jiffies_delta_msecs(unsigned long at, unsigned long now)
284	{
285	if (time_after(at, now))
286	return jiffies_to_msecs(at - now);
287	else
288	return -(long)jiffies_to_msecs(now - at);
289	}
290
291	static bool u32_before(u32 a, u32 b)
292	{
293	return (s32)(a - b) < 0;
294	}
295
296	/**
297	* scx_is_descendant - Test whether sched is a descendant
298	* @sch: sched to test
299	* @ancestor: ancestor sched to test against
300	*
301	* Test whether @sch is a descendant of @ancestor.
302	*/
303	bool scx_is_descendant(struct scx_sched sch, struct scx_sched ancestor)
304	{
305	if (sch->level < ancestor->level)
306	return false;
307	return sch->ancestors[ancestor->level] == ancestor;
308	}
309
310	static struct scx_dispatch_q find_global_dsq(struct scx_sched sch, s32 cpu)
311	{
312	return &sch->pnode[cpu_to_node(cpu)]->global_dsq;
313	}
314
315	static struct scx_dispatch_q find_user_dsq(struct scx_sched sch, u64 dsq_id)
316	{
317	return rhashtable_lookup(&sch->dsq_hash, &dsq_id, dsq_hash_params);
318	}
319
320	static const struct sched_class scx_setscheduler_class(struct task_struct p)
321	{
322	if (p->sched_class == &stop_sched_class)
323	return &stop_sched_class;
324
325	return __setscheduler_class(p->policy, p->prio);
326	}
327
328	static struct scx_dispatch_q bypass_enq_target_dsq(struct scx_sched sch, s32 cpu)
329	{
330	#ifdef CONFIG_EXT_SUB_SCHED
331	/*
332	* If @sch is a sub-sched which is bypassing, its tasks should go into
333	* the bypass DSQs of the nearest ancestor which is not bypassing. The
334	* not-bypassing ancestor is responsible for scheduling all tasks from
335	* bypassing sub-trees. If all ancestors including root are bypassing,
336	* all tasks should go to the root's bypass DSQs.
337	*
338	* Whenever a sched starts bypassing, all runnable tasks in its subtree
339	* are re-enqueued after scx_bypassing() is turned on, guaranteeing that
340	* all tasks are transferred to the right DSQs.
341	*/
342	while (scx_parent(sch) && scx_bypassing(sch, cpu))
343	sch = scx_parent(sch);
344	#endif /* CONFIG_EXT_SUB_SCHED */
345
346	return scx_bypass_dsq(sch, cpu);
347	}
348
349	/**
350	* rq_is_open - Is the rq available for immediate execution of an SCX task?
351	* @rq: rq to test
352	* @enq_flags: optional %SCX_ENQ_* of the task being enqueued
353	*
354	* Returns %true if @rq is currently open for executing an SCX task. After a
355	* %false return, @rq is guaranteed to invoke SCX dispatch path at least once
356	* before going to idle and not inserting a task into @rq's local DSQ after a
357	* %false return doesn't cause @rq to stall.
358	*/
359	static bool rq_is_open(struct rq *rq, u64 enq_flags)
360	{
361	lockdep_assert_rq_held(rq);
362
363	/*
364	* A higher-priority class task is either running or in the process of
365	* waking up on @rq.
366	*/
367	if (sched_class_above(rq->next_class, &ext_sched_class))
368	return false;
369
370	/*
371	* @rq is either in transition to or in idle and there is no
372	* higher-priority class task waking up on it.
373	*/
374	if (sched_class_above(&ext_sched_class, rq->next_class))
375	return true;
376
377	/*
378	* @rq is either picking, in transition to, or running an SCX task.
379	*/
380
381	/*
382	* If we're in the dispatch path holding rq lock, $curr may or may not
383	* be ready depending on whether the on-going dispatch decides to extend
384	* $curr's slice. We say yes here and resolve it at the end of dispatch.
385	* See dispatch_one().
386	*/
387	if (rq->scx.flags & SCX_RQ_IN_DISPATCH)
388	return true;
389
390	/*
391	* %SCX_ENQ_PREEMPT clears $curr's slice if on SCX and kicks dispatch,
392	* so allow it to avoid spuriously triggering reenq on a combined
393	* PREEMPT\|IMMED insertion.
394	*/
395	if (enq_flags & SCX_ENQ_PREEMPT) {
396	struct task_struct *curr = rq->curr;
397
398	/*
399	* A protected slice refuses the preemption and the cpu stays
400	* occupied. See rq_owned_post_enq().
401	*/
402	return curr->sched_class != &ext_sched_class \|\|
403	likely(!(curr->scx.flags & SCX_TASK_PROTECTED));
404	}
405
406	/*
407	* @rq is either in transition to or running an SCX task and can't go
408	* idle without another SCX dispatch cycle.
409	*/
410	return false;
411	}
412
413	/*
414	* Track the rq currently locked.
415	*
416	* This allows kfuncs to safely operate on rq from any scx ops callback,
417	* knowing which rq is already locked.
418	*/
419	DEFINE_PER_CPU(struct rq *, scx_locked_rq_state);
420
421	/*
422	* Under core scheduling, a pick that releases the rq lock invalidates the
423	* core-wide selection it is part of. Count the releases so that the core-sched
424	* pick can tell whether one happened across dispatch.
425	*/
426	static void scx_rq_lock_drop(struct rq *rq)
427	{
428	lockdep_assert_rq_held(rq);
429	#ifdef CONFIG_SCHED_CORE
430	if (sched_core_enabled(rq))
431	rq->scx.lock_drop_seq++;
432	#endif
433	}
434
435	static void switch_rq_lock(struct rq from, struct rq to)
436	{
437	bool tracked = scx_locked_rq() == from;
438
439	if (tracked)
440	update_locked_rq(NULL);
441	scx_rq_lock_drop(from);
442	raw_spin_rq_unlock(from);
443	raw_spin_rq_lock(to);
444	if (tracked)
445	update_locked_rq(to);
446	}
447
448	/*
449	* Flipped on enable per sch->is_cid_type. Declared in internal.h so
450	* subsystem inlines can read it.
451	*/
452	DEFINE_STATIC_KEY_FALSE(__scx_is_cid_type);
453
454	/**
455	* scx_fill_cmask_scratch - Build this cpu's arena cmask from @cpumask
456	* @sch: scx_sched whose scratch to fill
457	* @cpumask: cpus to translate into cids
458	*
459	* The scratch lives in BPF-writable arena memory and its header can't be
460	* trusted, so it is rewritten from kernel geometry rather than read. Caller
461	* must hold an rq lock so this cpu is the sole kernel writer for as long as the
462	* returned address is in use.
463	*/
464	static struct scx_cmask scx_fill_cmask_scratch(struct scx_sched sch,
465	const struct cpumask *cpumask)
466	{
467	struct scx_cmask kern_va = this_cpu_ptr(sch->set_cmask_scratch);
468	struct scx_cmask_ref ref;
469
470	scx_cmask_ref_init_kern(sch, kern_va, 0, num_possible_cpus(), &ref);
471	scx_cmask_ref_from_cpumask(&ref, cpumask);
472	return kern_va;
473	}
474
475	/**
476	* scx_call_op_set_cpumask - Invoke the set_cpumask or set_cmask op for @task
477	* @sch: scx_sched being invoked
478	* @rq: rq to update as the currently-locked rq, or NULL
479	* @task: task whose affinity is changing
480	* @cpumask: new cpumask
481	*
482	* For cid-form schedulers, translate @cpumask to a cmask in the per-cpu scratch
483	* and dispatch through the ops_cid union view. Caller must hold @rq's rq lock.
484	*/
485	static inline void scx_call_op_set_cpumask(struct scx_sched sch, struct rq rq,
486	struct task_struct *task,
487	const struct cpumask *cpumask)
488	{
489	if (scx_is_cid_type())
490	SCX_CALL_CID_OP_TASK(sch, set_cmask, rq, task,
491	scx_fill_cmask_scratch(sch, cpumask));
492	else
493	SCX_CALL_OP_TASK(sch, set_cpumask, rq, task, cpumask);
494	}
495
496	enum scx_dsq_iter_flags {
497	/* iterate in the reverse dispatch order */
498	SCX_DSQ_ITER_REV = 1U << 16,
499
500	__SCX_DSQ_ITER_HAS_SLICE = 1U << 30,
501	__SCX_DSQ_ITER_HAS_VTIME = 1U << 31,
502
503	__SCX_DSQ_ITER_USER_FLAGS = SCX_DSQ_ITER_REV,
504	__SCX_DSQ_ITER_ALL_FLAGS = __SCX_DSQ_ITER_USER_FLAGS \|
505	__SCX_DSQ_ITER_HAS_SLICE \|
506	__SCX_DSQ_ITER_HAS_VTIME,
507	};
508
509	/**
510	* nldsq_next_task - Iterate to the next task in a non-local DSQ
511	* @dsq: non-local dsq being iterated
512	* @cur: current position, %NULL to start iteration
513	* @rev: walk backwards
514	*
515	* Returns %NULL when iteration is finished.
516	*/
517	static struct task_struct nldsq_next_task(struct scx_dispatch_q dsq,
518	struct task_struct *cur, bool rev)
519	{
520	struct list_head *list_node;
521	struct scx_dsq_list_node *dsq_lnode;
522
523	lockdep_assert_held(&dsq->lock);
524
525	if (cur)
526	list_node = &cur->scx.dsq_list.node;
527	else
528	list_node = &dsq->list;
529
530	/* find the next task, need to skip BPF iteration cursors */
531	do {
532	if (rev)
533	list_node = list_node->prev;
534	else
535	list_node = list_node->next;
536
537	if (list_node == &dsq->list)
538	return NULL;
539
540	dsq_lnode = container_of(list_node, struct scx_dsq_list_node,
541	node);
542	} while (dsq_lnode->flags & SCX_DSQ_LNODE_ITER_CURSOR);
543
544	return container_of(dsq_lnode, struct task_struct, scx.dsq_list);
545	}
546
547	#define nldsq_for_each_task(p, dsq) \
548	for ((p) = nldsq_next_task((dsq), NULL, false); (p); \
549	(p) = nldsq_next_task((dsq), (p), false))
550
551	/**
552	* nldsq_cursor_next_task - Iterate to the next task given a cursor in a non-local DSQ
553	* @cursor: scx_dsq_list_node initialized with INIT_DSQ_LIST_CURSOR()
554	* @dsq: non-local dsq being iterated
555	*
556	* Find the next task in a cursor based iteration. The caller must have
557	* initialized @cursor using INIT_DSQ_LIST_CURSOR() and can release the DSQ lock
558	* between the iteration steps.
559	*
560	* Only tasks which were queued before @cursor was initialized are visible. This
561	* bounds the iteration and guarantees that vtime never jumps in the other
562	* direction while iterating.
563	*/
564	static struct task_struct nldsq_cursor_next_task(struct scx_dsq_list_node cursor,
565	struct scx_dispatch_q *dsq)
566	{
567	bool rev = cursor->flags & SCX_DSQ_ITER_REV;
568	struct task_struct *p;
569
570	lockdep_assert_held(&dsq->lock);
571	BUG_ON(!(cursor->flags & SCX_DSQ_LNODE_ITER_CURSOR));
572
573	if (list_empty(&cursor->node))
574	p = NULL;
575	else
576	p = container_of(cursor, struct task_struct, scx.dsq_list);
577
578	/* skip cursors and tasks that were queued after @cursor init */
579	do {
580	p = nldsq_next_task(dsq, p, rev);
581	} while (p && unlikely(u32_before(cursor->priv, p->scx.dsq_seq)));
582
583	if (p) {
584	if (rev)
585	list_move_tail(&cursor->node, &p->scx.dsq_list.node);
586	else
587	list_move(&cursor->node, &p->scx.dsq_list.node);
588	} else {
589	list_del_init(&cursor->node);
590	}
591
592	return p;
593	}
594
595	/**
596	* nldsq_cursor_lost_task - Test whether someone else took the task since iteration
597	* @cursor: scx_dsq_list_node initialized with INIT_DSQ_LIST_CURSOR()
598	* @rq: rq @p was on
599	* @dsq: dsq @p was on
600	* @p: target task
601	*
602	* @p is a task returned by nldsq_cursor_next_task(). The locks may have been
603	* dropped and re-acquired inbetween. Verify that no one else took or is in the
604	* process of taking @p from @dsq.
605	*
606	* On %false return, the caller can assume full ownership of @p.
607	*/
608	static bool nldsq_cursor_lost_task(struct scx_dsq_list_node *cursor,
609	struct rq rq, struct scx_dispatch_q dsq,
610	struct task_struct *p)
611	{
612	lockdep_assert_rq_held(rq);
613	lockdep_assert_held(&dsq->lock);
614
615	/*
616	* @p could have already left $src_dsq, got re-enqueud, or be in the
617	* process of being consumed by someone else.
618	*/
619	if (unlikely(p->scx.dsq != dsq \|\|
620	u32_before(cursor->priv, p->scx.dsq_seq) \|\|
621	p->scx.holding_cpu >= 0))
622	return true;
623
624	/* if @p has stayed on @dsq, its rq couldn't have changed */
625	if (WARN_ON_ONCE(rq != task_rq(p)))
626	return true;
627
628	return false;
629	}
630
631	/*
632	* BPF DSQ iterator. Tasks in a non-local DSQ can be iterated in [reverse]
633	* dispatch order. BPF-visible iterator is opaque and larger to allow future
634	* changes without breaking backward compatibility. Can be used with
635	* bpf_for_each(). See bpf_iter_scx_dsq_*().
636	*/
637	struct bpf_iter_scx_dsq_kern {
638	struct scx_dsq_list_node cursor;
639	struct scx_dispatch_q *dsq;
640	u64 slice;
641	u64 vtime;
642	} __attribute__((aligned(8)));
643
644	struct bpf_iter_scx_dsq {
645	u64 __opaque[6];
646	} __attribute__((aligned(8)));
647
648
649	u32 scx_get_task_state(const struct task_struct *p)
650	{
651	return p->scx.flags & SCX_TASK_STATE_MASK;
652	}
653
654	void scx_set_task_state(struct task_struct *p, u32 state)
655	{
656	u32 prev_state = scx_get_task_state(p);
657	bool warn = false;
658
659	switch (state) {
660	case SCX_TASK_NONE:
661	warn = prev_state == SCX_TASK_DEAD;
662	break;
663	case SCX_TASK_INIT_BEGIN:
664	warn = prev_state != SCX_TASK_NONE;
665	break;
666	case SCX_TASK_INIT:
667	warn = prev_state != SCX_TASK_INIT_BEGIN;
668	p->scx.flags \|= SCX_TASK_RESET_RUNNABLE_AT;
669	break;
670	case SCX_TASK_READY:
671	warn = !(prev_state == SCX_TASK_INIT \|\|
672	prev_state == SCX_TASK_ENABLED);
673	break;
674	case SCX_TASK_ENABLED:
675	warn = prev_state != SCX_TASK_READY;
676	break;
677	case SCX_TASK_DEAD:
678	warn = !(prev_state == SCX_TASK_NONE \|\|
679	prev_state == SCX_TASK_INIT_BEGIN);
680	break;
681	default:
682	WARN_ONCE(1, "sched_ext: Invalid task state %d -> %d for %s[%d]",
683	prev_state, state, p->comm, p->pid);
684	return;
685	}
686
687	WARN_ONCE(warn, "sched_ext: Invalid task state transition 0x%x -> 0x%x for %s[%d]",
688	prev_state, state, p->comm, p->pid);
689
690	p->scx.flags &= ~SCX_TASK_STATE_MASK;
691	p->scx.flags \|= state;
692	}
693
694	/**
695	* scx_task_iter_start - Lock scx_tasks_lock and start a task iteration
696	* @iter: iterator to init
697	* @cgrp: Optional root of cgroup subhierarchy to iterate
698	*
699	* Initialize @iter. Once initialized, @iter must eventually be stopped with
700	* scx_task_iter_stop().
701	*
702	* If @cgrp is %NULL, scx_tasks is used for iteration and this function returns
703	* with scx_tasks_lock held and @iter->cursor inserted into scx_tasks.
704	*
705	* If @cgrp is not %NULL, @cgrp and its descendants' tasks are walked using
706	* @iter->css_iter. The caller must be holding cgroup_lock() to prevent cgroup
707	* task migrations.
708	*
709	* The two modes of iterations are largely independent and it's likely that
710	* scx_tasks can be removed in favor of always using cgroup iteration if
711	* CONFIG_SCHED_CLASS_EXT depends on CONFIG_CGROUPS.
712	*
713	* scx_tasks_lock and the rq lock may be released using scx_task_iter_unlock()
714	* between this and the first next() call or between any two next() calls. If
715	* the locks are released between two next() calls, the caller is responsible
716	* for ensuring that the task being iterated remains accessible either through
717	* RCU read lock or obtaining a reference count.
718	*
719	* All tasks which existed when the iteration started are guaranteed to be
720	* visited as long as they are not dead.
721	*/
722	void scx_task_iter_start(struct scx_task_iter iter, struct cgroup cgrp)
723	{
724	memset(iter, 0, sizeof(*iter));
725
726	#ifdef CONFIG_EXT_SUB_SCHED
727	if (cgrp) {
728	lockdep_assert_held(&cgroup_mutex);
729	iter->cgrp = cgrp;
730	iter->css_pos = css_next_descendant_pre(NULL, &iter->cgrp->self);
731	css_task_iter_start(iter->css_pos, CSS_TASK_ITER_WITH_DEAD,
732	&iter->css_iter);
733	return;
734	}
735	#endif
736	raw_spin_lock_irq(&scx_tasks_lock);
737
738	iter->cursor = (struct sched_ext_entity){ .flags = SCX_TASK_CURSOR };
739	list_add(&iter->cursor.tasks_node, &scx_tasks);
740	iter->list_locked = true;
741	}
742
743	static void __scx_task_iter_rq_unlock(struct scx_task_iter *iter)
744	{
745	if (iter->locked_task) {
746	__balance_callbacks(iter->rq, &iter->rf);
747	task_rq_unlock(iter->rq, iter->locked_task, &iter->rf);
748	iter->locked_task = NULL;
749	}
750	}
751
752	/**
753	* scx_task_iter_unlock - Unlock rq and scx_tasks_lock held by a task iterator
754	* @iter: iterator to unlock
755	*
756	* If @iter is in the middle of a locked iteration, it may be locking the rq of
757	* the task currently being visited in addition to scx_tasks_lock. Unlock both.
758	* This function can be safely called anytime during an iteration. The next
759	* iterator operation will automatically restore the necessary locking.
760	*/
761	void scx_task_iter_unlock(struct scx_task_iter *iter)
762	{
763	__scx_task_iter_rq_unlock(iter);
764	if (iter->list_locked) {
765	iter->list_locked = false;
766	raw_spin_unlock_irq(&scx_tasks_lock);
767	}
768	}
769
770	static void __scx_task_iter_maybe_relock(struct scx_task_iter *iter)
771	{
772	if (!iter->list_locked) {
773	raw_spin_lock_irq(&scx_tasks_lock);
774	iter->list_locked = true;
775	}
776	}
777
778	/**
779	* scx_task_iter_relock - Re-acquire scx_tasks_lock and, optionally, @p's rq
780	* @iter: iterator to relock
781	* @p: task whose rq to lock, or %NULL for scx_tasks_lock only
782	*
783	* Counterpart to scx_task_iter_unlock(). Locking @p's rq is optional. Once
784	* re-acquired, both locks are managed by the iterator from here on.
785	*/
786	static void scx_task_iter_relock(struct scx_task_iter *iter,
787	struct task_struct *p)
788	{
789	__scx_task_iter_maybe_relock(iter);
790	if (p) {
791	iter->rq = task_rq_lock(p, &iter->rf);
792	iter->locked_task = p;
793	}
794	}
795
796	/**
797	* scx_task_iter_stop - Stop a task iteration and unlock scx_tasks_lock
798	* @iter: iterator to exit
799	*
800	* Exit a previously initialized @iter. Must be called with scx_tasks_lock held
801	* which is released on return. If the iterator holds a task's rq lock, that rq
802	* lock is also released. See scx_task_iter_start() for details.
803	*/
804	void scx_task_iter_stop(struct scx_task_iter *iter)
805	{
806	#ifdef CONFIG_EXT_SUB_SCHED
807	if (iter->cgrp) {
808	if (iter->css_pos)
809	css_task_iter_end(&iter->css_iter);
810	__scx_task_iter_rq_unlock(iter);
811	return;
812	}
813	#endif
814	__scx_task_iter_maybe_relock(iter);
815	list_del_init(&iter->cursor.tasks_node);
816	scx_task_iter_unlock(iter);
817	}
818
819	/**
820	* scx_task_iter_next - Next task
821	* @iter: iterator to walk
822	*
823	* Visit the next task. See scx_task_iter_start() for details. Locks are dropped
824	* and re-acquired every %SCX_TASK_ITER_BATCH iterations to avoid causing stalls
825	* by holding scx_tasks_lock for too long.
826	*/
827	static struct task_struct scx_task_iter_next(struct scx_task_iter iter)
828	{
829	struct list_head *cursor = &iter->cursor.tasks_node;
830	struct sched_ext_entity *pos;
831
832	if (!(++iter->cnt % SCX_TASK_ITER_BATCH)) {
833	scx_task_iter_unlock(iter);
834	cond_resched();
835	}
836
837	#ifdef CONFIG_EXT_SUB_SCHED
838	if (iter->cgrp) {
839	while (iter->css_pos) {
840	struct task_struct *p;
841
842	p = css_task_iter_next(&iter->css_iter);
843	if (p)
844	return p;
845
846	css_task_iter_end(&iter->css_iter);
847	iter->css_pos = css_next_descendant_pre(iter->css_pos,
848	&iter->cgrp->self);
849	if (iter->css_pos)
850	css_task_iter_start(iter->css_pos, CSS_TASK_ITER_WITH_DEAD,
851	&iter->css_iter);
852	}
853	return NULL;
854	}
855	#endif
856	__scx_task_iter_maybe_relock(iter);
857
858	list_for_each_entry(pos, cursor, tasks_node) {
859	if (&pos->tasks_node == &scx_tasks)
860	return NULL;
861	if (!(pos->flags & SCX_TASK_CURSOR)) {
862	list_move(cursor, &pos->tasks_node);
863	return container_of(pos, struct task_struct, scx);
864	}
865	}
866
867	/* can't happen, should always terminate at scx_tasks above */
868	BUG();
869	}
870
871	/**
872	* scx_task_iter_next_locked - Next non-idle task with its rq locked
873	* @iter: iterator to walk
874	*
875	* Visit the non-idle task with its rq lock held. Allows callers to specify
876	* whether they would like to filter out dead tasks. See scx_task_iter_start()
877	* for details.
878	*/
879	struct task_struct scx_task_iter_next_locked(struct scx_task_iter iter)
880	{
881	struct task_struct *p;
882
883	__scx_task_iter_rq_unlock(iter);
884
885	while ((p = scx_task_iter_next(iter))) {
886	/*
887	* scx_task_iter is used to prepare and move tasks into SCX
888	* while loading the BPF scheduler and vice-versa while
889	* unloading. The init_tasks ("swappers") should be excluded
890	* from the iteration because:
891	*
892	* - It's unsafe to use __setscheduler_class() on an init_task
893	* to determine the sched_class to use as it won't preserve
894	* its idle_sched_class.
895	*
896	* - ops.init/exit_task() can easily be confused if called with
897	* init_tasks as they, e.g., share PID 0.
898	*
899	* As init_tasks are never scheduled through SCX, they can be
900	* skipped safely. Note that is_idle_task() which tests %PF_IDLE
901	* doesn't work here:
902	*
903	* - %PF_IDLE may not be set for an init_task whose CPU hasn't
904	* yet been onlined.
905	*
906	* - %PF_IDLE can be set on tasks that are not init_tasks. See
907	* play_idle_precise() used by CONFIG_IDLE_INJECT.
908	*
909	* Test for idle_sched_class as only init_tasks are on it.
910	*/
911	if (p->sched_class == &idle_sched_class)
912	continue;
913
914	iter->rq = task_rq_lock(p, &iter->rf);
915	iter->locked_task = p;
916
917	/*
918	* cgroup_task_dead() removes the dead tasks from cset->tasks
919	* after sched_ext_dead() and cgroup iteration may see tasks
920	* which already finished sched_ext_dead(). %SCX_TASK_DEAD is
921	* set by sched_ext_dead() under @p's rq lock. Test it to
922	* avoid visiting tasks which are already dead from SCX POV.
923	*/
924	if (scx_get_task_state(p) == SCX_TASK_DEAD) {
925	__scx_task_iter_rq_unlock(iter);
926	continue;
927	}
928
929	return p;
930	}
931	return NULL;
932	}
933
934	/**
935	* scx_dump_event - Dump an event 'kind' in 'events' to 's'
936	* @s: output seq_buf
937	* @events: event stats
938	* @kind: a kind of event to dump
939	*/
940	#define scx_dump_event(s, events, kind) do { \
941	scx_dump_line(&(s), "%40s: %16lld", #kind, (events)->kind); \
942	} while (0)
943
944
945	static void scx_read_events(struct scx_sched *sch,
946	struct scx_event_stats *events);
947
948	static enum scx_enable_state scx_enable_state(void)
949	{
950	return atomic_read(&scx_enable_state_var);
951	}
952
953	static enum scx_enable_state scx_set_enable_state(enum scx_enable_state to)
954	{
955	return atomic_xchg(&scx_enable_state_var, to);
956	}
957
958	static bool scx_tryset_enable_state(enum scx_enable_state to,
959	enum scx_enable_state from)
960	{
961	int from_v = from;
962
963	return atomic_try_cmpxchg(&scx_enable_state_var, &from_v, to);
964	}
965
966	/**
967	* wait_ops_state - Busy-wait the specified ops state to end
968	* @p: target task
969	* @opss: state to wait the end of
970	*
971	* Busy-wait for @p to transition out of @opss. This can only be used when the
972	* state part of @opss is %SCX_QUEUEING or %SCX_DISPATCHING. This function also
973	* has load_acquire semantics to ensure that the caller can see the updates made
974	* in the enqueueing and dispatching paths.
975	*/
976	static void wait_ops_state(struct task_struct *p, unsigned long opss)
977	{
978	do {
979	cpu_relax();
980	} while (atomic_long_read_acquire(&p->scx.ops_state) == opss);
981	}
982
983	static inline bool __cpu_valid(s32 cpu)
984	{
985	return likely(cpu >= 0 && cpu < nr_cpu_ids && cpu_possible(cpu));
986	}
987
988	/**
989	* scx_cpu_valid - Verify a cpu number, to be used on ops input args
990	* @sch: scx_sched to abort on error
991	* @cpu: cpu number which came from a BPF ops
992	* @where: extra information reported on error
993	*
994	* @cpu is a cpu number which came from the BPF scheduler and can be any value.
995	* Verify that it is in range and one of the possible cpus. If invalid, trigger
996	* an ops error.
997	*/
998	bool scx_cpu_valid(struct scx_sched sch, s32 cpu, const char where)
999	{
1000	if (__cpu_valid(cpu)) {
1001	return true;
1002	} else {
1003	scx_error(sch, "invalid CPU %d%s%s", cpu, where ? " " : "", where ?: "");
1004	return false;
1005	}
1006	}
1007
1008	static void deferred_bal_cb_workfn(struct rq *rq)
1009	{
1010	run_deferred(rq);
1011	}
1012
1013	static void deferred_irq_workfn(struct irq_work *irq_work)
1014	{
1015	struct rq *rq = container_of(irq_work, struct rq, scx.deferred_irq_work);
1016
1017	raw_spin_rq_lock(rq);
1018	run_deferred(rq);
1019	scx_rq_lock_drop(rq);
1020	raw_spin_rq_unlock(rq);
1021	}
1022
1023	/**
1024	* schedule_deferred - Schedule execution of deferred actions on an rq
1025	* @rq: target rq
1026	*
1027	* Schedule execution of deferred actions on @rq. Deferred actions are executed
1028	* with @rq locked but unpinned, and thus can unlock @rq to e.g. migrate tasks
1029	* to other rqs.
1030	*/
1031	static void schedule_deferred(struct rq *rq)
1032	{
1033	/*
1034	* This is the fallback when schedule_deferred_locked() can't use
1035	* the cheaper balance callback or wakeup hook paths (the target
1036	* CPU is not in dispatch or wakeup). Currently, this is primarily
1037	* hit by reenqueue operations targeting a remote CPU.
1038	*
1039	* Queue on the target CPU. The deferred work can run from any CPU
1040	* correctly - the _locked() path already processes remote rqs from
1041	* the calling CPU - but targeting the owning CPU allows IPI delivery
1042	* without waiting for the calling CPU to re-enable IRQs and is
1043	* cheaper as the reenqueue runs locally.
1044	*/
1045	irq_work_queue_on(&rq->scx.deferred_irq_work, cpu_of(rq));
1046	}
1047
1048	/**
1049	* schedule_deferred_locked - Schedule execution of deferred actions on an rq
1050	* @rq: target rq
1051	*
1052	* Schedule execution of deferred actions on @rq. Equivalent to
1053	* schedule_deferred() but requires @rq to be locked and can be more efficient.
1054	*/
1055	static void schedule_deferred_locked(struct rq *rq)
1056	{
1057	lockdep_assert_rq_held(rq);
1058
1059	/*
1060	* If in the middle of waking up a task, task_woken_scx() will be called
1061	* afterwards which will then run the deferred actions, no need to
1062	* schedule anything.
1063	*/
1064	if (rq->scx.flags & SCX_RQ_IN_WAKEUP)
1065	return;
1066
1067	/* Don't do anything if there already is a deferred operation. */
1068	if (rq->scx.flags & SCX_RQ_BAL_CB_PENDING)
1069	return;
1070
1071	/*
1072	* If in dispatch, the balance callbacks will be called before rq lock
1073	* is released. Schedule one.
1074	*
1075	*
1076	* We can't directly insert the callback into the
1077	* rq's list: The call can drop its lock and make the pending balance
1078	* callback visible to unrelated code paths that call rq_pin_lock().
1079	*
1080	* Just let dispatch_one() know that it must do it itself.
1081	*/
1082	if (rq->scx.flags & SCX_RQ_IN_DISPATCH) {
1083	rq->scx.flags \|= SCX_RQ_BAL_CB_PENDING;
1084	return;
1085	}
1086
1087	/*
1088	* No scheduler hooks available. Use the generic irq_work path. The
1089	* above WAKEUP and DISPATCH paths should cover most of the cases and
1090	* the time to IRQ re-enable shouldn't be long.
1091	*/
1092	schedule_deferred(rq);
1093	}
1094
1095	void scx_proxy_reenqueue_retry(struct rq rq, struct task_struct next)
1096	{
1097	}
1098
1099	void schedule_dsq_reenq(struct scx_sched sch, struct scx_dispatch_q dsq,
1100	u64 reenq_flags, struct rq *locked_rq)
1101	{
1102	struct rq *rq;
1103
1104	/*
1105	* Allowing reenqueues doesn't make sense while bypassing. This also
1106	* blocks from new reenqueues to be scheduled on dead scheds.
1107	*/
1108	if (unlikely(READ_ONCE(sch->bypass_depth)))
1109	return;
1110
1111	if (dsq->id == SCX_DSQ_LOCAL) {
1112	rq = container_of(dsq, struct rq, scx.local_dsq);
1113
1114	/*
1115	* A sub-sched lacking baseline access on the target cid has no
1116	* business triggering IPIs. The lockless test is fine: slipping
1117	* through right after a revoke is harmless and a wrong denial
1118	* can't happen - if the caller has seen its ownership, so does
1119	* this test.
1120	*/
1121	if (unlikely(scx_missing_caps(sch, cpu_of(rq), SCX_CAP_BASE))) {
1122	__scx_add_event(sch, SCX_EV_SUB_REENQ_DENIED, 1);
1123	return;
1124	}
1125
1126	struct scx_sched_pcpu *sch_pcpu = per_cpu_ptr(sch->pcpu, cpu_of(rq));
1127	struct scx_deferred_reenq_local *drl = &sch_pcpu->deferred_reenq_local;
1128
1129	/*
1130	* Pairs with smp_mb() in process_deferred_reenq_locals() and
1131	* guarantees that there is a reenq_local() afterwards.
1132	*/
1133	smp_mb();
1134
1135	if (list_empty(&drl->node) \|\|
1136	(READ_ONCE(drl->flags) & reenq_flags) != reenq_flags) {
1137
1138	guard(raw_spinlock_irqsave)(&rq->scx.deferred_reenq_lock);
1139
1140	if (list_empty(&drl->node))
1141	list_move_tail(&drl->node, &rq->scx.deferred_reenq_locals);
1142	WRITE_ONCE(drl->flags, drl->flags \| reenq_flags);
1143	}
1144	} else if (!(dsq->id & SCX_DSQ_FLAG_BUILTIN)) {
1145	rq = this_rq();
1146
1147	struct scx_dsq_pcpu *dsq_pcpu = per_cpu_ptr(dsq->pcpu, cpu_of(rq));
1148	struct scx_deferred_reenq_user *dru = &dsq_pcpu->deferred_reenq_user;
1149
1150	/*
1151	* Pairs with smp_mb() in process_deferred_reenq_users() and
1152	* guarantees that there is a reenq_user() afterwards.
1153	*/
1154	smp_mb();
1155
1156	if (list_empty(&dru->node) \|\|
1157	(READ_ONCE(dru->flags) & reenq_flags) != reenq_flags) {
1158
1159	guard(raw_spinlock_irqsave)(&rq->scx.deferred_reenq_lock);
1160
1161	/* @dsq is being destroyed, see free_dsq_irq_workfn() */
1162	if (unlikely(READ_ONCE(dsq->id) == SCX_DSQ_INVALID))
1163	return;
1164
1165	if (list_empty(&dru->node))
1166	list_move_tail(&dru->node, &rq->scx.deferred_reenq_users);
1167	WRITE_ONCE(dru->flags, dru->flags \| reenq_flags);
1168	}
1169	} else {
1170	scx_error(sch, "DSQ 0x%llx not allowed for reenq", dsq->id);
1171	return;
1172	}
1173
1174	if (rq == locked_rq)
1175	schedule_deferred_locked(rq);
1176	else
1177	schedule_deferred(rq);
1178	}
1179
1180	/*
1181	* p->scx.slice_oob packs an out-of-band slice request into one atomic64. A zero
1182	* word means no request. Otherwise the fields are:
1183	*
1184	* 63 SCX_SLICE_OOB_PENDING, set on every request
1185	* 62-43 lower bits of issuing scheduler's id
1186	* 42-0 requested slice duration in nsecs
1187	*
1188	* A duration of SCX_SLICE_OOB_DUR_MASK means SCX_SLICE_INF. A finite dur
1189	* saturates at SCX_SLICE_OOB_DUR_MASK - 1. The id is used to detect and ignore
1190	* a request that outlived a task ownership change.
1191	*
1192	* Only the low 20 bits of sch->id are packed, which is enough to make
1193	* collisions practically impossible. A theoretical collision just lets a stale
1194	* request through once.
1195	*/
1196	enum scx_slice_oob_consts {
1197	SCX_SLICE_OOB_DUR_BITS = 43,
1198	SCX_SLICE_OOB_ID_BITS = 64 - SCX_SLICE_OOB_DUR_BITS - 1,
1199
1200	SCX_SLICE_OOB_DUR_MASK = (1LLU << SCX_SLICE_OOB_DUR_BITS) - 1,
1201	SCX_SLICE_OOB_ID_SHIFT = SCX_SLICE_OOB_DUR_BITS,
1202	SCX_SLICE_OOB_ID_MASK = (1LLU << SCX_SLICE_OOB_ID_BITS) - 1,
1203	SCX_SLICE_OOB_PENDING = 1LLU << 63,
1204	};
1205
1206	/*
1207	* Slice and dsq_vtime write rules
1208	*
1209	* While @p is running, sleeping or queued on an rq-owned DSQ, both fields are
1210	* protected by the rq lock. While running, the rq lock is required because
1211	* update_curr_scx() RMWs the slice and the cap check for slice extension is
1212	* only reliable under the rq lock.
1213	*
1214	* While @p is queued on a user DSQ or on the BPF side, the kernel neither
1215	* consumes nor decides on the fields. Synchronizing the writers is the BPF
1216	* scheduler's responsibility. An rq-locked scx_bpf_task_set_slice() write and a
1217	* concurrent DSQ insertion commit can race each other and whichever lands last
1218	* wins.
1219	*
1220	* A DSQ insert kfunc doesn't update the fields directly. The verdict carries
1221	* the values and apply_slice_vtime() commits them at the insertion.
1222	*
1223	* scx_bpf_task_set_slice() may be called from any context and writes directly
1224	* only if @p's rq lock is already held, otherwise it bounces through
1225	* p->scx.slice_oob, applied under @p's rq lock at the next slice consideration.
1226	*
1227	* While %SCX_TASK_PROTECTED is set, every scheduler-reachable slice update is
1228	* refused. See set_task_slice_keep_oob().
1229	*
1230	* dsq_vtime orders the next PRIQ insertion and has no running-side consumer, so
1231	* scx_bpf_task_set_dsq_vtime() writes it directly. Fork-time init and direct
1232	* BPF stores from non-cid-form schedulers are outside these rules.
1233	*/
1234
1235	/* clear a pending slice request */
1236	static void clear_task_slice_oob(struct task_struct *p)
1237	{
1238	if (unlikely(atomic64_read(&p->scx.slice_oob)))
1239	atomic64_set(&p->scx.slice_oob, 0);
1240	}
1241
1242	/**
1243	* dsq_insert_head - FIFO head insertion honoring %SCX_TASK_PROTECTED
1244	* @dsq: DSQ to insert into
1245	* @p: task being inserted
1246	*
1247	* A HEAD insert should land behind any leading protected tasks. Return %true
1248	* indicates whether @p became the first entry.
1249	*/
1250	static bool dsq_insert_head(struct scx_dispatch_q dsq, struct task_struct p)
1251	{
1252	struct list_head *pos = &dsq->list;
1253	struct scx_dsq_list_node *node;
1254
1255	/*
1256	* Only rq-owned DSQs can hold protected tasks and the associated rq
1257	* lock keeps their flags stable.
1258	*/
1259	if (!dsq_is_rq_owned(dsq)) {
1260	list_add(&p->scx.dsq_list.node, &dsq->list);
1261	return true;
1262	}
1263
1264	list_for_each_entry(node, &dsq->list, node) {
1265	struct task_struct *q;
1266
1267	if (WARN_ON_ONCE(node->flags & SCX_DSQ_LNODE_ITER_CURSOR))
1268	continue;
1269
1270	q = container_of(node, struct task_struct, scx.dsq_list);
1271	if (!(q->scx.flags & SCX_TASK_PROTECTED))
1272	break;
1273
1274	pos = &node->node;
1275	}
1276
1277	list_add(&p->scx.dsq_list.node, pos);
1278
1279	return pos == &dsq->list;
1280	}
1281
1282	/**
1283	* set_task_slice_keep_oob - Set @p's slice, leaving any pending oob request
1284	* @p: task of interest
1285	* @slice: slice to set
1286	*
1287	* While %SCX_TASK_PROTECTED is set, BPF schedulers may not modify the slice.
1288	* Refuse and return %false.
1289	*/
1290	static bool set_task_slice_keep_oob(struct task_struct *p, u64 slice)
1291	{
1292	lockdep_assert_rq_held(task_rq(p));
1293
1294	if (unlikely(p->scx.flags & SCX_TASK_PROTECTED))
1295	return false;
1296
1297	p->scx.slice = slice;
1298	return true;
1299	}
1300
1301	/* set @p's slice, superseding any pending out-of-band request */
1302	bool scx_set_task_slice(struct task_struct *p, u64 slice)
1303	{
1304	if (!set_task_slice_keep_oob(p, slice))
1305	return false;
1306	clear_task_slice_oob(p);
1307	return true;
1308	}
1309
1310	/**
1311	* scx_task_slice_ended - @p's slice is consumed or given up
1312	* @rq: rq @p is on
1313	* @p: task of interest
1314	*
1315	* End what rides on the slice - the protection, and the rescue if @p is being
1316	* rescued.
1317	*
1318	* A dequeue normally ends the slice too. The exception is a save/restore pair
1319	* on the running task. Attribute changes like renice cycle the task through
1320	* dequeue and enqueue while it keeps executing, so the slice continues. A
1321	* queued task instead loses its DSQ position on any dequeue and the slice ends
1322	* with it.
1323	*/
1324	void scx_task_slice_ended(struct rq rq, struct task_struct p)
1325	{
1326	lockdep_assert_rq_held(rq);
1327
1328	p->scx.flags &= ~SCX_TASK_PROTECTED;
1329	if (unlikely(p == scx_rescuee(rq)))
1330	scx_rescue_end(rq);
1331	}
1332
1333	/* request @p's slice to be set to @slice, see the write rules above */
1334	static void set_task_slice_oob(struct scx_sched sch, struct task_struct p, u64 slice)
1335	{
1336	u64 dur;
1337
1338	if (slice == SCX_SLICE_INF) {
1339	dur = SCX_SLICE_OOB_DUR_MASK;
1340	} else if (unlikely(slice >= SCX_SLICE_OOB_DUR_MASK)) {
1341	dur = SCX_SLICE_OOB_DUR_MASK - 1;
1342	scx_add_event(sch, SCX_EV_SLICE_CLAMPED, 1);
1343	} else {
1344	dur = slice;
1345	}
1346
1347	atomic64_set(&p->scx.slice_oob, SCX_SLICE_OOB_PENDING \|
1348	((sch->id & SCX_SLICE_OOB_ID_MASK) << SCX_SLICE_OOB_ID_SHIFT) \| dur);
1349	}
1350
1351	/*
1352	* Apply a pending out-of-band slice request under @rq's lock. A request whose
1353	* packed id no longer matches @p's current owner is dropped. An extension needs
1354	* baseline cpu access on @p's cid, shortening is always allowed, and a
1355	* protected slice refuses both. %SCX_EV_SLICE_DENIED counts the denials. See
1356	* the write rules above.
1357	*/
1358	static void apply_task_slice_oob(struct rq rq, struct task_struct p)
1359	{
1360	u64 oob, dur, slice;
1361
1362	lockdep_assert_rq_held(rq);
1363
1364	if (likely(!atomic64_read(&p->scx.slice_oob)))
1365	return;
1366
1367	oob = atomic64_xchg(&p->scx.slice_oob, 0);
1368	if (unlikely(!oob))
1369	return;
1370
1371	/* the issuing scheduler no longer owns @p, drop the request */
1372	if (unlikely(((oob >> SCX_SLICE_OOB_ID_SHIFT) & SCX_SLICE_OOB_ID_MASK) !=
1373	(scx_task_sched(p)->id & SCX_SLICE_OOB_ID_MASK)))
1374	return;
1375
1376	dur = oob & SCX_SLICE_OOB_DUR_MASK;
1377	slice = dur == SCX_SLICE_OOB_DUR_MASK ? SCX_SLICE_INF : dur;
1378
1379	if (slice > p->scx.slice &&
1380	unlikely(scx_missing_caps(scx_task_sched(p), cpu_of(rq), SCX_CAP_BASE))) {
1381	__scx_add_event(scx_task_sched(p), SCX_EV_SLICE_DENIED, 1);
1382	return;
1383	}
1384
1385	if (unlikely(!set_task_slice_keep_oob(p, slice)))
1386	__scx_add_event(scx_task_sched(p), SCX_EV_SLICE_DENIED, 1);
1387	}
1388
1389	/*
1390	* A dsq insert kfunc doesn't write slice or dsq_vtime. The verdict carries them
1391	* and they are committed here, at the insertion. A zero @slice keeps the
1392	* current value, floored at 1 so the task isn't treated as expired.
1393	*/
1394	static void apply_slice_vtime(struct task_struct *p, u64 slice, u64 vtime, u64 enq_flags)
1395	{
1396	if (slice) {
1397	p->scx.slice = slice;
1398	/*
1399	* An explicit slice supersedes a pending oob request. A carried
1400	* default refill is not an explicit request and must keep it.
1401	*/
1402	if (!(enq_flags & SCX_ENQ_SLICE_DFL))
1403	clear_task_slice_oob(p);
1404	} else if (!p->scx.slice) {
1405	p->scx.slice = 1;
1406	}
1407
1408	if (enq_flags & SCX_ENQ_DSQ_PRIQ)
1409	p->scx.dsq_vtime = vtime;
1410	}
1411
1412	static void update_curr_scx(struct rq *rq)
1413	{
1414	struct task_struct *curr = rq->curr;
1415	s64 delta_exec;
1416
1417	/* apply even on 0 delta_exec, callers may still act on the slice */
1418	apply_task_slice_oob(rq, curr);
1419
1420	delta_exec = update_curr_common(rq);
1421	if (unlikely(delta_exec <= 0))
1422	return;
1423
1424	if (curr->scx.slice != SCX_SLICE_INF)
1425	curr->scx.slice -= min_t(u64, curr->scx.slice, delta_exec);
1426
1427	if (unlikely(curr == scx_rescuee(rq)))
1428	scx_rescue_charge(rq, delta_exec);
1429
1430	dl_server_update(&rq->ext_server, delta_exec);
1431	}
1432
1433	static bool scx_dsq_priq_less(struct rb_node *node_a,
1434	const struct rb_node *node_b)
1435	{
1436	const struct task_struct *a =
1437	container_of(node_a, struct task_struct, scx.dsq_priq);
1438	const struct task_struct *b =
1439	container_of(node_b, struct task_struct, scx.dsq_priq);
1440
1441	return time_before64(a->scx.dsq_vtime, b->scx.dsq_vtime);
1442	}
1443
1444	static void dsq_inc_nr(struct scx_dispatch_q dsq, struct task_struct p, u64 enq_flags)
1445	{
1446	/* scx_bpf_dsq_nr_queued() reads ->nr without locking, use WRITE_ONCE() */
1447	WRITE_ONCE(dsq->nr, dsq->nr + 1);
1448
1449	/*
1450	* Once @p reaches a local DSQ, it can only leave it by being dispatched
1451	* to the CPU or dequeued. In both cases, the only way @p can go back to
1452	* the BPF sched is through enqueueing. If being inserted into a local
1453	* DSQ with IMMED, persist the state until the next enqueueing event in
1454	* scx_do_enqueue_task() so that we can maintain IMMED protection
1455	* through e.g. SAVE/RESTORE cycles and slice extensions.
1456	*/
1457	if (enq_flags & SCX_ENQ_IMMED) {
1458	if (unlikely(dsq->id != SCX_DSQ_LOCAL)) {
1459	WARN_ON_ONCE(!(enq_flags & SCX_ENQ_GDSQ_FALLBACK));
1460	return;
1461	}
1462	p->scx.flags \|= SCX_TASK_IMMED;
1463	}
1464
1465	if (p->scx.flags & SCX_TASK_IMMED) {
1466	struct rq *rq = container_of(dsq, struct rq, scx.local_dsq);
1467
1468	if (WARN_ON_ONCE(dsq->id != SCX_DSQ_LOCAL))
1469	return;
1470
1471	rq->scx.nr_immed++;
1472
1473	/*
1474	* If @rq already had other tasks or the current task is not
1475	* done yet, @p can't go on the CPU immediately. Re-enqueue.
1476	*/
1477	if (unlikely(dsq->nr > 1 \|\| !rq_is_open(rq, enq_flags)))
1478	scx_schedule_reenq_local(rq, 0);
1479	}
1480	}
1481
1482	static void dsq_dec_nr(struct scx_dispatch_q dsq, struct task_struct p)
1483	{
1484	/* see dsq_inc_nr() */
1485	WRITE_ONCE(dsq->nr, dsq->nr - 1);
1486
1487	if (p->scx.flags & SCX_TASK_IMMED) {
1488	struct rq *rq = container_of(dsq, struct rq, scx.local_dsq);
1489
1490	if (WARN_ON_ONCE(dsq->id != SCX_DSQ_LOCAL) \|\|
1491	WARN_ON_ONCE(rq->scx.nr_immed <= 0))
1492	return;
1493
1494	rq->scx.nr_immed--;
1495	}
1496	}
1497
1498	static void refill_task_slice_dfl(struct scx_sched sch, struct task_struct p)
1499	{
1500	/*
1501	* A default refill is not an explicit request, so it must not drop a
1502	* pending out-of-band one, which is applied when @p next runs.
1503	*/
1504	set_task_slice_keep_oob(p, READ_ONCE(sch->slice_dfl));
1505	__scx_add_event(sch, SCX_EV_REFILL_SLICE_DFL, 1);
1506	}
1507
1508	/*
1509	* Return true if @p is moving due to an internal SCX migration, false
1510	* otherwise.
1511	*/
1512	static inline bool task_scx_migrating(struct task_struct *p)
1513	{
1514	/*
1515	* We only need to check sticky_cpu: it is set to the destination
1516	* CPU in move_remote_task_to_local_dsq() before deactivate_task()
1517	* and cleared when the task is enqueued on the destination, so it
1518	* is only non-negative during an internal SCX migration.
1519	*/
1520	return p->scx.sticky_cpu >= 0;
1521	}
1522
1523	/* Must be called under the lock serializing @p's custody transfers. */
1524	static bool task_leave_custody(struct task_struct *p)
1525	{
1526	if (!(p->scx.flags & SCX_TASK_IN_CUSTODY) \|\| task_scx_migrating(p))
1527	return false;
1528
1529	p->scx.flags &= ~SCX_TASK_IN_CUSTODY;
1530	return true;
1531	}
1532
1533	static void rq_owned_post_enq(struct scx_sched sch, struct rq rq,
1534	struct scx_dispatch_q dsq, struct task_struct p,
1535	u64 enq_flags)
1536	{
1537	if (task_leave_custody(p) && SCX_HAS_OP(sch, dequeue))
1538	SCX_CALL_OP_TASK(sch, dequeue, rq, p, 0);
1539
1540	/*
1541	* Only local inserts get the wakeup treatment below. Rejects kick the
1542	* deferred reenq and rescue parks are paced by the rescue timer.
1543	*/
1544	if (unlikely(dsq->id != SCX_DSQ_LOCAL)) {
1545	if (dsq->id == SCX_DSQ_REJECT)
1546	schedule_deferred_locked(rq);
1547	return;
1548	}
1549
1550	/*
1551	* Note that @rq's lock may be dropped between this enqueue and @p
1552	* actually getting on CPU. This gives higher-class tasks (e.g. RT)
1553	* an opportunity to wake up on @rq and prevent @p from running.
1554	* Here are some concrete examples:
1555	*
1556	* Example 1:
1557	*
1558	* We dispatch two tasks from a single ops.dispatch():
1559	* - First, a local task to this CPU's local DSQ;
1560	* - Second, a local/remote task to a remote CPU's local DSQ.
1561	* We must drop the local rq lock in order to finish the second
1562	* dispatch. In that time, an RT task can wake up on the local rq.
1563	*
1564	* Example 2:
1565	*
1566	* We dispatch a local/remote task to a remote CPU's local DSQ.
1567	* We must drop the remote rq lock before the dispatched task can run,
1568	* which gives an RT task an opportunity to wake up on the remote rq.
1569	*
1570	* Both examples work the same if we replace dispatching with moving
1571	* the tasks from a user-created DSQ.
1572	*
1573	* We must detect these wakeups so that we can re-enqueue IMMED tasks
1574	* from @rq's local DSQ. scx_wakeup_preempt() serves exactly this
1575	* purpose, but for it to be invoked, we must ensure that we bump
1576	* @rq->next_class to &ext_sched_class if it's currently idle.
1577	*
1578	* wakeup_preempt() does the bumping, and since we only invoke it if
1579	* @rq->next_class is below &ext_sched_class, it will also
1580	* resched_curr(rq).
1581	*/
1582	if (sched_class_above(p->sched_class, rq->next_class))
1583	wakeup_preempt(rq, p, 0);
1584
1585	/*
1586	* If @rq is in dispatch, the CPU is already vacant and looking for the
1587	* next task to run. No need to preempt or trigger resched after moving
1588	* @p into its local DSQ.
1589	* Note that the wakeup_preempt() above may have already triggered
1590	* a resched if @rq->next_class was idle. It's harmless, since
1591	* need_resched is cleared immediately after task pick.
1592	*/
1593	if (rq->scx.flags & SCX_RQ_IN_DISPATCH)
1594	return;
1595
1596	if ((enq_flags & SCX_ENQ_PREEMPT) && p != rq->curr &&
1597	rq->curr->sched_class == &ext_sched_class) {
1598	if (likely(scx_set_task_slice(rq->curr, 0)))
1599	resched_curr(rq);
1600	else
1601	__scx_add_event(sch, SCX_EV_SLICE_DENIED, 1);
1602	}
1603	}
1604
1605	static void scx_dispatch_enqueue(struct scx_sched sch, struct rq rq,
1606	struct scx_dispatch_q dsq, struct task_struct p,
1607	u64 slice, u64 vtime, u64 enq_flags)
1608	{
1609	bool is_rq_owned = false;
1610
1611	if (dsq->id == SCX_DSQ_LOCAL) {
1612	dsq = scx_resolve_local_dsq(sch, rq, p, &enq_flags);
1613	is_rq_owned = true;
1614	}
1615
1616	WARN_ON_ONCE(p->scx.dsq \|\| !list_empty(&p->scx.dsq_list.node));
1617	WARN_ON_ONCE((p->scx.dsq_flags & SCX_TASK_DSQ_ON_PRIQ) \|\|
1618	!RB_EMPTY_NODE(&p->scx.dsq_priq));
1619
1620	if (!is_rq_owned) {
1621	raw_spin_lock_nested(&dsq->lock,
1622	(enq_flags & SCX_ENQ_NESTED) ? SINGLE_DEPTH_NESTING : 0);
1623
1624	if (unlikely(dsq->id == SCX_DSQ_INVALID)) {
1625	scx_error(sch, "attempting to dispatch to a destroyed dsq");
1626	/* fall back to the global dsq */
1627	raw_spin_unlock(&dsq->lock);
1628	dsq = find_global_dsq(sch, task_cpu(p));
1629	raw_spin_lock(&dsq->lock);
1630	}
1631	}
1632
1633	if (unlikely((dsq->id & SCX_DSQ_FLAG_BUILTIN) &&
1634	(enq_flags & SCX_ENQ_DSQ_PRIQ))) {
1635	/*
1636	* SCX_DSQ_LOCAL and SCX_DSQ_GLOBAL DSQs always consume from
1637	* their FIFO queues. To avoid confusion and accidentally
1638	* starving vtime-dispatched tasks by FIFO-dispatched tasks, we
1639	* disallow any internal DSQ from doing vtime ordering of
1640	* tasks.
1641	*/
1642	scx_error(sch, "cannot use vtime ordering for built-in DSQs");
1643	enq_flags &= ~SCX_ENQ_DSQ_PRIQ;
1644	}
1645
1646	/*
1647	* @dsq is locked and @enq_flags is sanitized. Commit the carried slice
1648	* and vtime before the PRIQ insertion below reads the new dsq_vtime.
1649	*/
1650	if (enq_flags & SCX_ENQ_APPLY_SLICE)
1651	apply_slice_vtime(p, slice, vtime, enq_flags);
1652
1653	if (enq_flags & SCX_ENQ_DSQ_PRIQ) {
1654	struct rb_node *rbp;
1655
1656	/*
1657	* A PRIQ DSQ shouldn't be using FIFO enqueueing. As tasks are
1658	* linked to both the rbtree and list on PRIQs, this can only be
1659	* tested easily when adding the first task.
1660	*/
1661	if (unlikely(RB_EMPTY_ROOT(&dsq->priq) &&
1662	nldsq_next_task(dsq, NULL, false)))
1663	scx_error(sch, "DSQ ID 0x%016llx already had FIFO-enqueued tasks",
1664	dsq->id);
1665
1666	p->scx.dsq_flags \|= SCX_TASK_DSQ_ON_PRIQ;
1667	rb_add(&p->scx.dsq_priq, &dsq->priq, scx_dsq_priq_less);
1668
1669	/*
1670	* Find the previous task and insert after it on the list so
1671	* that @dsq->list is vtime ordered.
1672	*/
1673	rbp = rb_prev(&p->scx.dsq_priq);
1674	if (rbp) {
1675	struct task_struct *prev =
1676	container_of(rbp, struct task_struct,
1677	scx.dsq_priq);
1678	list_add(&p->scx.dsq_list.node, &prev->scx.dsq_list.node);
1679	/* first task unchanged - no update needed */
1680	} else {
1681	list_add(&p->scx.dsq_list.node, &dsq->list);
1682	/* not builtin and new task is at head - use fastpath */
1683	rcu_assign_pointer(dsq->first_task, p);
1684	}
1685	} else {
1686	/* a FIFO DSQ shouldn't be using PRIQ enqueuing */
1687	if (unlikely(!RB_EMPTY_ROOT(&dsq->priq)))
1688	scx_error(sch, "DSQ ID 0x%016llx already had PRIQ-enqueued tasks",
1689	dsq->id);
1690
1691	if (enq_flags & (SCX_ENQ_HEAD \| SCX_ENQ_PREEMPT)) {
1692	/* new task inserted at head - use fastpath */
1693	if (dsq_insert_head(dsq, p) && !(dsq->id & SCX_DSQ_FLAG_BUILTIN))
1694	rcu_assign_pointer(dsq->first_task, p);
1695	} else {
1696	/*
1697	* dsq->list can contain parked BPF iterator cursors, so
1698	* list_empty() here isn't a reliable proxy for "no real
1699	* task in the DSQ". Test dsq->first_task directly.
1700	*/
1701	list_add_tail(&p->scx.dsq_list.node, &dsq->list);
1702	if (!dsq->first_task && !(dsq->id & SCX_DSQ_FLAG_BUILTIN))
1703	rcu_assign_pointer(dsq->first_task, p);
1704	}
1705	}
1706
1707	/* seq records the order tasks are queued, used by BPF DSQ iterator */
1708	WRITE_ONCE(dsq->seq, dsq->seq + 1);
1709	p->scx.dsq_seq = dsq->seq;
1710
1711	dsq_inc_nr(dsq, p, enq_flags);
1712	p->scx.dsq = dsq;
1713
1714	/*
1715	* Update custody and call ops.dequeue() before clearing ops_state:
1716	* once ops_state is cleared, waiters in ops_dequeue() can proceed
1717	* and dequeue_task_scx() will RMW p->scx.flags. If we clear
1718	* ops_state first, both sides would modify p->scx.flags
1719	* concurrently in a non-atomic way.
1720	*/
1721	if (is_rq_owned) {
1722	rq_owned_post_enq(sch, rq, dsq, p, enq_flags);
1723	} else {
1724	bool call_dequeue = false;
1725
1726	/*
1727	* Global and bypass DSQs are terminal - the task leaves the
1728	* scheduler's custody, so ops.dequeue() fires. It can run
1729	* without @p's rq lock (finish_dispatch() passes the dispatch
1730	* rq); that's safe because dequeue_task_scx() waits on
1731	* SCX_OPSS_DISPATCHING (see the ops_state note above) and so
1732	* can't race it. A non-terminal DSQ keeps the task in custody.
1733	* The custody transfer happens under @dsq->lock so that later
1734	* consumers see the flag clear; the callback runs after
1735	* @dsq->lock is dropped because it may lock a DSQ itself.
1736	*/
1737	if (dsq->id == SCX_DSQ_GLOBAL \|\| dsq->id == SCX_DSQ_BYPASS)
1738	call_dequeue = task_leave_custody(p);
1739	else
1740	p->scx.flags \|= SCX_TASK_IN_CUSTODY;
1741
1742	raw_spin_unlock(&dsq->lock);
1743
1744	if (call_dequeue && SCX_HAS_OP(sch, dequeue))
1745	SCX_CALL_OP_TASK(sch, dequeue, rq, p, 0);
1746	}
1747
1748	/*
1749	* We're transitioning out of QUEUEING or DISPATCHING. store_release to
1750	* match waiters' load_acquire.
1751	*/
1752	if (enq_flags & SCX_ENQ_CLEAR_OPSS)
1753	atomic_long_set_release(&p->scx.ops_state, SCX_OPSS_NONE);
1754	}
1755
1756	void scx_task_unlink_from_dsq(struct task_struct p, struct scx_dispatch_q dsq)
1757	{
1758	WARN_ON_ONCE(list_empty(&p->scx.dsq_list.node));
1759
1760	if (p->scx.dsq_flags & SCX_TASK_DSQ_ON_PRIQ) {
1761	rb_erase(&p->scx.dsq_priq, &dsq->priq);
1762	RB_CLEAR_NODE(&p->scx.dsq_priq);
1763	p->scx.dsq_flags &= ~SCX_TASK_DSQ_ON_PRIQ;
1764	}
1765
1766	list_del_init(&p->scx.dsq_list.node);
1767	dsq_dec_nr(dsq, p);
1768
1769	if (!(dsq->id & SCX_DSQ_FLAG_BUILTIN) && rcu_access_pointer(dsq->first_task) == p) {
1770	struct task_struct *first_task;
1771
1772	first_task = nldsq_next_task(dsq, NULL, false);
1773	rcu_assign_pointer(dsq->first_task, first_task);
1774	}
1775	}
1776
1777	void scx_dispatch_dequeue(struct rq rq, struct task_struct p)
1778	{
1779	struct scx_dispatch_q *dsq = p->scx.dsq;
1780	bool is_rq_owned = dsq && dsq_is_rq_owned(dsq);
1781
1782	lockdep_assert_rq_held(rq);
1783
1784	if (!dsq) {
1785	/*
1786	* If !dsq && on-list, @p is on @rq's ddsp_deferred_locals.
1787	* Unlinking is all that's needed to cancel.
1788	*/
1789	if (unlikely(!list_empty(&p->scx.dsq_list.node)))
1790	list_del_init(&p->scx.dsq_list.node);
1791
1792	/*
1793	* When dispatching directly from the BPF scheduler to a local
1794	* DSQ, the task isn't associated with any DSQ but
1795	* @p->scx.holding_cpu may be set under the protection of
1796	* %SCX_OPSS_DISPATCHING.
1797	*/
1798	if (p->scx.holding_cpu >= 0)
1799	p->scx.holding_cpu = -1;
1800
1801	return;
1802	}
1803
1804	if (!is_rq_owned)
1805	raw_spin_lock(&dsq->lock);
1806
1807	/*
1808	* Now that we hold @dsq->lock, @p->holding_cpu and @p->scx.dsq_* can't
1809	* change underneath us.
1810	*/
1811	if (p->scx.holding_cpu < 0) {
1812	/* @p must still be on @dsq, dequeue */
1813	scx_task_unlink_from_dsq(p, dsq);
1814	} else {
1815	/*
1816	* We're racing against dispatch_to_local_dsq() which already
1817	* removed @p from @dsq and set @p->scx.holding_cpu. Clear the
1818	* holding_cpu which tells dispatch_to_local_dsq() that it lost
1819	* the race.
1820	*/
1821	WARN_ON_ONCE(!list_empty(&p->scx.dsq_list.node));
1822	p->scx.holding_cpu = -1;
1823	}
1824	p->scx.dsq = NULL;
1825
1826	if (!is_rq_owned)
1827	raw_spin_unlock(&dsq->lock);
1828	}
1829
1830	/*
1831	* Abbreviated version of scx_dispatch_dequeue() that can be used when both
1832	* @p's rq and dsq are locked.
1833	*/
1834	static void dispatch_dequeue_locked(struct task_struct *p,
1835	struct scx_dispatch_q *dsq)
1836	{
1837	lockdep_assert_rq_held(task_rq(p));
1838	lockdep_assert_held(&dsq->lock);
1839
1840	scx_task_unlink_from_dsq(p, dsq);
1841	p->scx.dsq = NULL;
1842	}
1843
1844	static struct scx_dispatch_q find_dsq_for_dispatch(struct scx_sched sch,
1845	struct rq *rq, u64 dsq_id,
1846	s32 tcpu)
1847	{
1848	struct scx_dispatch_q *dsq;
1849
1850	if (dsq_id == SCX_DSQ_LOCAL)
1851	return &rq->scx.local_dsq;
1852
1853	if ((dsq_id & SCX_DSQ_LOCAL_ON) == SCX_DSQ_LOCAL_ON) {
1854	s32 cpu = scx_cpu_ret(sch, dsq_id & SCX_DSQ_LOCAL_CPU_MASK);
1855
1856	if (!scx_cpu_valid(sch, cpu, "in SCX_DSQ_LOCAL_ON dispatch verdict"))
1857	return find_global_dsq(sch, tcpu);
1858
1859	return &cpu_rq(cpu)->scx.local_dsq;
1860	}
1861
1862	if (dsq_id == SCX_DSQ_GLOBAL)
1863	dsq = find_global_dsq(sch, tcpu);
1864	else
1865	dsq = find_user_dsq(sch, dsq_id);
1866
1867	/*
1868	* Built-in DSQs are never inserted into dsq_hash, so REJECT and RESCUE
1869	* hit the error below. They cannot be reached with an ID.
1870	*/
1871	if (unlikely(!dsq)) {
1872	scx_error(sch, "non-existent DSQ 0x%llx", dsq_id);
1873	return find_global_dsq(sch, tcpu);
1874	}
1875
1876	return dsq;
1877	}
1878
1879	static void mark_direct_dispatch(struct scx_sched *sch,
1880	struct task_struct *ddsp_task,
1881	struct task_struct *p, u64 dsq_id,
1882	u64 slice, u64 vtime, u64 enq_flags)
1883	{
1884	/*
1885	* Mark that dispatch already happened from ops.select_cpu() or
1886	* ops.enqueue() by spoiling direct_dispatch_task with a non-NULL value
1887	* which can never match a valid task pointer.
1888	*/
1889	__this_cpu_write(direct_dispatch_task, ERR_PTR(-ESRCH));
1890
1891	/* @p must match the task on the enqueue path */
1892	if (unlikely(p != ddsp_task)) {
1893	if (IS_ERR(ddsp_task))
1894	scx_error(sch, "%s[%d] already direct-dispatched",
1895	p->comm, p->pid);
1896	else
1897	scx_error(sch, "scheduling for %s[%d] but trying to direct-dispatch %s[%d]",
1898	ddsp_task->comm, ddsp_task->pid,
1899	p->comm, p->pid);
1900	return;
1901	}
1902
1903	WARN_ON_ONCE(p->scx.ddsp_dsq_id != SCX_DSQ_INVALID);
1904	WARN_ON_ONCE(p->scx.ddsp_enq_flags);
1905
1906	p->scx.ddsp_slice = slice;
1907	p->scx.ddsp_vtime = vtime;
1908	p->scx.ddsp_dsq_id = dsq_id;
1909	p->scx.ddsp_enq_flags = enq_flags;
1910	}
1911
1912	/*
1913	* Clear @p direct dispatch state when leaving the scheduler.
1914	*
1915	* Direct dispatch state must be cleared in the following cases:
1916	* - direct_dispatch(): cleared on the synchronous enqueue path, deferred
1917	* dispatch keeps the state until consumed
1918	* - process_ddsp_deferred_locals(): cleared after consuming deferred state,
1919	* - scx_do_enqueue_task(): cleared on enqueue fallbacks where the dispatch
1920	* verdict is ignored (local/global/bypass)
1921	* - dequeue_task_scx(): cleared after scx_dispatch_dequeue(), covering
1922	* deferred cancellation and holding_cpu races
1923	* - scx_disable_task(): cleared for queued wakeup tasks, which are excluded by
1924	* the scx_bypass() loop, so that stale state is not reused by a subsequent
1925	* scheduler instance
1926	*/
1927	static inline void clear_direct_dispatch(struct task_struct *p)
1928	{
1929	p->scx.ddsp_dsq_id = SCX_DSQ_INVALID;
1930	p->scx.ddsp_enq_flags = 0;
1931	}
1932
1933	static void direct_dispatch(struct scx_sched sch, struct task_struct p,
1934	u64 enq_flags)
1935	{
1936	struct rq *rq = task_rq(p);
1937	struct scx_dispatch_q *dsq =
1938	find_dsq_for_dispatch(sch, rq, p->scx.ddsp_dsq_id, task_cpu(p));
1939	u64 ddsp_enq_flags, slice, vtime;
1940
1941	p->scx.ddsp_enq_flags \|= enq_flags;
1942
1943	/*
1944	* We are in the enqueue path with @rq locked and pinned, and thus can't
1945	* double lock a remote rq and enqueue to its local DSQ. For
1946	* DSQ_LOCAL_ON verdicts targeting the local DSQ of a remote CPU, defer
1947	* the enqueue so that it's executed when @rq can be unlocked.
1948	*/
1949	if (dsq->id == SCX_DSQ_LOCAL && dsq != &rq->scx.local_dsq) {
1950	unsigned long opss;
1951
1952	opss = atomic_long_read(&p->scx.ops_state) & SCX_OPSS_STATE_MASK;
1953
1954	switch (opss & SCX_OPSS_STATE_MASK) {
1955	case SCX_OPSS_NONE:
1956	break;
1957	case SCX_OPSS_QUEUEING:
1958	/*
1959	* As @p was never passed to the BPF side, _release is
1960	* not strictly necessary. Still do it for consistency.
1961	*/
1962	atomic_long_set_release(&p->scx.ops_state, SCX_OPSS_NONE);
1963	break;
1964	default:
1965	WARN_ONCE(true, "sched_ext: %s[%d] has invalid ops state 0x%lx in direct_dispatch()",
1966	p->comm, p->pid, opss);
1967	atomic_long_set_release(&p->scx.ops_state, SCX_OPSS_NONE);
1968	break;
1969	}
1970
1971	WARN_ON_ONCE(p->scx.dsq \|\| !list_empty(&p->scx.dsq_list.node));
1972	list_add_tail(&p->scx.dsq_list.node,
1973	&rq->scx.ddsp_deferred_locals);
1974	schedule_deferred_locked(rq);
1975	return;
1976	}
1977
1978	ddsp_enq_flags = p->scx.ddsp_enq_flags;
1979	slice = p->scx.ddsp_slice;
1980	vtime = p->scx.ddsp_vtime;
1981	clear_direct_dispatch(p);
1982
1983	scx_dispatch_enqueue(sch, rq, dsq, p, slice, vtime,
1984	ddsp_enq_flags \| SCX_ENQ_APPLY_SLICE \| SCX_ENQ_CLEAR_OPSS);
1985	}
1986
1987	bool scx_rq_online(struct rq *rq)
1988	{
1989	/*
1990	* Test both cpu_active() and %SCX_RQ_ONLINE. %SCX_RQ_ONLINE indicates
1991	* the online state as seen from the BPF scheduler. cpu_active() test
1992	* guarantees that, if this function returns %true, %SCX_RQ_ONLINE will
1993	* stay set until the current scheduling operation is complete even if
1994	* we aren't locking @rq.
1995	*/
1996	return likely((rq->scx.flags & SCX_RQ_ONLINE) && cpu_active(cpu_of(rq)));
1997	}
1998
1999	void scx_do_enqueue_task(struct rq rq, struct task_struct p, u64 enq_flags,
2000	int sticky_cpu)
2001	{
2002	struct scx_sched *sch = scx_task_sched(p);
2003	struct task_struct **ddsp_taskp;
2004	struct scx_dispatch_q *dsq;
2005	unsigned long qseq;
2006
2007	WARN_ON_ONCE(!(p->scx.flags & SCX_TASK_QUEUED));
2008
2009	/* internal movements - rq migration / RESTORE */
2010	if (sticky_cpu == cpu_of(rq))
2011	goto local_norefill;
2012
2013	/*
2014	* Clear persistent TASK_IMMED for fresh enqueues, see dsq_inc_nr().
2015	* Note that exiting and migration-disabled tasks that skip
2016	* ops.enqueue() below will lose IMMED protection unless
2017	* %SCX_OPS_ENQ_EXITING / %SCX_OPS_ENQ_MIGRATION_DISABLED are set.
2018	*/
2019	p->scx.flags &= ~SCX_TASK_IMMED;
2020
2021	/*
2022	* A task reenqueued too many times without running means the scheduler
2023	* keeps re-deciding a placement it can't honor, e.g. re-inserting to a
2024	* cid it lacks caps on. Eject the owning scheduler and strand the task
2025	* to be picked up during sched exit.
2026	*/
2027	if (enq_flags & SCX_ENQ_REENQ) {
2028	if (++p->scx.reenq_cnt > 1)
2029	__scx_add_event(sch, SCX_EV_REENQ_REPEAT, 1);
2030
2031	if (unlikely(p->scx.reenq_cnt > SCX_REENQ_MAX_REPEAT)) {
2032	__scx_exit(sch, SCX_EXIT_ERROR_REENQ, 0, cpu_of(rq),
2033	"%s[%d] reenqueued %u times without running",
2034	p->comm, p->pid, p->scx.reenq_cnt);
2035	return;
2036	}
2037	}
2038
2039	/*
2040	* If !scx_rq_online(), we already told the BPF scheduler that the CPU
2041	* is offline and are just running the hotplug path. Don't bother the
2042	* BPF scheduler.
2043	*/
2044	if (!scx_rq_online(rq))
2045	goto local;
2046
2047	if (scx_bypassing(sch, cpu_of(rq))) {
2048	__scx_add_event(sch, SCX_EV_BYPASS_DISPATCH, 1);
2049	goto bypass;
2050	}
2051
2052	if (p->scx.ddsp_dsq_id != SCX_DSQ_INVALID)
2053	goto direct;
2054
2055	/* see %SCX_OPS_ENQ_EXITING */
2056	if (!(sch->ops.flags & SCX_OPS_ENQ_EXITING) &&
2057	unlikely(p->flags & PF_EXITING)) {
2058	__scx_add_event(sch, SCX_EV_ENQ_SKIP_EXITING, 1);
2059	enq_flags \|= SCX_ENQ_RESCUE; /* avoid looping on cap rejection */
2060	goto local;
2061	}
2062
2063	/* see %SCX_OPS_ENQ_MIGRATION_DISABLED */
2064	if (!(sch->ops.flags & SCX_OPS_ENQ_MIGRATION_DISABLED) &&
2065	is_migration_disabled(p)) {
2066	__scx_add_event(sch, SCX_EV_ENQ_SKIP_MIGRATION_DISABLED, 1);
2067	goto local;
2068	}
2069
2070	if (unlikely(!SCX_HAS_OP(sch, enqueue)))
2071	goto global;
2072
2073	/*
2074	* DSQ bypass didn't trigger, enqueue on the BPF scheduler. Wrap the
2075	* per-task qseq counter where the QSEQ field wraps and skip 0, which is
2076	* what scx_bpf_dsq_insert() records for a task in NONE or DISPATCHING.
2077	*/
2078	p->scx.ops_qseq = ((p->scx.ops_qseq + 1) &
2079	(SCX_OPSS_QSEQ_MASK >> SCX_OPSS_QSEQ_SHIFT)) ?: 1;
2080	qseq = (unsigned long)p->scx.ops_qseq << SCX_OPSS_QSEQ_SHIFT;
2081
2082	WARN_ON_ONCE(atomic_long_read(&p->scx.ops_state) != SCX_OPSS_NONE);
2083	atomic_long_set(&p->scx.ops_state, SCX_OPSS_QUEUEING \| qseq);
2084
2085	ddsp_taskp = this_cpu_ptr(&direct_dispatch_task);
2086	WARN_ON_ONCE(*ddsp_taskp);
2087	*ddsp_taskp = p;
2088
2089	SCX_CALL_OP_TASK(sch, enqueue, rq, p, enq_flags);
2090
2091	*ddsp_taskp = NULL;
2092	if (p->scx.ddsp_dsq_id != SCX_DSQ_INVALID)
2093	goto direct;
2094
2095	/*
2096	* Task is now in BPF scheduler's custody. Set %SCX_TASK_IN_CUSTODY
2097	* so ops.dequeue() is called when it leaves custody.
2098	*/
2099	p->scx.flags \|= SCX_TASK_IN_CUSTODY;
2100
2101	/*
2102	* If not directly dispatched, QUEUEING isn't clear yet and dispatch or
2103	* dequeue may be waiting. The store_release matches their load_acquire.
2104	*/
2105	atomic_long_set_release(&p->scx.ops_state, SCX_OPSS_QUEUED \| qseq);
2106	return;
2107
2108	direct:
2109	direct_dispatch(sch, p, enq_flags);
2110	return;
2111	local_norefill:
2112	scx_dispatch_enqueue(sch, rq, &rq->scx.local_dsq, p, 0, 0, enq_flags);
2113	return;
2114	local:
2115	dsq = &rq->scx.local_dsq;
2116	goto enqueue;
2117	global:
2118	dsq = find_global_dsq(sch, task_cpu(p));
2119	goto enqueue;
2120	bypass:
2121	dsq = bypass_enq_target_dsq(sch, task_cpu(p));
2122	goto enqueue;
2123
2124	enqueue:
2125	refill_task_slice_dfl(sch, p);
2126	clear_direct_dispatch(p);
2127	scx_dispatch_enqueue(sch, rq, dsq, p, 0, 0, enq_flags);
2128	}
2129
2130	static bool task_runnable(const struct task_struct *p)
2131	{
2132	return !list_empty(&p->scx.runnable_node);
2133	}
2134
2135	static void set_task_runnable(struct rq rq, struct task_struct p)
2136	{
2137	lockdep_assert_rq_held(rq);
2138
2139	if (p->scx.flags & SCX_TASK_RESET_RUNNABLE_AT) {
2140	p->scx.runnable_at = jiffies;
2141	p->scx.flags &= ~SCX_TASK_RESET_RUNNABLE_AT;
2142	}
2143
2144	/*
2145	* list_add_tail() must be used. scx_bypass() and rq_offline_scx()
2146	* depend on tasks being appended to the runnable_list.
2147	*/
2148	list_add_tail(&p->scx.runnable_node, &rq->scx.runnable_list);
2149
2150	/*
2151	* Record the rq @p is runnable on, maintained under the rq lock so it
2152	* stays valid unlike task_cpu(), which a remote wakeup can move under
2153	* pi_lock alone.
2154	*/
2155	WRITE_ONCE(p->scx.runnable_cpu, cpu_of(rq));
2156	}
2157
2158	static void clr_task_runnable(struct task_struct *p, bool reset_runnable_at)
2159	{
2160	list_del_init(&p->scx.runnable_node);
2161	WRITE_ONCE(p->scx.runnable_cpu, -1);
2162	if (reset_runnable_at) {
2163	p->scx.flags \|= SCX_TASK_RESET_RUNNABLE_AT;
2164	p->scx.reenq_cnt = 0;
2165	}
2166	}
2167
2168	static void enqueue_task_scx(struct rq rq, struct task_struct p, int core_enq_flags)
2169	{
2170	struct scx_sched *sch = scx_task_sched(p);
2171	int sticky_cpu = p->scx.sticky_cpu;
2172	u64 enq_flags = core_enq_flags \| rq->scx.remote_activate_enq_flags;
2173
2174	/*
2175	* An SCX-internal migration ends on arrival. Clear sticky_cpu so @p can
2176	* leave custody when inserted into the destination DSQ.
2177	*/
2178	if (sticky_cpu >= 0)
2179	p->scx.sticky_cpu = -1;
2180
2181	/*
2182	* SCX_RQ_IN_WAKEUP promises a task_woken_scx() call once this enqueue
2183	* returns. Only the core's wakeup path delivers one. The flags stashed
2184	* for a remote activation may carry the wakeup bit without it.
2185	*/
2186	if (core_enq_flags & ENQUEUE_WAKEUP)
2187	rq->scx.flags \|= SCX_RQ_IN_WAKEUP;
2188
2189	/*
2190	* Restoring a running task will be immediately followed by
2191	* set_next_task_scx() which expects the task to not be on the BPF
2192	* scheduler as tasks can only start running through local DSQs. Force
2193	* direct-dispatch into the local DSQ by setting the sticky_cpu. Mark
2194	* IGNORE_CAPS to force entry into the local DSQ.
2195	*/
2196	if (unlikely(enq_flags & ENQUEUE_RESTORE) && task_current(rq, p)) {
2197	sticky_cpu = cpu_of(rq);
2198	enq_flags \|= SCX_ENQ_IGNORE_CAPS;
2199	}
2200
2201	if (p->scx.flags & SCX_TASK_QUEUED) {
2202	WARN_ON_ONCE(!task_runnable(p));
2203	goto out;
2204	}
2205
2206	set_task_runnable(rq, p);
2207	p->scx.flags \|= SCX_TASK_QUEUED;
2208	rq->scx.nr_running++;
2209	add_nr_running(rq, 1);
2210
2211	if (SCX_HAS_OP(sch, runnable) && !task_on_rq_migrating(p))
2212	SCX_CALL_OP_TASK(sch, runnable, rq, p, enq_flags);
2213
2214	/* Start dl_server if this is the first task being enqueued */
2215	if (rq->scx.nr_running == 1)
2216	dl_server_start(&rq->ext_server);
2217
2218	scx_do_enqueue_task(rq, p, enq_flags, sticky_cpu);
2219	out:
2220	rq->scx.flags &= ~SCX_RQ_IN_WAKEUP;
2221
2222	if ((enq_flags & SCX_ENQ_CPU_SELECTED) &&
2223	unlikely(cpu_of(rq) != p->scx.selected_cpu))
2224	__scx_add_event(sch, SCX_EV_SELECT_CPU_FALLBACK, 1);
2225	}
2226
2227	static void ops_dequeue(struct rq rq, struct task_struct p, u64 deq_flags)
2228	{
2229	struct scx_sched *sch = scx_task_sched(p);
2230	unsigned long opss;
2231
2232	/* dequeue is always temporary, don't reset runnable_at */
2233	clr_task_runnable(p, false);
2234
2235	retry:
2236	/* acquire ensures that we see the preceding updates on QUEUED */
2237	opss = atomic_long_read_acquire(&p->scx.ops_state);
2238
2239	switch (opss & SCX_OPSS_STATE_MASK) {
2240	case SCX_OPSS_NONE:
2241	break;
2242	case SCX_OPSS_QUEUEING:
2243	/*
2244	* QUEUEING is started and finished while holding @p's rq lock.
2245	* As we're holding the rq lock now, we shouldn't see QUEUEING.
2246	*/
2247	BUG();
2248	case SCX_OPSS_QUEUED:
2249	/*
2250	* A queued task must always be in BPF scheduler's custody. If
2251	* SCX_TASK_IN_CUSTODY is clear, finish_dispatch() on another
2252	* CPU has already passed task_leave_custody() (which clears the
2253	* flag), but has not yet written SCX_OPSS_NONE. That final
2254	* store does not require this rq's lock, so retrying with
2255	* cpu_relax() is bounded: we will observe NONE (or DISPATCHING,
2256	* handled by the fallthrough) on a subsequent iteration.
2257	*/
2258	if (unlikely(!(READ_ONCE(p->scx.flags) & SCX_TASK_IN_CUSTODY))) {
2259	cpu_relax();
2260	goto retry;
2261	}
2262
2263	if (atomic_long_try_cmpxchg(&p->scx.ops_state, &opss,
2264	SCX_OPSS_NONE))
2265	break;
2266	fallthrough;
2267	case SCX_OPSS_DISPATCHING:
2268	/*
2269	* If @p is being dispatched from the BPF scheduler to a DSQ,
2270	* wait for the transfer to complete so that @p doesn't get
2271	* added to its DSQ after dequeueing is complete.
2272	*
2273	* As we're waiting on DISPATCHING with the rq locked, the
2274	* dispatching side shouldn't try to lock the rq while
2275	* DISPATCHING is set. See dispatch_to_local_dsq().
2276	*
2277	* DISPATCHING shouldn't have qseq set and control can reach
2278	* here with NONE @opss from the above QUEUED case block.
2279	* Explicitly wait on %SCX_OPSS_DISPATCHING instead of @opss.
2280	*/
2281	wait_ops_state(p, SCX_OPSS_DISPATCHING);
2282	BUG_ON(atomic_long_read(&p->scx.ops_state) != SCX_OPSS_NONE);
2283	break;
2284	}
2285
2286	/*
2287	* Call ops.dequeue() if the task is still in BPF custody.
2288	*
2289	* The code that clears ops_state to %SCX_OPSS_NONE does not always
2290	* clear %SCX_TASK_IN_CUSTODY: in dispatch_to_local_dsq(), when
2291	* we're moving a task that was in %SCX_OPSS_DISPATCHING to a
2292	* remote CPU's local DSQ, we only set ops_state to %SCX_OPSS_NONE
2293	* so that a concurrent dequeue can proceed, but we clear
2294	* %SCX_TASK_IN_CUSTODY only when we later enqueue or move the
2295	* task. So we can see NONE + IN_CUSTODY here and we must handle
2296	* it. Similarly, after waiting on %SCX_OPSS_DISPATCHING we see
2297	* NONE but the task may still have %SCX_TASK_IN_CUSTODY set until
2298	* it is enqueued on the destination.
2299	*/
2300	if (task_leave_custody(p) && SCX_HAS_OP(sch, dequeue))
2301	SCX_CALL_OP_TASK(sch, dequeue, rq, p, deq_flags);
2302	}
2303
2304	static bool dequeue_task_scx(struct rq rq, struct task_struct p, int core_deq_flags)
2305	{
2306	struct scx_sched *sch = scx_task_sched(p);
2307	u64 deq_flags = core_deq_flags;
2308
2309	/*
2310	* Set %SCX_DEQ_SCHED_CHANGE when the dequeue is due to a property
2311	* change (not sleep).
2312	*/
2313	if (!(deq_flags & DEQUEUE_SLEEP))
2314	deq_flags \|= SCX_DEQ_SCHED_CHANGE;
2315
2316	if (!(p->scx.flags & SCX_TASK_QUEUED)) {
2317	WARN_ON_ONCE(task_runnable(p));
2318	return true;
2319	}
2320
2321	ops_dequeue(rq, p, deq_flags);
2322
2323	/*
2324	* A currently running task which is going off @rq first gets dequeued
2325	* and then stops running. As we want running <-> stopping transitions
2326	* to be contained within runnable <-> quiescent transitions, trigger
2327	* ->stopping() early here instead of in put_prev_task_scx().
2328	*
2329	* @p may go through multiple stopping <-> running transitions between
2330	* here and put_prev_task_scx() if task attribute changes occur while
2331	* dispatch_one() leaves @rq unlocked. However, they don't contain any
2332	* information meaningful to the BPF scheduler and can be suppressed by
2333	* skipping the callbacks if the task is !QUEUED.
2334	*/
2335	if (task_current(rq, p) &&
2336	(SCX_HAS_OP(sch, stopping) \|\| unlikely(p == scx_rescuee(rq)))) {
2337	update_curr_scx(rq);
2338	if (SCX_HAS_OP(sch, stopping))
2339	SCX_CALL_OP_TASK(sch, stopping, rq, p, false);
2340	}
2341
2342	if (SCX_HAS_OP(sch, quiescent) && !task_on_rq_migrating(p))
2343	SCX_CALL_OP_TASK(sch, quiescent, rq, p, deq_flags);
2344
2345	if (deq_flags & SCX_DEQ_SLEEP)
2346	p->scx.flags \|= SCX_TASK_DEQD_FOR_SLEEP;
2347	else
2348	p->scx.flags &= ~SCX_TASK_DEQD_FOR_SLEEP;
2349
2350	p->scx.flags &= ~SCX_TASK_QUEUED;
2351	rq->scx.nr_running--;
2352	sub_nr_running(rq, 1);
2353
2354	scx_dispatch_dequeue(rq, p);
2355
2356	/* see scx_task_slice_ended() for the save/restore exception */
2357	if (!((deq_flags & DEQUEUE_SAVE) && task_current(rq, p)))
2358	scx_task_slice_ended(rq, p);
2359
2360	clear_direct_dispatch(p);
2361	return true;
2362	}
2363
2364	static void yield_task_scx(struct rq *rq)
2365	{
2366	struct task_struct *p = rq->donor;
2367	struct scx_sched *sch = scx_task_sched(p);
2368
2369	/* a yield gives the slice up */
2370	scx_task_slice_ended(rq, p);
2371
2372	if (SCX_HAS_OP(sch, yield))
2373	SCX_CALL_OP_2TASKS_RET(sch, yield, rq, p, NULL);
2374	else
2375	scx_set_task_slice(p, 0);
2376	}
2377
2378	static bool yield_to_task_scx(struct rq rq, struct task_struct to)
2379	{
2380	struct task_struct *from = rq->donor;
2381	struct scx_sched *sch = scx_task_sched(from);
2382
2383	/* like a plain yield, giving the slice up ends the protection */
2384	scx_task_slice_ended(rq, from);
2385
2386	if (SCX_HAS_OP(sch, yield) && sch == scx_task_sched(to))
2387	return SCX_CALL_OP_2TASKS_RET(sch, yield, rq, from, to);
2388	else
2389	return false;
2390	}
2391
2392	static void wakeup_preempt_scx(struct rq rq, struct task_struct p, int wake_flags)
2393	{
2394	/*
2395	* Preemption between SCX tasks is implemented by resetting the victim
2396	* task's slice to 0 and triggering reschedule on the target CPU.
2397	* Nothing to do.
2398	*/
2399	if (p->sched_class == &ext_sched_class)
2400	return;
2401
2402	/*
2403	* Getting preempted by a higher-priority class. Reenqueue IMMED tasks.
2404	* This captures all preemption cases including:
2405	*
2406	* - A SCX task is currently running.
2407	*
2408	* - @rq is waking from idle due to a SCX task waking to it.
2409	*
2410	* - A higher-priority wakes up while SCX dispatch is in progress.
2411	*/
2412	if (rq->scx.nr_immed)
2413	scx_schedule_reenq_local(rq, 0);
2414	}
2415
2416	void scx_move_local_task_to_local_dsq(struct scx_sched sch, struct task_struct p,
2417	u64 enq_flags, struct rq *dst_rq)
2418	{
2419	struct scx_dispatch_q *dst_dsq;
2420
2421	lockdep_assert_rq_held(dst_rq);
2422
2423	WARN_ON_ONCE(p->scx.holding_cpu >= 0);
2424	scx_wait_dispatching(p);
2425
2426	dst_dsq = scx_resolve_local_dsq(sch, dst_rq, p, &enq_flags);
2427
2428	if (enq_flags & (SCX_ENQ_HEAD \| SCX_ENQ_PREEMPT))
2429	dsq_insert_head(dst_dsq, p);
2430	else
2431	list_add_tail(&p->scx.dsq_list.node, &dst_dsq->list);
2432
2433	dsq_inc_nr(dst_dsq, p, enq_flags);
2434	p->scx.dsq = dst_dsq;
2435
2436	rq_owned_post_enq(sch, dst_rq, dst_dsq, p, enq_flags);
2437	}
2438
2439	/**
2440	* move_remote_task_to_local_dsq - Move a task from a foreign rq to a local DSQ
2441	* @sch: scheduler placing @p
2442	* @p: task to move
2443	* @enq_flags: %SCX_ENQ_*
2444	* @src_rq: rq to move the task from, locked on entry, released on return
2445	* @dst_rq: rq to move the task into, locked on return
2446	*
2447	* Move @p which is currently on @src_rq to @dst_rq's local DSQ.
2448	*/
2449	static void move_remote_task_to_local_dsq(struct scx_sched *sch,
2450	struct task_struct *p, u64 enq_flags,
2451	struct rq src_rq, struct rq dst_rq)
2452	{
2453	lockdep_assert_rq_held(src_rq);
2454
2455	/*
2456	* Set sticky_cpu before deactivate_task() to properly mark the
2457	* beginning of an SCX-internal migration.
2458	*/
2459	p->scx.sticky_cpu = cpu_of(dst_rq);
2460	deactivate_task(src_rq, p, 0);
2461	set_task_cpu(p, cpu_of(dst_rq));
2462
2463	switch_rq_lock(src_rq, dst_rq);
2464
2465	/*
2466	* activate_task() below truncates enq_flags to 32 bits and re-derives
2467	* @p's owner, dropping our scx flags and the placing @sch. We own @rq,
2468	* so stash both across the call. The enqueue reads them back, keeping
2469	* the scx flags and checking caps against the placer, not the owner.
2470	*/
2471	WARN_ON_ONCE(!cpumask_test_cpu(cpu_of(dst_rq), p->cpus_ptr));
2472	WARN_ON_ONCE(dst_rq->scx.remote_activate_enq_flags \|\|
2473	dst_rq->scx.remote_activate_sch);
2474	dst_rq->scx.remote_activate_enq_flags = enq_flags;
2475	dst_rq->scx.remote_activate_sch = sch;
2476	activate_task(dst_rq, p, 0);
2477	dst_rq->scx.remote_activate_enq_flags = 0;
2478	dst_rq->scx.remote_activate_sch = NULL;
2479	}
2480
2481	/*
2482	* Similar to kernel/sched/core.c::is_cpu_allowed(). However, there are two
2483	* differences:
2484	*
2485	* - is_cpu_allowed() asks "Can this task run on this CPU?" while
2486	* task_can_run_on_remote_rq() asks "Can the BPF scheduler migrate the task to
2487	* this CPU?".
2488	*
2489	* While migration is disabled, is_cpu_allowed() has to say "yes" as the task
2490	* must be allowed to finish on the CPU that it's currently on regardless of
2491	* the CPU state. However, task_can_run_on_remote_rq() must say "no" as the
2492	* BPF scheduler shouldn't attempt to migrate a task which has migration
2493	* disabled.
2494	*
2495	* - The BPF scheduler is bypassed while the rq is offline and we can always say
2496	* no to the BPF scheduler initiated migrations while offline.
2497	*
2498	* The caller must ensure that @p and @rq are on different CPUs.
2499	* If enforce == true, caller must hold @p's rq lock.
2500	*/
2501	static bool task_can_run_on_remote_rq(struct scx_sched *sch,
2502	struct task_struct p, struct rq rq,
2503	bool enforce)
2504	{
2505	s32 cpu = cpu_of(rq);
2506
2507	/*
2508	* To prevent races with @p still running on its old CPU while switching
2509	* out, make sure we're holding @p's rq lock so as not to risk
2510	* erroneously killing the BPF scheduler.
2511	*/
2512	if (enforce)
2513	lockdep_assert_rq_held(task_rq(p));
2514
2515	WARN_ON_ONCE(task_cpu(p) == cpu);
2516
2517	/*
2518	* If @p has migration disabled, @p->cpus_ptr is updated to contain only
2519	* the pinned CPU in migrate_disable_switch() while @p is being switched
2520	* out. However, put_prev_task_scx() is called before @p->cpus_ptr is
2521	* updated and thus another CPU may see @p on a DSQ inbetween leading to
2522	* @p passing the below task_allowed_on_cpu() check while migration is
2523	* disabled.
2524	*
2525	* Test the migration disabled state first as the race window is narrow
2526	* and the BPF scheduler failing to check migration disabled state can
2527	* easily be masked if task_allowed_on_cpu() is done first.
2528	*/
2529	if (unlikely(is_migration_disabled(p))) {
2530	if (enforce)
2531	scx_error(sch, "SCX_DSQ_LOCAL[_ON] cannot move migration disabled %s[%d] from CPU %d to %d",
2532	p->comm, p->pid, task_cpu(p), cpu);
2533	return false;
2534	}
2535
2536	/*
2537	* We don't require the BPF scheduler to avoid dispatching to offline
2538	* CPUs mostly for convenience but also because CPUs can go offline
2539	* between scx_bpf_dsq_insert() calls and here. Trigger error iff the
2540	* picked CPU is outside the allowed mask.
2541	*/
2542	if (!task_allowed_on_cpu(p, cpu)) {
2543	if (enforce)
2544	scx_error(sch, "SCX_DSQ_LOCAL[_ON] target CPU %d not allowed for %s[%d]",
2545	cpu, p->comm, p->pid);
2546	return false;
2547	}
2548
2549	if (!scx_rq_online(rq)) {
2550	if (enforce)
2551	__scx_add_event(sch, SCX_EV_DISPATCH_LOCAL_DSQ_OFFLINE, 1);
2552	return false;
2553	}
2554
2555	return true;
2556	}
2557
2558	/**
2559	* unlink_dsq_and_switch_rq_lock() - Unlink task and switch to its rq lock
2560	* @p: target task
2561	* @dsq: locked DSQ @p is currently on
2562	* @locked_rq: currently locked rq
2563	* @src_rq: rq @p is currently on, stable with @dsq locked
2564	*
2565	* Called with @dsq and @locked_rq locked. We want to move @p to a different DSQ,
2566	* including any local DSQ, but are not locking @src_rq. Locking @src_rq is
2567	* required when transferring into a local DSQ. Even when transferring into a
2568	* non-local DSQ, it's better to use the same mechanism to protect against
2569	* dequeues and maintain the invariant that @p->scx.dsq can only change while
2570	* @src_rq is locked, which e.g. scx_dump_task() depends on.
2571	*
2572	* We want to grab @src_rq but that can deadlock if we try while locking @dsq,
2573	* so we want to unlink @p from @dsq, drop its lock and then lock @src_rq. As
2574	* this may race with dequeue, which can't drop the rq lock or fail, do a little
2575	* dancing from our side.
2576	*
2577	* @p->scx.holding_cpu is set to this CPU before @dsq is unlocked. If @p gets
2578	* dequeued after we unlock @dsq but before locking @src_rq, the holding_cpu
2579	* would be cleared to -1. While other cpus may have updated it to different
2580	* values afterwards, as this operation can't be preempted or recurse, the
2581	* holding_cpu can never become this CPU again before we're done. Thus, we can
2582	* tell whether we lost to dequeue by testing whether the holding_cpu still
2583	* points to this CPU. See scx_dispatch_dequeue() for the counterpart.
2584	*
2585	* On return, @dsq is unlocked and @src_rq is locked. Returns %true if @p is
2586	* still valid. %false if lost to dequeue.
2587	*/
2588	static bool unlink_dsq_and_switch_rq_lock(struct task_struct *p,
2589	struct scx_dispatch_q *dsq,
2590	struct rq *locked_rq,
2591	struct rq *src_rq)
2592	{
2593	s32 cpu = raw_smp_processor_id();
2594
2595	lockdep_assert_held(&dsq->lock);
2596	lockdep_assert_rq_held(locked_rq);
2597
2598	WARN_ON_ONCE(p->scx.holding_cpu >= 0);
2599	scx_task_unlink_from_dsq(p, dsq);
2600	p->scx.holding_cpu = cpu;
2601
2602	raw_spin_unlock(&dsq->lock);
2603	switch_rq_lock(locked_rq, src_rq);
2604
2605	/* task_rq couldn't have changed if we're still the holding cpu */
2606	if (likely(p->scx.holding_cpu == cpu) && !WARN_ON_ONCE(src_rq != task_rq(p))) {
2607	scx_wait_dispatching(p);
2608	return true;
2609	}
2610
2611	return false;
2612	}
2613
2614	static bool consume_remote_task(struct scx_sched sch, struct rq this_rq,
2615	struct task_struct *p, u64 enq_flags,
2616	struct scx_dispatch_q dsq, struct rq src_rq)
2617	{
2618	if (unlink_dsq_and_switch_rq_lock(p, dsq, this_rq, src_rq)) {
2619	move_remote_task_to_local_dsq(sch, p, enq_flags, src_rq, this_rq);
2620	return true;
2621	} else {
2622	switch_rq_lock(src_rq, this_rq);
2623	return false;
2624	}
2625	}
2626
2627	/**
2628	* move_task_between_dsqs() - Move a task from one DSQ to another
2629	* @sch: scx_sched being operated on
2630	* @p: target task
2631	* @enq_flags: %SCX_ENQ_*
2632	* @src_dsq: DSQ @p is currently on, must not be a local DSQ
2633	* @dst_dsq: DSQ @p is being moved to, can be any DSQ
2634	*
2635	* Must be called with @p's task_rq and @src_dsq locked. If @dst_dsq is a local
2636	* DSQ and @p is on a different CPU, @p will be migrated and thus its task_rq
2637	* will change. As @p's task_rq is locked, this function doesn't need to use the
2638	* holding_cpu mechanism.
2639	*
2640	* On return, @src_dsq is unlocked and only @p's new task_rq, which is the
2641	* return value, is locked.
2642	*/
2643	static struct rq move_task_between_dsqs(struct scx_sched sch,
2644	struct task_struct *p, u64 enq_flags,
2645	struct scx_dispatch_q *src_dsq,
2646	struct scx_dispatch_q *dst_dsq)
2647	{
2648	struct rq src_rq = task_rq(p), dst_rq;
2649
2650	BUG_ON(src_dsq->id == SCX_DSQ_LOCAL);
2651	lockdep_assert_held(&src_dsq->lock);
2652	lockdep_assert_rq_held(src_rq);
2653
2654	scx_wait_dispatching(p);
2655
2656	if (dst_dsq->id == SCX_DSQ_LOCAL) {
2657	dst_rq = container_of(dst_dsq, struct rq, scx.local_dsq);
2658	if (src_rq != dst_rq &&
2659	unlikely(!task_can_run_on_remote_rq(sch, p, dst_rq, true))) {
2660	dst_dsq = find_global_dsq(sch, task_cpu(p));
2661	dst_rq = src_rq;
2662	enq_flags \|= SCX_ENQ_GDSQ_FALLBACK;
2663	}
2664	} else {
2665	/* no need to migrate if destination is a non-local DSQ */
2666	dst_rq = src_rq;
2667	}
2668
2669	/*
2670	* Move @p into $dst_dsq. If $dst_dsq is the local DSQ of a different
2671	* CPU, @p will be migrated.
2672	*/
2673	if (dst_dsq->id == SCX_DSQ_LOCAL) {
2674	/* @p is going from a non-local DSQ to a local DSQ */
2675	if (src_rq == dst_rq) {
2676	scx_task_unlink_from_dsq(p, src_dsq);
2677	raw_spin_unlock(&src_dsq->lock);
2678	scx_move_local_task_to_local_dsq(sch, p, enq_flags, dst_rq);
2679	} else {
2680	raw_spin_unlock(&src_dsq->lock);
2681	move_remote_task_to_local_dsq(sch, p, enq_flags, src_rq, dst_rq);
2682	}
2683	} else {
2684	/*
2685	* @p is going from a non-local DSQ to a non-local DSQ. As
2686	* $src_dsq is already locked, do an abbreviated dequeue.
2687	*/
2688	dispatch_dequeue_locked(p, src_dsq);
2689	raw_spin_unlock(&src_dsq->lock);
2690
2691	scx_dispatch_enqueue(sch, dst_rq, dst_dsq, p, 0, 0, enq_flags);
2692	}
2693
2694	return dst_rq;
2695	}
2696
2697	bool scx_consume_dispatch_q(struct scx_sched sch, struct rq rq,
2698	struct scx_dispatch_q *dsq, u64 enq_flags)
2699	{
2700	struct task_struct *p;
2701	retry:
2702	/*
2703	* The caller can't expect to successfully consume a task if the task's
2704	* addition to @dsq isn't guaranteed to be visible somehow. Test
2705	* @dsq->list without locking and skip if it seems empty.
2706	*/
2707	if (list_empty(&dsq->list))
2708	return false;
2709
2710	raw_spin_lock(&dsq->lock);
2711
2712	nldsq_for_each_task(p, dsq) {
2713	struct rq *task_rq = task_rq(p);
2714
2715	/*
2716	* This loop can lead to multiple lockup scenarios, e.g. the BPF
2717	* scheduler can put an enormous number of affinitized tasks into
2718	* a contended DSQ, or the outer retry loop can repeatedly race
2719	* against scx_bypass() dequeueing tasks from @dsq trying to put
2720	* the system into the bypass mode. This can easily live-lock the
2721	* machine. If aborting, exit from all non-bypass DSQs.
2722	*/
2723	if (unlikely(READ_ONCE(sch->aborting)) && dsq->id != SCX_DSQ_BYPASS)
2724	break;
2725
2726	if (rq == task_rq) {
2727	scx_task_unlink_from_dsq(p, dsq);
2728	raw_spin_unlock(&dsq->lock);
2729	scx_move_local_task_to_local_dsq(sch, p, enq_flags, rq);
2730	return true;
2731	}
2732
2733	if (task_can_run_on_remote_rq(sch, p, rq, false)) {
2734	if (likely(consume_remote_task(sch, rq, p, enq_flags, dsq, task_rq)))
2735	return true;
2736	goto retry;
2737	}
2738	}
2739
2740	raw_spin_unlock(&dsq->lock);
2741	return false;
2742	}
2743
2744	bool scx_consume_global_dsq(struct scx_sched sch, struct rq rq)
2745	{
2746	int node = cpu_to_node(cpu_of(rq));
2747
2748	return scx_consume_dispatch_q(sch, rq, &sch->pnode[node]->global_dsq, 0);
2749	}
2750
2751	/**
2752	* dispatch_to_local_dsq - Dispatch a task to a local dsq
2753	* @sch: scx_sched being operated on
2754	* @rq: current rq which is locked
2755	* @dst_dsq: destination DSQ
2756	* @p: task to dispatch
2757	* @slice: slice carried by the insert verdict, 0 keeps the current value
2758	* @vtime: vtime carried by the insert verdict, committed on PRIQ inserts
2759	* @enq_flags: %SCX_ENQ_*
2760	*
2761	* We're holding @rq lock and want to dispatch @p to @dst_dsq which is a local
2762	* DSQ. This function performs all the synchronization dancing needed because
2763	* local DSQs are protected with rq locks.
2764	*
2765	* The caller must have exclusive ownership of @p (e.g. through
2766	* %SCX_OPSS_DISPATCHING).
2767	*/
2768	static void dispatch_to_local_dsq(struct scx_sched sch, struct rq rq,
2769	struct scx_dispatch_q dst_dsq, struct task_struct p,
2770	u64 slice, u64 vtime, u64 enq_flags)
2771	{
2772	struct rq *src_rq = task_rq(p);
2773	struct rq *dst_rq = container_of(dst_dsq, struct rq, scx.local_dsq);
2774	struct rq *locked_rq = rq;
2775
2776	/*
2777	* We're synchronized against dequeue through DISPATCHING. As @p can't
2778	* be dequeued, its task_rq and cpus_allowed are stable too.
2779	*
2780	* If dispatching to @rq that @p is already on, no lock dancing needed.
2781	*/
2782	if (rq == src_rq && rq == dst_rq) {
2783	scx_dispatch_enqueue(sch, rq, dst_dsq, p, slice, vtime,
2784	enq_flags \| SCX_ENQ_APPLY_SLICE \| SCX_ENQ_CLEAR_OPSS);
2785	return;
2786	}
2787
2788	/*
2789	* @p is on a possibly remote @src_rq which we need to lock to move the
2790	* task. If dequeue is in progress, it'd be locking @src_rq and waiting
2791	* on DISPATCHING, so we can't grab @src_rq lock while holding
2792	* DISPATCHING.
2793	*
2794	* As DISPATCHING guarantees that @p is wholly ours, we can pretend that
2795	* we're moving from a DSQ and use the same mechanism - mark the task
2796	* under transfer with holding_cpu, release DISPATCHING and then follow
2797	* the same protocol. See unlink_dsq_and_switch_rq_lock().
2798	*/
2799	p->scx.holding_cpu = raw_smp_processor_id();
2800
2801	/* store_release ensures that dequeue sees the above */
2802	atomic_long_set_release(&p->scx.ops_state, SCX_OPSS_NONE);
2803
2804	/* switch to @src_rq lock */
2805	if (locked_rq != src_rq) {
2806	switch_rq_lock(locked_rq, src_rq);
2807	locked_rq = src_rq;
2808	}
2809
2810	/* task_rq couldn't have changed if we're still the holding cpu */
2811	if (likely(p->scx.holding_cpu == raw_smp_processor_id()) &&
2812	!WARN_ON_ONCE(src_rq != task_rq(p))) {
2813	bool fallback = false;
2814	/*
2815	* If @p is staying on the same rq, there's no need to go
2816	* through the full deactivate/activate cycle. Optimize by
2817	* abbreviating move_remote_task_to_local_dsq().
2818	*/
2819	if (src_rq == dst_rq) {
2820	p->scx.holding_cpu = -1;
2821	scx_dispatch_enqueue(sch, dst_rq, &dst_rq->scx.local_dsq, p,
2822	slice, vtime, enq_flags \| SCX_ENQ_APPLY_SLICE);
2823	} else if (unlikely(!task_can_run_on_remote_rq(sch, p, dst_rq, true))) {
2824	p->scx.holding_cpu = -1;
2825	fallback = true;
2826	scx_dispatch_enqueue(sch, src_rq, find_global_dsq(sch, task_cpu(p)),
2827	p, slice, vtime,
2828	enq_flags \| SCX_ENQ_APPLY_SLICE \|
2829	SCX_ENQ_GDSQ_FALLBACK);
2830	} else {
2831	apply_slice_vtime(p, slice, vtime, enq_flags);
2832	move_remote_task_to_local_dsq(sch, p, enq_flags, src_rq, dst_rq);
2833	/* task has been moved to dst_rq, which is now locked */
2834	locked_rq = dst_rq;
2835	}
2836
2837	/* if the destination CPU is idle, wake it up */
2838	if (!fallback && sched_class_above(p->sched_class, dst_rq->curr->sched_class))
2839	resched_curr(dst_rq);
2840	}
2841
2842	/* switch back to @rq lock */
2843	if (locked_rq != rq)
2844	switch_rq_lock(locked_rq, rq);
2845	}
2846
2847	/**
2848	* finish_dispatch - Asynchronously finish dispatching a task
2849	* @sch: the scheduler
2850	* @rq: current rq which is locked
2851	* @p: task to finish dispatching
2852	* @qseq_at_dispatch: qseq when @p started getting dispatched
2853	* @dsq_id: destination DSQ ID
2854	* @slice: slice carried by the insert verdict, 0 keeps the current value
2855	* @vtime: vtime carried by the insert verdict, committed on PRIQ inserts
2856	* @enq_flags: %SCX_ENQ_*
2857	*
2858	* Dispatching to local DSQs may need to wait for queueing to complete or
2859	* require rq lock dancing. As we don't wanna do either while inside
2860	* ops.dispatch() to avoid locking order inversion, we split dispatching into
2861	* two parts. scx_bpf_dsq_insert() which is called by ops.dispatch() records the
2862	* task and its qseq. Once ops.dispatch() returns, this function is called to
2863	* finish up.
2864	*
2865	* There is no guarantee that @p is still valid for dispatching or even that it
2866	* was valid in the first place. Make sure that the task is still owned by the
2867	* BPF scheduler and claim the ownership before dispatching.
2868	*/
2869	static void finish_dispatch(struct scx_sched sch, struct rq rq, struct task_struct *p,
2870	unsigned long qseq_at_dispatch, u64 dsq_id,
2871	u64 slice, u64 vtime, u64 enq_flags)
2872	{
2873	struct scx_dispatch_q *dsq;
2874	unsigned long opss;
2875
2876	retry:
2877	/*
2878	* No need for _acquire here. @p is accessed only after a successful
2879	* try_cmpxchg to DISPATCHING.
2880	*/
2881	opss = atomic_long_read(&p->scx.ops_state);
2882
2883	switch (opss & SCX_OPSS_STATE_MASK) {
2884	case SCX_OPSS_DISPATCHING:
2885	case SCX_OPSS_NONE:
2886	/* someone else already got to it */
2887	return;
2888	case SCX_OPSS_QUEUED:
2889	/*
2890	* If qseq doesn't match, @p has gone through at least one
2891	* dispatch/dequeue and re-enqueue cycle between
2892	* scx_bpf_dsq_insert() and here and we have no claim on it.
2893	*/
2894	if ((opss & SCX_OPSS_QSEQ_MASK) != qseq_at_dispatch)
2895	return;
2896
2897	/* see SCX_EV_INSERT_NOT_OWNED definition */
2898	if (unlikely(!scx_task_on_sched(sch, p))) {
2899	__scx_add_event(sch, SCX_EV_INSERT_NOT_OWNED, 1);
2900	return;
2901	}
2902
2903	/*
2904	* While we know @p is accessible, we don't yet have a claim on
2905	* it - the BPF scheduler is allowed to dispatch tasks
2906	* spuriously and there can be a racing dequeue attempt. Let's
2907	* claim @p by atomically transitioning it from QUEUED to
2908	* DISPATCHING.
2909	*/
2910	if (likely(atomic_long_try_cmpxchg(&p->scx.ops_state, &opss,
2911	SCX_OPSS_DISPATCHING)))
2912	break;
2913	goto retry;
2914	case SCX_OPSS_QUEUEING:
2915	/*
2916	* scx_do_enqueue_task() is in the process of transferring the
2917	* task to the BPF scheduler while holding @p's rq lock. As we
2918	* aren't holding any kernel or BPF resource that the enqueue
2919	* path may depend upon, it's safe to wait.
2920	*/
2921	wait_ops_state(p, opss);
2922	goto retry;
2923	}
2924
2925	BUG_ON(!(p->scx.flags & SCX_TASK_QUEUED));
2926
2927	dsq = find_dsq_for_dispatch(sch, rq, dsq_id, task_cpu(p));
2928
2929	if (dsq->id == SCX_DSQ_LOCAL)
2930	dispatch_to_local_dsq(sch, rq, dsq, p, slice, vtime, enq_flags);
2931	else
2932	scx_dispatch_enqueue(sch, rq, dsq, p, slice, vtime,
2933	enq_flags \| SCX_ENQ_APPLY_SLICE \| SCX_ENQ_CLEAR_OPSS);
2934	}
2935
2936	void scx_flush_dispatch_buf(struct scx_sched sch, struct rq rq)
2937	{
2938	struct scx_dsp_ctx *dspc = &this_cpu_ptr(sch->pcpu)->dsp_ctx;
2939	u32 u;
2940
2941	for (u = 0; u < dspc->cursor; u++) {
2942	struct scx_dsp_buf_ent *ent = &dspc->buf[u];
2943
2944	finish_dispatch(sch, rq, ent->task, ent->qseq, ent->dsq_id,
2945	ent->slice, ent->vtime, ent->enq_flags);
2946	}
2947
2948	dspc->nr_tasks += dspc->cursor;
2949	dspc->cursor = 0;
2950	}
2951
2952	static inline void maybe_queue_balance_callback(struct rq *rq)
2953	{
2954	lockdep_assert_rq_held(rq);
2955
2956	if (!(rq->scx.flags & SCX_RQ_BAL_CB_PENDING))
2957	return;
2958
2959	queue_balance_callback(rq, &rq->scx.deferred_bal_cb,
2960	deferred_bal_cb_workfn);
2961
2962	rq->scx.flags &= ~SCX_RQ_BAL_CB_PENDING;
2963	}
2964
2965	static enum scx_dsp_verdict dispatch_one(struct rq rq, struct task_struct prev)
2966	{
2967	struct scx_sched *root_sch = scx_root_protected_live();
2968	enum scx_dsp_verdict verdict;
2969	s32 cpu = cpu_of(rq);
2970
2971	lockdep_assert_rq_held(rq);
2972	rq->scx.flags \|= SCX_RQ_IN_DISPATCH;
2973
2974	scx_process_sync_ecaps(rq, prev);
2975
2976	if ((root_sch->ops.flags & SCX_OPS_HAS_CPU_PREEMPT) &&
2977	unlikely(rq->scx.cpu_released)) {
2978	/*
2979	* If the previous sched_class for the current CPU was not SCX,
2980	* notify the BPF scheduler that it again has control of the
2981	* core. This callback complements ->cpu_release(), which is
2982	* emitted in switch_class().
2983	*/
2984	if (root_sch->ops.cpu_acquire)
2985	SCX_CALL_OP(root_sch, cpu_acquire, rq, cpu, NULL);
2986	rq->scx.cpu_released = false;
2987	}
2988
2989	if (prev->sched_class == &ext_sched_class) {
2990	update_curr_scx(rq);
2991
2992	/*
2993	* If @prev is runnable & has slice left, it has priority and
2994	* fetching more just increases latency for the fetched tasks.
2995	* Tell pick_task_scx() to keep running @prev. If the BPF
2996	* scheduler wants to handle this explicitly, it should
2997	* implement ->cpu_release().
2998	*
2999	* See scx_disable_workfn() for the explanation on the bypassing
3000	* test.
3001	*/
3002	if ((prev->scx.flags & SCX_TASK_QUEUED) && prev->scx.slice &&
3003	!scx_bypassing(scx_task_sched(prev), cpu)) {
3004	verdict = SCX_DSP_PREV;
3005	goto has_tasks;
3006	}
3007	}
3008
3009	/* if there already are tasks to run, nothing to do */
3010	if (rq->scx.local_dsq.nr) {
3011	verdict = SCX_DSP_LOCAL;
3012	goto has_tasks;
3013	}
3014
3015	verdict = scx_dispatch_sched(root_sch, rq, prev, false);
3016	if (verdict != SCX_DSP_NONE)
3017	goto has_tasks;
3018
3019	/*
3020	* Didn't find another task to run. Keep running @prev unless its own
3021	* scheduler set %SCX_OPS_ENQ_LAST and takes the enqueue instead, see
3022	* put_prev_task_scx(). Read the scheduler here as the dispatch above
3023	* may have dropped the rq lock while @prev changed class or scheduler.
3024	*/
3025	if (prev->scx.flags & SCX_TASK_QUEUED) {
3026	struct scx_sched *prev_sch = scx_task_sched(prev);
3027
3028	if ((!(prev_sch->ops.flags & SCX_OPS_ENQ_LAST) \|\|
3029	scx_bypassing(prev_sch, cpu)) && scx_task_can_stay_on_cpu(rq, prev)) {
3030	__scx_add_event(prev_sch, SCX_EV_DISPATCH_KEEP_LAST, 1);
3031	verdict = SCX_DSP_PREV;
3032	goto has_tasks;
3033	}
3034	}
3035	rq->scx.flags &= ~SCX_RQ_IN_DISPATCH;
3036	return SCX_DSP_NONE;
3037
3038	has_tasks:
3039	/*
3040	* @rq may have extra IMMED tasks without reenq scheduled:
3041	*
3042	* - rq_is_open() can't reliably tell when and how slice is going to be
3043	* modified for $curr and allows IMMED tasks to be queued while
3044	* dispatch is in progress.
3045	*
3046	* - A non-IMMED HEAD task can get queued in front of an IMMED task
3047	* between the IMMED queueing and the subsequent scheduling event.
3048	*/
3049	if (unlikely(rq->scx.local_dsq.nr > 1 && rq->scx.nr_immed))
3050	scx_schedule_reenq_local(rq, 0);
3051
3052	rq->scx.flags &= ~SCX_RQ_IN_DISPATCH;
3053	return verdict;
3054	}
3055
3056	static void set_next_task_scx(struct rq rq, struct task_struct p, enum snt_e type)
3057	{
3058	struct scx_sched *sch = scx_task_sched(p);
3059
3060	if (type == SNT_REPICK)
3061	return;
3062
3063	if (p->scx.flags & SCX_TASK_QUEUED) {
3064	/*
3065	* Core-sched might decide to execute @p before it is
3066	* dispatched. Call ops_dequeue() to notify the BPF scheduler.
3067	*/
3068	ops_dequeue(rq, p, SCX_DEQ_CORE_SCHED_EXEC);
3069	scx_dispatch_dequeue(rq, p);
3070	}
3071
3072	p->se.exec_start = rq_clock_task(rq);
3073
3074	/* see dequeue_task_scx() on why we skip when !QUEUED */
3075	if (SCX_HAS_OP(sch, running) && (p->scx.flags & SCX_TASK_QUEUED))
3076	SCX_CALL_OP_TASK(sch, running, rq, p);
3077
3078	clr_task_runnable(p, true);
3079
3080	/* apply any pending out-of-band slice request before the tick decision */
3081	apply_task_slice_oob(rq, p);
3082
3083	/*
3084	* @p is getting newly scheduled or got kicked after someone updated its
3085	* slice. Update SCX_RQ_CAN_STOP_TICK to reflect whether the tick can be
3086	* stopped. See scx_can_stop_tick().
3087	*
3088	* Moreover, refresh the load_avgs just when transitioning in and out of
3089	* nohz. In the future, we might want to add a mechanism to update
3090	* load_avgs periodically on tick-stopped CPUs.
3091	*/
3092	if (p->scx.slice == SCX_SLICE_INF) {
3093	if (!(rq->scx.flags & SCX_RQ_CAN_STOP_TICK)) {
3094	/*
3095	* Bypass mode always assigns finite slices, so @p
3096	* can't have an infinite slice while bypassing.
3097	* Therefore, sched_update_tick_dependency() can safely
3098	* evaluate the outgoing task.
3099	*/
3100	rq->scx.flags \|= SCX_RQ_CAN_STOP_TICK;
3101	sched_update_tick_dependency(rq);
3102
3103	update_other_load_avgs(rq);
3104	}
3105	} else {
3106	if (rq->scx.flags & SCX_RQ_CAN_STOP_TICK) {
3107	rq->scx.flags &= ~SCX_RQ_CAN_STOP_TICK;
3108	update_other_load_avgs(rq);
3109	}
3110
3111	/*
3112	* @rq still references the outgoing scheduling context. A finite
3113	* slice is sufficient by itself to require the tick.
3114	*/
3115	if (tick_nohz_full_cpu(cpu_of(rq)))
3116	tick_nohz_dep_set_cpu(cpu_of(rq), TICK_DEP_BIT_SCHED);
3117	}
3118	}
3119
3120	void scx_proxy_donor_start(struct rq *rq)
3121	{
3122	}
3123
3124	static enum scx_cpu_preempt_reason
3125	preempt_reason_from_class(const struct sched_class *class)
3126	{
3127	if (class == &stop_sched_class)
3128	return SCX_CPU_PREEMPT_STOP;
3129	if (class == &dl_sched_class)
3130	return SCX_CPU_PREEMPT_DL;
3131	if (class == &rt_sched_class)
3132	return SCX_CPU_PREEMPT_RT;
3133	return SCX_CPU_PREEMPT_UNKNOWN;
3134	}
3135
3136	static void switch_class(struct rq rq, struct task_struct next)
3137	{
3138	struct scx_sched *sch = scx_root_protected_live();
3139	const struct sched_class *next_class = next->sched_class;
3140
3141	if (!(sch->ops.flags & SCX_OPS_HAS_CPU_PREEMPT))
3142	return;
3143
3144	/*
3145	* The callback is conceptually meant to convey that the CPU is no
3146	* longer under the control of SCX. Therefore, don't invoke the callback
3147	* if the next class is below SCX (in which case the BPF scheduler has
3148	* actively decided not to schedule any tasks on the CPU).
3149	*/
3150	if (sched_class_above(&ext_sched_class, next_class))
3151	return;
3152
3153	/*
3154	* At this point we know that SCX was preempted by a higher priority
3155	* sched_class, so invoke the ->cpu_release() callback if we have not
3156	* done so already. We only send the callback once between SCX being
3157	* preempted, and it regaining control of the CPU.
3158	*
3159	* ->cpu_release() complements ->cpu_acquire(), which is emitted the
3160	* next time that dispatch_one() is invoked.
3161	*/
3162	if (!rq->scx.cpu_released) {
3163	if (sch->ops.cpu_release) {
3164	struct scx_cpu_release_args args = {
3165	.reason = preempt_reason_from_class(next_class),
3166	.task = next,
3167	};
3168
3169	SCX_CALL_OP(sch, cpu_release, rq, cpu_of(rq), &args);
3170	}
3171	rq->scx.cpu_released = true;
3172	}
3173	}
3174
3175	static void put_prev_task_scx(struct rq rq, struct task_struct p,
3176	struct task_struct *next)
3177	{
3178	struct scx_sched *sch = scx_task_sched(p);
3179	bool rescue_keep = false;
3180
3181	/* see kick_sync_wait_bal_cb() */
3182	smp_store_release(&rq->scx.kick_sync, rq->scx.kick_sync + 1);
3183
3184	update_curr_scx(rq);
3185
3186	/*
3187	* If the slice is consumed, protection ends with it. A rescuee
3188	* preempted beforehand keeps going, see scx_rescue_keep().
3189	*/
3190	if (!p->scx.slice) {
3191	if (unlikely(p == scx_rescuee(rq)))
3192	rescue_keep = scx_rescue_keep(rq, p);
3193	if (!rescue_keep)
3194	scx_task_slice_ended(rq, p);
3195	}
3196
3197	/* see dequeue_task_scx() on why we skip when !QUEUED */
3198	if (SCX_HAS_OP(sch, stopping) && (p->scx.flags & SCX_TASK_QUEUED))
3199	SCX_CALL_OP_TASK(sch, stopping, rq, p, true);
3200
3201	if (p->scx.flags & SCX_TASK_QUEUED) {
3202	set_task_runnable(rq, p);
3203
3204	/*
3205	* If @p has slice left and is being put, @p is getting
3206	* preempted by a higher priority scheduler class or core-sched
3207	* forcing a different task. Leave it at the head of the local
3208	* DSQ unless it was an IMMED task. IMMED tasks should not
3209	* linger on a busy CPU, reenqueue them to the BPF scheduler.
3210	*
3211	* An open rescue must keep @p on the local DSQ even if the
3212	* scheduler zeroed the slice in ops.stopping() above.
3213	*/
3214	if ((p->scx.slice \|\| unlikely(p == scx_rescuee(rq))) &&
3215	!scx_bypassing(sch, cpu_of(rq))) {
3216	if (p->scx.flags & SCX_TASK_IMMED) {
3217	p->scx.flags \|= SCX_TASK_REENQ_PREEMPTED;
3218	scx_do_enqueue_task(rq, p, SCX_ENQ_REENQ, -1);
3219	p->scx.flags &= ~SCX_TASK_REENQ_REASON_MASK;
3220	} else {
3221	u64 enq_flags = 0;
3222
3223	/*
3224	* Keep a preempted rescue going. If preempted
3225	* by another SCX task, append to the local DSQ,
3226	* see scx_rescue_keep().
3227	*/
3228	if (unlikely(p == scx_rescuee(rq))) {
3229	enq_flags \|= SCX_ENQ_IGNORE_CAPS;
3230	if (!rescue_keep)
3231	enq_flags \|= SCX_ENQ_HEAD;
3232	} else {
3233	enq_flags \|= SCX_ENQ_HEAD;
3234	}
3235
3236	scx_dispatch_enqueue(sch, rq, &rq->scx.local_dsq, p, 0, 0,
3237	enq_flags);
3238	}
3239	goto switch_class;
3240	}
3241
3242	/*
3243	* If @p is runnable but we're about to enter a lower
3244	* sched_class, %SCX_OPS_ENQ_LAST must be set. Tell
3245	* ops.enqueue() that @p is the only one available for this cpu.
3246	* This doesn't apply if the baseline access on the CPU is lost.
3247	*
3248	* Under core scheduling, a pick dispatches only when nothing is
3249	* locally runnable and can legitimately go idle with @p still
3250	* runnable (see do_pick_task_scx()).
3251	*/
3252	if (next && sched_class_above(&ext_sched_class, next->sched_class) &&
3253	scx_task_can_stay_on_cpu(rq, p)) {
3254	WARN_ON_ONCE(!sched_core_enabled(rq) &&
3255	!(sch->ops.flags & SCX_OPS_ENQ_LAST));
3256	scx_do_enqueue_task(rq, p, SCX_ENQ_LAST, -1);
3257
3258	/*
3259	* A task put on the local DSQ runs, as anywhere else.
3260	* Here the insert can't reschedule on its own: the pick
3261	* has settled on @next and @p is still curr, so
3262	* resched_curr() would flag @p and __schedule() clears
3263	* that right after the pick. Flag @next instead.
3264	*/
3265	if (p->scx.dsq == &rq->scx.local_dsq)
3266	set_tsk_need_resched(next);
3267	} else {
3268	scx_do_enqueue_task(rq, p, 0, -1);
3269	}
3270	}
3271
3272	switch_class:
3273	if (next && next->sched_class != &ext_sched_class)
3274	switch_class(rq, next);
3275	}
3276
3277	static void kick_sync_wait_bal_cb(struct rq *rq)
3278	{
3279	struct scx_kick_syncs __rcu *ks;
3280	unsigned long *ksyncs;
3281	bool waited;
3282	s32 cpu;
3283
3284	/*
3285	* This callback is queued and normally flushed within @rq's own
3286	* scheduling pass. However, dispatch can drop the rq lock while it sits
3287	* queued, and lock takers in that window (the sched class change paths,
3288	* the scx task iterator) flush pending balance callbacks on release,
3289	* running this one on a foreign CPU whose snapshots are unrelated. The
3290	* kicked CPUs are already on their way to advance the kick_syncs being
3291	* waited on. Don't get in the way.
3292	*/
3293	if (unlikely(cpu_of(rq) != smp_processor_id()))
3294	return;
3295
3296	ks = __this_cpu_read(scx_kick_syncs);
3297	ksyncs = rcu_dereference_sched(ks)->syncs;
3298
3299	/*
3300	* Drop rq lock and enable IRQs while waiting. IRQs must be enabled
3301	* — a target CPU may be waiting for us to process an IPI (e.g. TLB
3302	* flush) while we wait for its kick_sync to advance.
3303	*
3304	* Also, keep advancing our own kick_sync so that new kick_sync waits
3305	* targeting us, which can start after we drop the lock, cannot form
3306	* cyclic dependencies.
3307	*/
3308	retry:
3309	waited = false;
3310	for_each_cpu(cpu, rq->scx.cpus_to_sync) {
3311	/*
3312	* smp_load_acquire() pairs with smp_store_release() on
3313	* kick_sync updates on the target CPUs.
3314	*/
3315	if (cpu == cpu_of(rq) \|\|
3316	smp_load_acquire(&cpu_rq(cpu)->scx.kick_sync) != ksyncs[cpu]) {
3317	cpumask_clear_cpu(cpu, rq->scx.cpus_to_sync);
3318	continue;
3319	}
3320
3321	scx_rq_lock_drop(rq);
3322	raw_spin_rq_unlock_irq(rq);
3323	while (READ_ONCE(cpu_rq(cpu)->scx.kick_sync) == ksyncs[cpu]) {
3324	smp_store_release(&rq->scx.kick_sync, rq->scx.kick_sync + 1);
3325	cpu_relax();
3326	}
3327	raw_spin_rq_lock_irq(rq);
3328	waited = true;
3329	}
3330
3331	if (waited)
3332	goto retry;
3333	}
3334
3335	static struct task_struct first_local_task(struct rq rq)
3336	{
3337	return list_first_entry_or_null(&rq->scx.local_dsq.list,
3338	struct task_struct, scx.dsq_list.node);
3339	}
3340
3341	/*
3342	* Run dispatch and queue the follow-up work for a pick.
3343	*/
3344	static enum scx_dsp_verdict dispatch_pick(struct rq rq, struct rq_flags rf,
3345	struct task_struct *prev)
3346	{
3347	enum scx_dsp_verdict verdict;
3348
3349	rq_unpin_lock(rq, rf);
3350	verdict = dispatch_one(rq, prev);
3351	rq_repin_lock(rq, rf);
3352	maybe_queue_balance_callback(rq);
3353
3354	/*
3355	* Defer to a balance callback which can drop rq lock and enable IRQs.
3356	* Waiting directly in the pick path would deadlock against CPUs sending
3357	* us IPIs (e.g. TLB flushes) while we wait for them.
3358	*/
3359	if (unlikely(rq->scx.kick_sync_pending)) {
3360	rq->scx.kick_sync_pending = false;
3361	queue_balance_callback(rq, &rq->scx.kick_sync_bal_cb,
3362	kick_sync_wait_bal_cb);
3363	}
3364
3365	return verdict;
3366	}
3367
3368	#ifdef CONFIG_SCHED_CORE
3369	/*
3370	* Dispatch for a pick when core scheduling is enabled. The selection picks for
3371	* all SMT siblings and the rq_i->core_pick state it builds must stay atomic
3372	* throughout. If the dispatch released the rq lock, anything can have happened
3373	* in between - return %SCX_DSP_RETRY to restart the selection against current
3374	* state.
3375	*/
3376	static enum scx_dsp_verdict dispatch_core_pick(struct rq rq, struct rq_flags rf,
3377	struct task_struct *prev)
3378	{
3379	enum scx_dsp_verdict verdict;
3380	u32 seq = rq->scx.lock_drop_seq;
3381
3382	/* another dispatch is in flight on @rq, let that handle it */
3383	if (rq->scx.flags & SCX_RQ_IN_DISPATCH)
3384	return SCX_DSP_NONE;
3385
3386	rq_unpin_lock(rq, rf);
3387
3388	verdict = dispatch_one(rq, prev);
3389
3390	if (cpu_of(rq) == smp_processor_id()) {
3391	maybe_queue_balance_callback(rq);
3392
3393	/* see dispatch_pick() */
3394	if (unlikely(rq->scx.kick_sync_pending)) {
3395	rq->scx.kick_sync_pending = false;
3396	queue_balance_callback(rq, &rq->scx.kick_sync_bal_cb,
3397	kick_sync_wait_bal_cb);
3398	}
3399	} else if (unlikely(rq->scx.flags & SCX_RQ_BAL_CB_PENDING)) {
3400	/*
3401	* Balance callbacks must run in the context that queued them,
3402	* so they can't be queued on another CPU's rq. Run the deferred
3403	* work directly instead.
3404	*/
3405	rq->scx.flags &= ~SCX_RQ_BAL_CB_PENDING;
3406	run_deferred(rq);
3407	}
3408
3409	rq_repin_lock(rq, rf);
3410
3411	/* if dispatch_one() released the rq lock, restart the selection */
3412	if (rq->scx.lock_drop_seq != seq)
3413	return SCX_DSP_RETRY;
3414
3415	return verdict;
3416	}
3417	#else /* CONFIG_SCHED_CORE */
3418	static enum scx_dsp_verdict dispatch_core_pick(struct rq rq, struct rq_flags rf,
3419	struct task_struct *prev)
3420	{
3421	return SCX_DSP_NONE;
3422	}
3423	#endif /* CONFIG_SCHED_CORE */
3424
3425	static struct task_struct *
3426	do_pick_task_scx(struct rq rq, struct rq_flags rf, bool force_scx)
3427	{
3428	struct task_struct *prev = rq->curr;
3429	enum scx_dsp_verdict verdict;
3430	struct task_struct *p;
3431
3432	/* see kick_sync_wait_bal_cb() */
3433	smp_store_release(&rq->scx.kick_sync, rq->scx.kick_sync + 1);
3434
3435	rq_modified_begin(rq, &ext_sched_class);
3436
3437	if (sched_core_enabled(rq))
3438	verdict = dispatch_core_pick(rq, rf, prev);
3439	else
3440	verdict = dispatch_pick(rq, rf, prev);
3441
3442	if (verdict == SCX_DSP_RETRY)
3443	return RETRY_TASK;
3444
3445	/*
3446	* If any higher-priority sched class enqueued a runnable task on this
3447	* rq during dispatch_one(), abort and return RETRY_TASK, so that the
3448	* scheduler loop can restart.
3449	*
3450	* If @force_scx is true, always try to pick a SCHED_EXT task,
3451	* regardless of any higher-priority sched classes activity.
3452	*/
3453	if (!force_scx && rq_modified_above(rq, &ext_sched_class))
3454	return RETRY_TASK;
3455
3456	/*
3457	* If we're keeping @prev, replenish slice if necessary and keep running
3458	* @prev. Otherwise, pop the first one from the local DSQ.
3459	*/
3460	if (verdict == SCX_DSP_PREV) {
3461	p = prev;
3462	if (!p->scx.slice) {
3463	/* the slice is consumed, protection ends */
3464	scx_task_slice_ended(rq, p);
3465	refill_task_slice_dfl(scx_task_sched(p), p);
3466	}
3467	} else {
3468	p = first_local_task(rq);
3469	if (!p)
3470	return NULL;
3471
3472	if (unlikely(!p->scx.slice) && scx_task_can_stay_on_cpu(rq, p)) {
3473	struct scx_sched *sch = scx_task_sched(p);
3474
3475	if (!scx_bypassing(sch, cpu_of(rq)) &&
3476	!sch->warned_zero_slice) {
3477	printk_deferred(KERN_WARNING "sched_ext: %s[%d] has zero slice in %s()\n",
3478	p->comm, p->pid, __func__);
3479	sch->warned_zero_slice = true;
3480	}
3481	refill_task_slice_dfl(sch, p);
3482	}
3483	}
3484
3485	return p;
3486	}
3487
3488	static struct task_struct pick_task_scx(struct rq rq, struct rq_flags *rf)
3489	{
3490	return do_pick_task_scx(rq, rf, false);
3491	}
3492
3493	/*
3494	* Select the next task to run from the ext scheduling class.
3495	*
3496	* Use do_pick_task_scx() directly with @force_scx enabled, since the
3497	* dl_server must always select a sched_ext task.
3498	*/
3499	static struct task_struct *
3500	ext_server_pick_task(struct sched_dl_entity dl_se, struct rq_flags rf)
3501	{
3502	if (!scx_enabled())
3503	return NULL;
3504
3505	return do_pick_task_scx(dl_se->rq, rf, true);
3506	}
3507
3508	/*
3509	* Initialize the ext server deadline entity.
3510	*/
3511	void ext_server_init(struct rq *rq)
3512	{
3513	struct sched_dl_entity *dl_se = &rq->ext_server;
3514
3515	init_dl_entity(dl_se);
3516
3517	dl_server_init(dl_se, rq, ext_server_pick_task);
3518	}
3519
3520	#ifdef CONFIG_SCHED_CORE
3521	/**
3522	* scx_prio_less - Task ordering for core-sched
3523	* @a: task A
3524	* @b: task B
3525	* @in_fi: in forced idle state
3526	*
3527	* Core-sched is implemented as an additional scheduling layer on top of the
3528	* usual sched_class'es and needs to find out the expected task ordering. For
3529	* SCX, core-sched calls this function to interrogate the task ordering.
3530	*
3531	* A pair of tasks owned by one scheduler is ordered by the owner's
3532	* ops.core_sched_before(). A pair spanning two schedulers is ordered by their
3533	* nearest common ancestor which implements the op - the one case where the op
3534	* is called on tasks that the scheduler delegated to its sub-schedulers and may
3535	* not be scheduling anymore.
3536	*
3537	* When neither applies, or the deciding scheduler is bypassing on either task's
3538	* CPU, the default ordering runs the task which has been waiting longer first.
3539	* A running task counts as the most recently serviced and orders after every
3540	* waiting task. Waiting tasks are compared by @p->scx.runnable_at.
3541	*
3542	* Return: %true if @a should run after @b.
3543	*/
3544	bool scx_prio_less(const struct task_struct a, const struct task_struct b,
3545	bool in_fi)
3546	{
3547	struct scx_sched *sch_a = scx_task_sched(a);
3548	struct scx_sched *sch_b = scx_task_sched(b);
3549	struct scx_sched *sch = NULL;
3550	bool a_running, b_running;
3551
3552	if (sch_a == sch_b) {
3553	if (SCX_HAS_OP(sch_a, core_sched_before))
3554	sch = sch_a;
3555	} else {
3556	s32 level;
3557
3558	for (level = min(sch_a->level, sch_b->level); level >= 0; level--) {
3559	struct scx_sched *anc = sch_a->ancestors[level];
3560
3561	if (anc == sch_b->ancestors[level] &&
3562	SCX_HAS_OP(anc, core_sched_before)) {
3563	sch = anc;
3564	break;
3565	}
3566	}
3567	}
3568
3569	/*
3570	* scx_prio_less() returns whether @a should run after @b while
3571	* ops.core_sched_before() returns whether its first argument should run
3572	* before the second. Swap the arguments.
3573	*
3574	* The const qualifiers are dropped from task_struct pointers when
3575	* calling ops.core_sched_before(). Accesses are controlled by the
3576	* verifier.
3577	*/
3578	if (sch && !scx_bypassing(sch, task_cpu(a)) && !scx_bypassing(sch, task_cpu(b)))
3579	return SCX_CALL_OP_2TASKS_RET(sch, core_sched_before, task_rq(a),
3580	(struct task_struct *)b,
3581	(struct task_struct *)a);
3582
3583	/*
3584	* runnable_at is refreshed only on enqueue, so a task which keeps
3585	* occupying its CPU carries a stale stamp. A running task is the most
3586	* recently serviced whatever its stamp says. Order it after every
3587	* waiting task.
3588	*/
3589	a_running = a->on_cpu;
3590	b_running = b->on_cpu;
3591	if (a_running != b_running)
3592	return a_running;
3593
3594	return time_after(a->scx.runnable_at, b->scx.runnable_at);
3595	}
3596	#endif /* CONFIG_SCHED_CORE */
3597
3598	static int select_task_rq_scx(struct task_struct *p, int prev_cpu, int wake_flags)
3599	{
3600	struct scx_sched *sch = scx_task_sched(p);
3601	bool bypassing;
3602
3603	/*
3604	* sched_exec() calls with %WF_EXEC when @p is about to exec(2) as it
3605	* can be a good migration opportunity with low cache and memory
3606	* footprint. Returning a CPU different than @prev_cpu triggers
3607	* immediate rq migration. However, for SCX, as the current rq
3608	* association doesn't dictate where the task is going to run, this
3609	* doesn't fit well. If necessary, we can later add a dedicated method
3610	* which can decide to preempt self to force it through the regular
3611	* scheduling path.
3612	*/
3613	if (unlikely(wake_flags & WF_EXEC))
3614	return prev_cpu;
3615
3616	bypassing = scx_bypassing(sch, task_cpu(p));
3617	if (likely(SCX_HAS_OP(sch, select_cpu)) && !bypassing) {
3618	s32 cpu;
3619	struct task_struct **ddsp_taskp;
3620
3621	ddsp_taskp = this_cpu_ptr(&direct_dispatch_task);
3622	WARN_ON_ONCE(*ddsp_taskp);
3623	*ddsp_taskp = p;
3624
3625	this_rq()->scx.in_select_cpu = true;
3626	cpu = SCX_CALL_OP_TASK_RET(sch, select_cpu, NULL, p,
3627	scx_cpu_arg(prev_cpu), wake_flags);
3628	cpu = scx_cpu_ret(sch, cpu);
3629	this_rq()->scx.in_select_cpu = false;
3630	p->scx.selected_cpu = cpu;
3631	*ddsp_taskp = NULL;
3632	if (scx_cpu_valid(sch, cpu, "from ops.select_cpu()"))
3633	return cpu;
3634	else
3635	return prev_cpu;
3636	} else {
3637	s32 cpu;
3638
3639	/*
3640	* While bypassing, the enqueue path routes @p to a bypass DSQ
3641	* without consulting the direct-dispatch target, making the
3642	* default selection pointless. It doesn't work anyway when the
3643	* scheduler does its own idle tracking and the built-in idle
3644	* cpumasks are not updated. Leave @p on @prev_cpu.
3645	*/
3646	if (bypassing) {
3647	__scx_add_event(sch, SCX_EV_BYPASS_DISPATCH, 1);
3648	p->scx.selected_cpu = prev_cpu;
3649	return prev_cpu;
3650	}
3651
3652	cpu = scx_select_cpu_dfl(p, prev_cpu, wake_flags, NULL, 0);
3653	if (cpu >= 0) {
3654	/*
3655	* Carry the slice refill and let the insertion commit
3656	* it under rq lock. See the write rules.
3657	*/
3658	__scx_add_event(sch, SCX_EV_REFILL_SLICE_DFL, 1);
3659	p->scx.ddsp_slice = READ_ONCE(sch->slice_dfl);
3660	p->scx.ddsp_enq_flags = SCX_ENQ_SLICE_DFL;
3661	p->scx.ddsp_dsq_id = SCX_DSQ_LOCAL;
3662	} else {
3663	cpu = prev_cpu;
3664	}
3665	p->scx.selected_cpu = cpu;
3666
3667	return cpu;
3668	}
3669	}
3670
3671	static void task_woken_scx(struct rq rq, struct task_struct p)
3672	{
3673	run_deferred(rq);
3674	}
3675
3676	static void set_cpus_allowed_scx(struct task_struct *p,
3677	struct affinity_context *ac)
3678	{
3679	struct scx_sched *sch = scx_task_sched(p);
3680
3681	set_cpus_allowed_common(p, ac);
3682
3683	if (task_dead_and_done(p))
3684	return;
3685
3686	/*
3687	* The effective cpumask is stored in @p->cpus_ptr which may temporarily
3688	* differ from the configured one in @p->cpus_mask. Always tell the bpf
3689	* scheduler the effective one.
3690	*
3691	* Fine-grained memory write control is enforced by BPF making the const
3692	* designation pointless. Cast it away when calling the operation.
3693	*
3694	* The cid form receives the initial mask when the task is enabled and
3695	* hears about changes only afterwards, see struct scx_enable_args.
3696	*/
3697	if (SCX_HAS_OP(sch, set_cpumask) &&
3698	(!scx_is_cid_type() \|\| scx_get_task_state(p) == SCX_TASK_ENABLED))
3699	scx_call_op_set_cpumask(sch, task_rq(p), p, (struct cpumask *)p->cpus_ptr);
3700	}
3701
3702	static void handle_hotplug(struct rq *rq, bool online)
3703	{
3704	struct scx_sched *sch = scx_root_protected();
3705	s32 cpu = cpu_of(rq);
3706	s32 cpu_or_cid = cpu;
3707
3708	atomic_long_inc(&scx_hotplug_seq);
3709
3710	/*
3711	* scx_root updates are protected by cpus_read_lock() and will stay
3712	* stable here. Note that we can't depend on scx_enabled() test as the
3713	* hotplug ops need to be enabled before __scx_enabled is set.
3714	*/
3715	if (unlikely(!sch))
3716	return;
3717
3718	if (scx_enabled())
3719	scx_idle_update_selcpu_topology(&sch->ops);
3720
3721	if (online)
3722	scx_online_ecaps(rq);
3723	else
3724	scx_offline_ecaps(rq);
3725
3726	/*
3727	* The tables can't be retired while this function is running as the
3728	* retirement is inside cpus_read_lock. However, scx_cpu_arg() is
3729	* awkward here as the tables can be NULL after root enable failure and
3730	* lockdep would trigger without surrounding rcu_read_lock(). Open code
3731	* the translation. If the table is NULL, the ops are also cleared and
3732	* @cpu_or_cid goes unused.
3733	*/
3734	if (scx_is_cid_type()) {
3735	s16 *tbl = rcu_dereference_check(scx_cpu_to_cid_tbl,
3736	lockdep_is_cpus_held());
3737
3738	if (tbl) {
3739	struct scx_sched *pos;
3740
3741	cpu_or_cid = tbl[cpu];
3742
3743	guard(raw_spinlock_irqsave)(&scx_sched_lock);
3744	list_for_each_entry(pos, &scx_sched_all, all) {
3745	struct scx_cmask *mask = pos->online_cmask;
3746
3747	if (mask)
3748	__assign_bit(cpu_or_cid, (unsigned long *)mask->bits,
3749	online);
3750	}
3751	}
3752	}
3753
3754	if (online && SCX_HAS_OP(sch, cpu_online))
3755	SCX_CALL_OP(sch, cpu_online, NULL, cpu_or_cid);
3756	else if (!online && SCX_HAS_OP(sch, cpu_offline))
3757	SCX_CALL_OP(sch, cpu_offline, NULL, cpu_or_cid);
3758	else
3759	scx_exit(sch, SCX_EXIT_UNREG_KERN,
3760	SCX_ECODE_ACT_RESTART \| SCX_ECODE_RSN_HOTPLUG,
3761	"cpu %d going %s, exiting scheduler", cpu,
3762	online ? "online" : "offline");
3763	}
3764
3765	void scx_rq_activate(struct rq *rq)
3766	{
3767	handle_hotplug(rq, true);
3768	}
3769
3770	void scx_rq_deactivate(struct rq *rq)
3771	{
3772	handle_hotplug(rq, false);
3773	}
3774
3775	static void rq_online_scx(struct rq *rq)
3776	{
3777	rq->scx.flags \|= SCX_RQ_ONLINE;
3778	}
3779
3780	static void rq_offline_scx(struct rq *rq)
3781	{
3782	struct task_struct p, n;
3783
3784	rq->scx.flags &= ~SCX_RQ_ONLINE;
3785
3786	/* sched domain rebuilds call rq_offline with the CPU staying alive */
3787	if (cpu_active(cpu_of(rq)))
3788	return;
3789
3790	scx_rescue_flush(rq);
3791
3792	/*
3793	* An offline CPU no longer calls ops.dispatch(). Re-enqueue its tasks
3794	* onto the local DSQ so that they run here and balance_push() moves
3795	* them off.
3796	*/
3797	list_for_each_entry_safe_reverse(p, n, &rq->scx.runnable_list, scx.runnable_node) {
3798	if (p->scx.dsq == &rq->scx.local_dsq)
3799	continue;
3800	guard(sched_change)(p, DEQUEUE_SAVE \| DEQUEUE_MOVE \| DEQUEUE_NOCLOCK);
3801	}
3802	}
3803
3804	static bool check_rq_for_timeouts(struct rq *rq)
3805	{
3806	struct scx_sched *sch;
3807	struct task_struct *p;
3808	struct rq_flags rf;
3809	bool timed_out = false;
3810
3811	rq_lock_irqsave(rq, &rf);
3812	sch = rcu_dereference_bh(scx_root);
3813	if (unlikely(!sch))
3814	goto out_unlock;
3815
3816	list_for_each_entry(p, &rq->scx.runnable_list, scx.runnable_node) {
3817	struct scx_sched *sch = scx_task_sched(p);
3818	unsigned long last_runnable = p->scx.runnable_at;
3819
3820	if (unlikely(time_after(jiffies,
3821	last_runnable + READ_ONCE(sch->watchdog_timeout)))) {
3822	struct scx_dispatch_q *dsq = READ_ONCE(p->scx.dsq);
3823	u32 dur_ms = jiffies_to_msecs(jiffies - last_runnable);
3824
3825	/*
3826	* A task can be stuck on a DSQ that a sched other than
3827	* its owner is responsible for draining, e.g. an
3828	* ancestor's bypass DSQ while the owner is bypassing.
3829	* Blame the drainer. The local DSQ is consumed by the
3830	* cpu itself and keeps blame on the owner.
3831	*/
3832	if (dsq && dsq->sched && dsq->id != SCX_DSQ_LOCAL)
3833	sch = dsq->sched;
3834
3835	__scx_exit(sch, SCX_EXIT_ERROR_STALL, 0, cpu_of(rq),
3836	"%s[%d] failed to run for %u.%03us",
3837	p->comm, p->pid, dur_ms / 1000,
3838	dur_ms % 1000);
3839	timed_out = true;
3840	break;
3841	}
3842	}
3843	out_unlock:
3844	rq_unlock_irqrestore(rq, &rf);
3845	return timed_out;
3846	}
3847
3848	static void scx_watchdog_workfn(struct work_struct *work)
3849	{
3850	unsigned long intv;
3851	int cpu;
3852
3853	WRITE_ONCE(scx_watchdog_timestamp, jiffies);
3854
3855	for_each_online_cpu(cpu) {
3856	if (unlikely(check_rq_for_timeouts(cpu_rq(cpu))))
3857	break;
3858
3859	cond_resched();
3860	}
3861
3862	intv = READ_ONCE(scx_watchdog_interval);
3863	if (intv < ULONG_MAX)
3864	queue_delayed_work(system_dfl_wq, to_delayed_work(work), intv);
3865	}
3866
3867	void scx_tick(struct rq *rq)
3868	{
3869	struct scx_sched *root;
3870	unsigned long last_check;
3871
3872	if (!scx_enabled())
3873	return;
3874
3875	root = rcu_dereference_bh(scx_root);
3876	if (unlikely(!root))
3877	return;
3878
3879	last_check = READ_ONCE(scx_watchdog_timestamp);
3880	if (unlikely(time_after(jiffies,
3881	last_check + READ_ONCE(root->watchdog_timeout)))) {
3882	u32 dur_ms = jiffies_to_msecs(jiffies - last_check);
3883
3884	scx_exit(root, SCX_EXIT_ERROR_STALL, 0,
3885	"watchdog failed to check in for %u.%03us",
3886	dur_ms / 1000, dur_ms % 1000);
3887	}
3888
3889	update_other_load_avgs(rq);
3890	}
3891
3892	static void task_tick_scx(struct rq rq, struct task_struct curr, int queued)
3893	{
3894	struct scx_sched *sch = scx_task_sched(curr);
3895
3896	update_curr_scx(rq);
3897
3898	/*
3899	* While disabling, always resched as we can't trust the slice
3900	* management.
3901	*/
3902	if (scx_bypassing(sch, cpu_of(rq)))
3903	scx_set_task_slice(curr, 0);
3904	else if (SCX_HAS_OP(sch, tick))
3905	SCX_CALL_OP_TASK(sch, tick, rq, curr);
3906
3907	if (!curr->scx.slice)
3908	resched_curr(rq);
3909	}
3910
3911	#ifdef CONFIG_EXT_GROUP_SCHED
3912	static struct cgroup tg_cgrp(struct task_group tg)
3913	{
3914	/*
3915	* If CGROUP_SCHED is disabled, @tg is NULL. If @tg is an autogroup,
3916	* @tg->css.cgroup is NULL. In both cases, @tg can be treated as the
3917	* root cgroup.
3918	*/
3919	if (tg && tg->css.cgroup)
3920	return tg->css.cgroup;
3921	else
3922	return &cgrp_dfl_root.cgrp;
3923	}
3924
3925	#define SCX_INIT_TASK_ARGS_CGROUP(cgrp) .cgroup = (cgrp),
3926
3927	#else /* CONFIG_EXT_GROUP_SCHED */
3928
3929	#define SCX_INIT_TASK_ARGS_CGROUP(cgrp)
3930
3931	#endif /* CONFIG_EXT_GROUP_SCHED */
3932
3933	/**
3934	* __scx_init_task - Initialize a task for a sched
3935	* @sch: sched to initialize @p for
3936	* @p: task of interest
3937	* @cgrp: cgroup @p is joining, %NULL for @p's current task_group's cgroup
3938	* @fork: %true if @p is being forked
3939	*
3940	* Pre-commit cgroup migration passes @cgrp explicitly as @p's task_group
3941	* still reflects the source.
3942	*
3943	* Return 0 on success, -errno on failure.
3944	*/
3945	int __scx_init_task(struct scx_sched sch, struct task_struct p,
3946	struct cgroup *cgrp, bool fork)
3947	{
3948	int ret;
3949
3950	p->scx.disallow = false;
3951
3952	if (SCX_HAS_OP(sch, init_task)) {
3953	struct scx_init_task_args args = {
3954	SCX_INIT_TASK_ARGS_CGROUP(cgrp ?: tg_cgrp(task_group(p)))
3955	.fork = fork,
3956	};
3957
3958	ret = SCX_CALL_OP_RET(sch, init_task, NULL, p, &args);
3959	if (unlikely(ret)) {
3960	ret = scx_ops_sanitize_err(sch, "init_task", ret);
3961	return ret;
3962	}
3963	}
3964
3965	if (p->scx.disallow) {
3966	if (unlikely(scx_parent(sch))) {
3967	scx_error(sch, "non-root ops.init_task() set task->scx.disallow for %s[%d]",
3968	p->comm, p->pid);
3969	} else if (unlikely(fork)) {
3970	scx_error(sch, "ops.init_task() set task->scx.disallow for %s[%d] during fork",
3971	p->comm, p->pid);
3972	} else if (unlikely(scx_enable_state() != SCX_ENABLING)) {
3973	scx_error(sch, "ops.init_task() set task->scx.disallow for %s[%d] outside the enable path",
3974	p->comm, p->pid);
3975	} else {
3976	struct rq *rq;
3977	struct rq_flags rf;
3978
3979	rq = task_rq_lock(p, &rf);
3980
3981	/*
3982	* We're in the load path and @p->policy will be applied
3983	* right after. Reverting @p->policy here and rejecting
3984	* %SCHED_EXT transitions from scx_check_setscheduler()
3985	* guarantees that if ops.init_task() sets @p->disallow,
3986	* @p can never be in SCX.
3987	*/
3988	if (p->policy == SCHED_EXT) {
3989	p->policy = SCHED_NORMAL;
3990	atomic_long_inc(&scx_nr_rejected);
3991	}
3992
3993	task_rq_unlock(rq, p, &rf);
3994	}
3995	}
3996
3997	return 0;
3998	}
3999
4000	static void __scx_enable_task(struct scx_sched sch, struct task_struct p)
4001	{
4002	struct rq *rq = task_rq(p);
4003	u32 weight;
4004
4005	lockdep_assert_rq_held(rq);
4006
4007	/*
4008	* Verify the task is not in BPF scheduler's custody. If flag
4009	* transitions are consistent, the flag should always be clear
4010	* here.
4011	*/
4012	WARN_ON_ONCE(p->scx.flags & SCX_TASK_IN_CUSTODY);
4013
4014	/*
4015	* Set the weight before calling ops.enable() so that the scheduler
4016	* doesn't see a stale value if they inspect the task struct.
4017	*/
4018	if (task_has_idle_policy(p))
4019	weight = WEIGHT_IDLEPRIO;
4020	else
4021	weight = sched_prio_to_weight[p->static_prio - MAX_RT_PRIO];
4022
4023	p->scx.weight = sched_weight_to_cgroup(weight);
4024
4025	if (SCX_HAS_OP(sch, enable)) {
4026	if (scx_is_cid_type()) {
4027	struct scx_cmask *cmask = scx_fill_cmask_scratch(sch, p->cpus_ptr);
4028	struct scx_enable_args args = {
4029	.cmask_arena_addr = scx_kaddr_to_arena(sch, cmask),
4030	};
4031
4032	SCX_CALL_CID_OP_TASK(sch, enable, rq, p, &args);
4033	} else {
4034	SCX_CALL_OP_TASK(sch, enable, rq, p);
4035	}
4036	}
4037
4038	/*
4039	* The initial mask also goes out through set_cmask() so a scheduler can
4040	* track affinity there alone, and before set_weight() so that the mask
4041	* is in place when weight-dependent state is derived, see struct
4042	* scx_enable_args.
4043	*/
4044	if (scx_is_cid_type() && SCX_HAS_OP(sch, set_cpumask))
4045	scx_call_op_set_cpumask(sch, rq, p, p->cpus_ptr);
4046
4047	if (SCX_HAS_OP(sch, set_weight))
4048	SCX_CALL_OP_TASK(sch, set_weight, rq, p, p->scx.weight);
4049	}
4050
4051	void scx_enable_task(struct scx_sched sch, struct task_struct p)
4052	{
4053	__scx_enable_task(sch, p);
4054	scx_set_task_state(p, SCX_TASK_ENABLED);
4055	}
4056
4057	static void scx_disable_task(struct scx_sched sch, struct task_struct p)
4058	{
4059	struct rq *rq = task_rq(p);
4060
4061	lockdep_assert_rq_held(rq);
4062	WARN_ON_ONCE(scx_get_task_state(p) != SCX_TASK_ENABLED);
4063
4064	clear_direct_dispatch(p);
4065
4066	if (SCX_HAS_OP(sch, disable))
4067	SCX_CALL_OP_TASK(sch, disable, rq, p);
4068	scx_set_task_state(p, SCX_TASK_READY);
4069
4070	/*
4071	* Reset the SCX-managed fields when @p leaves the BPF scheduler's
4072	* control, after ops.disable() has observed their final values.
4073	*/
4074	p->scx.dsq_vtime = 0;
4075	scx_task_slice_ended(rq, p);
4076	scx_set_task_slice(p, 0);
4077	p->scx.reenq_cnt = 0;
4078
4079	/*
4080	* Verify the task is not in BPF scheduler's custody. If flag
4081	* transitions are consistent, the flag should always be clear
4082	* here.
4083	*/
4084	WARN_ON_ONCE(p->scx.flags & SCX_TASK_IN_CUSTODY);
4085	}
4086
4087	void __scx_disable_and_exit_task(struct scx_sched sch, struct task_struct p)
4088	{
4089	struct scx_exit_task_args args = {
4090	.cancelled = false,
4091	};
4092
4093	lockdep_assert_held(&p->pi_lock);
4094	lockdep_assert_rq_held(task_rq(p));
4095
4096	switch (scx_get_task_state(p)) {
4097	case SCX_TASK_NONE:
4098	return;
4099	case SCX_TASK_INIT:
4100	args.cancelled = true;
4101	break;
4102	case SCX_TASK_READY:
4103	break;
4104	case SCX_TASK_ENABLED:
4105	scx_disable_task(sch, p);
4106	break;
4107	default:
4108	WARN_ON_ONCE(true);
4109	return;
4110	}
4111
4112	if (SCX_HAS_OP(sch, exit_task))
4113	SCX_CALL_OP_TASK(sch, exit_task, task_rq(p), p, &args);
4114	}
4115
4116	/*
4117	* Undo a completed __scx_init_task(sch, p, false) when scx_enable_task() never
4118	* ran. The task state has not been transitioned, so this mirrors the
4119	* SCX_TASK_INIT branch in __scx_disable_and_exit_task().
4120	*/
4121	void scx_sub_init_cancel_task(struct scx_sched sch, struct task_struct p)
4122	{
4123	struct scx_exit_task_args args = { .cancelled = true };
4124
4125	lockdep_assert_held(&p->pi_lock);
4126	lockdep_assert_rq_held(task_rq(p));
4127
4128	/* @p was never associated with @sch, dispatch on the explicit @sch */
4129	if (SCX_HAS_OP(sch, exit_task))
4130	__SCX_CALL_OP_TASK(sch, ops, exit_task, task_rq(p), p, &args);
4131	}
4132
4133	void scx_disable_and_exit_task(struct scx_sched sch, struct task_struct p)
4134	{
4135	__scx_disable_and_exit_task(sch, p);
4136
4137	/*
4138	* If set, @p exited between __scx_init_task() and scx_enable_task() in
4139	* scx_sub_enable() and is initialized for both the associated sched and
4140	* its parent. Exit for the child too - scx_enable_task() never ran for
4141	* it, so undo only init_task. The flag is only set on the sub-enable
4142	* path, so it's always clear when @p arrives here in %SCX_TASK_NONE.
4143	*/
4144	if (p->scx.flags & SCX_TASK_SUB_INIT) {
4145	if (!WARN_ON_ONCE(!scx_enabling_sub_sched))
4146	scx_sub_init_cancel_task(scx_enabling_sub_sched, p);
4147	p->scx.flags &= ~SCX_TASK_SUB_INIT;
4148	}
4149
4150	scx_set_task_sched(p, NULL);
4151	scx_set_task_state(p, SCX_TASK_NONE);
4152	}
4153
4154	void init_scx_entity(struct sched_ext_entity *scx)
4155	{
4156	memset(scx, 0, sizeof(*scx));
4157	INIT_LIST_HEAD(&scx->dsq_list.node);
4158	RB_CLEAR_NODE(&scx->dsq_priq);
4159	scx->sticky_cpu = -1;
4160	scx->holding_cpu = -1;
4161	scx->runnable_cpu = -1;
4162	INIT_LIST_HEAD(&scx->runnable_node);
4163	scx->runnable_at = jiffies;
4164	scx->ddsp_dsq_id = SCX_DSQ_INVALID;
4165	scx->slice = SCX_SLICE_DFL;
4166	}
4167
4168	/* See scx_tid_alloc / scx_tid_cursor. */
4169	static u64 scx_alloc_tid(void)
4170	{
4171	struct scx_tid_alloc *ta;
4172
4173	guard(preempt)();
4174	ta = this_cpu_ptr(&scx_tid_alloc);
4175
4176	if (unlikely(ta->next >= ta->end)) {
4177	ta->next = atomic64_fetch_add(SCX_TID_CHUNK, &scx_tid_cursor);
4178	ta->end = ta->next + SCX_TID_CHUNK;
4179	}
4180	return ta->next++;
4181	}
4182
4183	static void scx_tid_hash_insert(struct task_struct *p)
4184	{
4185	int ret;
4186
4187	lockdep_assert_held(&scx_tasks_lock);
4188
4189	ret = rhashtable_lookup_insert_fast(&scx_tid_hash,
4190	&p->scx.tid_hash_node,
4191	scx_tid_hash_params);
4192	WARN_ON_ONCE(ret);
4193	}
4194
4195	void scx_pre_fork(struct task_struct *p)
4196	{
4197	/*
4198	* BPF scheduler enable/disable paths want to be able to iterate and
4199	* update all tasks which can become complex when racing forks. As
4200	* enable/disable are very cold paths, let's use a percpu_rwsem to
4201	* exclude forks.
4202	*/
4203	percpu_down_read(&scx_fork_rwsem);
4204	}
4205
4206	int scx_fork(struct task_struct p, struct kernel_clone_args kargs)
4207	{
4208	s32 ret;
4209
4210	percpu_rwsem_assert_held(&scx_fork_rwsem);
4211
4212	p->scx.tid = scx_alloc_tid();
4213
4214	if (scx_init_task_enabled) {
4215	#ifdef CONFIG_EXT_SUB_SCHED
4216	struct scx_sched *sch = scx_cgroup_sched(kargs->cset->dfl_cgrp);
4217	#else
4218	struct scx_sched *sch = scx_root_protected_live();
4219	#endif
4220	scx_set_task_state(p, SCX_TASK_INIT_BEGIN);
4221	ret = __scx_init_task(sch, p, NULL, true);
4222	if (unlikely(ret)) {
4223	scx_set_task_state(p, SCX_TASK_NONE);
4224	return ret;
4225	}
4226	scx_set_task_state(p, SCX_TASK_INIT);
4227	scx_set_task_sched(p, sch);
4228	}
4229
4230	return 0;
4231	}
4232
4233	void scx_post_fork(struct task_struct *p)
4234	{
4235	if (scx_init_task_enabled) {
4236	scx_set_task_state(p, SCX_TASK_READY);
4237
4238	/*
4239	* Enable the task immediately if it's running on sched_ext.
4240	* Otherwise, it'll be enabled in switching_to_scx() if and
4241	* when it's ever configured to run with a SCHED_EXT policy.
4242	*/
4243	if (p->sched_class == &ext_sched_class) {
4244	struct rq_flags rf;
4245	struct rq *rq;
4246
4247	rq = task_rq_lock(p, &rf);
4248	scx_enable_task(scx_task_sched(p), p);
4249	task_rq_unlock(rq, p, &rf);
4250	}
4251	}
4252
4253	scoped_guard(raw_spinlock_irq, &scx_tasks_lock) {
4254	list_add_tail(&p->scx.tasks_node, &scx_tasks);
4255	if (scx_tid_to_task_enabled())
4256	scx_tid_hash_insert(p);
4257	}
4258
4259	percpu_up_read(&scx_fork_rwsem);
4260	}
4261
4262	void scx_cancel_fork(struct task_struct *p)
4263	{
4264	if (scx_init_task_enabled) {
4265	struct rq *rq;
4266	struct rq_flags rf;
4267
4268	rq = task_rq_lock(p, &rf);
4269	WARN_ON_ONCE(scx_get_task_state(p) >= SCX_TASK_READY);
4270	scx_disable_and_exit_task(scx_task_sched(p), p);
4271	task_rq_unlock(rq, p, &rf);
4272	}
4273
4274	percpu_up_read(&scx_fork_rwsem);
4275	}
4276
4277	/**
4278	* task_dead_and_done - Is a task dead and done running?
4279	* @p: target task
4280	*
4281	* Once sched_ext_dead() removes the dead task from scx_tasks and exits it, the
4282	* task no longer exists from SCX's POV. However, certain sched_class ops may be
4283	* invoked on these dead tasks leading to failures - e.g. sched_setscheduler()
4284	* may try to switch a task which finished sched_ext_dead() back into SCX
4285	* triggering invalid SCX task state transitions and worse.
4286	*
4287	* Once a task has finished the final switch, sched_ext_dead() is the only thing
4288	* that needs to happen on the task. Use this test to short-circuit sched_class
4289	* operations which may be called on dead tasks.
4290	*/
4291	static bool task_dead_and_done(struct task_struct *p)
4292	{
4293	struct rq *rq = task_rq(p);
4294
4295	lockdep_assert_rq_held(rq);
4296
4297	/*
4298	* In do_task_dead(), a dying task sets %TASK_DEAD with preemption
4299	* disabled and __schedule(). If @p has %TASK_DEAD set and off CPU, @p
4300	* won't ever run again.
4301	*/
4302	return unlikely(READ_ONCE(p->__state) == TASK_DEAD) &&
4303	!task_on_cpu(rq, p);
4304	}
4305
4306	void sched_ext_dead(struct task_struct *p)
4307	{
4308	/*
4309	* By the time control reaches here, @p has %TASK_DEAD set, switched out
4310	* for the last time and then dropped the rq lock - task_dead_and_done()
4311	* should be returning %true nullifying the straggling sched_class ops.
4312	* Remove from scx_tasks and exit @p.
4313	*/
4314	scoped_guard(raw_spinlock_irqsave, &scx_tasks_lock) {
4315	list_del_init(&p->scx.tasks_node);
4316	if (scx_tid_to_task_enabled())
4317	rhashtable_remove_fast(&scx_tid_hash,
4318	&p->scx.tid_hash_node,
4319	scx_tid_hash_params);
4320	}
4321
4322	/*
4323	* @p is off scx_tasks and wholly ours. scx_root_enable()'s READY ->
4324	* ENABLED transitions can't race us. Disable ops for @p.
4325	*
4326	* %SCX_TASK_DEAD synchronizes against cgroup task iteration - see
4327	* scx_task_iter_next_locked(). NONE tasks need no marking: cgroup
4328	* iteration is only used from sub-sched paths, which require root
4329	* enabled. Root enable transitions every live task to at least READY.
4330	*
4331	* %INIT_BEGIN means ops.init_task() is running for @p. Don't call
4332	* into ops; transition to %DEAD so the post-init recheck unwinds
4333	* via scx_sub_init_cancel_task().
4334	*/
4335	if (scx_get_task_state(p) != SCX_TASK_NONE) {
4336	struct rq_flags rf;
4337	struct rq *rq;
4338
4339	rq = task_rq_lock(p, &rf);
4340	if (scx_get_task_state(p) != SCX_TASK_INIT_BEGIN)
4341	scx_disable_and_exit_task(scx_task_sched(p), p);
4342	scx_set_task_state(p, SCX_TASK_DEAD);
4343	task_rq_unlock(rq, p, &rf);
4344	}
4345	}
4346
4347	static void reweight_task_scx(struct rq rq, struct task_struct p,
4348	const struct load_weight *lw)
4349	{
4350	struct scx_sched *sch = scx_task_sched(p);
4351
4352	lockdep_assert_rq_held(task_rq(p));
4353
4354	if (task_dead_and_done(p))
4355	return;
4356
4357	/*
4358	* When switching sched_class away from SCX, reweight_task_scx()
4359	* is called _after_ scx_disable_task(). Skip calling ops.set_weight()
4360	* since the BPF scheduler may have already forgotten the task in
4361	* ops.disable().
4362	* p->scx.weight will be recalculated in scx_enable_task() if the task
4363	* ever returns to SCX class.
4364	*/
4365	if (scx_get_task_state(p) != SCX_TASK_ENABLED)
4366	return;
4367
4368	p->scx.weight = sched_weight_to_cgroup(scale_load_down(lw->weight));
4369	if (SCX_HAS_OP(sch, set_weight))
4370	SCX_CALL_OP_TASK(sch, set_weight, rq, p, p->scx.weight);
4371	}
4372
4373	static void prio_changed_scx(struct rq rq, struct task_struct p, u64 oldprio)
4374	{
4375	}
4376
4377	static void switching_to_scx(struct rq rq, struct task_struct p)
4378	{
4379	struct scx_sched *sch = scx_task_sched(p);
4380
4381	if (task_dead_and_done(p))
4382	return;
4383
4384	scx_enable_task(sch, p);
4385
4386	/*
4387	* set_cpus_allowed_scx() is not called while @p is associated with a
4388	* different scheduler class. Keep the BPF scheduler up-to-date. The cid
4389	* form gets its mask from scx_enable_task().
4390	*/
4391	if (!scx_is_cid_type() && SCX_HAS_OP(sch, set_cpumask))
4392	scx_call_op_set_cpumask(sch, rq, p, (struct cpumask *)p->cpus_ptr);
4393	}
4394
4395	static void switched_from_scx(struct rq rq, struct task_struct p)
4396	{
4397	if (task_dead_and_done(p))
4398	return;
4399
4400	/*
4401	* %NONE means SCX is no longer tracking @p at the task level (e.g.
4402	* scx_fail_parent() handed @p back to the parent at NONE pending the
4403	* parent's own teardown). There is nothing to disable; calling
4404	* scx_disable_task() would WARN on the non-%ENABLED state and trigger a
4405	* NONE -> READY validation failure.
4406	*/
4407	if (scx_get_task_state(p) == SCX_TASK_NONE)
4408	return;
4409
4410	scx_disable_task(scx_task_sched(p), p);
4411	}
4412
4413	static void switched_to_scx(struct rq rq, struct task_struct p) {}
4414
4415	int scx_check_setscheduler(struct task_struct *p, int policy)
4416	{
4417	lockdep_assert_rq_held(task_rq(p));
4418
4419	/* if disallow, reject transitioning into SCX */
4420	if (scx_enabled() && READ_ONCE(p->scx.disallow) &&
4421	p->policy != policy && policy == SCHED_EXT)
4422	return -EACCES;
4423
4424	return 0;
4425	}
4426
4427	static void process_ddsp_deferred_locals(struct rq *rq)
4428	{
4429	struct task_struct *p;
4430
4431	lockdep_assert_rq_held(rq);
4432
4433	/*
4434	* Now that @rq can be unlocked, execute the deferred enqueueing of
4435	* tasks directly dispatched to the local DSQs of other CPUs. See
4436	* direct_dispatch(). Keep popping from the head instead of using
4437	* list_for_each_entry_safe() as dispatch_local_dsq() may unlock @rq
4438	* temporarily.
4439	*/
4440	while ((p = list_first_entry_or_null(&rq->scx.ddsp_deferred_locals,
4441	struct task_struct, scx.dsq_list.node))) {
4442	struct scx_sched *sch = scx_task_sched(p);
4443	struct scx_dispatch_q *dsq;
4444	u64 dsq_id = p->scx.ddsp_dsq_id;
4445	u64 enq_flags = p->scx.ddsp_enq_flags;
4446	u64 slice = p->scx.ddsp_slice;
4447	u64 vtime = p->scx.ddsp_vtime;
4448
4449	list_del_init(&p->scx.dsq_list.node);
4450	clear_direct_dispatch(p);
4451
4452	dsq = find_dsq_for_dispatch(sch, rq, dsq_id, task_cpu(p));
4453	if (!WARN_ON_ONCE(dsq->id != SCX_DSQ_LOCAL))
4454	dispatch_to_local_dsq(sch, rq, dsq, p, slice, vtime, enq_flags);
4455	}
4456	}
4457
4458	/*
4459	* Determine whether @p should be reenqueued from a local DSQ.
4460	*
4461	* @reenq_flags is mutable and accumulates state across the DSQ walk:
4462	*
4463	* - %SCX_REENQ_TSR_NOT_FIRST: Set after the first task is visited. "First"
4464	* tracks position in the DSQ list, not among IMMED tasks. A non-IMMED task at
4465	* the head consumes the first slot.
4466	*
4467	* - %SCX_REENQ_TSR_RQ_OPEN: Set by reenq_local() before the walk if
4468	* rq_is_open() is true.
4469	*
4470	* An IMMED task is kept (returns %false) only if it's the first task in the DSQ
4471	* AND the current task is done — i.e. it will execute immediately. All other
4472	* IMMED tasks are reenqueued. This means if a non-IMMED task sits at the head,
4473	* every IMMED task behind it gets reenqueued.
4474	*
4475	* Reenqueued tasks go through ops.enqueue() with %SCX_ENQ_REENQ \|
4476	* %SCX_TASK_REENQ_IMMED. If the BPF scheduler dispatches back to the same local
4477	* DSQ with %SCX_ENQ_IMMED while the CPU is still unavailable, this triggers
4478	* another reenq cycle. Repetitions are bounded by %SCX_REENQ_MAX_REPEAT in
4479	* scx_do_enqueue_task(), which ejects the task's owning scheduler.
4480	*/
4481	static bool local_task_should_reenq(struct rq rq, struct task_struct p,
4482	u64 reenq_flags, u32 reason)
4483	{
4484	bool first;
4485
4486	first = !(*reenq_flags & SCX_REENQ_TSR_NOT_FIRST);
4487	*reenq_flags \|= SCX_REENQ_TSR_NOT_FIRST;
4488
4489	if (unlikely((p->scx.flags & SCX_TASK_PROTECTED) \|\| p == scx_rescuee(rq)))
4490	return false;
4491
4492	*reason = SCX_TASK_REENQ_KFUNC;
4493
4494	if ((p->scx.flags & SCX_TASK_IMMED) &&
4495	(!first \|\| !(*reenq_flags & SCX_REENQ_TSR_RQ_OPEN))) {
4496	__scx_add_event(scx_task_sched(p), SCX_EV_REENQ_IMMED, 1);
4497	*reason = SCX_TASK_REENQ_IMMED;
4498	return true;
4499	}
4500
4501	if ((*reenq_flags & SCX_REENQ_CAP_REVOKE) &&
4502	scx_task_reenq_on_cap_revoke(rq, p)) {
4503	*reason = SCX_TASK_REENQ_CAP;
4504	return true;
4505	}
4506
4507	return *reenq_flags & SCX_REENQ_ANY;
4508	}
4509
4510	/*
4511	* The dispatcher runs ops.dequeue() and stores the final ops_state after
4512	* dropping the DSQ lock, so @p can be found on a DSQ while still
4513	* %SCX_OPSS_DISPATCHING. Wait for the dispatch to finish before operating on
4514	* @p.
4515	*/
4516	void scx_wait_dispatching(struct task_struct *p)
4517	{
4518	if (unlikely(atomic_long_read_acquire(&p->scx.ops_state) == SCX_OPSS_DISPATCHING))
4519	wait_ops_state(p, SCX_OPSS_DISPATCHING);
4520	}
4521
4522	static u32 reenq_local(struct scx_sched sch, struct rq rq, u64 reenq_flags)
4523	{
4524	LIST_HEAD(tasks);
4525	u32 nr_enqueued = 0;
4526	struct task_struct p, n;
4527
4528	lockdep_assert_rq_held(rq);
4529
4530	if (WARN_ON_ONCE(reenq_flags & __SCX_REENQ_TSR_MASK))
4531	reenq_flags &= ~__SCX_REENQ_TSR_MASK;
4532	if (rq_is_open(rq, 0))
4533	reenq_flags \|= SCX_REENQ_TSR_RQ_OPEN;
4534
4535	/*
4536	* The BPF scheduler may choose to dispatch tasks back to
4537	* @rq->scx.local_dsq. Move all candidate tasks off to a private list
4538	* first to avoid processing the same tasks repeatedly.
4539	*/
4540	list_for_each_entry_safe(p, n, &rq->scx.local_dsq.list,
4541	scx.dsq_list.node) {
4542	struct scx_sched *task_sch = scx_task_sched(p);
4543	u32 reason;
4544
4545	/*
4546	* If @p is being migrated, @p's current CPU may not agree with
4547	* its allowed CPUs and the migration_cpu_stop is about to
4548	* deactivate and re-activate @p anyway. Skip re-enqueueing.
4549	*
4550	* While racing sched property changes may also dequeue and
4551	* re-enqueue a migrating task while its current CPU and allowed
4552	* CPUs disagree, they use %ENQUEUE_RESTORE which is bypassed to
4553	* the current local DSQ for running tasks and thus are not
4554	* visible to the BPF scheduler.
4555	*/
4556	if (p->migration_pending)
4557	continue;
4558
4559	if (!scx_is_descendant(task_sch, sch))
4560	continue;
4561
4562	if (!local_task_should_reenq(rq, p, &reenq_flags, &reason))
4563	continue;
4564
4565	scx_wait_dispatching(p);
4566	scx_dispatch_dequeue(rq, p);
4567
4568	if (WARN_ON_ONCE(p->scx.flags & SCX_TASK_REENQ_REASON_MASK))
4569	p->scx.flags &= ~SCX_TASK_REENQ_REASON_MASK;
4570	p->scx.flags \|= reason;
4571
4572	list_add_tail(&p->scx.dsq_list.node, &tasks);
4573	}
4574
4575	list_for_each_entry_safe(p, n, &tasks, scx.dsq_list.node) {
4576	list_del_init(&p->scx.dsq_list.node);
4577
4578	scx_do_enqueue_task(rq, p, SCX_ENQ_REENQ, -1);
4579
4580	p->scx.flags &= ~SCX_TASK_REENQ_REASON_MASK;
4581	nr_enqueued++;
4582	}
4583
4584	/*
4585	* The revoke that scheduled this scan may have raced the pick: curr
4586	* may be a now-capless task, either one that kept running or one
4587	* promoted off the local DSQ between the ecaps sync and this scan.
4588	* Zero the slice to evict it. The enqueue gate blocks new capless
4589	* inserts, so no later pick can slip through after the scan.
4590	*/
4591	if ((reenq_flags & SCX_REENQ_CAP_REVOKE) &&
4592	rq->curr->sched_class == &ext_sched_class &&
4593	scx_task_reenq_on_cap_revoke(rq, rq->curr)) {
4594	scx_set_task_slice(rq->curr, 0);
4595	resched_curr(rq);
4596	}
4597
4598	return nr_enqueued;
4599	}
4600
4601	static void process_deferred_reenq_locals(struct rq *rq)
4602	{
4603	lockdep_assert_rq_held(rq);
4604
4605	/*
4606	* A task can be re-queued within this loop when a reenqueued task
4607	* bounces straight back to the local DSQ. That recursion is bounded by
4608	* the per-task reenqueue cap in scx_do_enqueue_task().
4609	*/
4610	while (true) {
4611	struct scx_sched *sch;
4612	u64 reenq_flags;
4613
4614	scoped_guard (raw_spinlock, &rq->scx.deferred_reenq_lock) {
4615	struct scx_deferred_reenq_local *drl =
4616	list_first_entry_or_null(&rq->scx.deferred_reenq_locals,
4617	struct scx_deferred_reenq_local,
4618	node);
4619	struct scx_sched_pcpu *sch_pcpu;
4620
4621	if (!drl)
4622	return;
4623
4624	sch_pcpu = container_of(drl, struct scx_sched_pcpu,
4625	deferred_reenq_local);
4626	sch = sch_pcpu->sch;
4627
4628	reenq_flags = drl->flags;
4629	WRITE_ONCE(drl->flags, 0);
4630	list_del_init(&drl->node);
4631	}
4632
4633	/* see schedule_dsq_reenq() */
4634	smp_mb();
4635
4636	reenq_local(sch, rq, reenq_flags);
4637	}
4638	}
4639
4640	static bool user_task_should_reenq(struct task_struct p, u64 reenq_flags, u32 reason)
4641	{
4642	*reason = SCX_TASK_REENQ_KFUNC;
4643	return reenq_flags & SCX_REENQ_ANY;
4644	}
4645
4646	static void reenq_user(struct rq rq, struct scx_dispatch_q dsq, u64 reenq_flags)
4647	{
4648	struct rq *locked_rq = rq;
4649	struct scx_sched *sch = dsq->sched;
4650	struct scx_dsq_list_node cursor = INIT_DSQ_LIST_CURSOR(cursor, dsq, 0);
4651	struct task_struct *p;
4652	s32 nr_enqueued = 0;
4653
4654	lockdep_assert_rq_held(rq);
4655
4656	raw_spin_lock(&dsq->lock);
4657
4658	while (likely(!READ_ONCE(sch->bypass_depth))) {
4659	struct rq *task_rq;
4660	u32 reason;
4661
4662	p = nldsq_cursor_next_task(&cursor, dsq);
4663	if (!p)
4664	break;
4665
4666	if (!user_task_should_reenq(p, reenq_flags, &reason))
4667	continue;
4668
4669	task_rq = task_rq(p);
4670
4671	if (locked_rq != task_rq) {
4672	if (locked_rq) {
4673	scx_rq_lock_drop(locked_rq);
4674	raw_spin_rq_unlock(locked_rq);
4675	}
4676	if (unlikely(!raw_spin_rq_trylock(task_rq))) {
4677	raw_spin_unlock(&dsq->lock);
4678	raw_spin_rq_lock(task_rq);
4679	raw_spin_lock(&dsq->lock);
4680	}
4681	locked_rq = task_rq;
4682
4683	/* did we lose @p while switching locks? */
4684	if (nldsq_cursor_lost_task(&cursor, task_rq, dsq, p))
4685	continue;
4686	}
4687
4688	/* @p is on @dsq, its rq and @dsq are locked */
4689	scx_wait_dispatching(p);
4690	dispatch_dequeue_locked(p, dsq);
4691	raw_spin_unlock(&dsq->lock);
4692
4693	if (WARN_ON_ONCE(p->scx.flags & SCX_TASK_REENQ_REASON_MASK))
4694	p->scx.flags &= ~SCX_TASK_REENQ_REASON_MASK;
4695	p->scx.flags \|= reason;
4696
4697	scx_do_enqueue_task(task_rq, p, SCX_ENQ_REENQ, -1);
4698
4699	p->scx.flags &= ~SCX_TASK_REENQ_REASON_MASK;
4700
4701	if (!(++nr_enqueued % SCX_TASK_ITER_BATCH)) {
4702	scx_rq_lock_drop(locked_rq);
4703	raw_spin_rq_unlock(locked_rq);
4704	locked_rq = NULL;
4705	cpu_relax();
4706	}
4707
4708	raw_spin_lock(&dsq->lock);
4709	}
4710
4711	list_del_init(&cursor.node);
4712	raw_spin_unlock(&dsq->lock);
4713
4714	if (locked_rq != rq) {
4715	if (locked_rq) {
4716	scx_rq_lock_drop(locked_rq);
4717	raw_spin_rq_unlock(locked_rq);
4718	}
4719	raw_spin_rq_lock(rq);
4720	}
4721	}
4722
4723	static void process_deferred_reenq_users(struct rq *rq)
4724	{
4725	lockdep_assert_rq_held(rq);
4726
4727	while (true) {
4728	struct scx_dispatch_q *dsq;
4729	u64 dsq_id, reenq_flags;
4730
4731	scoped_guard (raw_spinlock, &rq->scx.deferred_reenq_lock) {
4732	struct scx_deferred_reenq_user *dru =
4733	list_first_entry_or_null(&rq->scx.deferred_reenq_users,
4734	struct scx_deferred_reenq_user,
4735	node);
4736	struct scx_dsq_pcpu *dsq_pcpu;
4737
4738	if (!dru)
4739	return;
4740
4741	dsq_pcpu = container_of(dru, struct scx_dsq_pcpu,
4742	deferred_reenq_user);
4743	dsq = dsq_pcpu->dsq;
4744	reenq_flags = dru->flags;
4745	WRITE_ONCE(dru->flags, 0);
4746	list_del_init(&dru->node);
4747	}
4748
4749	/* see schedule_dsq_reenq() */
4750	smp_mb();
4751
4752	/* destroy_dsq() may have raced and invalidated @dsq, nothing to reenq */
4753	dsq_id = READ_ONCE(dsq->id);
4754	if (unlikely(dsq_id == SCX_DSQ_INVALID))
4755	continue;
4756
4757	BUG_ON(dsq_id & SCX_DSQ_FLAG_BUILTIN);
4758	reenq_user(rq, dsq, reenq_flags);
4759	}
4760	}
4761
4762	static void run_deferred(struct rq *rq)
4763	{
4764	process_ddsp_deferred_locals(rq);
4765
4766	if (!list_empty(&rq->scx.deferred_reenq_locals))
4767	process_deferred_reenq_locals(rq);
4768
4769	if (!list_empty(&rq->scx.deferred_reenq_users))
4770	process_deferred_reenq_users(rq);
4771
4772	scx_reenq_reject(rq);
4773	}
4774
4775	#ifdef CONFIG_NO_HZ_FULL
4776	bool scx_can_stop_tick(struct rq *rq)
4777	{
4778	struct task_struct *p = rq->curr;
4779	struct scx_sched *sch = scx_task_sched(p);
4780
4781	if (p->sched_class != &ext_sched_class)
4782	return true;
4783
4784	/*
4785	* @rq->curr may still reference an outgoing EXT task after it has been
4786	* dequeued. If no EXT tasks are accounted on @rq, ignore its stale
4787	* slice state. If another task is dispatched from a DSQ,
4788	* set_next_task_scx() will update the dependency for the incoming task.
4789	*/
4790	if (!rq->scx.nr_running)
4791	return true;
4792
4793	if (scx_bypassing(sch, cpu_of(rq)))
4794	return false;
4795
4796	/*
4797	* A running rescuee's charging and expiry are tick-driven, see
4798	* scx_rescue_charge(). Keep the tick while rescue is in progress.
4799	*/
4800	if (unlikely(p == scx_rescuee(rq)))
4801	return false;
4802
4803	/*
4804	* @rq can dispatch from different DSQs, so we can't tell whether it
4805	* needs the tick or not by looking at nr_running. Allow stopping ticks
4806	* iff the BPF scheduler indicated so. See set_next_task_scx().
4807	*/
4808	return rq->scx.flags & SCX_RQ_CAN_STOP_TICK;
4809	}
4810	#endif
4811
4812	#ifdef CONFIG_EXT_GROUP_SCHED
4813
4814	DEFINE_STATIC_PERCPU_RWSEM(scx_cgroup_ops_rwsem);
4815
4816	void scx_tg_init(struct task_group *tg)
4817	{
4818	tg->scx.weight = CGROUP_WEIGHT_DFL;
4819	tg->scx.bw_period_us = default_bw_period_us();
4820	tg->scx.bw_quota_us = RUNTIME_INF;
4821	tg->scx.idle = false;
4822	}
4823
4824	/**
4825	* scx_tg_sched - Resolve a task_group's sched
4826	* @tg: task_group of interest
4827	*
4828	* Return the sched that @tg's ops.cgroup_init() succeeded on, %NULL if @tg
4829	* isn't inited. An autogroup tg has no cgroup of its own and resolves to the
4830	* root sched.
4831	*
4832	* When a child sched exits, its task_groups are moved to the parent and
4833	* re-inited on it. A failed re-init fails the parent in turn and leaves the
4834	* task_group without a sched it's inited on, resolving to %NULL. See
4835	* scx_cgroup_return_subtree().
4836	*
4837	* Safe for callers read-locking the ops rwsem. tg->scx.sched rewrites
4838	* write-lock it, and tg on/offline can't overlap such callers as a css's files
4839	* are created after online and drained before offline.
4840	*/
4841	static struct scx_sched scx_tg_sched(struct task_group tg)
4842	{
4843	lockdep_assert(lockdep_is_held(&cgroup_mutex) \|\|
4844	lockdep_is_held(&scx_cgroup_ops_rwsem));
4845
4846	if (!tg->css.cgroup)
4847	tg = &root_task_group;
4848	/* INITED means ops.cgroup_init() succeeded on @tg->scx.sched */
4849	return (tg->scx.flags & SCX_TG_INITED) ? tg->scx.sched : NULL;
4850	}
4851
4852	/**
4853	* scx_tg_knob_sched - Resolve the sched receiving a task_group's knob updates
4854	* @tg: task_group of interest
4855	*
4856	* Knobs of a cgroup belong to the parent. Deliver the set_* ops to the
4857	* parent task_group's sched, which equals @tg's own sched everywhere except
4858	* at a sub-scheduler attach point, where the sub's parent sched receives
4859	* them.
4860	*
4861	* Return %NULL if the parent task_group has no sched. That can happen when the
4862	* parent's ops.cgroup_init() fails while a sub-scheduler is being disabled.
4863	*
4864	* The callers sit in @tg's cgroup file writes holding the ops rwsem read
4865	* side. That extends scx_tg_sched()'s file-write argument to the parent's
4866	* sched read: a parent css outlives its children's files.
4867	*/
4868	static struct scx_sched scx_tg_knob_sched(struct task_group tg)
4869	{
4870	lockdep_assert(lockdep_is_held(&cgroup_mutex) \|\|
4871	lockdep_is_held(&scx_cgroup_ops_rwsem));
4872
4873	if (!tg->css.cgroup \|\| !tg->css.parent)
4874	return scx_tg_sched(&root_task_group);
4875	return scx_tg_sched(css_tg(tg->css.parent));
4876	}
4877
4878	int scx_tg_online(struct task_group *tg)
4879	{
4880	int ret = 0;
4881
4882	WARN_ON_ONCE(tg->scx.flags & (SCX_TG_ONLINE \| SCX_TG_INITED));
4883
4884	if (scx_cgroup_enabled) {
4885	struct scx_sched *sch;
4886
4887	/*
4888	* The cgroup lifetime notifier populates cgrp->scx_sched before
4889	* css_online, but only on the default hierarchy. Sub-scheds are
4890	* attached to the cgroup2 hierarchy, so a cgroup1 task_group
4891	* always belongs to the root sched.
4892	*/
4893	if (cgroup_on_dfl(tg->css.cgroup))
4894	sch = scx_cgroup_sched(tg->css.cgroup);
4895	else
4896	sch = scx_tg_sched(&root_task_group);
4897
4898	if (SCX_HAS_OP(sch, cgroup_init)) {
4899	struct scx_cgroup_init_args args =
4900	{ .weight = tg->scx.weight,
4901	.bw_period_us = tg->scx.bw_period_us,
4902	.bw_quota_us = tg->scx.bw_quota_us,
4903	.bw_burst_us = tg->scx.bw_burst_us,
4904	.sched_idle = tg->scx.idle };
4905
4906	ret = SCX_CALL_OP_RET(sch, cgroup_init,
4907	NULL, tg->css.cgroup, &args);
4908	if (ret)
4909	ret = scx_ops_sanitize_err(sch, "cgroup_init", ret);
4910	}
4911	if (ret == 0) {
4912	tg->scx.sched = sch;
4913	tg->scx.flags \|= SCX_TG_ONLINE \| SCX_TG_INITED;
4914	}
4915	} else {
4916	tg->scx.flags \|= SCX_TG_ONLINE;
4917	}
4918
4919	return ret;
4920	}
4921
4922	void scx_tg_offline(struct task_group *tg)
4923	{
4924	struct scx_sched *sch = tg->scx.sched;
4925
4926	WARN_ON_ONCE(!(tg->scx.flags & SCX_TG_ONLINE));
4927
4928	/* INITED implies non-NULL @sch, test before SCX_HAS_OP() derefs */
4929	if (scx_cgroup_enabled && (tg->scx.flags & SCX_TG_INITED) &&
4930	SCX_HAS_OP(sch, cgroup_exit))
4931	SCX_CALL_OP(sch, cgroup_exit, NULL, tg->css.cgroup);
4932	tg->scx.sched = NULL;
4933	tg->scx.flags &= ~(SCX_TG_ONLINE \| SCX_TG_INITED);
4934	}
4935
4936	/*
4937	* @p's sched for the cgroup migration paths. Stable as re-homes happen either
4938	* at CGROUP_TASK_MIGRATED of the same migration or under scx_cgroup_lock(),
4939	* both while holding cgroup_mutex.
4940	*/
4941	static struct scx_sched scx_cgroup_task_sched(struct task_struct p)
4942	{
4943	return rcu_dereference_protected(p->scx.sched, lockdep_is_held(&cgroup_mutex));
4944	}
4945
4946	int scx_cgroup_can_attach(struct cgroup_taskset *tset)
4947	{
4948	struct cgroup_subsys_state *css;
4949	struct task_struct *p;
4950	int ret;
4951
4952	if (!scx_cgroup_enabled)
4953	return 0;
4954
4955	cgroup_taskset_for_each(p, css, tset) {
4956	struct scx_sched *sch = scx_cgroup_task_sched(p);
4957	struct cgroup *from = tg_cgrp(task_group(p));
4958	struct cgroup *to = tg_cgrp(css_tg(css));
4959
4960	WARN_ON_ONCE(p->scx.cgrp_moving_from);
4961
4962	/*
4963	* sched_move_task() omits identity migrations. Let's match the
4964	* behavior so that ops.cgroup_prep_move() and ops.cgroup_move()
4965	* always match one-to-one.
4966	*/
4967	if (from == to)
4968	continue;
4969
4970	/*
4971	* The cgroup_move ops are delivered to @p's sched, and only for
4972	* moves that don't re-home @p. A re-homing move changes the dfl
4973	* cgroup's sched and is reported through the
4974	* exit_task/init_task pair that the re-homing generates.
4975	*/
4976	if (!sch \|\| sch != scx_cgroup_sched(task_css_set(p)->mg_dst_cset->dfl_cgrp))
4977	continue;
4978
4979	if (SCX_HAS_OP(sch, cgroup_prep_move)) {
4980	ret = SCX_CALL_OP_RET(sch, cgroup_prep_move, NULL,
4981	p, from, css->cgroup);
4982	if (ret) {
4983	ret = scx_ops_sanitize_err(sch, "cgroup_prep_move", ret);
4984	goto err;
4985	}
4986	}
4987
4988	p->scx.cgrp_moving_from = from;
4989	}
4990
4991	return 0;
4992
4993	err:
4994	cgroup_taskset_for_each(p, css, tset) {
4995	struct scx_sched *sch = scx_cgroup_task_sched(p);
4996
4997	/* cgrp_moving_from implies non-NULL @sch, test it first */
4998	if (p->scx.cgrp_moving_from && SCX_HAS_OP(sch, cgroup_cancel_move))
4999	SCX_CALL_OP(sch, cgroup_cancel_move, NULL,
5000	p, p->scx.cgrp_moving_from, css->cgroup);
5001	p->scx.cgrp_moving_from = NULL;
5002	}
5003
5004	return ret;
5005	}
5006
5007	void scx_cgroup_move_task(struct task_struct *p)
5008	{
5009	struct scx_sched *sch;
5010
5011	if (!scx_cgroup_enabled)
5012	return;
5013
5014	/*
5015	* Migration keys off css rather than cgroup identity, so it can hand an
5016	* unchanged-cgroup task here with cgrp_moving_from NULL. Nothing to
5017	* report to the BPF scheduler then, so skip it and keep prep_move and
5018	* move paired.
5019	*/
5020	sch = scx_cgroup_task_sched(p);
5021	if (p->scx.cgrp_moving_from && SCX_HAS_OP(sch, cgroup_move))
5022	SCX_CALL_OP_TASK(sch, cgroup_move, task_rq(p),
5023	p, p->scx.cgrp_moving_from,
5024	tg_cgrp(task_group(p)));
5025	p->scx.cgrp_moving_from = NULL;
5026	}
5027
5028	void scx_cgroup_cancel_attach(struct cgroup_taskset *tset)
5029	{
5030	struct cgroup_subsys_state *css;
5031	struct task_struct *p;
5032
5033	if (!scx_cgroup_enabled)
5034	return;
5035
5036	cgroup_taskset_for_each(p, css, tset) {
5037	struct scx_sched *sch = scx_cgroup_task_sched(p);
5038
5039	/* cgrp_moving_from implies non-NULL @sch, test it first */
5040	if (p->scx.cgrp_moving_from && SCX_HAS_OP(sch, cgroup_cancel_move))
5041	SCX_CALL_OP(sch, cgroup_cancel_move, NULL,
5042	p, p->scx.cgrp_moving_from, css->cgroup);
5043	p->scx.cgrp_moving_from = NULL;
5044	}
5045	}
5046
5047	void scx_group_set_weight(struct task_group *tg, unsigned long weight)
5048	{
5049	struct scx_sched *sch;
5050
5051	percpu_down_read(&scx_cgroup_ops_rwsem);
5052	sch = scx_tg_knob_sched(tg);
5053
5054	if (scx_cgroup_enabled && sch && SCX_HAS_OP(sch, cgroup_set_weight) &&
5055	tg->scx.weight != weight)
5056	SCX_CALL_OP(sch, cgroup_set_weight, NULL, tg_cgrp(tg), weight);
5057
5058	tg->scx.weight = weight;
5059
5060	percpu_up_read(&scx_cgroup_ops_rwsem);
5061	}
5062
5063	void scx_group_set_idle(struct task_group *tg, bool idle)
5064	{
5065	struct scx_sched *sch;
5066
5067	percpu_down_read(&scx_cgroup_ops_rwsem);
5068	sch = scx_tg_knob_sched(tg);
5069
5070	if (scx_cgroup_enabled && sch && SCX_HAS_OP(sch, cgroup_set_idle) &&
5071	tg->scx.idle != idle)
5072	SCX_CALL_OP(sch, cgroup_set_idle, NULL, tg_cgrp(tg), idle);
5073
5074	/* Update the task group's idle state */
5075	tg->scx.idle = idle;
5076
5077	percpu_up_read(&scx_cgroup_ops_rwsem);
5078	}
5079
5080	void scx_group_set_bandwidth(struct task_group *tg,
5081	u64 period_us, u64 quota_us, u64 burst_us)
5082	{
5083	struct scx_sched *sch;
5084
5085	percpu_down_read(&scx_cgroup_ops_rwsem);
5086	sch = scx_tg_knob_sched(tg);
5087
5088	if (scx_cgroup_enabled && sch && SCX_HAS_OP(sch, cgroup_set_bandwidth) &&
5089	(tg->scx.bw_period_us != period_us \|\|
5090	tg->scx.bw_quota_us != quota_us \|\|
5091	tg->scx.bw_burst_us != burst_us))
5092	SCX_CALL_OP(sch, cgroup_set_bandwidth, NULL,
5093	tg_cgrp(tg), period_us, quota_us, burst_us);
5094
5095	tg->scx.bw_period_us = period_us;
5096	tg->scx.bw_quota_us = quota_us;
5097	tg->scx.bw_burst_us = burst_us;
5098
5099	percpu_up_read(&scx_cgroup_ops_rwsem);
5100	}
5101	#endif /* CONFIG_EXT_GROUP_SCHED */
5102
5103	#if defined(CONFIG_EXT_GROUP_SCHED) \|\| defined(CONFIG_EXT_SUB_SCHED)
5104	static struct cgroup *root_cgroup(void)
5105	{
5106	return &cgrp_dfl_root.cgrp;
5107	}
5108
5109	/*
5110	* cgroup_lock() must nest outside the rwsem write side: a writer waiting
5111	* for cgroup_mutex deadlocks with cgroup teardown, which holds it while
5112	* draining a set_* file write blocked on the rwsem behind the writer.
5113	*/
5114	void scx_cgroup_lock(void)
5115	{
5116	cgroup_lock();
5117	#ifdef CONFIG_EXT_GROUP_SCHED
5118	percpu_down_write(&scx_cgroup_ops_rwsem);
5119	#endif
5120	}
5121
5122	void scx_cgroup_unlock(void)
5123	{
5124	#ifdef CONFIG_EXT_GROUP_SCHED
5125	percpu_up_write(&scx_cgroup_ops_rwsem);
5126	#endif
5127	cgroup_unlock();
5128	}
5129	#else /* CONFIG_EXT_GROUP_SCHED \|\| CONFIG_EXT_SUB_SCHED */
5130	static inline struct cgroup *root_cgroup(void) { return NULL; }
5131	static inline void scx_cgroup_lock(void) {}
5132	static inline void scx_cgroup_unlock(void) {}
5133	#endif /* CONFIG_EXT_GROUP_SCHED \|\| CONFIG_EXT_SUB_SCHED */
5134
5135	/*
5136	* Omitted operations:
5137	*
5138	* - migrate_task_rq: Unnecessary as task to cpu mapping is transient.
5139	*
5140	* - task_fork/dead: We need fork/dead notifications for all tasks regardless of
5141	* their current sched_class. Call them directly from sched core instead.
5142	*/
5143	DEFINE_SCHED_CLASS(ext) = {
5144	.enqueue_task = enqueue_task_scx,
5145	.dequeue_task = dequeue_task_scx,
5146	.yield_task = yield_task_scx,
5147	.yield_to_task = yield_to_task_scx,
5148
5149	.wakeup_preempt = wakeup_preempt_scx,
5150
5151	.pick_task = pick_task_scx,
5152
5153	.put_prev_task = put_prev_task_scx,
5154	.set_next_task = set_next_task_scx,
5155
5156	.select_task_rq = select_task_rq_scx,
5157	.task_woken = task_woken_scx,
5158	.set_cpus_allowed = set_cpus_allowed_scx,
5159
5160	.rq_online = rq_online_scx,
5161	.rq_offline = rq_offline_scx,
5162
5163	.task_tick = task_tick_scx,
5164
5165	.switching_to = switching_to_scx,
5166	.switched_from = switched_from_scx,
5167	.switched_to = switched_to_scx,
5168	.reweight_task = reweight_task_scx,
5169	.prio_changed = prio_changed_scx,
5170
5171	.update_curr = update_curr_scx,
5172
5173	#ifdef CONFIG_UCLAMP_TASK
5174	.uclamp_enabled = 1,
5175	#endif
5176	};
5177
5178	s32 scx_init_dsq(struct scx_dispatch_q dsq, u64 dsq_id, struct scx_sched sch)
5179	{
5180	s32 cpu;
5181
5182	memset(dsq, 0, sizeof(*dsq));
5183
5184	raw_spin_lock_init(&dsq->lock);
5185	INIT_LIST_HEAD(&dsq->list);
5186	dsq->id = dsq_id;
5187	dsq->sched = sch;
5188
5189	dsq->pcpu = alloc_percpu(struct scx_dsq_pcpu);
5190	if (!dsq->pcpu)
5191	return -ENOMEM;
5192
5193	for_each_possible_cpu(cpu) {
5194	struct scx_dsq_pcpu *pcpu = per_cpu_ptr(dsq->pcpu, cpu);
5195
5196	pcpu->dsq = dsq;
5197	INIT_LIST_HEAD(&pcpu->deferred_reenq_user.node);
5198	}
5199
5200	return 0;
5201	}
5202
5203	static void exit_dsq(struct scx_dispatch_q *dsq)
5204	{
5205	s32 cpu;
5206
5207	for_each_possible_cpu(cpu) {
5208	struct scx_dsq_pcpu *pcpu = per_cpu_ptr(dsq->pcpu, cpu);
5209	struct scx_deferred_reenq_user *dru = &pcpu->deferred_reenq_user;
5210	struct rq *rq = cpu_rq(cpu);
5211
5212	/*
5213	* free_dsq_irq_workfn() cancelled the reenqs before the grace
5214	* period and schedule_dsq_reenq() queues nothing after that.
5215	*/
5216	if (WARN_ON_ONCE(!list_empty(&dru->node))) {
5217	guard(raw_spinlock_irqsave)(&rq->scx.deferred_reenq_lock);
5218	list_del_init(&dru->node);
5219	}
5220	}
5221
5222	free_percpu(dsq->pcpu);
5223	}
5224
5225	static void free_dsq_rcufn(struct rcu_head *rcu)
5226	{
5227	struct scx_dispatch_q *dsq = container_of(rcu, struct scx_dispatch_q, rcu);
5228
5229	exit_dsq(dsq);
5230	kfree(dsq);
5231	}
5232
5233	static void free_dsq_irq_workfn(struct irq_work *irq_work)
5234	{
5235	struct llist_node *to_free = llist_del_all(&dsqs_to_free);
5236	struct scx_dispatch_q dsq, tmp_dsq;
5237
5238	llist_for_each_entry_safe(dsq, tmp_dsq, to_free, free_node) {
5239	s32 cpu;
5240
5241	/*
5242	* Cancel pending reenqs. Only a run_deferred() that started
5243	* before the grace period can hold one, and it keeps IRQs off,
5244	* so the grace period waits for it. After this sweep,
5245	* schedule_dsq_reenq() sees the invalid id under the same lock
5246	* and queues nothing.
5247	*/
5248	for_each_possible_cpu(cpu) {
5249	struct scx_dsq_pcpu *pcpu = per_cpu_ptr(dsq->pcpu, cpu);
5250	struct rq *rq = cpu_rq(cpu);
5251
5252	guard(raw_spinlock_irqsave)(&rq->scx.deferred_reenq_lock);
5253	list_del_init(&pcpu->deferred_reenq_user.node);
5254	}
5255
5256	call_rcu(&dsq->rcu, free_dsq_rcufn);
5257	}
5258	}
5259
5260	static DEFINE_IRQ_WORK(free_dsq_irq_work, free_dsq_irq_workfn);
5261
5262	static void destroy_dsq(struct scx_sched *sch, u64 dsq_id)
5263	{
5264	struct scx_dispatch_q *dsq;
5265	unsigned long flags;
5266
5267	rcu_read_lock();
5268
5269	dsq = find_user_dsq(sch, dsq_id);
5270	if (!dsq)
5271	goto out_unlock_rcu;
5272
5273	raw_spin_lock_irqsave(&dsq->lock, flags);
5274
5275	if (dsq->nr) {
5276	scx_error(sch, "attempting to destroy in-use dsq 0x%016llx (nr=%u)",
5277	dsq->id, dsq->nr);
5278	goto out_unlock_dsq;
5279	}
5280
5281	if (rhashtable_remove_fast(&sch->dsq_hash, &dsq->hash_node,
5282	dsq_hash_params))
5283	goto out_unlock_dsq;
5284
5285	/*
5286	* Mark dead by invalidating ->id to prevent scx_dispatch_enqueue() from
5287	* queueing more tasks. As this function can be called from anywhere,
5288	* freeing is bounced through an irq work to avoid nesting RCU
5289	* operations inside scheduler locks.
5290	*/
5291	dsq->id = SCX_DSQ_INVALID;
5292	if (llist_add(&dsq->free_node, &dsqs_to_free))
5293	irq_work_queue(&free_dsq_irq_work);
5294
5295	out_unlock_dsq:
5296	raw_spin_unlock_irqrestore(&dsq->lock, flags);
5297	out_unlock_rcu:
5298	rcu_read_unlock();
5299	}
5300
5301	#ifdef CONFIG_EXT_GROUP_SCHED
5302	static void scx_cgroup_exit(struct scx_sched *sch)
5303	{
5304	struct cgroup_subsys_state *css;
5305
5306	/*
5307	* scx_tg_on/offline() are excluded through cgroup_lock(). If we walk
5308	* cgroups and exit all the inited ones, all online cgroups are exited.
5309	*/
5310	css_for_each_descendant_post(css, &root_task_group.css) {
5311	struct task_group *tg = css_tg(css);
5312
5313	/* also clear the sched of tgs whose ops.cgroup_init() failed */
5314	tg->scx.sched = NULL;
5315	if (tg->scx.flags & SCX_TG_INITED) {
5316	tg->scx.flags &= ~SCX_TG_INITED;
5317	if (sch->ops.cgroup_exit)
5318	SCX_CALL_OP(sch, cgroup_exit, NULL, css->cgroup);
5319	}
5320	}
5321	}
5322
5323	static int scx_cgroup_init(struct scx_sched *sch)
5324	{
5325	struct cgroup_subsys_state *css;
5326	int ret;
5327
5328	/*
5329	* scx_tg_on/offline() are excluded through cgroup_lock(). If we walk
5330	* cgroups and init, all online cgroups are initialized.
5331	*/
5332	css_for_each_descendant_pre(css, &root_task_group.css) {
5333	struct task_group *tg = css_tg(css);
5334
5335	if ((tg->scx.flags & (SCX_TG_ONLINE \| SCX_TG_INITED)) != SCX_TG_ONLINE)
5336	continue;
5337
5338	if (sch->ops.cgroup_init) {
5339	struct scx_cgroup_init_args args = {
5340	.weight = tg->scx.weight,
5341	.bw_period_us = tg->scx.bw_period_us,
5342	.bw_quota_us = tg->scx.bw_quota_us,
5343	.bw_burst_us = tg->scx.bw_burst_us,
5344	.sched_idle = tg->scx.idle,
5345	};
5346
5347	ret = SCX_CALL_OP_RET(sch, cgroup_init, NULL, css->cgroup, &args);
5348	if (ret) {
5349	scx_error(sch, "ops.cgroup_init() failed (%d)", ret);
5350	return ret;
5351	}
5352	}
5353
5354	tg->scx.sched = sch;
5355	tg->scx.flags \|= SCX_TG_INITED;
5356	}
5357
5358	return 0;
5359	}
5360
5361	#else
5362	static void scx_cgroup_exit(struct scx_sched *sch) {}
5363	static int scx_cgroup_init(struct scx_sched *sch) { return 0; }
5364	#endif
5365
5366
5367	/********************************************************************************
5368	* Sysfs interface and ops enable/disable.
5369	*/
5370
5371	#define SCX_ATTR(_name) \
5372	static struct kobj_attribute scx_attr_##_name = { \
5373	.attr = { .name = __stringify(_name), .mode = 0444 }, \
5374	.show = scx_attr_##_name##_show, \
5375	}
5376
5377	static ssize_t scx_attr_state_show(struct kobject *kobj,
5378	struct kobj_attribute ka, char buf)
5379	{
5380	return sysfs_emit(buf, "%s\n", scx_enable_state_str[scx_enable_state()]);
5381	}
5382	SCX_ATTR(state);
5383
5384	static ssize_t scx_attr_switch_all_show(struct kobject *kobj,
5385	struct kobj_attribute ka, char buf)
5386	{
5387	return sysfs_emit(buf, "%d\n", READ_ONCE(scx_switching_all));
5388	}
5389	SCX_ATTR(switch_all);
5390
5391	static ssize_t scx_attr_nr_rejected_show(struct kobject *kobj,
5392	struct kobj_attribute ka, char buf)
5393	{
5394	return sysfs_emit(buf, "%ld\n", atomic_long_read(&scx_nr_rejected));
5395	}
5396	SCX_ATTR(nr_rejected);
5397
5398	static ssize_t scx_attr_hotplug_seq_show(struct kobject *kobj,
5399	struct kobj_attribute ka, char buf)
5400	{
5401	return sysfs_emit(buf, "%ld\n", atomic_long_read(&scx_hotplug_seq));
5402	}
5403	SCX_ATTR(hotplug_seq);
5404
5405	static ssize_t scx_attr_enable_seq_show(struct kobject *kobj,
5406	struct kobj_attribute ka, char buf)
5407	{
5408	return sysfs_emit(buf, "%ld\n", atomic_long_read(&scx_enable_seq));
5409	}
5410	SCX_ATTR(enable_seq);
5411
5412	static struct attribute *scx_global_attrs[] = {
5413	&scx_attr_state.attr,
5414	&scx_attr_switch_all.attr,
5415	&scx_attr_nr_rejected.attr,
5416	&scx_attr_hotplug_seq.attr,
5417	&scx_attr_enable_seq.attr,
5418	NULL,
5419	};
5420
5421	static const struct attribute_group scx_global_attr_group = {
5422	.attrs = scx_global_attrs,
5423	};
5424
5425	static void free_pnode(struct scx_sched_pnode *pnode);
5426	static void free_exit_info(struct scx_exit_info *ei);
5427	static const char *scx_exit_reason(enum scx_exit_kind kind);
5428	static bool scx_claim_exit(struct scx_sched *sch, enum scx_exit_kind kind);
5429
5430	s32 scx_alloc_kern_arena_objs(struct scx_sched *sch)
5431	{
5432	size_t size = struct_size_t(struct scx_cmask, bits,
5433	SCX_CMASK_NR_WORDS(num_possible_cpus()));
5434	struct scx_cmask *online;
5435	struct scx_cmask_ref ref;
5436	int cpu;
5437
5438	/* hotplug stays excluded until the online mask is published */
5439	lockdep_assert_cpus_held();
5440
5441	if (!sch->is_cid_type \|\| !sch->arena_pool)
5442	return 0;
5443
5444	sch->set_cmask_scratch = alloc_percpu(struct scx_cmask *);
5445	if (!sch->set_cmask_scratch)
5446	return -ENOMEM;
5447
5448	for_each_possible_cpu(cpu) {
5449	struct scx_cmask **slot = per_cpu_ptr(sch->set_cmask_scratch, cpu);
5450
5451	*slot = scx_arena_alloc(sch, size);
5452	if (!*slot)
5453	return -ENOMEM;
5454	scx_cmask_init(*slot, 0, num_possible_cpus());
5455	}
5456
5457	/* pack the online mask alongside the scratch masks */
5458	online = scx_arena_alloc(sch, size);
5459	if (!online)
5460	return -ENOMEM;
5461
5462	scoped_guard(rcu) {
5463	scx_cmask_ref_init_kern(sch, online, 0, num_possible_cpus(), &ref);
5464	scx_cmask_ref_from_cpumask(&ref, cpu_active_mask);
5465	}
5466	sch->online_cmask = online;
5467
5468	return 0;
5469	}
5470
5471	static void scx_free_kern_arena_objs(struct scx_sched *sch)
5472	{
5473	size_t size = struct_size_t(struct scx_cmask, bits,
5474	SCX_CMASK_NR_WORDS(num_possible_cpus()));
5475	int cpu;
5476
5477	scx_arena_free(sch, sch->online_cmask, size);
5478	if (!sch->set_cmask_scratch)
5479	return;
5480
5481	for_each_possible_cpu(cpu) {
5482	struct scx_cmask **slot = per_cpu_ptr(sch->set_cmask_scratch, cpu);
5483
5484	scx_arena_free(sch, *slot, size);
5485	}
5486	free_percpu(sch->set_cmask_scratch);
5487	sch->set_cmask_scratch = NULL;
5488	}
5489
5490	static void scx_sched_free_rcu_work(struct work_struct *work)
5491	{
5492	struct rcu_work *rcu_work = to_rcu_work(work);
5493	struct scx_sched *sch = container_of(rcu_work, struct scx_sched, rcu_work);
5494	struct rhashtable_iter rht_iter;
5495	struct scx_dispatch_q *dsq;
5496	int cpu, node;
5497
5498	irq_work_sync(&sch->propagate_exit_irq_work);
5499	irq_work_sync(&sch->disable_irq_work);
5500	kthread_destroy_worker(sch->helper);
5501	timer_shutdown_sync(&sch->bypass_lb_timer);
5502	free_cpumask_var(sch->bypass_lb_donee_cpumask);
5503	free_cpumask_var(sch->bypass_lb_resched_cpumask);
5504	free_cpumask_var(sch->stall_cpus);
5505
5506	#ifdef CONFIG_EXT_SUB_SCHED
5507	kfree(sch->cgrp_path);
5508	if (sch_cgroup(sch))
5509	cgroup_put(sch_cgroup(sch));
5510	if (sch->sub_kset)
5511	kobject_put(&sch->sub_kset->kobj);
5512	if (scx_parent(sch))
5513	kobject_put(&scx_parent(sch)->kobj);
5514	#endif /* CONFIG_EXT_SUB_SCHED */
5515
5516	for_each_possible_cpu(cpu) {
5517	struct scx_sched_pcpu *pcpu = per_cpu_ptr(sch->pcpu, cpu);
5518
5519	/*
5520	* $sch would have entered bypass mode before the RCU grace
5521	* period. As that blocks new deferrals, all
5522	* deferred_reenq_local_node's must be off-list by now.
5523	*/
5524	WARN_ON_ONCE(!list_empty(&pcpu->deferred_reenq_local.node));
5525
5526	/* remove the queued ecaps sync so the pcpu can be freed */
5527	scx_discard_ecaps_to_sync(cpu, pcpu);
5528
5529	/*
5530	* Bypass blocks new kicks. Flush the kick irq_work so this
5531	* pcpu's to_kick_node is off the list before it is freed.
5532	*/
5533	irq_work_sync(&cpu_rq(cpu)->scx.kick_cpus_irq_work);
5534	WARN_ON_ONCE(!list_empty(&pcpu->to_kick_node));
5535	free_cpumask_var(pcpu->cpus_to_kick);
5536	free_cpumask_var(pcpu->cpus_to_kick_if_idle);
5537	free_cpumask_var(pcpu->cpus_to_preempt);
5538	free_cpumask_var(pcpu->cpus_to_wait);
5539
5540	exit_dsq(scx_bypass_dsq(sch, cpu));
5541	}
5542
5543	free_percpu(sch->pcpu);
5544
5545	for_each_node_state(node, N_POSSIBLE)
5546	free_pnode(sch->pnode[node]);
5547	kfree(sch->pnode);
5548
5549	scx_free_pshards(sch);
5550
5551	rhashtable_walk_enter(&sch->dsq_hash, &rht_iter);
5552	do {
5553	rhashtable_walk_start(&rht_iter);
5554
5555	while (!IS_ERR_OR_NULL((dsq = rhashtable_walk_next(&rht_iter))))
5556	destroy_dsq(sch, dsq->id);
5557
5558	rhashtable_walk_stop(&rht_iter);
5559	} while (dsq == ERR_PTR(-EAGAIN));
5560	rhashtable_walk_exit(&rht_iter);
5561
5562	rhashtable_free_and_destroy(&sch->dsq_hash, NULL, NULL);
5563	free_exit_info(sch->exit_info);
5564	scx_free_kern_arena_objs(sch);
5565	scx_arena_pool_destroy(sch);
5566	if (sch->arena_map)
5567	bpf_map_put(sch->arena_map);
5568
5569	/* @sch is completely inactive by now */
5570	scx_dec_has_subs(sch);
5571
5572	kfree(sch);
5573	}
5574
5575	static void scx_kobj_release(struct kobject *kobj)
5576	{
5577	struct scx_sched *sch = container_of(kobj, struct scx_sched, kobj);
5578
5579	INIT_RCU_WORK(&sch->rcu_work, scx_sched_free_rcu_work);
5580	queue_rcu_work(system_dfl_wq, &sch->rcu_work);
5581	}
5582
5583	static ssize_t scx_attr_ops_show(struct kobject *kobj,
5584	struct kobj_attribute ka, char buf)
5585	{
5586	struct scx_sched *sch = container_of(kobj, struct scx_sched, kobj);
5587
5588	return sysfs_emit(buf, "%s\n", sch->ops.name);
5589	}
5590	SCX_ATTR(ops);
5591
5592	#define scx_attr_event_show(buf, at, events, kind) ({ \
5593	sysfs_emit_at(buf, at, "%s %llu\n", #kind, (events)->kind); \
5594	})
5595
5596	static ssize_t scx_attr_events_show(struct kobject *kobj,
5597	struct kobj_attribute ka, char buf)
5598	{
5599	struct scx_sched *sch = container_of(kobj, struct scx_sched, kobj);
5600	struct scx_event_stats events;
5601	int at = 0;
5602
5603	scx_read_events(sch, &events);
5604	#define SCX_EVENT(name) (at += scx_attr_event_show(buf, at, &events, name))
5605	SCX_EVENTS_LIST(SCX_EVENT);
5606	#undef SCX_EVENT
5607	return at;
5608	}
5609	SCX_ATTR(events);
5610
5611	#ifdef CONFIG_EXT_SUB_SCHED
5612	static const char *scx_cap_names[__SCX_NR_CAPS] = {
5613	[__SCX_CAP_ENQ_IMMED] = "enq_immed",
5614	[__SCX_CAP_ENQ] = "enq",
5615	[__SCX_CAP_PREEMPT] = "preempt",
5616	[__SCX_CAP_PERF] = "perf",
5617	};
5618
5619	static ssize_t scx_attr_caps_show(struct kobject *kobj,
5620	struct kobj_attribute ka, char buf)
5621	{
5622	struct scx_sched *sch = container_of(kobj, struct scx_sched, kobj);
5623	u32 npossible = num_possible_cpus();
5624	struct scx_cmask *agg __free(kfree) =
5625	kzalloc_flex(*agg, bits, SCX_CMASK_NR_WORDS(npossible));
5626	unsigned long *agg_bm __free(bitmap) = bitmap_zalloc(npossible, GFP_KERNEL);
5627	ssize_t count = 0;
5628	s32 cap, si;
5629
5630	if (!agg \|\| !agg_bm)
5631	return -ENOMEM;
5632
5633	for (cap = 0; cap < __SCX_NR_CAPS; cap++) {
5634	SCX_CMASK_DEFINE(snap, 0, SCX_CID_SHARD_MAX_CPUS);
5635
5636	scx_cmask_init(agg, 0, npossible);
5637	for (si = 0; si < sch->nr_pshards; si++) {
5638	struct scx_cmask *cm = &sch->pshard[si]->caps[cap].cmask;
5639
5640	scx_cmask_reframe(snap, cm->base, cm->nr_cids);
5641	scx_cmask_copy(snap, cm);
5642	scx_cmask_or(agg, snap);
5643	}
5644	/* %pbl takes unsigned long bitmap layout, convert from u64 /
5645	bitmap_from_arr64(agg_bm, agg->bits, npossible);
5646	count += sysfs_emit_at(buf, count, "%s: %*pbl\n",
5647	scx_cap_names[cap], npossible, agg_bm);
5648	}
5649	return count;
5650	}
5651	SCX_ATTR(caps);
5652	#endif /* CONFIG_EXT_SUB_SCHED */
5653
5654	static struct attribute *scx_sched_attrs[] = {
5655	&scx_attr_ops.attr,
5656	&scx_attr_events.attr,
5657	#ifdef CONFIG_EXT_SUB_SCHED
5658	&scx_attr_caps.attr,
5659	#endif
5660	NULL,
5661	};
5662	ATTRIBUTE_GROUPS(scx_sched);
5663
5664	static const struct kobj_type scx_ktype = {
5665	.release = scx_kobj_release,
5666	.sysfs_ops = &kobj_sysfs_ops,
5667	.default_groups = scx_sched_groups,
5668	};
5669
5670	static int scx_uevent(const struct kobject kobj, struct kobj_uevent_env env)
5671	{
5672	const struct scx_sched *sch;
5673
5674	/*
5675	* scx_uevent() can be reached by both scx_sched kobjects (scx_ktype)
5676	* and sub-scheduler kset kobjects (kset_ktype) through the parent
5677	* chain walk. Filter out the latter to avoid invalid casts.
5678	*/
5679	if (kobj->ktype != &scx_ktype)
5680	return 0;
5681
5682	sch = container_of(kobj, struct scx_sched, kobj);
5683
5684	return add_uevent_var(env, "SCXOPS=%s", sch->ops.name);
5685	}
5686
5687	static const struct kset_uevent_ops scx_uevent_ops = {
5688	.uevent = scx_uevent,
5689	};
5690
5691	/*
5692	* Used by sched_fork() and __setscheduler_class() to pick the matching
5693	* sched_class. dl/rt are already handled.
5694	*/
5695	bool task_should_scx(int policy)
5696	{
5697	/* if disabled, nothing should be on it */
5698	if (!scx_enabled())
5699	return false;
5700
5701	/* scx is taking over all SCHED_OTHER and SCHED_EXT tasks */
5702	if (READ_ONCE(scx_switching_all))
5703	return true;
5704
5705	/*
5706	* scx is tearing down - keep new SCHED_EXT tasks out.
5707	*
5708	* Must come after scx_switching_all test, which serves as a proxy
5709	* for __scx_switched_all. While __scx_switched_all is set, we must
5710	* return true via the branch above: a fork routed to fair would
5711	* stall because next_active_class() skips fair.
5712	*
5713	* This can develop into a deadlock - scx holds scx_enable_mutex across
5714	* kthread_create() in scx_alloc_and_add_sched(); if the new kthread is
5715	* the stalled task, the disable path can never grab the mutex to clear
5716	* scx_switching_all.
5717	*/
5718	if (unlikely(scx_enable_state() == SCX_DISABLING))
5719	return false;
5720
5721	return policy == SCHED_EXT;
5722	}
5723
5724	bool scx_allow_ttwu_queue(const struct task_struct *p)
5725	{
5726	struct scx_sched *sch;
5727
5728	if (!scx_enabled())
5729	return true;
5730
5731	sch = scx_task_sched(p);
5732	if (unlikely(!sch))
5733	return true;
5734
5735	if (sch->ops.flags & SCX_OPS_ALLOW_QUEUED_WAKEUP)
5736	return true;
5737
5738	if (unlikely(p->sched_class != &ext_sched_class))
5739	return true;
5740
5741	return false;
5742	}
5743
5744	/**
5745	* handle_lockup - sched_ext common lockup handler
5746	* @exit_cpu: CPU to record in exit_info. Pass the stalled/hung CPU, not current.
5747	* @fmt: format string
5748	*
5749	* Called on system stall or lockup condition and initiates abort of sched_ext
5750	* if enabled, which may resolve the reported lockup.
5751	*
5752	* Returns %true if sched_ext is enabled and abort was initiated, which may
5753	* resolve the lockup. %false if sched_ext is not enabled or abort was already
5754	* initiated by someone else.
5755	*/
5756	static __printf(2, 3) bool handle_lockup(int exit_cpu, const char *fmt, ...)
5757	{
5758	struct scx_sched *sch;
5759	va_list args;
5760	bool ret;
5761
5762	guard(rcu)();
5763
5764	sch = rcu_dereference(scx_root);
5765	if (unlikely(!sch))
5766	return false;
5767
5768	switch (scx_enable_state()) {
5769	case SCX_ENABLING:
5770	case SCX_ENABLED:
5771	va_start(args, fmt);
5772	ret = scx_vexit(sch, SCX_EXIT_ERROR, 0, exit_cpu, fmt, args);
5773	va_end(args);
5774	return ret;
5775	default:
5776	return false;
5777	}
5778	}
5779
5780	/**
5781	* scx_rcu_cpu_stall - sched_ext RCU CPU stall handler
5782	* @stalled_mask: bit mask of stalled CPUs
5783	*
5784	* While there are various reasons why RCU CPU stalls can occur on a system
5785	* that may not be caused by the current BPF scheduler, try kicking out the
5786	* current scheduler in an attempt to recover the system to a good state before
5787	* issuing panics.
5788	*
5789	* Returns %true if sched_ext is enabled and abort was initiated, which may
5790	* resolve the reported RCU stall. %false if sched_ext is not enabled or someone
5791	* else already initiated abort.
5792	*/
5793	bool scx_rcu_cpu_stall(const struct cpumask *stalled_mask)
5794	{
5795	struct scx_sched *sch;
5796	struct scx_exit_info *ei;
5797	int exit_cpu;
5798
5799	guard(rcu)();
5800
5801	sch = rcu_dereference(scx_root);
5802	if (unlikely(!sch))
5803	return false;
5804
5805	switch (scx_enable_state()) {
5806	case SCX_ENABLING:
5807	case SCX_ENABLED:
5808	break;
5809	default:
5810	return false;
5811	}
5812
5813	exit_cpu = cpumask_empty(stalled_mask) ? -1 : (int)cpumask_first(stalled_mask);
5814	ei = sch->exit_info;
5815
5816	guard(preempt)();
5817
5818	if (!scx_claim_exit(sch, SCX_EXIT_ERROR))
5819	return false;
5820
5821	#ifdef CONFIG_STACKTRACE
5822	ei->bt_len = stack_trace_save(ei->bt, SCX_EXIT_BT_LEN, 1);
5823	#endif
5824	scnprintf(ei->msg, SCX_EXIT_MSG_LEN, "RCU CPU stall on CPUs (%*pbl)",
5825	cpumask_pr_args(stalled_mask));
5826	ei->kind = SCX_EXIT_ERROR;
5827	ei->reason = scx_exit_reason(SCX_EXIT_ERROR);
5828	ei->exit_cpu = exit_cpu;
5829	cpumask_copy(sch->stall_cpus, stalled_mask);
5830
5831	irq_work_queue(&sch->disable_irq_work);
5832	return true;
5833	}
5834
5835	/**
5836	* scx_softlockup - sched_ext softlockup handler
5837	* @dur_s: number of seconds of CPU stuck due to soft lockup
5838	*
5839	* On some multi-socket setups (e.g. 2x Intel 8480c), the BPF scheduler can
5840	* live-lock the system by making many CPUs target the same DSQ to the point
5841	* where soft-lockup detection triggers. This function is called from
5842	* soft-lockup watchdog when the triggering point is close and tries to unjam
5843	* the system and aborting the BPF scheduler.
5844	*/
5845	void scx_softlockup(u32 dur_s)
5846	{
5847	int cpu = smp_processor_id();
5848
5849	if (!handle_lockup(cpu, "soft lockup - CPU %d stuck for %us", cpu, dur_s))
5850	return;
5851
5852	printk_deferred(KERN_ERR "sched_ext: Soft lockup - CPU %d stuck for %us, disabling BPF scheduler\n",
5853	cpu, dur_s);
5854	}
5855
5856	/**
5857	* scx_hardlockup - sched_ext hardlockup handler
5858	* @cpu: the target CPU
5859	*
5860	* A poorly behaving BPF scheduler can trigger hard lockup by e.g. putting
5861	* numerous affinitized tasks in a single queue and directing all CPUs at it.
5862	* Try kicking out the current scheduler in an attempt to recover the system to
5863	* a good state before taking more drastic actions.
5864	*
5865	* Called from NMI. Aborting the scheduler sets ->aborting throughout the
5866	* hierarchy before returning, which is what breaks the dispatch-path live-locks
5867	* that can hard-lock CPUs.
5868	*
5869	* Returns %true if sched_ext is enabled and abort was initiated, which may
5870	* resolve the lockup. %false if sched_ext is not enabled or abort was already
5871	* initiated by someone else.
5872	*/
5873	bool scx_hardlockup(int cpu)
5874	{
5875	if (!handle_lockup(cpu, "hard lockup - CPU %d", cpu))
5876	return false;
5877
5878	printk_deferred(KERN_ERR "sched_ext: Hard lockup - CPU %d, disabling BPF scheduler\n",
5879	cpu);
5880	return true;
5881	}
5882
5883	static u32 bypass_lb_cpu(struct scx_sched *sch, s32 donor,
5884	struct cpumask donee_mask, struct cpumask resched_mask,
5885	u32 nr_donor_target, u32 nr_donee_target)
5886	{
5887	struct rq *donor_rq = cpu_rq(donor);
5888	struct scx_dispatch_q *donor_dsq = scx_bypass_dsq(sch, donor);
5889	struct task_struct p, n;
5890	struct scx_dsq_list_node cursor = INIT_DSQ_LIST_CURSOR(cursor, donor_dsq, 0);
5891	s32 delta = READ_ONCE(donor_dsq->nr) - nr_donor_target;
5892	u32 nr_balanced = 0, min_delta_us;
5893
5894	/*
5895	* All we want to guarantee is reasonable forward progress. No reason to
5896	* fine tune. Assuming every task on @donor_dsq runs their full slice,
5897	* consider offloading iff the total queued duration is over the
5898	* threshold.
5899	*/
5900	min_delta_us = READ_ONCE(scx_bypass_lb_intv_us) / SCX_BYPASS_LB_MIN_DELTA_DIV;
5901	if (delta < DIV_ROUND_UP(min_delta_us, READ_ONCE(scx_slice_bypass_us)))
5902	return 0;
5903
5904	raw_spin_rq_lock_irq(donor_rq);
5905	raw_spin_lock(&donor_dsq->lock);
5906	list_add(&cursor.node, &donor_dsq->list);
5907	resume:
5908	n = container_of(&cursor, struct task_struct, scx.dsq_list);
5909	n = nldsq_next_task(donor_dsq, n, false);
5910
5911	while ((p = n)) {
5912	struct scx_dispatch_q *donee_dsq;
5913	int donee;
5914
5915	n = nldsq_next_task(donor_dsq, n, false);
5916
5917	if (donor_dsq->nr <= nr_donor_target)
5918	break;
5919
5920	if (cpumask_empty(donee_mask))
5921	break;
5922
5923	/*
5924	* If an earlier pass placed @p on @donor_dsq from a different
5925	* CPU and the donee hasn't consumed it yet, @p is still on the
5926	* previous CPU and task_rq(@p) != @donor_rq. @p can't be moved
5927	* without its rq locked. Skip.
5928	*/
5929	if (task_rq(p) != donor_rq)
5930	continue;
5931
5932	donee = cpumask_any_and_distribute(donee_mask, p->cpus_ptr);
5933	if (donee >= nr_cpu_ids)
5934	continue;
5935
5936	donee_dsq = scx_bypass_dsq(sch, donee);
5937
5938	/*
5939	* $p's rq is not locked but $p's DSQ lock protects its
5940	* scheduling properties making this test safe.
5941	*/
5942	if (!task_can_run_on_remote_rq(sch, p, cpu_rq(donee), false))
5943	continue;
5944
5945	/*
5946	* Moving $p from one non-local DSQ to another. The source rq
5947	* and DSQ are already locked. Do an abbreviated dequeue and
5948	* then perform enqueue without unlocking $donor_dsq.
5949	*
5950	* We don't want to drop and reacquire the lock on each
5951	* iteration as @donor_dsq can be very long and potentially
5952	* highly contended. Donee DSQs are less likely to be contended.
5953	* The nested locking is safe as only this LB moves tasks
5954	* between bypass DSQs.
5955	*/
5956	dispatch_dequeue_locked(p, donor_dsq);
5957	scx_dispatch_enqueue(sch, cpu_rq(donee), donee_dsq, p, 0, 0, SCX_ENQ_NESTED);
5958
5959	/*
5960	* $donee might have been idle and need to be woken up. No need
5961	* to be clever. Kick every CPU that receives tasks.
5962	*/
5963	cpumask_set_cpu(donee, resched_mask);
5964
5965	if (READ_ONCE(donee_dsq->nr) >= nr_donee_target)
5966	cpumask_clear_cpu(donee, donee_mask);
5967
5968	nr_balanced++;
5969	if (!(nr_balanced % SCX_BYPASS_LB_BATCH) && n) {
5970	list_move_tail(&cursor.node, &n->scx.dsq_list.node);
5971	raw_spin_unlock(&donor_dsq->lock);
5972	scx_rq_lock_drop(donor_rq);
5973	raw_spin_rq_unlock_irq(donor_rq);
5974	cpu_relax();
5975	raw_spin_rq_lock_irq(donor_rq);
5976	raw_spin_lock(&donor_dsq->lock);
5977	goto resume;
5978	}
5979	}
5980
5981	list_del_init(&cursor.node);
5982	raw_spin_unlock(&donor_dsq->lock);
5983	scx_rq_lock_drop(donor_rq);
5984	raw_spin_rq_unlock_irq(donor_rq);
5985
5986	return nr_balanced;
5987	}
5988
5989	static void bypass_lb_node(struct scx_sched *sch, int node)
5990	{
5991	const struct cpumask *node_mask = cpumask_of_node(node);
5992	struct cpumask *donee_mask = sch->bypass_lb_donee_cpumask;
5993	struct cpumask *resched_mask = sch->bypass_lb_resched_cpumask;
5994	u32 nr_tasks = 0, nr_cpus = 0, nr_balanced = 0;
5995	u32 nr_target, nr_donor_target;
5996	u32 before_min = U32_MAX, before_max = 0;
5997	u32 after_min = U32_MAX, after_max = 0;
5998	int cpu;
5999
6000	/* count the target tasks and CPUs */
6001	for_each_cpu_and(cpu, cpu_online_mask, node_mask) {
6002	u32 nr = READ_ONCE(scx_bypass_dsq(sch, cpu)->nr);
6003
6004	nr_tasks += nr;
6005	nr_cpus++;
6006
6007	before_min = min(nr, before_min);
6008	before_max = max(nr, before_max);
6009	}
6010
6011	if (!nr_cpus)
6012	return;
6013
6014	/*
6015	* We don't want CPUs to have more than $nr_donor_target tasks and
6016	* balancing to fill donee CPUs upto $nr_target. Once targets are
6017	* calculated, find the donee CPUs.
6018	*/
6019	nr_target = DIV_ROUND_UP(nr_tasks, nr_cpus);
6020	nr_donor_target = DIV_ROUND_UP(nr_target * SCX_BYPASS_LB_DONOR_PCT, 100);
6021
6022	cpumask_clear(donee_mask);
6023	for_each_cpu_and(cpu, cpu_online_mask, node_mask) {
6024	if (READ_ONCE(scx_bypass_dsq(sch, cpu)->nr) < nr_target)
6025	cpumask_set_cpu(cpu, donee_mask);
6026	}
6027
6028	/* iterate !donee CPUs and see if they should be offloaded */
6029	cpumask_clear(resched_mask);
6030	for_each_cpu_and(cpu, cpu_online_mask, node_mask) {
6031	if (cpumask_empty(donee_mask))
6032	break;
6033	if (cpumask_test_cpu(cpu, donee_mask))
6034	continue;
6035	if (READ_ONCE(scx_bypass_dsq(sch, cpu)->nr) <= nr_donor_target)
6036	continue;
6037
6038	nr_balanced += bypass_lb_cpu(sch, cpu, donee_mask, resched_mask,
6039	nr_donor_target, nr_target);
6040	}
6041
6042	for_each_cpu(cpu, resched_mask)
6043	resched_cpu(cpu);
6044
6045	for_each_cpu_and(cpu, cpu_online_mask, node_mask) {
6046	u32 nr = READ_ONCE(scx_bypass_dsq(sch, cpu)->nr);
6047
6048	after_min = min(nr, after_min);
6049	after_max = max(nr, after_max);
6050
6051	}
6052
6053	trace_sched_ext_bypass_lb(node, nr_cpus, nr_tasks, nr_balanced,
6054	before_min, before_max, after_min, after_max);
6055	}
6056
6057	/*
6058	* In bypass mode, all tasks are put on the per-CPU bypass DSQs. If the machine
6059	* is over-saturated and the BPF scheduler skewed tasks into few CPUs, some
6060	* bypass DSQs can be overloaded. If there are enough tasks to saturate other
6061	* lightly loaded CPUs, such imbalance can lead to very high execution latency
6062	* on the overloaded CPUs and thus to hung tasks and RCU stalls. To avoid such
6063	* outcomes, a simple load balancing mechanism is implemented by the following
6064	* timer which runs periodically while bypass mode is in effect.
6065	*/
6066	static void scx_bypass_lb_timerfn(struct timer_list *timer)
6067	{
6068	struct scx_sched *sch = container_of(timer, struct scx_sched, bypass_lb_timer);
6069	int node;
6070	u32 intv_us;
6071
6072	if (!scx_bypass_dsp_enabled(sch))
6073	return;
6074
6075	for_each_node_with_cpus(node)
6076	bypass_lb_node(sch, node);
6077
6078	intv_us = READ_ONCE(scx_bypass_lb_intv_us);
6079	if (intv_us)
6080	mod_timer(timer, jiffies + usecs_to_jiffies(intv_us));
6081	}
6082
6083	static bool inc_bypass_depth(struct scx_sched *sch)
6084	{
6085	lockdep_assert_held(&scx_bypass_lock);
6086
6087	WARN_ON_ONCE(sch->bypass_depth < 0);
6088	WRITE_ONCE(sch->bypass_depth, sch->bypass_depth + 1);
6089	if (sch->bypass_depth != 1)
6090	return false;
6091
6092	WRITE_ONCE(sch->slice_dfl, READ_ONCE(scx_slice_bypass_us) * NSEC_PER_USEC);
6093	sch->bypass_timestamp = ktime_get_ns();
6094	scx_add_event(sch, SCX_EV_BYPASS_ACTIVATE, 1);
6095	return true;
6096	}
6097
6098	static bool dec_bypass_depth(struct scx_sched *sch)
6099	{
6100	lockdep_assert_held(&scx_bypass_lock);
6101
6102	WARN_ON_ONCE(sch->bypass_depth < 1);
6103	WRITE_ONCE(sch->bypass_depth, sch->bypass_depth - 1);
6104	if (sch->bypass_depth != 0)
6105	return false;
6106
6107	WRITE_ONCE(sch->slice_dfl, SCX_SLICE_DFL);
6108	scx_add_event(sch, SCX_EV_BYPASS_DURATION,
6109	ktime_get_ns() - sch->bypass_timestamp);
6110	return true;
6111	}
6112
6113	static void enable_bypass_dsp(struct scx_sched *sch)
6114	{
6115	struct scx_sched *host = scx_parent(sch) ?: sch;
6116	u32 intv_us = READ_ONCE(scx_bypass_lb_intv_us);
6117	s32 ret;
6118
6119	/*
6120	* @sch->bypass_depth transitioning from 0 to 1 triggers enabling.
6121	* Shouldn't stagger.
6122	*/
6123	if (WARN_ON_ONCE(test_and_set_bit(0, &sch->bypass_dsp_claim)))
6124	return;
6125
6126	/*
6127	* When a sub-sched bypasses, its tasks are queued on the bypass DSQs of
6128	* the nearest non-bypassing ancestor or root. The parent hosts them
6129	* unless it is bypassing too. Then they sit further up and the parent
6130	* consumes only its own bypass DSQs, empty until it leaves bypass and
6131	* becomes the host. Either way the nearest non-bypassing ancestor or
6132	* root has bypass dispatch enabled while a descendant is bypassing,
6133	* which is all that's required.
6134	*
6135	* scx_bypass_dsp_enabled() test is used to determine whether to enter
6136	* the bypass dispatch handling path from both bypassing and hosting
6137	* scheds. Bump enable depth on both @sch and bypass dispatch host.
6138	*/
6139	ret = atomic_inc_return(&sch->bypass_dsp_enable_depth);
6140	WARN_ON_ONCE(ret <= 0);
6141
6142	if (host != sch) {
6143	ret = atomic_inc_return(&host->bypass_dsp_enable_depth);
6144	WARN_ON_ONCE(ret <= 0);
6145	}
6146
6147	/*
6148	* The LB timer will stop running if bypass dispatch is disabled. Start
6149	* after enabling bypass dispatch.
6150	*/
6151	if (intv_us && !timer_pending(&host->bypass_lb_timer))
6152	mod_timer(&host->bypass_lb_timer,
6153	jiffies + usecs_to_jiffies(intv_us));
6154	}
6155
6156	/* may be called without holding scx_bypass_lock */
6157	void scx_disable_bypass_dsp(struct scx_sched *sch)
6158	{
6159	s32 ret;
6160
6161	if (!test_and_clear_bit(0, &sch->bypass_dsp_claim))
6162	return;
6163
6164	ret = atomic_dec_return(&sch->bypass_dsp_enable_depth);
6165	WARN_ON_ONCE(ret < 0);
6166
6167	if (scx_parent(sch)) {
6168	ret = atomic_dec_return(&scx_parent(sch)->bypass_dsp_enable_depth);
6169	WARN_ON_ONCE(ret < 0);
6170	}
6171	}
6172
6173	/**
6174	* unbypass_renotify_idle - Arm an idle re-notify for a sched leaving bypass
6175	* @rq: rq of the cpu leaving bypass
6176	* @pos: scheduler that just left bypass on @rq's cpu
6177	* @pcpu: @pos's per-cpu state for @rq's cpu
6178	*
6179	* A sched leaving bypass is owed the ops.update_idle() calls suppressed while
6180	* bypassing. A cpu that goes idle during the bypass window and stays idle won't
6181	* produce a notification. Arm a re-notify that scx_bypass()'s resched flushes
6182	* on the next idle pick.
6183	*
6184	* An acute case is ops.sub_attach(). If the parent grants the child cids while
6185	* attaching, when attach is complete and bypass is lifted, the child may hold
6186	* idle cids it never saw go idle.
6187	*
6188	* The root is no exception as bypass suppresses its notifications the same way.
6189	* However, the root uses a separate per-rq flag so its re-notify keeps working
6190	* even when !CONFIG_EXT_SUB_SCHED.
6191	*/
6192	static void unbypass_renotify_idle(struct rq rq, struct scx_sched pos,
6193	struct scx_sched_pcpu *pcpu)
6194	{
6195	if (!pos->level) {
6196	rq->scx.flags \|= SCX_RQ_ROOT_IDLE_RENOTIFY;
6197	return;
6198	}
6199	#ifdef CONFIG_EXT_SUB_SCHED
6200	pcpu->idle_renotify = true;
6201	rq->scx.flags \|= SCX_RQ_SUB_IDLE_RENOTIFY;
6202	#endif
6203	}
6204
6205	/**
6206	* scx_bypass - [Un]bypass scx_ops and guarantee forward progress
6207	* @sch: sched to bypass
6208	* @bypass: true for bypass, false for unbypass
6209	*
6210	* Bypassing guarantees that all runnable tasks make forward progress without
6211	* trusting the BPF scheduler. We can't grab any mutexes or rwsems as they might
6212	* be held by tasks that the BPF scheduler is forgetting to run, which
6213	* unfortunately also excludes toggling the static branches.
6214	*
6215	* Let's work around by overriding a couple ops and modifying behaviors based on
6216	* the DISABLING state and then cycling the queued tasks through dequeue/enqueue
6217	* to force global FIFO scheduling.
6218	*
6219	* - ops.select_cpu() is ignored and the default select_cpu() is used.
6220	*
6221	* - ops.enqueue() is ignored and tasks are queued in simple global FIFO order.
6222	* %SCX_OPS_ENQ_LAST is also ignored.
6223	*
6224	* - ops.dispatch() is ignored.
6225	*
6226	* - dispatch_one() does not report %SCX_DSP_PREV on non-zero slice as slice
6227	* can't be trusted. Whenever a tick triggers, the running task is rotated to
6228	* the tail of the queue.
6229	*
6230	* - pick_next_task() suppresses zero slice warning.
6231	*
6232	* - scx_kick_cpu() is disabled to avoid irq_work malfunction during PM
6233	* operations.
6234	*
6235	* - scx_prio_less() reverts to the default runnable_at order.
6236	*/
6237	void scx_bypass(struct scx_sched *sch, bool bypass)
6238	{
6239	struct scx_sched *pos;
6240	unsigned long flags;
6241	int cpu;
6242	bool changed;
6243
6244	raw_spin_lock_irqsave(&scx_bypass_lock, flags);
6245
6246	if (bypass) {
6247	changed = inc_bypass_depth(sch);
6248	if (changed)
6249	enable_bypass_dsp(sch);
6250	} else {
6251	changed = dec_bypass_depth(sch);
6252	}
6253
6254	/*
6255	* Bypass state is propagated to all descendants - an scx_sched bypasses
6256	* if itself or any of its ancestors are in bypass mode. A descendant's
6257	* depth is the sum of its own bypasses and its ancestors', so every
6258	* call propagates whether or not @sch changed state. The descendants'
6259	* claims are released below with @sch's, after their tasks have moved.
6260	*/
6261	raw_spin_lock(&scx_sched_lock);
6262	scx_for_each_descendant_pre(pos, sch) {
6263	if (pos == sch)
6264	continue;
6265	if (bypass) {
6266	if (inc_bypass_depth(pos))
6267	enable_bypass_dsp(pos);
6268	} else {
6269	dec_bypass_depth(pos);
6270	}
6271	}
6272	raw_spin_unlock(&scx_sched_lock);
6273
6274	/*
6275	* A descendant's depth is at least @sch's, so no bypass state changed
6276	* unless @sch's did.
6277	*/
6278	if (!changed)
6279	goto unlock;
6280
6281	/*
6282	* No task property is changing. We just need to make sure all currently
6283	* queued tasks are re-queued according to the new scx_bypassing()
6284	* state. As an optimization, walk each rq's runnable_list instead of
6285	* the scx_tasks list.
6286	*
6287	* This function can't trust the scheduler and thus can't use
6288	* cpus_read_lock(). Walk all possible CPUs instead of online.
6289	*/
6290	for_each_possible_cpu(cpu) {
6291	struct rq *rq = cpu_rq(cpu);
6292	struct task_struct p, n;
6293
6294	raw_spin_rq_lock(rq);
6295	raw_spin_lock(&scx_sched_lock);
6296
6297	scx_for_each_descendant_pre(pos, sch) {
6298	struct scx_sched_pcpu *pcpu = per_cpu_ptr(pos->pcpu, cpu);
6299	bool was_bypassing = pcpu->flags & SCX_SCHED_PCPU_BYPASSING;
6300
6301	if (pos->bypass_depth) {
6302	pcpu->flags \|= SCX_SCHED_PCPU_BYPASSING;
6303	} else {
6304	pcpu->flags &= ~SCX_SCHED_PCPU_BYPASSING;
6305	if (was_bypassing) {
6306	unbypass_renotify_idle(rq, pos, pcpu);
6307	scx_unbypass_replay_ecaps(rq, pos);
6308	}
6309	}
6310	}
6311
6312	raw_spin_unlock(&scx_sched_lock);
6313
6314	/*
6315	* We need to guarantee that no tasks are on the BPF scheduler
6316	* while bypassing. Either we see enabled or the enable path
6317	* sees scx_bypassing() before moving tasks to SCX.
6318	*/
6319	if (!scx_enabled()) {
6320	scx_rq_lock_drop(rq);
6321	raw_spin_rq_unlock(rq);
6322	continue;
6323	}
6324
6325	/*
6326	* The use of list_for_each_entry_safe_reverse() is required
6327	* because each task is going to be removed from and added back
6328	* to the runnable_list during iteration. Because they're added
6329	* to the tail of the list, safe reverse iteration can still
6330	* visit all nodes.
6331	*/
6332	list_for_each_entry_safe_reverse(p, n, &rq->scx.runnable_list,
6333	scx.runnable_node) {
6334	if (!scx_is_descendant(scx_task_sched(p), sch))
6335	continue;
6336
6337	/*
6338	* Bypass trumps protection. Cycling clears for queued
6339	* tasks but current task needs explicit stripping.
6340	*/
6341	if (bypass && task_current(rq, p))
6342	scx_task_slice_ended(rq, p);
6343
6344	/* cycling deq/enq is enough, see the function comment */
6345	scoped_guard (sched_change, p, DEQUEUE_SAVE \| DEQUEUE_MOVE) {
6346	/* nothing */ ;
6347	}
6348	}
6349
6350	/* resched to restore ticks and idle state */
6351	if (cpu_online(cpu) \|\| cpu == smp_processor_id())
6352	resched_curr(rq);
6353
6354	scx_rq_lock_drop(rq);
6355	raw_spin_rq_unlock(rq);
6356	}
6357
6358	/* disarming must come after moving all tasks out of the bypass DSQs */
6359	if (!bypass) {
6360	guard(raw_spinlock)(&scx_sched_lock);
6361	scx_for_each_descendant_pre(pos, sch)
6362	if (!pos->bypass_depth)
6363	scx_disable_bypass_dsp(pos);
6364	}
6365	unlock:
6366	raw_spin_unlock_irqrestore(&scx_bypass_lock, flags);
6367	}
6368
6369	static void free_exit_info(struct scx_exit_info *ei)
6370	{
6371	kvfree(ei->dump);
6372	kfree(ei->msg);
6373	kfree(ei->bt);
6374	kfree(ei);
6375	}
6376
6377	static struct scx_exit_info *alloc_exit_info(size_t exit_dump_len)
6378	{
6379	struct scx_exit_info *ei;
6380
6381	ei = kzalloc_obj(*ei);
6382	if (!ei)
6383	return NULL;
6384
6385	ei->exit_cpu = -1;
6386	ei->bt = kzalloc_objs(ei->bt[0], SCX_EXIT_BT_LEN);
6387	ei->msg = kzalloc(SCX_EXIT_MSG_LEN, GFP_KERNEL);
6388	ei->dump = kvzalloc(exit_dump_len, GFP_KERNEL);
6389
6390	if (!ei->bt \|\| !ei->msg \|\| !ei->dump) {
6391	free_exit_info(ei);
6392	return NULL;
6393	}
6394
6395	return ei;
6396	}
6397
6398	static const char *scx_exit_reason(enum scx_exit_kind kind)
6399	{
6400	switch (kind) {
6401	case SCX_EXIT_UNREG:
6402	return "unregistered from user space";
6403	case SCX_EXIT_UNREG_BPF:
6404	return "unregistered from BPF";
6405	case SCX_EXIT_UNREG_KERN:
6406	return "unregistered from the main kernel";
6407	case SCX_EXIT_SYSRQ:
6408	return "disabled by sysrq-S";
6409	case SCX_EXIT_PARENT:
6410	return "parent exiting";
6411	case SCX_EXIT_PARENT_KILL:
6412	return "killed by parent scheduler";
6413	case SCX_EXIT_ERROR:
6414	return "runtime error";
6415	case SCX_EXIT_ERROR_BPF:
6416	return "scx_bpf_error";
6417	case SCX_EXIT_ERROR_STALL:
6418	return "runnable task stall";
6419	case SCX_EXIT_ERROR_REENQ:
6420	return "reenqueue limit";
6421	case SCX_EXIT_ERROR_RESCUE:
6422	return "rescue bandwidth overload";
6423	default:
6424	return "<UNKNOWN>";
6425	}
6426	}
6427
6428	static void free_kick_syncs(void)
6429	{
6430	int cpu;
6431
6432	for_each_possible_cpu(cpu) {
6433	struct scx_kick_syncs __rcu **ksyncs = per_cpu_ptr(&scx_kick_syncs, cpu);
6434	struct scx_kick_syncs *to_free;
6435
6436	/* flush the pending kick before freeing @ksyncs */
6437	irq_work_sync(&cpu_rq(cpu)->scx.kick_cpus_irq_work);
6438	to_free = rcu_replace_pointer(*ksyncs, NULL, true);
6439	if (to_free)
6440	kvfree_rcu(to_free, rcu);
6441	}
6442	}
6443
6444	static void refresh_watchdog(void)
6445	{
6446	struct scx_sched *sch;
6447	unsigned long intv = ULONG_MAX;
6448
6449	/* take the shortest timeout and use its half for watchdog interval */
6450	rcu_read_lock();
6451	list_for_each_entry_rcu(sch, &scx_sched_all, all)
6452	intv = max(min(intv, sch->watchdog_timeout / 2), 1);
6453	rcu_read_unlock();
6454
6455	WRITE_ONCE(scx_watchdog_timestamp, jiffies);
6456	WRITE_ONCE(scx_watchdog_interval, intv);
6457
6458	if (intv < ULONG_MAX)
6459	mod_delayed_work(system_dfl_wq, &scx_watchdog_work, intv);
6460	else
6461	cancel_delayed_work_sync(&scx_watchdog_work);
6462	}
6463
6464	s32 scx_link_sched(struct scx_sched *sch)
6465	{
6466	scoped_guard(raw_spinlock_irqsave, &scx_bypass_lock) /* for the parent bypass check */
6467	scoped_guard(raw_spinlock, &scx_sched_lock) {
6468	#ifdef CONFIG_EXT_SUB_SCHED
6469	struct scx_sched *parent = scx_parent(sch);
6470
6471	if (parent) {
6472	s32 ret;
6473
6474	/*
6475	* Bypass state is spread across per-cpu flags and a
6476	* depth count, so inheriting it is tricky and has no
6477	* valid use case. Refuse it.
6478	*/
6479	if (READ_ONCE(parent->bypass_depth)) {
6480	scx_error(sch, "parent bypassing (%d)", -EBUSY);
6481	return -EBUSY;
6482	}
6483
6484	ret = rhashtable_lookup_insert_fast(&scx_sched_hash,
6485	&sch->hash_node, scx_sched_hash_params);
6486	if (ret) {
6487	scx_error(sch, "failed to insert into scx_sched_hash (%d)",
6488	ret);
6489	return ret;
6490	}
6491
6492	list_add_tail_rcu(&sch->sibling, &parent->children);
6493
6494	/*
6495	* Pairs with the mb after the ->aborting assertion in
6496	* scx_claim_exit(). Either we see ->aborting and back
6497	* out, or the exit path sees us and exits us.
6498	*/
6499	smp_mb();
6500	if (unlikely(READ_ONCE(parent->aborting))) {
6501	rhashtable_remove_fast(&scx_sched_hash, &sch->hash_node,
6502	scx_sched_hash_params);
6503	list_del_rcu(&sch->sibling);
6504	scx_error(sch, "parent disabled (%d)", -ENOENT);
6505	return -ENOENT;
6506	}
6507
6508	sch->linked = true;
6509	}
6510	#endif /* CONFIG_EXT_SUB_SCHED */
6511
6512	list_add_tail_rcu(&sch->all, &scx_sched_all);
6513	}
6514
6515	refresh_watchdog();
6516	return 0;
6517	}
6518
6519	void scx_unlink_sched(struct scx_sched *sch)
6520	{
6521	scoped_guard(raw_spinlock_irq, &scx_sched_lock) {
6522	#ifdef CONFIG_EXT_SUB_SCHED
6523	if (sch->linked) {
6524	rhashtable_remove_fast(&scx_sched_hash, &sch->hash_node,
6525	scx_sched_hash_params);
6526	list_del_rcu(&sch->sibling);
6527	sch->linked = false;
6528	}
6529	#endif /* CONFIG_EXT_SUB_SCHED */
6530	list_del_rcu(&sch->all);
6531	}
6532
6533	refresh_watchdog();
6534	}
6535
6536	/*
6537	* Called to disable future dumps and wait for in-progress one while disabling
6538	* @sch. Once @sch becomes empty during disable, there's no point in dumping it.
6539	* This prevents calling dump ops on a dead sch.
6540	*/
6541	void scx_disable_dump(struct scx_sched *sch)
6542	{
6543	guard(raw_spinlock_irqsave)(&scx_dump_lock);
6544	sch->dump_disabled = true;
6545	}
6546
6547	void scx_log_sched_disable(struct scx_sched *sch)
6548	{
6549	struct scx_exit_info *ei = sch->exit_info;
6550	const char *type = scx_parent(sch) ? "sub-scheduler" : "scheduler";
6551
6552	if (ei->kind >= SCX_EXIT_ERROR) {
6553	pr_err("sched_ext: BPF %s \"%s\" disabled (%s)\n", type,
6554	sch->ops.name, ei->reason);
6555
6556	if (ei->msg[0] != '\0')
6557	pr_err("sched_ext: %s: %s\n", sch->ops.name, ei->msg);
6558	#ifdef CONFIG_STACKTRACE
6559	stack_trace_print(ei->bt, ei->bt_len, 2);
6560	#endif
6561	} else {
6562	pr_info("sched_ext: BPF %s \"%s\" disabled (%s)\n", type,
6563	sch->ops.name, ei->reason);
6564	}
6565	}
6566
6567	static void scx_root_disable(struct scx_sched *sch)
6568	{
6569	struct scx_task_iter sti;
6570	struct task_struct *p;
6571	bool was_switched_all;
6572	int cpu;
6573
6574	/* guarantee forward progress and wait for descendants to be disabled */
6575	scx_bypass(sch, true);
6576	drain_descendants(sch);
6577
6578	switch (scx_set_enable_state(SCX_DISABLING)) {
6579	case SCX_DISABLING:
6580	WARN_ONCE(true, "sched_ext: duplicate disabling instance?");
6581	break;
6582	case SCX_DISABLED:
6583	pr_warn("sched_ext: ops error detected without ops (%s)\n",
6584	sch->exit_info->msg);
6585	WARN_ON_ONCE(scx_set_enable_state(SCX_DISABLED) != SCX_DISABLING);
6586	goto done;
6587	default:
6588	break;
6589	}
6590
6591	/*
6592	* Here, every runnable task is guaranteed to make forward progress and
6593	* we can safely use blocking synchronization constructs. Actually
6594	* disable ops.
6595	*/
6596	mutex_lock(&scx_enable_mutex);
6597
6598	was_switched_all = scx_switched_all();
6599
6600	static_branch_disable(&__scx_switched_all);
6601	WRITE_ONCE(scx_switching_all, false);
6602
6603	/*
6604	* Shut down cgroup support before tasks so that the cgroup attach and
6605	* migration paths don't race against scx_disable_and_exit_task().
6606	*/
6607	scx_cgroup_lock();
6608	scx_cgroup_enabled = false;
6609	scx_cgroup_exit(sch);
6610	scx_cgroup_unlock();
6611
6612	/*
6613	* The BPF scheduler is going away. All tasks including %TASK_DEAD ones
6614	* must be switched out and exited synchronously.
6615	*/
6616	percpu_down_write(&scx_fork_rwsem);
6617
6618	scx_init_task_enabled = false;
6619
6620	scx_task_iter_start(&sti, NULL);
6621	while ((p = scx_task_iter_next_locked(&sti))) {
6622	unsigned int queue_flags = DEQUEUE_SAVE \| DEQUEUE_MOVE \| DEQUEUE_NOCLOCK;
6623	const struct sched_class *old_class = p->sched_class;
6624	const struct sched_class *new_class = scx_setscheduler_class(p);
6625
6626	update_rq_clock(task_rq(p));
6627
6628	if (old_class != new_class)
6629	queue_flags \|= DEQUEUE_CLASS;
6630
6631	scoped_guard (sched_change, p, queue_flags) {
6632	p->sched_class = new_class;
6633	}
6634
6635	scx_disable_and_exit_task(scx_task_sched(p), p);
6636	}
6637	scx_task_iter_stop(&sti);
6638
6639	scx_disable_dump(sch);
6640
6641	scx_cgroup_lock();
6642	set_cgroup_sched(sch_cgroup(sch), NULL);
6643	scx_cgroup_unlock();
6644
6645	percpu_up_write(&scx_fork_rwsem);
6646
6647	/*
6648	* Re-balance the dl_server bandwidth reservations: detach ext_server
6649	* (no more sched_ext tasks) and reinstate fair_server if it was
6650	* previously detached because we were running in full mode.
6651	*
6652	* Unlike the enable path, this runs on a recovery path that cannot
6653	* fail, so we use dl_server_swap_bw() to atomically free ext_server's
6654	* bandwidth and reclaim it for fair_server under the same dl_b lock.
6655	*
6656	* The swap can still fail with -EBUSY if someone bumped ext_server's
6657	* runtime via debugfs between enable and disable; in that narrow case
6658	* both servers end up detached and we just WARN.
6659	*/
6660	for_each_possible_cpu(cpu) {
6661	struct rq *rq = cpu_rq(cpu);
6662
6663	scoped_guard(rq_lock_irqsave, rq) {
6664	update_rq_clock(rq);
6665	if (was_switched_all) {
6666	if (WARN_ON_ONCE(dl_server_swap_bw(&rq->ext_server,
6667	&rq->fair_server)))
6668	pr_warn("failed to re-attach fair_server on CPU %d\n", cpu);
6669	} else {
6670	dl_server_detach_bw(&rq->ext_server);
6671	}
6672	}
6673	}
6674
6675	/* no task is on scx, turn off all the switches and flush in-progress calls */
6676	static_branch_disable(&__scx_enabled);
6677	static_branch_disable(&__scx_is_cid_type);
6678	if (sch->ops.flags & SCX_OPS_TID_TO_TASK)
6679	static_branch_disable(&__scx_tid_to_task_enabled);
6680	bitmap_zero(sch->has_op, SCX_OPI_END);
6681	scx_idle_disable();
6682	synchronize_rcu();
6683	if (sch->ops.flags & SCX_OPS_TID_TO_TASK)
6684	rhashtable_free_and_destroy(&scx_tid_hash, NULL, NULL);
6685
6686	scx_log_sched_disable(sch);
6687
6688	if (sch->ops.exit)
6689	SCX_CALL_OP(sch, exit, NULL, sch->exit_info);
6690
6691	/*
6692	* @sch's non-ops programs such as timers and tracers can fire after
6693	* ops.exit(). Now that exit is complete, stop scx_prog_sched() from
6694	* resolving to @sch and drain in-flight resolvers.
6695	*/
6696	WRITE_ONCE(sch->dead, true);
6697	synchronize_rcu();
6698
6699	scx_unlink_sched(sch);
6700
6701	/*
6702	* scx_root clearing and cid table retirement must be inside
6703	* cpus_read_lock(). See handle_hotplug().
6704	*/
6705	cpus_read_lock();
6706	RCU_INIT_POINTER(scx_root, NULL);
6707	scx_cid_retire_tables();
6708	cpus_read_unlock();
6709
6710	/*
6711	* Delete the kobject from the hierarchy synchronously. Otherwise, sysfs
6712	* could observe an object of the same name still in the hierarchy when
6713	* the next scheduler is loaded.
6714	*/
6715	#ifdef CONFIG_EXT_SUB_SCHED
6716	if (sch->sub_kset)
6717	kobject_del(&sch->sub_kset->kobj);
6718	#endif
6719	/* not added if enable failed before scx_sched_sysfs_add() */
6720	if (sch->kobj.state_in_sysfs)
6721	kobject_del(&sch->kobj);
6722
6723	free_kick_syncs();
6724
6725	mutex_unlock(&scx_enable_mutex);
6726
6727	WARN_ON_ONCE(scx_set_enable_state(SCX_DISABLED) != SCX_DISABLING);
6728	done:
6729	scx_bypass(sch, false);
6730	}
6731
6732	/**
6733	* scx_propagate_exit_irq_workfn - Claim SCX_EXIT_PARENT on the exiting subtree
6734	* @irq_work: &scx_sched.propagate_exit_irq_work
6735	*
6736	* Queued by scx_claim_exit() after a non-PARENT claim. Claims SCX_EXIT_PARENT
6737	* on each descendant, giving every one its own disable work - most of disabling
6738	* is serialized but ops.exit() can take arbitrarily long and running them in
6739	* separate helper kthreads parallelizes it. No recursion as only non-PARENT
6740	* claims propagate.
6741	*/
6742	static void scx_propagate_exit_irq_workfn(struct irq_work *irq_work)
6743	{
6744	struct scx_sched *sch = container_of(irq_work, struct scx_sched,
6745	propagate_exit_irq_work);
6746	struct scx_sched *pos;
6747
6748	scoped_guard (raw_spinlock_irqsave, &scx_sched_lock) {
6749	scx_for_each_descendant_pre(pos, sch)
6750	scx_disable(pos, SCX_EXIT_PARENT);
6751	}
6752	}
6753
6754	/*
6755	* Claim the exit on @sch. The caller must ensure that the helper kthread work
6756	* is kicked before the current task can be preempted. Once exit_kind is
6757	* claimed, scx_error() can no longer trigger, so if the current task gets
6758	* preempted and the BPF scheduler fails to schedule it back, the helper work
6759	* will never be kicked and the whole system can wedge.
6760	*
6761	* Lock-free and safe to call from any context including NMI.
6762	*/
6763	static bool scx_claim_exit(struct scx_sched *sch, enum scx_exit_kind kind)
6764	{
6765	int none = SCX_EXIT_NONE;
6766
6767	lockdep_assert_preemption_disabled();
6768
6769	if (WARN_ON_ONCE(kind == SCX_EXIT_NONE \|\| kind == SCX_EXIT_DONE))
6770	kind = SCX_EXIT_ERROR;
6771
6772	if (!atomic_try_cmpxchg(&sch->exit_kind, &none, kind))
6773	return false;
6774
6775	if (kind == SCX_EXIT_PARENT) {
6776	/* an ancestor is already sweeping the subtree */
6777	WRITE_ONCE(sch->aborting, true);
6778	} else {
6779	struct scx_sched *pos;
6780
6781	/*
6782	* CPUs may be live-locked in the dispatch paths of @sch or its
6783	* descendants, which ->aborting breaks. Sweep the subtree
6784	* locklessly so that this works from NMI. smp_store_mb() orders
6785	* each node's ->aborting store before its children are walked -
6786	* either we see a racing scx_link_sched() on ->children or it
6787	* sees ->aborting.
6788	*/
6789	scoped_guard (rcu) {
6790	scx_for_each_descendant_pre(pos, sch)
6791	smp_store_mb(pos->aborting, true);
6792	}
6793
6794	irq_work_queue(&sch->propagate_exit_irq_work);
6795	}
6796
6797	/* fired after ->aborting is set so callbacks can't delay recovery */
6798	trace_sched_ext_exit(sch, kind);
6799
6800	return true;
6801	}
6802
6803	static void scx_disable_workfn(struct kthread_work *work)
6804	{
6805	struct scx_sched *sch = container_of(work, struct scx_sched, disable_work);
6806	struct scx_exit_info *ei = sch->exit_info;
6807	int kind;
6808
6809	kind = atomic_read(&sch->exit_kind);
6810	while (true) {
6811	if (kind == SCX_EXIT_DONE) /* already disabled? */
6812	return;
6813	WARN_ON_ONCE(kind == SCX_EXIT_NONE);
6814	if (atomic_try_cmpxchg(&sch->exit_kind, &kind, SCX_EXIT_DONE))
6815	break;
6816	}
6817	ei->kind = kind;
6818	ei->reason = scx_exit_reason(ei->kind);
6819
6820	if (scx_parent(sch))
6821	scx_sub_disable(sch);
6822	else
6823	scx_root_disable(sch);
6824	}
6825
6826	static void scx_disable(struct scx_sched *sch, enum scx_exit_kind kind)
6827	{
6828	guard(preempt)();
6829	if (scx_claim_exit(sch, kind))
6830	irq_work_queue(&sch->disable_irq_work);
6831	}
6832
6833	/**
6834	* scx_flush_disable_work - flush the disable work and wait for it to finish
6835	* @sch: the scheduler
6836	*
6837	* sch->disable_work might still not queued, causing kthread_flush_work()
6838	* as a noop. Syncing the irq_work first is required to guarantee the
6839	* kthread work has been queued before waiting for it.
6840	*/
6841	void scx_flush_disable_work(struct scx_sched *sch)
6842	{
6843	int kind;
6844
6845	do {
6846	irq_work_sync(&sch->disable_irq_work);
6847	kthread_flush_work(&sch->disable_work);
6848	kind = atomic_read(&sch->exit_kind);
6849	} while (kind != SCX_EXIT_NONE && kind != SCX_EXIT_DONE);
6850	}
6851
6852	static void dump_newline(struct seq_buf *s)
6853	{
6854	trace_sched_ext_dump("");
6855
6856	/* @s may be zero sized and seq_buf triggers WARN if so */
6857	if (s->size)
6858	seq_buf_putc(s, '\n');
6859	}
6860
6861	__printf(2, 3) void scx_dump_line(struct seq_buf s, const char fmt, ...)
6862	{
6863	va_list args;
6864
6865	#ifdef CONFIG_TRACEPOINTS
6866	if (trace_sched_ext_dump_enabled()) {
6867	/* protected by scx_dump_lock */
6868	static char line_buf[SCX_EXIT_MSG_LEN];
6869
6870	va_start(args, fmt);
6871	vscnprintf(line_buf, sizeof(line_buf), fmt, args);
6872	va_end(args);
6873
6874	trace_call__sched_ext_dump(line_buf);
6875	}
6876	#endif
6877	/* @s may be zero sized and seq_buf triggers WARN if so */
6878	if (s->size) {
6879	va_start(args, fmt);
6880	seq_buf_vprintf(s, fmt, args);
6881	va_end(args);
6882
6883	seq_buf_putc(s, '\n');
6884	}
6885	}
6886
6887	static void dump_stack_trace(struct seq_buf s, const char prefix,
6888	const unsigned long *bt, unsigned int len)
6889	{
6890	unsigned int i;
6891
6892	for (i = 0; i < len; i++)
6893	scx_dump_line(s, "%s%pS", prefix, (void *)bt[i]);
6894	}
6895
6896	static void ops_dump_init(struct seq_buf s, const char prefix)
6897	{
6898	struct scx_dump_data *dd = &scx_dump_data;
6899
6900	lockdep_assert_irqs_disabled();
6901
6902	dd->cpu = smp_processor_id(); /* allow scx_bpf_dump() */
6903	dd->first = true;
6904	dd->cursor = 0;
6905	dd->s = s;
6906	dd->prefix = prefix;
6907	}
6908
6909	static void ops_dump_flush(void)
6910	{
6911	struct scx_dump_data *dd = &scx_dump_data;
6912	char *line = dd->buf.line;
6913
6914	if (!dd->cursor)
6915	return;
6916
6917	/*
6918	* There's something to flush and this is the first line. Insert a blank
6919	* line to distinguish ops dump.
6920	*/
6921	if (dd->first) {
6922	dump_newline(dd->s);
6923	dd->first = false;
6924	}
6925
6926	/*
6927	* There may be multiple lines in $line. Scan and emit each line
6928	* separately.
6929	*/
6930	while (true) {
6931	char *end = line;
6932	char c;
6933
6934	while (end != '\n' && end != '\0')
6935	end++;
6936
6937	/*
6938	* If $line overflowed, it may not have newline at the end.
6939	* Always emit with a newline.
6940	*/
6941	c = *end;
6942	*end = '\0';
6943	scx_dump_line(dd->s, "%s%s", dd->prefix, line);
6944	if (c == '\0')
6945	break;
6946
6947	/* move to the next line */
6948	end++;
6949	if (*end == '\0')
6950	break;
6951	line = end;
6952	}
6953
6954	dd->cursor = 0;
6955	}
6956
6957	static void ops_dump_exit(void)
6958	{
6959	ops_dump_flush();
6960	scx_dump_data.cpu = -1;
6961	}
6962
6963	static void scx_dump_task(struct scx_sched sch, struct seq_buf s, struct scx_dump_ctx *dctx,
6964	struct rq rq, struct task_struct p, char marker)
6965	{
6966	static unsigned long bt[SCX_EXIT_BT_LEN];
6967	struct scx_sched *task_sch = scx_task_sched(p);
6968	const char *own_marker;
6969	char sch_id_buf[32];
6970	char dsq_id_buf[19] = "(n/a)";
6971	unsigned long ops_state = atomic_long_read(&p->scx.ops_state);
6972	unsigned int bt_len = 0;
6973
6974	own_marker = task_sch == sch ? "*" : "";
6975
6976	if (task_sch->level == 0)
6977	scnprintf(sch_id_buf, sizeof(sch_id_buf), "root");
6978	else
6979	scnprintf(sch_id_buf, sizeof(sch_id_buf), "sub%d-%llu",
6980	task_sch->level, task_sch->ops.sub_cgroup_id);
6981
6982	if (p->scx.dsq)
6983	scnprintf(dsq_id_buf, sizeof(dsq_id_buf), "0x%llx",
6984	(unsigned long long)p->scx.dsq->id);
6985
6986	dump_newline(s);
6987	scx_dump_line(s, " %c%c %s[%d] %s%s %+ldms",
6988	marker, task_state_to_char(p), p->comm, p->pid, own_marker, sch_id_buf,
6989	jiffies_delta_msecs(p->scx.runnable_at, dctx->at_jiffies));
6990	scx_dump_line(s, " scx_state/flags=%u/0x%x dsq_flags=0x%x ops_state/qseq=%lu/%lu",
6991	scx_get_task_state(p) >> SCX_TASK_STATE_SHIFT,
6992	p->scx.flags & ~SCX_TASK_STATE_MASK, p->scx.dsq_flags,
6993	ops_state & SCX_OPSS_STATE_MASK, ops_state >> SCX_OPSS_QSEQ_SHIFT);
6994	scx_dump_line(s, " sticky/holding_cpu=%d/%d dsq_id=%s",
6995	p->scx.sticky_cpu, p->scx.holding_cpu, dsq_id_buf);
6996	scx_dump_line(s, " dsq_vtime=%llu slice=%llu weight=%u",
6997	p->scx.dsq_vtime, p->scx.slice, p->scx.weight);
6998	scx_dump_line(s, " cpus=%*pb no_mig=%u", cpumask_pr_args(p->cpus_ptr),
6999	p->migration_disabled);
7000
7001	if (SCX_HAS_OP(sch, dump_task)) {
7002	ops_dump_init(s, " ");
7003	SCX_CALL_OP(sch, dump_task, rq, dctx, p);
7004	ops_dump_exit();
7005	}
7006
7007	#ifdef CONFIG_STACKTRACE
7008	bt_len = stack_trace_save_tsk(p, bt, SCX_EXIT_BT_LEN, 1);
7009	#endif
7010	if (bt_len) {
7011	dump_newline(s);
7012	dump_stack_trace(s, " ", bt, bt_len);
7013	}
7014	}
7015
7016	static void scx_dump_cpu(struct scx_sched sch, struct seq_buf s,
7017	struct scx_dump_ctx *dctx, int cpu,
7018	bool dump_all_tasks)
7019	{
7020	struct rq *rq = cpu_rq(cpu);
7021	struct scx_sched_pcpu *pcpu = per_cpu_ptr(sch->pcpu, cpu);
7022	struct rq_flags rf;
7023	struct task_struct *p;
7024	struct seq_buf ns;
7025	size_t avail, used;
7026	char *buf;
7027	bool idle;
7028
7029	rq_lock_irqsave(rq, &rf);
7030
7031	idle = list_empty(&rq->scx.runnable_list) &&
7032	rq->curr->sched_class == &idle_sched_class;
7033
7034	if (idle && !SCX_HAS_OP(sch, dump_cpu))
7035	goto next;
7036
7037	/*
7038	* We don't yet know whether ops.dump_cpu() will produce output
7039	* and we may want to skip the default CPU dump if it doesn't.
7040	* Use a nested seq_buf to generate the standard dump so that we
7041	* can decide whether to commit later.
7042	*/
7043	avail = seq_buf_get_buf(s, &buf);
7044	seq_buf_init(&ns, buf, avail);
7045
7046	dump_newline(&ns);
7047	scx_dump_line(&ns, "CPU %-4d: nr_run=%u flags=0x%x cpu_rel=%d ksync=%lu",
7048	cpu, rq->scx.nr_running, rq->scx.flags, rq->scx.cpu_released,
7049	rq->scx.kick_sync);
7050	scx_rescue_dump(&ns, rq);
7051	scx_dump_line(&ns, " curr=%s[%d] class=%ps",
7052	rq->curr->comm, rq->curr->pid, rq->curr->sched_class);
7053	if (!cpumask_empty(pcpu->cpus_to_kick))
7054	scx_dump_line(&ns, " cpus_to_kick : %*pb",
7055	cpumask_pr_args(pcpu->cpus_to_kick));
7056	if (!cpumask_empty(pcpu->cpus_to_kick_if_idle))
7057	scx_dump_line(&ns, " idle_to_kick : %*pb",
7058	cpumask_pr_args(pcpu->cpus_to_kick_if_idle));
7059	if (!cpumask_empty(pcpu->cpus_to_preempt))
7060	scx_dump_line(&ns, " cpus_to_preempt: %*pb",
7061	cpumask_pr_args(pcpu->cpus_to_preempt));
7062	if (!cpumask_empty(pcpu->cpus_to_wait))
7063	scx_dump_line(&ns, " cpus_to_wait : %*pb",
7064	cpumask_pr_args(pcpu->cpus_to_wait));
7065	if (!cpumask_empty(rq->scx.cpus_to_sync))
7066	scx_dump_line(&ns, " cpus_to_sync : %*pb",
7067	cpumask_pr_args(rq->scx.cpus_to_sync));
7068
7069	used = seq_buf_used(&ns);
7070	if (SCX_HAS_OP(sch, dump_cpu)) {
7071	ops_dump_init(&ns, " ");
7072	SCX_CALL_OP(sch, dump_cpu, rq, dctx, scx_cpu_arg(cpu), idle);
7073	ops_dump_exit();
7074	}
7075
7076	/*
7077	* If idle && nothing generated by ops.dump_cpu(), there's
7078	* nothing interesting. Skip.
7079	*/
7080	if (idle && used == seq_buf_used(&ns))
7081	goto next;
7082
7083	/*
7084	* $s may already have overflowed when $ns was created. If so,
7085	* calling commit on it will trigger BUG.
7086	*/
7087	if (avail) {
7088	seq_buf_commit(s, seq_buf_used(&ns));
7089	if (seq_buf_has_overflowed(&ns))
7090	seq_buf_set_overflow(s);
7091	}
7092
7093	if (rq->curr->sched_class == &ext_sched_class &&
7094	(dump_all_tasks \|\| scx_task_on_sched(sch, rq->curr)))
7095	scx_dump_task(sch, s, dctx, rq, rq->curr, '*');
7096
7097	list_for_each_entry(p, &rq->scx.runnable_list, scx.runnable_node)
7098	if (dump_all_tasks \|\| scx_task_on_sched(sch, p))
7099	scx_dump_task(sch, s, dctx, rq, p, ' ');
7100	next:
7101	rq_unlock_irqrestore(rq, &rf);
7102	}
7103
7104	/*
7105	* Dump scheduler state. If @dump_all_tasks is true, dump all tasks regardless
7106	* of which scheduler they belong to. If false, only dump tasks owned by @sch.
7107	* For SysRq-D dumps, @dump_all_tasks=false since all schedulers are dumped
7108	* separately. For error dumps, @dump_all_tasks=true since only the failing
7109	* scheduler is dumped.
7110	*/
7111	static void scx_dump_state(struct scx_sched sch, struct scx_exit_info ei,
7112	size_t dump_len, bool dump_all_tasks)
7113	{
7114	static const char trunc_marker[] = "\n\n~~~~ TRUNCATED ~~~~\n";
7115	struct scx_dump_ctx dctx = {
7116	.kind = ei->kind,
7117	.exit_code = ei->exit_code,
7118	.reason = ei->reason,
7119	.at_ns = ktime_get_ns(),
7120	.at_jiffies = jiffies,
7121	};
7122	struct seq_buf s;
7123	struct scx_event_stats events;
7124	int cpu;
7125
7126	guard(raw_spinlock_irqsave)(&scx_dump_lock);
7127
7128	if (sch->dump_disabled)
7129	return;
7130
7131	seq_buf_init(&s, ei->dump, dump_len);
7132
7133	#ifdef CONFIG_EXT_SUB_SCHED
7134	if (sch->level == 0)
7135	scx_dump_line(&s, "%s: root", sch->ops.name);
7136	else
7137	scx_dump_line(&s, "%s: sub%d-%llu %s",
7138	sch->ops.name, sch->level, sch->ops.sub_cgroup_id,
7139	sch->cgrp_path);
7140	#endif
7141	if (ei->kind == SCX_EXIT_NONE) {
7142	scx_dump_line(&s, "Debug dump triggered by %s", ei->reason);
7143	} else {
7144	if (ei->exit_cpu >= 0)
7145	scx_dump_line(&s, "%s[%d] triggered exit kind %d on CPU %d:",
7146	current->comm, current->pid, ei->kind,
7147	ei->exit_cpu);
7148	else
7149	scx_dump_line(&s, "%s[%d] triggered exit kind %d:",
7150	current->comm, current->pid, ei->kind);
7151	scx_dump_line(&s, " %s (%s)", ei->reason, ei->msg);
7152	dump_newline(&s);
7153	scx_dump_line(&s, "Backtrace:");
7154	dump_stack_trace(&s, " ", ei->bt, ei->bt_len);
7155	}
7156
7157	if (SCX_HAS_OP(sch, dump)) {
7158	ops_dump_init(&s, "");
7159	SCX_CALL_OP(sch, dump, NULL, &dctx);
7160	ops_dump_exit();
7161	}
7162
7163	dump_newline(&s);
7164	scx_dump_line(&s, "CPU states");
7165	scx_dump_line(&s, "----------");
7166
7167	/*
7168	* Dump stalled CPUs first so they aren't lost to dump truncation, then
7169	* walk the rest in order. Fall back to exit_cpu if no stall mask set.
7170	*/
7171	if (!cpumask_empty(sch->stall_cpus)) {
7172	for_each_cpu(cpu, sch->stall_cpus)
7173	scx_dump_cpu(sch, &s, &dctx, cpu, dump_all_tasks);
7174	for_each_possible_cpu(cpu) {
7175	if (!cpumask_test_cpu(cpu, sch->stall_cpus))
7176	scx_dump_cpu(sch, &s, &dctx, cpu, dump_all_tasks);
7177	}
7178	} else {
7179	if (ei->exit_cpu >= 0)
7180	scx_dump_cpu(sch, &s, &dctx, ei->exit_cpu, dump_all_tasks);
7181	for_each_possible_cpu(cpu) {
7182	if (cpu != ei->exit_cpu)
7183	scx_dump_cpu(sch, &s, &dctx, cpu, dump_all_tasks);
7184	}
7185	}
7186
7187	dump_newline(&s);
7188	scx_dump_line(&s, "Event counters");
7189	scx_dump_line(&s, "--------------");
7190
7191	scx_read_events(sch, &events);
7192	#define SCX_EVENT(name) scx_dump_event(s, &events, name)
7193	SCX_EVENTS_LIST(SCX_EVENT);
7194	#undef SCX_EVENT
7195
7196	if (seq_buf_has_overflowed(&s) && dump_len >= sizeof(trunc_marker))
7197	memcpy(ei->dump + dump_len - sizeof(trunc_marker),
7198	trunc_marker, sizeof(trunc_marker));
7199	}
7200
7201	static void scx_disable_irq_workfn(struct irq_work *irq_work)
7202	{
7203	struct scx_sched *sch = container_of(irq_work, struct scx_sched, disable_irq_work);
7204	struct scx_exit_info *ei = sch->exit_info;
7205
7206	if (ei->kind >= SCX_EXIT_ERROR)
7207	scx_dump_state(sch, ei, sch->ops.exit_dump_len, true);
7208
7209	kthread_queue_work(sch->helper, &sch->disable_work);
7210	}
7211
7212	/* finish exit_info and kick the disable work, ei->msg must already be set */
7213	static void scx_finish_exit(struct scx_sched *sch, enum scx_exit_kind kind,
7214	s64 exit_code, s32 exit_cpu)
7215	{
7216	struct scx_exit_info *ei = sch->exit_info;
7217
7218	ei->exit_code = exit_code;
7219	#ifdef CONFIG_STACKTRACE
7220	/*
7221	* stack_trace_save()'s NMI-safety is arch-dependent and undocumented.
7222	* Skip the backtrace when exiting from NMI.
7223	*/
7224	if (kind >= SCX_EXIT_ERROR && !in_nmi())
7225	ei->bt_len = stack_trace_save(ei->bt, SCX_EXIT_BT_LEN, 1);
7226	#endif
7227	/*
7228	* Set ei->kind and ->reason for scx_dump_state(). They'll be set again
7229	* in scx_disable_workfn().
7230	*/
7231	ei->kind = kind;
7232	ei->reason = scx_exit_reason(ei->kind);
7233	ei->exit_cpu = exit_cpu;
7234
7235	irq_work_queue(&sch->disable_irq_work);
7236	}
7237
7238	bool scx_vexit(struct scx_sched *sch,
7239	enum scx_exit_kind kind, s64 exit_code, s32 exit_cpu,
7240	const char *fmt, va_list args)
7241	{
7242	struct scx_exit_info *ei = sch->exit_info;
7243
7244	guard(preempt)();
7245
7246	if (!scx_claim_exit(sch, kind))
7247	return false;
7248
7249	vscnprintf(ei->msg, SCX_EXIT_MSG_LEN, fmt, args);
7250
7251	scx_finish_exit(sch, kind, exit_code, exit_cpu);
7252	return true;
7253	}
7254
7255	static int alloc_kick_syncs(void)
7256	{
7257	int cpu;
7258
7259	/*
7260	* Allocate per-CPU arrays sized by nr_cpu_ids. Use kvzalloc as size
7261	* can exceed percpu allocator limits on large machines.
7262	*/
7263	for_each_possible_cpu(cpu) {
7264	struct scx_kick_syncs __rcu **ksyncs = per_cpu_ptr(&scx_kick_syncs, cpu);
7265	struct scx_kick_syncs *new_ksyncs;
7266
7267	WARN_ON_ONCE(rcu_access_pointer(*ksyncs));
7268
7269	new_ksyncs = kvzalloc_node(struct_size(new_ksyncs, syncs, nr_cpu_ids),
7270	GFP_KERNEL, cpu_to_node(cpu));
7271	if (!new_ksyncs) {
7272	free_kick_syncs();
7273	return -ENOMEM;
7274	}
7275
7276	rcu_assign_pointer(*ksyncs, new_ksyncs);
7277	}
7278
7279	return 0;
7280	}
7281
7282	static void free_pnode(struct scx_sched_pnode *pnode)
7283	{
7284	if (!pnode)
7285	return;
7286	exit_dsq(&pnode->global_dsq);
7287	kfree(pnode);
7288	}
7289
7290	static struct scx_sched_pnode alloc_pnode(struct scx_sched sch, int node)
7291	{
7292	struct scx_sched_pnode *pnode;
7293
7294	pnode = kzalloc_node(sizeof(*pnode), GFP_KERNEL, node);
7295	if (!pnode)
7296	return NULL;
7297
7298	if (scx_init_dsq(&pnode->global_dsq, SCX_DSQ_GLOBAL, sch)) {
7299	kfree(pnode);
7300	return NULL;
7301	}
7302
7303	return pnode;
7304	}
7305
7306	/*
7307	* Allocate and initialize a new scx_sched. @cgrp's reference is always
7308	* consumed whether the function succeeds or fails.
7309	*/
7310	struct scx_sched scx_alloc_and_add_sched(struct scx_enable_cmd cmd,
7311	struct cgroup *cgrp,
7312	struct scx_sched *parent)
7313	{
7314	struct sched_ext_ops *ops = cmd->ops;
7315	struct scx_sched *sch;
7316	s32 level = parent ? parent->level + 1 : 0;
7317	s32 node, cpu, ret, bypass_fail_cpu = nr_cpu_ids;
7318
7319	sch = kzalloc_flex(*sch, ancestors, level + 1);
7320	if (!sch) {
7321	ret = -ENOMEM;
7322	goto err_put_cgrp;
7323	}
7324
7325	sch->exit_info = alloc_exit_info(ops->exit_dump_len);
7326	if (!sch->exit_info) {
7327	ret = -ENOMEM;
7328	goto err_free_sch;
7329	}
7330
7331	ret = rhashtable_init(&sch->dsq_hash, &dsq_hash_params);
7332	if (ret < 0)
7333	goto err_free_ei;
7334
7335	sch->pnode = kzalloc_objs(sch->pnode[0], nr_node_ids);
7336	if (!sch->pnode) {
7337	ret = -ENOMEM;
7338	goto err_free_hash;
7339	}
7340
7341	for_each_node_state(node, N_POSSIBLE) {
7342	sch->pnode[node] = alloc_pnode(sch, node);
7343	if (!sch->pnode[node]) {
7344	ret = -ENOMEM;
7345	goto err_free_pnode;
7346	}
7347	}
7348
7349	sch->dsp_max_batch = ops->dispatch_max_batch ?: SCX_DSP_DFL_MAX_BATCH;
7350	sch->pcpu = __alloc_percpu(struct_size_t(struct scx_sched_pcpu,
7351	dsp_ctx.buf, sch->dsp_max_batch),
7352	__alignof__(struct scx_sched_pcpu));
7353	if (!sch->pcpu) {
7354	ret = -ENOMEM;
7355	goto err_free_pnode;
7356	}
7357
7358	for_each_possible_cpu(cpu) {
7359	ret = scx_init_dsq(scx_bypass_dsq(sch, cpu), SCX_DSQ_BYPASS, sch);
7360	if (ret) {
7361	bypass_fail_cpu = cpu;
7362	goto err_free_pcpu;
7363	}
7364	}
7365
7366	for_each_possible_cpu(cpu) {
7367	struct scx_sched_pcpu *pcpu = per_cpu_ptr(sch->pcpu, cpu);
7368
7369	node = cpu_to_node(cpu);
7370	pcpu->sch = sch;
7371	INIT_LIST_HEAD(&pcpu->deferred_reenq_local.node);
7372	#ifdef CONFIG_EXT_SUB_SCHED
7373	init_llist_node(&pcpu->ecaps_to_sync_node);
7374	#endif
7375	INIT_LIST_HEAD(&pcpu->to_kick_node);
7376	if (!zalloc_cpumask_var_node(&pcpu->cpus_to_kick, GFP_KERNEL, node) \|\|
7377	!zalloc_cpumask_var_node(&pcpu->cpus_to_kick_if_idle, GFP_KERNEL, node) \|\|
7378	!zalloc_cpumask_var_node(&pcpu->cpus_to_preempt, GFP_KERNEL, node) \|\|
7379	!zalloc_cpumask_var_node(&pcpu->cpus_to_wait, GFP_KERNEL, node)) {
7380	ret = -ENOMEM;
7381	goto err_free_pcpu;
7382	}
7383	}
7384
7385	sch->helper = kthread_run_worker(0, "sched_ext_helper");
7386	if (IS_ERR(sch->helper)) {
7387	ret = PTR_ERR(sch->helper);
7388	goto err_free_pcpu;
7389	}
7390
7391	sched_set_fifo(sch->helper->task);
7392
7393	if (parent)
7394	memcpy(sch->ancestors, parent->ancestors,
7395	level * sizeof(parent->ancestors[0]));
7396	sch->ancestors[level] = sch;
7397	sch->level = level;
7398	sch->id = atomic64_inc_return(&scx_sched_id_cursor);
7399
7400	if (ops->timeout_ms)
7401	sch->watchdog_timeout = msecs_to_jiffies(ops->timeout_ms);
7402	else
7403	sch->watchdog_timeout = SCX_WATCHDOG_MAX_TIMEOUT;
7404
7405	sch->slice_dfl = SCX_SLICE_DFL;
7406	atomic_set(&sch->exit_kind, SCX_EXIT_NONE);
7407	sch->disable_irq_work = IRQ_WORK_INIT_HARD(scx_disable_irq_workfn);
7408	sch->propagate_exit_irq_work = IRQ_WORK_INIT_HARD(scx_propagate_exit_irq_workfn);
7409	kthread_init_work(&sch->disable_work, scx_disable_workfn);
7410	timer_setup(&sch->bypass_lb_timer, scx_bypass_lb_timerfn, 0);
7411
7412	if (!alloc_cpumask_var(&sch->bypass_lb_donee_cpumask, GFP_KERNEL)) {
7413	ret = -ENOMEM;
7414	goto err_stop_helper;
7415	}
7416	if (!alloc_cpumask_var(&sch->bypass_lb_resched_cpumask, GFP_KERNEL)) {
7417	ret = -ENOMEM;
7418	goto err_free_lb_cpumask;
7419	}
7420	if (!zalloc_cpumask_var(&sch->stall_cpus, GFP_KERNEL)) {
7421	ret = -ENOMEM;
7422	goto err_free_lb_resched_cpumask;
7423	}
7424	/*
7425	* Copy ops through the right union view. For cid-form the source is
7426	* struct sched_ext_ops_cid which lacks the trailing cpu_acquire/
7427	* cpu_release; those stay zero from kzalloc.
7428	*/
7429	if (cmd->is_cid_type) {
7430	sch->ops_cid = *cmd->ops_cid;
7431	sch->is_cid_type = true;
7432	} else {
7433	sch->ops = *cmd->ops;
7434	}
7435
7436	#ifdef CONFIG_EXT_SUB_SCHED
7437	char *buf = kzalloc(PATH_MAX, GFP_KERNEL);
7438	if (!buf) {
7439	ret = -ENOMEM;
7440	goto err_free_lb_resched;
7441	}
7442	cgroup_path(cgrp, buf, PATH_MAX);
7443	sch->cgrp_path = kstrdup(buf, GFP_KERNEL);
7444	kfree(buf);
7445	if (!sch->cgrp_path) {
7446	ret = -ENOMEM;
7447	goto err_free_lb_resched;
7448	}
7449
7450	sch->cgrp = cgrp;
7451	INIT_LIST_HEAD(&sch->children);
7452	INIT_LIST_HEAD(&sch->sibling);
7453	#endif /* CONFIG_EXT_SUB_SCHED */
7454
7455	/*
7456	* Publishing makes @sch visible to scx_prog_sched() readers. Failure
7457	* paths after this point must free @sch through kobject_put() whose
7458	* release path defers the actual freeing by an RCU grace period.
7459	*/
7460	rcu_assign_pointer(ops->priv, sch);
7461
7462	sch->kobj.kset = scx_kset;
7463	INIT_LIST_HEAD(&sch->all);
7464
7465	#ifdef CONFIG_EXT_SUB_SCHED
7466	if (parent) {
7467	/*
7468	* Pin @parent for @sch's lifetime. The kobject hierarchy pins
7469	* it only via @parent->sub_kset, which is dropped during
7470	* disable. Released in scx_sched_free_rcu_work().
7471	*/
7472	kobject_get(&parent->kobj);
7473	}
7474	#endif /* CONFIG_EXT_SUB_SCHED */
7475
7476	/*
7477	* Init the kobj but don't add to sysfs yet. The enable path calls
7478	* scx_sched_sysfs_add() once @sch's sysfs-visible state is initialized.
7479	*/
7480	kobject_init(&sch->kobj, &scx_ktype);
7481
7482	/*
7483	* Consume the arena_map ref bpf_scx_reg_cid() took. Defer to here so
7484	* earlier failure paths leave cmd->arena_map set and bpf_scx_reg_cid
7485	* drops the ref. After this point, sch owns the ref and any cleanup
7486	* runs through scx_sched_free_rcu_work() which puts it.
7487	*/
7488	sch->arena_map = cmd->arena_map;
7489	/* BPF arena is only available on MMU && 64BIT */
7490	#if defined(CONFIG_MMU) && defined(CONFIG_64BIT)
7491	if (sch->arena_map)
7492	sch->arena_kern_base = bpf_arena_map_kern_vm_start(sch->arena_map);
7493	#endif
7494	cmd->arena_map = NULL;
7495	return sch;
7496
7497	#ifdef CONFIG_EXT_SUB_SCHED
7498	err_free_lb_resched:
7499	free_cpumask_var(sch->stall_cpus);
7500	#endif
7501	err_free_lb_resched_cpumask:
7502	free_cpumask_var(sch->bypass_lb_resched_cpumask);
7503	err_free_lb_cpumask:
7504	free_cpumask_var(sch->bypass_lb_donee_cpumask);
7505	err_stop_helper:
7506	kthread_destroy_worker(sch->helper);
7507	err_free_pcpu:
7508	for_each_possible_cpu(cpu) {
7509	struct scx_sched_pcpu *pcpu = per_cpu_ptr(sch->pcpu, cpu);
7510
7511	free_cpumask_var(pcpu->cpus_to_kick);
7512	free_cpumask_var(pcpu->cpus_to_kick_if_idle);
7513	free_cpumask_var(pcpu->cpus_to_preempt);
7514	free_cpumask_var(pcpu->cpus_to_wait);
7515	}
7516	for_each_possible_cpu(cpu) {
7517	if (cpu == bypass_fail_cpu)
7518	break;
7519	exit_dsq(scx_bypass_dsq(sch, cpu));
7520	}
7521	free_percpu(sch->pcpu);
7522	err_free_pnode:
7523	for_each_node_state(node, N_POSSIBLE)
7524	free_pnode(sch->pnode[node]);
7525	kfree(sch->pnode);
7526	err_free_hash:
7527	rhashtable_free_and_destroy(&sch->dsq_hash, NULL, NULL);
7528	err_free_ei:
7529	free_exit_info(sch->exit_info);
7530	err_free_sch:
7531	kfree(sch);
7532	err_put_cgrp:
7533	#ifdef CONFIG_EXT_SUB_SCHED
7534	cgroup_put(cgrp);
7535	#endif
7536	return ERR_PTR(ret);
7537	}
7538
7539	/*
7540	* Add @sch's kobject to sysfs, and create its sub_kset if the scheduler
7541	* implements ops.sub_attach. Called by the enable workfns once @sch's
7542	* sysfs-visible state is initialized.
7543	*/
7544	int scx_sched_sysfs_add(struct scx_sched *sch)
7545	{
7546	#ifdef CONFIG_EXT_SUB_SCHED
7547	struct scx_sched *parent = scx_parent(sch);
7548	int ret;
7549
7550	if (parent)
7551	ret = kobject_add(&sch->kobj, &parent->sub_kset->kobj,
7552	"sub-%llu", cgroup_id(sch_cgroup(sch)));
7553	else
7554	ret = kobject_add(&sch->kobj, NULL, "root");
7555	if (ret < 0)
7556	return ret;
7557
7558	if (sch->ops.sub_attach) {
7559	sch->sub_kset = kset_create_and_add("sub", NULL, &sch->kobj);
7560	if (!sch->sub_kset)
7561	return -ENOMEM;
7562	}
7563	return 0;
7564	#else
7565	return kobject_add(&sch->kobj, NULL, "root");
7566	#endif
7567	}
7568
7569	static int check_hotplug_seq(struct scx_sched *sch,
7570	const struct sched_ext_ops *ops)
7571	{
7572	unsigned long long global_hotplug_seq;
7573
7574	/*
7575	* If a hotplug event has occurred between when a scheduler was
7576	* initialized, and when we were able to attach, exit and notify user
7577	* space about it.
7578	*/
7579	if (ops->hotplug_seq) {
7580	global_hotplug_seq = atomic_long_read(&scx_hotplug_seq);
7581	if (ops->hotplug_seq != global_hotplug_seq) {
7582	scx_exit(sch, SCX_EXIT_UNREG_KERN,
7583	SCX_ECODE_ACT_RESTART \| SCX_ECODE_RSN_HOTPLUG,
7584	"expected hotplug seq %llu did not match actual %llu",
7585	ops->hotplug_seq, global_hotplug_seq);
7586	return -EBUSY;
7587	}
7588	}
7589
7590	return 0;
7591	}
7592
7593	int scx_validate_ops(struct scx_sched sch, const struct sched_ext_ops ops)
7594	{
7595	/*
7596	* It doesn't make sense to specify the SCX_OPS_ENQ_LAST flag if the
7597	* ops.enqueue() callback isn't implemented.
7598	*/
7599	if ((ops->flags & SCX_OPS_ENQ_LAST) && !ops->enqueue) {
7600	scx_error(sch, "SCX_OPS_ENQ_LAST requires ops.enqueue() to be implemented");
7601	return -EINVAL;
7602	}
7603
7604	/*
7605	* SCX_OPS_TID_TO_TASK is enabled by the root scheduler. A sub-sched
7606	* may set it to declare a dependency; reject if the root hasn't
7607	* enabled it.
7608	*/
7609	if ((ops->flags & SCX_OPS_TID_TO_TASK) && scx_parent(sch) &&
7610	!(sch->ancestors[0]->ops.flags & SCX_OPS_TID_TO_TASK)) {
7611	scx_error(sch, "SCX_OPS_TID_TO_TASK requires root scheduler to enable it");
7612	return -EINVAL;
7613	}
7614
7615	/*
7616	* SCX_OPS_BUILTIN_IDLE_PER_NODE requires built-in CPU idle
7617	* selection policy to be enabled.
7618	*/
7619	if ((ops->flags & SCX_OPS_BUILTIN_IDLE_PER_NODE) &&
7620	(ops->update_idle && !(ops->flags & SCX_OPS_KEEP_BUILTIN_IDLE))) {
7621	scx_error(sch, "SCX_OPS_BUILTIN_IDLE_PER_NODE requires CPU idle selection enabled");
7622	return -EINVAL;
7623	}
7624
7625	/*
7626	* cid-form's struct is shorter and doesn't include the cpu_acquire /
7627	* cpu_release tail; reading those fields off a cid-form @ops would
7628	* run past the BPF allocation. Skip for cid-form.
7629	*/
7630	if (!sch->is_cid_type && (ops->cpu_acquire \|\| ops->cpu_release))
7631	pr_warn_ratelimited("ops->cpu_acquire/release() are deprecated, use sched_switch TP instead\n");
7632
7633	/*
7634	* Sub-scheduler support is tied to the cid-form struct_ops. A sub-sched
7635	* attaches through a cid-form-only interface (sub_attach/sub_detach),
7636	* and a root that accepts sub-scheds must expose cid-form state to
7637	* them. Reject cpu-form schedulers on either side.
7638	*/
7639	if (!sch->is_cid_type) {
7640	if (scx_parent(sch)) {
7641	scx_error(sch, "sub-sched requires cid-form struct_ops");
7642	return -EINVAL;
7643	}
7644	if (ops->sub_attach \|\| ops->sub_detach) {
7645	scx_error(sch, "sub_attach/sub_detach requires cid-form struct_ops");
7646	return -EINVAL;
7647	}
7648	}
7649
7650	return 0;
7651	}
7652
7653	static void scx_root_enable_workfn(struct kthread_work *work)
7654	{
7655	struct scx_enable_cmd *cmd = container_of(work, struct scx_enable_cmd, work);
7656	struct sched_ext_ops *ops = cmd->ops;
7657	struct cgroup *cgrp = root_cgroup();
7658	struct scx_sched *sch;
7659	struct scx_task_iter sti;
7660	struct task_struct *p;
7661	int i, cpu, ret;
7662
7663	mutex_lock(&scx_enable_mutex);
7664
7665	if (scx_enable_state() != SCX_DISABLED) {
7666	ret = -EBUSY;
7667	goto err_unlock;
7668	}
7669
7670	/*
7671	* @ops->priv binds @ops to its scx_sched instance. It is set here by
7672	* scx_alloc_and_add_sched() and cleared at the tail of bpf_scx_unreg(),
7673	* which runs after scx_root_disable() has dropped scx_enable_mutex. If
7674	* it's still non-NULL here, a previous attachment on @ops has not
7675	* finished tearing down; proceeding would let the in-flight unreg's
7676	* RCU_INIT_POINTER(NULL) clobber the @ops->priv we are about to assign.
7677	*/
7678	if (rcu_access_pointer(ops->priv)) {
7679	ret = -EBUSY;
7680	goto err_unlock;
7681	}
7682
7683	ret = alloc_kick_syncs();
7684	if (ret)
7685	goto err_unlock;
7686
7687	if (ops->flags & SCX_OPS_TID_TO_TASK) {
7688	ret = rhashtable_init(&scx_tid_hash, &scx_tid_hash_params);
7689	if (ret)
7690	goto err_free_ksyncs;
7691	}
7692
7693	#ifdef CONFIG_EXT_SUB_SCHED
7694	cgroup_get(cgrp);
7695	#endif
7696	/*
7697	* Transition to ENABLING to arm the disable path. Allocation failure
7698	* still unwinds locally. Full disabling on failure applies only after
7699	* scx_alloc_and_add_sched() succeeds.
7700	*/
7701	WARN_ON_ONCE(scx_set_enable_state(SCX_ENABLING) != SCX_DISABLED);
7702	WARN_ON_ONCE(scx_root);
7703
7704	sch = scx_alloc_and_add_sched(cmd, cgrp, NULL);
7705	if (IS_ERR(sch)) {
7706	ret = PTR_ERR(sch);
7707	WARN_ON_ONCE(scx_set_enable_state(SCX_DISABLED) != SCX_ENABLING);
7708	goto err_free_tid_hash;
7709	}
7710
7711	if (sch->is_cid_type)
7712	static_branch_enable(&__scx_is_cid_type);
7713
7714	atomic_long_set(&scx_nr_rejected, 0);
7715
7716	for_each_possible_cpu(cpu) {
7717	struct rq *rq = cpu_rq(cpu);
7718
7719	rq->scx.local_dsq.sched = sch;
7720	rq->scx.cpuperf_target = SCX_CPUPERF_ONE;
7721	}
7722
7723	scx_discard_stale_ecaps_syncs();
7724	scx_rescue_set_knobs(sch);
7725
7726	/*
7727	* Keep CPUs stable during enable so that the BPF scheduler can track
7728	* online CPUs by watching ->on/offline_cpu() after ->init().
7729	*/
7730	cpus_read_lock();
7731
7732	/*
7733	* Build the cid mapping into a private under-construction set. It
7734	* becomes visible to readers only through scx_cid_publish_tables() once
7735	* ops.init_cids() has finalized the layout.
7736	*/
7737	ret = scx_cid_init(sch);
7738	if (ret) {
7739	cpus_read_unlock();
7740	goto err_disable;
7741	}
7742
7743	/*
7744	* Make the scheduler instance visible. Must be inside cpus_read_lock().
7745	* See handle_hotplug().
7746	*/
7747	rcu_assign_pointer(scx_root, sch);
7748
7749	ret = scx_link_sched(sch);
7750	if (ret) {
7751	cpus_read_unlock();
7752	goto err_disable;
7753	}
7754
7755	scx_idle_enable(ops);
7756
7757	/*
7758	* A cid-form scheduler finalizes its cid layout in ops.init_cids(),
7759	* which may call scx_bpf_cid_override(). Run it before the caps and
7760	* shard state are built so the final layout is in effect.
7761	*/
7762	if (sch->is_cid_type && sch->ops_cid.init_cids) {
7763	ret = SCX_CALL_OP_RET(sch, init_cids, NULL);
7764	if (ret) {
7765	ret = scx_ops_sanitize_err(sch, "init_cids", ret);
7766	cpus_read_unlock();
7767	scx_error(sch, "ops.init_cids() failed (%d)", ret);
7768	goto err_disable;
7769	}
7770	}
7771
7772	/* the cid layout is final, expose it to readers */
7773	scx_cid_publish_tables();
7774
7775	ret = scx_arena_pool_init(sch);
7776	if (ret) {
7777	cpus_read_unlock();
7778	goto err_disable;
7779	}
7780
7781	ret = scx_alloc_kern_arena_objs(sch);
7782	if (ret) {
7783	cpus_read_unlock();
7784	goto err_disable;
7785	}
7786
7787	ret = scx_alloc_pshards(sch);
7788	if (ret) {
7789	cpus_read_unlock();
7790	goto err_disable;
7791	}
7792
7793	scx_init_root_caps(sch);
7794
7795	/* the cid caps and shards are live now, so ops.init() can query them */
7796	if (sch->ops.init) {
7797	ret = SCX_CALL_OP_RET(sch, init, NULL);
7798	if (ret) {
7799	ret = scx_ops_sanitize_err(sch, "init", ret);
7800	cpus_read_unlock();
7801	scx_error(sch, "ops.init() failed (%d)", ret);
7802	goto err_disable;
7803	}
7804	sch->exit_info->flags \|= SCX_EFLAG_INITIALIZED;
7805	}
7806
7807	ret = scx_sched_sysfs_add(sch);
7808	if (ret) {
7809	cpus_read_unlock();
7810	goto err_disable;
7811	}
7812
7813	for (i = SCX_OPI_CPU_HOTPLUG_BEGIN; i < SCX_OPI_CPU_HOTPLUG_END; i++)
7814	if (((void (**)(void))ops)[i])
7815	set_bit(i, sch->has_op);
7816
7817	ret = check_hotplug_seq(sch, ops);
7818	if (ret) {
7819	cpus_read_unlock();
7820	goto err_disable;
7821	}
7822	scx_idle_update_selcpu_topology(ops);
7823
7824	cpus_read_unlock();
7825
7826	ret = scx_validate_ops(sch, ops);
7827	if (ret)
7828	goto err_disable;
7829
7830	/*
7831	* Attach the ext_server bandwidth reservation before anything is
7832	* committed so that we can fail the enable if the root domain cannot
7833	* accommodate it. The matching fair_server detach is deferred to the
7834	* tail of this function, after the switch is fully committed and can no
7835	* longer fail.
7836	*
7837	* On failure, err_disable funnels into scx_root_disable() which
7838	* detaches ext_server, so partially-attached state is cleaned up
7839	* automatically.
7840	*/
7841	for_each_possible_cpu(cpu) {
7842	struct rq *rq = cpu_rq(cpu);
7843
7844	scoped_guard(rq_lock_irqsave, rq) {
7845	update_rq_clock(rq);
7846	ret = dl_server_attach_bw(&rq->ext_server);
7847	}
7848	if (ret) {
7849	pr_warn("sched_ext: failed to attach ext_server on CPU %d (%d)\n",
7850	cpu, ret);
7851	goto err_disable;
7852	}
7853	}
7854
7855	/*
7856	* Once __scx_enabled is set, %current can be switched to SCX anytime.
7857	* This can lead to stalls as some BPF schedulers (e.g. userspace
7858	* scheduling) may not function correctly before all tasks are switched.
7859	* Init in bypass mode to guarantee forward progress.
7860	*/
7861	scx_bypass(sch, true);
7862
7863	for (i = SCX_OPI_NORMAL_BEGIN; i < SCX_OPI_NORMAL_END; i++)
7864	if (((void (**)(void))ops)[i])
7865	set_bit(i, sch->has_op);
7866
7867	if (sch->ops.cpu_acquire \|\| sch->ops.cpu_release)
7868	sch->ops.flags \|= SCX_OPS_HAS_CPU_PREEMPT;
7869
7870	/*
7871	* Lock out forks, cgroup on/offlining and moves before opening the
7872	* floodgate so that they don't wander into the operations prematurely.
7873	*/
7874	percpu_down_write(&scx_fork_rwsem);
7875
7876	WARN_ON_ONCE(scx_init_task_enabled);
7877	scx_init_task_enabled = true;
7878
7879	/* flip under fork_rwsem; the iter below covers existing tasks */
7880	if (ops->flags & SCX_OPS_TID_TO_TASK)
7881	static_branch_enable(&__scx_tid_to_task_enabled);
7882
7883	/*
7884	* Enable ops for every task. Fork is excluded by scx_fork_rwsem
7885	* preventing new tasks from being added. No need to exclude tasks
7886	* leaving as sched_ext_dead() can handle both prepped and enabled
7887	* tasks. Prep all tasks first and then enable them with preemption
7888	* disabled.
7889	*
7890	* All cgroups should be initialized before scx_init_task() so that the
7891	* BPF scheduler can reliably track each task's cgroup membership from
7892	* scx_init_task(). Lock out cgroup on/offlining and task migrations
7893	* while tasks are being initialized so that scx_cgroup_can_attach()
7894	* never sees uninitialized tasks.
7895	*/
7896	scx_cgroup_lock();
7897	set_cgroup_sched(sch_cgroup(sch), sch);
7898	ret = scx_cgroup_init(sch);
7899	if (ret)
7900	goto err_disable_unlock_all;
7901
7902	WARN_ON_ONCE(scx_cgroup_enabled);
7903	scx_cgroup_enabled = true;
7904
7905	scx_task_iter_start(&sti, NULL);
7906	while ((p = scx_task_iter_next_locked(&sti))) {
7907	/*
7908	* @p is in scx_tasks under scx_tasks_lock, and SCX_TASK_DEAD
7909	* tasks are filtered by scx_task_iter_next_locked().
7910	* sched_ext_dead() removes @p from scx_tasks under the same
7911	* lock before put_task_struct_rcu_user() runs, so @p->usage
7912	* is guaranteed > 0 here.
7913	*/
7914	get_task_struct(p);
7915
7916	/*
7917	* Set %INIT_BEGIN under the iter's rq lock so that a concurrent
7918	* sched_ext_dead() does not call ops.exit_task() on @p while
7919	* ops.init_task() is running. If sched_ext_dead() runs before
7920	* this store, it has already removed @p from scx_tasks and the
7921	* iter won't visit @p; if it runs after, it observes
7922	* %INIT_BEGIN and transitions to %DEAD without calling ops,
7923	* leaving the post-init recheck below to unwind.
7924	*/
7925	scx_set_task_state(p, SCX_TASK_INIT_BEGIN);
7926	scx_task_iter_unlock(&sti);
7927
7928	ret = __scx_init_task(sch, p, NULL, false);
7929
7930	scx_task_iter_relock(&sti, p);
7931
7932	if (unlikely(ret)) {
7933	if (scx_get_task_state(p) != SCX_TASK_DEAD)
7934	scx_set_task_state(p, SCX_TASK_NONE);
7935	scx_task_iter_stop(&sti);
7936	scx_error(sch, "ops.init_task() failed (%d) for %s[%d]",
7937	ret, p->comm, p->pid);
7938	put_task_struct(p);
7939	goto err_disable_unlock_all;
7940	}
7941
7942	if (scx_get_task_state(p) == SCX_TASK_DEAD) {
7943	/*
7944	* sched_ext_dead() observed %INIT_BEGIN and set %DEAD.
7945	* ops.exit_task() is owed to the sched __scx_init_task()
7946	* ran against; call it now.
7947	*/
7948	scx_sub_init_cancel_task(sch, p);
7949	} else {
7950	scx_set_task_state(p, SCX_TASK_INIT);
7951	scx_set_task_sched(p, sch);
7952	scx_set_task_state(p, SCX_TASK_READY);
7953	}
7954
7955	/*
7956	* Insert into the tid hash. scx_tasks_lock is held by the iter;
7957	* list_empty() guards against sched_ext_dead() having taken @p
7958	* off the list while init ran unlocked.
7959	*/
7960	if (scx_tid_to_task_enabled() && !list_empty(&p->scx.tasks_node))
7961	scx_tid_hash_insert(p);
7962
7963	put_task_struct(p);
7964	}
7965	scx_task_iter_stop(&sti);
7966	scx_cgroup_unlock();
7967	percpu_up_write(&scx_fork_rwsem);
7968
7969	/*
7970	* All tasks are READY. It's safe to turn on scx_enabled() and switch
7971	* all eligible tasks.
7972	*/
7973	WRITE_ONCE(scx_switching_all, !(ops->flags & SCX_OPS_SWITCH_PARTIAL));
7974	static_branch_enable(&__scx_enabled);
7975
7976	/*
7977	* We're fully committed and can't fail. The task READY -> ENABLED
7978	* transitions here are synchronized against sched_ext_dead() through
7979	* scx_tasks_lock.
7980	*/
7981	percpu_down_write(&scx_fork_rwsem);
7982	scx_task_iter_start(&sti, NULL);
7983	while ((p = scx_task_iter_next_locked(&sti))) {
7984	unsigned int queue_flags = DEQUEUE_SAVE \| DEQUEUE_MOVE;
7985	const struct sched_class *old_class = p->sched_class;
7986	const struct sched_class *new_class = scx_setscheduler_class(p);
7987
7988	if (scx_get_task_state(p) != SCX_TASK_READY)
7989	continue;
7990
7991	if (old_class != new_class)
7992	queue_flags \|= DEQUEUE_CLASS;
7993
7994	scoped_guard (sched_change, p, queue_flags) {
7995	scx_set_task_slice(p, READ_ONCE(sch->slice_dfl));
7996	p->sched_class = new_class;
7997	}
7998	}
7999	scx_task_iter_stop(&sti);
8000	percpu_up_write(&scx_fork_rwsem);
8001
8002	scx_bypass(sch, false);
8003
8004	if (!scx_tryset_enable_state(SCX_ENABLED, SCX_ENABLING)) {
8005	WARN_ON_ONCE(atomic_read(&sch->exit_kind) == SCX_EXIT_NONE);
8006	ret = -EBUSY;
8007	goto err_disable;
8008	}
8009
8010	if (!(ops->flags & SCX_OPS_SWITCH_PARTIAL))
8011	static_branch_enable(&__scx_switched_all);
8012
8013	/*
8014	* Detach the fair_server bandwidth reservation now that the switch
8015	* is fully committed. In full mode (!SCX_OPS_SWITCH_PARTIAL) no
8016	* task will ever run in the fair class, so give that bandwidth
8017	* back to the RT class. The matching ext_server attach already
8018	* happened earlier; this only releases bandwidth and cannot fail.
8019	*
8020	* In partial mode keep fair_server attached.
8021	*/
8022	if (scx_switched_all()) {
8023	for_each_possible_cpu(cpu) {
8024	struct rq *rq = cpu_rq(cpu);
8025
8026	guard(rq_lock_irqsave)(rq);
8027	update_rq_clock(rq);
8028	dl_server_detach_bw(&rq->fair_server);
8029	}
8030	}
8031
8032	pr_info("sched_ext: BPF scheduler \"%s\" enabled%s\n",
8033	sch->ops.name, scx_switched_all() ? "" : " (partial)");
8034	kobject_uevent(&sch->kobj, KOBJ_ADD);
8035	mutex_unlock(&scx_enable_mutex);
8036
8037	atomic_long_inc(&scx_enable_seq);
8038
8039	cmd->ret = 0;
8040	return;
8041
8042	err_free_tid_hash:
8043	if (ops->flags & SCX_OPS_TID_TO_TASK)
8044	rhashtable_free_and_destroy(&scx_tid_hash, NULL, NULL);
8045	err_free_ksyncs:
8046	free_kick_syncs();
8047	err_unlock:
8048	mutex_unlock(&scx_enable_mutex);
8049	cmd->ret = ret;
8050	return;
8051
8052	err_disable_unlock_all:
8053	scx_cgroup_unlock();
8054	percpu_up_write(&scx_fork_rwsem);
8055	/* we'll soon enter disable path, keep bypass on */
8056	err_disable:
8057	mutex_unlock(&scx_enable_mutex);
8058	/*
8059	* Returning an error code here would not pass all the error information
8060	* to userspace. Record errno using scx_error() for cases scx_error()
8061	* wasn't already invoked and exit indicating success so that the error
8062	* is notified through ops.exit() with all the details.
8063	*
8064	* Flush scx_disable_work to ensure that error is reported before init
8065	* completion. sch's base reference will be put by bpf_scx_unreg().
8066	*/
8067	scx_error(sch, "scx_root_enable() failed (%d)", ret);
8068	scx_flush_disable_work(sch);
8069	cmd->ret = 0;
8070	}
8071
8072	static s32 scx_enable(struct scx_enable_cmd cmd, struct bpf_link link)
8073	{
8074	static struct kthread_worker *helper;
8075	static DEFINE_MUTEX(helper_mutex);
8076
8077	if (housekeeping_enabled(HK_TYPE_DOMAIN_BOOT)) {
8078	pr_err("sched_ext: Not compatible with \"isolcpus=\" domain isolation\n");
8079	return -EINVAL;
8080	}
8081
8082	if (!READ_ONCE(helper)) {
8083	mutex_lock(&helper_mutex);
8084	if (!helper) {
8085	struct kthread_worker *w =
8086	kthread_run_worker(0, "scx_enable_helper");
8087	if (IS_ERR_OR_NULL(w)) {
8088	mutex_unlock(&helper_mutex);
8089	return -ENOMEM;
8090	}
8091	sched_set_fifo(w->task);
8092	WRITE_ONCE(helper, w);
8093	}
8094	mutex_unlock(&helper_mutex);
8095	}
8096
8097	#ifdef CONFIG_EXT_SUB_SCHED
8098	if (cmd->ops->sub_cgroup_id > 1)
8099	kthread_init_work(&cmd->work, scx_sub_enable_workfn);
8100	else
8101	#endif /* CONFIG_EXT_SUB_SCHED */
8102	kthread_init_work(&cmd->work, scx_root_enable_workfn);
8103
8104	kthread_queue_work(READ_ONCE(helper), &cmd->work);
8105	kthread_flush_work(&cmd->work);
8106	return cmd->ret;
8107	}
8108
8109
8110	/********************************************************************************
8111	* bpf_struct_ops plumbing.
8112	*/
8113	#include <linux/bpf_verifier.h>
8114	#include <linux/bpf.h>
8115	#include <linux/btf.h>
8116
8117	static const struct btf_type *task_struct_type;
8118
8119	static bool bpf_scx_is_valid_access(int off, int size,
8120	enum bpf_access_type type,
8121	const struct bpf_prog *prog,
8122	struct bpf_insn_access_aux *info)
8123	{
8124	if (type != BPF_READ)
8125	return false;
8126	if (off < 0 \|\| off >= sizeof(__u64) * MAX_BPF_FUNC_ARGS)
8127	return false;
8128	if (off % size != 0)
8129	return false;
8130
8131	return btf_ctx_access(off, size, type, prog, info);
8132	}
8133
8134	/* common to both forms: only scx.disallow is writable */
8135	static int bpf_scx_btf_struct_access_common(const struct bpf_reg_state *reg,
8136	int off, int size)
8137	{
8138	const struct btf_type *t;
8139
8140	t = btf_type_by_id(reg->btf, reg->btf_id);
8141	if (t == task_struct_type &&
8142	off >= offsetof(struct task_struct, scx.disallow) &&
8143	off + size <= offsetofend(struct task_struct, scx.disallow))
8144	return SCALAR_VALUE;
8145
8146	return -EACCES;
8147	}
8148
8149	static int bpf_scx_btf_struct_access(struct bpf_verifier_log *log,
8150	const struct bpf_reg_state *reg, int off,
8151	int size)
8152	{
8153	const struct btf_type *t;
8154
8155	t = btf_type_by_id(reg->btf, reg->btf_id);
8156	if (t == task_struct_type) {
8157	if ((off >= offsetof(struct task_struct, scx.slice) &&
8158	off + size <= offsetofend(struct task_struct, scx.slice)) \|\|
8159	(off >= offsetof(struct task_struct, scx.dsq_vtime) &&
8160	off + size <= offsetofend(struct task_struct, scx.dsq_vtime)))
8161	return SCALAR_VALUE;
8162	}
8163
8164	return bpf_scx_btf_struct_access_common(reg, off, size);
8165	}
8166
8167	/* cid-form rejects direct slice and dsq_vtime writes in favor of the kfuncs */
8168	static int bpf_scx_cid_btf_struct_access(struct bpf_verifier_log *log,
8169	const struct bpf_reg_state *reg, int off,
8170	int size)
8171	{
8172	return bpf_scx_btf_struct_access_common(reg, off, size);
8173	}
8174
8175	static const struct bpf_verifier_ops bpf_scx_verifier_ops = {
8176	.get_func_proto = bpf_base_func_proto,
8177	.is_valid_access = bpf_scx_is_valid_access,
8178	.btf_struct_access = bpf_scx_btf_struct_access,
8179	};
8180
8181	static const struct bpf_verifier_ops bpf_scx_cid_verifier_ops = {
8182	.get_func_proto = bpf_base_func_proto,
8183	.is_valid_access = bpf_scx_is_valid_access,
8184	.btf_struct_access = bpf_scx_cid_btf_struct_access,
8185	};
8186
8187	static int bpf_scx_init_member(const struct btf_type *t,
8188	const struct btf_member *member,
8189	void kdata, const void udata)
8190	{
8191	const struct sched_ext_ops *uops = udata;
8192	struct sched_ext_ops *ops = kdata;
8193	u32 moff = __btf_member_bit_offset(t, member) / 8;
8194	int ret;
8195
8196	switch (moff) {
8197	case offsetof(struct sched_ext_ops, dispatch_max_batch):
8198	if ((u32 )(udata + moff) > INT_MAX)
8199	return -E2BIG;
8200	ops->dispatch_max_batch = (u32 )(udata + moff);
8201	return 1;
8202	case offsetof(struct sched_ext_ops, flags):
8203	if ((u64 )(udata + moff) & ~SCX_OPS_ALL_FLAGS)
8204	return -EINVAL;
8205	ops->flags = (u64 )(udata + moff);
8206	return 1;
8207	case offsetof(struct sched_ext_ops, name):
8208	ret = bpf_obj_name_cpy(ops->name, uops->name,
8209	sizeof(ops->name));
8210	if (ret < 0)
8211	return ret;
8212	if (ret == 0)
8213	return -EINVAL;
8214	return 1;
8215	case offsetof(struct sched_ext_ops, timeout_ms):
8216	if (msecs_to_jiffies((u32 )(udata + moff)) >
8217	SCX_WATCHDOG_MAX_TIMEOUT)
8218	return -E2BIG;
8219	ops->timeout_ms = (u32 )(udata + moff);
8220	return 1;
8221	case offsetof(struct sched_ext_ops, exit_dump_len):
8222	ops->exit_dump_len =
8223	(u32 )(udata + moff) ?: SCX_EXIT_DUMP_DFL_LEN;
8224	return 1;
8225	case offsetof(struct sched_ext_ops, hotplug_seq):
8226	ops->hotplug_seq = (u64 )(udata + moff);
8227	return 1;
8228	case offsetof(struct sched_ext_ops, cid_shard_size):
8229	ops->cid_shard_size = (u32 )(udata + moff);
8230	return 1;
8231	case offsetof(struct sched_ext_ops, rescue_bandwidth_ppt): {
8232	u32 bw_ppt = (u32 )(udata + moff);
8233
8234	if (bw_ppt > SCX_RESCUE_MAX_BW_PPT && bw_ppt != SCX_RESCUE_DISABLE)
8235	return -E2BIG;
8236	ops->rescue_bandwidth_ppt = bw_ppt;
8237	return 1;
8238	}
8239	case offsetof(struct sched_ext_ops, rescue_quantum_us): {
8240	u32 quantum_us = (u32 )(udata + moff);
8241
8242	if (quantum_us > SCX_RESCUE_MAX_QUANTUM_US)
8243	return -E2BIG;
8244	if (quantum_us && quantum_us < SCX_RESCUE_MIN_QUANTUM_US)
8245	return -EINVAL;
8246	ops->rescue_quantum_us = quantum_us;
8247	return 1;
8248	}
8249	#ifdef CONFIG_EXT_SUB_SCHED
8250	case offsetof(struct sched_ext_ops, sub_cgroup_id):
8251	ops->sub_cgroup_id = (u64 )(udata + moff);
8252	return 1;
8253	#endif /* CONFIG_EXT_SUB_SCHED */
8254	}
8255
8256	return 0;
8257	}
8258
8259	static int bpf_scx_check_member(const struct btf_type *t,
8260	const struct btf_member *member,
8261	const struct bpf_prog *prog)
8262	{
8263	u32 moff = __btf_member_bit_offset(t, member) / 8;
8264
8265	switch (moff) {
8266	case offsetof(struct sched_ext_ops, init_task):
8267	#ifdef CONFIG_EXT_GROUP_SCHED
8268	case offsetof(struct sched_ext_ops, cgroup_init):
8269	case offsetof(struct sched_ext_ops, cgroup_exit):
8270	case offsetof(struct sched_ext_ops, cgroup_prep_move):
8271	case offsetof(struct sched_ext_ops, cgroup_set_bandwidth):
8272	#endif
8273	case offsetof(struct sched_ext_ops, cpu_online):
8274	case offsetof(struct sched_ext_ops, cpu_offline):
8275	case offsetof(struct sched_ext_ops, init_cids):
8276	case offsetof(struct sched_ext_ops, init):
8277	case offsetof(struct sched_ext_ops, exit):
8278	case offsetof(struct sched_ext_ops, sub_attach):
8279	case offsetof(struct sched_ext_ops, sub_detach):
8280	break;
8281	default:
8282	if (prog->sleepable)
8283	return -EINVAL;
8284	}
8285
8286	#ifdef CONFIG_EXT_SUB_SCHED
8287	/*
8288	* Enable private stack for operations that can nest along the
8289	* hierarchy.
8290	*
8291	* XXX - Ideally, we should only do this for scheds that allow
8292	* sub-scheds and sub-scheds themselves but I don't know how to access
8293	* struct_ops from here.
8294	*/
8295	switch (moff) {
8296	case offsetof(struct sched_ext_ops, dispatch):
8297	prog->aux->priv_stack_requested = true;
8298	prog->aux->recursion_detected = scx_pstack_recursion_on_dispatch;
8299	break;
8300	case offsetof(struct sched_ext_ops, sub_caps_updated):
8301	prog->aux->priv_stack_requested = true;
8302	prog->aux->recursion_detected = scx_pstack_recursion_on_caps_updated;
8303	break;
8304	}
8305	#endif /* CONFIG_EXT_SUB_SCHED */
8306
8307	return 0;
8308	}
8309
8310	static int bpf_scx_reg(void kdata, struct bpf_link link)
8311	{
8312	struct scx_enable_cmd cmd = { .ops = kdata };
8313
8314	return scx_enable(&cmd, link);
8315	}
8316
8317	struct scx_arena_scan {
8318	struct bpf_map *arena;
8319	int err;
8320	};
8321
8322	/*
8323	* The verifier enforces one arena per BPF program, so each struct_ops
8324	* member prog contributes at most one arena via bpf_prog_arena().
8325	* Require all non-NULL contributions to match.
8326	*/
8327	static int scx_arena_scan_prog(struct bpf_prog prog, void data)
8328	{
8329	struct scx_arena_scan *s = data;
8330	struct bpf_map *arena = NULL;
8331
8332	/* arena.o, which defines these, is built only on MMU && 64BIT */
8333	#if defined(CONFIG_MMU) && defined(CONFIG_64BIT)
8334	arena = bpf_prog_arena(prog);
8335	#endif
8336	if (!arena)
8337	return 0;
8338	if (s->arena && s->arena != arena) {
8339	s->err = -EINVAL;
8340	return 1;
8341	}
8342	s->arena = arena;
8343	return 0;
8344	}
8345
8346	static int bpf_scx_reg_cid(void kdata, struct bpf_link link)
8347	{
8348	struct scx_enable_cmd cmd = { .ops_cid = kdata, .is_cid_type = true };
8349	struct scx_arena_scan scan = {};
8350	int ret;
8351
8352	bpf_struct_ops_for_each_prog(kdata, scx_arena_scan_prog, &scan);
8353	if (scan.err) {
8354	pr_err("sched_ext: cid-form scheduler uses multiple arena maps\n");
8355	return scan.err;
8356	}
8357	if (!scan.arena) {
8358	pr_err("sched_ext: cid-form scheduler must use a BPF arena map\n");
8359	return -EINVAL;
8360	}
8361
8362	bpf_map_inc(scan.arena);
8363	cmd.arena_map = scan.arena;
8364	ret = scx_enable(&cmd, link);
8365	if (cmd.arena_map) /* not consumed by scx_alloc_and_add_sched() */
8366	bpf_map_put(cmd.arena_map);
8367	return ret;
8368	}
8369
8370	static void bpf_scx_unreg(void kdata, struct bpf_link link)
8371	{
8372	struct sched_ext_ops *ops = kdata;
8373	struct scx_sched *sch = rcu_dereference_protected(ops->priv, true);
8374
8375	scx_disable(sch, SCX_EXIT_UNREG);
8376	scx_flush_disable_work(sch);
8377	RCU_INIT_POINTER(ops->priv, NULL);
8378	kobject_put(&sch->kobj);
8379	}
8380
8381	static int bpf_scx_init(struct btf *btf)
8382	{
8383	task_struct_type = btf_type_by_id(btf, btf_tracing_ids[BTF_TRACING_TYPE_TASK]);
8384
8385	return 0;
8386	}
8387
8388	static int bpf_scx_update(void kdata, void old_kdata, struct bpf_link *link)
8389	{
8390	/*
8391	* sched_ext does not support updating the actively-loaded BPF
8392	* scheduler, as registering a BPF scheduler can always fail if the
8393	* scheduler returns an error code for e.g. ops.init(), ops.init_task(),
8394	* etc. Similarly, we can always race with unregistration happening
8395	* elsewhere, such as with sysrq.
8396	*/
8397	return -EOPNOTSUPP;
8398	}
8399
8400	static int bpf_scx_validate(void *kdata)
8401	{
8402	return 0;
8403	}
8404
8405	static s32 sched_ext_ops__select_cpu(struct task_struct *p, s32 prev_cpu, u64 wake_flags) { return -EINVAL; }
8406	static void sched_ext_ops__enqueue(struct task_struct *p, u64 enq_flags) {}
8407	static void sched_ext_ops__dequeue(struct task_struct *p, u64 enq_flags) {}
8408	static void sched_ext_ops__dispatch(s32 prev_cpu, struct task_struct *prev__nullable) {}
8409	static void sched_ext_ops__tick(struct task_struct *p) {}
8410	static void sched_ext_ops__runnable(struct task_struct *p, u64 enq_flags) {}
8411	static void sched_ext_ops__running(struct task_struct *p) {}
8412	static void sched_ext_ops__stopping(struct task_struct *p, bool runnable) {}
8413	static void sched_ext_ops__quiescent(struct task_struct *p, u64 deq_flags) {}
8414	static bool sched_ext_ops__yield(struct task_struct from, struct task_struct to__nullable) { return false; }
8415	static bool sched_ext_ops__core_sched_before(struct task_struct a, struct task_struct b) { return false; }
8416	static void sched_ext_ops__set_weight(struct task_struct *p, u32 weight) {}
8417	static void sched_ext_ops__set_cpumask(struct task_struct p, const struct cpumask mask) {}
8418	static void sched_ext_ops__update_idle(s32 cpu, bool idle) {}
8419	static void sched_ext_ops__cpu_acquire(s32 cpu, struct scx_cpu_acquire_args *args) {}
8420	static void sched_ext_ops__cpu_release(s32 cpu, struct scx_cpu_release_args *args) {}
8421	static s32 sched_ext_ops__init_task(struct task_struct p, struct scx_init_task_args args) { return -EINVAL; }
8422	static void sched_ext_ops__exit_task(struct task_struct p, struct scx_exit_task_args args) {}
8423	static void sched_ext_ops__enable(struct task_struct *p) {}
8424	static void sched_ext_ops__disable(struct task_struct *p) {}
8425	#ifdef CONFIG_EXT_GROUP_SCHED
8426	static s32 sched_ext_ops__cgroup_init(struct cgroup cgrp, struct scx_cgroup_init_args args) { return -EINVAL; }
8427	static void sched_ext_ops__cgroup_exit(struct cgroup *cgrp) {}
8428	static s32 sched_ext_ops__cgroup_prep_move(struct task_struct p, struct cgroup from, struct cgroup *to) { return -EINVAL; }
8429	static void sched_ext_ops__cgroup_move(struct task_struct p, struct cgroup from, struct cgroup *to) {}
8430	static void sched_ext_ops__cgroup_cancel_move(struct task_struct p, struct cgroup from, struct cgroup *to) {}
8431	static void sched_ext_ops__cgroup_set_weight(struct cgroup *cgrp, u32 weight) {}
8432	static void sched_ext_ops__cgroup_set_bandwidth(struct cgroup *cgrp, u64 period_us, u64 quota_us, u64 burst_us) {}
8433	static void sched_ext_ops__cgroup_set_idle(struct cgroup *cgrp, bool idle) {}
8434	#endif /* CONFIG_EXT_GROUP_SCHED */
8435	static s32 sched_ext_ops__sub_attach(struct scx_sub_attach_args *args) { return -EINVAL; }
8436	static void sched_ext_ops__sub_detach(struct scx_sub_detach_args *args) {}
8437	static void sched_ext_ops__cpu_online(s32 cpu) {}
8438	static void sched_ext_ops__cpu_offline(s32 cpu) {}
8439	static s32 sched_ext_ops__init_cids(void) { return -EINVAL; }
8440	static s32 sched_ext_ops__init(void) { return -EINVAL; }
8441	static void sched_ext_ops__exit(struct scx_exit_info *info) {}
8442	static void sched_ext_ops__dump(struct scx_dump_ctx *ctx) {}
8443	static void sched_ext_ops__dump_cpu(struct scx_dump_ctx *ctx, s32 cpu, bool idle) {}
8444	static void sched_ext_ops__dump_task(struct scx_dump_ctx ctx, struct task_struct p) {}
8445
8446	static struct sched_ext_ops __bpf_ops_sched_ext_ops = {
8447	.select_cpu = sched_ext_ops__select_cpu,
8448	.enqueue = sched_ext_ops__enqueue,
8449	.dequeue = sched_ext_ops__dequeue,
8450	.dispatch = sched_ext_ops__dispatch,
8451	.tick = sched_ext_ops__tick,
8452	.runnable = sched_ext_ops__runnable,
8453	.running = sched_ext_ops__running,
8454	.stopping = sched_ext_ops__stopping,
8455	.quiescent = sched_ext_ops__quiescent,
8456	.yield = sched_ext_ops__yield,
8457	.core_sched_before = sched_ext_ops__core_sched_before,
8458	.set_weight = sched_ext_ops__set_weight,
8459	.set_cpumask = sched_ext_ops__set_cpumask,
8460	.update_idle = sched_ext_ops__update_idle,
8461	.cpu_acquire = sched_ext_ops__cpu_acquire,
8462	.cpu_release = sched_ext_ops__cpu_release,
8463	.init_task = sched_ext_ops__init_task,
8464	.exit_task = sched_ext_ops__exit_task,
8465	.enable = sched_ext_ops__enable,
8466	.disable = sched_ext_ops__disable,
8467	#ifdef CONFIG_EXT_GROUP_SCHED
8468	.cgroup_init = sched_ext_ops__cgroup_init,
8469	.cgroup_exit = sched_ext_ops__cgroup_exit,
8470	.cgroup_prep_move = sched_ext_ops__cgroup_prep_move,
8471	.cgroup_move = sched_ext_ops__cgroup_move,
8472	.cgroup_cancel_move = sched_ext_ops__cgroup_cancel_move,
8473	.cgroup_set_weight = sched_ext_ops__cgroup_set_weight,
8474	.cgroup_set_bandwidth = sched_ext_ops__cgroup_set_bandwidth,
8475	.cgroup_set_idle = sched_ext_ops__cgroup_set_idle,
8476	#endif
8477	.sub_attach = sched_ext_ops__sub_attach,
8478	.sub_detach = sched_ext_ops__sub_detach,
8479	.cpu_online = sched_ext_ops__cpu_online,
8480	.cpu_offline = sched_ext_ops__cpu_offline,
8481	.init_cids = sched_ext_ops__init_cids,
8482	.init = sched_ext_ops__init,
8483	.exit = sched_ext_ops__exit,
8484	.dump = sched_ext_ops__dump,
8485	.dump_cpu = sched_ext_ops__dump_cpu,
8486	.dump_task = sched_ext_ops__dump_task,
8487	};
8488
8489	static struct bpf_struct_ops bpf_sched_ext_ops = {
8490	.verifier_ops = &bpf_scx_verifier_ops,
8491	.reg = bpf_scx_reg,
8492	.unreg = bpf_scx_unreg,
8493	.check_member = bpf_scx_check_member,
8494	.init_member = bpf_scx_init_member,
8495	.init = bpf_scx_init,
8496	.update = bpf_scx_update,
8497	.validate = bpf_scx_validate,
8498	.name = "sched_ext_ops",
8499	.owner = THIS_MODULE,
8500	.cfi_stubs = &__bpf_ops_sched_ext_ops
8501	};
8502
8503	/*
8504	* cid-form cfi stubs. Stubs whose signatures match the cpu-form (param types
8505	* identical, only param names differ across structs) are reused. Some need
8506	* fresh stubs, set_cmask and enable due to argument differences and the
8507	* sub-sched notifiers because no cpu-form stub exists to reuse.
8508	*/
8509	static void sched_ext_ops_cid__set_cmask(struct task_struct p, const struct scx_cmask cmask__arena) {}
8510	static void sched_ext_ops_cid__enable(struct task_struct p, struct scx_enable_args args) {}
8511	static void sched_ext_ops__sub_caps_updated(const struct scx_cmask *cmask__arena, u64 caps) {}
8512	static void sched_ext_ops__sub_ecaps_updated(s32 cid, u64 before, u64 after) {}
8513
8514	static struct sched_ext_ops_cid __bpf_ops_sched_ext_ops_cid = {
8515	.select_cid = sched_ext_ops__select_cpu,
8516	.enqueue = sched_ext_ops__enqueue,
8517	.dequeue = sched_ext_ops__dequeue,
8518	.dispatch = sched_ext_ops__dispatch,
8519	.tick = sched_ext_ops__tick,
8520	.runnable = sched_ext_ops__runnable,
8521	.running = sched_ext_ops__running,
8522	.stopping = sched_ext_ops__stopping,
8523	.quiescent = sched_ext_ops__quiescent,
8524	.yield = sched_ext_ops__yield,
8525	.core_sched_before = sched_ext_ops__core_sched_before,
8526	.set_weight = sched_ext_ops__set_weight,
8527	.set_cmask = sched_ext_ops_cid__set_cmask,
8528	.update_idle = sched_ext_ops__update_idle,
8529	.init_task = sched_ext_ops__init_task,
8530	.exit_task = sched_ext_ops__exit_task,
8531	.enable = sched_ext_ops_cid__enable,
8532	.disable = sched_ext_ops__disable,
8533	#ifdef CONFIG_EXT_GROUP_SCHED
8534	.cpuctl_init = sched_ext_ops__cgroup_init,
8535	.cpuctl_exit = sched_ext_ops__cgroup_exit,
8536	.cpuctl_prep_move = sched_ext_ops__cgroup_prep_move,
8537	.cpuctl_move = sched_ext_ops__cgroup_move,
8538	.cpuctl_cancel_move = sched_ext_ops__cgroup_cancel_move,
8539	.cpuctl_set_weight = sched_ext_ops__cgroup_set_weight,
8540	.cpuctl_set_bandwidth = sched_ext_ops__cgroup_set_bandwidth,
8541	.cpuctl_set_idle = sched_ext_ops__cgroup_set_idle,
8542	#endif
8543	.sub_attach = sched_ext_ops__sub_attach,
8544	.sub_detach = sched_ext_ops__sub_detach,
8545	.sub_caps_updated = sched_ext_ops__sub_caps_updated,
8546	.sub_ecaps_updated = sched_ext_ops__sub_ecaps_updated,
8547	.cid_online = sched_ext_ops__cpu_online,
8548	.cid_offline = sched_ext_ops__cpu_offline,
8549	.init_cids = sched_ext_ops__init_cids,
8550	.init = sched_ext_ops__init,
8551	.exit = sched_ext_ops__exit,
8552	.dump = sched_ext_ops__dump,
8553	.dump_cid = sched_ext_ops__dump_cpu,
8554	.dump_task = sched_ext_ops__dump_task,
8555	};
8556
8557	/*
8558	* The cid-form struct_ops shares all bpf_struct_ops hooks with the cpu form.
8559	* init_member, check_member, reg, unreg, etc. process kdata as the byte block
8560	* verified to match by the BUILD_BUG_ON checks in scx_init().
8561	*/
8562	static struct bpf_struct_ops bpf_sched_ext_ops_cid = {
8563	.verifier_ops = &bpf_scx_cid_verifier_ops,
8564	.reg = bpf_scx_reg_cid,
8565	.unreg = bpf_scx_unreg,
8566	.check_member = bpf_scx_check_member,
8567	.init_member = bpf_scx_init_member,
8568	.init = bpf_scx_init,
8569	.update = bpf_scx_update,
8570	.validate = bpf_scx_validate,
8571	.name = "sched_ext_ops_cid",
8572	.owner = THIS_MODULE,
8573	.cfi_stubs = &__bpf_ops_sched_ext_ops_cid
8574	};
8575
8576
8577	/********************************************************************************
8578	* System integration and init.
8579	*/
8580
8581	static void sysrq_handle_sched_ext_reset(u8 key)
8582	{
8583	struct scx_sched *sch;
8584
8585	sch = rcu_dereference(scx_root);
8586	if (likely(sch))
8587	scx_disable(sch, SCX_EXIT_SYSRQ);
8588	else
8589	pr_info("sched_ext: BPF schedulers not loaded\n");
8590	}
8591
8592	static const struct sysrq_key_op sysrq_sched_ext_reset_op = {
8593	.handler = sysrq_handle_sched_ext_reset,
8594	.help_msg = "reset-sched-ext(S)",
8595	.action_msg = "Disable sched_ext and revert all tasks to CFS",
8596	.enable_mask = SYSRQ_ENABLE_RTNICE,
8597	};
8598
8599	static void sysrq_handle_sched_ext_dump(u8 key)
8600	{
8601	struct scx_exit_info ei = {
8602	.kind = SCX_EXIT_NONE,
8603	.exit_cpu = -1,
8604	.reason = "SysRq-D",
8605	};
8606	struct scx_sched *sch;
8607
8608	list_for_each_entry_rcu(sch, &scx_sched_all, all)
8609	scx_dump_state(sch, &ei, 0, false);
8610	}
8611
8612	static const struct sysrq_key_op sysrq_sched_ext_dump_op = {
8613	.handler = sysrq_handle_sched_ext_dump,
8614	.help_msg = "dump-sched-ext(D)",
8615	.action_msg = "Trigger sched_ext debug dump",
8616	.enable_mask = SYSRQ_ENABLE_RTNICE,
8617	};
8618
8619	static bool can_skip_idle_kick(struct rq *rq)
8620	{
8621	lockdep_assert_rq_held(rq);
8622
8623	/*
8624	* We can skip idle kicking if @rq is going to go through at least one
8625	* full SCX scheduling cycle before going idle. Just checking whether
8626	* curr is not idle is insufficient because we could be racing
8627	* dispatch_one() trying to pull the next task from a remote rq, which
8628	* may fail, and @rq may become idle afterwards.
8629	*
8630	* The race window is small and we don't and can't guarantee that @rq is
8631	* only kicked while idle anyway. Skip only when sure.
8632	*/
8633	return !is_idle_task(rq->curr) && !(rq->scx.flags & SCX_RQ_IN_DISPATCH);
8634	}
8635
8636	static bool kick_one_cpu(s32 cpu, struct scx_sched_pcpu pcpu, struct rq this_rq,
8637	unsigned long *ksyncs)
8638	{
8639	struct rq *rq = cpu_rq(cpu);
8640	struct scx_rq *this_scx = &this_rq->scx;
8641	const struct sched_class *cur_class;
8642	bool should_wait = false;
8643	bool kickable;
8644	unsigned long flags;
8645
8646	raw_spin_rq_lock_irqsave(rq, flags);
8647	cur_class = rq->curr->sched_class;
8648
8649	/*
8650	* During CPU hotplug, a CPU may depend on kicking itself to make
8651	* forward progress. Allow kicking self regardless of online state. If
8652	* @cpu is running a higher class task, we have no control over @cpu.
8653	* Skip kicking. A sub-sched lacking baseline access on @cid has no
8654	* business forcing a reschedule there - skip. This is the authoritative
8655	* cap check: ecaps is read here under @rq's lock.
8656	*/
8657	kickable = (cpu_online(cpu) \|\| cpu == cpu_of(this_rq)) &&
8658	!sched_class_above(cur_class, &ext_sched_class);
8659
8660	if (kickable && !scx_missing_caps(pcpu->sch, cpu, SCX_CAP_BASE)) {
8661	if (cpumask_test_cpu(cpu, pcpu->cpus_to_preempt)) {
8662	if (cur_class == &ext_sched_class) {
8663	u64 caps = scx_caps_for_preempt(pcpu->sch, rq, 0);
8664
8665	if (unlikely(scx_missing_caps(pcpu->sch, cpu, caps)))
8666	__scx_add_event(pcpu->sch, SCX_EV_SUB_PREEMPT_DENIED, 1);
8667	else if (unlikely(!scx_set_task_slice(rq->curr, 0)))
8668	__scx_add_event(pcpu->sch, SCX_EV_SLICE_DENIED, 1);
8669	}
8670	cpumask_clear_cpu(cpu, pcpu->cpus_to_preempt);
8671	}
8672
8673	if (cpumask_test_cpu(cpu, pcpu->cpus_to_wait)) {
8674	if (cur_class == &ext_sched_class) {
8675	cpumask_set_cpu(cpu, this_scx->cpus_to_sync);
8676	ksyncs[cpu] = rq->scx.kick_sync;
8677	should_wait = true;
8678	}
8679	cpumask_clear_cpu(cpu, pcpu->cpus_to_wait);
8680	}
8681
8682	resched_curr(rq);
8683	} else {
8684	/* a kickable cpu was skipped solely for the missing caps */
8685	if (kickable)
8686	__scx_add_event(pcpu->sch, SCX_EV_SUB_KICK_DENIED, 1);
8687	cpumask_clear_cpu(cpu, pcpu->cpus_to_preempt);
8688	cpumask_clear_cpu(cpu, pcpu->cpus_to_wait);
8689	}
8690
8691	scx_rq_lock_drop(rq);
8692	raw_spin_rq_unlock_irqrestore(rq, flags);
8693
8694	return should_wait;
8695	}
8696
8697	static void kick_one_cpu_if_idle(s32 cpu, struct scx_sched_pcpu *pcpu,
8698	struct rq *this_rq)
8699	{
8700	struct rq *rq = cpu_rq(cpu);
8701	unsigned long flags;
8702
8703	raw_spin_rq_lock_irqsave(rq, flags);
8704
8705	/* idle kicks need baseline access too, see kick_one_cpu() */
8706	if (!can_skip_idle_kick(rq) &&
8707	(cpu_online(cpu) \|\| cpu == cpu_of(this_rq))) {
8708	if (likely(!scx_missing_caps(pcpu->sch, cpu, SCX_CAP_BASE)))
8709	resched_curr(rq);
8710	else
8711	__scx_add_event(pcpu->sch, SCX_EV_SUB_KICK_DENIED, 1);
8712	}
8713
8714	scx_rq_lock_drop(rq);
8715	raw_spin_rq_unlock_irqrestore(rq, flags);
8716	}
8717
8718	static void kick_cpus_irq_workfn(struct irq_work *irq_work)
8719	{
8720	struct rq *this_rq = this_rq();
8721	struct scx_rq *this_scx = &this_rq->scx;
8722	struct scx_kick_syncs __rcu *ksyncs_pcpu = __this_cpu_read(scx_kick_syncs);
8723	struct scx_sched_pcpu pcpu, tmp;
8724	bool should_wait = false;
8725	unsigned long *ksyncs;
8726	s32 cpu;
8727
8728	/* can race with free_kick_syncs() during scheduler disable */
8729	if (unlikely(!ksyncs_pcpu))
8730	return;
8731
8732	ksyncs = rcu_dereference_bh(ksyncs_pcpu)->syncs;
8733
8734	/*
8735	* Walk scheds with pending kicks on this cpu. scx_kick_cpu() adds to
8736	* the list under local_irq_save() and only this irq_work consumes it.
8737	* A plain list without locking is sufficient.
8738	*/
8739	list_for_each_entry_safe(pcpu, tmp, &this_scx->sched_pcpus_to_kick, to_kick_node) {
8740	list_del_init(&pcpu->to_kick_node);
8741
8742	for_each_cpu(cpu, pcpu->cpus_to_kick) {
8743	should_wait \|= kick_one_cpu(cpu, pcpu, this_rq, ksyncs);
8744	cpumask_clear_cpu(cpu, pcpu->cpus_to_kick);
8745	cpumask_clear_cpu(cpu, pcpu->cpus_to_kick_if_idle);
8746	}
8747
8748	for_each_cpu(cpu, pcpu->cpus_to_kick_if_idle) {
8749	kick_one_cpu_if_idle(cpu, pcpu, this_rq);
8750	cpumask_clear_cpu(cpu, pcpu->cpus_to_kick_if_idle);
8751	}
8752	}
8753
8754	/*
8755	* Can't wait in hardirq — kick_sync can't advance, deadlocking if
8756	* CPUs wait for each other. Defer to kick_sync_wait_bal_cb().
8757	*/
8758	if (should_wait) {
8759	raw_spin_rq_lock(this_rq);
8760	this_scx->kick_sync_pending = true;
8761	resched_curr(this_rq);
8762	scx_rq_lock_drop(this_rq);
8763	raw_spin_rq_unlock(this_rq);
8764	}
8765	}
8766
8767	/**
8768	* print_scx_info - print out sched_ext scheduler state
8769	* @log_lvl: the log level to use when printing
8770	* @p: target task
8771	*
8772	* If a sched_ext scheduler is enabled, print the name and state of the
8773	* scheduler. If @p is on sched_ext, print further information about the task.
8774	*
8775	* This function can be safely called on any task as long as the task_struct
8776	* itself is accessible. While safe, this function isn't synchronized and may
8777	* print out mixups or garbages of limited length.
8778	*/
8779	void print_scx_info(const char log_lvl, struct task_struct p)
8780	{
8781	struct scx_sched *sch;
8782	enum scx_enable_state state = scx_enable_state();
8783	const char *all = READ_ONCE(scx_switching_all) ? "+all" : "";
8784	char runnable_at_buf[22] = "?";
8785	struct sched_class *class;
8786	unsigned long runnable_at;
8787
8788	guard(rcu)();
8789
8790	sch = scx_task_sched_rcu(p);
8791
8792	if (!sch)
8793	return;
8794
8795	/*
8796	* Carefully check if the task was running on sched_ext, and then
8797	* carefully copy the time it's been runnable, and its state.
8798	*/
8799	if (copy_from_kernel_nofault(&class, &p->sched_class, sizeof(class)) \|\|
8800	class != &ext_sched_class) {
8801	printk("%sSched_ext: %s (%s%s)", log_lvl, sch->ops.name,
8802	scx_enable_state_str[state], all);
8803	return;
8804	}
8805
8806	if (!copy_from_kernel_nofault(&runnable_at, &p->scx.runnable_at,
8807	sizeof(runnable_at)))
8808	scnprintf(runnable_at_buf, sizeof(runnable_at_buf), "%+ldms",
8809	jiffies_delta_msecs(runnable_at, jiffies));
8810
8811	/* print everything onto one line to conserve console space */
8812	printk("%sSched_ext: %s (%s%s), task: runnable_at=%s",
8813	log_lvl, sch->ops.name, scx_enable_state_str[state], all,
8814	runnable_at_buf);
8815	}
8816
8817	static int scx_pm_handler(struct notifier_block nb, unsigned long event, void ptr)
8818	{
8819	struct scx_sched *sch;
8820
8821	guard(rcu)();
8822
8823	sch = rcu_dereference(scx_root);
8824	if (!sch)
8825	return NOTIFY_OK;
8826
8827	/*
8828	* SCX schedulers often have userspace components which are sometimes
8829	* involved in critial scheduling paths. PM operations involve freezing
8830	* userspace which can lead to scheduling misbehaviors including stalls.
8831	* Let's bypass while PM operations are in progress.
8832	*/
8833	switch (event) {
8834	case PM_HIBERNATION_PREPARE:
8835	case PM_SUSPEND_PREPARE:
8836	case PM_RESTORE_PREPARE:
8837	scx_bypass(sch, true);
8838	break;
8839	case PM_POST_HIBERNATION:
8840	case PM_POST_SUSPEND:
8841	case PM_POST_RESTORE:
8842	scx_bypass(sch, false);
8843	break;
8844	}
8845
8846	return NOTIFY_OK;
8847	}
8848
8849	static struct notifier_block scx_pm_notifier = {
8850	.notifier_call = scx_pm_handler,
8851	};
8852
8853	void __init init_sched_ext_class(void)
8854	{
8855	s32 cpu, v;
8856
8857	/*
8858	* The following is to prevent the compiler from optimizing out the enum
8859	* definitions so that BPF scheduler implementations can use them
8860	* through the generated vmlinux.h.
8861	*/
8862	WRITE_ONCE(v, SCX_ENQ_WAKEUP \| SCX_DEQ_SLEEP \| SCX_KICK_PREEMPT \|
8863	SCX_TG_ONLINE);
8864
8865	scx_idle_init_masks();
8866
8867	for_each_possible_cpu(cpu) {
8868	struct rq *rq = cpu_rq(cpu);
8869	int n = cpu_to_node(cpu);
8870
8871	/* local_dsq's sch will be set during scx_root_enable() */
8872	BUG_ON(scx_init_dsq(&rq->scx.local_dsq, SCX_DSQ_LOCAL, NULL));
8873	#ifdef CONFIG_EXT_SUB_SCHED
8874	BUG_ON(scx_init_dsq(&rq->scx.reject_dsq, SCX_DSQ_REJECT, NULL));
8875	scx_rescue_init(rq);
8876	#endif
8877
8878	INIT_LIST_HEAD(&rq->scx.runnable_list);
8879	INIT_LIST_HEAD(&rq->scx.ddsp_deferred_locals);
8880
8881	BUG_ON(!zalloc_cpumask_var_node(&rq->scx.cpus_to_sync, GFP_KERNEL, n));
8882	INIT_LIST_HEAD(&rq->scx.sched_pcpus_to_kick);
8883	raw_spin_lock_init(&rq->scx.deferred_reenq_lock);
8884	INIT_LIST_HEAD(&rq->scx.deferred_reenq_locals);
8885	INIT_LIST_HEAD(&rq->scx.deferred_reenq_users);
8886	rq->scx.deferred_irq_work = IRQ_WORK_INIT_HARD(deferred_irq_workfn);
8887	rq->scx.kick_cpus_irq_work = IRQ_WORK_INIT_HARD(kick_cpus_irq_workfn);
8888
8889	if (cpu_online(cpu))
8890	cpu_rq(cpu)->scx.flags \|= SCX_RQ_ONLINE;
8891	}
8892
8893	register_sysrq_key('S', &sysrq_sched_ext_reset_op);
8894	register_sysrq_key('D', &sysrq_sched_ext_dump_op);
8895	INIT_DELAYED_WORK(&scx_watchdog_work, scx_watchdog_workfn);
8896
8897	#ifdef CONFIG_EXT_SUB_SCHED
8898	BUG_ON(rhashtable_init(&scx_sched_hash, &scx_sched_hash_params));
8899	#endif /* CONFIG_EXT_SUB_SCHED */
8900	}
8901
8902
8903	/********************************************************************************
8904	* Helpers that can be called from the BPF scheduler.
8905	*/
8906	static bool scx_vet_enq_flags(struct scx_sched sch, u64 dsq_id, u64 enq_flags)
8907	{
8908	bool is_local = dsq_id == SCX_DSQ_LOCAL \|\|
8909	(dsq_id & SCX_DSQ_LOCAL_ON) == SCX_DSQ_LOCAL_ON;
8910
8911	if (unlikely(*enq_flags & __SCX_ENQ_INTERNAL_MASK)) {
8912	scx_error(sch, "invalid enq_flags 0x%llx", *enq_flags);
8913	return false;
8914	}
8915
8916	if (*enq_flags & SCX_ENQ_IMMED) {
8917	if (unlikely(!is_local)) {
8918	scx_error(sch, "SCX_ENQ_IMMED on a non-local DSQ 0x%llx", dsq_id);
8919	return false;
8920	}
8921	} else if ((sch->ops.flags & SCX_OPS_ALWAYS_ENQ_IMMED) && is_local) {
8922	*enq_flags \|= SCX_ENQ_IMMED;
8923	}
8924
8925	if (unlikely((*enq_flags & SCX_ENQ_RESCUE) && !is_local)) {
8926	scx_error(sch, "SCX_ENQ_RESCUE on a non-local DSQ 0x%llx", dsq_id);
8927	return false;
8928	}
8929
8930	return true;
8931	}
8932
8933	static bool scx_dsq_insert_preamble(struct scx_sched sch, struct task_struct p,
8934	u64 dsq_id, u64 *enq_flags)
8935	{
8936	lockdep_assert_irqs_disabled();
8937
8938	if (unlikely(!p)) {
8939	scx_error(sch, "called with NULL task");
8940	return false;
8941	}
8942
8943	/* see SCX_EV_INSERT_NOT_OWNED definition */
8944	if (unlikely(!scx_task_on_sched(sch, p))) {
8945	__scx_add_event(sch, SCX_EV_INSERT_NOT_OWNED, 1);
8946	return false;
8947	}
8948
8949	if (!scx_vet_enq_flags(sch, dsq_id, enq_flags))
8950	return false;
8951
8952	return true;
8953	}
8954
8955	static void scx_dsq_insert_commit(struct scx_sched sch, struct task_struct p,
8956	u64 dsq_id, u64 slice, u64 vtime, u64 enq_flags)
8957	{
8958	struct scx_dsp_ctx *dspc = &this_cpu_ptr(sch->pcpu)->dsp_ctx;
8959	struct task_struct *ddsp_task;
8960
8961	ddsp_task = __this_cpu_read(direct_dispatch_task);
8962	if (ddsp_task) {
8963	mark_direct_dispatch(sch, ddsp_task, p, dsq_id, slice, vtime, enq_flags);
8964	return;
8965	}
8966
8967	if (unlikely(dspc->cursor >= sch->dsp_max_batch)) {
8968	scx_error(sch, "dispatch buffer overflow");
8969	return;
8970	}
8971
8972	dspc->buf[dspc->cursor++] = (struct scx_dsp_buf_ent){
8973	.task = p,
8974	.qseq = atomic_long_read(&p->scx.ops_state) & SCX_OPSS_QSEQ_MASK,
8975	.dsq_id = dsq_id,
8976	.slice = slice,
8977	.vtime = vtime,
8978	.enq_flags = enq_flags,
8979	};
8980	}
8981
8982	__bpf_kfunc_start_defs();
8983
8984	/**
8985	* scx_bpf_dsq_insert___v2 - Insert a task into the FIFO queue of a DSQ
8986	* @p: task_struct to insert
8987	* @dsq_id: DSQ to insert into
8988	* @slice: duration @p can run for in nsecs, 0 to keep the current value
8989	* @enq_flags: SCX_ENQ_*
8990	* @aux: implicit BPF argument to access bpf_prog_aux hidden from BPF progs
8991	*
8992	* Insert @p into the FIFO queue of the DSQ identified by @dsq_id. It is safe to
8993	* call this function spuriously. Can be called from ops.enqueue(),
8994	* ops.select_cpu(), and ops.dispatch().
8995	*
8996	* When called from ops.select_cpu() or ops.enqueue(), it's for direct dispatch
8997	* and @p must match the task being enqueued.
8998	*
8999	* When called from ops.select_cpu(), @enq_flags and @dsq_id are stored, and @p
9000	* will be directly inserted into the corresponding dispatch queue after
9001	* ops.select_cpu() returns. If @p is inserted into SCX_DSQ_LOCAL, it will be
9002	* inserted into the local DSQ of the CPU returned by ops.select_cpu().
9003	* @enq_flags are OR'd with the enqueue flags on the enqueue path before the
9004	* task is inserted.
9005	*
9006	* When called from ops.dispatch(), there are no restrictions on @p or @dsq_id
9007	* and this function can be called upto ops.dispatch_max_batch times to insert
9008	* multiple tasks. scx_bpf_dispatch_nr_slots() returns the number of the
9009	* remaining slots. scx_bpf_dsq_move_to_local() flushes the batch and resets the
9010	* counter.
9011	*
9012	* This function doesn't have any locking restrictions and may be called under
9013	* BPF locks (in the future when BPF introduces more flexible locking).
9014	*
9015	* @p is allowed to run for @slice. The scheduling path is triggered on slice
9016	* exhaustion. If zero, the current residual slice is maintained. If
9017	* %SCX_SLICE_INF, @p never expires and the BPF scheduler must kick the CPU with
9018	* scx_bpf_kick_cpu() to trigger scheduling.
9019	*
9020	* Returns %true on successful insertion, %false on failure. On the root
9021	* scheduler, %false return triggers scheduler abort and the caller doesn't need
9022	* to check the return value.
9023	*/
9024	__bpf_kfunc bool scx_bpf_dsq_insert___v2(struct task_struct *p, u64 dsq_id,
9025	u64 slice, u64 enq_flags,
9026	const struct bpf_prog_aux *aux)
9027	{
9028	struct scx_sched *sch;
9029
9030	guard(rcu)();
9031	sch = scx_prog_sched(aux);
9032	if (unlikely(!sch))
9033	return false;
9034
9035	if (!scx_dsq_insert_preamble(sch, p, dsq_id, &enq_flags))
9036	return false;
9037
9038	scx_dsq_insert_commit(sch, p, dsq_id, slice, 0, enq_flags);
9039
9040	return true;
9041	}
9042
9043	/*
9044	* COMPAT: Will be removed in v6.23 along with the ___v2 suffix.
9045	*/
9046	__bpf_kfunc void scx_bpf_dsq_insert(struct task_struct *p, u64 dsq_id,
9047	u64 slice, u64 enq_flags,
9048	const struct bpf_prog_aux *aux)
9049	{
9050	scx_bpf_dsq_insert___v2(p, dsq_id, slice, enq_flags, aux);
9051	}
9052
9053	static bool scx_dsq_insert_vtime(struct scx_sched sch, struct task_struct p,
9054	u64 dsq_id, u64 slice, u64 vtime, u64 enq_flags)
9055	{
9056	if (!scx_dsq_insert_preamble(sch, p, dsq_id, &enq_flags))
9057	return false;
9058
9059	scx_dsq_insert_commit(sch, p, dsq_id, slice, vtime, enq_flags \| SCX_ENQ_DSQ_PRIQ);
9060
9061	return true;
9062	}
9063
9064	struct scx_bpf_dsq_insert_vtime_args {
9065	/* @p can't be packed together as KF_RCU is not transitive */
9066	u64 dsq_id;
9067	u64 slice;
9068	u64 vtime;
9069	u64 enq_flags;
9070	};
9071
9072	/**
9073	* __scx_bpf_dsq_insert_vtime - Arg-wrapped vtime DSQ insertion
9074	* @p: task_struct to insert
9075	* @args: struct containing the rest of the arguments
9076	* @args->dsq_id: DSQ to insert into
9077	* @args->slice: duration @p can run for in nsecs, 0 to keep the current value
9078	* @args->vtime: @p's ordering inside the vtime-sorted queue of the target DSQ
9079	* @args->enq_flags: SCX_ENQ_*
9080	* @aux: implicit BPF argument to access bpf_prog_aux hidden from BPF progs
9081	*
9082	* Wrapper kfunc that takes arguments via struct to work around BPF's 5 argument
9083	* limit. BPF programs should use scx_bpf_dsq_insert_vtime() which is provided
9084	* as an inline wrapper in common.bpf.h.
9085	*
9086	* Insert @p into the vtime priority queue of the DSQ identified by
9087	* @args->dsq_id. Tasks queued into the priority queue are ordered by
9088	* @args->vtime. All other aspects are identical to scx_bpf_dsq_insert().
9089	*
9090	* @args->vtime ordering is according to time_before64() which considers
9091	* wrapping. A numerically larger vtime may indicate an earlier position in the
9092	* ordering and vice-versa.
9093	*
9094	* A DSQ can only be used as a FIFO or priority queue at any given time and this
9095	* function must not be called on a DSQ which already has one or more FIFO tasks
9096	* queued and vice-versa. Also, the built-in DSQs (SCX_DSQ_LOCAL and
9097	* SCX_DSQ_GLOBAL) cannot be used as priority queues.
9098	*
9099	* Returns %true on successful insertion, %false on failure. On the root
9100	* scheduler, %false return triggers scheduler abort and the caller doesn't need
9101	* to check the return value.
9102	*/
9103	__bpf_kfunc bool
9104	__scx_bpf_dsq_insert_vtime(struct task_struct *p,
9105	struct scx_bpf_dsq_insert_vtime_args *args,
9106	const struct bpf_prog_aux *aux)
9107	{
9108	struct scx_sched *sch;
9109
9110	guard(rcu)();
9111
9112	sch = scx_prog_sched(aux);
9113	if (unlikely(!sch))
9114	return false;
9115
9116	return scx_dsq_insert_vtime(sch, p, args->dsq_id, args->slice,
9117	args->vtime, args->enq_flags);
9118	}
9119
9120	/*
9121	* COMPAT: Will be removed in v6.23.
9122	*/
9123	__bpf_kfunc void scx_bpf_dsq_insert_vtime(struct task_struct *p, u64 dsq_id,
9124	u64 slice, u64 vtime, u64 enq_flags)
9125	{
9126	struct scx_sched *sch;
9127
9128	guard(rcu)();
9129
9130	sch = rcu_dereference(scx_root);
9131	if (unlikely(!sch))
9132	return;
9133
9134	#ifdef CONFIG_EXT_SUB_SCHED
9135	/*
9136	* Disallow if any sub-scheds are attached. There is no way to tell
9137	* which scheduler called us, so error out @p's scheduler -- read it
9138	* under RCU as @p's locks aren't necessarily held here. @p may be a
9139	* task past sched_ext_dead() or an idle task, in which case its
9140	* scheduler can't be determined and there is nothing obviously wrong
9141	* to report; just refuse the call.
9142	*/
9143	if (unlikely(!list_empty(&sch->children))) {
9144	struct scx_sched *tsch = scx_task_sched_rcu(p);
9145
9146	if (tsch)
9147	scx_error(tsch, "__scx_bpf_dsq_insert_vtime() must be used");
9148	return;
9149	}
9150	#endif
9151
9152	scx_dsq_insert_vtime(sch, p, dsq_id, slice, vtime, enq_flags);
9153	}
9154
9155	__bpf_kfunc_end_defs();
9156
9157	BTF_KFUNCS_START(scx_kfunc_ids_enqueue_dispatch)
9158	BTF_ID_FLAGS(func, scx_bpf_dsq_insert, KF_IMPLICIT_ARGS \| KF_RCU)
9159	BTF_ID_FLAGS(func, scx_bpf_dsq_insert___v2, KF_IMPLICIT_ARGS \| KF_RCU)
9160	BTF_ID_FLAGS(func, __scx_bpf_dsq_insert_vtime, KF_IMPLICIT_ARGS \| KF_RCU)
9161	BTF_ID_FLAGS(func, scx_bpf_dsq_insert_vtime, KF_RCU)
9162	BTF_KFUNCS_END(scx_kfunc_ids_enqueue_dispatch)
9163
9164	static const struct btf_kfunc_id_set scx_kfunc_set_enqueue_dispatch = {
9165	.owner = THIS_MODULE,
9166	.set = &scx_kfunc_ids_enqueue_dispatch,
9167	.filter = scx_kfunc_context_filter,
9168	};
9169
9170	static bool scx_dsq_move(struct bpf_iter_scx_dsq_kern *kit,
9171	struct task_struct *p, u64 dsq_id, u64 enq_flags,
9172	bool priq)
9173	{
9174	struct scx_dispatch_q src_dsq = kit->dsq, dst_dsq;
9175	struct scx_sched *sch;
9176	struct rq p_rq, src_rq, *locked_rq;
9177	bool dispatched = false;
9178	unsigned long flags;
9179
9180	/*
9181	* The verifier considers an iterator slot initialized on any
9182	* KF_ITER_NEW return, so a BPF program may legally reach here after
9183	* bpf_iter_scx_dsq_new() failed and left @kit->dsq NULL.
9184	*/
9185	if (unlikely(!src_dsq))
9186	return false;
9187
9188	sch = src_dsq->sched;
9189
9190	if (!scx_vet_enq_flags(sch, dsq_id, &enq_flags))
9191	return false;
9192
9193	/* internal bit, can only go in after @enq_flags is vetted */
9194	if (priq)
9195	enq_flags \|= SCX_ENQ_DSQ_PRIQ;
9196
9197	/*
9198	* If the BPF scheduler keeps calling this function repeatedly, it can
9199	* cause similar live-lock conditions as scx_consume_dispatch_q().
9200	*/
9201	if (unlikely(READ_ONCE(sch->aborting)))
9202	return false;
9203
9204	/*
9205	* Can be called from either ops.dispatch() holding the dispatched rq's
9206	* lock or any context where no rq lock is held. If latter, lock @p's
9207	* task_rq which we'll likely need anyway.
9208	*/
9209	src_rq = task_rq(p);
9210
9211	local_irq_save(flags);
9212
9213	/*
9214	* Under core scheduling, dispatch can run for a sibling rq, so the
9215	* locked rq is not necessarily this CPU's.
9216	*/
9217	locked_rq = scx_locked_rq();
9218
9219	if (locked_rq) {
9220	if (locked_rq != src_rq)
9221	switch_rq_lock(locked_rq, src_rq);
9222	} else {
9223	raw_spin_rq_lock(src_rq);
9224	}
9225
9226	p_rq = src_rq;
9227	raw_spin_lock(&src_dsq->lock);
9228
9229	/* did someone else get to it while we dropped the locks? */
9230	if (nldsq_cursor_lost_task(&kit->cursor, src_rq, src_dsq, p)) {
9231	raw_spin_unlock(&src_dsq->lock);
9232	goto out;
9233	}
9234
9235	/*
9236	* @p has been on $src_dsq and can't move anymore. If @p is not on @sch,
9237	* the caller didn't have authority over @p at the time of the call.
9238	*/
9239	if (unlikely(!scx_task_on_sched(sch, p))) {
9240	scx_error(sch, "scx_bpf_dsq_move[_vtime]() on %s[%d] but the task belongs to a different scheduler",
9241	p->comm, p->pid);
9242	raw_spin_unlock(&src_dsq->lock);
9243	goto out;
9244	}
9245
9246	/* @p is still on $src_dsq and stable, determine the destination */
9247	dst_dsq = find_dsq_for_dispatch(sch, locked_rq ?: this_rq(), dsq_id, task_cpu(p));
9248
9249	/*
9250	* Apply vtime and slice updates before moving. @p is still on $src_dsq
9251	* with both $src_dsq and its task_rq locked, satisfying the write
9252	* rules, and the PRIQ insertion into $dst_dsq reads the new vtime.
9253	*/
9254	if (kit->cursor.flags & __SCX_DSQ_ITER_HAS_VTIME)
9255	p->scx.dsq_vtime = kit->vtime;
9256	if (kit->cursor.flags & __SCX_DSQ_ITER_HAS_SLICE)
9257	scx_set_task_slice(p, kit->slice);
9258
9259	/* execute move */
9260	p_rq = move_task_between_dsqs(sch, p, enq_flags, src_dsq, dst_dsq);
9261	dispatched = true;
9262	out:
9263	if (locked_rq) {
9264	if (locked_rq != p_rq)
9265	switch_rq_lock(p_rq, locked_rq);
9266	} else {
9267	scx_rq_lock_drop(p_rq);
9268	raw_spin_rq_unlock_irqrestore(p_rq, flags);
9269	}
9270
9271	kit->cursor.flags &= ~(__SCX_DSQ_ITER_HAS_SLICE \|
9272	__SCX_DSQ_ITER_HAS_VTIME);
9273	return dispatched;
9274	}
9275
9276	__bpf_kfunc_start_defs();
9277
9278	/**
9279	* scx_bpf_dispatch_nr_slots - Return the number of remaining dispatch slots
9280	* @aux: implicit BPF argument to access bpf_prog_aux hidden from BPF progs
9281	*
9282	* Can only be called from ops.dispatch().
9283	*/
9284	__bpf_kfunc u32 scx_bpf_dispatch_nr_slots(const struct bpf_prog_aux *aux)
9285	{
9286	struct scx_sched *sch;
9287
9288	guard(rcu)();
9289
9290	sch = scx_prog_sched(aux);
9291	if (unlikely(!sch))
9292	return 0;
9293
9294	return sch->dsp_max_batch - __this_cpu_read(sch->pcpu->dsp_ctx.cursor);
9295	}
9296
9297	/**
9298	* scx_bpf_dispatch_cancel - Cancel the latest dispatch
9299	* @aux: implicit BPF argument to access bpf_prog_aux hidden from BPF progs
9300	*
9301	* Cancel the latest dispatch. Can be called multiple times to cancel further
9302	* dispatches. Can only be called from ops.dispatch().
9303	*/
9304	__bpf_kfunc void scx_bpf_dispatch_cancel(const struct bpf_prog_aux *aux)
9305	{
9306	struct scx_sched *sch;
9307	struct scx_dsp_ctx *dspc;
9308
9309	guard(rcu)();
9310
9311	sch = scx_prog_sched(aux);
9312	if (unlikely(!sch))
9313	return;
9314
9315	dspc = &this_cpu_ptr(sch->pcpu)->dsp_ctx;
9316
9317	if (dspc->cursor > 0)
9318	dspc->cursor--;
9319	else
9320	scx_error(sch, "dispatch buffer underflow");
9321	}
9322
9323	/**
9324	* scx_bpf_dsq_move_to_local___v2 - move a task from a DSQ to the current CPU's local DSQ
9325	* @dsq_id: DSQ to move task from. Must be a user-created DSQ
9326	* @aux: implicit BPF argument to access bpf_prog_aux hidden from BPF progs
9327	* @enq_flags: %SCX_ENQ_*
9328	*
9329	* Move a task from the non-local DSQ identified by @dsq_id to the current CPU's
9330	* local DSQ for execution with @enq_flags applied. Can only be called from
9331	* ops.dispatch().
9332	*
9333	* Built-in DSQs (%SCX_DSQ_GLOBAL and %SCX_DSQ_LOCAL*) are not supported as
9334	* sources. Local DSQs support reenqueueing (a task can be picked up for
9335	* execution, dequeued for property changes, or reenqueued), but the BPF
9336	* scheduler cannot directly iterate or move tasks from them. %SCX_DSQ_GLOBAL
9337	* is similar but also doesn't support reenqueueing, as it maps to multiple
9338	* per-node DSQs making the scope difficult to define; this may change in the
9339	* future.
9340	*
9341	* This function flushes the in-flight dispatches from scx_bpf_dsq_insert()
9342	* before trying to move from the specified DSQ. It may also grab rq locks and
9343	* thus can't be called under any BPF locks.
9344	*
9345	* Returns %true if a task has been moved, %false if there isn't any task to
9346	* move.
9347	*/
9348	__bpf_kfunc bool scx_bpf_dsq_move_to_local___v2(u64 dsq_id, u64 enq_flags,
9349	const struct bpf_prog_aux *aux)
9350	{
9351	struct scx_dispatch_q *dsq;
9352	struct scx_sched *sch;
9353	struct scx_dsp_ctx *dspc;
9354
9355	guard(rcu)();
9356
9357	sch = scx_prog_sched(aux);
9358	if (unlikely(!sch))
9359	return false;
9360
9361	if (!scx_vet_enq_flags(sch, SCX_DSQ_LOCAL, &enq_flags))
9362	return false;
9363
9364	dspc = &this_cpu_ptr(sch->pcpu)->dsp_ctx;
9365
9366	scx_flush_dispatch_buf(sch, dspc->rq);
9367
9368	dsq = find_user_dsq(sch, dsq_id);
9369	if (unlikely(!dsq)) {
9370	scx_error(sch, "invalid DSQ ID 0x%016llx", dsq_id);
9371	return false;
9372	}
9373
9374	if (scx_consume_dispatch_q(sch, dspc->rq, dsq, enq_flags)) {
9375	/*
9376	* A successfully consumed task can be dequeued before it starts
9377	* running while the CPU is trying to migrate other dispatched
9378	* tasks. Bump nr_tasks to tell dispatch_one() to retry on empty
9379	* local DSQ.
9380	*/
9381	dspc->nr_tasks++;
9382	return true;
9383	} else {
9384	return false;
9385	}
9386	}
9387
9388	/*
9389	* COMPAT: ___v2 was introduced in v7.1. Remove this and ___v2 tag in the future.
9390	*/
9391	__bpf_kfunc bool scx_bpf_dsq_move_to_local(u64 dsq_id, const struct bpf_prog_aux *aux)
9392	{
9393	return scx_bpf_dsq_move_to_local___v2(dsq_id, 0, aux);
9394	}
9395
9396	/**
9397	* scx_bpf_dsq_move_set_slice - Override slice when moving between DSQs
9398	* @it__iter: DSQ iterator in progress
9399	* @slice: duration the moved task can run for in nsecs
9400	*
9401	* Override the slice of the next task that will be moved from @it__iter using
9402	* scx_bpf_dsq_move[_vtime](). If this function is not called, the previous
9403	* slice duration is kept.
9404	*/
9405	__bpf_kfunc void scx_bpf_dsq_move_set_slice(struct bpf_iter_scx_dsq *it__iter,
9406	u64 slice)
9407	{
9408	struct bpf_iter_scx_dsq_kern kit = (void )it__iter;
9409
9410	kit->slice = slice;
9411	kit->cursor.flags \|= __SCX_DSQ_ITER_HAS_SLICE;
9412	}
9413
9414	/**
9415	* scx_bpf_dsq_move_set_vtime - Override vtime when moving between DSQs
9416	* @it__iter: DSQ iterator in progress
9417	* @vtime: task's ordering inside the vtime-sorted queue of the target DSQ
9418	*
9419	* Override the vtime of the next task that will be moved from @it__iter using
9420	* scx_bpf_dsq_move_vtime(). If this function is not called, the previous slice
9421	* vtime is kept. If scx_bpf_dsq_move() is used to dispatch the next task, the
9422	* override is ignored and cleared.
9423	*/
9424	__bpf_kfunc void scx_bpf_dsq_move_set_vtime(struct bpf_iter_scx_dsq *it__iter,
9425	u64 vtime)
9426	{
9427	struct bpf_iter_scx_dsq_kern kit = (void )it__iter;
9428
9429	kit->vtime = vtime;
9430	kit->cursor.flags \|= __SCX_DSQ_ITER_HAS_VTIME;
9431	}
9432
9433	/**
9434	* scx_bpf_dsq_move - Move a task from DSQ iteration to a DSQ
9435	* @it__iter: DSQ iterator in progress
9436	* @p: task to transfer
9437	* @dsq_id: DSQ to move @p to
9438	* @enq_flags: SCX_ENQ_*
9439	*
9440	* Transfer @p which is on the DSQ currently iterated by @it__iter to the DSQ
9441	* specified by @dsq_id. All DSQs - local DSQs, global DSQ and user DSQs - can
9442	* be the destination.
9443	*
9444	* For the transfer to be successful, @p must still be on the DSQ and have been
9445	* queued before the DSQ iteration started. This function doesn't care whether
9446	* @p was obtained from the DSQ iteration. @p just has to be on the DSQ and have
9447	* been queued before the iteration started.
9448	*
9449	* @p's slice is kept by default. Use scx_bpf_dsq_move_set_slice() to update.
9450	*
9451	* Can be called from ops.dispatch() or any BPF context which doesn't hold a rq
9452	* lock (e.g. BPF timers or SYSCALL programs).
9453	*
9454	* Returns %true if @p has been consumed, %false if @p had already been
9455	* consumed, dequeued, or, for sub-scheds, @dsq_id points to a disallowed local
9456	* DSQ.
9457	*/
9458	__bpf_kfunc bool scx_bpf_dsq_move(struct bpf_iter_scx_dsq *it__iter,
9459	struct task_struct *p, u64 dsq_id,
9460	u64 enq_flags)
9461	{
9462	return scx_dsq_move((struct bpf_iter_scx_dsq_kern *)it__iter,
9463	p, dsq_id, enq_flags, false);
9464	}
9465
9466	/**
9467	* scx_bpf_dsq_move_vtime - Move a task from DSQ iteration to a PRIQ DSQ
9468	* @it__iter: DSQ iterator in progress
9469	* @p: task to transfer
9470	* @dsq_id: DSQ to move @p to
9471	* @enq_flags: SCX_ENQ_*
9472	*
9473	* Transfer @p which is on the DSQ currently iterated by @it__iter to the
9474	* priority queue of the DSQ specified by @dsq_id. The destination must be a
9475	* user DSQ as only user DSQs support priority queue.
9476	*
9477	* @p's slice and vtime are kept by default. Use scx_bpf_dsq_move_set_slice()
9478	* and scx_bpf_dsq_move_set_vtime() to update.
9479	*
9480	* All other aspects are identical to scx_bpf_dsq_move(). See
9481	* scx_bpf_dsq_insert_vtime() for more information on @vtime.
9482	*/
9483	__bpf_kfunc bool scx_bpf_dsq_move_vtime(struct bpf_iter_scx_dsq *it__iter,
9484	struct task_struct *p, u64 dsq_id,
9485	u64 enq_flags)
9486	{
9487	return scx_dsq_move((struct bpf_iter_scx_dsq_kern *)it__iter,
9488	p, dsq_id, enq_flags, true);
9489	}
9490
9491	__bpf_kfunc_end_defs();
9492
9493	BTF_KFUNCS_START(scx_kfunc_ids_dispatch)
9494	BTF_ID_FLAGS(func, scx_bpf_dispatch_nr_slots, KF_IMPLICIT_ARGS)
9495	BTF_ID_FLAGS(func, scx_bpf_dispatch_cancel, KF_IMPLICIT_ARGS)
9496	BTF_ID_FLAGS(func, scx_bpf_dsq_move_to_local, KF_IMPLICIT_ARGS)
9497	BTF_ID_FLAGS(func, scx_bpf_dsq_move_to_local___v2, KF_IMPLICIT_ARGS)
9498	/* scx_bpf_dsq_move() also in scx_kfunc_ids_unlocked: callable from unlocked contexts /
9499	BTF_ID_FLAGS(func, scx_bpf_dsq_move_set_slice, KF_RCU)
9500	BTF_ID_FLAGS(func, scx_bpf_dsq_move_set_vtime, KF_RCU)
9501	BTF_ID_FLAGS(func, scx_bpf_dsq_move, KF_RCU)
9502	BTF_ID_FLAGS(func, scx_bpf_dsq_move_vtime, KF_RCU)
9503	#ifdef CONFIG_EXT_SUB_SCHED
9504	BTF_ID_FLAGS(func, scx_bpf_sub_dispatch, KF_IMPLICIT_ARGS)
9505	#endif
9506	BTF_KFUNCS_END(scx_kfunc_ids_dispatch)
9507
9508	static const struct btf_kfunc_id_set scx_kfunc_set_dispatch = {
9509	.owner = THIS_MODULE,
9510	.set = &scx_kfunc_ids_dispatch,
9511	.filter = scx_kfunc_context_filter,
9512	};
9513
9514	__bpf_kfunc_start_defs();
9515
9516	/**
9517	* scx_bpf_reenqueue_local - Re-enqueue tasks on a local DSQ
9518	* @aux: implicit BPF argument to access bpf_prog_aux hidden from BPF progs
9519	*
9520	* Iterate over all of the tasks currently enqueued on the local DSQ of the
9521	* caller's CPU, and re-enqueue them in the BPF scheduler. Returns the number of
9522	* processed tasks. Can only be called from ops.cpu_release().
9523	*/
9524	__bpf_kfunc u32 scx_bpf_reenqueue_local(const struct bpf_prog_aux *aux)
9525	{
9526	struct scx_sched *sch;
9527	struct rq *rq;
9528
9529	guard(rcu)();
9530	sch = scx_prog_sched(aux);
9531	if (unlikely(!sch))
9532	return 0;
9533
9534	rq = cpu_rq(smp_processor_id());
9535	lockdep_assert_rq_held(rq);
9536
9537	return reenq_local(sch, rq, SCX_REENQ_ANY);
9538	}
9539
9540	__bpf_kfunc_end_defs();
9541
9542	BTF_KFUNCS_START(scx_kfunc_ids_cpu_release)
9543	BTF_ID_FLAGS(func, scx_bpf_reenqueue_local, KF_IMPLICIT_ARGS)
9544	BTF_KFUNCS_END(scx_kfunc_ids_cpu_release)
9545
9546	static const struct btf_kfunc_id_set scx_kfunc_set_cpu_release = {
9547	.owner = THIS_MODULE,
9548	.set = &scx_kfunc_ids_cpu_release,
9549	.filter = scx_kfunc_context_filter,
9550	};
9551
9552	__bpf_kfunc_start_defs();
9553
9554	/**
9555	* scx_bpf_create_dsq - Create a custom DSQ
9556	* @dsq_id: DSQ to create
9557	* @node: NUMA node to allocate from
9558	* @aux: implicit BPF argument to access bpf_prog_aux hidden from BPF progs
9559	*
9560	* Create a custom DSQ identified by @dsq_id. Can be called from any sleepable
9561	* scx callback, and any BPF_PROG_TYPE_SYSCALL prog.
9562	*/
9563	__bpf_kfunc s32 scx_bpf_create_dsq(u64 dsq_id, s32 node, const struct bpf_prog_aux *aux)
9564	{
9565	struct scx_dispatch_q *dsq;
9566	struct scx_sched *sch;
9567	s32 ret;
9568
9569	if (unlikely(node >= (int)nr_node_ids \|\|
9570	(node < 0 && node != NUMA_NO_NODE)))
9571	return -EINVAL;
9572
9573	if (unlikely(dsq_id & SCX_DSQ_FLAG_BUILTIN))
9574	return -EINVAL;
9575
9576	dsq = kmalloc_node(sizeof(*dsq), GFP_KERNEL, node);
9577	if (!dsq)
9578	return -ENOMEM;
9579
9580	/*
9581	* scx_init_dsq() must be called in GFP_KERNEL context. Init it with
9582	* NULL @sch and update afterwards.
9583	*/
9584	ret = scx_init_dsq(dsq, dsq_id, NULL);
9585	if (ret) {
9586	kfree(dsq);
9587	return ret;
9588	}
9589
9590	rcu_read_lock();
9591
9592	sch = scx_prog_sched(aux);
9593	if (sch) {
9594	dsq->sched = sch;
9595	ret = rhashtable_lookup_insert_fast(&sch->dsq_hash, &dsq->hash_node,
9596	dsq_hash_params);
9597	} else {
9598	ret = -ENODEV;
9599	}
9600
9601	rcu_read_unlock();
9602	if (ret) {
9603	exit_dsq(dsq);
9604	kfree(dsq);
9605	}
9606	return ret;
9607	}
9608
9609	__bpf_kfunc_end_defs();
9610
9611	BTF_KFUNCS_START(scx_kfunc_ids_unlocked)
9612	BTF_ID_FLAGS(func, scx_bpf_create_dsq, KF_IMPLICIT_ARGS \| KF_SLEEPABLE)
9613	/* also in scx_kfunc_ids_dispatch: also callable from ops.dispatch() */
9614	BTF_ID_FLAGS(func, scx_bpf_dsq_move_set_slice, KF_RCU)
9615	BTF_ID_FLAGS(func, scx_bpf_dsq_move_set_vtime, KF_RCU)
9616	BTF_ID_FLAGS(func, scx_bpf_dsq_move, KF_RCU)
9617	BTF_ID_FLAGS(func, scx_bpf_dsq_move_vtime, KF_RCU)
9618	/* also in scx_kfunc_ids_select_cpu: also callable from ops.select_cpu()/ops.enqueue() */
9619	BTF_ID_FLAGS(func, __scx_bpf_select_cpu_and, KF_IMPLICIT_ARGS \| KF_RCU)
9620	BTF_ID_FLAGS(func, scx_bpf_select_cpu_and, KF_RCU)
9621	BTF_ID_FLAGS(func, scx_bpf_select_cpu_dfl, KF_IMPLICIT_ARGS \| KF_RCU)
9622	BTF_KFUNCS_END(scx_kfunc_ids_unlocked)
9623
9624	static const struct btf_kfunc_id_set scx_kfunc_set_unlocked = {
9625	.owner = THIS_MODULE,
9626	.set = &scx_kfunc_ids_unlocked,
9627	.filter = scx_kfunc_context_filter,
9628	};
9629
9630	__bpf_kfunc_start_defs();
9631
9632	/**
9633	* scx_bpf_task_set_slice - Set task's time slice
9634	* @p: task of interest
9635	* @slice: time slice to set in nsecs
9636	* @aux: implicit BPF argument to access bpf_prog_aux hidden from BPF progs
9637	*
9638	* Set @p's time slice. @p must be on the calling scheduler. The value is
9639	* applied whether or not the caller holds @p's rq lock - see the slice write
9640	* rules above for the ownership model.
9641	*
9642	* Raising the slice is honored only while the scheduler holds %SCX_CAP_BASE on
9643	* @p's cpu, otherwise it is counted in %SCX_EV_SLICE_DENIED. Shortening is
9644	* always allowed. On the stashed path the slice is packed into an atomic64_t
9645	* with the scheduler id and a flag bit, so a slice too large to fit is clamped
9646	* and counted in %SCX_EV_SLICE_CLAMPED. %SCX_SLICE_INF is preserved.
9647	*
9648	* Return %true on success, %false if @p is not on the calling scheduler.
9649	*/
9650	__bpf_kfunc bool scx_bpf_task_set_slice(struct task_struct *p, u64 slice,
9651	const struct bpf_prog_aux *aux)
9652	{
9653	struct scx_sched *sch;
9654	struct rq *locked_rq;
9655
9656	guard(rcu)();
9657	sch = scx_prog_sched(aux);
9658	if (unlikely(!sch \|\| !scx_task_on_sched(sch, p)))
9659	return false;
9660
9661	/*
9662	* Directly write only when we hold the lock of the rq @p is queued or
9663	* running on. See the write rules above.
9664	*
9665	* While @p is queued on a user DSQ or in the BPF scheduler,
9666	* synchronization is the scheduler's responsibility. This write can
9667	* race a concurrent dispatch's commit, see apply_slice_vtime().
9668	*
9669	* Making this kfunc always go through the oob stash would leave the
9670	* commit as the only direct writer and close the race, but that would
9671	* require two more oob application points - the dispatch keep-prev test
9672	* and the tick-time expiry check.
9673	*/
9674	locked_rq = scx_locked_rq();
9675	if (!locked_rq \|\|
9676	(READ_ONCE(p->scx.runnable_cpu) != cpu_of(locked_rq) &&
9677	!task_current(locked_rq, p))) {
9678	set_task_slice_oob(sch, p, slice);
9679	return true;
9680	}
9681
9682	/* under the rq lock: apply now, extensions gated on baseline access */
9683	if (slice > p->scx.slice &&
9684	unlikely(scx_missing_caps(sch, cpu_of(locked_rq), SCX_CAP_BASE))) {
9685	__scx_add_event(sch, SCX_EV_SLICE_DENIED, 1);
9686	return true;
9687	}
9688
9689	if (unlikely(!scx_set_task_slice(p, slice)))
9690	__scx_add_event(sch, SCX_EV_SLICE_DENIED, 1);
9691
9692	return true;
9693	}
9694
9695	/**
9696	* scx_bpf_task_set_dsq_vtime - Set task's virtual time for DSQ ordering
9697	* @p: task of interest
9698	* @vtime: virtual time to set
9699	* @aux: implicit BPF argument to access bpf_prog_aux hidden from BPF progs
9700	*
9701	* Set @p's virtual time to @vtime. Returns %true on success, %false if the
9702	* calling scheduler doesn't have authority over @p.
9703	*/
9704	__bpf_kfunc bool scx_bpf_task_set_dsq_vtime(struct task_struct *p, u64 vtime,
9705	const struct bpf_prog_aux *aux)
9706	{
9707	struct scx_sched *sch;
9708
9709	guard(rcu)();
9710	sch = scx_prog_sched(aux);
9711	if (unlikely(!sch \|\| !scx_task_on_sched(sch, p)))
9712	return false;
9713
9714	p->scx.dsq_vtime = vtime;
9715	return true;
9716	}
9717
9718	void scx_kick_cpu(struct scx_sched *sch, s32 cpu, u64 flags)
9719	{
9720	struct scx_sched_pcpu *pcpu;
9721	struct rq *this_rq;
9722	unsigned long irq_flags;
9723
9724	/*
9725	* The per-cpu kick list is guarded only by local_irq_save(), which does
9726	* not mask NMIs, so kicking from NMI could corrupt it and is unsupported.
9727	*/
9728	if (unlikely(in_nmi())) {
9729	scx_error(sch, "scx_bpf_kick_cpu() called from NMI");
9730	return;
9731	}
9732
9733	local_irq_save(irq_flags);
9734
9735	this_rq = this_rq();
9736	pcpu = this_cpu_ptr(sch->pcpu);
9737
9738	/*
9739	* While bypassing for PM ops, IRQ handling may not be online which can
9740	* lead to irq_work_queue() malfunction such as infinite busy wait for
9741	* IRQ status update. Suppress kicking.
9742	*/
9743	if (scx_bypassing(sch, cpu_of(this_rq)))
9744	goto out;
9745
9746	/*
9747	* Actual kicking is bounced to kick_cpus_irq_workfn() to avoid nesting
9748	* rq locks. We can probably be smarter and avoid bouncing if called
9749	* from ops which don't hold a rq lock.
9750	*
9751	* The kick masks are owned by @sch->pcpu, so that a preempt kick can be
9752	* attributed to @sch.
9753	*/
9754	if (flags & SCX_KICK_IDLE) {
9755	struct rq *target_rq = cpu_rq(cpu);
9756
9757	if (unlikely(flags & (SCX_KICK_PREEMPT \| SCX_KICK_WAIT)))
9758	scx_error(sch, "PREEMPT/WAIT cannot be used with SCX_KICK_IDLE");
9759
9760	if (raw_spin_rq_trylock(target_rq)) {
9761	if (can_skip_idle_kick(target_rq)) {
9762	scx_rq_lock_drop(target_rq);
9763	raw_spin_rq_unlock(target_rq);
9764	goto out;
9765	}
9766	scx_rq_lock_drop(target_rq);
9767	raw_spin_rq_unlock(target_rq);
9768	}
9769	cpumask_set_cpu(cpu, pcpu->cpus_to_kick_if_idle);
9770	} else {
9771	cpumask_set_cpu(cpu, pcpu->cpus_to_kick);
9772
9773	if (flags & SCX_KICK_PREEMPT)
9774	cpumask_set_cpu(cpu, pcpu->cpus_to_preempt);
9775	if (flags & SCX_KICK_WAIT)
9776	cpumask_set_cpu(cpu, pcpu->cpus_to_wait);
9777	}
9778
9779	if (list_empty(&pcpu->to_kick_node))
9780	list_add_tail(&pcpu->to_kick_node, &this_rq->scx.sched_pcpus_to_kick);
9781	irq_work_queue(&this_rq->scx.kick_cpus_irq_work);
9782	out:
9783	local_irq_restore(irq_flags);
9784	}
9785
9786	/**
9787	* scx_bpf_kick_cpu - Trigger reschedule on a CPU
9788	* @cpu: cpu to kick
9789	* @flags: %SCX_KICK_* flags
9790	* @aux: implicit BPF argument to access bpf_prog_aux hidden from BPF progs
9791	*
9792	* Kick @cpu into rescheduling. This can be used to wake up an idle CPU or
9793	* trigger rescheduling on a busy CPU. This can be called from any online
9794	* scx_ops operation and the actual kicking is performed asynchronously through
9795	* an irq work.
9796	*/
9797	__bpf_kfunc void scx_bpf_kick_cpu(s32 cpu, u64 flags, const struct bpf_prog_aux *aux)
9798	{
9799	struct scx_sched *sch;
9800
9801	guard(rcu)();
9802	sch = scx_prog_sched(aux);
9803	if (likely(sch) && scx_cpu_valid(sch, cpu, NULL))
9804	scx_kick_cpu(sch, cpu, flags);
9805	}
9806
9807	/**
9808	* scx_bpf_kick_cid - Trigger reschedule on the CPU mapped to @cid
9809	* @cid: cid to kick
9810	* @flags: %SCX_KICK_* flags
9811	* @aux: implicit BPF argument to access bpf_prog_aux hidden from BPF progs
9812	*
9813	* cid-addressed equivalent of scx_bpf_kick_cpu(). An invalid @cid aborts the
9814	* scheduler via scx_cid_to_cpu(). Caps are enforced on the delivery path: a
9815	* kick is dropped if the caller lacks baseline access on @cid, and a
9816	* %SCX_KICK_PREEMPT degrades to a plain reschedule if the caller lacks
9817	* %SCX_CAP_PREEMPT for a task outside its subtree.
9818	*/
9819	__bpf_kfunc void scx_bpf_kick_cid(s32 cid, u64 flags, const struct bpf_prog_aux *aux)
9820	{
9821	struct scx_sched *sch;
9822	s32 cpu;
9823
9824	guard(rcu)();
9825	sch = scx_prog_sched(aux);
9826	if (unlikely(!sch))
9827	return;
9828	cpu = scx_cid_to_cpu(sch, cid);
9829	if (cpu < 0)
9830	return;
9831	scx_kick_cpu(sch, cpu, flags);
9832	}
9833
9834	/**
9835	* scx_bpf_dsq_nr_queued - Return the number of queued tasks
9836	* @dsq_id: id of the DSQ
9837	* @aux: implicit BPF argument to access bpf_prog_aux hidden from BPF progs
9838	*
9839	* Return the number of tasks in the DSQ matching @dsq_id. If not found,
9840	* -%ENOENT is returned.
9841	*
9842	* %SCX_DSQ_LOCAL resolves to the local DSQ of the rq the current scheduler
9843	* operation is locked to - e.g. the rq being dispatched for in ops.dispatch() -
9844	* or the calling CPU's when no rq is locked.
9845	*/
9846	__bpf_kfunc s32 scx_bpf_dsq_nr_queued(u64 dsq_id, const struct bpf_prog_aux *aux)
9847	{
9848	struct scx_sched *sch;
9849	struct scx_dispatch_q *dsq;
9850	s32 ret;
9851
9852	preempt_disable();
9853
9854	sch = scx_prog_sched(aux);
9855	if (unlikely(!sch)) {
9856	ret = -ENODEV;
9857	goto out;
9858	}
9859
9860	if (dsq_id == SCX_DSQ_LOCAL) {
9861	ret = READ_ONCE((scx_locked_rq() ?: this_rq())->scx.local_dsq.nr);
9862	goto out;
9863	} else if ((dsq_id & SCX_DSQ_LOCAL_ON) == SCX_DSQ_LOCAL_ON) {
9864	s32 cpu = scx_cpu_ret(sch, dsq_id & SCX_DSQ_LOCAL_CPU_MASK);
9865
9866	if (scx_cpu_valid(sch, cpu, NULL)) {
9867	ret = READ_ONCE(cpu_rq(cpu)->scx.local_dsq.nr);
9868	goto out;
9869	}
9870	} else {
9871	dsq = find_user_dsq(sch, dsq_id);
9872	if (dsq) {
9873	ret = READ_ONCE(dsq->nr);
9874	goto out;
9875	}
9876	}
9877	ret = -ENOENT;
9878	out:
9879	preempt_enable();
9880	return ret;
9881	}
9882
9883	/**
9884	* scx_bpf_destroy_dsq - Destroy a custom DSQ
9885	* @dsq_id: DSQ to destroy
9886	* @aux: implicit BPF argument to access bpf_prog_aux hidden from BPF progs
9887	*
9888	* Destroy the custom DSQ identified by @dsq_id. Only DSQs created with
9889	* scx_bpf_create_dsq() can be destroyed. The caller must ensure that the DSQ is
9890	* empty and no further tasks are dispatched to it. Ignored if called on a DSQ
9891	* which doesn't exist. Can be called from any online scx_ops operations.
9892	*/
9893	__bpf_kfunc void scx_bpf_destroy_dsq(u64 dsq_id, const struct bpf_prog_aux *aux)
9894	{
9895	struct scx_sched *sch;
9896
9897	guard(rcu)();
9898	sch = scx_prog_sched(aux);
9899	if (sch)
9900	destroy_dsq(sch, dsq_id);
9901	}
9902
9903	/**
9904	* bpf_iter_scx_dsq_new - Create a DSQ iterator
9905	* @it: iterator to initialize
9906	* @dsq_id: DSQ to iterate
9907	* @flags: %SCX_DSQ_ITER_*
9908	* @aux: implicit BPF argument to access bpf_prog_aux hidden from BPF progs
9909	*
9910	* Initialize BPF iterator @it which can be used with bpf_for_each() to walk
9911	* tasks in the DSQ specified by @dsq_id. Iteration using @it only includes
9912	* tasks which are already queued when this function is invoked.
9913	*/
9914	__bpf_kfunc int bpf_iter_scx_dsq_new(struct bpf_iter_scx_dsq *it, u64 dsq_id,
9915	u64 flags, const struct bpf_prog_aux *aux)
9916	{
9917	struct bpf_iter_scx_dsq_kern kit = (void )it;
9918	struct scx_sched *sch;
9919
9920	BUILD_BUG_ON(sizeof(struct bpf_iter_scx_dsq_kern) >
9921	sizeof(struct bpf_iter_scx_dsq));
9922	BUILD_BUG_ON(__alignof__(struct bpf_iter_scx_dsq_kern) !=
9923	__alignof__(struct bpf_iter_scx_dsq));
9924	BUILD_BUG_ON(__SCX_DSQ_ITER_ALL_FLAGS &
9925	((1U << __SCX_DSQ_LNODE_PRIV_SHIFT) - 1));
9926
9927	/*
9928	* next() and destroy() will be called regardless of the return value.
9929	* Always clear $kit->dsq.
9930	*/
9931	kit->dsq = NULL;
9932
9933	sch = scx_prog_sched(aux);
9934	if (unlikely(!sch))
9935	return -ENODEV;
9936
9937	if (flags & ~__SCX_DSQ_ITER_USER_FLAGS)
9938	return -EINVAL;
9939
9940	kit->dsq = find_user_dsq(sch, dsq_id);
9941	if (!kit->dsq)
9942	return -ENOENT;
9943
9944	kit->cursor = INIT_DSQ_LIST_CURSOR(kit->cursor, kit->dsq, flags);
9945
9946	return 0;
9947	}
9948
9949	/**
9950	* bpf_iter_scx_dsq_next - Progress a DSQ iterator
9951	* @it: iterator to progress
9952	*
9953	* Return the next task. See bpf_iter_scx_dsq_new().
9954	*/
9955	__bpf_kfunc struct task_struct bpf_iter_scx_dsq_next(struct bpf_iter_scx_dsq it)
9956	{
9957	struct bpf_iter_scx_dsq_kern kit = (void )it;
9958
9959	if (!kit->dsq)
9960	return NULL;
9961
9962	guard(raw_spinlock_irqsave)(&kit->dsq->lock);
9963
9964	return nldsq_cursor_next_task(&kit->cursor, kit->dsq);
9965	}
9966
9967	/**
9968	* bpf_iter_scx_dsq_destroy - Destroy a DSQ iterator
9969	* @it: iterator to destroy
9970	*
9971	* Undo bpf_iter_scx_dsq_new().
9972	*/
9973	__bpf_kfunc void bpf_iter_scx_dsq_destroy(struct bpf_iter_scx_dsq *it)
9974	{
9975	struct bpf_iter_scx_dsq_kern kit = (void )it;
9976
9977	if (!kit->dsq)
9978	return;
9979
9980	if (!list_empty(&kit->cursor.node)) {
9981	unsigned long flags;
9982
9983	raw_spin_lock_irqsave(&kit->dsq->lock, flags);
9984	list_del_init(&kit->cursor.node);
9985	raw_spin_unlock_irqrestore(&kit->dsq->lock, flags);
9986	}
9987	kit->dsq = NULL;
9988	}
9989
9990	/**
9991	* scx_bpf_dsq_peek - Lockless peek at the first element.
9992	* @dsq_id: DSQ to examine.
9993	* @aux: implicit BPF argument to access bpf_prog_aux hidden from BPF progs
9994	*
9995	* Read the first element in the DSQ. This is semantically equivalent to using
9996	* the DSQ iterator, but is lockfree. Of course, like any lockless operation,
9997	* this provides only a point-in-time snapshot, and the contents may change
9998	* by the time any subsequent locking operation reads the queue.
9999	*
10000	* Returns the pointer, or NULL indicates an empty queue OR internal error.
10001	*/
10002	__bpf_kfunc struct task_struct *scx_bpf_dsq_peek(u64 dsq_id,
10003	const struct bpf_prog_aux *aux)
10004	{
10005	struct scx_sched *sch;
10006	struct scx_dispatch_q *dsq;
10007
10008	sch = scx_prog_sched(aux);
10009	if (unlikely(!sch))
10010	return NULL;
10011
10012	if (unlikely(dsq_id & SCX_DSQ_FLAG_BUILTIN)) {
10013	scx_error(sch, "peek disallowed on builtin DSQ 0x%llx", dsq_id);
10014	return NULL;
10015	}
10016
10017	dsq = find_user_dsq(sch, dsq_id);
10018	if (unlikely(!dsq)) {
10019	scx_error(sch, "peek on non-existent DSQ 0x%llx", dsq_id);
10020	return NULL;
10021	}
10022
10023	return rcu_dereference(dsq->first_task);
10024	}
10025
10026	/**
10027	* scx_bpf_dsq_reenq - Re-enqueue tasks on a DSQ
10028	* @dsq_id: DSQ to re-enqueue
10029	* @reenq_flags: %SCX_RENQ_*
10030	* @aux: implicit BPF argument to access bpf_prog_aux hidden from BPF progs
10031	*
10032	* Iterate over all of the tasks currently enqueued on the DSQ identified by
10033	* @dsq_id, and re-enqueue them in the BPF scheduler. The following DSQs are
10034	* supported:
10035	*
10036	* - Local DSQs (%SCX_DSQ_LOCAL or %SCX_DSQ_LOCAL_ON \| $cpu)
10037	* - User DSQs
10038	*
10039	* Re-enqueues are performed asynchronously. Can be called from anywhere.
10040	*
10041	* %SCX_DSQ_LOCAL resolves to the local DSQ of the rq the current scheduler
10042	* operation is locked to - e.g. the rq being dispatched for in ops.dispatch() -
10043	* or the calling CPU's when no rq is locked.
10044	*/
10045	__bpf_kfunc void scx_bpf_dsq_reenq(u64 dsq_id, u64 reenq_flags,
10046	const struct bpf_prog_aux *aux)
10047	{
10048	struct rq *locked_rq = scx_locked_rq();
10049	struct scx_sched *sch;
10050	struct scx_dispatch_q *dsq;
10051
10052	guard(preempt)();
10053
10054	sch = scx_prog_sched(aux);
10055	if (unlikely(!sch))
10056	return;
10057
10058	if (unlikely(reenq_flags & ~__SCX_REENQ_USER_MASK)) {
10059	scx_error(sch, "invalid SCX_REENQ flags 0x%llx", reenq_flags);
10060	return;
10061	}
10062
10063	/* not specifying any filter bits is the same as %SCX_REENQ_ANY */
10064	if (!(reenq_flags & __SCX_REENQ_FILTER_MASK))
10065	reenq_flags \|= SCX_REENQ_ANY;
10066
10067	dsq = find_dsq_for_dispatch(sch, locked_rq ?: this_rq(), dsq_id, smp_processor_id());
10068	schedule_dsq_reenq(sch, dsq, reenq_flags, locked_rq);
10069	}
10070
10071	/**
10072	* scx_bpf_reenqueue_local___v2 - Re-enqueue tasks on a local DSQ
10073	* @aux: implicit BPF argument to access bpf_prog_aux hidden from BPF progs
10074	*
10075	* Iterate over all of the tasks currently enqueued on the local DSQ of the
10076	* caller's CPU, and re-enqueue them in the BPF scheduler. Can be called from
10077	* anywhere.
10078	*
10079	* This is now a special case of scx_bpf_dsq_reenq() and may be removed in the
10080	* future.
10081	*/
10082	__bpf_kfunc void scx_bpf_reenqueue_local___v2(const struct bpf_prog_aux *aux)
10083	{
10084	scx_bpf_dsq_reenq(SCX_DSQ_LOCAL, 0, aux);
10085	}
10086
10087	__bpf_kfunc_end_defs();
10088
10089	__printf(5, 0)
10090	static s32 __bstr_format(struct scx_sched sch, u64 data_buf, char *line_buf,
10091	size_t line_size, char fmt, unsigned long long data,
10092	u32 data__sz)
10093	{
10094	struct bpf_bprintf_data bprintf_data = { .get_bin_args = true };
10095	s32 ret;
10096
10097	if (data__sz % 8 \|\| data__sz > MAX_BPRINTF_VARARGS * 8 \|\|
10098	(data__sz && !data)) {
10099	scx_error(sch, "invalid data=%p and data__sz=%u", (void *)data, data__sz);
10100	return -EINVAL;
10101	}
10102
10103	ret = copy_from_kernel_nofault(data_buf, data, data__sz);
10104	if (ret < 0) {
10105	scx_error(sch, "failed to read data fields (%d)", ret);
10106	return ret;
10107	}
10108
10109	ret = bpf_bprintf_prepare(fmt, UINT_MAX, data_buf, data__sz / 8,
10110	&bprintf_data);
10111	if (ret < 0) {
10112	scx_error(sch, "format preparation failed (%d)", ret);
10113	return ret;
10114	}
10115
10116	ret = bstr_printf(line_buf, line_size, fmt,
10117	bprintf_data.bin_args);
10118	bpf_bprintf_cleanup(&bprintf_data);
10119	if (ret < 0) {
10120	scx_error(sch, "(\"%s\", %p, %u) failed to format", fmt, data, data__sz);
10121	return ret;
10122	}
10123
10124	return ret;
10125	}
10126
10127	/*
10128	* Exit @sch with the reason formatted from a BPF-supplied bstr format. The exit
10129	* is claimed first and the reason is formatted directly into the winner-owned
10130	* exit_info buffer, which allows use from any context including NMI.
10131	*
10132	* @fmt_blame is the sched blamed for formatting failures through the
10133	* scx_error() calls in __bstr_format() and differs from @sch when a parent
10134	* supplies the kill reason for a child. A formatting failure doesn't revert the
10135	* claim - @sch still exits with the claimed kind and a fallback message.
10136	*/
10137	__printf(5, 0)
10138	bool scx_exit_bstr(struct scx_sched *sch, enum scx_exit_kind kind,
10139	s64 exit_code, struct scx_sched fmt_blame, char fmt,
10140	unsigned long long *data, u32 data__sz)
10141	{
10142	struct scx_exit_info *ei = sch->exit_info;
10143	u64 data_buf[MAX_BPRINTF_VARARGS];
10144	s32 ret;
10145
10146	guard(preempt)();
10147
10148	if (!scx_claim_exit(sch, kind))
10149	return false;
10150
10151	ret = __bstr_format(fmt_blame, data_buf, ei->msg, SCX_EXIT_MSG_LEN,
10152	fmt, data, data__sz);
10153	if (ret < 0)
10154	scnprintf(ei->msg, SCX_EXIT_MSG_LEN,
10155	"exit message formatting failed (%d)", ret);
10156
10157	scx_finish_exit(sch, kind, exit_code, raw_smp_processor_id());
10158	return true;
10159	}
10160
10161	__bpf_kfunc_start_defs();
10162
10163	/**
10164	* scx_bpf_exit_bstr - Gracefully exit the BPF scheduler.
10165	* @exit_code: Exit value to pass to user space via struct scx_exit_info.
10166	* @fmt: error message format string
10167	* @data: format string parameters packaged using ___bpf_fill() macro
10168	* @data__sz: @data len, must end in '__sz' for the verifier
10169	* @aux: implicit BPF argument to access bpf_prog_aux hidden from BPF progs
10170	*
10171	* Indicate that the BPF scheduler wants to exit gracefully, and initiate ops
10172	* disabling.
10173	*/
10174	__printf(2, 0)
10175	__bpf_kfunc void scx_bpf_exit_bstr(s64 exit_code, char *fmt,
10176	unsigned long long *data, u32 data__sz,
10177	const struct bpf_prog_aux *aux)
10178	{
10179	struct scx_sched *sch;
10180
10181	guard(rcu)();
10182
10183	sch = scx_prog_sched(aux);
10184	if (likely(sch))
10185	scx_exit_bstr(sch, SCX_EXIT_UNREG_BPF, exit_code, sch, fmt,
10186	data, data__sz);
10187	}
10188
10189	/**
10190	* scx_bpf_error_bstr - Indicate fatal error
10191	* @fmt: error message format string
10192	* @data: format string parameters packaged using ___bpf_fill() macro
10193	* @data__sz: @data len, must end in '__sz' for the verifier
10194	* @aux: implicit BPF argument to access bpf_prog_aux hidden from BPF progs
10195	*
10196	* Indicate that the BPF scheduler encountered a fatal error and initiate ops
10197	* disabling.
10198	*/
10199	__printf(1, 0)
10200	__bpf_kfunc void scx_bpf_error_bstr(char fmt, unsigned long long data,
10201	u32 data__sz, const struct bpf_prog_aux *aux)
10202	{
10203	struct scx_sched *sch;
10204
10205	guard(rcu)();
10206
10207	sch = scx_prog_sched(aux);
10208	if (likely(sch))
10209	scx_exit_bstr(sch, SCX_EXIT_ERROR_BPF, 0, sch, fmt, data,
10210	data__sz);
10211	}
10212
10213	/**
10214	* scx_bpf_dump_bstr - Generate extra debug dump specific to the BPF scheduler
10215	* @fmt: format string
10216	* @data: format string parameters packaged using ___bpf_fill() macro
10217	* @data__sz: @data len, must end in '__sz' for the verifier
10218	* @aux: implicit BPF argument to access bpf_prog_aux hidden from BPF progs
10219	*
10220	* To be called through scx_bpf_dump() helper from ops.dump(), dump_cpu() and
10221	* dump_task() to generate extra debug dump specific to the BPF scheduler.
10222	*
10223	* The extra dump may be multiple lines. A single line may be split over
10224	* multiple calls. The last line is automatically terminated.
10225	*/
10226	__printf(1, 0)
10227	__bpf_kfunc void scx_bpf_dump_bstr(char fmt, unsigned long long data,
10228	u32 data__sz, const struct bpf_prog_aux *aux)
10229	{
10230	struct scx_sched *sch;
10231	struct scx_dump_data *dd = &scx_dump_data;
10232	struct scx_bstr_buf *buf = &dd->buf;
10233	s32 ret;
10234
10235	guard(rcu)();
10236
10237	sch = scx_prog_sched(aux);
10238	if (unlikely(!sch))
10239	return;
10240
10241	if (raw_smp_processor_id() != dd->cpu) {
10242	scx_error(sch, "scx_bpf_dump() must only be called from ops.dump() and friends");
10243	return;
10244	}
10245
10246	/* append the formatted string to the line buf */
10247	ret = __bstr_format(sch, buf->data, buf->line + dd->cursor,
10248	sizeof(buf->line) - dd->cursor, fmt, data, data__sz);
10249	if (ret < 0) {
10250	scx_dump_line(dd->s, "%s[!] (\"%s\", %p, %u) failed to format (%d)",
10251	dd->prefix, fmt, data, data__sz, ret);
10252	return;
10253	}
10254
10255	dd->cursor += ret;
10256	dd->cursor = min_t(s32, dd->cursor, sizeof(buf->line));
10257
10258	if (!dd->cursor)
10259	return;
10260
10261	/*
10262	* If the line buf overflowed or ends in a newline, flush it into the
10263	* dump. This is to allow the caller to generate a single line over
10264	* multiple calls. As ops_dump_flush() can also handle multiple lines in
10265	* the line buf, the only case which can lead to an unexpected
10266	* truncation is when the caller keeps generating newlines in the middle
10267	* instead of the end consecutively. Don't do that.
10268	*/
10269	if (dd->cursor >= sizeof(buf->line) \|\| buf->line[dd->cursor - 1] == '\n')
10270	ops_dump_flush();
10271	}
10272
10273	/**
10274	* scx_bpf_cpuperf_cap - Query the maximum relative capacity of a CPU
10275	* @cpu: CPU of interest
10276	* @aux: implicit BPF argument to access bpf_prog_aux hidden from BPF progs
10277	*
10278	* Return the maximum relative capacity of @cpu in relation to the most
10279	* performant CPU in the system. The return value is in the range [1,
10280	* %SCX_CPUPERF_ONE]. See scx_bpf_cpuperf_cur().
10281	*/
10282	__bpf_kfunc u32 scx_bpf_cpuperf_cap(s32 cpu, const struct bpf_prog_aux *aux)
10283	{
10284	struct scx_sched *sch;
10285
10286	guard(rcu)();
10287
10288	sch = scx_prog_sched(aux);
10289	if (likely(sch) && scx_cpu_valid(sch, cpu, NULL))
10290	return arch_scale_cpu_capacity(cpu);
10291	else
10292	return SCX_CPUPERF_ONE;
10293	}
10294
10295	/**
10296	* scx_bpf_cidperf_cap - Query the maximum relative capacity of the CPU at @cid
10297	* @cid: cid of the CPU to query
10298	* @aux: implicit BPF argument to access bpf_prog_aux hidden from BPF progs
10299	*
10300	* cid-addressed equivalent of scx_bpf_cpuperf_cap().
10301	*/
10302	__bpf_kfunc u32 scx_bpf_cidperf_cap(s32 cid, const struct bpf_prog_aux *aux)
10303	{
10304	struct scx_sched *sch;
10305	s32 cpu;
10306
10307	guard(rcu)();
10308
10309	sch = scx_prog_sched(aux);
10310	if (unlikely(!sch))
10311	return SCX_CPUPERF_ONE;
10312	cpu = scx_cid_to_cpu(sch, cid);
10313	if (cpu < 0)
10314	return SCX_CPUPERF_ONE;
10315	return arch_scale_cpu_capacity(cpu);
10316	}
10317
10318	/**
10319	* scx_bpf_cpuperf_cur - Query the current relative performance of a CPU
10320	* @cpu: CPU of interest
10321	* @aux: implicit BPF argument to access bpf_prog_aux hidden from BPF progs
10322	*
10323	* Return the current relative performance of @cpu in relation to its maximum.
10324	* The return value is in the range [1, %SCX_CPUPERF_ONE].
10325	*
10326	* The current performance level of a CPU in relation to the maximum performance
10327	* available in the system can be calculated as follows:
10328	*
10329	* scx_bpf_cpuperf_cap() * scx_bpf_cpuperf_cur() / %SCX_CPUPERF_ONE
10330	*
10331	* The result is in the range [1, %SCX_CPUPERF_ONE].
10332	*/
10333	__bpf_kfunc u32 scx_bpf_cpuperf_cur(s32 cpu, const struct bpf_prog_aux *aux)
10334	{
10335	struct scx_sched *sch;
10336
10337	guard(rcu)();
10338
10339	sch = scx_prog_sched(aux);
10340	if (likely(sch) && scx_cpu_valid(sch, cpu, NULL))
10341	return arch_scale_freq_capacity(cpu);
10342	else
10343	return SCX_CPUPERF_ONE;
10344	}
10345
10346	/**
10347	* scx_bpf_cidperf_cur - Query the current performance of the CPU at @cid
10348	* @cid: cid of the CPU to query
10349	* @aux: implicit BPF argument to access bpf_prog_aux hidden from BPF progs
10350	*
10351	* cid-addressed equivalent of scx_bpf_cpuperf_cur().
10352	*/
10353	__bpf_kfunc u32 scx_bpf_cidperf_cur(s32 cid, const struct bpf_prog_aux *aux)
10354	{
10355	struct scx_sched *sch;
10356	s32 cpu;
10357
10358	guard(rcu)();
10359
10360	sch = scx_prog_sched(aux);
10361	if (unlikely(!sch))
10362	return SCX_CPUPERF_ONE;
10363	cpu = scx_cid_to_cpu(sch, cid);
10364	if (cpu < 0)
10365	return SCX_CPUPERF_ONE;
10366	return arch_scale_freq_capacity(cpu);
10367	}
10368
10369	/* validate and apply a cpuperf target, see scx_bpf_cpuperf_set() */
10370	static s32 scx_cpuperf_set(struct scx_sched *sch, s32 cpu, u32 perf)
10371	{
10372	struct rq rq, locked_rq;
10373	struct rq_flags rf;
10374	s32 ret;
10375
10376	if (unlikely(perf > SCX_CPUPERF_ONE)) {
10377	scx_error(sch, "Invalid cpuperf target %u for CPU %d", perf, cpu);
10378	return -EINVAL;
10379	}
10380
10381	if (!scx_cpu_valid(sch, cpu, NULL))
10382	return -EINVAL;
10383
10384	rq = cpu_rq(cpu);
10385	locked_rq = scx_locked_rq();
10386
10387	/*
10388	* When called with an rq lock held, restrict the operation to the
10389	* corresponding CPU to prevent ABBA deadlocks.
10390	*/
10391	if (locked_rq && rq != locked_rq) {
10392	scx_error(sch, "Invalid target CPU %d", cpu);
10393	return -EINVAL;
10394	}
10395
10396	/*
10397	* If no rq lock is held, allow to operate on any CPU by acquiring
10398	* the corresponding rq lock.
10399	*/
10400	if (!locked_rq) {
10401	rq_lock_irqsave(rq, &rf);
10402	update_rq_clock(rq);
10403	}
10404
10405	/*
10406	* ecaps updates are folded under the rq lock, making this test
10407	* authoritative: a write can never land after a revoke has taken
10408	* effect on @cpu.
10409	*/
10410	if (likely(!scx_missing_caps(sch, cpu, SCX_CAP_PERF))) {
10411	rq->scx.cpuperf_target = perf;
10412	cpufreq_update_util(rq, 0);
10413	ret = 0;
10414	} else {
10415	__scx_add_event(sch, SCX_EV_SUB_CIDPERF_DENIED, 1);
10416	ret = -EACCES;
10417	}
10418
10419	if (!locked_rq)
10420	rq_unlock_irqrestore(rq, &rf);
10421
10422	return ret;
10423	}
10424
10425	/**
10426	* scx_bpf_cpuperf_set - Set the relative performance target of a CPU
10427	* @cpu: CPU of interest
10428	* @perf: target performance level [0, %SCX_CPUPERF_ONE]
10429	* @aux: implicit BPF argument to access bpf_prog_aux hidden from BPF progs
10430	*
10431	* Set the target performance level of @cpu to @perf. @perf is in linear
10432	* relative scale between 0 and %SCX_CPUPERF_ONE. This determines how the
10433	* schedutil cpufreq governor chooses the target frequency.
10434	*
10435	* The actual performance level chosen, CPU grouping, and the overhead and
10436	* latency of the operations are dependent on the hardware and cpufreq driver in
10437	* use. Consult hardware and cpufreq documentation for more information. The
10438	* current performance level can be monitored using scx_bpf_cpuperf_cur().
10439	*/
10440	__bpf_kfunc void scx_bpf_cpuperf_set(s32 cpu, u32 perf, const struct bpf_prog_aux *aux)
10441	{
10442	struct scx_sched *sch;
10443
10444	guard(rcu)();
10445
10446	sch = scx_prog_sched(aux);
10447	if (unlikely(!sch))
10448	return;
10449
10450	scx_cpuperf_set(sch, cpu, perf);
10451	}
10452
10453	/**
10454	* scx_bpf_cidperf_set - Set the performance target of the CPU at @cid
10455	* @cid: cid of the CPU to target
10456	* @perf: target performance level [0, %SCX_CPUPERF_ONE]
10457	* @aux: implicit BPF argument to access bpf_prog_aux hidden from BPF progs
10458	*
10459	* cid-addressed equivalent of scx_bpf_cpuperf_set(). A sub-sched needs
10460	* SCX_CAP_PERF on @cid. Returns 0 if the target was applied, -%EACCES if
10461	* the write was denied for missing caps, other -errnos if @cid didn't
10462	* resolve.
10463	*/
10464	__bpf_kfunc s32 scx_bpf_cidperf_set(s32 cid, u32 perf,
10465	const struct bpf_prog_aux *aux)
10466	{
10467	struct scx_sched *sch;
10468	s32 cpu;
10469
10470	guard(rcu)();
10471
10472	sch = scx_prog_sched(aux);
10473	if (unlikely(!sch))
10474	return -ENODEV;
10475	cpu = scx_cid_to_cpu(sch, cid);
10476	if (cpu < 0)
10477	return cpu;
10478
10479	return scx_cpuperf_set(sch, cpu, perf);
10480	}
10481
10482	/**
10483	* scx_bpf_nr_node_ids - Return the number of possible node IDs
10484	*
10485	* All valid node IDs in the system are smaller than the returned value.
10486	*/
10487	__bpf_kfunc u32 scx_bpf_nr_node_ids(void)
10488	{
10489	return nr_node_ids;
10490	}
10491
10492	/**
10493	* scx_bpf_nr_cpu_ids - Return the number of possible CPU IDs
10494	*
10495	* All valid CPU IDs in the system are smaller than the returned value.
10496	*/
10497	__bpf_kfunc u32 scx_bpf_nr_cpu_ids(void)
10498	{
10499	return nr_cpu_ids;
10500	}
10501
10502	/**
10503	* scx_bpf_nr_cids - Return the size of the cid space
10504	*
10505	* Equals num_possible_cpus(). All valid cids are in [0, return value).
10506	*/
10507	__bpf_kfunc u32 scx_bpf_nr_cids(void)
10508	{
10509	return num_possible_cpus();
10510	}
10511
10512	/**
10513	* scx_bpf_nr_online_cids - Return current count of online CPUs in cid space
10514	*
10515	* Return num_online_cpus(). The standard model restarts the scheduler on
10516	* hotplug, which lets schedulers treat [0, nr_online_cids) as the online
10517	* range. Schedulers that prefer to handle hotplug without a restart should
10518	* install a custom mapping via scx_bpf_cid_override() and track onlining
10519	* through the ops.cid_online / ops.cid_offline callbacks, starting from the
10520	* mask scx_bpf_online_cmask() returns.
10521	*/
10522	__bpf_kfunc u32 scx_bpf_nr_online_cids(void)
10523	{
10524	return num_online_cpus();
10525	}
10526
10527	/**
10528	* scx_bpf_online_cmask - Return the online cid mask in the scheduler arena
10529	* @aux: implicit BPF argument to access bpf_prog_aux hidden from BPF progs
10530	*
10531	* Return a kernel-maintained cmask covering [0, scx_bpf_nr_cids()), or NULL if
10532	* the calling program is not associated with a live cid-form scheduler or the
10533	* mask is not allocated yet, as in ops.init_cids(). Treat the mask as read-only
10534	* even though arena memory stays writable by the BPF scheduler. The mask
10535	* follows the SCX hotplug notifications: a cid's bit is updated before
10536	* ops.cid_online/offline() runs for it. The pointer is valid from ops.init()
10537	* through ops.exit(). Root ops.init() runs with hotplug excluded. Other
10538	* contexts can observe concurrent updates.
10539	*/
10540	__bpf_kfunc const void scx_bpf_online_cmask(const struct bpf_prog_aux aux)
10541	{
10542	struct scx_sched *sch;
10543	struct scx_cmask *online;
10544
10545	guard(rcu)();
10546
10547	sch = scx_prog_sched(aux);
10548	if (unlikely(!sch))
10549	return NULL;
10550	online = sch->online_cmask;
10551	if (unlikely(!online))
10552	return NULL;
10553
10554	return (void *)scx_kaddr_to_arena(sch, online);
10555	}
10556
10557	/**
10558	* scx_bpf_this_cid - Return the cid of the CPU this program is running on
10559	*
10560	* cid-addressed equivalent of bpf_get_smp_processor_id() for scx programs.
10561	* The current cpu is trivially valid, so this is just a table lookup. Return
10562	* -EINVAL if called before any scheduler has ever published its cid tables.
10563	*/
10564	__bpf_kfunc s32 scx_bpf_this_cid(void)
10565	{
10566	s16 *tbl;
10567
10568	guard(rcu)();
10569
10570	tbl = rcu_dereference(scx_cpu_to_cid_tbl);
10571	if (!tbl)
10572	return -EINVAL;
10573	return tbl[raw_smp_processor_id()];
10574	}
10575
10576	/**
10577	* scx_bpf_get_possible_cpumask - Get a referenced kptr to cpu_possible_mask
10578	*/
10579	__bpf_kfunc const struct cpumask *scx_bpf_get_possible_cpumask(void)
10580	{
10581	return cpu_possible_mask;
10582	}
10583
10584	/**
10585	* scx_bpf_get_online_cpumask - Get a referenced kptr to cpu_online_mask
10586	*/
10587	__bpf_kfunc const struct cpumask *scx_bpf_get_online_cpumask(void)
10588	{
10589	return cpu_online_mask;
10590	}
10591
10592	/**
10593	* scx_bpf_put_cpumask - Release a possible/online cpumask
10594	* @cpumask: cpumask to release
10595	*/
10596	__bpf_kfunc void scx_bpf_put_cpumask(const struct cpumask *cpumask)
10597	{
10598	/*
10599	* Empty function body because we aren't actually acquiring or releasing
10600	* a reference to a global cpumask, which is read-only in the caller and
10601	* is never released. The acquire / release semantics here are just used
10602	* to make the cpumask is a trusted pointer in the caller.
10603	*/
10604	}
10605
10606	/**
10607	* scx_bpf_task_running - Is task currently running?
10608	* @p: task of interest
10609	*/
10610	__bpf_kfunc bool scx_bpf_task_running(const struct task_struct *p)
10611	{
10612	return task_rq(p)->curr == p;
10613	}
10614
10615	/**
10616	* scx_bpf_task_cpu - CPU a task is currently associated with
10617	* @p: task of interest
10618	*/
10619	__bpf_kfunc s32 scx_bpf_task_cpu(const struct task_struct *p)
10620	{
10621	return task_cpu(p);
10622	}
10623
10624	/**
10625	* scx_bpf_task_cid - cid a task is currently associated with
10626	* @p: task of interest
10627	*
10628	* cid-addressed equivalent of scx_bpf_task_cpu(). task_cpu(p) is always a
10629	* valid cpu, so this is just a table lookup. Return -EINVAL if called before
10630	* any scheduler has ever published its cid tables.
10631	*/
10632	__bpf_kfunc s32 scx_bpf_task_cid(const struct task_struct *p)
10633	{
10634	s16 *tbl;
10635
10636	/* KF_RCU covers only @p - a sleepable program holds no RCU lock */
10637	guard(rcu)();
10638
10639	tbl = rcu_dereference(scx_cpu_to_cid_tbl);
10640	if (!tbl)
10641	return -EINVAL;
10642	return tbl[task_cpu(p)];
10643	}
10644
10645	/**
10646	* scx_bpf_locked_rq - Return the rq currently locked by SCX
10647	* @aux: implicit BPF argument to access bpf_prog_aux hidden from BPF progs
10648	*
10649	* Returns the rq if a rq lock is currently held by SCX.
10650	* Otherwise emits an error and returns NULL.
10651	*/
10652	__bpf_kfunc struct rq scx_bpf_locked_rq(const struct bpf_prog_aux aux)
10653	{
10654	struct scx_sched *sch;
10655	struct rq *rq;
10656
10657	guard(preempt)();
10658
10659	sch = scx_prog_sched(aux);
10660	if (unlikely(!sch))
10661	return NULL;
10662
10663	rq = scx_locked_rq();
10664	if (!rq) {
10665	scx_error(sch, "accessing rq without holding rq lock");
10666	return NULL;
10667	}
10668
10669	return rq;
10670	}
10671
10672	/**
10673	* scx_bpf_cpu_curr - Return remote CPU's curr task
10674	* @cpu: CPU of interest
10675	* @aux: implicit BPF argument to access bpf_prog_aux hidden from BPF progs
10676	*
10677	* Callers must hold RCU read lock (KF_RCU).
10678	*/
10679	__bpf_kfunc struct task_struct scx_bpf_cpu_curr(s32 cpu, const struct bpf_prog_aux aux)
10680	{
10681	struct scx_sched *sch;
10682
10683	guard(rcu)();
10684
10685	sch = scx_prog_sched(aux);
10686	if (unlikely(!sch))
10687	return NULL;
10688
10689	if (!scx_cpu_valid(sch, cpu, NULL))
10690	return NULL;
10691
10692	return rcu_dereference(cpu_rq(cpu)->curr);
10693	}
10694
10695	/**
10696	* scx_bpf_cid_curr - Return the curr task on the CPU at @cid
10697	* @cid: cid of interest
10698	* @aux: implicit BPF argument to access bpf_prog_aux hidden from BPF progs
10699	*
10700	* cid-addressed equivalent of scx_bpf_cpu_curr(). Callers must hold RCU
10701	* read lock (KF_RCU).
10702	*/
10703	__bpf_kfunc struct task_struct scx_bpf_cid_curr(s32 cid, const struct bpf_prog_aux aux)
10704	{
10705	struct scx_sched *sch;
10706	s32 cpu;
10707
10708	guard(rcu)();
10709
10710	sch = scx_prog_sched(aux);
10711	if (unlikely(!sch))
10712	return NULL;
10713	cpu = scx_cid_to_cpu(sch, cid);
10714	if (cpu < 0)
10715	return NULL;
10716	return rcu_dereference(cpu_rq(cpu)->curr);
10717	}
10718
10719	/**
10720	* scx_bpf_tid_to_task - Look up a task by its scx tid
10721	* @tid: task ID previously read from p->scx.tid
10722	*
10723	* Returns the task with the given tid, or NULL if no such task exists. The
10724	* returned pointer is valid until the end of the current RCU read section
10725	* (KF_RCU_PROTECTED). Requires SCX_OPS_TID_TO_TASK to be set on the root
10726	* scheduler; otherwise an error is raised and NULL returned.
10727	*/
10728	__bpf_kfunc struct task_struct *scx_bpf_tid_to_task(u64 tid)
10729	{
10730	struct sched_ext_entity *scx;
10731
10732	if (!scx_tid_to_task_enabled()) {
10733	struct scx_sched *sch = rcu_dereference(scx_root);
10734
10735	if (sch)
10736	scx_error(sch, "scx_bpf_tid_to_task() called without SCX_OPS_TID_TO_TASK");
10737	return NULL;
10738	}
10739
10740	scx = rhashtable_lookup(&scx_tid_hash, &tid, scx_tid_hash_params);
10741	if (!scx)
10742	return NULL;
10743
10744	return container_of(scx, struct task_struct, scx);
10745	}
10746
10747	u64 __scx_bpf_now(struct rq *rq)
10748	{
10749	/* the caller must be on @rq's cpu or hold its lock */
10750	lockdep_assert((rq == this_rq() && !preemptible()) \|\|
10751	lockdep_is_held(__rq_lockp(rq)));
10752
10753	if (smp_load_acquire(&rq->scx.flags) & SCX_RQ_CLK_VALID) {
10754	/* if the rq clock is valid, use the cached rq clock */
10755	return READ_ONCE(rq->scx.clock);
10756	} else {
10757	/*
10758	* Otherwise, return a fresh rq clock.
10759	*
10760	* The rq clock is updated outside of the rq lock.
10761	* In this case, keep the updated rq clock invalid so the next
10762	* read outside the rq lock gets a fresh rq clock.
10763	*/
10764	return sched_clock_cpu(cpu_of(rq));
10765	}
10766	}
10767
10768	/**
10769	* scx_bpf_now - Returns a high-performance monotonically non-decreasing
10770	* clock for the current CPU. The clock returned is in nanoseconds.
10771	*
10772	* It provides the following properties:
10773	*
10774	* 1) High performance: Many BPF schedulers call bpf_ktime_get_ns() frequently
10775	* to account for execution time and track tasks' runtime properties.
10776	* Unfortunately, in some hardware platforms, bpf_ktime_get_ns() -- which
10777	* eventually reads a hardware timestamp counter -- is neither performant nor
10778	* scalable. scx_bpf_now() aims to provide a high-performance clock by
10779	* using the rq clock in the scheduler core whenever possible.
10780	*
10781	* 2) High enough resolution for the BPF scheduler use cases: In most BPF
10782	* scheduler use cases, the required clock resolution is lower than the most
10783	* accurate hardware clock (e.g., rdtsc in x86). scx_bpf_now() basically
10784	* uses the rq clock in the scheduler core whenever it is valid. It considers
10785	* that the rq clock is valid from the time the rq clock is updated
10786	* (update_rq_clock) until the rq is unlocked (rq_unpin_lock).
10787	*
10788	* 3) Monotonically non-decreasing clock for the same CPU: scx_bpf_now()
10789	* guarantees the clock never goes backward when comparing them in the same
10790	* CPU. On the other hand, when comparing clocks in different CPUs, there
10791	* is no such guarantee -- the clock can go backward. It provides a
10792	* monotonically non-decreasing clock so that it would provide the same
10793	* clock values in two different scx_bpf_now() calls in the same CPU
10794	* during the same period of when the rq clock is valid.
10795	*/
10796	__bpf_kfunc u64 scx_bpf_now(void)
10797	{
10798	/*
10799	* Note that scx_bpf_now() is re-entrant between a process context and
10800	* an interrupt context (e.g., timer interrupt). However, we don't need
10801	* to consider the race between them because such race is not observable
10802	* from a caller.
10803	*/
10804	guard(preempt)();
10805	return __scx_bpf_now(this_rq());
10806	}
10807
10808	static void scx_read_events(struct scx_sched sch, struct scx_event_stats events)
10809	{
10810	int cpu;
10811
10812	/* Aggregate per-CPU event counters into @events. */
10813	memset(events, 0, sizeof(*events));
10814	for_each_possible_cpu(cpu) {
10815	struct scx_event_stats *e_cpu = &per_cpu_ptr(sch->pcpu, cpu)->event_stats;
10816	#define SCX_EVENT(name) (events->name += READ_ONCE(e_cpu->name))
10817	SCX_EVENTS_LIST(SCX_EVENT);
10818	#undef SCX_EVENT
10819	}
10820	}
10821
10822	/**
10823	* scx_bpf_events - Read the event counters of the calling scheduler
10824	* @events: output buffer from a BPF program
10825	* @events__sz: @events len, must end in '__sz' for the verifier
10826	* @aux: implicit BPF argument to access bpf_prog_aux hidden from BPF progs
10827	*
10828	* Read the event counters of the scheduler associated with the calling program.
10829	* @events is zeroed when no scheduler can be resolved.
10830	*/
10831	__bpf_kfunc void scx_bpf_events(struct scx_event_stats *events, size_t events__sz,
10832	const struct bpf_prog_aux *aux)
10833	{
10834	struct scx_sched *sch;
10835	struct scx_event_stats e_sys;
10836
10837	rcu_read_lock();
10838	sch = scx_prog_sched(aux);
10839	if (sch)
10840	scx_read_events(sch, &e_sys);
10841	else
10842	memset(&e_sys, 0, sizeof(e_sys));
10843	rcu_read_unlock();
10844
10845	/*
10846	* We cannot entirely trust a BPF-provided size since a BPF program
10847	* might be compiled against a different vmlinux.h, of which
10848	* scx_event_stats would be larger (a newer vmlinux.h) or smaller
10849	* (an older vmlinux.h). Hence, we use the smaller size to avoid
10850	* memory corruption.
10851	*/
10852	events__sz = min(events__sz, sizeof(*events));
10853	memcpy(events, &e_sys, events__sz);
10854	}
10855
10856	#ifdef CONFIG_CGROUP_SCHED
10857	/**
10858	* scx_bpf_task_cgroup - Return the sched cgroup of a task
10859	* @p: task of interest
10860	* @aux: implicit BPF argument to access bpf_prog_aux hidden from BPF progs
10861	*
10862	* @p->sched_task_group->css.cgroup represents the cgroup @p is associated with
10863	* from the scheduler's POV. SCX operations should use this function to
10864	* determine @p's current cgroup as, unlike following @p->cgroups,
10865	* @p->sched_task_group is stable for the duration of the SCX op. See
10866	* SCX_CALL_OP_TASK() for details.
10867	*/
10868	__bpf_kfunc struct cgroup scx_bpf_task_cgroup(struct task_struct p,
10869	const struct bpf_prog_aux *aux)
10870	{
10871	struct task_group *tg = p->sched_task_group;
10872	struct cgroup *cgrp = &cgrp_dfl_root.cgrp;
10873	struct scx_sched *sch;
10874
10875	guard(rcu)();
10876
10877	sch = scx_prog_sched(aux);
10878	if (unlikely(!sch))
10879	goto out;
10880
10881	if (!scx_kf_arg_task_ok(sch, p))
10882	goto out;
10883
10884	cgrp = tg_cgrp(tg);
10885
10886	out:
10887	cgroup_get(cgrp);
10888	return cgrp;
10889	}
10890	#endif /* CONFIG_CGROUP_SCHED */
10891
10892	__bpf_kfunc_end_defs();
10893
10894	BTF_KFUNCS_START(scx_kfunc_ids_any)
10895	BTF_ID_FLAGS(func, scx_bpf_task_set_slice, KF_IMPLICIT_ARGS \| KF_RCU);
10896	BTF_ID_FLAGS(func, scx_bpf_task_set_dsq_vtime, KF_IMPLICIT_ARGS \| KF_RCU);
10897	BTF_ID_FLAGS(func, scx_bpf_kick_cpu, KF_IMPLICIT_ARGS)
10898	BTF_ID_FLAGS(func, scx_bpf_kick_cid, KF_IMPLICIT_ARGS)
10899	BTF_ID_FLAGS(func, scx_bpf_dsq_nr_queued, KF_IMPLICIT_ARGS)
10900	BTF_ID_FLAGS(func, scx_bpf_destroy_dsq, KF_IMPLICIT_ARGS)
10901	BTF_ID_FLAGS(func, scx_bpf_dsq_peek, KF_IMPLICIT_ARGS \| KF_RCU_PROTECTED \| KF_RET_NULL)
10902	BTF_ID_FLAGS(func, scx_bpf_dsq_reenq, KF_IMPLICIT_ARGS)
10903	BTF_ID_FLAGS(func, scx_bpf_reenqueue_local___v2, KF_IMPLICIT_ARGS)
10904	BTF_ID_FLAGS(func, bpf_iter_scx_dsq_new, KF_IMPLICIT_ARGS \| KF_ITER_NEW \| KF_RCU_PROTECTED)
10905	BTF_ID_FLAGS(func, bpf_iter_scx_dsq_next, KF_ITER_NEXT \| KF_RET_NULL)
10906	BTF_ID_FLAGS(func, bpf_iter_scx_dsq_destroy, KF_ITER_DESTROY)
10907	BTF_ID_FLAGS(func, scx_bpf_exit_bstr, KF_IMPLICIT_ARGS)
10908	BTF_ID_FLAGS(func, scx_bpf_error_bstr, KF_IMPLICIT_ARGS)
10909	BTF_ID_FLAGS(func, scx_bpf_dump_bstr, KF_IMPLICIT_ARGS)
10910	BTF_ID_FLAGS(func, scx_bpf_cpuperf_cap, KF_IMPLICIT_ARGS)
10911	BTF_ID_FLAGS(func, scx_bpf_cpuperf_cur, KF_IMPLICIT_ARGS)
10912	BTF_ID_FLAGS(func, scx_bpf_cpuperf_set, KF_IMPLICIT_ARGS)
10913	BTF_ID_FLAGS(func, scx_bpf_cidperf_cap, KF_IMPLICIT_ARGS)
10914	BTF_ID_FLAGS(func, scx_bpf_cidperf_cur, KF_IMPLICIT_ARGS)
10915	BTF_ID_FLAGS(func, scx_bpf_cidperf_set, KF_IMPLICIT_ARGS)
10916	BTF_ID_FLAGS(func, scx_bpf_nr_node_ids)
10917	BTF_ID_FLAGS(func, scx_bpf_nr_cpu_ids)
10918	BTF_ID_FLAGS(func, scx_bpf_nr_cids)
10919	BTF_ID_FLAGS(func, scx_bpf_nr_online_cids)
10920	BTF_ID_FLAGS(func, scx_bpf_online_cmask, KF_IMPLICIT_ARGS \| KF_ARENA_RET)
10921	BTF_ID_FLAGS(func, scx_bpf_this_cid)
10922	BTF_ID_FLAGS(func, scx_bpf_get_possible_cpumask, KF_ACQUIRE)
10923	BTF_ID_FLAGS(func, scx_bpf_get_online_cpumask, KF_ACQUIRE)
10924	BTF_ID_FLAGS(func, scx_bpf_put_cpumask, KF_RELEASE)
10925	BTF_ID_FLAGS(func, scx_bpf_task_running, KF_RCU)
10926	BTF_ID_FLAGS(func, scx_bpf_task_cpu, KF_RCU)
10927	BTF_ID_FLAGS(func, scx_bpf_task_cid, KF_RCU)
10928	BTF_ID_FLAGS(func, scx_bpf_locked_rq, KF_IMPLICIT_ARGS \| KF_RET_NULL)
10929	BTF_ID_FLAGS(func, scx_bpf_cpu_curr, KF_IMPLICIT_ARGS \| KF_RET_NULL \| KF_RCU_PROTECTED)
10930	BTF_ID_FLAGS(func, scx_bpf_cid_curr, KF_IMPLICIT_ARGS \| KF_RET_NULL \| KF_RCU_PROTECTED)
10931	BTF_ID_FLAGS(func, scx_bpf_tid_to_task, KF_RET_NULL \| KF_RCU_PROTECTED)
10932	BTF_ID_FLAGS(func, scx_bpf_now)
10933	BTF_ID_FLAGS(func, scx_bpf_events, KF_IMPLICIT_ARGS)
10934	#ifdef CONFIG_CGROUP_SCHED
10935	BTF_ID_FLAGS(func, scx_bpf_task_cgroup, KF_IMPLICIT_ARGS \| KF_RCU \| KF_ACQUIRE)
10936	#endif
10937	BTF_ID_FLAGS(func, scx_bpf_sub_grant, KF_IMPLICIT_ARGS)
10938	BTF_ID_FLAGS(func, scx_bpf_sub_revoke, KF_IMPLICIT_ARGS)
10939	BTF_ID_FLAGS(func, scx_bpf_sub_caps, KF_IMPLICIT_ARGS)
10940	BTF_ID_FLAGS(func, scx_bpf_sub_kill_bstr, KF_IMPLICIT_ARGS)
10941	BTF_KFUNCS_END(scx_kfunc_ids_any)
10942
10943	static const struct btf_kfunc_id_set scx_kfunc_set_any = {
10944	.owner = THIS_MODULE,
10945	.set = &scx_kfunc_ids_any,
10946	.filter = scx_kfunc_context_filter,
10947	};
10948
10949	/*
10950	* cpu-form kfuncs that are forbidden from cid-form schedulers
10951	* (bpf_sched_ext_ops_cid). Programs targeting the cid struct_ops type must
10952	* use the cid-form alternative (cid/cmask kfuncs).
10953	*
10954	* Membership overlaps with scx_kfunc_ids_{any,idle,select_cpu}; the filter
10955	* tests this set independently and rejects matches before the per-op
10956	* allow-list check runs.
10957	*
10958	* pahole/resolve_btfids scans every BTF_ID_FLAGS() at build time and
10959	* intersects flags across duplicate entries, so each entry must carry the
10960	* same flags as the kfunc's primary declaration; otherwise the flags get
10961	* dropped globally.
10962	*/
10963	BTF_KFUNCS_START(scx_kfunc_ids_cpu_only)
10964	BTF_ID_FLAGS(func, scx_bpf_kick_cpu, KF_IMPLICIT_ARGS)
10965	BTF_ID_FLAGS(func, scx_bpf_task_cpu, KF_RCU)
10966	BTF_ID_FLAGS(func, scx_bpf_cpu_curr, KF_IMPLICIT_ARGS \| KF_RET_NULL \| KF_RCU_PROTECTED)
10967	BTF_ID_FLAGS(func, scx_bpf_cpu_node, KF_IMPLICIT_ARGS)
10968	BTF_ID_FLAGS(func, scx_bpf_cpuperf_cap, KF_IMPLICIT_ARGS)
10969	BTF_ID_FLAGS(func, scx_bpf_cpuperf_cur, KF_IMPLICIT_ARGS)
10970	BTF_ID_FLAGS(func, scx_bpf_cpuperf_set, KF_IMPLICIT_ARGS)
10971	BTF_ID_FLAGS(func, scx_bpf_get_possible_cpumask, KF_ACQUIRE)
10972	BTF_ID_FLAGS(func, scx_bpf_get_online_cpumask, KF_ACQUIRE)
10973	BTF_ID_FLAGS(func, scx_bpf_put_cpumask, KF_RELEASE)
10974	BTF_ID_FLAGS(func, scx_bpf_select_cpu_dfl, KF_IMPLICIT_ARGS \| KF_RCU)
10975	BTF_ID_FLAGS(func, __scx_bpf_select_cpu_and, KF_IMPLICIT_ARGS \| KF_RCU)
10976	BTF_ID_FLAGS(func, scx_bpf_select_cpu_and, KF_RCU)
10977	BTF_ID_FLAGS(func, scx_bpf_get_idle_cpumask, KF_IMPLICIT_ARGS \| KF_ACQUIRE)
10978	BTF_ID_FLAGS(func, scx_bpf_get_idle_cpumask_node, KF_IMPLICIT_ARGS \| KF_ACQUIRE)
10979	BTF_ID_FLAGS(func, scx_bpf_get_idle_smtmask, KF_IMPLICIT_ARGS \| KF_ACQUIRE)
10980	BTF_ID_FLAGS(func, scx_bpf_get_idle_smtmask_node, KF_IMPLICIT_ARGS \| KF_ACQUIRE)
10981	BTF_ID_FLAGS(func, scx_bpf_put_idle_cpumask, KF_RELEASE)
10982	BTF_ID_FLAGS(func, scx_bpf_test_and_clear_cpu_idle, KF_IMPLICIT_ARGS)
10983	BTF_ID_FLAGS(func, scx_bpf_pick_idle_cpu, KF_IMPLICIT_ARGS \| KF_RCU)
10984	BTF_ID_FLAGS(func, scx_bpf_pick_idle_cpu_node, KF_IMPLICIT_ARGS \| KF_RCU)
10985	BTF_ID_FLAGS(func, scx_bpf_pick_any_cpu, KF_IMPLICIT_ARGS \| KF_RCU)
10986	BTF_ID_FLAGS(func, scx_bpf_pick_any_cpu_node, KF_IMPLICIT_ARGS \| KF_RCU)
10987	BTF_KFUNCS_END(scx_kfunc_ids_cpu_only)
10988
10989	/*
10990	* Per-op kfunc allow flags. Each bit corresponds to a context-sensitive kfunc
10991	* group; an op may permit zero or more groups, with the union expressed in
10992	* scx_kf_allow_flags[]. The verifier-time filter (scx_kfunc_context_filter())
10993	* consults this table to decide whether a context-sensitive kfunc is callable
10994	* from a given SCX op.
10995	*/
10996	enum scx_kf_allow_flags {
10997	SCX_KF_ALLOW_UNLOCKED = 1 << 0,
10998	SCX_KF_ALLOW_INIT_CIDS = 1 << 1,
10999	SCX_KF_ALLOW_CPU_RELEASE = 1 << 2,
11000	SCX_KF_ALLOW_DISPATCH = 1 << 3,
11001	SCX_KF_ALLOW_ENQUEUE = 1 << 4,
11002	SCX_KF_ALLOW_SELECT_CPU = 1 << 5,
11003	};
11004
11005	/*
11006	* Map each SCX op to the union of kfunc groups it permits, indexed by
11007	* SCX_OP_IDX(op). Ops not listed only permit kfuncs that are not
11008	* context-sensitive.
11009	*/
11010	static const u32 scx_kf_allow_flags[] = {
11011	[SCX_OP_IDX(select_cpu)] = SCX_KF_ALLOW_SELECT_CPU \| SCX_KF_ALLOW_ENQUEUE,
11012	[SCX_OP_IDX(enqueue)] = SCX_KF_ALLOW_SELECT_CPU \| SCX_KF_ALLOW_ENQUEUE,
11013	[SCX_OP_IDX(dispatch)] = SCX_KF_ALLOW_ENQUEUE \| SCX_KF_ALLOW_DISPATCH,
11014	[SCX_OP_IDX(cpu_release)] = SCX_KF_ALLOW_CPU_RELEASE,
11015	[SCX_OP_IDX(init_task)] = SCX_KF_ALLOW_UNLOCKED,
11016	[SCX_OP_IDX(dump)] = SCX_KF_ALLOW_UNLOCKED,
11017	#ifdef CONFIG_EXT_GROUP_SCHED
11018	[SCX_OP_IDX(cgroup_init)] = SCX_KF_ALLOW_UNLOCKED,
11019	[SCX_OP_IDX(cgroup_exit)] = SCX_KF_ALLOW_UNLOCKED,
11020	[SCX_OP_IDX(cgroup_prep_move)] = SCX_KF_ALLOW_UNLOCKED,
11021	[SCX_OP_IDX(cgroup_cancel_move)] = SCX_KF_ALLOW_UNLOCKED,
11022	[SCX_OP_IDX(cgroup_set_weight)] = SCX_KF_ALLOW_UNLOCKED,
11023	[SCX_OP_IDX(cgroup_set_bandwidth)] = SCX_KF_ALLOW_UNLOCKED,
11024	[SCX_OP_IDX(cgroup_set_idle)] = SCX_KF_ALLOW_UNLOCKED,
11025	#endif /* CONFIG_EXT_GROUP_SCHED */
11026	[SCX_OP_IDX(sub_attach)] = SCX_KF_ALLOW_UNLOCKED,
11027	[SCX_OP_IDX(sub_detach)] = SCX_KF_ALLOW_UNLOCKED,
11028	[SCX_OP_IDX(sub_ecaps_updated)] = SCX_KF_ALLOW_ENQUEUE \| SCX_KF_ALLOW_DISPATCH,
11029	[SCX_OP_IDX(cpu_online)] = SCX_KF_ALLOW_UNLOCKED,
11030	[SCX_OP_IDX(cpu_offline)] = SCX_KF_ALLOW_UNLOCKED,
11031	[SCX_OP_IDX(init_cids)] = SCX_KF_ALLOW_UNLOCKED \| SCX_KF_ALLOW_INIT_CIDS,
11032	[SCX_OP_IDX(init)] = SCX_KF_ALLOW_UNLOCKED,
11033	[SCX_OP_IDX(exit)] = SCX_KF_ALLOW_UNLOCKED,
11034	};
11035
11036	/*
11037	* Verifier-time filter for SCX kfuncs. Registered via the .filter field on
11038	* each per-group btf_kfunc_id_set. The BPF core invokes this for every kfunc
11039	* call in the registered hook (BPF_PROG_TYPE_STRUCT_OPS or
11040	* BPF_PROG_TYPE_SYSCALL), regardless of which set originally introduced the
11041	* kfunc - so the filter must short-circuit on kfuncs it doesn't govern by
11042	* falling through to "allow" when none of the SCX sets contain the kfunc.
11043	*/
11044	int scx_kfunc_context_filter(const struct bpf_prog *prog, u32 kfunc_id)
11045	{
11046	bool in_unlocked = btf_id_set8_contains(&scx_kfunc_ids_unlocked, kfunc_id);
11047	bool in_init_cids = btf_id_set8_contains(&scx_kfunc_ids_init_cids, kfunc_id);
11048	bool in_select_cpu = btf_id_set8_contains(&scx_kfunc_ids_select_cpu, kfunc_id);
11049	bool in_enqueue = btf_id_set8_contains(&scx_kfunc_ids_enqueue_dispatch, kfunc_id);
11050	bool in_dispatch = btf_id_set8_contains(&scx_kfunc_ids_dispatch, kfunc_id);
11051	bool in_cpu_release = btf_id_set8_contains(&scx_kfunc_ids_cpu_release, kfunc_id);
11052	bool in_idle = btf_id_set8_contains(&scx_kfunc_ids_idle, kfunc_id);
11053	bool in_any = btf_id_set8_contains(&scx_kfunc_ids_any, kfunc_id);
11054	bool in_cpu_only = btf_id_set8_contains(&scx_kfunc_ids_cpu_only, kfunc_id);
11055	bool in_cid = btf_id_set8_contains(&scx_kfunc_ids_cid, kfunc_id);
11056	u32 moff, flags;
11057
11058	/* Not an SCX kfunc - allow. */
11059	if (!(in_unlocked \|\| in_init_cids \|\| in_select_cpu \|\| in_enqueue \|\| in_dispatch \|\|
11060	in_cpu_release \|\| in_idle \|\| in_any \|\| in_cid))
11061	return 0;
11062
11063	/* SYSCALL progs (e.g. BPF test_run()) may call unlocked and select_cpu kfuncs. */
11064	if (prog->type == BPF_PROG_TYPE_SYSCALL)
11065	return (in_unlocked \|\| in_select_cpu \|\| in_idle \|\| in_any \|\| in_cid) ? 0 : -EACCES;
11066
11067	if (prog->type != BPF_PROG_TYPE_STRUCT_OPS)
11068	return (in_any \|\| in_idle \|\| in_cid) ? 0 : -EACCES;
11069
11070	/*
11071	* add_subprog_and_kfunc() collects all kfunc calls, including dead code
11072	* guarded by bpf_ksym_exists(), before check_attach_btf_id() sets
11073	* prog->aux->st_ops. Allow all kfuncs when st_ops is not yet set;
11074	* do_check_main() re-runs the filter with st_ops set and enforces the
11075	* actual restrictions.
11076	*/
11077	if (!prog->aux->st_ops)
11078	return 0;
11079
11080	/*
11081	* Non-SCX struct_ops: SCX kfuncs are not permitted.
11082	*
11083	* Both bpf_sched_ext_ops (cpu-form) and bpf_sched_ext_ops_cid
11084	* (cid-form) are valid SCX struct_ops. Member offsets match between
11085	* the two (verified by BUILD_BUG_ON in scx_init()), so the shared
11086	* scx_kf_allow_flags[] table indexed by SCX_MOFF_IDX(moff) applies to
11087	* both.
11088	*/
11089	if (prog->aux->st_ops != &bpf_sched_ext_ops &&
11090	prog->aux->st_ops != &bpf_sched_ext_ops_cid)
11091	return -EACCES;
11092
11093	/*
11094	* cid-form schedulers must use cid/cmask kfuncs. cid and cpu are both
11095	* small s32s and trivially confused, so cpu-only kfuncs are rejected at
11096	* load time. The reverse (cpu-form calling cid-form kfuncs) is
11097	* intentionally permissive to ease gradual cpumask -> cid migration.
11098	*/
11099	if (prog->aux->st_ops == &bpf_sched_ext_ops_cid && in_cpu_only)
11100	return -EACCES;
11101
11102	/* SCX struct_ops: check the per-op allow list. */
11103	if (in_any \|\| in_idle \|\| in_cid)
11104	return 0;
11105
11106	moff = prog->aux->attach_st_ops_member_off;
11107	flags = scx_kf_allow_flags[SCX_MOFF_IDX(moff)];
11108
11109	if ((flags & SCX_KF_ALLOW_UNLOCKED) && in_unlocked)
11110	return 0;
11111	if ((flags & SCX_KF_ALLOW_INIT_CIDS) && in_init_cids)
11112	return 0;
11113	if ((flags & SCX_KF_ALLOW_CPU_RELEASE) && in_cpu_release)
11114	return 0;
11115	if ((flags & SCX_KF_ALLOW_DISPATCH) && in_dispatch)
11116	return 0;
11117	if ((flags & SCX_KF_ALLOW_ENQUEUE) && in_enqueue)
11118	return 0;
11119	if ((flags & SCX_KF_ALLOW_SELECT_CPU) && in_select_cpu)
11120	return 0;
11121
11122	return -EACCES;
11123	}
11124
11125	static int __init scx_init(void)
11126	{
11127	int ret;
11128
11129	/*
11130	* sched_ext_ops_cid mirrors sched_ext_ops up to and including @priv.
11131	* Both bpf_scx_init_member() and bpf_scx_check_member() use offsets
11132	* from struct sched_ext_ops; sched_ext_ops_cid relies on those offsets
11133	* matching for the shared fields. Catch any drift at boot.
11134	*/
11135	#define CID_OFFSET_MATCH(cpu_field, cid_field) \
11136	BUILD_BUG_ON(offsetof(struct sched_ext_ops, cpu_field) != \
11137	offsetof(struct sched_ext_ops_cid, cid_field))
11138	/* data fields used by bpf_scx_init_member() */
11139	CID_OFFSET_MATCH(dispatch_max_batch, dispatch_max_batch);
11140	CID_OFFSET_MATCH(flags, flags);
11141	CID_OFFSET_MATCH(name, name);
11142	CID_OFFSET_MATCH(timeout_ms, timeout_ms);
11143	CID_OFFSET_MATCH(exit_dump_len, exit_dump_len);
11144	CID_OFFSET_MATCH(hotplug_seq, hotplug_seq);
11145	CID_OFFSET_MATCH(cid_shard_size, cid_shard_size);
11146	CID_OFFSET_MATCH(rescue_bandwidth_ppt, rescue_bandwidth_ppt);
11147	CID_OFFSET_MATCH(rescue_quantum_us, rescue_quantum_us);
11148	CID_OFFSET_MATCH(sub_cgroup_id, sub_cgroup_id);
11149	/* shared callbacks: the union view requires byte-for-byte offset match */
11150	CID_OFFSET_MATCH(enqueue, enqueue);
11151	CID_OFFSET_MATCH(dequeue, dequeue);
11152	CID_OFFSET_MATCH(dispatch, dispatch);
11153	CID_OFFSET_MATCH(tick, tick);
11154	CID_OFFSET_MATCH(runnable, runnable);
11155	CID_OFFSET_MATCH(running, running);
11156	CID_OFFSET_MATCH(stopping, stopping);
11157	CID_OFFSET_MATCH(quiescent, quiescent);
11158	CID_OFFSET_MATCH(yield, yield);
11159	CID_OFFSET_MATCH(core_sched_before, core_sched_before);
11160	CID_OFFSET_MATCH(set_weight, set_weight);
11161	CID_OFFSET_MATCH(update_idle, update_idle);
11162	CID_OFFSET_MATCH(init_task, init_task);
11163	CID_OFFSET_MATCH(exit_task, exit_task);
11164	CID_OFFSET_MATCH(enable, enable);
11165	CID_OFFSET_MATCH(disable, disable);
11166	CID_OFFSET_MATCH(dump, dump);
11167	CID_OFFSET_MATCH(dump_task, dump_task);
11168	CID_OFFSET_MATCH(sub_attach, sub_attach);
11169	CID_OFFSET_MATCH(sub_detach, sub_detach);
11170	CID_OFFSET_MATCH(sub_caps_updated, sub_caps_updated);
11171	CID_OFFSET_MATCH(sub_ecaps_updated, sub_ecaps_updated);
11172	CID_OFFSET_MATCH(init_cids, init_cids);
11173	CID_OFFSET_MATCH(init, init);
11174	CID_OFFSET_MATCH(exit, exit);
11175	/* renamed callbacks must occupy the same slot as their cpu-form sibling */
11176	CID_OFFSET_MATCH(select_cpu, select_cid);
11177	CID_OFFSET_MATCH(set_cpumask, set_cmask);
11178	CID_OFFSET_MATCH(cpu_online, cid_online);
11179	CID_OFFSET_MATCH(cpu_offline, cid_offline);
11180	CID_OFFSET_MATCH(dump_cpu, dump_cid);
11181	#ifdef CONFIG_EXT_GROUP_SCHED
11182	CID_OFFSET_MATCH(cgroup_init, cpuctl_init);
11183	CID_OFFSET_MATCH(cgroup_exit, cpuctl_exit);
11184	CID_OFFSET_MATCH(cgroup_prep_move, cpuctl_prep_move);
11185	CID_OFFSET_MATCH(cgroup_move, cpuctl_move);
11186	CID_OFFSET_MATCH(cgroup_cancel_move, cpuctl_cancel_move);
11187	CID_OFFSET_MATCH(cgroup_set_weight, cpuctl_set_weight);
11188	CID_OFFSET_MATCH(cgroup_set_bandwidth, cpuctl_set_bandwidth);
11189	CID_OFFSET_MATCH(cgroup_set_idle, cpuctl_set_idle);
11190	#endif
11191	/* @priv tail must align since both share the same data block */
11192	CID_OFFSET_MATCH(priv, priv);
11193	/*
11194	* cid-form must end exactly at @priv - scx_validate_ops() skips
11195	* cpu_acquire/cpu_release for cid-form because reading those fields
11196	* past the BPF allocation would be UB.
11197	*/
11198	BUILD_BUG_ON(offsetof(struct sched_ext_ops_cid, __end) !=
11199	offsetofend(struct sched_ext_ops, priv));
11200	#undef CID_OFFSET_MATCH
11201
11202	/*
11203	* kfunc registration can't be done from init_sched_ext_class() as
11204	* register_btf_kfunc_id_set() needs most of the system to be up.
11205	*
11206	* Some kfuncs are context-sensitive and can only be called from
11207	* specific SCX ops. They are grouped into per-context BTF sets, each
11208	* registered with scx_kfunc_context_filter as its .filter callback. The
11209	* BPF core dedups identical filter pointers per hook
11210	* (btf_populate_kfunc_set()), so the filter is invoked exactly once per
11211	* kfunc lookup; it consults scx_kf_allow_flags[] to enforce per-op
11212	* restrictions at verify time.
11213	*/
11214	if ((ret = register_btf_kfunc_id_set(BPF_PROG_TYPE_STRUCT_OPS,
11215	&scx_kfunc_set_enqueue_dispatch)) \|\|
11216	(ret = register_btf_kfunc_id_set(BPF_PROG_TYPE_STRUCT_OPS,
11217	&scx_kfunc_set_dispatch)) \|\|
11218	(ret = register_btf_kfunc_id_set(BPF_PROG_TYPE_STRUCT_OPS,
11219	&scx_kfunc_set_cpu_release)) \|\|
11220	(ret = register_btf_kfunc_id_set(BPF_PROG_TYPE_STRUCT_OPS,
11221	&scx_kfunc_set_unlocked)) \|\|
11222	(ret = register_btf_kfunc_id_set(BPF_PROG_TYPE_SYSCALL,
11223	&scx_kfunc_set_unlocked)) \|\|
11224	(ret = register_btf_kfunc_id_set(BPF_PROG_TYPE_STRUCT_OPS,
11225	&scx_kfunc_set_any)) \|\|
11226	(ret = register_btf_kfunc_id_set(BPF_PROG_TYPE_TRACING,
11227	&scx_kfunc_set_any)) \|\|
11228	(ret = register_btf_kfunc_id_set(BPF_PROG_TYPE_SYSCALL,
11229	&scx_kfunc_set_any))) {
11230	pr_err("sched_ext: Failed to register kfunc sets (%d)\n", ret);
11231	return ret;
11232	}
11233
11234	ret = scx_idle_init();
11235	if (ret) {
11236	pr_err("sched_ext: Failed to initialize idle tracking (%d)\n", ret);
11237	return ret;
11238	}
11239
11240	ret = scx_cid_kfunc_init();
11241	if (ret) {
11242	pr_err("sched_ext: Failed to register cid kfuncs (%d)\n", ret);
11243	return ret;
11244	}
11245
11246	ret = register_bpf_struct_ops(&bpf_sched_ext_ops, sched_ext_ops);
11247	if (ret) {
11248	pr_err("sched_ext: Failed to register struct_ops (%d)\n", ret);
11249	return ret;
11250	}
11251
11252	ret = register_bpf_struct_ops(&bpf_sched_ext_ops_cid, sched_ext_ops_cid);
11253	if (ret) {
11254	pr_err("sched_ext: Failed to register cid struct_ops (%d)\n", ret);
11255	return ret;
11256	}
11257
11258	ret = register_pm_notifier(&scx_pm_notifier);
11259	if (ret) {
11260	pr_err("sched_ext: Failed to register PM notifier (%d)\n", ret);
11261	return ret;
11262	}
11263
11264	scx_kset = kset_create_and_add("sched_ext", &scx_uevent_ops, kernel_kobj);
11265	if (!scx_kset) {
11266	pr_err("sched_ext: Failed to create /sys/kernel/sched_ext\n");
11267	return -ENOMEM;
11268	}
11269
11270	ret = sysfs_create_group(&scx_kset->kobj, &scx_global_attr_group);
11271	if (ret < 0) {
11272	pr_err("sched_ext: Failed to add global attributes\n");
11273	return ret;
11274	}
11275
11276	return 0;
11277	}
11278	__initcall(scx_init);
11279
11280	/*
11281	* Compatibility markers for userspace. Existence of a marker function
11282	* represents that the kernel supports that sched-ext feature.
11283	*/
11284
11285	/*
11286	* scx_compat_marker_cgroup_set_bandwidth_may_sleep: advertises that
11287	* ops.cgroup_set_bandwidth() may be implemented as a sleepable callback.
11288	*/
11289	#ifdef CONFIG_EXT_GROUP_SCHED
11290	DEFINE_SCX_COMPAT_MARKER(cgroup_set_bandwidth_may_sleep);
11291	#endif /* CONFIG_EXT_GROUP_SCHED */