linux - kernel/sched/rt.c

2946 lines · 69.6 KiB

1	// SPDX-License-Identifier: GPL-2.0
2	/*
3	* Real-Time Scheduling Class (mapped to the SCHED_FIFO and SCHED_RR
4	* policies)
5	*/
6
7	#include "sched.h"
8	#include "pelt.h"
9
10	int sched_rr_timeslice = RR_TIMESLICE;
11	/* More than 4 hours if BW_SHIFT equals 20. */
12	static const u64 max_rt_runtime = MAX_BW;
13
14	/*
15	* period over which we measure -rt task CPU usage in us.
16	* default: 1s
17	*/
18	int sysctl_sched_rt_period = 1000000;
19
20	/*
21	* part of the period that we allow rt tasks to run in us.
22	* default: 1s
23	*/
24	int sysctl_sched_rt_runtime = 1000000;
25
26	#ifdef CONFIG_SYSCTL
27	static int sysctl_sched_rr_timeslice = (MSEC_PER_SEC * RR_TIMESLICE) / HZ;
28	static int sched_rt_handler(const struct ctl_table table, int write, void buffer,
29	size_t lenp, loff_t ppos);
30	static int sched_rr_handler(const struct ctl_table table, int write, void buffer,
31	size_t lenp, loff_t ppos);
32	static const struct ctl_table sched_rt_sysctls[] = {
33	{
34	.procname = "sched_rt_period_us",
35	.data = &sysctl_sched_rt_period,
36	.maxlen = sizeof(int),
37	.mode = 0644,
38	.proc_handler = sched_rt_handler,
39	.extra1 = SYSCTL_ONE,
40	.extra2 = SYSCTL_INT_MAX,
41	},
42	{
43	.procname = "sched_rt_runtime_us",
44	.data = &sysctl_sched_rt_runtime,
45	.maxlen = sizeof(int),
46	.mode = 0644,
47	.proc_handler = sched_rt_handler,
48	.extra1 = SYSCTL_NEG_ONE,
49	.extra2 = (void *)&sysctl_sched_rt_period,
50	},
51	{
52	.procname = "sched_rr_timeslice_ms",
53	.data = &sysctl_sched_rr_timeslice,
54	.maxlen = sizeof(int),
55	.mode = 0644,
56	.proc_handler = sched_rr_handler,
57	},
58	};
59
60	static int __init sched_rt_sysctl_init(void)
61	{
62	register_sysctl_init("kernel", sched_rt_sysctls);
63	return 0;
64	}
65	late_initcall(sched_rt_sysctl_init);
66	#endif /* CONFIG_SYSCTL */
67
68	void init_rt_rq(struct rt_rq *rt_rq)
69	{
70	struct rt_prio_array *array;
71	int i;
72
73	array = &rt_rq->active;
74	for (i = 0; i < MAX_RT_PRIO; i++) {
75	INIT_LIST_HEAD(array->queue + i);
76	__clear_bit(i, array->bitmap);
77	}
78	/* delimiter for bitsearch: */
79	__set_bit(MAX_RT_PRIO, array->bitmap);
80
81	rt_rq->highest_prio.curr = MAX_RT_PRIO-1;
82	rt_rq->highest_prio.next = MAX_RT_PRIO-1;
83	rt_rq->overloaded = 0;
84	plist_head_init(&rt_rq->pushable_tasks);
85	/* We start is dequeued state, because no RT tasks are queued */
86	rt_rq->rt_queued = 0;
87
88	#ifdef CONFIG_RT_GROUP_SCHED
89	rt_rq->rt_time = 0;
90	rt_rq->rt_throttled = 0;
91	rt_rq->rt_runtime = 0;
92	raw_spin_lock_init(&rt_rq->rt_runtime_lock);
93	rt_rq->tg = &root_task_group;
94	#endif
95	}
96
97	#ifdef CONFIG_RT_GROUP_SCHED
98
99	static int do_sched_rt_period_timer(struct rt_bandwidth *rt_b, int overrun);
100
101	static enum hrtimer_restart sched_rt_period_timer(struct hrtimer *timer)
102	{
103	struct rt_bandwidth *rt_b =
104	container_of(timer, struct rt_bandwidth, rt_period_timer);
105	int idle = 0;
106	int overrun;
107
108	raw_spin_lock(&rt_b->rt_runtime_lock);
109	for (;;) {
110	overrun = hrtimer_forward_now(timer, rt_b->rt_period);
111	if (!overrun)
112	break;
113
114	raw_spin_unlock(&rt_b->rt_runtime_lock);
115	idle = do_sched_rt_period_timer(rt_b, overrun);
116	raw_spin_lock(&rt_b->rt_runtime_lock);
117	}
118	if (idle)
119	rt_b->rt_period_active = 0;
120	raw_spin_unlock(&rt_b->rt_runtime_lock);
121
122	return idle ? HRTIMER_NORESTART : HRTIMER_RESTART;
123	}
124
125	void init_rt_bandwidth(struct rt_bandwidth *rt_b, u64 period, u64 runtime)
126	{
127	rt_b->rt_period = ns_to_ktime(period);
128	rt_b->rt_runtime = runtime;
129
130	raw_spin_lock_init(&rt_b->rt_runtime_lock);
131
132	hrtimer_setup(&rt_b->rt_period_timer, sched_rt_period_timer, CLOCK_MONOTONIC,
133	HRTIMER_MODE_REL_HARD);
134	}
135
136	static inline void do_start_rt_bandwidth(struct rt_bandwidth *rt_b)
137	{
138	raw_spin_lock(&rt_b->rt_runtime_lock);
139	if (!rt_b->rt_period_active) {
140	rt_b->rt_period_active = 1;
141	/*
142	* SCHED_DEADLINE updates the bandwidth, as a run away
143	* RT task with a DL task could hog a CPU. But DL does
144	* not reset the period. If a deadline task was running
145	* without an RT task running, it can cause RT tasks to
146	* throttle when they start up. Kick the timer right away
147	* to update the period.
148	*/
149	hrtimer_forward_now(&rt_b->rt_period_timer, ns_to_ktime(0));
150	hrtimer_start_expires(&rt_b->rt_period_timer,
151	HRTIMER_MODE_ABS_PINNED_HARD);
152	}
153	raw_spin_unlock(&rt_b->rt_runtime_lock);
154	}
155
156	static void start_rt_bandwidth(struct rt_bandwidth *rt_b)
157	{
158	if (!rt_bandwidth_enabled() \|\| rt_b->rt_runtime == RUNTIME_INF)
159	return;
160
161	do_start_rt_bandwidth(rt_b);
162	}
163
164	static void destroy_rt_bandwidth(struct rt_bandwidth *rt_b)
165	{
166	hrtimer_cancel(&rt_b->rt_period_timer);
167	}
168
169	#define rt_entity_is_task(rt_se) (!(rt_se)->my_q)
170
171	static inline struct task_struct rt_task_of(struct sched_rt_entity rt_se)
172	{
173	WARN_ON_ONCE(!rt_entity_is_task(rt_se));
174
175	return container_of(rt_se, struct task_struct, rt);
176	}
177
178	static inline struct rq rq_of_rt_rq(struct rt_rq rt_rq)
179	{
180	/* Cannot fold with non-CONFIG_RT_GROUP_SCHED version, layout */
181	WARN_ON(!rt_group_sched_enabled() && rt_rq->tg != &root_task_group);
182	return rt_rq->rq;
183	}
184
185	static inline struct rt_rq rt_rq_of_se(struct sched_rt_entity rt_se)
186	{
187	WARN_ON(!rt_group_sched_enabled() && rt_se->rt_rq->tg != &root_task_group);
188	return rt_se->rt_rq;
189	}
190
191	static inline struct rq rq_of_rt_se(struct sched_rt_entity rt_se)
192	{
193	struct rt_rq *rt_rq = rt_se->rt_rq;
194
195	WARN_ON(!rt_group_sched_enabled() && rt_rq->tg != &root_task_group);
196	return rt_rq->rq;
197	}
198
199	void unregister_rt_sched_group(struct task_group *tg)
200	{
201	if (!rt_group_sched_enabled())
202	return;
203
204	if (tg->rt_se)
205	destroy_rt_bandwidth(&tg->rt_bandwidth);
206	}
207
208	void free_rt_sched_group(struct task_group *tg)
209	{
210	int i;
211
212	if (!rt_group_sched_enabled())
213	return;
214
215	for_each_possible_cpu(i) {
216	if (tg->rt_rq)
217	kfree(tg->rt_rq[i]);
218	if (tg->rt_se)
219	kfree(tg->rt_se[i]);
220	}
221
222	kfree(tg->rt_rq);
223	kfree(tg->rt_se);
224	}
225
226	void init_tg_rt_entry(struct task_group tg, struct rt_rq rt_rq,
227	struct sched_rt_entity *rt_se, int cpu,
228	struct sched_rt_entity *parent)
229	{
230	struct rq *rq = cpu_rq(cpu);
231
232	rt_rq->highest_prio.curr = MAX_RT_PRIO-1;
233	rt_rq->rt_nr_boosted = 0;
234	rt_rq->rq = rq;
235	rt_rq->tg = tg;
236
237	tg->rt_rq[cpu] = rt_rq;
238	tg->rt_se[cpu] = rt_se;
239
240	if (!rt_se)
241	return;
242
243	if (!parent)
244	rt_se->rt_rq = &rq->rt;
245	else
246	rt_se->rt_rq = parent->my_q;
247
248	rt_se->my_q = rt_rq;
249	rt_se->parent = parent;
250	INIT_LIST_HEAD(&rt_se->run_list);
251	}
252
253	int alloc_rt_sched_group(struct task_group tg, struct task_group parent)
254	{
255	struct rt_rq *rt_rq;
256	struct sched_rt_entity *rt_se;
257	int i;
258
259	if (!rt_group_sched_enabled())
260	return 1;
261
262	tg->rt_rq = kzalloc_objs(rt_rq, nr_cpu_ids);
263	if (!tg->rt_rq)
264	goto err;
265	tg->rt_se = kzalloc_objs(rt_se, nr_cpu_ids);
266	if (!tg->rt_se)
267	goto err;
268
269	init_rt_bandwidth(&tg->rt_bandwidth, ktime_to_ns(global_rt_period()), 0);
270
271	for_each_possible_cpu(i) {
272	rt_rq = kzalloc_node(sizeof(struct rt_rq),
273	GFP_KERNEL, cpu_to_node(i));
274	if (!rt_rq)
275	goto err;
276
277	rt_se = kzalloc_node(sizeof(struct sched_rt_entity),
278	GFP_KERNEL, cpu_to_node(i));
279	if (!rt_se)
280	goto err_free_rq;
281
282	init_rt_rq(rt_rq);
283	rt_rq->rt_runtime = tg->rt_bandwidth.rt_runtime;
284	init_tg_rt_entry(tg, rt_rq, rt_se, i, parent->rt_se[i]);
285	}
286
287	return 1;
288
289	err_free_rq:
290	kfree(rt_rq);
291	err:
292	return 0;
293	}
294
295	#else /* !CONFIG_RT_GROUP_SCHED: */
296
297	#define rt_entity_is_task(rt_se) (1)
298
299	static inline struct task_struct rt_task_of(struct sched_rt_entity rt_se)
300	{
301	return container_of(rt_se, struct task_struct, rt);
302	}
303
304	static inline struct rq rq_of_rt_rq(struct rt_rq rt_rq)
305	{
306	return container_of(rt_rq, struct rq, rt);
307	}
308
309	static inline struct rq rq_of_rt_se(struct sched_rt_entity rt_se)
310	{
311	struct task_struct *p = rt_task_of(rt_se);
312
313	return task_rq(p);
314	}
315
316	static inline struct rt_rq rt_rq_of_se(struct sched_rt_entity rt_se)
317	{
318	struct rq *rq = rq_of_rt_se(rt_se);
319
320	return &rq->rt;
321	}
322
323	void unregister_rt_sched_group(struct task_group *tg) { }
324
325	void free_rt_sched_group(struct task_group *tg) { }
326
327	int alloc_rt_sched_group(struct task_group tg, struct task_group parent)
328	{
329	return 1;
330	}
331	#endif /* !CONFIG_RT_GROUP_SCHED */
332
333	static inline bool need_pull_rt_task(struct rq rq, struct task_struct prev)
334	{
335	/* Try to pull RT tasks here if we lower this rq's prio */
336	return rq->online && rq->rt.highest_prio.curr > prev->prio;
337	}
338
339	static inline int rt_overloaded(struct rq *rq)
340	{
341	return atomic_read(&rq->rd->rto_count);
342	}
343
344	static inline void rt_set_overload(struct rq *rq)
345	{
346	if (!rq->online)
347	return;
348
349	cpumask_set_cpu(rq->cpu, rq->rd->rto_mask);
350	/*
351	* Make sure the mask is visible before we set
352	* the overload count. That is checked to determine
353	* if we should look at the mask. It would be a shame
354	* if we looked at the mask, but the mask was not
355	* updated yet.
356	*
357	* Matched by the barrier in pull_rt_task().
358	*/
359	smp_wmb();
360	atomic_inc(&rq->rd->rto_count);
361	}
362
363	static inline void rt_clear_overload(struct rq *rq)
364	{
365	if (!rq->online)
366	return;
367
368	/* the order here really doesn't matter */
369	atomic_dec(&rq->rd->rto_count);
370	cpumask_clear_cpu(rq->cpu, rq->rd->rto_mask);
371	}
372
373	static inline int has_pushable_tasks(struct rq *rq)
374	{
375	return !plist_head_empty(&rq->rt.pushable_tasks);
376	}
377
378	static DEFINE_PER_CPU(struct balance_callback, rt_push_head);
379	static DEFINE_PER_CPU(struct balance_callback, rt_pull_head);
380
381	static void push_rt_tasks(struct rq *);
382	static void pull_rt_task(struct rq *);
383
384	static inline void rt_queue_push_tasks(struct rq *rq)
385	{
386	if (!has_pushable_tasks(rq))
387	return;
388
389	queue_balance_callback(rq, &per_cpu(rt_push_head, rq->cpu), push_rt_tasks);
390	}
391
392	static inline void rt_queue_pull_task(struct rq *rq)
393	{
394	queue_balance_callback(rq, &per_cpu(rt_pull_head, rq->cpu), pull_rt_task);
395	}
396
397	static void enqueue_pushable_task(struct rq rq, struct task_struct p)
398	{
399	plist_del(&p->pushable_tasks, &rq->rt.pushable_tasks);
400	plist_node_init(&p->pushable_tasks, p->prio);
401	plist_add(&p->pushable_tasks, &rq->rt.pushable_tasks);
402
403	/* Update the highest prio pushable task */
404	if (p->prio < rq->rt.highest_prio.next)
405	rq->rt.highest_prio.next = p->prio;
406
407	if (!rq->rt.overloaded) {
408	rt_set_overload(rq);
409	rq->rt.overloaded = 1;
410	}
411	}
412
413	static void dequeue_pushable_task(struct rq rq, struct task_struct p)
414	{
415	plist_del(&p->pushable_tasks, &rq->rt.pushable_tasks);
416
417	/* Update the new highest prio pushable task */
418	if (has_pushable_tasks(rq)) {
419	p = plist_first_entry(&rq->rt.pushable_tasks,
420	struct task_struct, pushable_tasks);
421	rq->rt.highest_prio.next = p->prio;
422	} else {
423	rq->rt.highest_prio.next = MAX_RT_PRIO-1;
424
425	if (rq->rt.overloaded) {
426	rt_clear_overload(rq);
427	rq->rt.overloaded = 0;
428	}
429	}
430	}
431
432	static void enqueue_top_rt_rq(struct rt_rq *rt_rq);
433	static void dequeue_top_rt_rq(struct rt_rq *rt_rq, unsigned int count);
434
435	static inline int on_rt_rq(struct sched_rt_entity *rt_se)
436	{
437	return rt_se->on_rq;
438	}
439
440	#ifdef CONFIG_UCLAMP_TASK
441	/*
442	* Verify the fitness of task @p to run on @cpu taking into account the uclamp
443	* settings.
444	*
445	* This check is only important for heterogeneous systems where uclamp_min value
446	* is higher than the capacity of a @cpu. For non-heterogeneous system this
447	* function will always return true.
448	*
449	* The function will return true if the capacity of the @cpu is >= the
450	* uclamp_min and false otherwise.
451	*
452	* Note that uclamp_min will be clamped to uclamp_max if uclamp_min
453	* > uclamp_max.
454	*/
455	static inline bool rt_task_fits_capacity(struct task_struct *p, int cpu)
456	{
457	unsigned int min_cap;
458	unsigned int max_cap;
459	unsigned int cpu_cap;
460
461	/* Only heterogeneous systems can benefit from this check */
462	if (!sched_asym_cpucap_active())
463	return true;
464
465	min_cap = uclamp_eff_value(p, UCLAMP_MIN);
466	max_cap = uclamp_eff_value(p, UCLAMP_MAX);
467
468	cpu_cap = arch_scale_cpu_capacity(cpu);
469
470	return cpu_cap >= min(min_cap, max_cap);
471	}
472	#else /* !CONFIG_UCLAMP_TASK: */
473	static inline bool rt_task_fits_capacity(struct task_struct *p, int cpu)
474	{
475	return true;
476	}
477	#endif /* !CONFIG_UCLAMP_TASK */
478
479	#ifdef CONFIG_RT_GROUP_SCHED
480
481	static inline u64 sched_rt_runtime(struct rt_rq *rt_rq)
482	{
483	return rt_rq->rt_runtime;
484	}
485
486	static inline u64 sched_rt_period(struct rt_rq *rt_rq)
487	{
488	return ktime_to_ns(rt_rq->tg->rt_bandwidth.rt_period);
489	}
490
491	typedef struct task_group *rt_rq_iter_t;
492
493	static inline struct task_group next_task_group(struct task_group tg)
494	{
495	if (!rt_group_sched_enabled()) {
496	WARN_ON(tg != &root_task_group);
497	return NULL;
498	}
499
500	do {
501	tg = list_entry_rcu(tg->list.next,
502	typeof(struct task_group), list);
503	} while (&tg->list != &task_groups && task_group_is_autogroup(tg));
504
505	if (&tg->list == &task_groups)
506	tg = NULL;
507
508	return tg;
509	}
510
511	#define for_each_rt_rq(rt_rq, iter, rq) \
512	for (iter = &root_task_group; \
513	iter && (rt_rq = iter->rt_rq[cpu_of(rq)]); \
514	iter = next_task_group(iter))
515
516	#define for_each_sched_rt_entity(rt_se) \
517	for (; rt_se; rt_se = rt_se->parent)
518
519	static inline struct rt_rq group_rt_rq(struct sched_rt_entity rt_se)
520	{
521	return rt_se->my_q;
522	}
523
524	static void enqueue_rt_entity(struct sched_rt_entity *rt_se, unsigned int flags);
525	static void dequeue_rt_entity(struct sched_rt_entity *rt_se, unsigned int flags);
526
527	static void sched_rt_rq_enqueue(struct rt_rq *rt_rq)
528	{
529	struct task_struct *donor = rq_of_rt_rq(rt_rq)->donor;
530	struct rq *rq = rq_of_rt_rq(rt_rq);
531	struct sched_rt_entity *rt_se;
532
533	int cpu = cpu_of(rq);
534
535	rt_se = rt_rq->tg->rt_se[cpu];
536
537	if (rt_rq->rt_nr_running) {
538	if (!rt_se)
539	enqueue_top_rt_rq(rt_rq);
540	else if (!on_rt_rq(rt_se))
541	enqueue_rt_entity(rt_se, 0);
542
543	if (rt_rq->highest_prio.curr < donor->prio)
544	resched_curr(rq);
545	}
546	}
547
548	static void sched_rt_rq_dequeue(struct rt_rq *rt_rq)
549	{
550	struct sched_rt_entity *rt_se;
551	int cpu = cpu_of(rq_of_rt_rq(rt_rq));
552
553	rt_se = rt_rq->tg->rt_se[cpu];
554
555	if (!rt_se) {
556	dequeue_top_rt_rq(rt_rq, rt_rq->rt_nr_running);
557	/* Kick cpufreq (see the comment in kernel/sched/sched.h). */
558	cpufreq_update_util(rq_of_rt_rq(rt_rq), 0);
559	}
560	else if (on_rt_rq(rt_se))
561	dequeue_rt_entity(rt_se, 0);
562	}
563
564	static inline int rt_rq_throttled(struct rt_rq *rt_rq)
565	{
566	return rt_rq->rt_throttled && !rt_rq->rt_nr_boosted;
567	}
568
569	static int rt_se_boosted(struct sched_rt_entity *rt_se)
570	{
571	struct rt_rq *rt_rq = group_rt_rq(rt_se);
572	struct task_struct *p;
573
574	if (rt_rq)
575	return !!rt_rq->rt_nr_boosted;
576
577	p = rt_task_of(rt_se);
578	return p->prio != p->normal_prio;
579	}
580
581	static inline const struct cpumask *sched_rt_period_mask(void)
582	{
583	return this_rq()->rd->span;
584	}
585
586	static inline
587	struct rt_rq sched_rt_period_rt_rq(struct rt_bandwidth rt_b, int cpu)
588	{
589	return container_of(rt_b, struct task_group, rt_bandwidth)->rt_rq[cpu];
590	}
591
592	static inline struct rt_bandwidth sched_rt_bandwidth(struct rt_rq rt_rq)
593	{
594	return &rt_rq->tg->rt_bandwidth;
595	}
596
597	bool sched_rt_bandwidth_account(struct rt_rq *rt_rq)
598	{
599	struct rt_bandwidth *rt_b = sched_rt_bandwidth(rt_rq);
600
601	return (hrtimer_active(&rt_b->rt_period_timer) \|\|
602	rt_rq->rt_time < rt_b->rt_runtime);
603	}
604
605	/*
606	* We ran out of runtime, see if we can borrow some from our neighbours.
607	*/
608	static void do_balance_runtime(struct rt_rq *rt_rq)
609	{
610	struct rt_bandwidth *rt_b = sched_rt_bandwidth(rt_rq);
611	struct root_domain *rd = rq_of_rt_rq(rt_rq)->rd;
612	int i, weight;
613	u64 rt_period;
614
615	weight = cpumask_weight(rd->span);
616
617	raw_spin_lock(&rt_b->rt_runtime_lock);
618	rt_period = ktime_to_ns(rt_b->rt_period);
619	for_each_cpu(i, rd->span) {
620	struct rt_rq *iter = sched_rt_period_rt_rq(rt_b, i);
621	s64 diff;
622
623	if (iter == rt_rq)
624	continue;
625
626	raw_spin_lock(&iter->rt_runtime_lock);
627	/*
628	* Either all rqs have inf runtime and there's nothing to steal
629	* or __disable_runtime() below sets a specific rq to inf to
630	* indicate its been disabled and disallow stealing.
631	*/
632	if (iter->rt_runtime == RUNTIME_INF)
633	goto next;
634
635	/*
636	* From runqueues with spare time, take 1/n part of their
637	* spare time, but no more than our period.
638	*/
639	diff = iter->rt_runtime - iter->rt_time;
640	if (diff > 0) {
641	diff = div_u64((u64)diff, weight);
642	if (rt_rq->rt_runtime + diff > rt_period)
643	diff = rt_period - rt_rq->rt_runtime;
644	iter->rt_runtime -= diff;
645	rt_rq->rt_runtime += diff;
646	if (rt_rq->rt_runtime == rt_period) {
647	raw_spin_unlock(&iter->rt_runtime_lock);
648	break;
649	}
650	}
651	next:
652	raw_spin_unlock(&iter->rt_runtime_lock);
653	}
654	raw_spin_unlock(&rt_b->rt_runtime_lock);
655	}
656
657	/*
658	* Ensure this RQ takes back all the runtime it lend to its neighbours.
659	*/
660	static void __disable_runtime(struct rq *rq)
661	{
662	struct root_domain *rd = rq->rd;
663	rt_rq_iter_t iter;
664	struct rt_rq *rt_rq;
665
666	if (unlikely(!scheduler_running))
667	return;
668
669	for_each_rt_rq(rt_rq, iter, rq) {
670	struct rt_bandwidth *rt_b = sched_rt_bandwidth(rt_rq);
671	s64 want;
672	int i;
673
674	raw_spin_lock(&rt_b->rt_runtime_lock);
675	raw_spin_lock(&rt_rq->rt_runtime_lock);
676	/*
677	* Either we're all inf and nobody needs to borrow, or we're
678	* already disabled and thus have nothing to do, or we have
679	* exactly the right amount of runtime to take out.
680	*/
681	if (rt_rq->rt_runtime == RUNTIME_INF \|\|
682	rt_rq->rt_runtime == rt_b->rt_runtime)
683	goto balanced;
684	raw_spin_unlock(&rt_rq->rt_runtime_lock);
685
686	/*
687	* Calculate the difference between what we started out with
688	* and what we current have, that's the amount of runtime
689	* we lend and now have to reclaim.
690	*/
691	want = rt_b->rt_runtime - rt_rq->rt_runtime;
692
693	/*
694	* Greedy reclaim, take back as much as we can.
695	*/
696	for_each_cpu(i, rd->span) {
697	struct rt_rq *iter = sched_rt_period_rt_rq(rt_b, i);
698	s64 diff;
699
700	/*
701	* Can't reclaim from ourselves or disabled runqueues.
702	*/
703	if (iter == rt_rq \|\| iter->rt_runtime == RUNTIME_INF)
704	continue;
705
706	raw_spin_lock(&iter->rt_runtime_lock);
707	if (want > 0) {
708	diff = min_t(s64, iter->rt_runtime, want);
709	iter->rt_runtime -= diff;
710	want -= diff;
711	} else {
712	iter->rt_runtime -= want;
713	want -= want;
714	}
715	raw_spin_unlock(&iter->rt_runtime_lock);
716
717	if (!want)
718	break;
719	}
720
721	raw_spin_lock(&rt_rq->rt_runtime_lock);
722	/*
723	* We cannot be left wanting - that would mean some runtime
724	* leaked out of the system.
725	*/
726	WARN_ON_ONCE(want);
727	balanced:
728	/*
729	* Disable all the borrow logic by pretending we have inf
730	* runtime - in which case borrowing doesn't make sense.
731	*/
732	rt_rq->rt_runtime = RUNTIME_INF;
733	rt_rq->rt_throttled = 0;
734	raw_spin_unlock(&rt_rq->rt_runtime_lock);
735	raw_spin_unlock(&rt_b->rt_runtime_lock);
736
737	/* Make rt_rq available for pick_next_task() */
738	sched_rt_rq_enqueue(rt_rq);
739	}
740	}
741
742	static void __enable_runtime(struct rq *rq)
743	{
744	rt_rq_iter_t iter;
745	struct rt_rq *rt_rq;
746
747	if (unlikely(!scheduler_running))
748	return;
749
750	/*
751	* Reset each runqueue's bandwidth settings
752	*/
753	for_each_rt_rq(rt_rq, iter, rq) {
754	struct rt_bandwidth *rt_b = sched_rt_bandwidth(rt_rq);
755
756	raw_spin_lock(&rt_b->rt_runtime_lock);
757	raw_spin_lock(&rt_rq->rt_runtime_lock);
758	rt_rq->rt_runtime = rt_b->rt_runtime;
759	rt_rq->rt_time = 0;
760	rt_rq->rt_throttled = 0;
761	raw_spin_unlock(&rt_rq->rt_runtime_lock);
762	raw_spin_unlock(&rt_b->rt_runtime_lock);
763	}
764	}
765
766	static void balance_runtime(struct rt_rq *rt_rq)
767	{
768	if (!sched_feat(RT_RUNTIME_SHARE))
769	return;
770
771	if (rt_rq->rt_time > rt_rq->rt_runtime) {
772	raw_spin_unlock(&rt_rq->rt_runtime_lock);
773	do_balance_runtime(rt_rq);
774	raw_spin_lock(&rt_rq->rt_runtime_lock);
775	}
776	}
777
778	static int do_sched_rt_period_timer(struct rt_bandwidth *rt_b, int overrun)
779	{
780	int i, idle = 1, throttled = 0;
781	const struct cpumask *span;
782
783	span = sched_rt_period_mask();
784
785	/*
786	* FIXME: isolated CPUs should really leave the root task group,
787	* whether they are isolcpus or were isolated via cpusets, lest
788	* the timer run on a CPU which does not service all runqueues,
789	* potentially leaving other CPUs indefinitely throttled. If
790	* isolation is really required, the user will turn the throttle
791	* off to kill the perturbations it causes anyway. Meanwhile,
792	* this maintains functionality for boot and/or troubleshooting.
793	*/
794	if (rt_b == &root_task_group.rt_bandwidth)
795	span = cpu_online_mask;
796
797	for_each_cpu(i, span) {
798	int enqueue = 0;
799	struct rt_rq *rt_rq = sched_rt_period_rt_rq(rt_b, i);
800	struct rq *rq = rq_of_rt_rq(rt_rq);
801	struct rq_flags rf;
802	int skip;
803
804	/*
805	* When span == cpu_online_mask, taking each rq->lock
806	* can be time-consuming. Try to avoid it when possible.
807	*/
808	raw_spin_lock(&rt_rq->rt_runtime_lock);
809	if (!sched_feat(RT_RUNTIME_SHARE) && rt_rq->rt_runtime != RUNTIME_INF)
810	rt_rq->rt_runtime = rt_b->rt_runtime;
811	skip = !rt_rq->rt_time && !rt_rq->rt_nr_running;
812	raw_spin_unlock(&rt_rq->rt_runtime_lock);
813	if (skip)
814	continue;
815
816	rq_lock(rq, &rf);
817	update_rq_clock(rq);
818
819	if (rt_rq->rt_time) {
820	u64 runtime;
821
822	raw_spin_lock(&rt_rq->rt_runtime_lock);
823	if (rt_rq->rt_throttled)
824	balance_runtime(rt_rq);
825	runtime = rt_rq->rt_runtime;
826	rt_rq->rt_time -= min(rt_rq->rt_time, overrun*runtime);
827	if (rt_rq->rt_throttled && rt_rq->rt_time < runtime) {
828	rt_rq->rt_throttled = 0;
829	enqueue = 1;
830
831	/*
832	* When we're idle and a woken (rt) task is
833	* throttled wakeup_preempt() will set
834	* skip_update and the time between the wakeup
835	* and this unthrottle will get accounted as
836	* 'runtime'.
837	*/
838	if (rt_rq->rt_nr_running && rq->curr == rq->idle)
839	rq_clock_cancel_skipupdate(rq);
840	}
841	if (rt_rq->rt_time \|\| rt_rq->rt_nr_running)
842	idle = 0;
843	raw_spin_unlock(&rt_rq->rt_runtime_lock);
844	} else if (rt_rq->rt_nr_running) {
845	idle = 0;
846	if (!rt_rq_throttled(rt_rq))
847	enqueue = 1;
848	}
849	if (rt_rq->rt_throttled)
850	throttled = 1;
851
852	if (enqueue)
853	sched_rt_rq_enqueue(rt_rq);
854	rq_unlock(rq, &rf);
855	}
856
857	if (!throttled && (!rt_bandwidth_enabled() \|\| rt_b->rt_runtime == RUNTIME_INF))
858	return 1;
859
860	return idle;
861	}
862
863	static int sched_rt_runtime_exceeded(struct rt_rq *rt_rq)
864	{
865	u64 runtime = sched_rt_runtime(rt_rq);
866
867	if (rt_rq->rt_throttled)
868	return rt_rq_throttled(rt_rq);
869
870	if (runtime >= sched_rt_period(rt_rq))
871	return 0;
872
873	balance_runtime(rt_rq);
874	runtime = sched_rt_runtime(rt_rq);
875	if (runtime == RUNTIME_INF)
876	return 0;
877
878	if (rt_rq->rt_time > runtime) {
879	struct rt_bandwidth *rt_b = sched_rt_bandwidth(rt_rq);
880
881	/*
882	* Don't actually throttle groups that have no runtime assigned
883	* but accrue some time due to boosting.
884	*/
885	if (likely(rt_b->rt_runtime)) {
886	rt_rq->rt_throttled = 1;
887	printk_deferred_once("sched: RT throttling activated\n");
888	} else {
889	/*
890	* In case we did anyway, make it go away,
891	* replenishment is a joke, since it will replenish us
892	* with exactly 0 ns.
893	*/
894	rt_rq->rt_time = 0;
895	}
896
897	if (rt_rq_throttled(rt_rq)) {
898	sched_rt_rq_dequeue(rt_rq);
899	return 1;
900	}
901	}
902
903	return 0;
904	}
905
906	#else /* !CONFIG_RT_GROUP_SCHED: */
907
908	typedef struct rt_rq *rt_rq_iter_t;
909
910	#define for_each_rt_rq(rt_rq, iter, rq) \
911	for ((void) iter, rt_rq = &rq->rt; rt_rq; rt_rq = NULL)
912
913	#define for_each_sched_rt_entity(rt_se) \
914	for (; rt_se; rt_se = NULL)
915
916	static inline struct rt_rq group_rt_rq(struct sched_rt_entity rt_se)
917	{
918	return NULL;
919	}
920
921	static inline void sched_rt_rq_enqueue(struct rt_rq *rt_rq)
922	{
923	struct rq *rq = rq_of_rt_rq(rt_rq);
924
925	if (!rt_rq->rt_nr_running)
926	return;
927
928	enqueue_top_rt_rq(rt_rq);
929	resched_curr(rq);
930	}
931
932	static inline void sched_rt_rq_dequeue(struct rt_rq *rt_rq)
933	{
934	dequeue_top_rt_rq(rt_rq, rt_rq->rt_nr_running);
935	}
936
937	static inline int rt_rq_throttled(struct rt_rq *rt_rq)
938	{
939	return false;
940	}
941
942	static inline const struct cpumask *sched_rt_period_mask(void)
943	{
944	return cpu_online_mask;
945	}
946
947	static inline
948	struct rt_rq sched_rt_period_rt_rq(struct rt_bandwidth rt_b, int cpu)
949	{
950	return &cpu_rq(cpu)->rt;
951	}
952
953	static void __enable_runtime(struct rq *rq) { }
954	static void __disable_runtime(struct rq *rq) { }
955
956	#endif /* !CONFIG_RT_GROUP_SCHED */
957
958	static inline int rt_se_prio(struct sched_rt_entity *rt_se)
959	{
960	#ifdef CONFIG_RT_GROUP_SCHED
961	struct rt_rq *rt_rq = group_rt_rq(rt_se);
962
963	if (rt_rq)
964	return rt_rq->highest_prio.curr;
965	#endif
966
967	return rt_task_of(rt_se)->prio;
968	}
969
970	/*
971	* Update the current task's runtime statistics. Skip current tasks that
972	* are not in our scheduling class.
973	*/
974	static void update_curr_rt(struct rq *rq)
975	{
976	struct task_struct *donor = rq->donor;
977	s64 delta_exec;
978
979	if (donor->sched_class != &rt_sched_class)
980	return;
981
982	delta_exec = update_curr_common(rq);
983	if (unlikely(delta_exec <= 0))
984	return;
985
986	#ifdef CONFIG_RT_GROUP_SCHED
987	struct sched_rt_entity *rt_se = &donor->rt;
988
989	if (!rt_bandwidth_enabled())
990	return;
991
992	for_each_sched_rt_entity(rt_se) {
993	struct rt_rq *rt_rq = rt_rq_of_se(rt_se);
994	int exceeded;
995
996	if (sched_rt_runtime(rt_rq) != RUNTIME_INF) {
997	raw_spin_lock(&rt_rq->rt_runtime_lock);
998	rt_rq->rt_time += delta_exec;
999	exceeded = sched_rt_runtime_exceeded(rt_rq);
1000	if (exceeded)
1001	resched_curr(rq);
1002	raw_spin_unlock(&rt_rq->rt_runtime_lock);
1003	if (exceeded)
1004	do_start_rt_bandwidth(sched_rt_bandwidth(rt_rq));
1005	}
1006	}
1007	#endif /* CONFIG_RT_GROUP_SCHED */
1008	}
1009
1010	static void
1011	dequeue_top_rt_rq(struct rt_rq *rt_rq, unsigned int count)
1012	{
1013	struct rq *rq = rq_of_rt_rq(rt_rq);
1014
1015	BUG_ON(&rq->rt != rt_rq);
1016
1017	if (!rt_rq->rt_queued)
1018	return;
1019
1020	BUG_ON(!rq->nr_running);
1021
1022	sub_nr_running(rq, count);
1023	rt_rq->rt_queued = 0;
1024
1025	}
1026
1027	static void
1028	enqueue_top_rt_rq(struct rt_rq *rt_rq)
1029	{
1030	struct rq *rq = rq_of_rt_rq(rt_rq);
1031
1032	BUG_ON(&rq->rt != rt_rq);
1033
1034	if (rt_rq->rt_queued)
1035	return;
1036
1037	if (rt_rq_throttled(rt_rq))
1038	return;
1039
1040	if (rt_rq->rt_nr_running) {
1041	add_nr_running(rq, rt_rq->rt_nr_running);
1042	rt_rq->rt_queued = 1;
1043	}
1044
1045	/* Kick cpufreq (see the comment in kernel/sched/sched.h). */
1046	cpufreq_update_util(rq, 0);
1047	}
1048
1049	static void
1050	inc_rt_prio_smp(struct rt_rq *rt_rq, int prio, int prev_prio)
1051	{
1052	struct rq *rq = rq_of_rt_rq(rt_rq);
1053
1054	/*
1055	* Change rq's cpupri only if rt_rq is the top queue.
1056	*/
1057	if (IS_ENABLED(CONFIG_RT_GROUP_SCHED) && &rq->rt != rt_rq)
1058	return;
1059
1060	if (rq->online && prio < prev_prio)
1061	cpupri_set(&rq->rd->cpupri, rq->cpu, prio);
1062	}
1063
1064	static void
1065	dec_rt_prio_smp(struct rt_rq *rt_rq, int prio, int prev_prio)
1066	{
1067	struct rq *rq = rq_of_rt_rq(rt_rq);
1068
1069	/*
1070	* Change rq's cpupri only if rt_rq is the top queue.
1071	*/
1072	if (IS_ENABLED(CONFIG_RT_GROUP_SCHED) && &rq->rt != rt_rq)
1073	return;
1074
1075	if (rq->online && rt_rq->highest_prio.curr != prev_prio)
1076	cpupri_set(&rq->rd->cpupri, rq->cpu, rt_rq->highest_prio.curr);
1077	}
1078
1079	static void
1080	inc_rt_prio(struct rt_rq *rt_rq, int prio)
1081	{
1082	int prev_prio = rt_rq->highest_prio.curr;
1083
1084	if (prio < prev_prio)
1085	rt_rq->highest_prio.curr = prio;
1086
1087	inc_rt_prio_smp(rt_rq, prio, prev_prio);
1088	}
1089
1090	static void
1091	dec_rt_prio(struct rt_rq *rt_rq, int prio)
1092	{
1093	int prev_prio = rt_rq->highest_prio.curr;
1094
1095	if (rt_rq->rt_nr_running) {
1096
1097	WARN_ON(prio < prev_prio);
1098
1099	/*
1100	* This may have been our highest task, and therefore
1101	* we may have some re-computation to do
1102	*/
1103	if (prio == prev_prio) {
1104	struct rt_prio_array *array = &rt_rq->active;
1105
1106	rt_rq->highest_prio.curr =
1107	sched_find_first_bit(array->bitmap);
1108	}
1109
1110	} else {
1111	rt_rq->highest_prio.curr = MAX_RT_PRIO-1;
1112	}
1113
1114	dec_rt_prio_smp(rt_rq, prio, prev_prio);
1115	}
1116
1117	#ifdef CONFIG_RT_GROUP_SCHED
1118
1119	static void
1120	inc_rt_group(struct sched_rt_entity rt_se, struct rt_rq rt_rq)
1121	{
1122	if (rt_se_boosted(rt_se))
1123	rt_rq->rt_nr_boosted++;
1124
1125	start_rt_bandwidth(&rt_rq->tg->rt_bandwidth);
1126	}
1127
1128	static void
1129	dec_rt_group(struct sched_rt_entity rt_se, struct rt_rq rt_rq)
1130	{
1131	if (rt_se_boosted(rt_se))
1132	rt_rq->rt_nr_boosted--;
1133
1134	WARN_ON(!rt_rq->rt_nr_running && rt_rq->rt_nr_boosted);
1135	}
1136
1137	#else /* !CONFIG_RT_GROUP_SCHED: */
1138
1139	static void
1140	inc_rt_group(struct sched_rt_entity rt_se, struct rt_rq rt_rq)
1141	{
1142	}
1143
1144	static inline
1145	void dec_rt_group(struct sched_rt_entity rt_se, struct rt_rq rt_rq) {}
1146
1147	#endif /* !CONFIG_RT_GROUP_SCHED */
1148
1149	static inline
1150	unsigned int rt_se_nr_running(struct sched_rt_entity *rt_se)
1151	{
1152	struct rt_rq *group_rq = group_rt_rq(rt_se);
1153
1154	if (group_rq)
1155	return group_rq->rt_nr_running;
1156	else
1157	return 1;
1158	}
1159
1160	static inline
1161	unsigned int rt_se_rr_nr_running(struct sched_rt_entity *rt_se)
1162	{
1163	struct rt_rq *group_rq = group_rt_rq(rt_se);
1164	struct task_struct *tsk;
1165
1166	if (group_rq)
1167	return group_rq->rr_nr_running;
1168
1169	tsk = rt_task_of(rt_se);
1170
1171	return (tsk->policy == SCHED_RR) ? 1 : 0;
1172	}
1173
1174	static inline
1175	void inc_rt_tasks(struct sched_rt_entity rt_se, struct rt_rq rt_rq)
1176	{
1177	int prio = rt_se_prio(rt_se);
1178
1179	WARN_ON(!rt_prio(prio));
1180	rt_rq->rt_nr_running += rt_se_nr_running(rt_se);
1181	rt_rq->rr_nr_running += rt_se_rr_nr_running(rt_se);
1182
1183	inc_rt_prio(rt_rq, prio);
1184	inc_rt_group(rt_se, rt_rq);
1185	}
1186
1187	static inline
1188	void dec_rt_tasks(struct sched_rt_entity rt_se, struct rt_rq rt_rq)
1189	{
1190	WARN_ON(!rt_prio(rt_se_prio(rt_se)));
1191	WARN_ON(!rt_rq->rt_nr_running);
1192	rt_rq->rt_nr_running -= rt_se_nr_running(rt_se);
1193	rt_rq->rr_nr_running -= rt_se_rr_nr_running(rt_se);
1194
1195	dec_rt_prio(rt_rq, rt_se_prio(rt_se));
1196	dec_rt_group(rt_se, rt_rq);
1197	}
1198
1199	/*
1200	* Change rt_se->run_list location unless SAVE && !MOVE
1201	*
1202	* assumes ENQUEUE/DEQUEUE flags match
1203	*/
1204	static inline bool move_entity(unsigned int flags)
1205	{
1206	if ((flags & (DEQUEUE_SAVE \| DEQUEUE_MOVE)) == DEQUEUE_SAVE)
1207	return false;
1208
1209	return true;
1210	}
1211
1212	static void __delist_rt_entity(struct sched_rt_entity rt_se, struct rt_prio_array array)
1213	{
1214	list_del_init(&rt_se->run_list);
1215
1216	if (list_empty(array->queue + rt_se_prio(rt_se)))
1217	__clear_bit(rt_se_prio(rt_se), array->bitmap);
1218
1219	rt_se->on_list = 0;
1220	}
1221
1222	static inline struct sched_statistics *
1223	__schedstats_from_rt_se(struct sched_rt_entity *rt_se)
1224	{
1225	/* schedstats is not supported for rt group. */
1226	if (!rt_entity_is_task(rt_se))
1227	return NULL;
1228
1229	return &rt_task_of(rt_se)->stats;
1230	}
1231
1232	static inline void
1233	update_stats_wait_start_rt(struct rt_rq rt_rq, struct sched_rt_entity rt_se)
1234	{
1235	struct sched_statistics *stats;
1236	struct task_struct *p = NULL;
1237
1238	if (!schedstat_enabled())
1239	return;
1240
1241	if (rt_entity_is_task(rt_se))
1242	p = rt_task_of(rt_se);
1243
1244	stats = __schedstats_from_rt_se(rt_se);
1245	if (!stats)
1246	return;
1247
1248	__update_stats_wait_start(rq_of_rt_rq(rt_rq), p, stats);
1249	}
1250
1251	static inline void
1252	update_stats_enqueue_sleeper_rt(struct rt_rq rt_rq, struct sched_rt_entity rt_se)
1253	{
1254	struct sched_statistics *stats;
1255	struct task_struct *p = NULL;
1256
1257	if (!schedstat_enabled())
1258	return;
1259
1260	if (rt_entity_is_task(rt_se))
1261	p = rt_task_of(rt_se);
1262
1263	stats = __schedstats_from_rt_se(rt_se);
1264	if (!stats)
1265	return;
1266
1267	__update_stats_enqueue_sleeper(rq_of_rt_rq(rt_rq), p, stats);
1268	}
1269
1270	static inline void
1271	update_stats_enqueue_rt(struct rt_rq rt_rq, struct sched_rt_entity rt_se,
1272	int flags)
1273	{
1274	if (!schedstat_enabled())
1275	return;
1276
1277	if (flags & ENQUEUE_WAKEUP)
1278	update_stats_enqueue_sleeper_rt(rt_rq, rt_se);
1279	}
1280
1281	static inline void
1282	update_stats_wait_end_rt(struct rt_rq rt_rq, struct sched_rt_entity rt_se)
1283	{
1284	struct sched_statistics *stats;
1285	struct task_struct *p = NULL;
1286
1287	if (!schedstat_enabled())
1288	return;
1289
1290	if (rt_entity_is_task(rt_se))
1291	p = rt_task_of(rt_se);
1292
1293	stats = __schedstats_from_rt_se(rt_se);
1294	if (!stats)
1295	return;
1296
1297	__update_stats_wait_end(rq_of_rt_rq(rt_rq), p, stats);
1298	}
1299
1300	static inline void
1301	update_stats_dequeue_rt(struct rt_rq rt_rq, struct sched_rt_entity rt_se,
1302	int flags)
1303	{
1304	struct task_struct *p = NULL;
1305	struct rq *rq = rq_of_rt_rq(rt_rq);
1306
1307	if (!schedstat_enabled())
1308	return;
1309
1310	if (rt_entity_is_task(rt_se)) {
1311	p = rt_task_of(rt_se);
1312
1313	if (p != rq->curr)
1314	update_stats_wait_end_rt(rt_rq, rt_se);
1315	}
1316
1317	if ((flags & DEQUEUE_SLEEP) && p) {
1318	unsigned int state;
1319
1320	state = READ_ONCE(p->__state);
1321	if (state & TASK_INTERRUPTIBLE)
1322	__schedstat_set(p->stats.sleep_start,
1323	rq_clock(rq_of_rt_rq(rt_rq)));
1324
1325	if (state & TASK_UNINTERRUPTIBLE)
1326	__schedstat_set(p->stats.block_start,
1327	rq_clock(rq_of_rt_rq(rt_rq)));
1328	}
1329	}
1330
1331	static void __enqueue_rt_entity(struct sched_rt_entity *rt_se, unsigned int flags)
1332	{
1333	struct rt_rq *rt_rq = rt_rq_of_se(rt_se);
1334	struct rt_prio_array *array = &rt_rq->active;
1335	struct rt_rq *group_rq = group_rt_rq(rt_se);
1336	struct list_head *queue = array->queue + rt_se_prio(rt_se);
1337
1338	/*
1339	* Don't enqueue the group if its throttled, or when empty.
1340	* The latter is a consequence of the former when a child group
1341	* get throttled and the current group doesn't have any other
1342	* active members.
1343	*/
1344	if (group_rq && (rt_rq_throttled(group_rq) \|\| !group_rq->rt_nr_running)) {
1345	if (rt_se->on_list)
1346	__delist_rt_entity(rt_se, array);
1347	return;
1348	}
1349
1350	if (move_entity(flags)) {
1351	WARN_ON_ONCE(rt_se->on_list);
1352	if (flags & ENQUEUE_HEAD)
1353	list_add(&rt_se->run_list, queue);
1354	else
1355	list_add_tail(&rt_se->run_list, queue);
1356
1357	__set_bit(rt_se_prio(rt_se), array->bitmap);
1358	rt_se->on_list = 1;
1359	}
1360	rt_se->on_rq = 1;
1361
1362	inc_rt_tasks(rt_se, rt_rq);
1363	}
1364
1365	static void __dequeue_rt_entity(struct sched_rt_entity *rt_se, unsigned int flags)
1366	{
1367	struct rt_rq *rt_rq = rt_rq_of_se(rt_se);
1368	struct rt_prio_array *array = &rt_rq->active;
1369
1370	if (move_entity(flags)) {
1371	WARN_ON_ONCE(!rt_se->on_list);
1372	__delist_rt_entity(rt_se, array);
1373	}
1374	rt_se->on_rq = 0;
1375
1376	dec_rt_tasks(rt_se, rt_rq);
1377	}
1378
1379	/*
1380	* Because the prio of an upper entry depends on the lower
1381	* entries, we must remove entries top - down.
1382	*/
1383	static void dequeue_rt_stack(struct sched_rt_entity *rt_se, unsigned int flags)
1384	{
1385	struct sched_rt_entity *back = NULL;
1386	unsigned int rt_nr_running;
1387
1388	for_each_sched_rt_entity(rt_se) {
1389	rt_se->back = back;
1390	back = rt_se;
1391	}
1392
1393	rt_nr_running = rt_rq_of_se(back)->rt_nr_running;
1394
1395	for (rt_se = back; rt_se; rt_se = rt_se->back) {
1396	if (on_rt_rq(rt_se))
1397	__dequeue_rt_entity(rt_se, flags);
1398	}
1399
1400	dequeue_top_rt_rq(rt_rq_of_se(back), rt_nr_running);
1401	}
1402
1403	static void enqueue_rt_entity(struct sched_rt_entity *rt_se, unsigned int flags)
1404	{
1405	struct rq *rq = rq_of_rt_se(rt_se);
1406
1407	update_stats_enqueue_rt(rt_rq_of_se(rt_se), rt_se, flags);
1408
1409	dequeue_rt_stack(rt_se, flags);
1410	for_each_sched_rt_entity(rt_se)
1411	__enqueue_rt_entity(rt_se, flags);
1412	enqueue_top_rt_rq(&rq->rt);
1413	}
1414
1415	static void dequeue_rt_entity(struct sched_rt_entity *rt_se, unsigned int flags)
1416	{
1417	struct rq *rq = rq_of_rt_se(rt_se);
1418
1419	update_stats_dequeue_rt(rt_rq_of_se(rt_se), rt_se, flags);
1420
1421	dequeue_rt_stack(rt_se, flags);
1422
1423	for_each_sched_rt_entity(rt_se) {
1424	struct rt_rq *rt_rq = group_rt_rq(rt_se);
1425
1426	if (rt_rq && rt_rq->rt_nr_running)
1427	__enqueue_rt_entity(rt_se, flags);
1428	}
1429	enqueue_top_rt_rq(&rq->rt);
1430	}
1431
1432	/*
1433	* Adding/removing a task to/from a priority array:
1434	*/
1435	static void
1436	enqueue_task_rt(struct rq rq, struct task_struct p, int flags)
1437	{
1438	struct sched_rt_entity *rt_se = &p->rt;
1439
1440	if (flags & ENQUEUE_WAKEUP)
1441	rt_se->timeout = 0;
1442
1443	check_schedstat_required();
1444	update_stats_wait_start_rt(rt_rq_of_se(rt_se), rt_se);
1445
1446	enqueue_rt_entity(rt_se, flags);
1447
1448	if (task_is_blocked(p))
1449	return;
1450
1451	if (!task_current(rq, p) && p->nr_cpus_allowed > 1)
1452	enqueue_pushable_task(rq, p);
1453	}
1454
1455	static bool dequeue_task_rt(struct rq rq, struct task_struct p, int flags)
1456	{
1457	struct sched_rt_entity *rt_se = &p->rt;
1458
1459	update_curr_rt(rq);
1460	dequeue_rt_entity(rt_se, flags);
1461
1462	dequeue_pushable_task(rq, p);
1463
1464	return true;
1465	}
1466
1467	/*
1468	* Put task to the head or the end of the run list without the overhead of
1469	* dequeue followed by enqueue.
1470	*/
1471	static void
1472	requeue_rt_entity(struct rt_rq rt_rq, struct sched_rt_entity rt_se, int head)
1473	{
1474	if (on_rt_rq(rt_se)) {
1475	struct rt_prio_array *array = &rt_rq->active;
1476	struct list_head *queue = array->queue + rt_se_prio(rt_se);
1477
1478	if (head)
1479	list_move(&rt_se->run_list, queue);
1480	else
1481	list_move_tail(&rt_se->run_list, queue);
1482	}
1483	}
1484
1485	static void requeue_task_rt(struct rq rq, struct task_struct p, int head)
1486	{
1487	struct sched_rt_entity *rt_se = &p->rt;
1488	struct rt_rq *rt_rq;
1489
1490	for_each_sched_rt_entity(rt_se) {
1491	rt_rq = rt_rq_of_se(rt_se);
1492	requeue_rt_entity(rt_rq, rt_se, head);
1493	}
1494	}
1495
1496	static void yield_task_rt(struct rq *rq)
1497	{
1498	requeue_task_rt(rq, rq->donor, 0);
1499	}
1500
1501	static int find_lowest_rq(struct task_struct *task);
1502
1503	static int
1504	select_task_rq_rt(struct task_struct *p, int cpu, int flags)
1505	{
1506	struct task_struct curr, donor;
1507	struct rq *rq;
1508	bool test;
1509
1510	/* For anything but wake ups, just return the task_cpu */
1511	if (!(flags & (WF_TTWU \| WF_FORK)))
1512	goto out;
1513
1514	rq = cpu_rq(cpu);
1515
1516	rcu_read_lock();
1517	curr = READ_ONCE(rq->curr); /* unlocked access */
1518	donor = READ_ONCE(rq->donor);
1519
1520	/*
1521	* If the current task on @p's runqueue is an RT task, then
1522	* try to see if we can wake this RT task up on another
1523	* runqueue. Otherwise simply start this RT task
1524	* on its current runqueue.
1525	*
1526	* We want to avoid overloading runqueues. If the woken
1527	* task is a higher priority, then it will stay on this CPU
1528	* and the lower prio task should be moved to another CPU.
1529	* Even though this will probably make the lower prio task
1530	* lose its cache, we do not want to bounce a higher task
1531	* around just because it gave up its CPU, perhaps for a
1532	* lock?
1533	*
1534	* For equal prio tasks, we just let the scheduler sort it out.
1535	*
1536	* Otherwise, just let it ride on the affine RQ and the
1537	* post-schedule router will push the preempted task away
1538	*
1539	* This test is optimistic, if we get it wrong the load-balancer
1540	* will have to sort it out.
1541	*
1542	* We take into account the capacity of the CPU to ensure it fits the
1543	* requirement of the task - which is only important on heterogeneous
1544	* systems like big.LITTLE.
1545	*/
1546	test = curr &&
1547	unlikely(rt_task(donor)) &&
1548	(curr->nr_cpus_allowed < 2 \|\| donor->prio <= p->prio);
1549
1550	if (test \|\| !rt_task_fits_capacity(p, cpu)) {
1551	int target = find_lowest_rq(p);
1552
1553	/*
1554	* Bail out if we were forcing a migration to find a better
1555	* fitting CPU but our search failed.
1556	*/
1557	if (!test && target != -1 && !rt_task_fits_capacity(p, target))
1558	goto out_unlock;
1559
1560	/*
1561	* Don't bother moving it if the destination CPU is
1562	* not running a lower priority task.
1563	*/
1564	if (target != -1 &&
1565	p->prio < cpu_rq(target)->rt.highest_prio.curr)
1566	cpu = target;
1567	}
1568
1569	out_unlock:
1570	rcu_read_unlock();
1571
1572	out:
1573	return cpu;
1574	}
1575
1576	static void check_preempt_equal_prio(struct rq rq, struct task_struct p)
1577	{
1578	if (rq->curr->nr_cpus_allowed == 1 \|\|
1579	!cpupri_find(&rq->rd->cpupri, rq->donor, NULL))
1580	return;
1581
1582	/*
1583	* p is migratable, so let's not schedule it and
1584	* see if it is pushed or pulled somewhere else.
1585	*/
1586	if (p->nr_cpus_allowed != 1 &&
1587	cpupri_find(&rq->rd->cpupri, p, NULL))
1588	return;
1589
1590	/*
1591	* There appear to be other CPUs that can accept
1592	* the current task but none can run 'p', so lets reschedule
1593	* to try and push the current task away:
1594	*/
1595	requeue_task_rt(rq, p, 1);
1596	resched_curr(rq);
1597	}
1598
1599	static int balance_rt(struct rq rq, struct rq_flags rf)
1600	{
1601	/*
1602	* Note, rq->donor may change during rq lock drops,
1603	* so don't re-use p across lock drops
1604	*/
1605	struct task_struct *p = rq->donor;
1606
1607	if (!on_rt_rq(&p->rt) && need_pull_rt_task(rq, p)) {
1608	/*
1609	* This is OK, because current is on_cpu, which avoids it being
1610	* picked for load-balance and preemption/IRQs are still
1611	* disabled avoiding further scheduler activity on it and we've
1612	* not yet started the picking loop.
1613	*/
1614	rq_unpin_lock(rq, rf);
1615	pull_rt_task(rq);
1616	rq_repin_lock(rq, rf);
1617	}
1618
1619	return sched_stop_runnable(rq) \|\| sched_dl_runnable(rq) \|\| sched_rt_runnable(rq);
1620	}
1621
1622	/*
1623	* Preempt the current task with a newly woken task if needed:
1624	*/
1625	static void wakeup_preempt_rt(struct rq rq, struct task_struct p, int flags)
1626	{
1627	struct task_struct *donor = rq->donor;
1628
1629	/*
1630	* XXX If we're preempted by DL, queue a push?
1631	*/
1632	if (p->sched_class != &rt_sched_class \|\|
1633	donor->sched_class != &rt_sched_class)
1634	return;
1635
1636	if (p->prio < donor->prio) {
1637	resched_curr(rq);
1638	return;
1639	}
1640
1641	/*
1642	* If:
1643	*
1644	* - the newly woken task is of equal priority to the current task
1645	* - the newly woken task is non-migratable while current is migratable
1646	* - current will be preempted on the next reschedule
1647	*
1648	* we should check to see if current can readily move to a different
1649	* cpu. If so, we will reschedule to allow the push logic to try
1650	* to move current somewhere else, making room for our non-migratable
1651	* task.
1652	*/
1653	if (p->prio == donor->prio && !test_tsk_need_resched(rq->curr))
1654	check_preempt_equal_prio(rq, p);
1655	}
1656
1657	static inline void set_next_task_rt(struct rq rq, struct task_struct p, bool first)
1658	{
1659	struct sched_rt_entity *rt_se = &p->rt;
1660	struct rt_rq *rt_rq = &rq->rt;
1661
1662	p->se.exec_start = rq_clock_task(rq);
1663	if (on_rt_rq(&p->rt))
1664	update_stats_wait_end_rt(rt_rq, rt_se);
1665
1666	/* The running task is never eligible for pushing */
1667	dequeue_pushable_task(rq, p);
1668
1669	if (!first)
1670	return;
1671
1672	/*
1673	* If prev task was rt, put_prev_task() has already updated the
1674	* utilization. We only care of the case where we start to schedule a
1675	* rt task
1676	*/
1677	if (rq->donor->sched_class != &rt_sched_class)
1678	update_rt_rq_load_avg(rq_clock_pelt(rq), rq, 0);
1679
1680	rt_queue_push_tasks(rq);
1681	}
1682
1683	static struct sched_rt_entity pick_next_rt_entity(struct rt_rq rt_rq)
1684	{
1685	struct rt_prio_array *array = &rt_rq->active;
1686	struct sched_rt_entity *next = NULL;
1687	struct list_head *queue;
1688	int idx;
1689
1690	idx = sched_find_first_bit(array->bitmap);
1691	BUG_ON(idx >= MAX_RT_PRIO);
1692
1693	queue = array->queue + idx;
1694	if (WARN_ON_ONCE(list_empty(queue)))
1695	return NULL;
1696	next = list_entry(queue->next, struct sched_rt_entity, run_list);
1697
1698	return next;
1699	}
1700
1701	static struct task_struct _pick_next_task_rt(struct rq rq)
1702	{
1703	struct sched_rt_entity *rt_se;
1704	struct rt_rq *rt_rq = &rq->rt;
1705
1706	do {
1707	rt_se = pick_next_rt_entity(rt_rq);
1708	if (unlikely(!rt_se))
1709	return NULL;
1710	rt_rq = group_rt_rq(rt_se);
1711	} while (rt_rq);
1712
1713	return rt_task_of(rt_se);
1714	}
1715
1716	static struct task_struct pick_task_rt(struct rq rq, struct rq_flags *rf)
1717	{
1718	struct task_struct *p;
1719
1720	if (!sched_rt_runnable(rq))
1721	return NULL;
1722
1723	p = _pick_next_task_rt(rq);
1724
1725	return p;
1726	}
1727
1728	static void put_prev_task_rt(struct rq rq, struct task_struct p, struct task_struct *next)
1729	{
1730	struct sched_rt_entity *rt_se = &p->rt;
1731	struct rt_rq *rt_rq = &rq->rt;
1732
1733	if (on_rt_rq(&p->rt))
1734	update_stats_wait_start_rt(rt_rq, rt_se);
1735
1736	update_curr_rt(rq);
1737
1738	update_rt_rq_load_avg(rq_clock_pelt(rq), rq, 1);
1739
1740	if (task_is_blocked(p))
1741	return;
1742	/*
1743	* The previous task needs to be made eligible for pushing
1744	* if it is still active
1745	*/
1746	if (on_rt_rq(&p->rt) && p->nr_cpus_allowed > 1)
1747	enqueue_pushable_task(rq, p);
1748	}
1749
1750	/* Only try algorithms three times */
1751	#define RT_MAX_TRIES 3
1752
1753	/*
1754	* Return the highest pushable rq's task, which is suitable to be executed
1755	* on the CPU, NULL otherwise
1756	*/
1757	static struct task_struct pick_highest_pushable_task(struct rq rq, int cpu)
1758	{
1759	struct plist_head *head = &rq->rt.pushable_tasks;
1760	struct task_struct *p;
1761
1762	if (!has_pushable_tasks(rq))
1763	return NULL;
1764
1765	plist_for_each_entry(p, head, pushable_tasks) {
1766	if (task_is_pushable(rq, p, cpu))
1767	return p;
1768	}
1769
1770	return NULL;
1771	}
1772
1773	static DEFINE_PER_CPU(cpumask_var_t, local_cpu_mask);
1774
1775	static int find_lowest_rq(struct task_struct *task)
1776	{
1777	struct sched_domain *sd;
1778	struct cpumask *lowest_mask = this_cpu_cpumask_var_ptr(local_cpu_mask);
1779	int this_cpu = smp_processor_id();
1780	int cpu = task_cpu(task);
1781	int ret;
1782
1783	/* Make sure the mask is initialized first */
1784	if (unlikely(!lowest_mask))
1785	return -1;
1786
1787	if (task->nr_cpus_allowed == 1)
1788	return -1; /* No other targets possible */
1789
1790	/*
1791	* If we're on asym system ensure we consider the different capacities
1792	* of the CPUs when searching for the lowest_mask.
1793	*/
1794	if (sched_asym_cpucap_active()) {
1795
1796	ret = cpupri_find_fitness(&task_rq(task)->rd->cpupri,
1797	task, lowest_mask,
1798	rt_task_fits_capacity);
1799	} else {
1800
1801	ret = cpupri_find(&task_rq(task)->rd->cpupri,
1802	task, lowest_mask);
1803	}
1804
1805	if (!ret)
1806	return -1; /* No targets found */
1807
1808	/*
1809	* At this point we have built a mask of CPUs representing the
1810	* lowest priority tasks in the system. Now we want to elect
1811	* the best one based on our affinity and topology.
1812	*
1813	* We prioritize the last CPU that the task executed on since
1814	* it is most likely cache-hot in that location.
1815	*/
1816	if (cpumask_test_cpu(cpu, lowest_mask))
1817	return cpu;
1818
1819	/*
1820	* Otherwise, we consult the sched_domains span maps to figure
1821	* out which CPU is logically closest to our hot cache data.
1822	*/
1823	if (!cpumask_test_cpu(this_cpu, lowest_mask))
1824	this_cpu = -1; /* Skip this_cpu opt if not among lowest */
1825
1826	rcu_read_lock();
1827	for_each_domain(cpu, sd) {
1828	if (sd->flags & SD_WAKE_AFFINE) {
1829	int best_cpu;
1830
1831	/*
1832	* "this_cpu" is cheaper to preempt than a
1833	* remote processor.
1834	*/
1835	if (this_cpu != -1 &&
1836	cpumask_test_cpu(this_cpu, sched_domain_span(sd))) {
1837	rcu_read_unlock();
1838	return this_cpu;
1839	}
1840
1841	best_cpu = cpumask_any_and_distribute(lowest_mask,
1842	sched_domain_span(sd));
1843	if (best_cpu < nr_cpu_ids) {
1844	rcu_read_unlock();
1845	return best_cpu;
1846	}
1847	}
1848	}
1849	rcu_read_unlock();
1850
1851	/*
1852	* And finally, if there were no matches within the domains
1853	* just give the caller something to work with from the compatible
1854	* locations.
1855	*/
1856	if (this_cpu != -1)
1857	return this_cpu;
1858
1859	cpu = cpumask_any_distribute(lowest_mask);
1860	if (cpu < nr_cpu_ids)
1861	return cpu;
1862
1863	return -1;
1864	}
1865
1866	static struct task_struct pick_next_pushable_task(struct rq rq)
1867	{
1868	struct plist_head *head = &rq->rt.pushable_tasks;
1869	struct task_struct i, p = NULL;
1870
1871	if (!has_pushable_tasks(rq))
1872	return NULL;
1873
1874	plist_for_each_entry(i, head, pushable_tasks) {
1875	/* skip tasks that cannot be migrated */
1876	if (!task_on_cpu(rq, i) && !is_migration_disabled(i)) {
1877	p = i;
1878	break;
1879	}
1880	}
1881
1882	if (!p)
1883	return NULL;
1884
1885	BUG_ON(rq->cpu != task_cpu(p));
1886	BUG_ON(task_current(rq, p));
1887	BUG_ON(task_current_donor(rq, p));
1888	BUG_ON(p->nr_cpus_allowed <= 1);
1889
1890	BUG_ON(!task_on_rq_queued(p));
1891	BUG_ON(!rt_task(p));
1892
1893	return p;
1894	}
1895
1896	/* Will lock the rq it finds */
1897	static struct rq find_lock_lowest_rq(struct task_struct task, struct rq *rq)
1898	{
1899	struct rq *lowest_rq = NULL;
1900	int tries;
1901	int cpu;
1902
1903	for (tries = 0; tries < RT_MAX_TRIES; tries++) {
1904	cpu = find_lowest_rq(task);
1905
1906	if ((cpu == -1) \|\| (cpu == rq->cpu))
1907	break;
1908
1909	lowest_rq = cpu_rq(cpu);
1910
1911	if (lowest_rq->rt.highest_prio.curr <= task->prio) {
1912	/*
1913	* Target rq has tasks of equal or higher priority,
1914	* retrying does not release any lock and is unlikely
1915	* to yield a different result.
1916	*/
1917	lowest_rq = NULL;
1918	break;
1919	}
1920
1921	/* if the prio of this runqueue changed, try again */
1922	if (double_lock_balance(rq, lowest_rq)) {
1923	/*
1924	* We had to unlock the run queue. In
1925	* the mean time, task could have
1926	* migrated already or had its affinity changed,
1927	* therefore check if the task is still at the
1928	* head of the pushable tasks list.
1929	* It is possible the task was scheduled, set
1930	* "migrate_disabled" and then got preempted, so we must
1931	* check the task migration disable flag here too.
1932	*/
1933	if (unlikely(is_migration_disabled(task) \|\|
1934	!cpumask_test_cpu(lowest_rq->cpu, &task->cpus_mask) \|\|
1935	task != pick_next_pushable_task(rq))) {
1936
1937	double_unlock_balance(rq, lowest_rq);
1938	lowest_rq = NULL;
1939	break;
1940	}
1941	}
1942
1943	/* If this rq is still suitable use it. */
1944	if (lowest_rq->rt.highest_prio.curr > task->prio)
1945	break;
1946
1947	/* try again */
1948	double_unlock_balance(rq, lowest_rq);
1949	lowest_rq = NULL;
1950	}
1951
1952	return lowest_rq;
1953	}
1954
1955	/*
1956	* If the current CPU has more than one RT task, see if the non
1957	* running task can migrate over to a CPU that is running a task
1958	* of lesser priority.
1959	*/
1960	static int push_rt_task(struct rq *rq, bool pull)
1961	{
1962	struct task_struct *next_task;
1963	struct rq *lowest_rq;
1964	int ret = 0;
1965
1966	if (!rq->rt.overloaded)
1967	return 0;
1968
1969	next_task = pick_next_pushable_task(rq);
1970	if (!next_task)
1971	return 0;
1972
1973	retry:
1974	/*
1975	* It's possible that the next_task slipped in of
1976	* higher priority than current. If that's the case
1977	* just reschedule current.
1978	*/
1979	if (unlikely(next_task->prio < rq->donor->prio)) {
1980	resched_curr(rq);
1981	return 0;
1982	}
1983
1984	if (is_migration_disabled(next_task)) {
1985	struct task_struct *push_task = NULL;
1986	int cpu;
1987
1988	if (!pull \|\| rq->push_busy)
1989	return 0;
1990
1991	/*
1992	* Invoking find_lowest_rq() on anything but an RT task doesn't
1993	* make sense. Per the above priority check, curr has to
1994	* be of higher priority than next_task, so no need to
1995	* reschedule when bailing out.
1996	*
1997	* Note that the stoppers are masqueraded as SCHED_FIFO
1998	* (cf. sched_set_stop_task()), so we can't rely on rt_task().
1999	*/
2000	if (rq->donor->sched_class != &rt_sched_class)
2001	return 0;
2002
2003	cpu = find_lowest_rq(rq->curr);
2004	if (cpu == -1 \|\| cpu == rq->cpu)
2005	return 0;
2006
2007	/*
2008	* Given we found a CPU with lower priority than @next_task,
2009	* therefore it should be running. However we cannot migrate it
2010	* to this other CPU, instead attempt to push the current
2011	* running task on this CPU away.
2012	*/
2013	push_task = get_push_task(rq);
2014	if (push_task) {
2015	preempt_disable();
2016	raw_spin_rq_unlock(rq);
2017	stop_one_cpu_nowait(rq->cpu, push_cpu_stop,
2018	push_task, &rq->push_work);
2019	preempt_enable();
2020	raw_spin_rq_lock(rq);
2021	}
2022
2023	return 0;
2024	}
2025
2026	if (WARN_ON(next_task == rq->curr))
2027	return 0;
2028
2029	/* We might release rq lock */
2030	get_task_struct(next_task);
2031
2032	/* find_lock_lowest_rq locks the rq if found */
2033	lowest_rq = find_lock_lowest_rq(next_task, rq);
2034	if (!lowest_rq) {
2035	struct task_struct *task;
2036	/*
2037	* find_lock_lowest_rq releases rq->lock
2038	* so it is possible that next_task has migrated.
2039	*
2040	* We need to make sure that the task is still on the same
2041	* run-queue and is also still the next task eligible for
2042	* pushing.
2043	*/
2044	task = pick_next_pushable_task(rq);
2045	if (task == next_task) {
2046	/*
2047	* The task hasn't migrated, and is still the next
2048	* eligible task, but we failed to find a run-queue
2049	* to push it to. Do not retry in this case, since
2050	* other CPUs will pull from us when ready.
2051	*/
2052	goto out;
2053	}
2054
2055	if (!task)
2056	/* No more tasks, just exit */
2057	goto out;
2058
2059	/*
2060	* Something has shifted, try again.
2061	*/
2062	put_task_struct(next_task);
2063	next_task = task;
2064	goto retry;
2065	}
2066
2067	move_queued_task_locked(rq, lowest_rq, next_task);
2068	resched_curr(lowest_rq);
2069	ret = 1;
2070
2071	double_unlock_balance(rq, lowest_rq);
2072	out:
2073	put_task_struct(next_task);
2074
2075	return ret;
2076	}
2077
2078	static void push_rt_tasks(struct rq *rq)
2079	{
2080	/* push_rt_task will return true if it moved an RT */
2081	while (push_rt_task(rq, false))
2082	;
2083	}
2084
2085	#ifdef HAVE_RT_PUSH_IPI
2086
2087	/*
2088	* When a high priority task schedules out from a CPU and a lower priority
2089	* task is scheduled in, a check is made to see if there's any RT tasks
2090	* on other CPUs that are waiting to run because a higher priority RT task
2091	* is currently running on its CPU. In this case, the CPU with multiple RT
2092	* tasks queued on it (overloaded) needs to be notified that a CPU has opened
2093	* up that may be able to run one of its non-running queued RT tasks.
2094	*
2095	* All CPUs with overloaded RT tasks need to be notified as there is currently
2096	* no way to know which of these CPUs have the highest priority task waiting
2097	* to run. Instead of trying to take a spinlock on each of these CPUs,
2098	* which has shown to cause large latency when done on machines with many
2099	* CPUs, sending an IPI to the CPUs to have them push off the overloaded
2100	* RT tasks waiting to run.
2101	*
2102	* Just sending an IPI to each of the CPUs is also an issue, as on large
2103	* count CPU machines, this can cause an IPI storm on a CPU, especially
2104	* if its the only CPU with multiple RT tasks queued, and a large number
2105	* of CPUs scheduling a lower priority task at the same time.
2106	*
2107	* Each root domain has its own IRQ work function that can iterate over
2108	* all CPUs with RT overloaded tasks. Since all CPUs with overloaded RT
2109	* task must be checked if there's one or many CPUs that are lowering
2110	* their priority, there's a single IRQ work iterator that will try to
2111	* push off RT tasks that are waiting to run.
2112	*
2113	* When a CPU schedules a lower priority task, it will kick off the
2114	* IRQ work iterator that will jump to each CPU with overloaded RT tasks.
2115	* As it only takes the first CPU that schedules a lower priority task
2116	* to start the process, the rto_start variable is incremented and if
2117	* the atomic result is one, then that CPU will try to take the rto_lock.
2118	* This prevents high contention on the lock as the process handles all
2119	* CPUs scheduling lower priority tasks.
2120	*
2121	* All CPUs that are scheduling a lower priority task will increment the
2122	* rt_loop_next variable. This will make sure that the IRQ work iterator
2123	* checks all RT overloaded CPUs whenever a CPU schedules a new lower
2124	* priority task, even if the iterator is in the middle of a scan. Incrementing
2125	* the rt_loop_next will cause the iterator to perform another scan.
2126	*
2127	*/
2128	static int rto_next_cpu(struct root_domain *rd)
2129	{
2130	int this_cpu = smp_processor_id();
2131	int next;
2132	int cpu;
2133
2134	/*
2135	* When starting the IPI RT pushing, the rto_cpu is set to -1,
2136	* rt_next_cpu() will simply return the first CPU found in
2137	* the rto_mask.
2138	*
2139	* If rto_next_cpu() is called with rto_cpu is a valid CPU, it
2140	* will return the next CPU found in the rto_mask.
2141	*
2142	* If there are no more CPUs left in the rto_mask, then a check is made
2143	* against rto_loop and rto_loop_next. rto_loop is only updated with
2144	* the rto_lock held, but any CPU may increment the rto_loop_next
2145	* without any locking.
2146	*/
2147	for (;;) {
2148
2149	/* When rto_cpu is -1 this acts like cpumask_first() */
2150	cpu = cpumask_next(rd->rto_cpu, rd->rto_mask);
2151
2152	rd->rto_cpu = cpu;
2153
2154	/* Do not send IPI to self */
2155	if (cpu == this_cpu)
2156	continue;
2157
2158	if (cpu < nr_cpu_ids)
2159	return cpu;
2160
2161	rd->rto_cpu = -1;
2162
2163	/*
2164	* ACQUIRE ensures we see the @rto_mask changes
2165	* made prior to the @next value observed.
2166	*
2167	* Matches WMB in rt_set_overload().
2168	*/
2169	next = atomic_read_acquire(&rd->rto_loop_next);
2170
2171	if (rd->rto_loop == next)
2172	break;
2173
2174	rd->rto_loop = next;
2175	}
2176
2177	return -1;
2178	}
2179
2180	static inline bool rto_start_trylock(atomic_t *v)
2181	{
2182	return !atomic_cmpxchg_acquire(v, 0, 1);
2183	}
2184
2185	static inline void rto_start_unlock(atomic_t *v)
2186	{
2187	atomic_set_release(v, 0);
2188	}
2189
2190	static void tell_cpu_to_push(struct rq *rq)
2191	{
2192	int cpu = -1;
2193
2194	/* Keep the loop going if the IPI is currently active */
2195	atomic_inc(&rq->rd->rto_loop_next);
2196
2197	/* Only one CPU can initiate a loop at a time */
2198	if (!rto_start_trylock(&rq->rd->rto_loop_start))
2199	return;
2200
2201	raw_spin_lock(&rq->rd->rto_lock);
2202
2203	/*
2204	* The rto_cpu is updated under the lock, if it has a valid CPU
2205	* then the IPI is still running and will continue due to the
2206	* update to loop_next, and nothing needs to be done here.
2207	* Otherwise it is finishing up and an IPI needs to be sent.
2208	*/
2209	if (rq->rd->rto_cpu < 0)
2210	cpu = rto_next_cpu(rq->rd);
2211
2212	raw_spin_unlock(&rq->rd->rto_lock);
2213
2214	rto_start_unlock(&rq->rd->rto_loop_start);
2215
2216	if (cpu >= 0) {
2217	/* Make sure the rd does not get freed while pushing */
2218	sched_get_rd(rq->rd);
2219	irq_work_queue_on(&rq->rd->rto_push_work, cpu);
2220	}
2221	}
2222
2223	/* Called from hardirq context */
2224	void rto_push_irq_work_func(struct irq_work *work)
2225	{
2226	struct root_domain *rd =
2227	container_of(work, struct root_domain, rto_push_work);
2228	struct rq *rq;
2229	int cpu;
2230
2231	rq = this_rq();
2232
2233	/*
2234	* We do not need to grab the lock to check for has_pushable_tasks.
2235	* When it gets updated, a check is made if a push is possible.
2236	*/
2237	if (has_pushable_tasks(rq)) {
2238	raw_spin_rq_lock(rq);
2239	while (push_rt_task(rq, true))
2240	;
2241	raw_spin_rq_unlock(rq);
2242	}
2243
2244	raw_spin_lock(&rd->rto_lock);
2245
2246	/* Pass the IPI to the next rt overloaded queue */
2247	cpu = rto_next_cpu(rd);
2248
2249	raw_spin_unlock(&rd->rto_lock);
2250
2251	if (cpu < 0) {
2252	sched_put_rd(rd);
2253	return;
2254	}
2255
2256	/* Try the next RT overloaded CPU */
2257	irq_work_queue_on(&rd->rto_push_work, cpu);
2258	}
2259	#endif /* HAVE_RT_PUSH_IPI */
2260
2261	static void pull_rt_task(struct rq *this_rq)
2262	{
2263	int this_cpu = this_rq->cpu, cpu;
2264	bool resched = false;
2265	struct task_struct p, push_task;
2266	struct rq *src_rq;
2267	int rt_overload_count = rt_overloaded(this_rq);
2268
2269	if (likely(!rt_overload_count))
2270	return;
2271
2272	/*
2273	* Match the barrier from rt_set_overloaded; this guarantees that if we
2274	* see overloaded we must also see the rto_mask bit.
2275	*/
2276	smp_rmb();
2277
2278	/* If we are the only overloaded CPU do nothing */
2279	if (rt_overload_count == 1 &&
2280	cpumask_test_cpu(this_rq->cpu, this_rq->rd->rto_mask))
2281	return;
2282
2283	#ifdef HAVE_RT_PUSH_IPI
2284	if (sched_feat(RT_PUSH_IPI)) {
2285	tell_cpu_to_push(this_rq);
2286	return;
2287	}
2288	#endif
2289
2290	for_each_cpu(cpu, this_rq->rd->rto_mask) {
2291	if (this_cpu == cpu)
2292	continue;
2293
2294	src_rq = cpu_rq(cpu);
2295
2296	/*
2297	* Don't bother taking the src_rq->lock if the next highest
2298	* task is known to be lower-priority than our current task.
2299	* This may look racy, but if this value is about to go
2300	* logically higher, the src_rq will push this task away.
2301	* And if its going logically lower, we do not care
2302	*/
2303	if (src_rq->rt.highest_prio.next >=
2304	this_rq->rt.highest_prio.curr)
2305	continue;
2306
2307	/*
2308	* We can potentially drop this_rq's lock in
2309	* double_lock_balance, and another CPU could
2310	* alter this_rq
2311	*/
2312	push_task = NULL;
2313	double_lock_balance(this_rq, src_rq);
2314
2315	/*
2316	* We can pull only a task, which is pushable
2317	* on its rq, and no others.
2318	*/
2319	p = pick_highest_pushable_task(src_rq, this_cpu);
2320
2321	/*
2322	* Do we have an RT task that preempts
2323	* the to-be-scheduled task?
2324	*/
2325	if (p && (p->prio < this_rq->rt.highest_prio.curr)) {
2326	WARN_ON(p == src_rq->curr);
2327	WARN_ON(!task_on_rq_queued(p));
2328
2329	/*
2330	* There's a chance that p is higher in priority
2331	* than what's currently running on its CPU.
2332	* This is just that p is waking up and hasn't
2333	* had a chance to schedule. We only pull
2334	* p if it is lower in priority than the
2335	* current task on the run queue
2336	*/
2337	if (p->prio < src_rq->donor->prio)
2338	goto skip;
2339
2340	if (is_migration_disabled(p)) {
2341	push_task = get_push_task(src_rq);
2342	} else {
2343	move_queued_task_locked(src_rq, this_rq, p);
2344	resched = true;
2345	}
2346	/*
2347	* We continue with the search, just in
2348	* case there's an even higher prio task
2349	* in another runqueue. (low likelihood
2350	* but possible)
2351	*/
2352	}
2353	skip:
2354	double_unlock_balance(this_rq, src_rq);
2355
2356	if (push_task) {
2357	preempt_disable();
2358	raw_spin_rq_unlock(this_rq);
2359	stop_one_cpu_nowait(src_rq->cpu, push_cpu_stop,
2360	push_task, &src_rq->push_work);
2361	preempt_enable();
2362	raw_spin_rq_lock(this_rq);
2363	}
2364	}
2365
2366	if (resched)
2367	resched_curr(this_rq);
2368	}
2369
2370	/*
2371	* If we are not running and we are not going to reschedule soon, we should
2372	* try to push tasks away now
2373	*/
2374	static void task_woken_rt(struct rq rq, struct task_struct p)
2375	{
2376	bool need_to_push = !task_on_cpu(rq, p) &&
2377	!test_tsk_need_resched(rq->curr) &&
2378	p->nr_cpus_allowed > 1 &&
2379	(dl_task(rq->donor) \|\| rt_task(rq->donor)) &&
2380	(rq->curr->nr_cpus_allowed < 2 \|\|
2381	rq->donor->prio <= p->prio);
2382
2383	if (need_to_push)
2384	push_rt_tasks(rq);
2385	}
2386
2387	/* Assumes rq->lock is held */
2388	static void rq_online_rt(struct rq *rq)
2389	{
2390	if (rq->rt.overloaded)
2391	rt_set_overload(rq);
2392
2393	__enable_runtime(rq);
2394
2395	cpupri_set(&rq->rd->cpupri, rq->cpu, rq->rt.highest_prio.curr);
2396	}
2397
2398	/* Assumes rq->lock is held */
2399	static void rq_offline_rt(struct rq *rq)
2400	{
2401	if (rq->rt.overloaded)
2402	rt_clear_overload(rq);
2403
2404	__disable_runtime(rq);
2405
2406	cpupri_set(&rq->rd->cpupri, rq->cpu, CPUPRI_INVALID);
2407	}
2408
2409	/*
2410	* When switch from the rt queue, we bring ourselves to a position
2411	* that we might want to pull RT tasks from other runqueues.
2412	*/
2413	static void switched_from_rt(struct rq rq, struct task_struct p)
2414	{
2415	/*
2416	* If there are other RT tasks then we will reschedule
2417	* and the scheduling of the other RT tasks will handle
2418	* the balancing. But if we are the last RT task
2419	* we may need to handle the pulling of RT tasks
2420	* now.
2421	*/
2422	if (!task_on_rq_queued(p) \|\| rq->rt.rt_nr_running)
2423	return;
2424
2425	rt_queue_pull_task(rq);
2426	}
2427
2428	void __init init_sched_rt_class(void)
2429	{
2430	unsigned int i;
2431
2432	for_each_possible_cpu(i) {
2433	zalloc_cpumask_var_node(&per_cpu(local_cpu_mask, i),
2434	GFP_KERNEL, cpu_to_node(i));
2435	}
2436	}
2437
2438	/*
2439	* When switching a task to RT, we may overload the runqueue
2440	* with RT tasks. In this case we try to push them off to
2441	* other runqueues.
2442	*/
2443	static void switched_to_rt(struct rq rq, struct task_struct p)
2444	{
2445	/*
2446	* If we are running, update the avg_rt tracking, as the running time
2447	* will now on be accounted into the latter.
2448	*/
2449	if (task_current(rq, p)) {
2450	update_rt_rq_load_avg(rq_clock_pelt(rq), rq, 0);
2451	return;
2452	}
2453
2454	/*
2455	* If we are not running we may need to preempt the current
2456	* running task. If that current running task is also an RT task
2457	* then see if we can move to another run queue.
2458	*/
2459	if (task_on_rq_queued(p)) {
2460	if (p->nr_cpus_allowed > 1 && rq->rt.overloaded)
2461	rt_queue_push_tasks(rq);
2462	if (p->prio < rq->donor->prio && cpu_online(cpu_of(rq)))
2463	resched_curr(rq);
2464	}
2465	}
2466
2467	/*
2468	* Priority of the task has changed. This may cause
2469	* us to initiate a push or pull.
2470	*/
2471	static void
2472	prio_changed_rt(struct rq rq, struct task_struct p, u64 oldprio)
2473	{
2474	if (!task_on_rq_queued(p))
2475	return;
2476
2477	if (p->prio == oldprio)
2478	return;
2479
2480	if (task_current_donor(rq, p)) {
2481	/*
2482	* If our priority decreases while running, we
2483	* may need to pull tasks to this runqueue.
2484	*/
2485	if (oldprio < p->prio)
2486	rt_queue_pull_task(rq);
2487
2488	/*
2489	* If there's a higher priority task waiting to run
2490	* then reschedule.
2491	*/
2492	if (p->prio > rq->rt.highest_prio.curr)
2493	resched_curr(rq);
2494	} else {
2495	/*
2496	* This task is not running, but if it is
2497	* greater than the current running task
2498	* then reschedule.
2499	*/
2500	if (p->prio < rq->donor->prio)
2501	resched_curr(rq);
2502	}
2503	}
2504
2505	#ifdef CONFIG_POSIX_TIMERS
2506	static void watchdog(struct rq rq, struct task_struct p)
2507	{
2508	unsigned long soft, hard;
2509
2510	/* max may change after cur was read, this will be fixed next tick */
2511	soft = task_rlimit(p, RLIMIT_RTTIME);
2512	hard = task_rlimit_max(p, RLIMIT_RTTIME);
2513
2514	if (soft != RLIM_INFINITY) {
2515	unsigned long next;
2516
2517	if (p->rt.watchdog_stamp != jiffies) {
2518	p->rt.timeout++;
2519	p->rt.watchdog_stamp = jiffies;
2520	}
2521
2522	next = DIV_ROUND_UP(min(soft, hard), USEC_PER_SEC/HZ);
2523	if (p->rt.timeout > next) {
2524	posix_cputimers_rt_watchdog(&p->posix_cputimers,
2525	p->se.sum_exec_runtime);
2526	}
2527	}
2528	}
2529	#else /* !CONFIG_POSIX_TIMERS: */
2530	static inline void watchdog(struct rq rq, struct task_struct p) { }
2531	#endif /* !CONFIG_POSIX_TIMERS */
2532
2533	/*
2534	* scheduler tick hitting a task of our scheduling class.
2535	*
2536	* NOTE: This function can be called remotely by the tick offload that
2537	* goes along full dynticks. Therefore no local assumption can be made
2538	* and everything must be accessed through the @rq and @curr passed in
2539	* parameters.
2540	*/
2541	static void task_tick_rt(struct rq rq, struct task_struct p, int queued)
2542	{
2543	struct sched_rt_entity *rt_se = &p->rt;
2544
2545	update_curr_rt(rq);
2546	update_rt_rq_load_avg(rq_clock_pelt(rq), rq, 1);
2547
2548	watchdog(rq, p);
2549
2550	/*
2551	* RR tasks need a special form of time-slice management.
2552	* FIFO tasks have no timeslices.
2553	*/
2554	if (p->policy != SCHED_RR)
2555	return;
2556
2557	if (--p->rt.time_slice)
2558	return;
2559
2560	p->rt.time_slice = sched_rr_timeslice;
2561
2562	/*
2563	* Requeue to the end of queue if we (and all of our ancestors) are not
2564	* the only element on the queue
2565	*/
2566	for_each_sched_rt_entity(rt_se) {
2567	if (rt_se->run_list.prev != rt_se->run_list.next) {
2568	requeue_task_rt(rq, p, 0);
2569	resched_curr(rq);
2570	return;
2571	}
2572	}
2573	}
2574
2575	static unsigned int get_rr_interval_rt(struct rq rq, struct task_struct task)
2576	{
2577	/*
2578	* Time slice is 0 for SCHED_FIFO tasks
2579	*/
2580	if (task->policy == SCHED_RR)
2581	return sched_rr_timeslice;
2582	else
2583	return 0;
2584	}
2585
2586	#ifdef CONFIG_SCHED_CORE
2587	static int task_is_throttled_rt(struct task_struct *p, int cpu)
2588	{
2589	struct rt_rq *rt_rq;
2590
2591	#ifdef CONFIG_RT_GROUP_SCHED // XXX maybe add task_rt_rq(), see also sched_rt_period_rt_rq
2592	rt_rq = task_group(p)->rt_rq[cpu];
2593	WARN_ON(!rt_group_sched_enabled() && rt_rq->tg != &root_task_group);
2594	#else
2595	rt_rq = &cpu_rq(cpu)->rt;
2596	#endif
2597
2598	return rt_rq_throttled(rt_rq);
2599	}
2600	#endif /* CONFIG_SCHED_CORE */
2601
2602	DEFINE_SCHED_CLASS(rt) = {
2603	.enqueue_task = enqueue_task_rt,
2604	.dequeue_task = dequeue_task_rt,
2605	.yield_task = yield_task_rt,
2606
2607	.wakeup_preempt = wakeup_preempt_rt,
2608
2609	.pick_task = pick_task_rt,
2610	.put_prev_task = put_prev_task_rt,
2611	.set_next_task = set_next_task_rt,
2612
2613	.balance = balance_rt,
2614	.select_task_rq = select_task_rq_rt,
2615	.set_cpus_allowed = set_cpus_allowed_common,
2616	.rq_online = rq_online_rt,
2617	.rq_offline = rq_offline_rt,
2618	.task_woken = task_woken_rt,
2619	.switched_from = switched_from_rt,
2620	.find_lock_rq = find_lock_lowest_rq,
2621
2622	.task_tick = task_tick_rt,
2623
2624	.get_rr_interval = get_rr_interval_rt,
2625
2626	.switched_to = switched_to_rt,
2627	.prio_changed = prio_changed_rt,
2628
2629	.update_curr = update_curr_rt,
2630
2631	#ifdef CONFIG_SCHED_CORE
2632	.task_is_throttled = task_is_throttled_rt,
2633	#endif
2634
2635	#ifdef CONFIG_UCLAMP_TASK
2636	.uclamp_enabled = 1,
2637	#endif
2638	};
2639
2640	#ifdef CONFIG_RT_GROUP_SCHED
2641	/*
2642	* Ensure that the real time constraints are schedulable.
2643	*/
2644	static DEFINE_MUTEX(rt_constraints_mutex);
2645
2646	static inline int tg_has_rt_tasks(struct task_group *tg)
2647	{
2648	struct task_struct *task;
2649	struct css_task_iter it;
2650	int ret = 0;
2651
2652	/*
2653	* Autogroups do not have RT tasks; see autogroup_create().
2654	*/
2655	if (task_group_is_autogroup(tg))
2656	return 0;
2657
2658	css_task_iter_start(&tg->css, 0, &it);
2659	while (!ret && (task = css_task_iter_next(&it)))
2660	ret \|= rt_task(task);
2661	css_task_iter_end(&it);
2662
2663	return ret;
2664	}
2665
2666	struct rt_schedulable_data {
2667	struct task_group *tg;
2668	u64 rt_period;
2669	u64 rt_runtime;
2670	};
2671
2672	static int tg_rt_schedulable(struct task_group tg, void data)
2673	{
2674	struct rt_schedulable_data *d = data;
2675	struct task_group *child;
2676	u64 total, sum = 0;
2677	u64 period, runtime;
2678
2679	period = ktime_to_ns(tg->rt_bandwidth.rt_period);
2680	runtime = tg->rt_bandwidth.rt_runtime;
2681
2682	if (tg == d->tg) {
2683	period = d->rt_period;
2684	runtime = d->rt_runtime;
2685	}
2686
2687	/*
2688	* Cannot have more runtime than the period.
2689	*/
2690	if (runtime > period && runtime != RUNTIME_INF)
2691	return -EINVAL;
2692
2693	/*
2694	* Ensure we don't starve existing RT tasks if runtime turns zero.
2695	*/
2696	if (rt_bandwidth_enabled() && !runtime &&
2697	tg->rt_bandwidth.rt_runtime && tg_has_rt_tasks(tg))
2698	return -EBUSY;
2699
2700	total = to_ratio(period, runtime);
2701
2702	/*
2703	* Nobody can have more than the global setting allows.
2704	*/
2705	if (total > to_ratio(global_rt_period(), global_rt_runtime()))
2706	return -EINVAL;
2707
2708	/*
2709	* The sum of our children's runtime should not exceed our own.
2710	*/
2711	list_for_each_entry_rcu(child, &tg->children, siblings) {
2712	period = ktime_to_ns(child->rt_bandwidth.rt_period);
2713	runtime = child->rt_bandwidth.rt_runtime;
2714
2715	if (child == d->tg) {
2716	period = d->rt_period;
2717	runtime = d->rt_runtime;
2718	}
2719
2720	sum += to_ratio(period, runtime);
2721	}
2722
2723	if (sum > total)
2724	return -EINVAL;
2725
2726	return 0;
2727	}
2728
2729	static int __rt_schedulable(struct task_group *tg, u64 period, u64 runtime)
2730	{
2731	int ret;
2732
2733	struct rt_schedulable_data data = {
2734	.tg = tg,
2735	.rt_period = period,
2736	.rt_runtime = runtime,
2737	};
2738
2739	rcu_read_lock();
2740	ret = walk_tg_tree(tg_rt_schedulable, tg_nop, &data);
2741	rcu_read_unlock();
2742
2743	return ret;
2744	}
2745
2746	static int tg_set_rt_bandwidth(struct task_group *tg,
2747	u64 rt_period, u64 rt_runtime)
2748	{
2749	int i, err = 0;
2750
2751	/*
2752	* Disallowing the root group RT runtime is BAD, it would disallow the
2753	* kernel creating (and or operating) RT threads.
2754	*/
2755	if (tg == &root_task_group && rt_runtime == 0)
2756	return -EINVAL;
2757
2758	/* No period doesn't make any sense. */
2759	if (rt_period == 0)
2760	return -EINVAL;
2761
2762	/*
2763	* Bound quota to defend quota against overflow during bandwidth shift.
2764	*/
2765	if (rt_runtime != RUNTIME_INF && rt_runtime > max_rt_runtime)
2766	return -EINVAL;
2767
2768	mutex_lock(&rt_constraints_mutex);
2769	err = __rt_schedulable(tg, rt_period, rt_runtime);
2770	if (err)
2771	goto unlock;
2772
2773	raw_spin_lock_irq(&tg->rt_bandwidth.rt_runtime_lock);
2774	tg->rt_bandwidth.rt_period = ns_to_ktime(rt_period);
2775	tg->rt_bandwidth.rt_runtime = rt_runtime;
2776
2777	for_each_possible_cpu(i) {
2778	struct rt_rq *rt_rq = tg->rt_rq[i];
2779
2780	raw_spin_lock(&rt_rq->rt_runtime_lock);
2781	rt_rq->rt_runtime = rt_runtime;
2782	raw_spin_unlock(&rt_rq->rt_runtime_lock);
2783	}
2784	raw_spin_unlock_irq(&tg->rt_bandwidth.rt_runtime_lock);
2785	unlock:
2786	mutex_unlock(&rt_constraints_mutex);
2787
2788	return err;
2789	}
2790
2791	int sched_group_set_rt_runtime(struct task_group *tg, long rt_runtime_us)
2792	{
2793	u64 rt_runtime, rt_period;
2794
2795	rt_period = ktime_to_ns(tg->rt_bandwidth.rt_period);
2796	rt_runtime = (u64)rt_runtime_us * NSEC_PER_USEC;
2797	if (rt_runtime_us < 0)
2798	rt_runtime = RUNTIME_INF;
2799	else if ((u64)rt_runtime_us > U64_MAX / NSEC_PER_USEC)
2800	return -EINVAL;
2801
2802	return tg_set_rt_bandwidth(tg, rt_period, rt_runtime);
2803	}
2804
2805	long sched_group_rt_runtime(struct task_group *tg)
2806	{
2807	u64 rt_runtime_us;
2808
2809	if (tg->rt_bandwidth.rt_runtime == RUNTIME_INF)
2810	return -1;
2811
2812	rt_runtime_us = tg->rt_bandwidth.rt_runtime;
2813	do_div(rt_runtime_us, NSEC_PER_USEC);
2814	return rt_runtime_us;
2815	}
2816
2817	int sched_group_set_rt_period(struct task_group *tg, u64 rt_period_us)
2818	{
2819	u64 rt_runtime, rt_period;
2820
2821	if (rt_period_us > U64_MAX / NSEC_PER_USEC)
2822	return -EINVAL;
2823
2824	rt_period = rt_period_us * NSEC_PER_USEC;
2825	rt_runtime = tg->rt_bandwidth.rt_runtime;
2826
2827	return tg_set_rt_bandwidth(tg, rt_period, rt_runtime);
2828	}
2829
2830	long sched_group_rt_period(struct task_group *tg)
2831	{
2832	u64 rt_period_us;
2833
2834	rt_period_us = ktime_to_ns(tg->rt_bandwidth.rt_period);
2835	do_div(rt_period_us, NSEC_PER_USEC);
2836	return rt_period_us;
2837	}
2838
2839	int sched_rt_can_attach(struct task_group tg, struct task_struct tsk)
2840	{
2841	/* Don't accept real-time tasks when there is no way for them to run */
2842	if (rt_group_sched_enabled() && rt_task(tsk) && tg->rt_bandwidth.rt_runtime == 0)
2843	return 0;
2844
2845	return 1;
2846	}
2847
2848	#endif /* !CONFIG_RT_GROUP_SCHED */
2849
2850	#ifdef CONFIG_SYSCTL
2851	static int sched_rt_global_validate(void)
2852	{
2853	if ((sysctl_sched_rt_runtime != RUNTIME_INF) &&
2854	((sysctl_sched_rt_runtime > sysctl_sched_rt_period) \|\|
2855	((u64)sysctl_sched_rt_runtime *
2856	NSEC_PER_USEC > max_rt_runtime)))
2857	return -EINVAL;
2858
2859	#ifdef CONFIG_RT_GROUP_SCHED
2860	if (!rt_group_sched_enabled())
2861	return 0;
2862
2863	scoped_guard(mutex, &rt_constraints_mutex)
2864	return __rt_schedulable(NULL, 0, 0);
2865	#endif
2866	return 0;
2867	}
2868
2869	static int sched_rt_handler(const struct ctl_table table, int write, void buffer,
2870	size_t lenp, loff_t ppos)
2871	{
2872	int old_period, old_runtime;
2873	static DEFINE_MUTEX(mutex);
2874	int ret;
2875
2876	mutex_lock(&mutex);
2877	sched_domains_mutex_lock();
2878	old_period = sysctl_sched_rt_period;
2879	old_runtime = sysctl_sched_rt_runtime;
2880
2881	ret = proc_dointvec_minmax(table, write, buffer, lenp, ppos);
2882
2883	if (!ret && write) {
2884	ret = sched_rt_global_validate();
2885	if (ret)
2886	goto undo;
2887
2888	ret = sched_dl_global_validate();
2889	if (ret)
2890	goto undo;
2891
2892	sched_dl_do_global();
2893	}
2894	if (0) {
2895	undo:
2896	sysctl_sched_rt_period = old_period;
2897	sysctl_sched_rt_runtime = old_runtime;
2898	}
2899	sched_domains_mutex_unlock();
2900	mutex_unlock(&mutex);
2901
2902	/*
2903	* After changing maximum available bandwidth for DEADLINE, we need to
2904	* recompute per root domain and per cpus variables accordingly.
2905	*/
2906	rebuild_sched_domains();
2907
2908	return ret;
2909	}
2910
2911	static int sched_rr_handler(const struct ctl_table table, int write, void buffer,
2912	size_t lenp, loff_t ppos)
2913	{
2914	int ret;
2915	static DEFINE_MUTEX(mutex);
2916
2917	mutex_lock(&mutex);
2918	ret = proc_dointvec(table, write, buffer, lenp, ppos);
2919	/*
2920	* Make sure that internally we keep jiffies.
2921	* Also, writing zero resets the time-slice to default:
2922	*/
2923	if (!ret && write) {
2924	sched_rr_timeslice =
2925	sysctl_sched_rr_timeslice <= 0 ? RR_TIMESLICE :
2926	msecs_to_jiffies(sysctl_sched_rr_timeslice);
2927
2928	if (sysctl_sched_rr_timeslice <= 0)
2929	sysctl_sched_rr_timeslice = jiffies_to_msecs(RR_TIMESLICE);
2930	}
2931	mutex_unlock(&mutex);
2932
2933	return ret;
2934	}
2935	#endif /* CONFIG_SYSCTL */
2936
2937	void print_rt_stats(struct seq_file *m, int cpu)
2938	{
2939	rt_rq_iter_t iter;
2940	struct rt_rq *rt_rq;
2941
2942	rcu_read_lock();
2943	for_each_rt_rq(rt_rq, iter, cpu_rq(cpu))
2944	print_rt_rq(m, cpu, rt_rq);
2945	rcu_read_unlock();
2946	}