linux - kernel/sched/fair.c

15960 lines · 433.8 KiB

1	// SPDX-License-Identifier: GPL-2.0
2	/*
3	* Completely Fair Scheduling (CFS) Class (SCHED_NORMAL/SCHED_BATCH)
4	*
5	* Copyright (C) 2007 Red Hat, Inc., Ingo Molnar <[email protected]>
6	*
7	* Interactivity improvements by Mike Galbraith
8	* (C) 2007 Mike Galbraith <[email protected]>
9	*
10	* Various enhancements by Dmitry Adamushko.
11	* (C) 2007 Dmitry Adamushko <[email protected]>
12	*
13	* Group scheduling enhancements by Srivatsa Vaddagiri
14	* Copyright IBM Corporation, 2007
15	* Author: Srivatsa Vaddagiri <[email protected]>
16	*
17	* Scaled math optimizations by Thomas Gleixner
18	* Copyright (C) 2007, Linutronix GmbH, Thomas Gleixner <[email protected]>
19	*
20	* Adaptive scheduling granularity, math enhancements by Peter Zijlstra
21	* Copyright (C) 2007 Red Hat, Inc., Peter Zijlstra
22	*/
23	#include <linux/energy_model.h>
24	#include <linux/mmap_lock.h>
25	#include <linux/hugetlb_inline.h>
26	#include <linux/jiffies.h>
27	#include <linux/math.h>
28	#include <linux/mm_api.h>
29	#include <linux/highmem.h>
30	#include <linux/hrtimer.h>
31	#include <linux/hrtimer_bases.h>
32	#include <linux/spinlock_api.h>
33	#include <linux/cpumask_api.h>
34	#include <linux/lockdep_api.h>
35	#include <linux/softirq.h>
36	#include <linux/refcount_api.h>
37	#include <linux/topology.h>
38	#include <linux/sched/clock.h>
39	#include <linux/sched/cond_resched.h>
40	#include <linux/sched/cputime.h>
41	#include <linux/sched/isolation.h>
42	#include <linux/sched/nohz.h>
43	#include <linux/sched/prio.h>
44	#include <linux/static_call.h>
45
46	#include <linux/cpuidle.h>
47	#include <linux/interrupt.h>
48	#include <linux/memory-tiers.h>
49	#include <linux/mempolicy.h>
50	#include <linux/mutex_api.h>
51	#include <linux/profile.h>
52	#include <linux/psi.h>
53	#include <linux/ratelimit.h>
54	#include <linux/task_work.h>
55	#include <linux/rbtree_augmented.h>
56
57	#include <asm/switch_to.h>
58
59	#include <uapi/linux/sched/types.h>
60
61	#include "sched.h"
62	#include "stats.h"
63	#include "autogroup.h"
64
65	/*
66	* The initial- and re-scaling of tunables is configurable
67	*
68	* Options are:
69	*
70	* SCHED_TUNABLESCALING_NONE - unscaled, always *1
71	* SCHED_TUNABLESCALING_LOG - scaled logarithmically, *1+ilog(ncpus)
72	* SCHED_TUNABLESCALING_LINEAR - scaled linear, *ncpus
73	*
74	* (default SCHED_TUNABLESCALING_LOG = *(1+ilog(ncpus))
75	*/
76	unsigned int sysctl_sched_tunable_scaling = SCHED_TUNABLESCALING_LOG;
77
78	/*
79	* Default base time slice (request size r_i) for SCHED_NORMAL/SCHED_BATCH:
80	*
81	* Under EEVDF this is the request size used to compute the virtual
82	* deadline; see update_deadline().
83	*
84	* (default: 0.70 msec * (1 + ilog(ncpus)), units: nanoseconds)
85	*/
86	unsigned int sysctl_sched_base_slice = 700000ULL;
87	static unsigned int normalized_sysctl_sched_base_slice = 700000ULL;
88
89	__read_mostly unsigned int sysctl_sched_migration_cost = 500000UL;
90
91	static int __init setup_sched_thermal_decay_shift(char *str)
92	{
93	pr_warn("Ignoring the deprecated sched_thermal_decay_shift= option\n");
94	return 1;
95	}
96	__setup("sched_thermal_decay_shift=", setup_sched_thermal_decay_shift);
97
98	/*
99	* For asym packing, by default the lower numbered CPU has higher priority.
100	*/
101	int __weak arch_asym_cpu_priority(int cpu)
102	{
103	return -cpu;
104	}
105
106	/*
107	* The margin used when comparing utilization with CPU capacity.
108	*
109	* (default: ~20%)
110	*/
111	#define fits_capacity(cap, max) ((cap) * 1280 < (max) * 1024)
112
113	/*
114	* The margin used when comparing CPU capacities.
115	* is 'cap1' noticeably greater than 'cap2'
116	*
117	* (default: ~5%)
118	*/
119	#define capacity_greater(cap1, cap2) ((cap1) * 1024 > (cap2) * 1078)
120
121	#ifdef CONFIG_CFS_BANDWIDTH
122	/*
123	* Amount of runtime to allocate from global (tg) to local (per-cfs_rq) pool
124	* each time a cfs_rq requests quota.
125	*
126	* Note: in the case that the slice exceeds the runtime remaining (either due
127	* to consumption or the quota being specified to be smaller than the slice)
128	* we will always only issue the remaining available time.
129	*
130	* (default: 5 msec, units: microseconds)
131	*/
132	static unsigned int sysctl_sched_cfs_bandwidth_slice = 5000UL;
133	#endif
134
135	#ifdef CONFIG_NUMA_BALANCING
136	/* Restrict the NUMA promotion throughput (MB/s) for each target node. */
137	static unsigned int sysctl_numa_balancing_promote_rate_limit = 65536;
138	#endif
139
140	#ifdef CONFIG_SYSCTL
141	static const struct ctl_table sched_fair_sysctls[] = {
142	#ifdef CONFIG_CFS_BANDWIDTH
143	{
144	.procname = "sched_cfs_bandwidth_slice_us",
145	.data = &sysctl_sched_cfs_bandwidth_slice,
146	.maxlen = sizeof(unsigned int),
147	.mode = 0644,
148	.proc_handler = proc_dointvec_minmax,
149	.extra1 = SYSCTL_ONE,
150	},
151	#endif
152	#ifdef CONFIG_NUMA_BALANCING
153	{
154	.procname = "numa_balancing_promote_rate_limit_MBps",
155	.data = &sysctl_numa_balancing_promote_rate_limit,
156	.maxlen = sizeof(unsigned int),
157	.mode = 0644,
158	.proc_handler = proc_dointvec_minmax,
159	.extra1 = SYSCTL_ZERO,
160	},
161	#endif /* CONFIG_NUMA_BALANCING */
162	};
163
164	static int __init sched_fair_sysctl_init(void)
165	{
166	register_sysctl_init("kernel", sched_fair_sysctls);
167	return 0;
168	}
169	late_initcall(sched_fair_sysctl_init);
170	#endif /* CONFIG_SYSCTL */
171
172	static inline void update_load_add(struct load_weight *lw, unsigned long inc)
173	{
174	lw->weight += inc;
175	lw->inv_weight = 0;
176	}
177
178	static inline void update_load_sub(struct load_weight *lw, unsigned long dec)
179	{
180	lw->weight -= dec;
181	lw->inv_weight = 0;
182	}
183
184	static inline void update_load_set(struct load_weight *lw, unsigned long w)
185	{
186	lw->weight = w;
187	lw->inv_weight = 0;
188	}
189
190	/*
191	* Increase the granularity value when there are more CPUs,
192	* because with more CPUs the 'effective latency' as visible
193	* to users decreases. But the relationship is not linear,
194	* so pick a second-best guess by going with the log2 of the
195	* number of CPUs.
196	*
197	* This idea comes from the SD scheduler of Con Kolivas:
198	*/
199	static unsigned int get_update_sysctl_factor(void)
200	{
201	unsigned int cpus = min_t(unsigned int, num_online_cpus(), 8);
202	unsigned int factor;
203
204	switch (sysctl_sched_tunable_scaling) {
205	case SCHED_TUNABLESCALING_NONE:
206	factor = 1;
207	break;
208	case SCHED_TUNABLESCALING_LINEAR:
209	factor = cpus;
210	break;
211	case SCHED_TUNABLESCALING_LOG:
212	default:
213	factor = 1 + ilog2(cpus);
214	break;
215	}
216
217	return factor;
218	}
219
220	static void update_sysctl(void)
221	{
222	unsigned int factor = get_update_sysctl_factor();
223
224	#define SET_SYSCTL(name) \
225	(sysctl_##name = (factor) * normalized_sysctl_##name)
226	SET_SYSCTL(sched_base_slice);
227	#undef SET_SYSCTL
228	}
229
230	void __init sched_init_granularity(void)
231	{
232	update_sysctl();
233	}
234
235	#ifndef CONFIG_64BIT
236	#define WMULT_CONST (~0U)
237	#define WMULT_SHIFT 32
238
239	static void __update_inv_weight(struct load_weight *lw)
240	{
241	unsigned long w;
242
243	if (likely(lw->inv_weight))
244	return;
245
246	w = scale_load_down(lw->weight);
247
248	if (BITS_PER_LONG > 32 && unlikely(w >= WMULT_CONST))
249	lw->inv_weight = 1;
250	else if (unlikely(!w))
251	lw->inv_weight = WMULT_CONST;
252	else
253	lw->inv_weight = WMULT_CONST / w;
254	}
255
256	/*
257	* delta_exec * weight / lw.weight
258	* OR
259	* (delta_exec * (weight * lw->inv_weight)) >> WMULT_SHIFT
260	*
261	* Either weight := NICE_0_LOAD and lw \e sched_prio_to_wmult[], in which case
262	* we're guaranteed shift stays positive because inv_weight is guaranteed to
263	* fit 32 bits, and NICE_0_LOAD gives another 10 bits; therefore shift >= 22.
264	*
265	* Or, weight =< lw.weight (because lw.weight is the runqueue weight), thus
266	* weight/lw.weight <= 1, and therefore our shift will also be positive.
267	*/
268	static u64 __calc_delta(u64 delta_exec, unsigned long weight, struct load_weight *lw)
269	{
270	u64 fact = scale_load_down(weight);
271	u32 fact_hi = (u32)(fact >> 32);
272	int shift = WMULT_SHIFT;
273	int fs;
274
275	__update_inv_weight(lw);
276
277	if (unlikely(fact_hi)) {
278	fs = fls(fact_hi);
279	shift -= fs;
280	fact >>= fs;
281	}
282
283	fact = mul_u32_u32(fact, lw->inv_weight);
284
285	fact_hi = (u32)(fact >> 32);
286	if (fact_hi) {
287	fs = fls(fact_hi);
288	shift -= fs;
289	fact >>= fs;
290	}
291
292	return mul_u64_u32_shr(delta_exec, fact, shift);
293	}
294	#else
295	static u64 __calc_delta(u64 delta_exec, unsigned long weight, struct load_weight *lw)
296	{
297	return (delta_exec * weight) / lw->weight;
298	}
299	#endif
300
301	/*
302	* delta /= w
303	*/
304	static inline u64 calc_delta_fair(u64 delta, struct sched_entity *se)
305	{
306	if (se->h_load.weight != NICE_0_LOAD)
307	delta = __calc_delta(delta, NICE_0_LOAD, &se->h_load);
308
309	return delta;
310	}
311
312	const struct sched_class fair_sched_class;
313
314	/**************************************************************
315	* CFS operations on generic schedulable entities:
316	*/
317
318	#ifdef CONFIG_FAIR_GROUP_SCHED
319
320	/* Walk up scheduling entities hierarchy */
321	#define for_each_sched_entity(se) \
322	for (; se; se = se->parent)
323
324	static inline bool list_add_leaf_cfs_rq(struct cfs_rq *cfs_rq)
325	{
326	struct rq *rq = rq_of(cfs_rq);
327	int cpu = cpu_of(rq);
328
329	if (cfs_rq->on_list)
330	return rq->tmp_alone_branch == &rq->leaf_cfs_rq_list;
331
332	cfs_rq->on_list = 1;
333
334	/*
335	* Ensure we either appear before our parent (if already
336	* enqueued) or force our parent to appear after us when it is
337	* enqueued. The fact that we always enqueue bottom-up
338	* reduces this to two cases and a special case for the root
339	* cfs_rq. Furthermore, it also means that we will always reset
340	* tmp_alone_branch either when the branch is connected
341	* to a tree or when we reach the top of the tree
342	*/
343	if (cfs_rq->tg->parent &&
344	tg_cfs_rq(cfs_rq->tg->parent, cpu)->on_list) {
345	/*
346	* If parent is already on the list, we add the child
347	* just before. Thanks to circular linked property of
348	* the list, this means to put the child at the tail
349	* of the list that starts by parent.
350	*/
351	list_add_tail_rcu(&cfs_rq->leaf_cfs_rq_list,
352	&(tg_cfs_rq(cfs_rq->tg->parent, cpu)->leaf_cfs_rq_list));
353	/*
354	* The branch is now connected to its tree so we can
355	* reset tmp_alone_branch to the beginning of the
356	* list.
357	*/
358	rq->tmp_alone_branch = &rq->leaf_cfs_rq_list;
359	return true;
360	}
361
362	if (!cfs_rq->tg->parent) {
363	/*
364	* cfs rq without parent should be put
365	* at the tail of the list.
366	*/
367	list_add_tail_rcu(&cfs_rq->leaf_cfs_rq_list,
368	&rq->leaf_cfs_rq_list);
369	/*
370	* We have reach the top of a tree so we can reset
371	* tmp_alone_branch to the beginning of the list.
372	*/
373	rq->tmp_alone_branch = &rq->leaf_cfs_rq_list;
374	return true;
375	}
376
377	/*
378	* The parent has not already been added so we want to
379	* make sure that it will be put after us.
380	* tmp_alone_branch points to the begin of the branch
381	* where we will add parent.
382	*/
383	list_add_rcu(&cfs_rq->leaf_cfs_rq_list, rq->tmp_alone_branch);
384	/*
385	* update tmp_alone_branch to points to the new begin
386	* of the branch
387	*/
388	rq->tmp_alone_branch = &cfs_rq->leaf_cfs_rq_list;
389	return false;
390	}
391
392	static inline void list_del_leaf_cfs_rq(struct cfs_rq *cfs_rq)
393	{
394	if (cfs_rq->on_list) {
395	struct rq *rq = rq_of(cfs_rq);
396
397	/*
398	* With cfs_rq being unthrottled/throttled during an enqueue,
399	* it can happen the tmp_alone_branch points to the leaf that
400	* we finally want to delete. In this case, tmp_alone_branch moves
401	* to the prev element but it will point to rq->leaf_cfs_rq_list
402	* at the end of the enqueue.
403	*/
404	if (rq->tmp_alone_branch == &cfs_rq->leaf_cfs_rq_list)
405	rq->tmp_alone_branch = cfs_rq->leaf_cfs_rq_list.prev;
406
407	list_del_rcu(&cfs_rq->leaf_cfs_rq_list);
408	cfs_rq->on_list = 0;
409	}
410	}
411
412	static inline void assert_list_leaf_cfs_rq(struct rq *rq)
413	{
414	WARN_ON_ONCE(rq->tmp_alone_branch != &rq->leaf_cfs_rq_list);
415	}
416
417	/* Iterate through all leaf cfs_rq's on a runqueue */
418	#define for_each_leaf_cfs_rq_safe(rq, cfs_rq, pos) \
419	list_for_each_entry_safe(cfs_rq, pos, &rq->leaf_cfs_rq_list, \
420	leaf_cfs_rq_list)
421
422	/* Do the two (enqueued) entities belong to the same group ? */
423	static inline struct cfs_rq *
424	is_same_group(struct sched_entity se, struct sched_entity pse)
425	{
426	if (se->cfs_rq == pse->cfs_rq)
427	return se->cfs_rq;
428
429	return NULL;
430	}
431
432	static inline struct sched_entity parent_entity(const struct sched_entity se)
433	{
434	return se->parent;
435	}
436
437	static int tg_is_idle(struct task_group *tg)
438	{
439	return tg->idle > 0;
440	}
441
442	static int cfs_rq_is_idle(struct cfs_rq *cfs_rq)
443	{
444	return cfs_rq->idle > 0;
445	}
446
447	static int se_is_idle(struct sched_entity *se)
448	{
449	if (entity_is_task(se))
450	return task_has_idle_policy(task_of(se));
451	return cfs_rq_is_idle(group_cfs_rq(se));
452	}
453
454	#else /* !CONFIG_FAIR_GROUP_SCHED: */
455
456	#define for_each_sched_entity(se) \
457	for (; se; se = NULL)
458
459	static inline bool list_add_leaf_cfs_rq(struct cfs_rq *cfs_rq)
460	{
461	return true;
462	}
463
464	static inline void list_del_leaf_cfs_rq(struct cfs_rq *cfs_rq)
465	{
466	}
467
468	static inline void assert_list_leaf_cfs_rq(struct rq *rq)
469	{
470	}
471
472	#define for_each_leaf_cfs_rq_safe(rq, cfs_rq, pos) \
473	for (cfs_rq = &rq->cfs, pos = NULL; cfs_rq; cfs_rq = pos)
474
475	static inline struct sched_entity parent_entity(struct sched_entity se)
476	{
477	return NULL;
478	}
479
480	static inline int tg_is_idle(struct task_group *tg)
481	{
482	return 0;
483	}
484
485	static int cfs_rq_is_idle(struct cfs_rq *cfs_rq)
486	{
487	return 0;
488	}
489
490	static int se_is_idle(struct sched_entity *se)
491	{
492	return task_has_idle_policy(task_of(se));
493	}
494
495	#endif /* !CONFIG_FAIR_GROUP_SCHED */
496
497	static __always_inline
498	bool account_cfs_rq_runtime(struct cfs_rq *cfs_rq, u64 delta_exec);
499
500	/**************************************************************
501	* Scheduling class tree data structure manipulation methods:
502	*/
503
504	extern void __BUILD_BUG_vruntime_cmp(void);
505
506	/* Use __builtin_strcmp() because of __HAVE_ARCH_STRCMP: */
507
508	#define vruntime_cmp(A, CMP_STR, B) ({ \
509	int __res = 0; \
510	\
511	if (!__builtin_strcmp(CMP_STR, "<")) { \
512	__res = ((s64)((A)-(B)) < 0); \
513	} else if (!__builtin_strcmp(CMP_STR, "<=")) { \
514	__res = ((s64)((A)-(B)) <= 0); \
515	} else if (!__builtin_strcmp(CMP_STR, ">")) { \
516	__res = ((s64)((A)-(B)) > 0); \
517	} else if (!__builtin_strcmp(CMP_STR, ">=")) { \
518	__res = ((s64)((A)-(B)) >= 0); \
519	} else { \
520	/* Unknown operator throws linker error: */ \
521	__BUILD_BUG_vruntime_cmp(); \
522	} \
523	\
524	__res; \
525	})
526
527	extern void __BUILD_BUG_vruntime_op(void);
528
529	#define vruntime_op(A, OP_STR, B) ({ \
530	s64 __res = 0; \
531	\
532	if (!__builtin_strcmp(OP_STR, "-")) { \
533	__res = (s64)((A)-(B)); \
534	} else { \
535	/* Unknown operator throws linker error: */ \
536	__BUILD_BUG_vruntime_op(); \
537	} \
538	\
539	__res; \
540	})
541
542
543	static inline __maybe_unused u64 max_vruntime(u64 max_vruntime, u64 vruntime)
544	{
545	if (vruntime_cmp(vruntime, ">", max_vruntime))
546	max_vruntime = vruntime;
547
548	return max_vruntime;
549	}
550
551	static inline __maybe_unused u64 min_vruntime(u64 min_vruntime, u64 vruntime)
552	{
553	if (vruntime_cmp(vruntime, "<", min_vruntime))
554	min_vruntime = vruntime;
555
556	return min_vruntime;
557	}
558
559	static inline bool entity_before(const struct sched_entity *a,
560	const struct sched_entity *b)
561	{
562	/*
563	* Tiebreak on vruntime seems unnecessary since it can
564	* hardly happen.
565	*/
566	return vruntime_cmp(a->deadline, "<", b->deadline);
567	}
568
569	/*
570	* Per avg_vruntime() below, cfs_rq::zero_vruntime is only slightly stale
571	* and this value should be no more than two lag bounds. Which puts it in the
572	* general order of:
573	*
574	* (slice + TICK_NSEC) << NICE_0_LOAD_SHIFT
575	*
576	* which is around 44 bits in size (on 64bit); that is 20 for
577	* NICE_0_LOAD_SHIFT, another 20 for NSEC_PER_MSEC and then a handful for
578	* however many msec the actual slice+tick ends up begin.
579	*
580	* (disregarding the actual divide-by-weight part makes for the worst case
581	* weight of 2, which nicely cancels vs the fuzz in zero_vruntime not actually
582	* being the zero-lag point).
583	*/
584	static inline s64 entity_key(struct cfs_rq cfs_rq, struct sched_entity se)
585	{
586	return vruntime_op(se->vruntime, "-", cfs_rq->zero_vruntime);
587	}
588
589	#define __node_2_se(node) \
590	rb_entry((node), struct sched_entity, run_node)
591
592	/*
593	* Compute virtual time from the per-task service numbers:
594	*
595	* Fair schedulers conserve lag:
596	*
597	* \Sum lag_i = 0
598	*
599	* Where lag_i is given by:
600	*
601	* lag_i = S - s_i = w_i * (V - v_i)
602	*
603	* Where S is the ideal service time and V is it's virtual time counterpart.
604	* Therefore:
605	*
606	* \Sum lag_i = 0
607	* \Sum w_i * (V - v_i) = 0
608	* \Sum (w_i * V - w_i * v_i) = 0
609	*
610	* From which we can solve an expression for V in v_i (which we have in
611	* se->vruntime):
612	*
613	* \Sum v_i * w_i \Sum v_i * w_i
614	* V = -------------- = --------------
615	* \Sum w_i W
616	*
617	* Specifically, this is the weighted average of all entity virtual runtimes.
618	*
619	* [[ NOTE: this is only equal to the ideal scheduler under the condition
620	* that join/leave operations happen at lag_i = 0, otherwise the
621	* virtual time has non-contiguous motion equivalent to:
622	*
623	* V +-= lag_i / W
624	*
625	* Also see the comment in place_entity() that deals with this. ]]
626	*
627	* However, since v_i is u64, and the multiplication could easily overflow
628	* transform it into a relative form that uses smaller quantities:
629	*
630	* Substitute: v_i == (v_i - v0) + v0
631	*
632	* \Sum ((v_i - v0) + v0) * w_i \Sum (v_i - v0) * w_i
633	* V = ---------------------------- = --------------------- + v0
634	* W W
635	*
636	* Which we track using:
637	*
638	* v0 := cfs_rq->zero_vruntime
639	* \Sum (v_i - v0) * w_i := cfs_rq->sum_w_vruntime
640	* \Sum w_i := cfs_rq->sum_weight
641	*
642	* Since zero_vruntime closely tracks the per-task service, these
643	* deltas: (v_i - v0), will be in the order of the maximal (virtual) lag
644	* induced in the system due to quantisation.
645	*/
646	static inline unsigned long avg_vruntime_weight(struct cfs_rq *cfs_rq, unsigned long w)
647	{
648	#ifdef CONFIG_64BIT
649	if (cfs_rq->sum_shift)
650	w = max(2UL, w >> cfs_rq->sum_shift);
651	#endif
652	return w;
653	}
654
655	static inline void
656	__sum_w_vruntime_add(struct cfs_rq cfs_rq, struct sched_entity se)
657	{
658	unsigned long weight = avg_vruntime_weight(cfs_rq, se->h_load.weight);
659	s64 w_vruntime, key = entity_key(cfs_rq, se);
660
661	w_vruntime = key * weight;
662	WARN_ON_ONCE((w_vruntime >> 63) != (w_vruntime >> 62));
663
664	cfs_rq->sum_w_vruntime += w_vruntime;
665	cfs_rq->sum_weight += weight;
666	}
667
668	static void
669	sum_w_vruntime_add_paranoid(struct cfs_rq cfs_rq, struct sched_entity se)
670	{
671	unsigned long weight;
672	s64 key, tmp;
673
674	again:
675	weight = avg_vruntime_weight(cfs_rq, se->h_load.weight);
676	key = entity_key(cfs_rq, se);
677
678	if (check_mul_overflow(key, weight, &key))
679	goto overflow;
680
681	if (check_add_overflow(cfs_rq->sum_w_vruntime, key, &tmp))
682	goto overflow;
683
684	cfs_rq->sum_w_vruntime = tmp;
685	cfs_rq->sum_weight += weight;
686	return;
687
688	overflow:
689	/*
690	* There's gotta be a limit -- if we're still failing at this point
691	* there's really nothing much to be done about things.
692	*/
693	BUG_ON(cfs_rq->sum_shift >= 10);
694	cfs_rq->sum_shift++;
695
696	/*
697	* Note: \Sum (k_i * (w_i >> 1)) != (\Sum (k_i * w_i)) >> 1
698	*/
699	cfs_rq->sum_w_vruntime = 0;
700	cfs_rq->sum_weight = 0;
701
702	for (struct rb_node *node = cfs_rq->tasks_timeline.rb_leftmost;
703	node; node = rb_next(node))
704	__sum_w_vruntime_add(cfs_rq, __node_2_se(node));
705
706	goto again;
707	}
708
709	static void
710	sum_w_vruntime_add(struct cfs_rq cfs_rq, struct sched_entity se)
711	{
712	if (sched_feat(PARANOID_AVG))
713	return sum_w_vruntime_add_paranoid(cfs_rq, se);
714
715	__sum_w_vruntime_add(cfs_rq, se);
716	}
717
718	static void
719	sum_w_vruntime_sub(struct cfs_rq cfs_rq, struct sched_entity se)
720	{
721	unsigned long weight = avg_vruntime_weight(cfs_rq, se->h_load.weight);
722	s64 key = entity_key(cfs_rq, se);
723
724	cfs_rq->sum_w_vruntime -= key * weight;
725	cfs_rq->sum_weight -= weight;
726	}
727
728	static inline
729	void update_zero_vruntime(struct cfs_rq *cfs_rq, s64 delta)
730	{
731	/*
732	* v' = v + d ==> sum_w_vruntime' = sum_w_vruntime - d*sum_weight
733	*/
734	cfs_rq->sum_w_vruntime -= cfs_rq->sum_weight * delta;
735	cfs_rq->zero_vruntime += delta;
736	}
737
738	/*
739	* Specifically: avg_vruntime() + 0 must result in entity_eligible() := true
740	* For this to be so, the result of this function must have a left bias.
741	*
742	* Called in:
743	* - place_entity() -- before enqueue
744	* - update_entity_lag() -- before dequeue
745	* - update_deadline() -- slice expiration
746	*
747	* This means it is one entry 'behind' but that puts it close enough to where
748	* the bound on entity_key() is at most two lag bounds.
749	*/
750	u64 avg_vruntime(struct cfs_rq *cfs_rq)
751	{
752	struct sched_entity *curr = cfs_rq->curr;
753	long weight = cfs_rq->sum_weight;
754	s64 delta = 0;
755
756	if (curr && !curr->on_rq)
757	curr = NULL;
758
759	if (weight) {
760	s64 runtime = cfs_rq->sum_w_vruntime;
761
762	if (curr) {
763	unsigned long w = avg_vruntime_weight(cfs_rq, curr->h_load.weight);
764
765	runtime += entity_key(cfs_rq, curr) * w;
766	weight += w;
767	}
768
769	/* sign flips effective floor / ceiling */
770	if (runtime < 0)
771	runtime -= (weight - 1);
772
773	delta = div64_long(runtime, weight);
774	} else if (curr) {
775	/*
776	* When there is but one element, it is the average.
777	*/
778	delta = curr->vruntime - cfs_rq->zero_vruntime;
779	}
780
781	update_zero_vruntime(cfs_rq, delta);
782
783	return cfs_rq->zero_vruntime;
784	}
785
786	/*
787	* \Sum (v_i - v0)*w_i
788	* V = ------------------- + v0
789	* \Sum w_i
790	*
791	* Let W = \Sum w_i, and move v_j such that 'v_j == V', thus:
792	*
793	* V = 1/W * {(v_j - v0)w_j + \Sum_i!=j (v_i - v0)w_i} + v0
794	*
795	* v_j = 1/W * {(v_j - v0)w_j + \Sum_i!=j (v_i - v0)w_i} + v0
796	*
797	* v_j = 1/W * (v_j - v0)w_j + 1/W \Sum_i!=j (v_i - v0)*w_i + v0
798	*
799	* v_j - 1/W * (v_j - v0)w_j = 1/W \Sum_i!=j (v_i - v0)*w_i + v0
800	*
801	* v_jW - (v_j - v0)w_j = \Sum_i!=j (v_i - v0)w_i + v0W
802	*
803	* v_j(W - w_j) + v0w_j = \Sum_i!=j (v_i - v0)w_i + v0W
804	*
805	* v_j(W - w_j) = \Sum_i!=j (v_i - v0)w_i + v0*(W - w_j)
806	*
807	* \Sum_i!=j (v_i - v0)*w_i
808	* v_j = ------------------------ + v0
809	* W - w_j
810	*
811	* When v_j happens to be curr, then '\Sum_i!=j (v_i - v0)*w_i'
812	* is cfs_rq->sum_w_runtime, and 'W - w_j' is cfs_rq->sum_weight, since curr
813	* is not included in the sum.
814	*/
815	static u64 ineligible_vruntime(struct cfs_rq *cfs_rq)
816	{
817	struct sched_entity *curr = cfs_rq->curr;
818	long weight = cfs_rq->sum_weight;
819	s64 delta = 0;
820
821	if (curr && !curr->on_rq)
822	curr = NULL;
823
824	if (weight) {
825	s64 runtime = cfs_rq->sum_w_vruntime;
826
827	/*
828	* Do not add @curr to obtain the effective '- w_j' terms.
829	*/
830
831	/* sign flips effective floor / ceiling */
832	if (runtime < 0)
833	runtime -= (weight - 1);
834
835	delta = div64_long(runtime, weight);
836	}
837
838	return cfs_rq->zero_vruntime + delta + 1;
839	}
840
841	static inline u64 cfs_rq_max_slice(struct cfs_rq *cfs_rq);
842
843	/*
844	* lag_i = S - s_i = w_i * (V - v_i)
845	*
846	* However, since V is approximated by the weighted average of all entities it
847	* is possible -- by addition/removal/reweight to the tree -- to move V around
848	* and end up with a larger lag than we started with.
849	*
850	* Limit this to either double the slice length with a minimum of TICK_NSEC
851	* since that is the timing granularity.
852	*
853	* EEVDF gives the following limit for a steady state system:
854	*
855	* -r_max < lag < max(r_max, q)
856	*/
857	static s64 entity_lag(struct cfs_rq cfs_rq, struct sched_entity se, u64 avruntime)
858	{
859	u64 max_slice = cfs_rq_max_slice(cfs_rq) + TICK_NSEC;
860	s64 vlag, limit;
861
862	vlag = avruntime - se->vruntime;
863	limit = calc_delta_fair(max_slice, se);
864
865	return clamp(vlag, -limit, limit);
866	}
867
868	/*
869	* Delayed dequeue aims to reduce the negative lag of a dequeued task. While
870	* updating the lag of an entity, check that negative lag didn't increase
871	* during the delayed dequeue period which would be unfair.
872	* Similarly, check that the entity didn't gain positive lag when DELAY_ZERO
873	* is set.
874	*
875	* Return true if the vlag has been modified. Specifically:
876	*
877	* se->vlag != avg_vruntime() - se->vruntime
878	*
879	* This can be due to clamping in entity_lag() or clamping due to
880	* sched_delayed. Either way, when vlag is modified and the entity is
881	* retained, the tree needs to be adjusted.
882	*/
883	static __always_inline
884	bool update_entity_lag(struct cfs_rq cfs_rq, struct sched_entity se)
885	{
886	u64 avruntime = avg_vruntime(cfs_rq);
887	s64 vlag = entity_lag(cfs_rq, se, avruntime);
888
889	if (se->sched_delayed) {
890	/* previous vlag < 0 otherwise se would not be delayed */
891	vlag = max(vlag, se->vlag);
892	if (sched_feat(DELAY_ZERO))
893	vlag = min(vlag, 0);
894	}
895	se->vlag = vlag;
896
897	return avruntime - vlag != se->vruntime;
898	}
899
900	/*
901	* Entity is eligible once it received less service than it ought to have,
902	* eg. lag >= 0.
903	*
904	* lag_i = S - s_i = w_i*(V - v_i)
905	*
906	* lag_i >= 0 -> V >= v_i
907	*
908	* \Sum (v_i - v0)*w_i
909	* V = ------------------- + v0
910	* \Sum w_i
911	*
912	* lag_i >= 0 -> \Sum (v_i - v0)w_i >= (v_i - v0)(\Sum w_i)
913	*
914	* Note: using 'avg_vruntime() > se->vruntime' is inaccurate due
915	* to the loss in precision caused by the division.
916	*/
917	static int vruntime_eligible(struct cfs_rq *cfs_rq, u64 vruntime)
918	{
919	struct sched_entity *curr = cfs_rq->curr;
920	s64 key, avg = cfs_rq->sum_w_vruntime;
921	long load = cfs_rq->sum_weight;
922
923	if (curr && curr->on_rq) {
924	unsigned long weight = avg_vruntime_weight(cfs_rq, curr->h_load.weight);
925
926	avg += entity_key(cfs_rq, curr) * weight;
927	load += weight;
928	}
929
930	key = vruntime_op(vruntime, "-", cfs_rq->zero_vruntime);
931
932	/*
933	* The worst case term for @key includes 'NSEC_TICK * NICE_0_LOAD'
934	* and @load obviously includes NICE_0_LOAD. NSEC_TICK is around 24
935	* bits, while NICE_0_LOAD is 20 on 64bit and 10 otherwise.
936	*
937	* This gives that on 64bit the product will be at least 64bit which
938	* overflows s64, while on 32bit it will only be 44bits and should fit
939	* comfortably.
940	*/
941	#ifdef CONFIG_64BIT
942	#ifdef CONFIG_ARCH_SUPPORTS_INT128
943	/* This often results in simpler code than __builtin_mul_overflow(). */
944	return avg >= (__int128)key * load;
945	#else
946	s64 rhs;
947	/*
948	* On overflow, the sign of key tells us the correct answer: a large
949	* positive key means vruntime >> V, so not eligible; a large negative
950	* key means vruntime << V, so eligible.
951	*/
952	if (check_mul_overflow(key, load, &rhs))
953	return key <= 0;
954
955	return avg >= rhs;
956	#endif
957	#else /* 32bit */
958	return avg >= key * load;
959	#endif
960	}
961
962	int entity_eligible(struct cfs_rq cfs_rq, struct sched_entity se)
963	{
964	return vruntime_eligible(cfs_rq, se->vruntime);
965	}
966
967	static inline u64 cfs_rq_min_slice(struct cfs_rq *cfs_rq)
968	{
969	struct sched_entity *root = __pick_root_entity(cfs_rq);
970	struct sched_entity *curr = cfs_rq->curr;
971	u64 min_slice = ~0ULL;
972
973	if (curr && curr->on_rq)
974	min_slice = curr->slice;
975
976	if (root)
977	min_slice = min(min_slice, root->min_slice);
978
979	return min_slice;
980	}
981
982	static inline u64 cfs_rq_max_slice(struct cfs_rq *cfs_rq)
983	{
984	struct sched_entity *root = __pick_root_entity(cfs_rq);
985	struct sched_entity *curr = cfs_rq->curr;
986	u64 max_slice = 0ULL;
987
988	if (curr && curr->on_rq)
989	max_slice = curr->slice;
990
991	if (root)
992	max_slice = max(max_slice, root->max_slice);
993
994	return max_slice;
995	}
996
997	static inline bool __entity_less(struct rb_node a, const struct rb_node b)
998	{
999	return entity_before(__node_2_se(a), __node_2_se(b));
1000	}
1001
1002	/* min() for wrapping vruntimes */
1003	static inline u64 __min_vruntime(u64 a, u64 b)
1004	{
1005	return vruntime_cmp(a, "<", b) ? a : b;
1006	}
1007
1008	RB_DECLARE_CALLBACKS(static, min_vruntime_cb,
1009	struct sched_entity, run_node,
1010	RB_AUG(vruntime, min_vruntime, __min_vruntime),
1011	RB_AUG(slice, min_slice, min),
1012	RB_AUG(slice, max_slice, max));
1013
1014	/*
1015	* Enqueue an entity into the rb-tree:
1016	*/
1017	static void __enqueue_entity(struct cfs_rq cfs_rq, struct sched_entity se)
1018	{
1019	WARN_ON_ONCE(&rq_of(cfs_rq)->cfs != cfs_rq);
1020	WARN_ON_ONCE(!entity_is_task(se));
1021
1022	sum_w_vruntime_add(cfs_rq, se);
1023	se->min_vruntime = se->vruntime;
1024	se->min_slice = se->slice;
1025	se->max_slice = se->slice;
1026
1027	rb_add_augmented_cached(&se->run_node, &cfs_rq->tasks_timeline,
1028	__entity_less, &min_vruntime_cb);
1029	}
1030
1031	static void __dequeue_entity(struct cfs_rq cfs_rq, struct sched_entity se)
1032	{
1033	WARN_ON_ONCE(&rq_of(cfs_rq)->cfs != cfs_rq);
1034	WARN_ON_ONCE(!entity_is_task(se));
1035
1036	rb_erase_augmented_cached(&se->run_node, &cfs_rq->tasks_timeline,
1037	&min_vruntime_cb);
1038	sum_w_vruntime_sub(cfs_rq, se);
1039	}
1040
1041	struct sched_entity __pick_root_entity(struct cfs_rq cfs_rq)
1042	{
1043	struct rb_node *root = cfs_rq->tasks_timeline.rb_root.rb_node;
1044
1045	if (!root)
1046	return NULL;
1047
1048	return __node_2_se(root);
1049	}
1050
1051	struct sched_entity __pick_first_entity(struct cfs_rq cfs_rq)
1052	{
1053	struct rb_node *left = rb_first_cached(&cfs_rq->tasks_timeline);
1054
1055	if (!left)
1056	return NULL;
1057
1058	return __node_2_se(left);
1059	}
1060
1061	/*
1062	* Set the vruntime up to which an entity can run before looking
1063	* for another entity to pick.
1064	* In case of run to parity, we use the shortest slice of the enqueued
1065	* entities to set the protected period.
1066	* When run to parity is disabled, we give a minimum quantum to the running
1067	* entity to ensure progress.
1068	*/
1069	static inline void set_protect_slice(struct cfs_rq cfs_rq, struct sched_entity se)
1070	{
1071	u64 slice = normalized_sysctl_sched_base_slice;
1072	u64 vprot = se->deadline;
1073
1074	if (sched_feat(RUN_TO_PARITY))
1075	slice = cfs_rq_min_slice(cfs_rq);
1076
1077	slice = min(slice, se->slice);
1078
1079	/* If there are shorter slices than se's one */
1080	if (slice != se->slice) {
1081	vprot = min_vruntime(vprot, se->vruntime + calc_delta_fair(slice, se));
1082	if (sched_feat(PREEMPT_SHORT))
1083	vprot = min_vruntime(vprot, ineligible_vruntime(cfs_rq));
1084	}
1085
1086	se->vprot = vprot;
1087	}
1088
1089	static inline void update_protect_slice(struct cfs_rq cfs_rq, struct sched_entity se)
1090	{
1091	u64 vruntime = min_vruntime(se->vruntime, avg_vruntime(cfs_rq));
1092	u64 slice = cfs_rq_min_slice(cfs_rq);
1093	u64 vprot = min_vruntime(se->vprot, vruntime + calc_delta_fair(slice, se));
1094
1095	if (sched_feat(PREEMPT_SHORT) && slice != se->slice)
1096	vprot = min_vruntime(vprot, ineligible_vruntime(cfs_rq));
1097
1098	se->vprot = vprot;
1099	}
1100
1101	static inline bool protect_slice(struct sched_entity *se)
1102	{
1103	return vruntime_cmp(se->vruntime, "<", se->vprot);
1104	}
1105
1106	static inline void cancel_protect_slice(struct sched_entity *se)
1107	{
1108	if (protect_slice(se))
1109	se->vprot = se->vruntime;
1110	}
1111
1112	/*
1113	* Earliest Eligible Virtual Deadline First
1114	*
1115	* In order to provide latency guarantees for different request sizes
1116	* EEVDF selects the best runnable task from two criteria:
1117	*
1118	* 1) the task must be eligible (must be owed service)
1119	*
1120	* 2) from those tasks that meet 1), we select the one
1121	* with the earliest virtual deadline.
1122	*
1123	* We can do this in O(log n) time due to an augmented RB-tree. The
1124	* tree keeps the entries sorted on deadline, but also functions as a
1125	* heap based on the vruntime by keeping:
1126	*
1127	* se->min_vruntime = min(se->vruntime, se->{left,right}->min_vruntime)
1128	*
1129	* Which allows tree pruning through eligibility.
1130	*/
1131	static struct sched_entity pick_eevdf(struct cfs_rq cfs_rq, bool protect)
1132	{
1133	struct rb_node *node = cfs_rq->tasks_timeline.rb_root.rb_node;
1134	struct sched_entity *se = __pick_first_entity(cfs_rq);
1135	struct sched_entity *curr = cfs_rq->curr;
1136	struct sched_entity *best = NULL;
1137
1138	/*
1139	* We can safely skip eligibility check if there is only one entity
1140	* in this cfs_rq, saving some cycles.
1141	*/
1142	if (cfs_rq->h_nr_queued == 1)
1143	return curr && curr->on_rq ? curr : se;
1144
1145	/*
1146	* Picking the ->next buddy will affect latency but not fairness.
1147	*/
1148	if (sched_feat(PICK_BUDDY) && protect &&
1149	cfs_rq->next && entity_eligible(cfs_rq, cfs_rq->next)) {
1150	/* ->next will never be delayed */
1151	WARN_ON_ONCE(cfs_rq->next->sched_delayed);
1152	return cfs_rq->next;
1153	}
1154
1155	if (curr && (!curr->on_rq \|\| !entity_eligible(cfs_rq, curr)))
1156	curr = NULL;
1157
1158	if (curr && protect && protect_slice(curr))
1159	return curr;
1160
1161	/* Pick the leftmost entity if it's eligible */
1162	if (se && entity_eligible(cfs_rq, se)) {
1163	best = se;
1164	goto found;
1165	}
1166
1167	/* Heap search for the EEVD entity */
1168	while (node) {
1169	struct rb_node *left = node->rb_left;
1170
1171	/*
1172	* Eligible entities in left subtree are always better
1173	* choices, since they have earlier deadlines.
1174	*/
1175	if (left && vruntime_eligible(cfs_rq,
1176	__node_2_se(left)->min_vruntime)) {
1177	node = left;
1178	continue;
1179	}
1180
1181	se = __node_2_se(node);
1182
1183	/*
1184	* The left subtree either is empty or has no eligible
1185	* entity, so check the current node since it is the one
1186	* with earliest deadline that might be eligible.
1187	*/
1188	if (entity_eligible(cfs_rq, se)) {
1189	best = se;
1190	break;
1191	}
1192
1193	node = node->rb_right;
1194	}
1195	found:
1196	if (!best \|\| (curr && entity_before(curr, best)))
1197	best = curr;
1198
1199	return best;
1200	}
1201
1202	struct sched_entity __pick_last_entity(struct cfs_rq cfs_rq)
1203	{
1204	struct rb_node *last = rb_last(&cfs_rq->tasks_timeline.rb_root);
1205
1206	if (!last)
1207	return NULL;
1208
1209	return __node_2_se(last);
1210	}
1211
1212	/**************************************************************
1213	* Scheduling class statistics methods:
1214	*/
1215	int sched_update_scaling(void)
1216	{
1217	unsigned int factor = get_update_sysctl_factor();
1218
1219	#define WRT_SYSCTL(name) \
1220	(normalized_sysctl_##name = sysctl_##name / (factor))
1221	WRT_SYSCTL(sched_base_slice);
1222	#undef WRT_SYSCTL
1223
1224	return 0;
1225	}
1226
1227	static void clear_buddies(struct cfs_rq cfs_rq, struct sched_entity se);
1228
1229	/*
1230	* XXX: strictly: vd_i += N*r_i/w_i such that: vd_i > ve_i
1231	* this is probably good enough.
1232	*/
1233	static bool update_deadline(struct cfs_rq cfs_rq, struct sched_entity se)
1234	{
1235	if (vruntime_cmp(se->vruntime, "<", se->deadline))
1236	return false;
1237
1238	/*
1239	* For EEVDF the virtual time slope is determined by w_i (iow.
1240	* nice) while the request time r_i is determined by
1241	* sysctl_sched_base_slice.
1242	*/
1243	if (!se->custom_slice)
1244	se->slice = sysctl_sched_base_slice;
1245
1246	/*
1247	* EEVDF: vd_i = ve_i + r_i / w_i
1248	*/
1249	se->deadline = se->vruntime + calc_delta_fair(se->slice, se);
1250	avg_vruntime(cfs_rq);
1251
1252	/*
1253	* The task has consumed its request, reschedule.
1254	*/
1255	return true;
1256	}
1257
1258	#include "pelt.h"
1259
1260	static int select_idle_sibling(struct task_struct *p, int prev_cpu, int cpu);
1261	static unsigned long task_h_load(struct task_struct *p);
1262	static unsigned long capacity_of(int cpu);
1263
1264	/* Give new sched_entity start runnable values to heavy its load in infant time */
1265	void init_entity_runnable_average(struct sched_entity *se)
1266	{
1267	struct sched_avg *sa = &se->avg;
1268
1269	memset(sa, 0, sizeof(*sa));
1270
1271	/*
1272	* Tasks are initialized with full load to be seen as heavy tasks until
1273	* they get a chance to stabilize to their real load level.
1274	* Group entities are initialized with zero load to reflect the fact that
1275	* nothing has been attached to the task group yet.
1276	*/
1277	if (entity_is_task(se))
1278	sa->load_avg = scale_load_down(se->load.weight);
1279
1280	/* when this task is enqueued, it will contribute to its cfs_rq's load_avg */
1281	}
1282
1283	/*
1284	* With new tasks being created, their initial util_avgs are extrapolated
1285	* based on the cfs_rq's current util_avg:
1286	*
1287	* util_avg = cfs_rq->avg.util_avg / (cfs_rq->avg.load_avg + 1)
1288	* * se_weight(se)
1289	*
1290	* However, in many cases, the above util_avg does not give a desired
1291	* value. Moreover, the sum of the util_avgs may be divergent, such
1292	* as when the series is a harmonic series.
1293	*
1294	* To solve this problem, we also cap the util_avg of successive tasks to
1295	* only 1/2 of the left utilization budget:
1296	*
1297	* util_avg_cap = (cpu_scale - cfs_rq->avg.util_avg) / 2^n
1298	*
1299	* where n denotes the nth task and cpu_scale the CPU capacity.
1300	*
1301	* For example, for a CPU with 1024 of capacity, a simplest series from
1302	* the beginning would be like:
1303	*
1304	* task util_avg: 512, 256, 128, 64, 32, 16, 8, ...
1305	* cfs_rq util_avg: 512, 768, 896, 960, 992, 1008, 1016, ...
1306	*
1307	* Finally, that extrapolated util_avg is clamped to the cap (util_avg_cap)
1308	* if util_avg > util_avg_cap.
1309	*/
1310	void post_init_entity_util_avg(struct task_struct *p)
1311	{
1312	struct sched_entity *se = &p->se;
1313	struct cfs_rq *cfs_rq = cfs_rq_of(se);
1314	struct sched_avg *sa = &se->avg;
1315	long cpu_scale = arch_scale_cpu_capacity(cpu_of(rq_of(cfs_rq)));
1316	long cap = (long)(cpu_scale - cfs_rq->avg.util_avg) / 2;
1317
1318	if (p->sched_class != &fair_sched_class) {
1319	/*
1320	* For !fair tasks do:
1321	*
1322	update_cfs_rq_load_avg(now, cfs_rq);
1323	attach_entity_load_avg(cfs_rq, se);
1324	switched_from_fair(rq, p);
1325	*
1326	* such that the next switched_to_fair() has the
1327	* expected state.
1328	*/
1329	se->avg.last_update_time = cfs_rq_clock_pelt(cfs_rq);
1330	return;
1331	}
1332
1333	if (cap > 0) {
1334	if (cfs_rq->avg.util_avg != 0) {
1335	sa->util_avg = cfs_rq->avg.util_avg * se_weight(se);
1336	sa->util_avg /= (cfs_rq->avg.load_avg + 1);
1337
1338	if (sa->util_avg > cap)
1339	sa->util_avg = cap;
1340	} else {
1341	sa->util_avg = cap;
1342	}
1343	}
1344
1345	sa->runnable_avg = sa->util_avg;
1346	}
1347
1348	static inline void account_mm_sched(struct rq rq, struct task_struct p, s64 delta_exec);
1349
1350	static s64 update_se(struct rq rq, struct sched_entity se)
1351	{
1352	u64 now = rq_clock_task(rq);
1353	s64 delta_exec;
1354
1355	delta_exec = now - se->exec_start;
1356	if (unlikely(delta_exec <= 0))
1357	return delta_exec;
1358
1359	se->exec_start = now;
1360	if (entity_is_task(se)) {
1361	struct task_struct *running = rq->curr;
1362	/*
1363	* If se is a task, we account the time against the running
1364	* task, as w/ proxy-exec they may not be the same.
1365	*/
1366	running->se.exec_start = now;
1367	running->se.sum_exec_runtime += delta_exec;
1368
1369	trace_sched_stat_runtime(running, delta_exec);
1370	account_group_exec_runtime(running, delta_exec);
1371	account_mm_sched(rq, running, delta_exec);
1372
1373	cgroup_account_cputime(running, delta_exec);
1374	} else {
1375	/* If not task, account the time against donor se */
1376	se->sum_exec_runtime += delta_exec;
1377	}
1378
1379	if (schedstat_enabled()) {
1380	struct sched_statistics *stats;
1381
1382	stats = __schedstats_from_se(se);
1383	__schedstat_set(stats->exec_max,
1384	max(delta_exec, stats->exec_max));
1385	}
1386
1387	return delta_exec;
1388	}
1389
1390	#ifdef CONFIG_SCHED_CACHE
1391
1392	/*
1393	* XXX numbers come from a place the sun don't shine -- probably wants to be SD
1394	* tunable or so.
1395	*/
1396	#define EPOCH_PERIOD (HZ / 100) /* 10 ms */
1397	#define EPOCH_LLC_AFFINITY_TIMEOUT 5 /* 50 ms */
1398	__read_mostly unsigned int llc_aggr_tolerance = 1;
1399	__read_mostly unsigned int llc_epoch_period = EPOCH_PERIOD;
1400	__read_mostly unsigned int llc_epoch_affinity_timeout = EPOCH_LLC_AFFINITY_TIMEOUT;
1401	__read_mostly unsigned int llc_imb_pct = 20;
1402	__read_mostly unsigned int llc_overaggr_pct = 50;
1403
1404	static int llc_id(int cpu)
1405	{
1406	if (cpu < 0)
1407	return -1;
1408
1409	return per_cpu(sd_llc_id, cpu);
1410	}
1411
1412	static inline int get_sched_cache_scale(int mul)
1413	{
1414	unsigned int tol = READ_ONCE(llc_aggr_tolerance);
1415
1416	if (!tol)
1417	return 0;
1418
1419	if (tol >= 100)
1420	return INT_MAX;
1421
1422	return (1 + (tol - 1) * mul);
1423	}
1424
1425	static bool exceed_llc_capacity(struct sched_cache_group *grp, int cpu)
1426	{
1427	#ifdef CONFIG_NUMA_BALANCING
1428	unsigned long llc, footprint;
1429	struct sched_domain *sd;
1430	int scale;
1431
1432	guard(rcu)();
1433
1434	sd = rcu_dereference_sched_domain(cpu_rq(cpu)->sd);
1435	if (!sd)
1436	return true;
1437
1438	if (static_branch_likely(&sched_numa_balancing)) {
1439	/*
1440	* TBD: RDT exclusive LLC ways reserved should be
1441	* excluded.
1442	*/
1443	llc = sd->llc_bytes;
1444	footprint = READ_ONCE(grp->footprint);
1445
1446	/*
1447	* Scale the LLC size by 256*llc_aggr_tolerance
1448	* and compare it to the task's footprint.
1449	*
1450	* Suppose the L3 size is 32MB. If the
1451	* llc_aggr_tolerance is 1:
1452	* When the footprint is larger than 32MB, the
1453	* process is regarded as exceeding the LLC
1454	* capacity. If the llc_aggr_tolerance is 99:
1455	* When the footprint is larger than 784GB, the
1456	* process is regarded as exceeding the LLC
1457	* capacity:
1458	* 784GB = (1 + (99 - 1) * 256) * 32MB
1459	* If the llc_aggr_tolerance is 100:
1460	* ignore the footprint and do the aggregation
1461	* anyway.
1462	*/
1463	scale = get_sched_cache_scale(256);
1464	if (scale == INT_MAX)
1465	return false;
1466
1467	return ((llc * (u64)scale) < (footprint * PAGE_SIZE));
1468	}
1469	#endif
1470	return false;
1471	}
1472
1473	static bool invalid_llc_nr(struct sched_cache_group grp, struct task_struct p,
1474	int cpu)
1475	{
1476	int scale;
1477
1478	if (get_nr_threads(p) <= 1)
1479	return true;
1480
1481	/*
1482	* Scale the number of 'cores' in a LLC by llc_aggr_tolerance
1483	* and compare it to the task's active threads.
1484	*/
1485	scale = get_sched_cache_scale(1);
1486	if (scale == INT_MAX)
1487	return false;
1488
1489	return !fits_capacity((READ_ONCE(grp->nr_running_avg) * cpu_smt_num_threads),
1490	(scale * per_cpu(sd_llc_size, cpu)));
1491	}
1492
1493	/*
1494	* A task counts in nr_pref_llc_running while it is queued on its preferred
1495	* LLC (pref_llc_queued) and runnable (!sched_delayed), keeping the counter in
1496	* the runnable domain so alb_break_llc() can compare it with h_nr_runnable.
1497	*/
1498	static bool task_pref_llc_runnable(struct task_struct *p)
1499	{
1500	return p->pref_llc_queued && !p->se.sched_delayed;
1501	}
1502
1503	static void pref_llc_running_inc(struct rq rq, struct task_struct p)
1504	{
1505	if (task_pref_llc_runnable(p))
1506	rq->nr_pref_llc_running++;
1507	}
1508
1509	static void pref_llc_running_dec(struct rq rq, struct task_struct p)
1510	{
1511	if (task_pref_llc_runnable(p))
1512	rq->nr_pref_llc_running--;
1513	}
1514
1515	static void account_llc_enqueue(struct rq rq, struct task_struct p)
1516	{
1517	int pref_llc, pref_llc_queued;
1518	struct sched_domain *sd;
1519
1520	pref_llc = p->preferred_llc;
1521	if (pref_llc < 0)
1522	return;
1523
1524	pref_llc_queued = (pref_llc == task_llc(p));
1525	rq->nr_llc_running++;
1526
1527	/*
1528	* Record whether p is enqueued on its preferred
1529	* LLC, in order to pair with account_llc_dequeue()
1530	* to maintain a consistent nr_pref_llc_running per
1531	* runqueue.
1532	* This is necessary because a race condition exists:
1533	* after a task is enqueued on a runqueue, task_llc(p)
1534	* may change due to CPU hotplug. Therefore, checking
1535	* task_llc(p) to determine whether the task is being
1536	* dequeued from its preferred LLC is unreliable and
1537	* can cause inconsistent values - checking the
1538	* p->pref_llc_queued in account_llc_dequeue() would
1539	* be reliable.
1540	*/
1541	p->pref_llc_queued = pref_llc_queued;
1542
1543	/* Skipped while delayed; clear_delayed() adds it back on wake. */
1544	pref_llc_running_inc(rq, p);
1545
1546	sd = rcu_dereference_all(rq->sd);
1547	if (sd && (unsigned int)pref_llc < sd->llc_max)
1548	sd->llc_counts[pref_llc]++;
1549	}
1550
1551	static void account_llc_dequeue(struct rq rq, struct task_struct p)
1552	{
1553	struct sched_domain *sd;
1554	int pref_llc;
1555
1556	pref_llc = p->preferred_llc;
1557	if (pref_llc < 0)
1558	return;
1559
1560	rq->nr_llc_running--;
1561	if (p->pref_llc_queued) {
1562	/*
1563	* Skipped if still delayed (set_delayed() already removed it);
1564	* clearing pref_llc_queued below also stops clear_delayed()
1565	* from re-adding it.
1566	*/
1567	pref_llc_running_dec(rq, p);
1568	/*
1569	* Update the status in case
1570	* other logic might query
1571	* this.
1572	*/
1573	p->pref_llc_queued = 0;
1574	}
1575
1576	sd = rcu_dereference_all(rq->sd);
1577	if (sd && (unsigned int)pref_llc < sd->llc_max) {
1578	/*
1579	* There is a race condition between dequeue
1580	* and CPU hotplug. After a task has been enqueued
1581	* on CPUx, a CPU hotplug event occurs, and all online
1582	* CPUs (including CPUx) rebuild their sched_domains
1583	* and reset statistics to zero(including sd->llc_counts).
1584	* This can cause temporary undercount and we have to
1585	* check for such underflow in sd->llc_counts.
1586	*
1587	* This undercount is temporary and accurate accounting
1588	* will resume once the rq has a chance to be idle.
1589	*/
1590	if (sd->llc_counts[pref_llc])
1591	sd->llc_counts[pref_llc]--;
1592	}
1593	}
1594
1595	int mm_init_sched(struct mm_struct *mm,
1596	struct sched_cache_time __percpu *_pcpu_sched)
1597	{
1598	struct sched_cache_group *grp;
1599	unsigned long epoch = 0;
1600	int i;
1601
1602	grp = kzalloc_obj(*grp);
1603	if (!grp) {
1604	free_percpu(_pcpu_sched);
1605	mm->sched_cache_grp = NULL;
1606	return -ENOMEM;
1607	}
1608
1609	for_each_possible_cpu(i) {
1610	struct sched_cache_time *pcpu_sched = per_cpu_ptr(_pcpu_sched, i);
1611	struct rq *rq = cpu_rq(i);
1612
1613	pcpu_sched->runtime = 0;
1614	/* a slightly stale cpu epoch is acceptible */
1615	pcpu_sched->epoch = rq->cpu_epoch;
1616	epoch = rq->cpu_epoch;
1617	}
1618
1619	raw_spin_lock_init(&grp->lock);
1620	grp->epoch = epoch;
1621	grp->cpu = -1;
1622	grp->next_scan = jiffies;
1623	grp->nr_running_avg = 0;
1624	grp->footprint = 0;
1625	refcount_set(&grp->refcnt, 1);
1626	/*
1627	* The update to grp->pcpu_sched should not be reordered
1628	* before initialization to grp's other fields, in case
1629	* the readers may get invalid mm_sched_epoch, etc.
1630	*/
1631	smp_store_release(&grp->pcpu_sched, _pcpu_sched);
1632	/*
1633	* Publish the group last. Not every reader qualifies it by
1634	* grp->pcpu_sched - can_migrate_llc_task() only checks that the
1635	* pointer is non-NULL before reading grp->footprint and
1636	* grp->nr_running_avg - so a reachable group must already be
1637	* fully initialized.
1638	*/
1639	smp_store_release(&mm->sched_cache_grp, grp);
1640	return 0;
1641	}
1642
1643	static void sched_cache_group_free_rcu(struct rcu_head *rcu)
1644	{
1645	struct sched_cache_group *grp =
1646	container_of(rcu, struct sched_cache_group, rcu);
1647
1648	free_percpu(grp->pcpu_sched);
1649	kfree(grp);
1650	}
1651
1652	static void sched_cache_group_put(struct sched_cache_group *grp)
1653	{
1654	if (!grp \|\| !refcount_dec_and_test(&grp->refcnt))
1655	return;
1656
1657	call_rcu(&grp->rcu, sched_cache_group_free_rcu);
1658	}
1659
1660	DEFINE_FREE(sched_cache_group_put, struct sched_cache_group *,
1661	sched_cache_group_put(_T));
1662
1663	#define rcu_deref_sched_cache_grp(tsk) \
1664	rcu_dereference_check((tsk)->sched_cache_grp, (tsk) == current)
1665
1666	static struct sched_cache_group sched_cache_replace_grp(struct task_struct p,
1667	struct sched_cache_group *new)
1668	{
1669	struct sched_cache_group *old;
1670
1671	old = rcu_deref_sched_cache_grp(p);
1672	rcu_assign_pointer(p->sched_cache_grp, new);
1673
1674	return old;
1675	}
1676
1677	struct sched_cache_group sched_cache_group_get(struct sched_cache_group grp)
1678	{
1679	/*
1680	* refcount_inc_not_zero() is the acquire primitive for lockless
1681	* (RCU) lookups; plain refcount_inc() would scribble the count if
1682	* it already reached zero. Return NULL in that case.
1683	*/
1684	if (grp && !refcount_inc_not_zero(&grp->refcnt))
1685	grp = NULL;
1686
1687	return grp;
1688	}
1689
1690	struct sched_cache_group task_cache_group_get(struct task_struct p)
1691	{
1692	guard(rcu)();
1693	return sched_cache_group_get(rcu_dereference(p->sched_cache_grp));
1694	}
1695
1696	void sched_cache_fork(struct task_struct *p)
1697	{
1698	/*
1699	* The child takes its own reference on the mm's cache group, separate
1700	* from the reference held by the mm. @p is not yet visible to readers,
1701	* so a plain initializing store is enough.
1702	*/
1703	RCU_INIT_POINTER(p->sched_cache_grp,
1704	sched_cache_group_get(p->mm->sched_cache_grp));
1705	}
1706
1707	void sched_cache_fork_cleanup(struct task_struct *p)
1708	{
1709	/*
1710	* A fork that fails after sched_cache_fork() never reaches exit_mm(),
1711	* so drop the reference here. @p never became visible, so there are no
1712	* concurrent readers and the reference we hold keeps the group alive.
1713	*/
1714	sched_cache_group_put(rcu_access_pointer(p->sched_cache_grp));
1715	RCU_INIT_POINTER(p->sched_cache_grp, NULL);
1716	}
1717
1718	void sched_cache_exec_mmap(struct task_struct p, struct mm_struct mm)
1719	{
1720	struct sched_cache_group *old;
1721
1722	/*
1723	* Acquire the new reference before publishing the pointer, then drop
1724	* the old one. @p is current and the only writer of its own pointer.
1725	*/
1726	old = sched_cache_replace_grp(p, sched_cache_group_get(mm->sched_cache_grp));
1727	sched_cache_group_put(old);
1728	}
1729
1730	void sched_cache_exit_mm(struct task_struct *p)
1731	{
1732	struct sched_cache_group *grp = sched_cache_replace_grp(p, NULL);
1733
1734	#ifdef CONFIG_NUMA_BALANCING
1735	/*
1736	* Subtract this task's footprint from the group before dropping the
1737	* reference, so the group footprint converges as its threads exit.
1738	* Unlocked for performance; clamp to avoid underflow.
1739	*/
1740	if (grp && p->total_numa_faults) {
1741	unsigned long fp = READ_ONCE(grp->footprint);
1742	unsigned long sub = min(fp, p->total_numa_faults);
1743
1744	WRITE_ONCE(grp->footprint, fp - sub);
1745	}
1746	#endif
1747	sched_cache_group_put(grp);
1748	}
1749
1750	void mm_destroy_sched(struct mm_struct *mm)
1751	{
1752	sched_cache_group_put(mm->sched_cache_grp);
1753	mm->sched_cache_grp = NULL;
1754	}
1755
1756	/* because why would C be fully specified */
1757	static __always_inline void __shr_u64(u64 *val, unsigned int n)
1758	{
1759	if (n >= 64) {
1760	*val = 0;
1761	return;
1762	}
1763	*val >>= n;
1764	}
1765
1766	static inline void __update_mm_sched(struct rq *rq,
1767	struct sched_cache_time *pcpu_sched)
1768	{
1769	lockdep_assert_held(&rq->cpu_epoch_lock);
1770
1771	unsigned int period = max(READ_ONCE(llc_epoch_period), 1U);
1772	unsigned long n, now = jiffies;
1773	long delta = now - rq->cpu_epoch_next;
1774
1775	if (delta > 0) {
1776	n = (delta + period - 1) / period;
1777	rq->cpu_epoch += n;
1778	rq->cpu_epoch_next += n * period;
1779	__shr_u64(&rq->cpu_runtime, n);
1780	}
1781
1782	n = rq->cpu_epoch - pcpu_sched->epoch;
1783	if (n) {
1784	pcpu_sched->epoch += n;
1785	__shr_u64(&pcpu_sched->runtime, n);
1786	}
1787	}
1788
1789	static unsigned long fraction_mm_sched(struct rq *rq,
1790	struct sched_cache_time *pcpu_sched)
1791	{
1792	guard(raw_spinlock_irqsave)(&rq->cpu_epoch_lock);
1793
1794	__update_mm_sched(rq, pcpu_sched);
1795
1796	/*
1797	* Runtime is a geometric series (r=0.5) and as such will sum to twice
1798	* the accumulation period, this means the multiplcation here should
1799	* not overflow.
1800	*/
1801	return div64_u64(NICE_0_LOAD * pcpu_sched->runtime, rq->cpu_runtime + 1);
1802	}
1803
1804	static int get_pref_llc(struct task_struct p, struct sched_cache_group grp)
1805	{
1806	int mm_sched_llc = -1, mm_sched_cpu;
1807
1808	if (!grp)
1809	return -1;
1810
1811	mm_sched_cpu = READ_ONCE(grp->cpu);
1812	if (mm_sched_cpu != -1) {
1813	mm_sched_llc = llc_id(mm_sched_cpu);
1814
1815	#ifdef CONFIG_NUMA_BALANCING
1816	/*
1817	* Don't assign preferred LLC if it
1818	* conflicts with NUMA balancing.
1819	* This can happen when sched_setnuma() gets
1820	* called, however it is not much of an issue
1821	* because we expect account_mm_sched() to get
1822	* called fairly regularly -- at a higher rate
1823	* than sched_setnuma() at least -- and thus the
1824	* conflict only exists for a short period of time.
1825	*/
1826	if (static_branch_likely(&sched_numa_balancing) &&
1827	p->numa_preferred_nid >= 0 &&
1828	cpu_to_node(mm_sched_cpu) != p->numa_preferred_nid)
1829	mm_sched_llc = -1;
1830	#endif
1831	}
1832
1833	return mm_sched_llc;
1834	}
1835
1836	static unsigned int task_running_on_cpu(int cpu, struct task_struct *p);
1837
1838	static inline
1839	void account_mm_sched(struct rq rq, struct task_struct p, s64 delta_exec)
1840	{
1841	struct sched_cache_group *grp = rcu_dereference_all(p->sched_cache_grp);
1842	struct sched_cache_time *pcpu_sched;
1843	int mm_sched_llc = -1;
1844	unsigned long epoch;
1845
1846	if (!sched_cache_enabled())
1847	return;
1848
1849	if (p->sched_class != &fair_sched_class)
1850	return;
1851	/*
1852	* init_task, kthreads and user thread created
1853	* by user_mode_thread() don't have a cache group.
1854	* In theory a kernel thread does not have any valid
1855	* cache group, because sched_cache_fork() is not
1856	* invoked for a kernel thread - !grp should gate the
1857	* kernel thread. Use the PF_KTHREAD check explicitly
1858	* here for safety reasons, to guard against future
1859	* modifications and to pair with task_tick_cache().
1860	*/
1861	if (p->flags & PF_KTHREAD \|\| !grp \|\| !grp->pcpu_sched)
1862	return;
1863
1864	pcpu_sched = per_cpu_ptr(grp->pcpu_sched, cpu_of(rq));
1865
1866	scoped_guard (raw_spinlock, &rq->cpu_epoch_lock) {
1867	__update_mm_sched(rq, pcpu_sched);
1868	pcpu_sched->runtime += delta_exec;
1869	rq->cpu_runtime += delta_exec;
1870	epoch = rq->cpu_epoch;
1871	}
1872
1873	/*
1874	* If this process hasn't hit task_cache_work() for a while invalidate
1875	* its preferred state.
1876	*/
1877	if ((long)(epoch - READ_ONCE(grp->epoch)) > llc_epoch_affinity_timeout \|\|
1878	invalid_llc_nr(grp, p, cpu_of(rq)) \|\|
1879	exceed_llc_capacity(grp, cpu_of(rq))) {
1880	if (READ_ONCE(grp->cpu) != -1)
1881	WRITE_ONCE(grp->cpu, -1);
1882	}
1883
1884	mm_sched_llc = get_pref_llc(p, grp);
1885
1886	/* task not on rq accounted later in account_entity_enqueue() */
1887	if (task_running_on_cpu(rq->cpu, p) &&
1888	READ_ONCE(p->preferred_llc) != mm_sched_llc) {
1889	account_llc_dequeue(rq, p);
1890	WRITE_ONCE(p->preferred_llc, mm_sched_llc);
1891	account_llc_enqueue(rq, p);
1892	}
1893	}
1894
1895	static void task_tick_cache(struct rq rq, struct task_struct p)
1896	{
1897	struct sched_cache_group *grp = rcu_dereference_all(p->sched_cache_grp);
1898	struct callback_head *work = &p->cache_work;
1899	unsigned long epoch;
1900
1901	if (!sched_cache_enabled())
1902	return;
1903
1904	if (!grp \|\| p->flags & PF_KTHREAD \|\|
1905	!grp->pcpu_sched)
1906	return;
1907
1908	epoch = rq->cpu_epoch;
1909	/* avoid moving backwards */
1910	if (time_after_eq(grp->epoch, epoch))
1911	return;
1912
1913	guard(raw_spinlock)(&grp->lock);
1914
1915	if (work->next == work) {
1916	task_work_add(p, work, TWA_RESUME);
1917	WRITE_ONCE(grp->epoch, epoch);
1918	}
1919	}
1920
1921	static void get_scan_cpumasks(cpumask_var_t cpus, struct task_struct *p,
1922	struct sched_cache_group *grp)
1923	{
1924	#ifdef CONFIG_NUMA_BALANCING
1925	int cpu, curr_cpu, nid, pref_nid;
1926
1927	if (!static_branch_likely(&sched_numa_balancing))
1928	goto out;
1929
1930	cpu = READ_ONCE(grp->cpu);
1931	if (cpu != -1)
1932	nid = cpu_to_node(cpu);
1933	curr_cpu = task_cpu(p);
1934
1935	/*
1936	* Scanning in the preferred NUMA node is ideal. However, the NUMA
1937	* preferred node is per-task rather than per-process. It is possible
1938	* for different threads of the process to have distinct preferred
1939	* nodes; consequently, the process-wide preferred LLC may bounce
1940	* between different nodes. As a workaround, maintain the scan
1941	* CPU mask to also cover the process's current preferred LLC and the
1942	* current running node to mitigate the bouncing risk.
1943	* TBD: numa_group should be considered during task aggregation.
1944	*/
1945	pref_nid = p->numa_preferred_nid;
1946	/* honor the task's preferred node */
1947	if (pref_nid == NUMA_NO_NODE)
1948	goto out;
1949
1950	cpumask_or(cpus, cpus, cpumask_of_node(pref_nid));
1951
1952	/* honor the task's preferred LLC CPU */
1953	if (cpu != -1 && !cpumask_test_cpu(cpu, cpus) && nid != NUMA_NO_NODE)
1954	cpumask_or(cpus, cpus, cpumask_of_node(nid));
1955
1956	/* make sure the task's current running node is included */
1957	if (!cpumask_test_cpu(curr_cpu, cpus))
1958	cpumask_or(cpus, cpus, cpumask_of_node(cpu_to_node(curr_cpu)));
1959
1960	return;
1961
1962	out:
1963	#endif
1964	cpumask_copy(cpus, cpu_online_mask);
1965	}
1966
1967	static inline void update_avg_scale(u64 *avg, u64 sample)
1968	{
1969	int factor = per_cpu(sd_llc_size, raw_smp_processor_id());
1970	s64 diff = sample - *avg;
1971	u32 divisor;
1972
1973	/*
1974	* Scale the divisor based on the number of CPUs contained
1975	* in the LLC. This scaling ensures smaller LLC domains use
1976	* a smaller divisor to achieve more precise sensitivity to
1977	* changes in nr_running, while larger LLC domains are capped
1978	* at a maximum divisor of 8 which is the default smoothing
1979	* factor of EWMA in update_avg().
1980	*/
1981	divisor = clamp_t(u32, (factor >> 2), 2, 8);
1982	*avg += div64_s64(diff, divisor);
1983	}
1984
1985	static void task_cache_work(struct callback_head *work)
1986	{
1987	struct sched_cache_group *grp __free(sched_cache_group_put) = NULL;
1988	cpumask_var_t cpus __free(free_cpumask_var) = CPUMASK_VAR_NULL;
1989	int cpu, m_a_cpu = -1, nr_running = 0, curr_cpu;
1990	unsigned long next_scan, now = jiffies;
1991	struct task_struct p = current, cur;
1992	unsigned long curr_m_a_occ = 0;
1993	unsigned long m_a_occ = 0;
1994
1995	WARN_ON_ONCE(work != &p->cache_work);
1996
1997	work->next = work;
1998
1999	if (p->flags & PF_EXITING)
2000	return;
2001
2002	/*
2003	* A reference makes sure grp is not released by others. The rcu
2004	* lock can not be held till after zalloc_cpumask_var() below,
2005	* because the latter might sleep.
2006	*/
2007	grp = task_cache_group_get(p);
2008	if (!grp)
2009	return;
2010
2011	next_scan = READ_ONCE(grp->next_scan);
2012	if (time_before(now, next_scan))
2013	return;
2014
2015	/* only 1 thread is allowed to scan */
2016	if (!try_cmpxchg(&grp->next_scan, &next_scan,
2017	now + max_t(unsigned long,
2018	READ_ONCE(llc_epoch_period), 1)))
2019	return;
2020
2021	curr_cpu = task_cpu(p);
2022	if (invalid_llc_nr(grp, p, curr_cpu) \|\|
2023	exceed_llc_capacity(grp, curr_cpu)) {
2024	if (READ_ONCE(grp->cpu) != -1)
2025	WRITE_ONCE(grp->cpu, -1);
2026
2027	return;
2028	}
2029
2030	if (!zalloc_cpumask_var(&cpus, GFP_KERNEL))
2031	return;
2032
2033	scoped_guard (cpus_read_lock) {
2034	guard(rcu)();
2035
2036	get_scan_cpumasks(cpus, p, grp);
2037
2038	for_each_cpu(cpu, cpus) {
2039	/* XXX sched_cluster_active */
2040	struct sched_domain *sd = rcu_dereference_all(per_cpu(sd_llc, cpu));
2041	unsigned long occ, m_occ = 0, a_occ = 0;
2042	int m_cpu = -1, i;
2043
2044	if (!sd)
2045	continue;
2046
2047	for_each_cpu(i, sched_domain_span(sd)) {
2048	occ = fraction_mm_sched(cpu_rq(i),
2049	per_cpu_ptr(grp->pcpu_sched, i));
2050	a_occ += occ;
2051	if (occ > m_occ) {
2052	m_occ = occ;
2053	m_cpu = i;
2054	}
2055
2056	/*
2057	* rcu_access_pointer() is used because the
2058	* pointer is only compared, never dereferenced.
2059	*/
2060	cur = rcu_dereference_all(cpu_rq(i)->curr);
2061	if (cur && !(cur->flags & (PF_EXITING \| PF_KTHREAD)) &&
2062	rcu_access_pointer(cur->sched_cache_grp) == grp)
2063	nr_running++;
2064	}
2065
2066	/*
2067	* Compare the accumulated occupancy of each LLC. The
2068	* reason for using accumulated occupancy rather than average
2069	* per CPU occupancy is that it works better in asymmetric LLC
2070	* scenarios.
2071	* For example, if there are 2 threads in a 4CPU LLC and 3
2072	* threads in an 8CPU LLC, it might be better to choose the one
2073	* with 3 threads. However, this would not be the case if the
2074	* occupancy is divided by the number of CPUs in an LLC (i.e.,
2075	* if average per CPU occupancy is used).
2076	* Besides, NUMA balancing fault statistics behave similarly:
2077	* the total number of faults per node is compared rather than
2078	* the average number of faults per CPU. This strategy is also
2079	* followed here.
2080	*/
2081	if (a_occ > m_a_occ) {
2082	m_a_occ = a_occ;
2083	m_a_cpu = m_cpu;
2084	}
2085
2086	if (llc_id(cpu) == llc_id(READ_ONCE(grp->cpu)))
2087	curr_m_a_occ = a_occ;
2088
2089	cpumask_andnot(cpus, cpus, sched_domain_span(sd));
2090	}
2091	}
2092
2093	if (m_a_occ > (2 * curr_m_a_occ)) {
2094	/*
2095	* Avoid switching sched_cache_grp->cpu too fast.
2096	* The reason to choose 2X is because:
2097	* 1. It is better to keep the preferred LLC stable,
2098	* rather than changing it frequently and cause migrations
2099	* 2. 2X means the new preferred LLC has at least 1 more
2100	* busy CPU than the old one(200% vs 100%, eg)
2101	* 3. 2X is chosen based on test results, as it delivers
2102	* the optimal performance gain so far.
2103	*/
2104	WRITE_ONCE(grp->cpu, m_a_cpu);
2105	}
2106
2107	update_avg_scale(&grp->nr_running_avg, nr_running);
2108	}
2109
2110	void init_sched_mm(struct task_struct *p)
2111	{
2112	struct callback_head *work = &p->cache_work;
2113
2114	init_task_work(work, task_cache_work);
2115	work->next = work;
2116	/*
2117	* dup_task_struct() copies the parent's task_struct, including its
2118	* sched_cache_grp, for which the child holds no reference. Clear it
2119	* here - before copy_mm() runs - so the child never carries a
2120	* borrowed pointer that the fork error path would put.
2121	*/
2122	RCU_INIT_POINTER(p->sched_cache_grp, NULL);
2123	/*
2124	* Reset new task's preference to avoid
2125	* polluting account_llc_enqueue().
2126	*/
2127	p->preferred_llc = -1;
2128	p->pref_llc_queued = 0;
2129	}
2130
2131	#else /* CONFIG_SCHED_CACHE */
2132
2133	static inline void account_mm_sched(struct rq rq, struct task_struct p,
2134	s64 delta_exec) { }
2135
2136	void init_sched_mm(struct task_struct *p) { }
2137
2138	static void task_tick_cache(struct rq rq, struct task_struct p) { }
2139
2140	static inline int get_pref_llc(struct task_struct *p,
2141	struct mm_struct *mm)
2142	{
2143	return -1;
2144	}
2145
2146	static void account_llc_enqueue(struct rq rq, struct task_struct p) {}
2147
2148	static void account_llc_dequeue(struct rq rq, struct task_struct p) {}
2149
2150	static void pref_llc_running_inc(struct rq rq, struct task_struct p) {}
2151
2152	static void pref_llc_running_dec(struct rq rq, struct task_struct p) {}
2153
2154	#endif /* CONFIG_SCHED_CACHE */
2155
2156	/*
2157	* Used by other classes to account runtime.
2158	*/
2159	s64 update_curr_common(struct rq *rq)
2160	{
2161	return update_se(rq, &rq->donor->se);
2162	}
2163
2164	/*
2165	* Update the current task's runtime statistics.
2166	*/
2167	static void update_curr(struct cfs_rq *cfs_rq)
2168	{
2169	/*
2170	* Note: cfs_rq->curr corresponds to the task picked to
2171	* run (ie: rq->donor.se) which due to proxy-exec may
2172	* not necessarily be the actual task running
2173	* (rq->curr.se). This is easy to confuse!
2174	*/
2175	struct sched_entity *curr = cfs_rq->h_curr;
2176	struct rq *rq = rq_of(cfs_rq);
2177	s64 delta_exec;
2178	bool resched;
2179
2180	if (unlikely(!curr))
2181	return;
2182
2183	delta_exec = update_se(rq, curr);
2184	if (unlikely(delta_exec <= 0))
2185	return;
2186
2187	account_cfs_rq_runtime(cfs_rq, delta_exec);
2188
2189	if (!entity_is_task(curr))
2190	return;
2191
2192	cfs_rq = &rq->cfs;
2193
2194	curr->vruntime += calc_delta_fair(delta_exec, curr);
2195	resched = update_deadline(cfs_rq, curr);
2196
2197	/*
2198	* If the fair_server is active, we need to account for the
2199	* fair_server time whether or not the task is running on
2200	* behalf of fair_server or not:
2201	* - If the task is running on behalf of fair_server, we need
2202	* to limit its time based on the assigned runtime.
2203	* - Fair task that runs outside of fair_server should account
2204	* against fair_server such that it can account for this time
2205	* and possibly avoid running this period.
2206	*/
2207	dl_server_update(&rq->fair_server, delta_exec);
2208
2209	if (cfs_rq->h_nr_queued == 1)
2210	return;
2211
2212	if (resched \|\| !protect_slice(curr)) {
2213	resched_curr_lazy(rq);
2214	clear_buddies(cfs_rq, curr);
2215	}
2216	}
2217
2218	static void update_curr_fair(struct rq *rq)
2219	{
2220	struct sched_entity *se = &rq->donor->se;
2221
2222	for_each_sched_entity(se)
2223	update_curr(cfs_rq_of(se));
2224	}
2225
2226	static inline void
2227	update_stats_wait_start_fair(struct cfs_rq cfs_rq, struct sched_entity se)
2228	{
2229	struct sched_statistics *stats;
2230	struct task_struct *p = NULL;
2231
2232	if (!schedstat_enabled())
2233	return;
2234
2235	stats = __schedstats_from_se(se);
2236
2237	if (entity_is_task(se))
2238	p = task_of(se);
2239
2240	__update_stats_wait_start(rq_of(cfs_rq), p, stats);
2241	}
2242
2243	static inline void
2244	update_stats_wait_end_fair(struct cfs_rq cfs_rq, struct sched_entity se)
2245	{
2246	struct sched_statistics *stats;
2247	struct task_struct *p = NULL;
2248
2249	if (!schedstat_enabled())
2250	return;
2251
2252	stats = __schedstats_from_se(se);
2253
2254	/*
2255	* When the sched_schedstat changes from 0 to 1, some sched se
2256	* maybe already in the runqueue, the se->statistics.wait_start
2257	* will be 0.So it will let the delta wrong. We need to avoid this
2258	* scenario.
2259	*/
2260	if (unlikely(!schedstat_val(stats->wait_start)))
2261	return;
2262
2263	if (entity_is_task(se))
2264	p = task_of(se);
2265
2266	__update_stats_wait_end(rq_of(cfs_rq), p, stats);
2267	}
2268
2269	static inline void
2270	update_stats_enqueue_sleeper_fair(struct cfs_rq cfs_rq, struct sched_entity se)
2271	{
2272	struct sched_statistics *stats;
2273	struct task_struct *tsk = NULL;
2274
2275	if (!schedstat_enabled())
2276	return;
2277
2278	stats = __schedstats_from_se(se);
2279
2280	if (entity_is_task(se))
2281	tsk = task_of(se);
2282
2283	__update_stats_enqueue_sleeper(rq_of(cfs_rq), tsk, stats);
2284	}
2285
2286	/*
2287	* Task is being enqueued - update stats:
2288	*/
2289	static inline void
2290	update_stats_enqueue_fair(struct cfs_rq cfs_rq, struct sched_entity se, int flags)
2291	{
2292	if (!schedstat_enabled())
2293	return;
2294
2295	/*
2296	* Are we enqueueing a waiting task? (for current tasks
2297	* a dequeue/enqueue event is a NOP)
2298	*/
2299	if (se != cfs_rq->h_curr)
2300	update_stats_wait_start_fair(cfs_rq, se);
2301
2302	if (flags & ENQUEUE_WAKEUP)
2303	update_stats_enqueue_sleeper_fair(cfs_rq, se);
2304	}
2305
2306	static inline void
2307	update_stats_dequeue_fair(struct cfs_rq cfs_rq, struct sched_entity se, int flags)
2308	{
2309
2310	if (!schedstat_enabled())
2311	return;
2312
2313	/*
2314	* Mark the end of the wait period if dequeueing a
2315	* waiting task:
2316	*/
2317	if (se != cfs_rq->h_curr)
2318	update_stats_wait_end_fair(cfs_rq, se);
2319
2320	if ((flags & DEQUEUE_SLEEP) && entity_is_task(se)) {
2321	struct task_struct *tsk = task_of(se);
2322	unsigned int state;
2323
2324	/* XXX racy against TTWU */
2325	state = READ_ONCE(tsk->__state);
2326	if (state & TASK_INTERRUPTIBLE)
2327	__schedstat_set(tsk->stats.sleep_start,
2328	rq_clock(rq_of(cfs_rq)));
2329	if (state & TASK_UNINTERRUPTIBLE)
2330	__schedstat_set(tsk->stats.block_start,
2331	rq_clock(rq_of(cfs_rq)));
2332	}
2333	}
2334
2335	/*
2336	* We are picking a new current task - update its stats:
2337	*/
2338	static inline void
2339	update_stats_curr_start(struct cfs_rq cfs_rq, struct sched_entity se)
2340	{
2341	/*
2342	* We are starting a new run period:
2343	*/
2344	se->exec_start = rq_clock_task(rq_of(cfs_rq));
2345	}
2346
2347	/* Check sched_smt_active before calling this to avoid overheads in fastpaths */
2348	static inline bool is_core_idle(int cpu)
2349	{
2350	int sibling;
2351
2352	for_each_cpu(sibling, cpu_smt_mask(cpu)) {
2353	if (cpu == sibling)
2354	continue;
2355
2356	if (!idle_cpu(sibling))
2357	return false;
2358	}
2359
2360	return true;
2361	}
2362
2363	#ifdef CONFIG_NUMA
2364	#define NUMA_IMBALANCE_MIN 2
2365
2366	static inline long
2367	adjust_numa_imbalance(int imbalance, int dst_running, int imb_numa_nr)
2368	{
2369	/*
2370	* Allow a NUMA imbalance if busy CPUs is less than the maximum
2371	* threshold. Above this threshold, individual tasks may be contending
2372	* for both memory bandwidth and any shared HT resources. This is an
2373	* approximation as the number of running tasks may not be related to
2374	* the number of busy CPUs due to sched_setaffinity.
2375	*/
2376	if (dst_running > imb_numa_nr)
2377	return imbalance;
2378
2379	/*
2380	* Allow a small imbalance based on a simple pair of communicating
2381	* tasks that remain local when the destination is lightly loaded.
2382	*/
2383	if (imbalance <= NUMA_IMBALANCE_MIN)
2384	return 0;
2385
2386	return imbalance;
2387	}
2388	#endif /* CONFIG_NUMA */
2389
2390	#ifdef CONFIG_NUMA_BALANCING
2391	/*
2392	* Approximate time to scan a full NUMA task in ms. The task scan period is
2393	* calculated based on the tasks virtual memory size and
2394	* numa_balancing_scan_size.
2395	*/
2396	unsigned int sysctl_numa_balancing_scan_period_min = 1000;
2397	unsigned int sysctl_numa_balancing_scan_period_max = 60000;
2398
2399	/* Portion of address space to scan in MB */
2400	unsigned int sysctl_numa_balancing_scan_size = 256;
2401
2402	/* Scan @scan_size MB every @scan_period after an initial @scan_delay in ms */
2403	unsigned int sysctl_numa_balancing_scan_delay = 1000;
2404
2405	/* The page with hint page fault latency < threshold in ms is considered hot */
2406	unsigned int sysctl_numa_balancing_hot_threshold = MSEC_PER_SEC;
2407
2408	struct numa_group {
2409	refcount_t refcount;
2410
2411	spinlock_t lock; /* nr_tasks, tasks */
2412	int nr_tasks;
2413	pid_t gid;
2414	int active_nodes;
2415
2416	struct rcu_head rcu;
2417	unsigned long total_faults;
2418	unsigned long max_faults_cpu;
2419	/*
2420	* faults[] array is split into two regions: faults_mem and faults_cpu.
2421	*
2422	* Faults_cpu is used to decide whether memory should move
2423	* towards the CPU. As a consequence, these stats are weighted
2424	* more by CPU use than by memory faults.
2425	*/
2426	unsigned long faults[];
2427	};
2428
2429	/*
2430	* For functions that can be called in multiple contexts that permit reading
2431	* ->numa_group (see struct task_struct for locking rules).
2432	*/
2433	static struct numa_group deref_task_numa_group(struct task_struct p)
2434	{
2435	return rcu_dereference_check(p->numa_group, p == current \|\|
2436	(lockdep_is_held(__rq_lockp(task_rq(p))) && !READ_ONCE(p->on_cpu)));
2437	}
2438
2439	static struct numa_group deref_curr_numa_group(struct task_struct p)
2440	{
2441	return rcu_dereference_protected(p->numa_group, p == current);
2442	}
2443
2444	static inline unsigned long group_faults_priv(struct numa_group *ng);
2445	static inline unsigned long group_faults_shared(struct numa_group *ng);
2446
2447	static unsigned int task_nr_scan_windows(struct task_struct *p)
2448	{
2449	unsigned long rss = 0;
2450	unsigned long nr_scan_pages;
2451
2452	/*
2453	* Calculations based on RSS as non-present and empty pages are skipped
2454	* by the PTE scanner and NUMA hinting faults should be trapped based
2455	* on resident pages
2456	*/
2457	nr_scan_pages = MB_TO_PAGES(sysctl_numa_balancing_scan_size);
2458	rss = get_mm_rss(p->mm);
2459	if (!rss)
2460	rss = nr_scan_pages;
2461
2462	rss = round_up(rss, nr_scan_pages);
2463	return rss / nr_scan_pages;
2464	}
2465
2466	/* For sanity's sake, never scan more PTEs than MAX_SCAN_WINDOW MB/sec. */
2467	#define MAX_SCAN_WINDOW 2560
2468
2469	static unsigned int task_scan_min(struct task_struct *p)
2470	{
2471	unsigned int scan_size = READ_ONCE(sysctl_numa_balancing_scan_size);
2472	unsigned int scan, floor;
2473	unsigned int windows = 1;
2474
2475	if (scan_size < MAX_SCAN_WINDOW)
2476	windows = MAX_SCAN_WINDOW / scan_size;
2477	floor = 1000 / windows;
2478
2479	scan = sysctl_numa_balancing_scan_period_min / task_nr_scan_windows(p);
2480	return max_t(unsigned int, floor, scan);
2481	}
2482
2483	static unsigned int task_scan_start(struct task_struct *p)
2484	{
2485	unsigned long smin = task_scan_min(p);
2486	unsigned long period = smin;
2487	struct numa_group *ng;
2488
2489	/* Scale the maximum scan period with the amount of shared memory. */
2490	rcu_read_lock();
2491	ng = rcu_dereference_all(p->numa_group);
2492	if (ng) {
2493	unsigned long shared = group_faults_shared(ng);
2494	unsigned long private = group_faults_priv(ng);
2495
2496	period *= refcount_read(&ng->refcount);
2497	period *= shared + 1;
2498	period /= private + shared + 1;
2499	}
2500	rcu_read_unlock();
2501
2502	return max(smin, period);
2503	}
2504
2505	static unsigned int task_scan_max(struct task_struct *p)
2506	{
2507	unsigned long smin = task_scan_min(p);
2508	unsigned long smax;
2509	struct numa_group *ng;
2510
2511	/* Watch for min being lower than max due to floor calculations */
2512	smax = sysctl_numa_balancing_scan_period_max / task_nr_scan_windows(p);
2513
2514	/* Scale the maximum scan period with the amount of shared memory. */
2515	ng = deref_curr_numa_group(p);
2516	if (ng) {
2517	unsigned long shared = group_faults_shared(ng);
2518	unsigned long private = group_faults_priv(ng);
2519	unsigned long period = smax;
2520
2521	period *= refcount_read(&ng->refcount);
2522	period *= shared + 1;
2523	period /= private + shared + 1;
2524
2525	smax = max(smax, period);
2526	}
2527
2528	return max(smin, smax);
2529	}
2530
2531	static void account_numa_enqueue(struct rq rq, struct task_struct p)
2532	{
2533	rq->nr_numa_running += (p->numa_preferred_nid != NUMA_NO_NODE);
2534	rq->nr_preferred_running += (p->numa_preferred_nid == task_node(p));
2535	}
2536
2537	static void account_numa_dequeue(struct rq rq, struct task_struct p)
2538	{
2539	rq->nr_numa_running -= (p->numa_preferred_nid != NUMA_NO_NODE);
2540	rq->nr_preferred_running -= (p->numa_preferred_nid == task_node(p));
2541	}
2542
2543	/* Shared or private faults. */
2544	#define NR_NUMA_HINT_FAULT_TYPES 2
2545
2546	/* Memory and CPU locality */
2547	#define NR_NUMA_HINT_FAULT_STATS (NR_NUMA_HINT_FAULT_TYPES * 2)
2548
2549	/* Averaged statistics, and temporary buffers. */
2550	#define NR_NUMA_HINT_FAULT_BUCKETS (NR_NUMA_HINT_FAULT_STATS * 2)
2551
2552	pid_t task_numa_group_id(struct task_struct *p)
2553	{
2554	struct numa_group *ng;
2555	pid_t gid = 0;
2556
2557	rcu_read_lock();
2558	ng = rcu_dereference_all(p->numa_group);
2559	if (ng)
2560	gid = ng->gid;
2561	rcu_read_unlock();
2562
2563	return gid;
2564	}
2565
2566	/*
2567	* The averaged statistics, shared & private, memory & CPU,
2568	* occupy the first half of the array. The second half of the
2569	* array is for current counters, which are averaged into the
2570	* first set by task_numa_placement.
2571	*/
2572	static inline int task_faults_idx(enum numa_faults_stats s, int nid, int priv)
2573	{
2574	return NR_NUMA_HINT_FAULT_TYPES * (s * nr_node_ids + nid) + priv;
2575	}
2576
2577	static inline unsigned long task_faults(struct task_struct *p, int nid)
2578	{
2579	if (!p->numa_faults)
2580	return 0;
2581
2582	return p->numa_faults[task_faults_idx(NUMA_MEM, nid, 0)] +
2583	p->numa_faults[task_faults_idx(NUMA_MEM, nid, 1)];
2584	}
2585
2586	static inline unsigned long group_faults(struct task_struct *p, int nid)
2587	{
2588	struct numa_group *ng = deref_task_numa_group(p);
2589
2590	if (!ng)
2591	return 0;
2592
2593	return ng->faults[task_faults_idx(NUMA_MEM, nid, 0)] +
2594	ng->faults[task_faults_idx(NUMA_MEM, nid, 1)];
2595	}
2596
2597	static inline unsigned long group_faults_cpu(struct numa_group *group, int nid)
2598	{
2599	return group->faults[task_faults_idx(NUMA_CPU, nid, 0)] +
2600	group->faults[task_faults_idx(NUMA_CPU, nid, 1)];
2601	}
2602
2603	static inline unsigned long group_faults_priv(struct numa_group *ng)
2604	{
2605	unsigned long faults = 0;
2606	int node;
2607
2608	for_each_online_node(node) {
2609	faults += ng->faults[task_faults_idx(NUMA_MEM, node, 1)];
2610	}
2611
2612	return faults;
2613	}
2614
2615	static inline unsigned long group_faults_shared(struct numa_group *ng)
2616	{
2617	unsigned long faults = 0;
2618	int node;
2619
2620	for_each_online_node(node) {
2621	faults += ng->faults[task_faults_idx(NUMA_MEM, node, 0)];
2622	}
2623
2624	return faults;
2625	}
2626
2627	/*
2628	* A node triggering more than 1/3 as many NUMA faults as the maximum is
2629	* considered part of a numa group's pseudo-interleaving set. Migrations
2630	* between these nodes are slowed down, to allow things to settle down.
2631	*/
2632	#define ACTIVE_NODE_FRACTION 3
2633
2634	static bool numa_is_active_node(int nid, struct numa_group *ng)
2635	{
2636	return group_faults_cpu(ng, nid) * ACTIVE_NODE_FRACTION > ng->max_faults_cpu;
2637	}
2638
2639	/* Handle placement on systems where not all nodes are directly connected. */
2640	static unsigned long score_nearby_nodes(struct task_struct *p, int nid,
2641	int lim_dist, bool task)
2642	{
2643	unsigned long score = 0;
2644	int node, max_dist;
2645
2646	/*
2647	* All nodes are directly connected, and the same distance
2648	* from each other. No need for fancy placement algorithms.
2649	*/
2650	if (sched_numa_topology_type == NUMA_DIRECT)
2651	return 0;
2652
2653	/* sched_max_numa_distance may be changed in parallel. */
2654	max_dist = READ_ONCE(sched_max_numa_distance);
2655	/*
2656	* This code is called for each node, introducing N^2 complexity,
2657	* which should be OK given the number of nodes rarely exceeds 8.
2658	*/
2659	for_each_online_node(node) {
2660	unsigned long faults;
2661	int dist = node_distance(nid, node);
2662
2663	/*
2664	* The furthest away nodes in the system are not interesting
2665	* for placement; nid was already counted.
2666	*/
2667	if (dist >= max_dist \|\| node == nid)
2668	continue;
2669
2670	/*
2671	* On systems with a backplane NUMA topology, compare groups
2672	* of nodes, and move tasks towards the group with the most
2673	* memory accesses. When comparing two nodes at distance
2674	* "hoplimit", only nodes closer by than "hoplimit" are part
2675	* of each group. Skip other nodes.
2676	*/
2677	if (sched_numa_topology_type == NUMA_BACKPLANE && dist >= lim_dist)
2678	continue;
2679
2680	/* Add up the faults from nearby nodes. */
2681	if (task)
2682	faults = task_faults(p, node);
2683	else
2684	faults = group_faults(p, node);
2685
2686	/*
2687	* On systems with a glueless mesh NUMA topology, there are
2688	* no fixed "groups of nodes". Instead, nodes that are not
2689	* directly connected bounce traffic through intermediate
2690	* nodes; a numa_group can occupy any set of nodes.
2691	* The further away a node is, the less the faults count.
2692	* This seems to result in good task placement.
2693	*/
2694	if (sched_numa_topology_type == NUMA_GLUELESS_MESH) {
2695	faults *= (max_dist - dist);
2696	faults /= (max_dist - LOCAL_DISTANCE);
2697	}
2698
2699	score += faults;
2700	}
2701
2702	return score;
2703	}
2704
2705	/*
2706	* These return the fraction of accesses done by a particular task, or
2707	* task group, on a particular numa node. The group weight is given a
2708	* larger multiplier, in order to group tasks together that are almost
2709	* evenly spread out between numa nodes.
2710	*/
2711	static inline unsigned long task_weight(struct task_struct *p, int nid,
2712	int dist)
2713	{
2714	unsigned long faults, total_faults;
2715
2716	if (!p->numa_faults)
2717	return 0;
2718
2719	total_faults = p->total_numa_faults;
2720
2721	if (!total_faults)
2722	return 0;
2723
2724	faults = task_faults(p, nid);
2725	faults += score_nearby_nodes(p, nid, dist, true);
2726
2727	return 1000 * faults / total_faults;
2728	}
2729
2730	static inline unsigned long group_weight(struct task_struct *p, int nid,
2731	int dist)
2732	{
2733	struct numa_group *ng = deref_task_numa_group(p);
2734	unsigned long faults, total_faults;
2735
2736	if (!ng)
2737	return 0;
2738
2739	total_faults = ng->total_faults;
2740
2741	if (!total_faults)
2742	return 0;
2743
2744	faults = group_faults(p, nid);
2745	faults += score_nearby_nodes(p, nid, dist, false);
2746
2747	return 1000 * faults / total_faults;
2748	}
2749
2750	/*
2751	* If memory tiering mode is enabled, cpupid of slow memory page is
2752	* used to record scan time instead of CPU and PID. When tiering mode
2753	* is disabled at run time, the scan time (in cpupid) will be
2754	* interpreted as CPU and PID. So CPU needs to be checked to avoid to
2755	* access out of array bound.
2756	*/
2757	static inline bool cpupid_valid(int cpupid)
2758	{
2759	return cpupid_to_cpu(cpupid) < nr_cpu_ids;
2760	}
2761
2762	/*
2763	* For memory tiering mode, if there are enough free pages (more than
2764	* enough watermark defined here) in fast memory node, to take full
2765	* advantage of fast memory capacity, all recently accessed slow
2766	* memory pages will be migrated to fast memory node without
2767	* considering hot threshold.
2768	*/
2769	static bool pgdat_free_space_enough(struct pglist_data *pgdat)
2770	{
2771	int z;
2772	unsigned long enough_wmark;
2773
2774	enough_wmark = max(1UL * 1024 * 1024 * 1024 >> PAGE_SHIFT,
2775	pgdat->node_present_pages >> 4);
2776	for (z = pgdat->nr_zones - 1; z >= 0; z--) {
2777	struct zone *zone = pgdat->node_zones + z;
2778
2779	if (!populated_zone(zone))
2780	continue;
2781
2782	if (zone_watermark_ok(zone, 0,
2783	promo_wmark_pages(zone) + enough_wmark,
2784	ZONE_MOVABLE, 0))
2785	return true;
2786	}
2787	return false;
2788	}
2789
2790	/*
2791	* For memory tiering mode, when page tables are scanned, the scan
2792	* time will be recorded in struct page in addition to make page
2793	* PROT_NONE for slow memory page. So when the page is accessed, in
2794	* hint page fault handler, the hint page fault latency is calculated
2795	* via,
2796	*
2797	* hint page fault latency = hint page fault time - scan time
2798	*
2799	* The smaller the hint page fault latency, the higher the possibility
2800	* for the page to be hot.
2801	*/
2802	static int numa_hint_fault_latency(struct folio *folio)
2803	{
2804	int last_time, time;
2805
2806	time = jiffies_to_msecs(jiffies);
2807	last_time = folio_xchg_access_time(folio, time);
2808
2809	return (time - last_time) & PAGE_ACCESS_TIME_MASK;
2810	}
2811
2812	/*
2813	* For memory tiering mode, too high promotion/demotion throughput may
2814	* hurt application latency. So we provide a mechanism to rate limit
2815	* the number of pages that are tried to be promoted.
2816	*/
2817	static bool numa_promotion_rate_limit(struct pglist_data *pgdat,
2818	unsigned long rate_limit, int nr)
2819	{
2820	unsigned long nr_cand;
2821	unsigned int now, start;
2822
2823	now = jiffies_to_msecs(jiffies);
2824	mod_node_page_state(pgdat, PGPROMOTE_CANDIDATE, nr);
2825	nr_cand = node_page_state(pgdat, PGPROMOTE_CANDIDATE);
2826	start = pgdat->nbp_rl_start;
2827	if (now - start > MSEC_PER_SEC &&
2828	cmpxchg(&pgdat->nbp_rl_start, start, now) == start)
2829	pgdat->nbp_rl_nr_cand = nr_cand;
2830	if (nr_cand - pgdat->nbp_rl_nr_cand >= rate_limit)
2831	return true;
2832	return false;
2833	}
2834
2835	#define NUMA_MIGRATION_ADJUST_STEPS 16
2836
2837	static void numa_promotion_adjust_threshold(struct pglist_data *pgdat,
2838	unsigned long rate_limit,
2839	unsigned int ref_th)
2840	{
2841	unsigned int now, start, th_period, unit_th, th;
2842	unsigned long nr_cand, ref_cand, diff_cand;
2843
2844	now = jiffies_to_msecs(jiffies);
2845	th_period = sysctl_numa_balancing_scan_period_max;
2846	start = pgdat->nbp_th_start;
2847	if (now - start > th_period &&
2848	cmpxchg(&pgdat->nbp_th_start, start, now) == start) {
2849	ref_cand = rate_limit *
2850	sysctl_numa_balancing_scan_period_max / MSEC_PER_SEC;
2851	nr_cand = node_page_state(pgdat, PGPROMOTE_CANDIDATE);
2852	diff_cand = nr_cand - pgdat->nbp_th_nr_cand;
2853	unit_th = ref_th * 2 / NUMA_MIGRATION_ADJUST_STEPS;
2854	th = pgdat->nbp_threshold ? : ref_th;
2855	if (diff_cand > ref_cand * 11 / 10)
2856	th = max(th - unit_th, unit_th);
2857	else if (diff_cand < ref_cand * 9 / 10)
2858	th = min(th + unit_th, ref_th * 2);
2859	pgdat->nbp_th_nr_cand = nr_cand;
2860	pgdat->nbp_threshold = th;
2861	}
2862	}
2863
2864	bool should_numa_migrate_memory(struct task_struct p, struct folio folio,
2865	int src_nid, int dst_cpu)
2866	{
2867	struct numa_group *ng = deref_curr_numa_group(p);
2868	int dst_nid = cpu_to_node(dst_cpu);
2869	int last_cpupid, this_cpupid;
2870
2871	/*
2872	* Cannot migrate to memoryless nodes.
2873	*/
2874	if (!node_state(dst_nid, N_MEMORY))
2875	return false;
2876
2877	/*
2878	* The pages in slow memory node should be migrated according
2879	* to hot/cold instead of private/shared.
2880	*/
2881	if (folio_use_access_time(folio)) {
2882	struct pglist_data *pgdat;
2883	unsigned long rate_limit;
2884	unsigned int latency, th, def_th;
2885	long nr = folio_nr_pages(folio);
2886
2887	pgdat = NODE_DATA(dst_nid);
2888	if (pgdat_free_space_enough(pgdat)) {
2889	/* workload changed, reset hot threshold */
2890	pgdat->nbp_threshold = 0;
2891	mod_node_page_state(pgdat, PGPROMOTE_CANDIDATE_NRL, nr);
2892	return true;
2893	}
2894
2895	def_th = sysctl_numa_balancing_hot_threshold;
2896	rate_limit = MB_TO_PAGES(sysctl_numa_balancing_promote_rate_limit);
2897	numa_promotion_adjust_threshold(pgdat, rate_limit, def_th);
2898
2899	th = pgdat->nbp_threshold ? : def_th;
2900	latency = numa_hint_fault_latency(folio);
2901	if (latency >= th)
2902	return false;
2903
2904	return !numa_promotion_rate_limit(pgdat, rate_limit, nr);
2905	}
2906
2907	this_cpupid = cpu_pid_to_cpupid(dst_cpu, current->pid);
2908	last_cpupid = folio_xchg_last_cpupid(folio, this_cpupid);
2909
2910	if (!(sysctl_numa_balancing_mode & NUMA_BALANCING_MEMORY_TIERING) &&
2911	!node_is_toptier(src_nid) && !cpupid_valid(last_cpupid))
2912	return false;
2913
2914	/*
2915	* Allow first faults or private faults to migrate immediately early in
2916	* the lifetime of a task. The magic number 4 is based on waiting for
2917	* two full passes of the "multi-stage node selection" test that is
2918	* executed below.
2919	*/
2920	if ((p->numa_preferred_nid == NUMA_NO_NODE \|\| p->numa_scan_seq <= 4) &&
2921	(cpupid_pid_unset(last_cpupid) \|\| cpupid_match_pid(p, last_cpupid)))
2922	return true;
2923
2924	/*
2925	* Multi-stage node selection is used in conjunction with a periodic
2926	* migration fault to build a temporal task<->page relation. By using
2927	* a two-stage filter we remove short/unlikely relations.
2928	*
2929	* Using P(p) ~ n_p / n_t as per frequentist probability, we can equate
2930	* a task's usage of a particular page (n_p) per total usage of this
2931	* page (n_t) (in a given time-span) to a probability.
2932	*
2933	* Our periodic faults will sample this probability and getting the
2934	* same result twice in a row, given these samples are fully
2935	* independent, is then given by P(n)^2, provided our sample period
2936	* is sufficiently short compared to the usage pattern.
2937	*
2938	* This quadric squishes small probabilities, making it less likely we
2939	* act on an unlikely task<->page relation.
2940	*/
2941	if (!cpupid_pid_unset(last_cpupid) &&
2942	cpupid_to_nid(last_cpupid) != dst_nid)
2943	return false;
2944
2945	/* Always allow migrate on private faults */
2946	if (cpupid_match_pid(p, last_cpupid))
2947	return true;
2948
2949	/* A shared fault, but p->numa_group has not been set up yet. */
2950	if (!ng)
2951	return true;
2952
2953	/*
2954	* Destination node is much more heavily used than the source
2955	* node? Allow migration.
2956	*/
2957	if (group_faults_cpu(ng, dst_nid) > group_faults_cpu(ng, src_nid) *
2958	ACTIVE_NODE_FRACTION)
2959	return true;
2960
2961	/*
2962	* Distribute memory according to CPU & memory use on each node,
2963	* with 3/4 hysteresis to avoid unnecessary memory migrations:
2964	*
2965	* faults_cpu(dst) 3 faults_cpu(src)
2966	* --------------- * - > ---------------
2967	* faults_mem(dst) 4 faults_mem(src)
2968	*/
2969	return group_faults_cpu(ng, dst_nid) * group_faults(p, src_nid) * 3 >
2970	group_faults_cpu(ng, src_nid) * group_faults(p, dst_nid) * 4;
2971	}
2972
2973	/*
2974	* 'numa_type' describes the node at the moment of load balancing.
2975	*/
2976	enum numa_type {
2977	/* The node has spare capacity that can be used to run more tasks. */
2978	node_has_spare = 0,
2979	/*
2980	* The node is fully used and the tasks don't compete for more CPU
2981	* cycles. Nevertheless, some tasks might wait before running.
2982	*/
2983	node_fully_busy,
2984	/*
2985	* The node is overloaded and can't provide expected CPU cycles to all
2986	* tasks.
2987	*/
2988	node_overloaded
2989	};
2990
2991	/* Cached statistics for all CPUs within a node */
2992	struct numa_stats {
2993	unsigned long load;
2994	unsigned long runnable;
2995	unsigned long util;
2996	/* Total compute capacity of CPUs on a node */
2997	unsigned long compute_capacity;
2998	unsigned int nr_running;
2999	unsigned int weight;
3000	enum numa_type node_type;
3001	int idle_cpu;
3002	};
3003
3004	struct task_numa_env {
3005	struct task_struct *p;
3006
3007	int src_cpu, src_nid;
3008	int dst_cpu, dst_nid;
3009	int imb_numa_nr;
3010
3011	struct numa_stats src_stats, dst_stats;
3012
3013	int imbalance_pct;
3014	int dist;
3015
3016	struct task_struct *best_task;
3017	long best_imp;
3018	int best_cpu;
3019	};
3020
3021	static unsigned long cpu_load(struct rq *rq);
3022	static unsigned long cpu_runnable(struct rq *rq);
3023
3024	static inline enum
3025	numa_type numa_classify(unsigned int imbalance_pct,
3026	struct numa_stats *ns)
3027	{
3028	if ((ns->nr_running > ns->weight) &&
3029	(((ns->compute_capacity * 100) < (ns->util * imbalance_pct)) \|\|
3030	((ns->compute_capacity * imbalance_pct) < (ns->runnable * 100))))
3031	return node_overloaded;
3032
3033	if ((ns->nr_running < ns->weight) \|\|
3034	(((ns->compute_capacity * 100) > (ns->util * imbalance_pct)) &&
3035	((ns->compute_capacity * imbalance_pct) > (ns->runnable * 100))))
3036	return node_has_spare;
3037
3038	return node_fully_busy;
3039	}
3040
3041	/* Forward declarations of select_idle_sibling helpers */
3042	static inline bool test_idle_cores(int cpu);
3043	static inline int numa_idle_core(int idle_core, int cpu)
3044	{
3045	if (!sched_smt_active() \|\|
3046	idle_core >= 0 \|\| !test_idle_cores(cpu))
3047	return idle_core;
3048
3049	/*
3050	* Prefer cores instead of packing HT siblings
3051	* and triggering future load balancing.
3052	*/
3053	if (is_core_idle(cpu))
3054	idle_core = cpu;
3055
3056	return idle_core;
3057	}
3058
3059	/*
3060	* Gather all necessary information to make NUMA balancing placement
3061	* decisions that are compatible with standard load balancer. This
3062	* borrows code and logic from update_sg_lb_stats but sharing a
3063	* common implementation is impractical.
3064	*/
3065	static void update_numa_stats(struct task_numa_env *env,
3066	struct numa_stats *ns, int nid,
3067	bool find_idle)
3068	{
3069	int cpu, idle_core = -1;
3070
3071	memset(ns, 0, sizeof(*ns));
3072	ns->idle_cpu = -1;
3073
3074	rcu_read_lock();
3075	for_each_cpu(cpu, cpumask_of_node(nid)) {
3076	struct rq *rq = cpu_rq(cpu);
3077
3078	ns->load += cpu_load(rq);
3079	ns->runnable += cpu_runnable(rq);
3080	ns->util += cpu_util_cfs(cpu);
3081	ns->nr_running += rq->cfs.h_nr_runnable;
3082	ns->compute_capacity += capacity_of(cpu);
3083
3084	if (find_idle && idle_core < 0 && !rq->nr_running && idle_cpu(cpu)) {
3085	if (READ_ONCE(rq->numa_migrate_on) \|\|
3086	!cpumask_test_cpu(cpu, env->p->cpus_ptr))
3087	continue;
3088
3089	if (ns->idle_cpu == -1)
3090	ns->idle_cpu = cpu;
3091
3092	idle_core = numa_idle_core(idle_core, cpu);
3093	}
3094	}
3095	rcu_read_unlock();
3096
3097	ns->weight = cpumask_weight(cpumask_of_node(nid));
3098
3099	ns->node_type = numa_classify(env->imbalance_pct, ns);
3100
3101	if (idle_core >= 0)
3102	ns->idle_cpu = idle_core;
3103	}
3104
3105	static void task_numa_assign(struct task_numa_env *env,
3106	struct task_struct *p, long imp)
3107	{
3108	struct rq *rq = cpu_rq(env->dst_cpu);
3109
3110	/* Check if run-queue part of active NUMA balance. */
3111	if (env->best_cpu != env->dst_cpu && xchg(&rq->numa_migrate_on, 1)) {
3112	int cpu;
3113	int start = env->dst_cpu;
3114
3115	/* Find alternative idle CPU. */
3116	for_each_cpu_wrap(cpu, cpumask_of_node(env->dst_nid), start + 1) {
3117	if (cpu == env->best_cpu \|\| !idle_cpu(cpu) \|\|
3118	!cpumask_test_cpu(cpu, env->p->cpus_ptr)) {
3119	continue;
3120	}
3121
3122	env->dst_cpu = cpu;
3123	rq = cpu_rq(env->dst_cpu);
3124	if (!xchg(&rq->numa_migrate_on, 1))
3125	goto assign;
3126	}
3127
3128	/* Failed to find an alternative idle CPU */
3129	return;
3130	}
3131
3132	assign:
3133	/*
3134	* Clear previous best_cpu/rq numa-migrate flag, since task now
3135	* found a better CPU to move/swap.
3136	*/
3137	if (env->best_cpu != -1 && env->best_cpu != env->dst_cpu) {
3138	rq = cpu_rq(env->best_cpu);
3139	WRITE_ONCE(rq->numa_migrate_on, 0);
3140	}
3141
3142	if (env->best_task)
3143	put_task_struct(env->best_task);
3144	if (p)
3145	get_task_struct(p);
3146
3147	env->best_task = p;
3148	env->best_imp = imp;
3149	env->best_cpu = env->dst_cpu;
3150	}
3151
3152	static bool load_too_imbalanced(long src_load, long dst_load,
3153	struct task_numa_env *env)
3154	{
3155	long imb, old_imb;
3156	long orig_src_load, orig_dst_load;
3157	long src_capacity, dst_capacity;
3158
3159	/*
3160	* The load is corrected for the CPU capacity available on each node.
3161	*
3162	* src_load dst_load
3163	* ------------ vs ---------
3164	* src_capacity dst_capacity
3165	*/
3166	src_capacity = env->src_stats.compute_capacity;
3167	dst_capacity = env->dst_stats.compute_capacity;
3168
3169	imb = abs(dst_load * src_capacity - src_load * dst_capacity);
3170
3171	orig_src_load = env->src_stats.load;
3172	orig_dst_load = env->dst_stats.load;
3173
3174	old_imb = abs(orig_dst_load * src_capacity - orig_src_load * dst_capacity);
3175
3176	/* Would this change make things worse? */
3177	return (imb > old_imb);
3178	}
3179
3180	/*
3181	* Maximum NUMA importance can be 1998 (2*999);
3182	* SMALLIMP @ 30 would be close to 1998/64.
3183	* Used to deter task migration.
3184	*/
3185	#define SMALLIMP 30
3186
3187	/*
3188	* This checks if the overall compute and NUMA accesses of the system would
3189	* be improved if the source tasks was migrated to the target dst_cpu taking
3190	* into account that it might be best if task running on the dst_cpu should
3191	* be exchanged with the source task
3192	*/
3193	static bool task_numa_compare(struct task_numa_env *env,
3194	long taskimp, long groupimp, bool maymove)
3195	{
3196	struct numa_group cur_ng, p_ng = deref_curr_numa_group(env->p);
3197	struct rq *dst_rq = cpu_rq(env->dst_cpu);
3198	long imp = p_ng ? groupimp : taskimp;
3199	struct task_struct *cur;
3200	long src_load, dst_load;
3201	int dist = env->dist;
3202	long moveimp = imp;
3203	long load;
3204	bool stopsearch = false;
3205
3206	if (READ_ONCE(dst_rq->numa_migrate_on))
3207	return false;
3208
3209	rcu_read_lock();
3210	cur = rcu_dereference_all(dst_rq->curr);
3211	if (cur && ((cur->flags & (PF_EXITING \| PF_KTHREAD)) \|\|
3212	!cur->mm))
3213	cur = NULL;
3214
3215	/*
3216	* Because we have preemption enabled we can get migrated around and
3217	* end try selecting ourselves (current == env->p) as a swap candidate.
3218	*/
3219	if (cur == env->p) {
3220	stopsearch = true;
3221	goto unlock;
3222	}
3223
3224	if (!cur) {
3225	if (maymove && moveimp >= env->best_imp)
3226	goto assign;
3227	else
3228	goto unlock;
3229	}
3230
3231	/* Skip this swap candidate if cannot move to the source cpu. */
3232	if (!cpumask_test_cpu(env->src_cpu, cur->cpus_ptr))
3233	goto unlock;
3234
3235	/*
3236	* Skip this swap candidate if it is not moving to its preferred
3237	* node and the best task is.
3238	*/
3239	if (env->best_task &&
3240	env->best_task->numa_preferred_nid == env->src_nid &&
3241	cur->numa_preferred_nid != env->src_nid) {
3242	goto unlock;
3243	}
3244
3245	/*
3246	* "imp" is the fault differential for the source task between the
3247	* source and destination node. Calculate the total differential for
3248	* the source task and potential destination task. The more negative
3249	* the value is, the more remote accesses that would be expected to
3250	* be incurred if the tasks were swapped.
3251	*
3252	* If dst and source tasks are in the same NUMA group, or not
3253	* in any group then look only at task weights.
3254	*/
3255	cur_ng = rcu_dereference_all(cur->numa_group);
3256	if (cur_ng == p_ng) {
3257	/*
3258	* Do not swap within a group or between tasks that have
3259	* no group if there is spare capacity. Swapping does
3260	* not address the load imbalance and helps one task at
3261	* the cost of punishing another.
3262	*/
3263	if (env->dst_stats.node_type == node_has_spare)
3264	goto unlock;
3265
3266	imp = taskimp + task_weight(cur, env->src_nid, dist) -
3267	task_weight(cur, env->dst_nid, dist);
3268	/*
3269	* Add some hysteresis to prevent swapping the
3270	* tasks within a group over tiny differences.
3271	*/
3272	if (cur_ng)
3273	imp -= imp / 16;
3274	} else {
3275	/*
3276	* Compare the group weights. If a task is all by itself
3277	* (not part of a group), use the task weight instead.
3278	*/
3279	if (cur_ng && p_ng)
3280	imp += group_weight(cur, env->src_nid, dist) -
3281	group_weight(cur, env->dst_nid, dist);
3282	else
3283	imp += task_weight(cur, env->src_nid, dist) -
3284	task_weight(cur, env->dst_nid, dist);
3285	}
3286
3287	/* Discourage picking a task already on its preferred node */
3288	if (cur->numa_preferred_nid == env->dst_nid)
3289	imp -= imp / 16;
3290
3291	/*
3292	* Encourage picking a task that moves to its preferred node.
3293	* This potentially makes imp larger than it's maximum of
3294	* 1998 (see SMALLIMP and task_weight for why) but in this
3295	* case, it does not matter.
3296	*/
3297	if (cur->numa_preferred_nid == env->src_nid)
3298	imp += imp / 8;
3299
3300	if (maymove && moveimp > imp && moveimp > env->best_imp) {
3301	imp = moveimp;
3302	cur = NULL;
3303	goto assign;
3304	}
3305
3306	/*
3307	* Prefer swapping with a task moving to its preferred node over a
3308	* task that is not.
3309	*/
3310	if (env->best_task && cur->numa_preferred_nid == env->src_nid &&
3311	env->best_task->numa_preferred_nid != env->src_nid) {
3312	goto assign;
3313	}
3314
3315	/*
3316	* If the NUMA importance is less than SMALLIMP,
3317	* task migration might only result in ping pong
3318	* of tasks and also hurt performance due to cache
3319	* misses.
3320	*/
3321	if (imp < SMALLIMP \|\| imp <= env->best_imp + SMALLIMP / 2)
3322	goto unlock;
3323
3324	/*
3325	* In the overloaded case, try and keep the load balanced.
3326	*/
3327	load = task_h_load(env->p) - task_h_load(cur);
3328	if (!load)
3329	goto assign;
3330
3331	dst_load = env->dst_stats.load + load;
3332	src_load = env->src_stats.load - load;
3333
3334	if (load_too_imbalanced(src_load, dst_load, env))
3335	goto unlock;
3336
3337	assign:
3338	/* Evaluate an idle CPU for a task numa move. */
3339	if (!cur) {
3340	int cpu = env->dst_stats.idle_cpu;
3341
3342	/* Nothing cached so current CPU went idle since the search. */
3343	if (cpu < 0)
3344	cpu = env->dst_cpu;
3345
3346	/*
3347	* If the CPU is no longer truly idle and the previous best CPU
3348	* is, keep using it.
3349	*/
3350	if (!idle_cpu(cpu) && env->best_cpu >= 0 &&
3351	idle_cpu(env->best_cpu)) {
3352	cpu = env->best_cpu;
3353	}
3354
3355	env->dst_cpu = cpu;
3356	}
3357
3358	task_numa_assign(env, cur, imp);
3359
3360	/*
3361	* If a move to idle is allowed because there is capacity or load
3362	* balance improves then stop the search. While a better swap
3363	* candidate may exist, a search is not free.
3364	*/
3365	if (maymove && !cur && env->best_cpu >= 0 && idle_cpu(env->best_cpu))
3366	stopsearch = true;
3367
3368	/*
3369	* If a swap candidate must be identified and the current best task
3370	* moves its preferred node then stop the search.
3371	*/
3372	if (!maymove && env->best_task &&
3373	env->best_task->numa_preferred_nid == env->src_nid) {
3374	stopsearch = true;
3375	}
3376	unlock:
3377	rcu_read_unlock();
3378
3379	return stopsearch;
3380	}
3381
3382	static void task_numa_find_cpu(struct task_numa_env *env,
3383	long taskimp, long groupimp)
3384	{
3385	bool maymove = false;
3386	int cpu;
3387
3388	/*
3389	* If dst node has spare capacity, then check if there is an
3390	* imbalance that would be overruled by the load balancer.
3391	*/
3392	if (env->dst_stats.node_type == node_has_spare) {
3393	unsigned int imbalance;
3394	int src_running, dst_running;
3395
3396	/*
3397	* Would movement cause an imbalance? Note that if src has
3398	* more running tasks that the imbalance is ignored as the
3399	* move improves the imbalance from the perspective of the
3400	* CPU load balancer.
3401	* */
3402	src_running = env->src_stats.nr_running - 1;
3403	dst_running = env->dst_stats.nr_running + 1;
3404	imbalance = max(0, dst_running - src_running);
3405	imbalance = adjust_numa_imbalance(imbalance, dst_running,
3406	env->imb_numa_nr);
3407
3408	/* Use idle CPU if there is no imbalance */
3409	if (!imbalance) {
3410	maymove = true;
3411	if (env->dst_stats.idle_cpu >= 0) {
3412	env->dst_cpu = env->dst_stats.idle_cpu;
3413	task_numa_assign(env, NULL, 0);
3414	return;
3415	}
3416	}
3417	} else {
3418	long src_load, dst_load, load;
3419	/*
3420	* If the improvement from just moving env->p direction is better
3421	* than swapping tasks around, check if a move is possible.
3422	*/
3423	load = task_h_load(env->p);
3424	dst_load = env->dst_stats.load + load;
3425	src_load = env->src_stats.load - load;
3426	maymove = !load_too_imbalanced(src_load, dst_load, env);
3427	}
3428
3429	/* Skip CPUs if the source task cannot migrate */
3430	for_each_cpu_and(cpu, cpumask_of_node(env->dst_nid), env->p->cpus_ptr) {
3431	env->dst_cpu = cpu;
3432	if (task_numa_compare(env, taskimp, groupimp, maymove))
3433	break;
3434	}
3435	}
3436
3437	static int task_numa_migrate(struct task_struct *p)
3438	{
3439	struct task_numa_env env = {
3440	.p = p,
3441
3442	.src_cpu = task_cpu(p),
3443	.src_nid = task_node(p),
3444
3445	.imbalance_pct = 112,
3446
3447	.best_task = NULL,
3448	.best_imp = 0,
3449	.best_cpu = -1,
3450	};
3451	unsigned long taskweight, groupweight;
3452	struct sched_domain *sd;
3453	long taskimp, groupimp;
3454	struct numa_group *ng;
3455	struct rq *best_rq;
3456	int nid, ret, dist;
3457
3458	/*
3459	* Pick the lowest SD_NUMA domain, as that would have the smallest
3460	* imbalance and would be the first to start moving tasks about.
3461	*
3462	* And we want to avoid any moving of tasks about, as that would create
3463	* random movement of tasks -- counter the numa conditions we're trying
3464	* to satisfy here.
3465	*/
3466	rcu_read_lock();
3467	sd = rcu_dereference_all(per_cpu(sd_numa, env.src_cpu));
3468	if (sd) {
3469	env.imbalance_pct = 100 + (sd->imbalance_pct - 100) / 2;
3470	env.imb_numa_nr = sd->imb_numa_nr;
3471	}
3472	rcu_read_unlock();
3473
3474	/*
3475	* Cpusets can break the scheduler domain tree into smaller
3476	* balance domains, some of which do not cross NUMA boundaries.
3477	* Tasks that are "trapped" in such domains cannot be migrated
3478	* elsewhere, so there is no point in (re)trying.
3479	*/
3480	if (unlikely(!sd)) {
3481	sched_setnuma(p, task_node(p));
3482	return -EINVAL;
3483	}
3484
3485	env.dst_nid = p->numa_preferred_nid;
3486	dist = env.dist = node_distance(env.src_nid, env.dst_nid);
3487	taskweight = task_weight(p, env.src_nid, dist);
3488	groupweight = group_weight(p, env.src_nid, dist);
3489	update_numa_stats(&env, &env.src_stats, env.src_nid, false);
3490	taskimp = task_weight(p, env.dst_nid, dist) - taskweight;
3491	groupimp = group_weight(p, env.dst_nid, dist) - groupweight;
3492	update_numa_stats(&env, &env.dst_stats, env.dst_nid, true);
3493
3494	/* Try to find a spot on the preferred nid. */
3495	task_numa_find_cpu(&env, taskimp, groupimp);
3496
3497	/*
3498	* Look at other nodes in these cases:
3499	* - there is no space available on the preferred_nid
3500	* - the task is part of a numa_group that is interleaved across
3501	* multiple NUMA nodes; in order to better consolidate the group,
3502	* we need to check other locations.
3503	*/
3504	ng = deref_curr_numa_group(p);
3505	if (env.best_cpu == -1 \|\| (ng && ng->active_nodes > 1)) {
3506	for_each_node_state(nid, N_CPU) {
3507	if (nid == env.src_nid \|\| nid == p->numa_preferred_nid)
3508	continue;
3509
3510	dist = node_distance(env.src_nid, env.dst_nid);
3511	if (sched_numa_topology_type == NUMA_BACKPLANE &&
3512	dist != env.dist) {
3513	taskweight = task_weight(p, env.src_nid, dist);
3514	groupweight = group_weight(p, env.src_nid, dist);
3515	}
3516
3517	/* Only consider nodes where both task and groups benefit */
3518	taskimp = task_weight(p, nid, dist) - taskweight;
3519	groupimp = group_weight(p, nid, dist) - groupweight;
3520	if (taskimp < 0 && groupimp < 0)
3521	continue;
3522
3523	env.dist = dist;
3524	env.dst_nid = nid;
3525	update_numa_stats(&env, &env.dst_stats, env.dst_nid, true);
3526	task_numa_find_cpu(&env, taskimp, groupimp);
3527	}
3528	}
3529
3530	/*
3531	* If the task is part of a workload that spans multiple NUMA nodes,
3532	* and is migrating into one of the workload's active nodes, remember
3533	* this node as the task's preferred numa node, so the workload can
3534	* settle down.
3535	* A task that migrated to a second choice node will be better off
3536	* trying for a better one later. Do not set the preferred node here.
3537	*/
3538	if (ng) {
3539	if (env.best_cpu == -1)
3540	nid = env.src_nid;
3541	else
3542	nid = cpu_to_node(env.best_cpu);
3543
3544	if (nid != p->numa_preferred_nid)
3545	sched_setnuma(p, nid);
3546	}
3547
3548	/* No better CPU than the current one was found. */
3549	if (env.best_cpu == -1) {
3550	trace_sched_stick_numa(p, env.src_cpu, NULL, -1);
3551	return -EAGAIN;
3552	}
3553
3554	best_rq = cpu_rq(env.best_cpu);
3555	if (env.best_task == NULL) {
3556	ret = migrate_task_to(p, env.best_cpu);
3557	WRITE_ONCE(best_rq->numa_migrate_on, 0);
3558	if (ret != 0)
3559	trace_sched_stick_numa(p, env.src_cpu, NULL, env.best_cpu);
3560	return ret;
3561	}
3562
3563	ret = migrate_swap(p, env.best_task, env.best_cpu, env.src_cpu);
3564	WRITE_ONCE(best_rq->numa_migrate_on, 0);
3565
3566	if (ret != 0)
3567	trace_sched_stick_numa(p, env.src_cpu, env.best_task, env.best_cpu);
3568	put_task_struct(env.best_task);
3569	return ret;
3570	}
3571
3572	/* Attempt to migrate a task to a CPU on the preferred node. */
3573	static void numa_migrate_preferred(struct task_struct *p)
3574	{
3575	unsigned long interval = HZ;
3576
3577	/* This task has no NUMA fault statistics yet */
3578	if (unlikely(p->numa_preferred_nid == NUMA_NO_NODE \|\| !p->numa_faults))
3579	return;
3580
3581	/* Periodically retry migrating the task to the preferred node */
3582	interval = min(interval, msecs_to_jiffies(p->numa_scan_period) / 16);
3583	p->numa_migrate_retry = jiffies + interval;
3584
3585	/* Success if task is already running on preferred CPU */
3586	if (task_node(p) == p->numa_preferred_nid)
3587	return;
3588
3589	/* Otherwise, try migrate to a CPU on the preferred node */
3590	task_numa_migrate(p);
3591	}
3592
3593	/*
3594	* Find out how many nodes the workload is actively running on. Do this by
3595	* tracking the nodes from which NUMA hinting faults are triggered. This can
3596	* be different from the set of nodes where the workload's memory is currently
3597	* located.
3598	*/
3599	static void numa_group_count_active_nodes(struct numa_group *numa_group)
3600	{
3601	unsigned long faults, max_faults = 0;
3602	int nid, active_nodes = 0;
3603
3604	for_each_node_state(nid, N_CPU) {
3605	faults = group_faults_cpu(numa_group, nid);
3606	if (faults > max_faults)
3607	max_faults = faults;
3608	}
3609
3610	for_each_node_state(nid, N_CPU) {
3611	faults = group_faults_cpu(numa_group, nid);
3612	if (faults * ACTIVE_NODE_FRACTION > max_faults)
3613	active_nodes++;
3614	}
3615
3616	numa_group->max_faults_cpu = max_faults;
3617	numa_group->active_nodes = active_nodes;
3618	}
3619
3620	/*
3621	* When adapting the scan rate, the period is divided into NUMA_PERIOD_SLOTS
3622	* increments. The more local the fault statistics are, the higher the scan
3623	* period will be for the next scan window. If local/(local+remote) ratio is
3624	* below NUMA_PERIOD_THRESHOLD (where range of ratio is 1..NUMA_PERIOD_SLOTS)
3625	* the scan period will decrease. Aim for 70% local accesses.
3626	*/
3627	#define NUMA_PERIOD_SLOTS 10
3628	#define NUMA_PERIOD_THRESHOLD 7
3629
3630	/*
3631	* Increase the scan period (slow down scanning) if the majority of
3632	* our memory is already on our local node, or if the majority of
3633	* the page accesses are shared with other processes.
3634	* Otherwise, decrease the scan period.
3635	*/
3636	static void update_task_scan_period(struct task_struct *p,
3637	unsigned long shared, unsigned long private)
3638	{
3639	unsigned int period_slot;
3640	int lr_ratio, ps_ratio;
3641	int diff;
3642
3643	unsigned long remote = p->numa_faults_locality[0];
3644	unsigned long local = p->numa_faults_locality[1];
3645
3646	/*
3647	* If there were no record hinting faults then either the task is
3648	* completely idle or all activity is in areas that are not of interest
3649	* to automatic numa balancing. Related to that, if there were failed
3650	* migration then it implies we are migrating too quickly or the local
3651	* node is overloaded. In either case, scan slower
3652	*/
3653	if (local + shared == 0 \|\| p->numa_faults_locality[2]) {
3654	p->numa_scan_period = min(p->numa_scan_period_max,
3655	p->numa_scan_period << 1);
3656
3657	p->mm->numa_next_scan = jiffies +
3658	msecs_to_jiffies(p->numa_scan_period);
3659
3660	goto out;
3661	}
3662
3663	/*
3664	* Prepare to scale scan period relative to the current period.
3665	* == NUMA_PERIOD_THRESHOLD scan period stays the same
3666	* < NUMA_PERIOD_THRESHOLD scan period decreases (scan faster)
3667	* >= NUMA_PERIOD_THRESHOLD scan period increases (scan slower)
3668	*/
3669	period_slot = DIV_ROUND_UP(p->numa_scan_period, NUMA_PERIOD_SLOTS);
3670	lr_ratio = (local * NUMA_PERIOD_SLOTS) / (local + remote);
3671	ps_ratio = (private * NUMA_PERIOD_SLOTS) / (private + shared);
3672
3673	if (ps_ratio >= NUMA_PERIOD_THRESHOLD) {
3674	/*
3675	* Most memory accesses are local. There is no need to
3676	* do fast NUMA scanning, since memory is already local.
3677	*/
3678	int slot = ps_ratio - NUMA_PERIOD_THRESHOLD;
3679	if (!slot)
3680	slot = 1;
3681	diff = slot * period_slot;
3682	} else if (lr_ratio >= NUMA_PERIOD_THRESHOLD) {
3683	/*
3684	* Most memory accesses are shared with other tasks.
3685	* There is no point in continuing fast NUMA scanning,
3686	* since other tasks may just move the memory elsewhere.
3687	*/
3688	int slot = lr_ratio - NUMA_PERIOD_THRESHOLD;
3689	if (!slot)
3690	slot = 1;
3691	diff = slot * period_slot;
3692	} else {
3693	/*
3694	* Private memory faults exceed (SLOTS-THRESHOLD)/SLOTS,
3695	* yet they are not on the local NUMA node. Speed up
3696	* NUMA scanning to get the memory moved over.
3697	*/
3698	int ratio = max(lr_ratio, ps_ratio);
3699	diff = -(NUMA_PERIOD_THRESHOLD - ratio) * period_slot;
3700	}
3701
3702	p->numa_scan_period = clamp(p->numa_scan_period + diff,
3703	task_scan_min(p), task_scan_max(p));
3704
3705	out:
3706	memset(p->numa_faults_locality, 0,
3707	sizeof(p->numa_faults_locality));
3708	}
3709
3710	/*
3711	* Get the fraction of time the task has been running since the last
3712	* NUMA placement cycle. The scheduler keeps similar statistics, but
3713	* decays those on a 32ms period, which is orders of magnitude off
3714	* from the dozens-of-seconds NUMA balancing period. Use the scheduler
3715	* stats only if the task is so new there are no NUMA statistics yet.
3716	*/
3717	static u64 numa_get_avg_runtime(struct task_struct p, u64 period)
3718	{
3719	u64 runtime, delta, now;
3720	/* Use the start of this time slice to avoid calculations. */
3721	now = p->se.exec_start;
3722	runtime = p->se.sum_exec_runtime;
3723
3724	if (p->last_task_numa_placement) {
3725	delta = runtime - p->last_sum_exec_runtime;
3726	*period = now - p->last_task_numa_placement;
3727
3728	/* Avoid time going backwards, prevent potential divide error: */
3729	if (unlikely((s64)*period < 0))
3730	*period = 0;
3731	} else {
3732	delta = p->se.avg.load_sum;
3733	*period = LOAD_AVG_MAX;
3734	}
3735
3736	p->last_sum_exec_runtime = runtime;
3737	p->last_task_numa_placement = now;
3738
3739	return delta;
3740	}
3741
3742	/*
3743	* Determine the preferred nid for a task in a numa_group. This needs to
3744	* be done in a way that produces consistent results with group_weight,
3745	* otherwise workloads might not converge.
3746	*/
3747	static int preferred_group_nid(struct task_struct *p, int nid)
3748	{
3749	nodemask_t nodes;
3750	int dist;
3751
3752	/* Direct connections between all NUMA nodes. */
3753	if (sched_numa_topology_type == NUMA_DIRECT)
3754	return nid;
3755
3756	/*
3757	* On a system with glueless mesh NUMA topology, group_weight
3758	* scores nodes according to the number of NUMA hinting faults on
3759	* both the node itself, and on nearby nodes.
3760	*/
3761	if (sched_numa_topology_type == NUMA_GLUELESS_MESH) {
3762	unsigned long score, max_score = 0;
3763	int node, max_node = nid;
3764
3765	dist = sched_max_numa_distance;
3766
3767	for_each_node_state(node, N_CPU) {
3768	score = group_weight(p, node, dist);
3769	if (score > max_score) {
3770	max_score = score;
3771	max_node = node;
3772	}
3773	}
3774	return max_node;
3775	}
3776
3777	/*
3778	* Finding the preferred nid in a system with NUMA backplane
3779	* interconnect topology is more involved. The goal is to locate
3780	* tasks from numa_groups near each other in the system, and
3781	* untangle workloads from different sides of the system. This requires
3782	* searching down the hierarchy of node groups, recursively searching
3783	* inside the highest scoring group of nodes. The nodemask tricks
3784	* keep the complexity of the search down.
3785	*/
3786	nodes = node_states[N_CPU];
3787	for (dist = sched_max_numa_distance; dist > LOCAL_DISTANCE; dist--) {
3788	unsigned long max_faults = 0;
3789	nodemask_t max_group = NODE_MASK_NONE;
3790	int a, b;
3791
3792	/* Are there nodes at this distance from each other? */
3793	if (!find_numa_distance(dist))
3794	continue;
3795
3796	for_each_node_mask(a, nodes) {
3797	unsigned long faults = 0;
3798	nodemask_t this_group;
3799	nodes_clear(this_group);
3800
3801	/* Sum group's NUMA faults; includes a==b case. */
3802	for_each_node_mask(b, nodes) {
3803	if (node_distance(a, b) < dist) {
3804	faults += group_faults(p, b);
3805	node_set(b, this_group);
3806	node_clear(b, nodes);
3807	}
3808	}
3809
3810	/* Remember the top group. */
3811	if (faults > max_faults) {
3812	max_faults = faults;
3813	max_group = this_group;
3814	/*
3815	* subtle: at the smallest distance there is
3816	* just one node left in each "group", the
3817	* winner is the preferred nid.
3818	*/
3819	nid = a;
3820	}
3821	}
3822	/* Next round, evaluate the nodes within max_group. */
3823	if (!max_faults)
3824	break;
3825	nodes = max_group;
3826	}
3827	return nid;
3828	}
3829
3830	static void task_numa_placement(struct task_struct *p)
3831	__context_unsafe(/* conditional locking */)
3832	{
3833	struct sched_cache_group __maybe_unused *grp;
3834	int seq, nid, max_nid = NUMA_NO_NODE;
3835	unsigned long max_faults = 0;
3836	unsigned long fault_types[2] = { 0, 0 };
3837	unsigned long total_faults;
3838	u64 runtime, period;
3839	spinlock_t *group_lock = NULL;
3840	long __maybe_unused new_fp;
3841	struct numa_group *ng;
3842
3843	/*
3844	* The p->mm->numa_scan_seq field gets updated without
3845	* exclusive access. Use READ_ONCE() here to ensure
3846	* that the field is read in a single access:
3847	*/
3848	seq = READ_ONCE(p->mm->numa_scan_seq);
3849	if (p->numa_scan_seq == seq)
3850	return;
3851	p->numa_scan_seq = seq;
3852	p->numa_scan_period_max = task_scan_max(p);
3853
3854	total_faults = p->numa_faults_locality[0] +
3855	p->numa_faults_locality[1];
3856	runtime = numa_get_avg_runtime(p, &period);
3857
3858	/* If the task is part of a group prevent parallel updates to group stats */
3859	ng = deref_curr_numa_group(p);
3860	if (ng) {
3861	group_lock = &ng->lock;
3862	spin_lock_irq(group_lock);
3863	}
3864
3865	/* Find the node with the highest number of faults */
3866	for_each_online_node(nid) {
3867	/* Keep track of the offsets in numa_faults array */
3868	int mem_idx, membuf_idx, cpu_idx, cpubuf_idx;
3869	unsigned long faults = 0, group_faults = 0;
3870	int priv;
3871
3872	for (priv = 0; priv < NR_NUMA_HINT_FAULT_TYPES; priv++) {
3873	long diff, f_diff, f_weight;
3874
3875	mem_idx = task_faults_idx(NUMA_MEM, nid, priv);
3876	membuf_idx = task_faults_idx(NUMA_MEMBUF, nid, priv);
3877	cpu_idx = task_faults_idx(NUMA_CPU, nid, priv);
3878	cpubuf_idx = task_faults_idx(NUMA_CPUBUF, nid, priv);
3879
3880	/* Decay existing window, copy faults since last scan */
3881	diff = p->numa_faults[membuf_idx] - p->numa_faults[mem_idx] / 2;
3882	fault_types[priv] += p->numa_faults[membuf_idx];
3883	p->numa_faults[membuf_idx] = 0;
3884
3885	/*
3886	* Normalize the faults_from, so all tasks in a group
3887	* count according to CPU use, instead of by the raw
3888	* number of faults. Tasks with little runtime have
3889	* little over-all impact on throughput, and thus their
3890	* faults are less important.
3891	*/
3892	f_weight = div64_u64(runtime << 16, period + 1);
3893	f_weight = (f_weight * p->numa_faults[cpubuf_idx]) /
3894	(total_faults + 1);
3895	f_diff = f_weight - p->numa_faults[cpu_idx] / 2;
3896	p->numa_faults[cpubuf_idx] = 0;
3897
3898	p->numa_faults[mem_idx] += diff;
3899	p->numa_faults[cpu_idx] += f_diff;
3900	faults += p->numa_faults[mem_idx];
3901	p->total_numa_faults += diff;
3902	if (ng) {
3903	/*
3904	* safe because we can only change our own group
3905	*
3906	* mem_idx represents the offset for a given
3907	* nid and priv in a specific region because it
3908	* is at the beginning of the numa_faults array.
3909	*/
3910	ng->faults[mem_idx] += diff;
3911	ng->faults[cpu_idx] += f_diff;
3912	ng->total_faults += diff;
3913	group_faults += ng->faults[mem_idx];
3914	}
3915	#ifdef CONFIG_SCHED_CACHE
3916	/*
3917	* Per task p->numa_faults[mem_idx] converges,
3918	* so the accumulation of each task's faults
3919	* converges too - Given the number of threads,
3920	* it cannot overflow an unsigned long.
3921	* Racy with concurrent updates from other threads
3922	* sharing this mm. Acceptable since footprint is a
3923	* heuristic and occasional lost updates are tolerable.
3924	*
3925	* If a task exits, its corresponding footprint must
3926	* be subtracted from p->sched_cache_grp->footprint,
3927	* otherwise the footprint will not converge: the
3928	* exiting thread's footprint remains unchanged/undecayed.
3929	* See exit_mm().
3930	*
3931	* Lost updates and unsynchronized subtraction
3932	* in exit_mm() can cause footprint + diff to
3933	* go negative. Clamp to zero to prevent the
3934	* unsigned footprint from wrapping.
3935	*/
3936	scoped_guard(rcu) {
3937	grp = rcu_dereference(p->sched_cache_grp);
3938
3939	if (grp) {
3940	new_fp = (long)READ_ONCE(grp->footprint) + diff;
3941	WRITE_ONCE(grp->footprint, max(new_fp, 0L));
3942	}
3943	}
3944	#endif
3945	}
3946
3947	if (!ng) {
3948	if (faults > max_faults) {
3949	max_faults = faults;
3950	max_nid = nid;
3951	}
3952	} else if (group_faults > max_faults) {
3953	max_faults = group_faults;
3954	max_nid = nid;
3955	}
3956	}
3957
3958	/* Cannot migrate task to CPU-less node */
3959	max_nid = numa_nearest_node(max_nid, N_CPU);
3960
3961	if (ng) {
3962	numa_group_count_active_nodes(ng);
3963	spin_unlock_irq(group_lock);
3964	max_nid = preferred_group_nid(p, max_nid);
3965	}
3966
3967	if (max_faults) {
3968	/* Set the new preferred node */
3969	if (max_nid != p->numa_preferred_nid)
3970	sched_setnuma(p, max_nid);
3971	}
3972
3973	update_task_scan_period(p, fault_types[0], fault_types[1]);
3974	}
3975
3976	static inline int get_numa_group(struct numa_group *grp)
3977	{
3978	return refcount_inc_not_zero(&grp->refcount);
3979	}
3980
3981	static inline void put_numa_group(struct numa_group *grp)
3982	{
3983	if (refcount_dec_and_test(&grp->refcount))
3984	kfree_rcu(grp, rcu);
3985	}
3986
3987	static void task_numa_group(struct task_struct *p, int cpupid, int flags,
3988	int *priv)
3989	{
3990	struct numa_group grp, my_grp;
3991	struct task_struct *tsk;
3992	bool join = false;
3993	int cpu = cpupid_to_cpu(cpupid);
3994	int i;
3995
3996	if (unlikely(!deref_curr_numa_group(p))) {
3997	unsigned int size = sizeof(struct numa_group) +
3998	NR_NUMA_HINT_FAULT_STATS *
3999	nr_node_ids * sizeof(unsigned long);
4000
4001	grp = kzalloc(size, GFP_KERNEL \| __GFP_NOWARN);
4002	if (!grp)
4003	return;
4004
4005	refcount_set(&grp->refcount, 1);
4006	grp->active_nodes = 1;
4007	grp->max_faults_cpu = 0;
4008	spin_lock_init(&grp->lock);
4009	grp->gid = p->pid;
4010
4011	for (i = 0; i < NR_NUMA_HINT_FAULT_STATS * nr_node_ids; i++)
4012	grp->faults[i] = p->numa_faults[i];
4013
4014	grp->total_faults = p->total_numa_faults;
4015
4016	grp->nr_tasks++;
4017	rcu_assign_pointer(p->numa_group, grp);
4018	}
4019
4020	rcu_read_lock();
4021	tsk = READ_ONCE(cpu_rq(cpu)->curr);
4022
4023	if (!cpupid_match_pid(tsk, cpupid))
4024	goto no_join;
4025
4026	grp = rcu_dereference_all(tsk->numa_group);
4027	if (!grp)
4028	goto no_join;
4029
4030	my_grp = deref_curr_numa_group(p);
4031	if (grp == my_grp)
4032	goto no_join;
4033
4034	/*
4035	* Only join the other group if its bigger; if we're the bigger group,
4036	* the other task will join us.
4037	*/
4038	if (my_grp->nr_tasks > grp->nr_tasks)
4039	goto no_join;
4040
4041	/*
4042	* Tie-break on the grp address.
4043	*/
4044	if (my_grp->nr_tasks == grp->nr_tasks && my_grp > grp)
4045	goto no_join;
4046
4047	/* Always join threads in the same process. */
4048	if (tsk->mm == current->mm)
4049	join = true;
4050
4051	/* Simple filter to avoid false positives due to PID collisions */
4052	if (flags & TNF_SHARED)
4053	join = true;
4054
4055	/* Update priv based on whether false sharing was detected */
4056	*priv = !join;
4057
4058	if (join && !get_numa_group(grp))
4059	goto no_join;
4060
4061	rcu_read_unlock();
4062
4063	if (!join)
4064	return;
4065
4066	WARN_ON_ONCE(irqs_disabled());
4067	double_lock_irq(&my_grp->lock, &grp->lock);
4068
4069	for (i = 0; i < NR_NUMA_HINT_FAULT_STATS * nr_node_ids; i++) {
4070	my_grp->faults[i] -= p->numa_faults[i];
4071	grp->faults[i] += p->numa_faults[i];
4072	}
4073	my_grp->total_faults -= p->total_numa_faults;
4074	grp->total_faults += p->total_numa_faults;
4075
4076	my_grp->nr_tasks--;
4077	grp->nr_tasks++;
4078
4079	spin_unlock(&my_grp->lock);
4080	spin_unlock_irq(&grp->lock);
4081
4082	rcu_assign_pointer(p->numa_group, grp);
4083
4084	put_numa_group(my_grp);
4085	return;
4086
4087	no_join:
4088	rcu_read_unlock();
4089	return;
4090	}
4091
4092	/*
4093	* Get rid of NUMA statistics associated with a task (either current or dead).
4094	* If @final is set, the task is dead and has reached refcount zero, so we can
4095	* safely free all relevant data structures. Otherwise, there might be
4096	* concurrent reads from places like load balancing and procfs, and we should
4097	* reset the data back to default state without freeing ->numa_faults.
4098	*/
4099	void task_numa_free(struct task_struct *p, bool final)
4100	{
4101	/* safe: p either is current or is being freed by current */
4102	struct numa_group *grp = rcu_dereference_raw(p->numa_group);
4103	unsigned long *numa_faults = p->numa_faults;
4104	unsigned long flags;
4105	int i;
4106
4107	if (!numa_faults)
4108	return;
4109
4110	if (grp) {
4111	spin_lock_irqsave(&grp->lock, flags);
4112	for (i = 0; i < NR_NUMA_HINT_FAULT_STATS * nr_node_ids; i++)
4113	grp->faults[i] -= p->numa_faults[i];
4114	grp->total_faults -= p->total_numa_faults;
4115
4116	grp->nr_tasks--;
4117	spin_unlock_irqrestore(&grp->lock, flags);
4118	RCU_INIT_POINTER(p->numa_group, NULL);
4119	put_numa_group(grp);
4120	}
4121
4122	if (final) {
4123	p->numa_faults = NULL;
4124	kfree(numa_faults);
4125	} else {
4126	p->total_numa_faults = 0;
4127	for (i = 0; i < NR_NUMA_HINT_FAULT_STATS * nr_node_ids; i++)
4128	numa_faults[i] = 0;
4129	}
4130	}
4131
4132	/*
4133	* Got a PROT_NONE fault for a page on @node.
4134	*/
4135	void task_numa_fault(int last_cpupid, int mem_node, int pages, int flags)
4136	{
4137	struct task_struct *p = current;
4138	bool migrated = flags & TNF_MIGRATED;
4139	int cpu_node = task_node(current);
4140	int local = !!(flags & TNF_FAULT_LOCAL);
4141	struct numa_group *ng;
4142	int priv;
4143
4144	if (!static_branch_likely(&sched_numa_balancing))
4145	return;
4146
4147	/* for example, ksmd faulting in a user's mm */
4148	if (!p->mm)
4149	return;
4150
4151	/*
4152	* NUMA faults statistics are unnecessary for the slow memory
4153	* node for memory tiering mode.
4154	*/
4155	if (!node_is_toptier(mem_node) &&
4156	(sysctl_numa_balancing_mode & NUMA_BALANCING_MEMORY_TIERING \|\|
4157	!cpupid_valid(last_cpupid)))
4158	return;
4159
4160	/* Allocate buffer to track faults on a per-node basis */
4161	if (unlikely(!p->numa_faults)) {
4162	int size = sizeof(p->numa_faults)
4163	NR_NUMA_HINT_FAULT_BUCKETS * nr_node_ids;
4164
4165	p->numa_faults = kzalloc(size, GFP_KERNEL\|__GFP_NOWARN);
4166	if (!p->numa_faults)
4167	return;
4168
4169	p->total_numa_faults = 0;
4170	memset(p->numa_faults_locality, 0, sizeof(p->numa_faults_locality));
4171	}
4172
4173	/*
4174	* First accesses are treated as private, otherwise consider accesses
4175	* to be private if the accessing pid has not changed
4176	*/
4177	if (unlikely(last_cpupid == (-1 & LAST_CPUPID_MASK))) {
4178	priv = 1;
4179	} else {
4180	priv = cpupid_match_pid(p, last_cpupid);
4181	if (!priv && !(flags & TNF_NO_GROUP))
4182	task_numa_group(p, last_cpupid, flags, &priv);
4183	}
4184
4185	/*
4186	* If a workload spans multiple NUMA nodes, a shared fault that
4187	* occurs wholly within the set of nodes that the workload is
4188	* actively using should be counted as local. This allows the
4189	* scan rate to slow down when a workload has settled down.
4190	*/
4191	ng = deref_curr_numa_group(p);
4192	if (!priv && !local && ng && ng->active_nodes > 1 &&
4193	numa_is_active_node(cpu_node, ng) &&
4194	numa_is_active_node(mem_node, ng))
4195	local = 1;
4196
4197	/*
4198	* Retry to migrate task to preferred node periodically, in case it
4199	* previously failed, or the scheduler moved us.
4200	*/
4201	if (time_after(jiffies, p->numa_migrate_retry)) {
4202	task_numa_placement(p);
4203	numa_migrate_preferred(p);
4204	}
4205
4206	if (migrated)
4207	p->numa_pages_migrated += pages;
4208	if (flags & TNF_MIGRATE_FAIL)
4209	p->numa_faults_locality[2] += pages;
4210
4211	p->numa_faults[task_faults_idx(NUMA_MEMBUF, mem_node, priv)] += pages;
4212	p->numa_faults[task_faults_idx(NUMA_CPUBUF, cpu_node, priv)] += pages;
4213	p->numa_faults_locality[local] += pages;
4214	}
4215
4216	static void reset_ptenuma_scan(struct task_struct *p)
4217	{
4218	/*
4219	* We only did a read acquisition of the mmap sem, so
4220	* p->mm->numa_scan_seq is written to without exclusive access
4221	* and the update is not guaranteed to be atomic. That's not
4222	* much of an issue though, since this is just used for
4223	* statistical sampling. Use READ_ONCE/WRITE_ONCE, which are not
4224	* expensive, to avoid any form of compiler optimizations:
4225	*/
4226	WRITE_ONCE(p->mm->numa_scan_seq, READ_ONCE(p->mm->numa_scan_seq) + 1);
4227	p->mm->numa_scan_offset = 0;
4228	}
4229
4230	static bool vma_is_accessed(struct mm_struct mm, struct vm_area_struct vma)
4231	{
4232	unsigned long pids;
4233	/*
4234	* Allow unconditional access first two times, so that all the (pages)
4235	* of VMAs get prot_none fault introduced irrespective of accesses.
4236	* This is also done to avoid any side effect of task scanning
4237	* amplifying the unfairness of disjoint set of VMAs' access.
4238	*/
4239	if ((READ_ONCE(current->mm->numa_scan_seq) - vma->numab_state->start_scan_seq) < 2)
4240	return true;
4241
4242	pids = vma->numab_state->pids_active[0] \| vma->numab_state->pids_active[1];
4243	if (test_bit(hash_32(current->pid, ilog2(BITS_PER_LONG)), &pids))
4244	return true;
4245
4246	/*
4247	* Complete a scan that has already started regardless of PID access, or
4248	* some VMAs may never be scanned in multi-threaded applications:
4249	*/
4250	if (mm->numa_scan_offset > vma->vm_start) {
4251	trace_sched_skip_vma_numa(mm, vma, NUMAB_SKIP_IGNORE_PID);
4252	return true;
4253	}
4254
4255	/*
4256	* This vma has not been accessed for a while, and if the number
4257	* the threads in the same process is low, which means no other
4258	* threads can help scan this vma, force a vma scan.
4259	*/
4260	if (READ_ONCE(mm->numa_scan_seq) >
4261	(vma->numab_state->prev_scan_seq + get_nr_threads(current)))
4262	return true;
4263
4264	return false;
4265	}
4266
4267	#define VMA_PID_RESET_PERIOD (4 * sysctl_numa_balancing_scan_delay)
4268
4269	/*
4270	* The expensive part of numa migration is done from task_work context.
4271	* Triggered from task_tick_numa().
4272	*/
4273	static void task_numa_work(struct callback_head *work)
4274	{
4275	unsigned long migrate, next_scan, now = jiffies;
4276	struct task_struct *p = current;
4277	struct mm_struct *mm = p->mm;
4278	u64 runtime = p->se.sum_exec_runtime;
4279	struct vm_area_struct *vma;
4280	unsigned long start, end;
4281	unsigned long nr_pte_updates = 0;
4282	long pages, virtpages;
4283	struct vma_iterator vmi;
4284	bool vma_pids_skipped;
4285	bool vma_pids_forced = false;
4286
4287	WARN_ON_ONCE(p != container_of(work, struct task_struct, numa_work));
4288
4289	work->next = work;
4290	/*
4291	* Who cares about NUMA placement when they're dying.
4292	*
4293	* NOTE: make sure not to dereference p->mm before this check,
4294	* exit_task_work() happens _after_ exit_mm() so we could be called
4295	* without p->mm even though we still had it when we enqueued this
4296	* work.
4297	*/
4298	if (p->flags & PF_EXITING)
4299	return;
4300
4301	/*
4302	* Memory is pinned to only one NUMA node via cpuset.mems, naturally
4303	* no page can be migrated.
4304	*/
4305	if (cpusets_enabled() && nodes_weight(cpuset_current_mems_allowed) == 1) {
4306	trace_sched_skip_cpuset_numa(current, &cpuset_current_mems_allowed);
4307	return;
4308	}
4309
4310	if (!mm->numa_next_scan) {
4311	mm->numa_next_scan = now +
4312	msecs_to_jiffies(sysctl_numa_balancing_scan_delay);
4313	}
4314
4315	/*
4316	* Enforce maximal scan/migration frequency..
4317	*/
4318	migrate = mm->numa_next_scan;
4319	if (time_before(now, migrate))
4320	return;
4321
4322	if (p->numa_scan_period == 0) {
4323	p->numa_scan_period_max = task_scan_max(p);
4324	p->numa_scan_period = task_scan_start(p);
4325	}
4326
4327	next_scan = now + msecs_to_jiffies(p->numa_scan_period);
4328	if (!try_cmpxchg(&mm->numa_next_scan, &migrate, next_scan))
4329	return;
4330
4331	/*
4332	* Delay this task enough that another task of this mm will likely win
4333	* the next time around.
4334	*/
4335	p->node_stamp += 2 * TICK_NSEC;
4336
4337	pages = sysctl_numa_balancing_scan_size;
4338	pages <<= 20 - PAGE_SHIFT; /* MB in pages */
4339	virtpages = pages * 8; /* Scan up to this much virtual space */
4340	if (!pages)
4341	return;
4342
4343
4344	if (!mmap_read_trylock(mm))
4345	return;
4346
4347	/*
4348	* VMAs are skipped if the current PID has not trapped a fault within
4349	* the VMA recently. Allow scanning to be forced if there is no
4350	* suitable VMA remaining.
4351	*/
4352	vma_pids_skipped = false;
4353
4354	retry_pids:
4355	start = mm->numa_scan_offset;
4356	vma_iter_init(&vmi, mm, start);
4357	vma = vma_next(&vmi);
4358	if (!vma) {
4359	reset_ptenuma_scan(p);
4360	start = 0;
4361	vma_iter_set(&vmi, start);
4362	vma = vma_next(&vmi);
4363	}
4364
4365	for (; vma; vma = vma_next(&vmi)) {
4366	if (!vma_migratable(vma) \|\| !vma_policy_mof(vma) \|\|
4367	is_vm_hugetlb_page(vma) \|\| (vma->vm_flags & VM_MIXEDMAP)) {
4368	trace_sched_skip_vma_numa(mm, vma, NUMAB_SKIP_UNSUITABLE);
4369	continue;
4370	}
4371
4372	/*
4373	* Shared library pages mapped by multiple processes are not
4374	* migrated as it is expected they are cache replicated. Avoid
4375	* hinting faults in read-only file-backed mappings or the vDSO
4376	* as migrating the pages will be of marginal benefit.
4377	*/
4378	if (!vma->vm_mm \|\|
4379	(vma->vm_file && (vma->vm_flags & (VM_READ\|VM_WRITE)) == (VM_READ))) {
4380	trace_sched_skip_vma_numa(mm, vma, NUMAB_SKIP_SHARED_RO);
4381	continue;
4382	}
4383
4384	/*
4385	* Skip inaccessible VMAs to avoid any confusion between
4386	* PROT_NONE and NUMA hinting PTEs
4387	*/
4388	if (!vma_is_accessible(vma)) {
4389	trace_sched_skip_vma_numa(mm, vma, NUMAB_SKIP_INACCESSIBLE);
4390	continue;
4391	}
4392
4393	/* Initialise new per-VMA NUMAB state. */
4394	if (!vma->numab_state) {
4395	struct vma_numab_state *ptr;
4396
4397	ptr = kzalloc_obj(*ptr);
4398	if (!ptr)
4399	continue;
4400
4401	if (cmpxchg(&vma->numab_state, NULL, ptr)) {
4402	kfree(ptr);
4403	continue;
4404	}
4405
4406	vma->numab_state->start_scan_seq = mm->numa_scan_seq;
4407
4408	vma->numab_state->next_scan = now +
4409	msecs_to_jiffies(sysctl_numa_balancing_scan_delay);
4410
4411	/* Reset happens after 4 times scan delay of scan start */
4412	vma->numab_state->pids_active_reset = vma->numab_state->next_scan +
4413	msecs_to_jiffies(VMA_PID_RESET_PERIOD);
4414
4415	/*
4416	* Ensure prev_scan_seq does not match numa_scan_seq,
4417	* to prevent VMAs being skipped prematurely on the
4418	* first scan:
4419	*/
4420	vma->numab_state->prev_scan_seq = mm->numa_scan_seq - 1;
4421	}
4422
4423	/*
4424	* Scanning the VMAs of short lived tasks add more overhead. So
4425	* delay the scan for new VMAs.
4426	*/
4427	if (mm->numa_scan_seq && time_before(jiffies,
4428	vma->numab_state->next_scan)) {
4429	trace_sched_skip_vma_numa(mm, vma, NUMAB_SKIP_SCAN_DELAY);
4430	continue;
4431	}
4432
4433	/* RESET access PIDs regularly for old VMAs. */
4434	if (mm->numa_scan_seq &&
4435	time_after(jiffies, vma->numab_state->pids_active_reset)) {
4436	vma->numab_state->pids_active_reset = vma->numab_state->pids_active_reset +
4437	msecs_to_jiffies(VMA_PID_RESET_PERIOD);
4438	vma->numab_state->pids_active[0] = READ_ONCE(vma->numab_state->pids_active[1]);
4439	vma->numab_state->pids_active[1] = 0;
4440	}
4441
4442	/* Do not rescan VMAs twice within the same sequence. */
4443	if (vma->numab_state->prev_scan_seq == mm->numa_scan_seq) {
4444	mm->numa_scan_offset = vma->vm_end;
4445	trace_sched_skip_vma_numa(mm, vma, NUMAB_SKIP_SEQ_COMPLETED);
4446	continue;
4447	}
4448
4449	/*
4450	* Do not scan the VMA if task has not accessed it, unless no other
4451	* VMA candidate exists.
4452	*/
4453	if (!vma_pids_forced && !vma_is_accessed(mm, vma)) {
4454	vma_pids_skipped = true;
4455	trace_sched_skip_vma_numa(mm, vma, NUMAB_SKIP_PID_INACTIVE);
4456	continue;
4457	}
4458
4459	do {
4460	start = max(start, vma->vm_start);
4461	end = ALIGN(start + (pages << PAGE_SHIFT), HPAGE_SIZE);
4462	end = min(end, vma->vm_end);
4463	nr_pte_updates = change_prot_numa(vma, start, end);
4464
4465	/*
4466	* Try to scan sysctl_numa_balancing_size worth of
4467	* hpages that have at least one present PTE that
4468	* is not already PTE-numa. If the VMA contains
4469	* areas that are unused or already full of prot_numa
4470	* PTEs, scan up to virtpages, to skip through those
4471	* areas faster.
4472	*/
4473	if (nr_pte_updates)
4474	pages -= (end - start) >> PAGE_SHIFT;
4475	virtpages -= (end - start) >> PAGE_SHIFT;
4476
4477	start = end;
4478	if (pages <= 0 \|\| virtpages <= 0)
4479	goto out;
4480
4481	cond_resched();
4482	} while (end != vma->vm_end);
4483
4484	/* VMA scan is complete, do not scan until next sequence. */
4485	vma->numab_state->prev_scan_seq = mm->numa_scan_seq;
4486
4487	/*
4488	* Only force scan within one VMA at a time, to limit the
4489	* cost of scanning a potentially uninteresting VMA.
4490	*/
4491	if (vma_pids_forced)
4492	break;
4493	}
4494
4495	/*
4496	* If no VMAs are remaining and VMAs were skipped due to the PID
4497	* not accessing the VMA previously, then force a scan to ensure
4498	* forward progress:
4499	*/
4500	if (!vma && !vma_pids_forced && vma_pids_skipped) {
4501	vma_pids_forced = true;
4502	goto retry_pids;
4503	}
4504
4505	out:
4506	/*
4507	* It is possible to reach the end of the VMA list but the last few
4508	* VMAs are not guaranteed to the vma_migratable. If they are not, we
4509	* would find the !migratable VMA on the next scan but not reset the
4510	* scanner to the start so check it now.
4511	*/
4512	if (vma)
4513	mm->numa_scan_offset = start;
4514	else
4515	reset_ptenuma_scan(p);
4516	mmap_read_unlock(mm);
4517
4518	/*
4519	* Make sure tasks use at least 32x as much time to run other code
4520	* than they used here, to limit NUMA PTE scanning overhead to 3% max.
4521	* Usually update_task_scan_period slows down scanning enough; on an
4522	* overloaded system we need to limit overhead on a per task basis.
4523	*/
4524	if (unlikely(p->se.sum_exec_runtime != runtime)) {
4525	u64 diff = p->se.sum_exec_runtime - runtime;
4526	p->node_stamp += 32 * diff;
4527	}
4528	}
4529
4530	void init_numa_balancing(u64 clone_flags, struct task_struct *p)
4531	{
4532	int mm_users = 0;
4533	struct mm_struct *mm = p->mm;
4534
4535	if (mm) {
4536	mm_users = atomic_read(&mm->mm_users);
4537	if (mm_users == 1) {
4538	mm->numa_next_scan = jiffies + msecs_to_jiffies(sysctl_numa_balancing_scan_delay);
4539	mm->numa_scan_seq = 0;
4540	}
4541	}
4542	p->node_stamp = 0;
4543	p->numa_scan_seq = mm ? mm->numa_scan_seq : 0;
4544	p->numa_scan_period = sysctl_numa_balancing_scan_delay;
4545	p->numa_migrate_retry = 0;
4546	/* Protect against double add, see task_tick_numa and task_numa_work */
4547	p->numa_work.next = &p->numa_work;
4548	p->numa_faults = NULL;
4549	p->numa_pages_migrated = 0;
4550	p->total_numa_faults = 0;
4551	RCU_INIT_POINTER(p->numa_group, NULL);
4552	p->last_task_numa_placement = 0;
4553	p->last_sum_exec_runtime = 0;
4554
4555	init_task_work(&p->numa_work, task_numa_work);
4556
4557	/* New address space, reset the preferred nid */
4558	if (!(clone_flags & CLONE_VM)) {
4559	p->numa_preferred_nid = NUMA_NO_NODE;
4560	return;
4561	}
4562
4563	/*
4564	* New thread, keep existing numa_preferred_nid which should be copied
4565	* already by arch_dup_task_struct but stagger when scans start.
4566	*/
4567	if (mm) {
4568	unsigned int delay;
4569
4570	delay = min_t(unsigned int, task_scan_max(current),
4571	current->numa_scan_period * mm_users * NSEC_PER_MSEC);
4572	delay += 2 * TICK_NSEC;
4573	p->node_stamp = delay;
4574	}
4575	}
4576
4577	/*
4578	* Drive the periodic memory faults..
4579	*/
4580	static void task_tick_numa(struct rq rq, struct task_struct curr)
4581	{
4582	struct callback_head *work = &curr->numa_work;
4583	u64 period, now;
4584
4585	/*
4586	* We don't care about NUMA placement if we don't have memory.
4587	*/
4588	if (!curr->mm \|\| (curr->flags & (PF_EXITING \| PF_KTHREAD)) \|\| work->next != work)
4589	return;
4590
4591	/*
4592	* Using runtime rather than walltime has the dual advantage that
4593	* we (mostly) drive the selection from busy threads and that the
4594	* task needs to have done some actual work before we bother with
4595	* NUMA placement.
4596	*/
4597	now = curr->se.sum_exec_runtime;
4598	period = (u64)curr->numa_scan_period * NSEC_PER_MSEC;
4599
4600	if (now > curr->node_stamp + period) {
4601	if (!curr->node_stamp)
4602	curr->numa_scan_period = task_scan_start(curr);
4603	curr->node_stamp += period;
4604
4605	if (!time_before(jiffies, curr->mm->numa_next_scan))
4606	task_work_add(curr, work, TWA_RESUME);
4607	}
4608	}
4609
4610	static void update_scan_period(struct task_struct *p, int new_cpu)
4611	{
4612	int src_nid = cpu_to_node(task_cpu(p));
4613	int dst_nid = cpu_to_node(new_cpu);
4614
4615	if (!static_branch_likely(&sched_numa_balancing))
4616	return;
4617
4618	if (!p->mm \|\| !p->numa_faults \|\| (p->flags & PF_EXITING))
4619	return;
4620
4621	if (src_nid == dst_nid)
4622	return;
4623
4624	/*
4625	* Allow resets if faults have been trapped before one scan
4626	* has completed. This is most likely due to a new task that
4627	* is pulled cross-node due to wakeups or load balancing.
4628	*/
4629	if (p->numa_scan_seq) {
4630	/*
4631	* Avoid scan adjustments if moving to the preferred
4632	* node or if the task was not previously running on
4633	* the preferred node.
4634	*/
4635	if (dst_nid == p->numa_preferred_nid \|\|
4636	(p->numa_preferred_nid != NUMA_NO_NODE &&
4637	src_nid != p->numa_preferred_nid))
4638	return;
4639	}
4640
4641	p->numa_scan_period = task_scan_start(p);
4642	}
4643
4644	#else /* !CONFIG_NUMA_BALANCING: */
4645
4646	static void task_tick_numa(struct rq rq, struct task_struct curr)
4647	{
4648	}
4649
4650	static inline void account_numa_enqueue(struct rq rq, struct task_struct p)
4651	{
4652	}
4653
4654	static inline void account_numa_dequeue(struct rq rq, struct task_struct p)
4655	{
4656	}
4657
4658	static inline void update_scan_period(struct task_struct *p, int new_cpu)
4659	{
4660	}
4661
4662	#endif /* !CONFIG_NUMA_BALANCING */
4663
4664	static void
4665	account_entity_enqueue(struct cfs_rq cfs_rq, struct sched_entity se)
4666	{
4667	WARN_ON_ONCE(cfs_rq != cfs_rq_of(se));
4668	update_load_add(&cfs_rq->load, se->load.weight);
4669	if (entity_is_task(se)) {
4670	struct task_struct *p = task_of(se);
4671	struct rq *rq = rq_of(cfs_rq);
4672
4673	account_numa_enqueue(rq, p);
4674	account_llc_enqueue(rq, p);
4675	list_add(&se->group_node, &rq->cfs_tasks);
4676	}
4677	cfs_rq->nr_queued++;
4678	}
4679
4680	static void
4681	account_entity_dequeue(struct cfs_rq cfs_rq, struct sched_entity se)
4682	{
4683	WARN_ON_ONCE(cfs_rq != cfs_rq_of(se));
4684	update_load_sub(&cfs_rq->load, se->load.weight);
4685	if (entity_is_task(se)) {
4686	struct task_struct *p = task_of(se);
4687	struct rq *rq = rq_of(cfs_rq);
4688
4689	account_numa_dequeue(rq, p);
4690	account_llc_dequeue(rq, p);
4691	list_del_init(&se->group_node);
4692	}
4693	cfs_rq->nr_queued--;
4694	}
4695
4696	/*
4697	* Signed add and clamp on underflow.
4698	*
4699	* Explicitly do a load-store to ensure the intermediate value never hits
4700	* memory. This allows lockless observations without ever seeing the negative
4701	* values.
4702	*/
4703	#define add_positive(_ptr, _val) do { \
4704	typeof(_ptr) ptr = (_ptr); \
4705	__signed_scalar_typeof(*ptr) val = (_val); \
4706	typeof(ptr) res, var = READ_ONCE(ptr); \
4707	\
4708	res = var + val; \
4709	\
4710	if (val < 0 && res > var) \
4711	res = 0; \
4712	\
4713	WRITE_ONCE(*ptr, res); \
4714	} while (0)
4715
4716	/*
4717	* Remove and clamp on negative, from a local variable.
4718	*
4719	* A variant of sub_positive(), which does not use explicit load-store
4720	* and is thus optimized for local variable updates.
4721	*/
4722	#define lsub_positive(_ptr, _val) do { \
4723	typeof(_ptr) ptr = (_ptr); \
4724	ptr -= min_t(typeof(ptr), *ptr, _val); \
4725	} while (0)
4726
4727
4728	/*
4729	* Because of rounding, se->util_sum might ends up being +1 more than
4730	* cfs->util_sum. Although this is not a problem by itself, detaching
4731	* a lot of tasks with the rounding problem between 2 updates of
4732	* util_avg (~1ms) can make cfs->util_sum becoming null whereas
4733	* cfs_util_avg is not.
4734	*
4735	* Check that util_sum is still above its lower bound for the new
4736	* util_avg. Given that period_contrib might have moved since the last
4737	* sync, we are only sure that util_sum must be above or equal to
4738	* util_avg * minimum possible divider
4739	*/
4740	#define __update_sa(sa, name, delta_avg, delta_sum) do { \
4741	add_positive(&(sa)->name##_avg, delta_avg); \
4742	add_positive(&(sa)->name##_sum, delta_sum); \
4743	(sa)->name##_sum = max_t(typeof((sa)->name##_sum), \
4744	(sa)->name##_sum, \
4745	(sa)->name##_avg * PELT_MIN_DIVIDER); \
4746	} while (0)
4747
4748	static inline void
4749	enqueue_load_avg(struct cfs_rq cfs_rq, struct sched_entity se)
4750	{
4751	__update_sa(&cfs_rq->avg, load, se->avg.load_avg,
4752	se_weight(se) * se->avg.load_sum);
4753	}
4754
4755	static inline void
4756	dequeue_load_avg(struct cfs_rq cfs_rq, struct sched_entity se)
4757	{
4758	__update_sa(&cfs_rq->avg, load, -se->avg.load_avg,
4759	se_weight(se) * -se->avg.load_sum);
4760	}
4761
4762	static void
4763	rescale_entity(struct sched_entity *se, unsigned long weight, bool rel_vprot)
4764	{
4765	long old_weight = se->h_load.weight;
4766
4767	/*
4768	* VRUNTIME
4769	* --------
4770	*
4771	* COROLLARY #1: The virtual runtime of the entity needs to be
4772	* adjusted if re-weight at !0-lag point.
4773	*
4774	* Proof: For contradiction assume this is not true, so we can
4775	* re-weight without changing vruntime at !0-lag point.
4776	*
4777	* Weight VRuntime Avg-VRuntime
4778	* before w v V
4779	* after w' v' V'
4780	*
4781	* Since lag needs to be preserved through re-weight:
4782	*
4783	* lag = (V - v)w = (V'- v')w', where v = v'
4784	* ==> V' = (V - v)*w/w' + v (1)
4785	*
4786	* Let W be the total weight of the entities before reweight,
4787	* since V' is the new weighted average of entities:
4788	*
4789	* V' = (WV + w'v - wv) / (W + w' - w) (2)
4790	*
4791	* by using (1) & (2) we obtain:
4792	*
4793	* (WV + w'v - wv) / (W + w' - w) = (V - v)*w/w' + v
4794	* ==> (WV-Wv+Wv+w'v-wv)/(W+w'-w) = (V - v)*w/w' + v
4795	* ==> (WV - Wv)/(W + w' - w) + v = (V - v)*w/w' + v
4796	* ==> (V - v)W/(W + w' - w) = (V - v)w/w' (3)
4797	*
4798	* Since we are doing at !0-lag point which means V != v, we
4799	* can simplify (3):
4800	*
4801	* ==> W / (W + w' - w) = w / w'
4802	* ==> Ww' = Ww + ww' - ww
4803	* ==> W * (w' - w) = w * (w' - w)
4804	* ==> W = w (re-weight indicates w' != w)
4805	*
4806	* So the cfs_rq contains only one entity, hence vruntime of
4807	* the entity @v should always equal to the cfs_rq's weighted
4808	* average vruntime @V, which means we will always re-weight
4809	* at 0-lag point, thus breach assumption. Proof completed.
4810	*
4811	*
4812	* COROLLARY #2: Re-weight does NOT affect weighted average
4813	* vruntime of all the entities.
4814	*
4815	* Proof: According to corollary #1, Eq. (1) should be:
4816	*
4817	* (V - v)w = (V' - v')w'
4818	* ==> v' = V' - (V - v)*w/w' (4)
4819	*
4820	* According to the weighted average formula, we have:
4821	*
4822	* V' = (WV - wv + w'v') / (W - w + w')
4823	* = (WV - wv + w'(V' - (V - v)w/w')) / (W - w + w')
4824	* = (WV - wv + w'V' - Vw + wv) / (W - w + w')
4825	* = (WV + w'V' - Vw) / (W - w + w')
4826	*
4827	* ==> V'*(W - w + w') = WV + w'V' - Vw
4828	* ==> V' * (W - w) = (W - w) * V (5)
4829	*
4830	* If the entity is the only one in the cfs_rq, then reweight
4831	* always occurs at 0-lag point, so V won't change. Or else
4832	* there are other entities, hence W != w, then Eq. (5) turns
4833	* into V' = V. So V won't change in either case, proof done.
4834	*
4835	*
4836	* So according to corollary #1 & #2, the effect of re-weight
4837	* on vruntime should be:
4838	*
4839	* v' = V' - (V - v) * w / w' (4)
4840	* = V - (V - v) * w / w'
4841	* = V - vl * w / w'
4842	* = V - vl'
4843	*/
4844	se->vlag = div64_long(se->vlag * old_weight, weight);
4845
4846	/*
4847	* DEADLINE
4848	* --------
4849	*
4850	* When the weight changes, the virtual time slope changes and
4851	* we should adjust the relative virtual deadline accordingly.
4852	*
4853	* d' = v' + (d - v)*w/w'
4854	* = V' - (V - v)w/w' + (d - v)w/w'
4855	* = V - (V - v)w/w' + (d - v)w/w'
4856	* = V + (d - V)*w/w'
4857	*/
4858	if (se->rel_deadline)
4859	se->deadline = div64_long(se->deadline * old_weight, weight);
4860
4861	if (rel_vprot)
4862	se->vprot = div64_long(se->vprot * old_weight, weight);
4863	}
4864
4865	static void reweight_eevdf(struct cfs_rq cfs_rq, struct sched_entity se,
4866	unsigned long weight, bool on_rq)
4867	{
4868	bool curr = cfs_rq->curr == se;
4869	bool rel_vprot = false;
4870	u64 avruntime = 0;
4871
4872	if (se->h_load.weight == weight)
4873	return;
4874
4875	if (on_rq) {
4876	avruntime = avg_vruntime(cfs_rq);
4877	se->vlag = entity_lag(cfs_rq, se, avruntime);
4878	se->deadline -= avruntime;
4879	se->rel_deadline = 1;
4880	if (curr && protect_slice(se)) {
4881	se->vprot -= avruntime;
4882	rel_vprot = true;
4883	}
4884
4885	cfs_rq->h_nr_queued--;
4886	if (!curr)
4887	__dequeue_entity(cfs_rq, se);
4888	}
4889
4890	rescale_entity(se, weight, rel_vprot);
4891
4892	update_load_set(&se->h_load, weight);
4893
4894	if (on_rq) {
4895	if (rel_vprot)
4896	se->vprot += avruntime;
4897	se->deadline += avruntime;
4898	se->rel_deadline = 0;
4899	se->vruntime = avruntime - se->vlag;
4900
4901	if (!curr)
4902	__enqueue_entity(cfs_rq, se);
4903	cfs_rq->h_nr_queued++;
4904	}
4905	}
4906
4907	static void reweight_entity(struct cfs_rq cfs_rq, struct sched_entity se,
4908	unsigned long weight)
4909	{
4910	if (se->load.weight == weight)
4911	return;
4912
4913	if (se->on_rq) {
4914	WARN_ON_ONCE(cfs_rq != cfs_rq_of(se));
4915	update_load_sub(&cfs_rq->load, se->load.weight);
4916	}
4917	dequeue_load_avg(cfs_rq, se);
4918
4919	update_load_set(&se->load, weight);
4920
4921	do {
4922	u32 divider = get_pelt_divider(&se->avg);
4923	se->avg.load_avg = div_u64(se_weight(se) * se->avg.load_sum, divider);
4924	} while (0);
4925
4926	enqueue_load_avg(cfs_rq, se);
4927
4928	if (se->on_rq)
4929	update_load_add(&cfs_rq->load, se->load.weight);
4930	}
4931
4932	/*
4933	* weight = NICE_0_LOAD;
4934	* for_each_entity_se(se)
4935	* weight = __calc_prop_weight(cfs_rq_of(se), se, weight);
4936	*/
4937	static __always_inline
4938	unsigned long __calc_prop_weight(struct cfs_rq cfs_rq, struct sched_entity se,
4939	unsigned long weight)
4940	{
4941	weight *= se->load.weight;
4942	if (parent_entity(se))
4943	weight /= cfs_rq->load.weight;
4944	else
4945	weight /= NICE_0_LOAD;
4946
4947	return max(weight, MIN_SHARES);
4948	}
4949
4950	static void reweight_task_fair(struct rq rq, struct task_struct p,
4951	const struct load_weight *lw)
4952	{
4953	struct sched_entity *se = &p->se;
4954	unsigned long weight = NICE_0_LOAD;
4955
4956	if (se->on_rq)
4957	update_curr_fair(rq);
4958
4959	reweight_entity(cfs_rq_of(se), se, lw->weight);
4960	se->load.inv_weight = lw->inv_weight;
4961
4962	if (!se->on_rq)
4963	return;
4964
4965	for_each_sched_entity(se)
4966	weight = __calc_prop_weight(cfs_rq_of(se), se, weight);
4967
4968	reweight_eevdf(&rq->cfs, &p->se, weight, p->se.on_rq);
4969	}
4970
4971	static inline int throttled_hierarchy(struct cfs_rq *cfs_rq);
4972
4973	#ifdef CONFIG_FAIR_GROUP_SCHED
4974	/*
4975	* All this does is approximate the hierarchical proportion which includes that
4976	* global sum we all love to hate.
4977	*
4978	* That is, the weight of a group entity, is the proportional share of the
4979	* group weight based on the group runqueue weights. That is:
4980	*
4981	* tg->weight * grq->load.weight
4982	* ge->load.weight = ----------------------------- (1)
4983	* \Sum grq->load.weight
4984	*
4985	* Now, because computing that sum is prohibitively expensive to compute (been
4986	* there, done that) we approximate it with this average stuff. The average
4987	* moves slower and therefore the approximation is cheaper and more stable.
4988	*
4989	* So instead of the above, we substitute:
4990	*
4991	* grq->load.weight -> grq->avg.load_avg (2)
4992	*
4993	* which yields the following:
4994	*
4995	* tg->weight * grq->avg.load_avg
4996	* ge->load.weight = ------------------------------ (3)
4997	* tg->load_avg
4998	*
4999	* Where: tg->load_avg ~= \Sum grq->avg.load_avg
5000	*
5001	* That is shares_avg, and it is right (given the approximation (2)).
5002	*
5003	* The problem with it is that because the average is slow -- it was designed
5004	* to be exactly that of course -- this leads to transients in boundary
5005	* conditions. In specific, the case where the group was idle and we start the
5006	* one task. It takes time for our CPU's grq->avg.load_avg to build up,
5007	* yielding bad latency etc..
5008	*
5009	* Now, in that special case (1) reduces to:
5010	*
5011	* tg->weight * grq->load.weight
5012	* ge->load.weight = ----------------------------- = tg->weight (4)
5013	* grp->load.weight
5014	*
5015	* That is, the sum collapses because all other CPUs are idle; the UP scenario.
5016	*
5017	* So what we do is modify our approximation (3) to approach (4) in the (near)
5018	* UP case, like:
5019	*
5020	* ge->load.weight =
5021	*
5022	* tg->weight * grq->load.weight
5023	* --------------------------------------------------- (5)
5024	* tg->load_avg - grq->avg.load_avg + grq->load.weight
5025	*
5026	* But because grq->load.weight can drop to 0, resulting in a divide by zero,
5027	* we need to use grq->avg.load_avg as its lower bound, which then gives:
5028	*
5029	*
5030	* tg->weight * grq->load.weight
5031	* ge->load.weight = ----------------------------- (6)
5032	* tg_load_avg'
5033	*
5034	* Where:
5035	*
5036	* tg_load_avg' = tg->load_avg - grq->avg.load_avg +
5037	* max(grq->load.weight, grq->avg.load_avg)
5038	*
5039	* And that is shares_weight and is icky. In the (near) UP case it approaches
5040	* (4) while in the normal case it approaches (3). It consistently
5041	* overestimates the ge->load.weight and therefore:
5042	*
5043	* \Sum ge->load.weight >= tg->weight
5044	*
5045	* hence icky!
5046	*/
5047	static long __calc_smp_shares(struct cfs_rq *cfs_rq, long tg_shares, long shares_max)
5048	{
5049	struct task_group *tg = cfs_rq->tg;
5050	long tg_weight, load, shares;
5051
5052	load = max(scale_load_down(cfs_rq->load.weight), cfs_rq->avg.load_avg);
5053
5054	tg_weight = atomic_long_read(&tg->load_avg);
5055
5056	/* Ensure tg_weight >= load */
5057	tg_weight -= cfs_rq->tg_load_avg_contrib;
5058	tg_weight += load;
5059
5060	shares = (tg_shares * load);
5061	if (tg_weight)
5062	shares /= tg_weight;
5063
5064	/*
5065	* MIN_SHARES has to be unscaled here to support per-CPU partitioning
5066	* of a group with small tg->shares value. It is a floor value which is
5067	* assigned as a minimum load.weight to the sched_entity representing
5068	* the group on a CPU.
5069	*
5070	* E.g. on 64-bit for a group with tg->shares of scale_load(15)=15*1024
5071	* on an 8-core system with 8 tasks each runnable on one CPU shares has
5072	* to be 1510241/8=1920 instead of scale_load(MIN_SHARES)=2*1024. In
5073	* case no task is runnable on a CPU MIN_SHARES=2 should be returned
5074	* instead of 0.
5075	*/
5076	return clamp_t(long, shares, MIN_SHARES, shares_max);
5077	}
5078
5079	static int tg_cpus(struct task_group *tg)
5080	{
5081	int nr = num_online_cpus();
5082
5083	if (cpusets_enabled()) {
5084	struct cgroup *cgrp = tg->css.cgroup;
5085	if (cgrp)
5086	nr = cpuset_num_cpus(cgrp);
5087	}
5088
5089	/*
5090	* An empty cpuset would propagate a 0 shares_max into
5091	* __calc_smp_shares(), where clamp() yields hi when hi < lo and so
5092	* defeats the MIN_SHARES floor. Match tg_tasks(), which floors at 1.
5093	*/
5094	return max(nr, 1);
5095	}
5096
5097	static inline int tg_tasks(struct task_group *tg)
5098	{
5099	return max(1, atomic_long_read(&tg->runnable_avg) >> SCHED_CAPACITY_SHIFT);
5100	}
5101
5102	/*
5103	* Func: fraction(nr_tasks * tg->shares)
5104	*
5105	* Scale tg->shares by the number of tasks.
5106	*/
5107	static long calc_tasks_shares(struct cfs_rq *cfs_rq)
5108	{
5109	struct task_group *tg = cfs_rq->tg;
5110	int nr = tg_tasks(tg);
5111	long tg_shares = READ_ONCE(tg->shares);
5112	return __calc_smp_shares(cfs_rq, nr * tg_shares, nr * tg_shares);
5113	}
5114
5115	/*
5116	* Func: min(fraction(nr_cpus * tg->shares), nice -20)
5117	*
5118	* Scale tg->shares by the maximal number of CPUs; but clip the max shares at
5119	* nice -20, otherwise a single spinner on a 512 CPU machine would result in
5120	* 512*NICE_0_LOAD, which is also crazy.
5121	*/
5122	static long calc_max_shares(struct cfs_rq *cfs_rq)
5123	{
5124	struct task_group *tg = cfs_rq->tg;
5125	int nr = tg_cpus(tg);
5126	long tg_shares = READ_ONCE(tg->shares);
5127	long max_shares = scale_load(sched_prio_to_weight[0]);
5128	return __calc_smp_shares(cfs_rq, tg_shares * nr, max_shares);
5129	}
5130
5131	/*
5132	* Func: fraction(nr * tg->shares); nr = min(nr_tasks, nr_cpus)
5133	*
5134	* Scales between "smp" and "max" in a natural way. No longer needs clipping
5135	* since there are no unnatural inflations like with "max".
5136	*/
5137	static long calc_concur_shares(struct cfs_rq *cfs_rq)
5138	{
5139	struct task_group *tg = cfs_rq->tg;
5140	int nr = min(tg_tasks(tg), tg_cpus(tg));
5141	long tg_shares = READ_ONCE(tg->shares);
5142	return __calc_smp_shares(cfs_rq, nr * tg_shares, nr * tg_shares);
5143	}
5144
5145	/*
5146	* Func: fraction(tg->shares)
5147	*
5148	* This infamously results in tiny shares when you have many CPUs.
5149	*/
5150	static long calc_smp_shares(struct cfs_rq *cfs_rq)
5151	{
5152	struct task_group *tg = cfs_rq->tg;
5153	long tg_shares = READ_ONCE(tg->shares);
5154	return __calc_smp_shares(cfs_rq, tg_shares, tg_shares);
5155	}
5156
5157	/*
5158	* Ignore this pesky SMP stuff, use (4).
5159	*/
5160	static long calc_up_shares(struct cfs_rq *cfs_rq)
5161	{
5162	struct task_group *tg = cfs_rq->tg;
5163	return READ_ONCE(tg->shares);
5164	}
5165
5166	DEFINE_STATIC_CALL(calc_group_shares, calc_concur_shares);
5167
5168	void __sched_cgroup_mode_update(int mode)
5169	{
5170	long (func)(struct cfs_rq );
5171	switch (mode) {
5172	case 0:
5173	func = &calc_up_shares;
5174	break;
5175	case 1:
5176	func = &calc_smp_shares;
5177	break;
5178	case 2:
5179	default:
5180	func = &calc_concur_shares;
5181	break;
5182	case 3:
5183	func = &calc_max_shares;
5184	break;
5185	case 4:
5186	func = &calc_tasks_shares;
5187	break;
5188	}
5189	static_call_update(calc_group_shares, func);
5190	}
5191
5192	/*
5193	* Recomputes the group entity based on the current state of its group
5194	* runqueue.
5195	*/
5196	static void update_cfs_group(struct sched_entity *se)
5197	{
5198	struct cfs_rq *gcfs_rq = group_cfs_rq(se);
5199	long shares;
5200
5201	/*
5202	* When a group becomes empty, preserve its weight. This matters for
5203	* DELAY_DEQUEUE.
5204	*/
5205	if (!gcfs_rq \|\| !gcfs_rq->load.weight)
5206	return;
5207
5208	shares = static_call(calc_group_shares)(gcfs_rq);
5209	reweight_entity(cfs_rq_of(se), se, shares);
5210	}
5211
5212	#else /* !CONFIG_FAIR_GROUP_SCHED: */
5213	static inline void update_cfs_group(struct sched_entity *se)
5214	{
5215	}
5216	#endif /* !CONFIG_FAIR_GROUP_SCHED */
5217
5218	static inline void cfs_rq_util_change(struct cfs_rq *cfs_rq, int flags)
5219	{
5220	struct rq *rq = rq_of(cfs_rq);
5221
5222	if (&rq->cfs == cfs_rq) {
5223	/*
5224	* There are a few boundary cases this might miss but it should
5225	* get called often enough that that should (hopefully) not be
5226	* a real problem.
5227	*
5228	* It will not get called when we go idle, because the idle
5229	* thread is a different class (!fair), nor will the utilization
5230	* number include things like RT tasks.
5231	*
5232	* As is, the util number is not freq-invariant (we'd have to
5233	* implement arch_scale_freq_capacity() for that).
5234	*
5235	* See cpu_util_cfs().
5236	*/
5237	cpufreq_update_util(rq, flags);
5238	}
5239	}
5240
5241	static inline bool load_avg_is_decayed(struct sched_avg *sa)
5242	{
5243	if (sa->load_sum)
5244	return false;
5245
5246	if (sa->util_sum)
5247	return false;
5248
5249	if (sa->runnable_sum)
5250	return false;
5251
5252	/*
5253	* _avg must be null when _sum are null because _avg = _sum / divider
5254	* Make sure that rounding and/or propagation of PELT values never
5255	* break this.
5256	*/
5257	WARN_ON_ONCE(sa->load_avg \|\|
5258	sa->util_avg \|\|
5259	sa->runnable_avg);
5260
5261	return true;
5262	}
5263
5264	static inline u64 cfs_rq_last_update_time(struct cfs_rq *cfs_rq)
5265	{
5266	return u64_u32_load_copy(cfs_rq->avg.last_update_time,
5267	cfs_rq->last_update_time_copy);
5268	}
5269	#ifdef CONFIG_FAIR_GROUP_SCHED
5270	/*
5271	* Because list_add_leaf_cfs_rq always places a child cfs_rq on the list
5272	* immediately before a parent cfs_rq, and cfs_rqs are removed from the list
5273	* bottom-up, we only have to test whether the cfs_rq before us on the list
5274	* is our child.
5275	* If cfs_rq is not on the list, test whether a child needs its to be added to
5276	* connect a branch to the tree * (see list_add_leaf_cfs_rq() for details).
5277	*/
5278	static inline bool child_cfs_rq_on_list(struct cfs_rq *cfs_rq)
5279	{
5280	struct cfs_rq *prev_cfs_rq;
5281	struct list_head *prev;
5282	struct rq *rq = rq_of(cfs_rq);
5283
5284	if (cfs_rq->on_list) {
5285	prev = cfs_rq->leaf_cfs_rq_list.prev;
5286	} else {
5287	prev = rq->tmp_alone_branch;
5288	}
5289
5290	if (prev == &rq->leaf_cfs_rq_list)
5291	return false;
5292
5293	prev_cfs_rq = container_of(prev, struct cfs_rq, leaf_cfs_rq_list);
5294
5295	return (prev_cfs_rq->tg->parent == cfs_rq->tg);
5296	}
5297
5298	static inline bool cfs_rq_is_decayed(struct cfs_rq *cfs_rq)
5299	{
5300	if (cfs_rq->load.weight)
5301	return false;
5302
5303	if (!load_avg_is_decayed(&cfs_rq->avg))
5304	return false;
5305
5306	if (child_cfs_rq_on_list(cfs_rq))
5307	return false;
5308
5309	if (cfs_rq->tg_load_avg_contrib)
5310	return false;
5311
5312	return true;
5313	}
5314
5315	/**
5316	* update_tg_load_avg - update the tg's load avg
5317	* @cfs_rq: the cfs_rq whose avg changed
5318	*
5319	* This function 'ensures': tg->load_avg := \Sum tg->cfs_rq[]->avg.load.
5320	* However, because tg->load_avg is a global value there are performance
5321	* considerations.
5322	*
5323	* In order to avoid having to look at the other cfs_rq's, we use a
5324	* differential update where we store the last value we propagated. This in
5325	* turn allows skipping updates if the differential is 'small'.
5326	*
5327	* Updating tg's load_avg is necessary before update_cfs_group().
5328	*/
5329	static inline void update_tg_load_avg(struct cfs_rq *cfs_rq)
5330	{
5331	long dl, dr;
5332	u64 now;
5333
5334	/*
5335	* No need to update load_avg for root_task_group as it is not used.
5336	*/
5337	if (cfs_rq->tg == &root_task_group)
5338	return;
5339
5340	/* rq has been offline and doesn't contribute to the share anymore: */
5341	if (!cpu_active(cpu_of(rq_of(cfs_rq))))
5342	return;
5343
5344	/*
5345	* For migration heavy workloads, access to tg->load_avg can be
5346	* unbound. Limit the update rate to at most once per ms.
5347	*/
5348	now = rq_clock(rq_of(cfs_rq));
5349	if (now - cfs_rq->last_update_tg_load_avg < NSEC_PER_MSEC)
5350	return;
5351
5352	dl = cfs_rq->avg.load_avg - cfs_rq->tg_load_avg_contrib;
5353	dr = cfs_rq->avg.runnable_avg - cfs_rq->tg_runnable_avg_contrib;
5354	if (abs(dl) > cfs_rq->tg_load_avg_contrib / 64 \|\|
5355	abs(dr) > cfs_rq->tg_runnable_avg_contrib / 64) {
5356	atomic_long_add(dl, &cfs_rq->tg->load_avg);
5357	atomic_long_add(dr, &cfs_rq->tg->runnable_avg);
5358	cfs_rq->tg_load_avg_contrib = cfs_rq->avg.load_avg;
5359	cfs_rq->tg_runnable_avg_contrib = cfs_rq->avg.runnable_avg;
5360	cfs_rq->last_update_tg_load_avg = now;
5361	}
5362	}
5363
5364	static inline void clear_tg_load_avg(struct cfs_rq *cfs_rq)
5365	{
5366	long dl, dr;
5367	u64 now;
5368
5369	/*
5370	* No need to update load_avg for root_task_group, as it is not used.
5371	*/
5372	if (cfs_rq->tg == &root_task_group)
5373	return;
5374
5375	now = rq_clock(rq_of(cfs_rq));
5376	dl = 0 - cfs_rq->tg_load_avg_contrib;
5377	dr = 0 - cfs_rq->tg_runnable_avg_contrib;
5378	atomic_long_add(dl, &cfs_rq->tg->load_avg);
5379	atomic_long_add(dr, &cfs_rq->tg->runnable_avg);
5380	cfs_rq->tg_load_avg_contrib = 0;
5381	cfs_rq->tg_runnable_avg_contrib = 0;
5382	cfs_rq->last_update_tg_load_avg = now;
5383	}
5384
5385	/* CPU offline callback: */
5386	static void __maybe_unused clear_tg_offline_cfs_rqs(struct rq *rq)
5387	{
5388	struct task_group *tg;
5389
5390	lockdep_assert_rq_held(rq);
5391
5392	/*
5393	* The rq clock has already been updated in
5394	* set_rq_offline(), so we should skip updating
5395	* the rq clock again in unthrottle_cfs_rq().
5396	*/
5397	rq_clock_start_loop_update(rq);
5398
5399	guard(rcu)();
5400
5401	list_for_each_entry_rcu(tg, &task_groups, list) {
5402	struct cfs_rq *cfs_rq = tg_cfs_rq(tg, cpu_of(rq));
5403
5404	clear_tg_load_avg(cfs_rq);
5405	}
5406
5407	rq_clock_stop_loop_update(rq);
5408	}
5409
5410	/*
5411	* Called within set_task_rq() right before setting a task's CPU. The
5412	* caller only guarantees p->pi_lock is held; no other assumptions,
5413	* including the state of rq->lock, should be made.
5414	*/
5415	void set_task_rq_fair(struct sched_entity *se,
5416	struct cfs_rq prev, struct cfs_rq next)
5417	{
5418	u64 p_last_update_time;
5419	u64 n_last_update_time;
5420
5421	if (!sched_feat(ATTACH_AGE_LOAD))
5422	return;
5423
5424	/*
5425	* We are supposed to update the task to "current" time, then its up to
5426	* date and ready to go to new CPU/cfs_rq. But we have difficulty in
5427	* getting what current time is, so simply throw away the out-of-date
5428	* time. This will result in the wakee task is less decayed, but giving
5429	* the wakee more load sounds not bad.
5430	*/
5431	if (!(se->avg.last_update_time && prev))
5432	return;
5433
5434	p_last_update_time = cfs_rq_last_update_time(prev);
5435	n_last_update_time = cfs_rq_last_update_time(next);
5436
5437	__update_load_avg_blocked_se(p_last_update_time, se);
5438	se->avg.last_update_time = n_last_update_time;
5439	}
5440
5441	/*
5442	* When on migration a sched_entity joins/leaves the PELT hierarchy, we need to
5443	* propagate its contribution. The key to this propagation is the invariant
5444	* that for each group:
5445	*
5446	* ge->avg == grq->avg (1)
5447	*
5448	* _IFF_ we look at the pure running and runnable sums. Because they
5449	* represent the very same entity, just at different points in the hierarchy.
5450	*
5451	* Per the above update_tg_cfs_util() and update_tg_cfs_runnable() are trivial
5452	* and simply copies the running/runnable sum over (but still wrong, because
5453	* the group entity and group rq do not have their PELT windows aligned).
5454	*
5455	* However, update_tg_cfs_load() is more complex. So we have:
5456	*
5457	* ge->avg.load_avg = ge->load.weight * ge->avg.runnable_avg (2)
5458	*
5459	* And since, like util, the runnable part should be directly transferable,
5460	* the following would _appear_ to be the straight forward approach:
5461	*
5462	* grq->avg.load_avg = grq->load.weight * grq->avg.runnable_avg (3)
5463	*
5464	* And per (1) we have:
5465	*
5466	* ge->avg.runnable_avg == grq->avg.runnable_avg
5467	*
5468	* Which gives:
5469	*
5470	* ge->load.weight * grq->avg.load_avg
5471	* ge->avg.load_avg = ----------------------------------- (4)
5472	* grq->load.weight
5473	*
5474	* Except that is wrong!
5475	*
5476	* Because while for entities historical weight is not important and we
5477	* really only care about our future and therefore can consider a pure
5478	* runnable sum, runqueues can NOT do this.
5479	*
5480	* We specifically want runqueues to have a load_avg that includes
5481	* historical weights. Those represent the blocked load, the load we expect
5482	* to (shortly) return to us. This only works by keeping the weights as
5483	* integral part of the sum. We therefore cannot decompose as per (3).
5484	*
5485	* Another reason this doesn't work is that runnable isn't a 0-sum entity.
5486	* Imagine a rq with 2 tasks that each are runnable 2/3 of the time. Then the
5487	* rq itself is runnable anywhere between 2/3 and 1 depending on how the
5488	* runnable section of these tasks overlap (or not). If they were to perfectly
5489	* align the rq as a whole would be runnable 2/3 of the time. If however we
5490	* always have at least 1 runnable task, the rq as a whole is always runnable.
5491	*
5492	* So we'll have to approximate.. :/
5493	*
5494	* Given the constraint:
5495	*
5496	* ge->avg.running_sum <= ge->avg.runnable_sum <= LOAD_AVG_MAX
5497	*
5498	* We can construct a rule that adds runnable to a rq by assuming minimal
5499	* overlap.
5500	*
5501	* On removal, we'll assume each task is equally runnable; which yields:
5502	*
5503	* grq->avg.runnable_sum = grq->avg.load_sum / grq->load.weight
5504	*
5505	* XXX: only do this for the part of runnable > running ?
5506	*
5507	*/
5508	static inline void
5509	update_tg_cfs_util(struct cfs_rq cfs_rq, struct sched_entity se, struct cfs_rq *gcfs_rq)
5510	{
5511	long delta_sum, delta_avg = gcfs_rq->avg.util_avg - se->avg.util_avg;
5512	u32 new_sum, divider;
5513
5514	/* Nothing to update */
5515	if (!delta_avg)
5516	return;
5517
5518	/*
5519	* cfs_rq->avg.period_contrib can be used for both cfs_rq and se.
5520	* See ___update_load_avg() for details.
5521	*/
5522	divider = get_pelt_divider(&cfs_rq->avg);
5523
5524	/* Set new sched_entity's utilization */
5525	se->avg.util_avg = gcfs_rq->avg.util_avg;
5526	new_sum = se->avg.util_avg * divider;
5527	delta_sum = (long)new_sum - (long)se->avg.util_sum;
5528	se->avg.util_sum = new_sum;
5529
5530	/* Update parent cfs_rq utilization */
5531	__update_sa(&cfs_rq->avg, util, delta_avg, delta_sum);
5532	}
5533
5534	static inline void
5535	update_tg_cfs_runnable(struct cfs_rq cfs_rq, struct sched_entity se, struct cfs_rq *gcfs_rq)
5536	{
5537	long delta_sum, delta_avg = gcfs_rq->avg.runnable_avg - se->avg.runnable_avg;
5538	u64 new_sum;
5539	u32 divider;
5540
5541	/* Nothing to update */
5542	if (!delta_avg)
5543	return;
5544
5545	/*
5546	* cfs_rq->avg.period_contrib can be used for both cfs_rq and se.
5547	* See ___update_load_avg() for details.
5548	*/
5549	divider = get_pelt_divider(&cfs_rq->avg);
5550
5551	/* Set new sched_entity's runnable */
5552	se->avg.runnable_avg = gcfs_rq->avg.runnable_avg;
5553	new_sum = (u64)se->avg.runnable_avg * divider;
5554	delta_sum = (long)new_sum - (long)se->avg.runnable_sum;
5555	se->avg.runnable_sum = new_sum;
5556
5557	/* Update parent cfs_rq runnable */
5558	__update_sa(&cfs_rq->avg, runnable, delta_avg, delta_sum);
5559	}
5560
5561	static inline void
5562	update_tg_cfs_load(struct cfs_rq cfs_rq, struct sched_entity se, struct cfs_rq *gcfs_rq)
5563	{
5564	long delta_avg, running_sum, runnable_sum = gcfs_rq->prop_runnable_sum;
5565	unsigned long load_avg;
5566	u64 load_sum = 0;
5567	s64 delta_sum;
5568	u32 divider;
5569
5570	if (!runnable_sum)
5571	return;
5572
5573	gcfs_rq->prop_runnable_sum = 0;
5574
5575	/*
5576	* cfs_rq->avg.period_contrib can be used for both cfs_rq and se.
5577	* See ___update_load_avg() for details.
5578	*/
5579	divider = get_pelt_divider(&cfs_rq->avg);
5580
5581	if (runnable_sum >= 0) {
5582	/*
5583	* Add runnable; clip at LOAD_AVG_MAX. Reflects that until
5584	* the CPU is saturated running == runnable.
5585	*/
5586	runnable_sum += se->avg.load_sum;
5587	runnable_sum = min_t(long, runnable_sum, divider);
5588	} else {
5589	/*
5590	* Estimate the new unweighted runnable_sum of the gcfs_rq by
5591	* assuming all tasks are equally runnable.
5592	*/
5593	if (scale_load_down(gcfs_rq->load.weight)) {
5594	load_sum = div_u64(gcfs_rq->avg.load_sum,
5595	scale_load_down(gcfs_rq->load.weight));
5596	}
5597
5598	/* But make sure to not inflate se's runnable */
5599	runnable_sum = min(se->avg.load_sum, load_sum);
5600	}
5601
5602	/*
5603	* runnable_sum can't be lower than running_sum
5604	* Rescale running sum to be in the same range as runnable sum
5605	* running_sum is in [0 : LOAD_AVG_MAX << SCHED_CAPACITY_SHIFT]
5606	* runnable_sum is in [0 : LOAD_AVG_MAX]
5607	*/
5608	running_sum = se->avg.util_sum >> SCHED_CAPACITY_SHIFT;
5609	runnable_sum = max(runnable_sum, running_sum);
5610
5611	load_sum = se_weight(se) * runnable_sum;
5612	load_avg = div_u64(load_sum, divider);
5613
5614	delta_avg = load_avg - se->avg.load_avg;
5615	if (!delta_avg)
5616	return;
5617
5618	delta_sum = load_sum - (s64)se_weight(se) * se->avg.load_sum;
5619
5620	se->avg.load_sum = runnable_sum;
5621	se->avg.load_avg = load_avg;
5622	__update_sa(&cfs_rq->avg, load, delta_avg, delta_sum);
5623	}
5624
5625	static inline void add_tg_cfs_propagate(struct cfs_rq *cfs_rq, long runnable_sum)
5626	{
5627	cfs_rq->propagate = 1;
5628	cfs_rq->prop_runnable_sum += runnable_sum;
5629	}
5630
5631	/* Update task and its cfs_rq load average */
5632	static inline int propagate_entity_load_avg(struct sched_entity *se)
5633	{
5634	struct cfs_rq cfs_rq, gcfs_rq;
5635
5636	if (entity_is_task(se))
5637	return 0;
5638
5639	gcfs_rq = group_cfs_rq(se);
5640	if (!gcfs_rq->propagate)
5641	return 0;
5642
5643	gcfs_rq->propagate = 0;
5644
5645	cfs_rq = cfs_rq_of(se);
5646
5647	add_tg_cfs_propagate(cfs_rq, gcfs_rq->prop_runnable_sum);
5648
5649	update_tg_cfs_util(cfs_rq, se, gcfs_rq);
5650	update_tg_cfs_runnable(cfs_rq, se, gcfs_rq);
5651	update_tg_cfs_load(cfs_rq, se, gcfs_rq);
5652
5653	trace_pelt_cfs_tp(cfs_rq);
5654	trace_pelt_se_tp(se);
5655
5656	return 1;
5657	}
5658
5659	/*
5660	* Check if we need to update the load and the utilization of a blocked
5661	* group_entity:
5662	*/
5663	static inline bool skip_blocked_update(struct sched_entity *se)
5664	{
5665	struct cfs_rq *gcfs_rq = group_cfs_rq(se);
5666
5667	/*
5668	* If sched_entity still have not zero load or utilization, we have to
5669	* decay it:
5670	*/
5671	if (se->avg.load_avg \|\| se->avg.util_avg)
5672	return false;
5673
5674	/*
5675	* If there is a pending propagation, we have to update the load and
5676	* the utilization of the sched_entity:
5677	*/
5678	if (gcfs_rq->propagate)
5679	return false;
5680
5681	/*
5682	* Otherwise, the load and the utilization of the sched_entity is
5683	* already zero and there is no pending propagation, so it will be a
5684	* waste of time to try to decay it:
5685	*/
5686	return true;
5687	}
5688
5689	#else /* !CONFIG_FAIR_GROUP_SCHED: */
5690
5691	static inline void update_tg_load_avg(struct cfs_rq *cfs_rq) {}
5692
5693	static inline void clear_tg_offline_cfs_rqs(struct rq *rq) {}
5694
5695	static inline int propagate_entity_load_avg(struct sched_entity *se)
5696	{
5697	return 0;
5698	}
5699
5700	static inline void add_tg_cfs_propagate(struct cfs_rq *cfs_rq, long runnable_sum) {}
5701
5702	#endif /* !CONFIG_FAIR_GROUP_SCHED */
5703
5704	#ifdef CONFIG_NO_HZ_COMMON
5705	static inline void migrate_se_pelt_lag(struct sched_entity *se)
5706	{
5707	u64 throttled = 0, now, lut;
5708	struct cfs_rq *cfs_rq;
5709	struct rq *rq;
5710	bool is_idle;
5711
5712	if (load_avg_is_decayed(&se->avg))
5713	return;
5714
5715	cfs_rq = cfs_rq_of(se);
5716	rq = rq_of(cfs_rq);
5717
5718	rcu_read_lock();
5719	is_idle = is_idle_task(rcu_dereference_all(rq->curr));
5720	rcu_read_unlock();
5721
5722	/*
5723	* The lag estimation comes with a cost we don't want to pay all the
5724	* time. Hence, limiting to the case where the source CPU is idle and
5725	* we know we are at the greatest risk to have an outdated clock.
5726	*/
5727	if (!is_idle)
5728	return;
5729
5730	/*
5731	* Estimated "now" is: last_update_time + cfs_idle_lag + rq_idle_lag, where:
5732	*
5733	* last_update_time (the cfs_rq's last_update_time)
5734	* = cfs_rq_clock_pelt()@cfs_rq_idle
5735	* = rq_clock_pelt()@cfs_rq_idle
5736	* - cfs->throttled_clock_pelt_time@cfs_rq_idle
5737	*
5738	* cfs_idle_lag (delta between rq's update and cfs_rq's update)
5739	* = rq_clock_pelt()@rq_idle - rq_clock_pelt()@cfs_rq_idle
5740	*
5741	* rq_idle_lag (delta between now and rq's update)
5742	* = sched_clock_cpu() - rq_clock()@rq_idle
5743	*
5744	* We can then write:
5745	*
5746	* now = rq_clock_pelt()@rq_idle - cfs->throttled_clock_pelt_time +
5747	* sched_clock_cpu() - rq_clock()@rq_idle
5748	* Where:
5749	* rq_clock_pelt()@rq_idle is rq->clock_pelt_idle
5750	* rq_clock()@rq_idle is rq->clock_idle
5751	* cfs->throttled_clock_pelt_time@cfs_rq_idle
5752	* is cfs_rq->throttled_pelt_idle
5753	*/
5754
5755	#ifdef CONFIG_CFS_BANDWIDTH
5756	throttled = u64_u32_load(cfs_rq->throttled_pelt_idle);
5757	/* The clock has been stopped for throttling */
5758	if (throttled == U64_MAX)
5759	return;
5760	#endif
5761	now = u64_u32_load(rq->clock_pelt_idle);
5762	/*
5763	* Paired with _update_idle_rq_clock_pelt(). It ensures at the worst case
5764	* is observed the old clock_pelt_idle value and the new clock_idle,
5765	* which lead to an underestimation. The opposite would lead to an
5766	* overestimation.
5767	*/
5768	smp_rmb();
5769	lut = cfs_rq_last_update_time(cfs_rq);
5770
5771	now -= throttled;
5772	if (now < lut)
5773	/*
5774	* cfs_rq->avg.last_update_time is more recent than our
5775	* estimation, let's use it.
5776	*/
5777	now = lut;
5778	else
5779	now += sched_clock_cpu(cpu_of(rq)) - u64_u32_load(rq->clock_idle);
5780
5781	__update_load_avg_blocked_se(now, se);
5782	}
5783	#else /* !CONFIG_NO_HZ_COMMON: */
5784	static void migrate_se_pelt_lag(struct sched_entity *se) {}
5785	#endif /* !CONFIG_NO_HZ_COMMON */
5786
5787	/**
5788	* update_cfs_rq_load_avg - update the cfs_rq's load/util averages
5789	* @now: current time, as per cfs_rq_clock_pelt()
5790	* @cfs_rq: cfs_rq to update
5791	*
5792	* The cfs_rq avg is the direct sum of all its entities (blocked and runnable)
5793	* avg. The immediate corollary is that all (fair) tasks must be attached.
5794	*
5795	* cfs_rq->avg is used for task_h_load() and update_cfs_group() for example.
5796	*
5797	* Return: true if the load decayed or we removed load.
5798	*
5799	* Since both these conditions indicate a changed cfs_rq->avg.load we should
5800	* call update_tg_load_avg() when this function returns true.
5801	*/
5802	static inline int
5803	update_cfs_rq_load_avg(u64 now, struct cfs_rq *cfs_rq)
5804	{
5805	unsigned long removed_load = 0, removed_util = 0, removed_runnable = 0;
5806	struct sched_avg *sa = &cfs_rq->avg;
5807	int decayed = 0;
5808
5809	if (cfs_rq->removed.nr) {
5810	unsigned long r;
5811	u32 divider = get_pelt_divider(&cfs_rq->avg);
5812
5813	raw_spin_lock(&cfs_rq->removed.lock);
5814	swap(cfs_rq->removed.util_avg, removed_util);
5815	swap(cfs_rq->removed.load_avg, removed_load);
5816	swap(cfs_rq->removed.runnable_avg, removed_runnable);
5817	cfs_rq->removed.nr = 0;
5818	raw_spin_unlock(&cfs_rq->removed.lock);
5819
5820	r = removed_load;
5821	__update_sa(sa, load, -r, -r*divider);
5822
5823	r = removed_util;
5824	__update_sa(sa, util, -r, -r*divider);
5825
5826	r = removed_runnable;
5827	__update_sa(sa, runnable, -r, -r*divider);
5828
5829	/*
5830	* removed_runnable is the unweighted version of removed_load so we
5831	* can use it to estimate removed_load_sum.
5832	*/
5833	add_tg_cfs_propagate(cfs_rq,
5834	-(long)(removed_runnable * divider) >> SCHED_CAPACITY_SHIFT);
5835
5836	decayed = 1;
5837	}
5838
5839	decayed \|= __update_load_avg_cfs_rq(now, cfs_rq);
5840	u64_u32_store_copy(sa->last_update_time,
5841	cfs_rq->last_update_time_copy,
5842	sa->last_update_time);
5843	return decayed;
5844	}
5845
5846	/**
5847	* attach_entity_load_avg - attach this entity to its cfs_rq load avg
5848	* @cfs_rq: cfs_rq to attach to
5849	* @se: sched_entity to attach
5850	*
5851	* Must call update_cfs_rq_load_avg() before this, since we rely on
5852	* cfs_rq->avg.last_update_time being current.
5853	*/
5854	static void attach_entity_load_avg(struct cfs_rq cfs_rq, struct sched_entity se)
5855	{
5856	/*
5857	* cfs_rq->avg.period_contrib can be used for both cfs_rq and se.
5858	* See ___update_load_avg() for details.
5859	*/
5860	u32 divider = get_pelt_divider(&cfs_rq->avg);
5861
5862	/*
5863	* When we attach the @se to the @cfs_rq, we must align the decay
5864	* window because without that, really weird and wonderful things can
5865	* happen.
5866	*
5867	* XXX illustrate
5868	*/
5869	se->avg.last_update_time = cfs_rq->avg.last_update_time;
5870	se->avg.period_contrib = cfs_rq->avg.period_contrib;
5871
5872	/*
5873	* Hell(o) Nasty stuff.. we need to recompute _sum based on the new
5874	* period_contrib. This isn't strictly correct, but since we're
5875	* entirely outside of the PELT hierarchy, nobody cares if we truncate
5876	* _sum a little.
5877	*/
5878	se->avg.util_sum = se->avg.util_avg * divider;
5879
5880	se->avg.runnable_sum = se->avg.runnable_avg * divider;
5881
5882	se->avg.load_sum = se->avg.load_avg * divider;
5883	if (se_weight(se) < se->avg.load_sum)
5884	se->avg.load_sum = div_u64(se->avg.load_sum, se_weight(se));
5885	else
5886	se->avg.load_sum = 1;
5887
5888	enqueue_load_avg(cfs_rq, se);
5889	cfs_rq->avg.util_avg += se->avg.util_avg;
5890	cfs_rq->avg.util_sum += se->avg.util_sum;
5891	cfs_rq->avg.runnable_avg += se->avg.runnable_avg;
5892	cfs_rq->avg.runnable_sum += se->avg.runnable_sum;
5893
5894	add_tg_cfs_propagate(cfs_rq, se->avg.load_sum);
5895
5896	cfs_rq_util_change(cfs_rq, 0);
5897
5898	trace_pelt_cfs_tp(cfs_rq);
5899	}
5900
5901	/**
5902	* detach_entity_load_avg - detach this entity from its cfs_rq load avg
5903	* @cfs_rq: cfs_rq to detach from
5904	* @se: sched_entity to detach
5905	*
5906	* Must call update_cfs_rq_load_avg() before this, since we rely on
5907	* cfs_rq->avg.last_update_time being current.
5908	*/
5909	static void detach_entity_load_avg(struct cfs_rq cfs_rq, struct sched_entity se)
5910	{
5911	dequeue_load_avg(cfs_rq, se);
5912	__update_sa(&cfs_rq->avg, util, -se->avg.util_avg, -se->avg.util_sum);
5913	__update_sa(&cfs_rq->avg, runnable, -se->avg.runnable_avg, -se->avg.runnable_sum);
5914
5915	add_tg_cfs_propagate(cfs_rq, -se->avg.load_sum);
5916
5917	cfs_rq_util_change(cfs_rq, 0);
5918
5919	trace_pelt_cfs_tp(cfs_rq);
5920	}
5921
5922	#define UTIL_EST_MARGIN (SCHED_CAPACITY_SCALE / 100)
5923
5924	static inline void util_est_update(struct sched_entity *se)
5925	{
5926	unsigned int ewma, dequeued, last_ewma_diff;
5927
5928	if (!sched_feat(UTIL_EST))
5929	return;
5930
5931	/* Get current estimate of utilization */
5932	ewma = READ_ONCE(se->avg.util_est);
5933
5934	/*
5935	* If the PELT values haven't changed since enqueue time,
5936	* skip the util_est update.
5937	*/
5938	if (ewma & UTIL_AVG_UNCHANGED)
5939	return;
5940
5941	/* Get utilization at dequeue */
5942	dequeued = READ_ONCE(se->avg.util_avg);
5943
5944	/*
5945	* Reset EWMA on utilization increases, the moving average is used only
5946	* to smooth utilization decreases.
5947	*/
5948	if (ewma <= dequeued) {
5949	ewma = dequeued;
5950	goto done;
5951	}
5952
5953	/*
5954	* Skip update of task's estimated utilization when its members are
5955	* already ~1% close to its last activation value.
5956	*/
5957	last_ewma_diff = ewma - dequeued;
5958	if (last_ewma_diff < UTIL_EST_MARGIN)
5959	goto done;
5960
5961	/*
5962	* To avoid underestimate of task utilization, skip updates of EWMA if
5963	* we cannot grant that thread got all CPU time it wanted.
5964	*/
5965	if ((dequeued + UTIL_EST_MARGIN) < READ_ONCE(se->avg.runnable_avg))
5966	goto done;
5967
5968	/*
5969	* Update Task's estimated utilization
5970	*
5971	* When *p completes an activation we can consolidate another sample
5972	* of the task size. This is done by using this value to update the
5973	* Exponential Weighted Moving Average (EWMA):
5974	*
5975	* ewma(t) = w * task_util(p) + (1-w) * ewma(t-1)
5976	* = w * task_util(p) + ewma(t-1) - w * ewma(t-1)
5977	* = w * (task_util(p) - ewma(t-1)) + ewma(t-1)
5978	* = w * ( -last_ewma_diff ) + ewma(t-1)
5979	* = w * (-last_ewma_diff + ewma(t-1) / w)
5980	*
5981	* Where 'w' is the weight of new samples, which is configured to be
5982	* 0.25, thus making w=1/4 ( >>= UTIL_EST_WEIGHT_SHIFT)
5983	*/
5984	ewma <<= UTIL_EST_WEIGHT_SHIFT;
5985	ewma -= last_ewma_diff;
5986	ewma >>= UTIL_EST_WEIGHT_SHIFT;
5987	done:
5988	ewma \|= UTIL_AVG_UNCHANGED;
5989	WRITE_ONCE(se->avg.util_est, ewma);
5990
5991	trace_sched_util_est_se_tp(se);
5992	}
5993
5994	/*
5995	* Optional action to be done while updating the load average
5996	*/
5997	#define UPDATE_TG 0x01
5998	#define SKIP_AGE_LOAD 0x02
5999	#define DO_ATTACH 0x04
6000	#define DO_DETACH 0x08
6001	#define UPDATE_UTIL_EST 0x10
6002
6003	/* Update task and its cfs_rq load average */
6004	static inline void update_load_avg(struct cfs_rq cfs_rq, struct sched_entity se, int flags)
6005	{
6006	u64 now = cfs_rq_clock_pelt(cfs_rq);
6007	int decayed;
6008
6009	/*
6010	* Track task load average for carrying it to new CPU after migrated, and
6011	* track group sched_entity load average for task_h_load calculation in migration
6012	*/
6013	if (se->avg.last_update_time && !(flags & SKIP_AGE_LOAD))
6014	__update_load_avg_se(now, cfs_rq, se);
6015
6016	decayed = update_cfs_rq_load_avg(now, cfs_rq);
6017	decayed \|= propagate_entity_load_avg(se);
6018
6019	if (!se->avg.last_update_time && (flags & DO_ATTACH)) {
6020
6021	/*
6022	* DO_ATTACH means we're here from enqueue_entity().
6023	* !last_update_time means we've passed through
6024	* migrate_task_rq_fair() indicating we migrated.
6025	*
6026	* IOW we're enqueueing a task on a new CPU.
6027	*/
6028	attach_entity_load_avg(cfs_rq, se);
6029	update_tg_load_avg(cfs_rq);
6030
6031	} else if (flags & DO_DETACH) {
6032	/*
6033	* DO_DETACH means we're here from dequeue_entity()
6034	* and we are migrating task out of the CPU.
6035	*/
6036	detach_entity_load_avg(cfs_rq, se);
6037	update_tg_load_avg(cfs_rq);
6038	} else if (decayed) {
6039	cfs_rq_util_change(cfs_rq, 0);
6040
6041	if (flags & UPDATE_TG)
6042	update_tg_load_avg(cfs_rq);
6043	}
6044
6045	if (flags & UPDATE_UTIL_EST)
6046	util_est_update(se);
6047	}
6048
6049	/*
6050	* Synchronize entity load avg of dequeued entity without locking
6051	* the previous rq.
6052	*/
6053	static void sync_entity_load_avg(struct sched_entity *se)
6054	{
6055	struct cfs_rq *cfs_rq = cfs_rq_of(se);
6056	u64 last_update_time;
6057
6058	last_update_time = cfs_rq_last_update_time(cfs_rq);
6059	__update_load_avg_blocked_se(last_update_time, se);
6060	}
6061
6062	/*
6063	* Task first catches up with cfs_rq, and then subtract
6064	* itself from the cfs_rq (task must be off the queue now).
6065	*/
6066	static void remove_entity_load_avg(struct sched_entity *se)
6067	{
6068	struct cfs_rq *cfs_rq = cfs_rq_of(se);
6069	unsigned long flags;
6070
6071	/*
6072	* tasks cannot exit without having gone through wake_up_new_task() ->
6073	* enqueue_task_fair() which will have added things to the cfs_rq,
6074	* so we can remove unconditionally.
6075	*/
6076
6077	sync_entity_load_avg(se);
6078
6079	raw_spin_lock_irqsave(&cfs_rq->removed.lock, flags);
6080	++cfs_rq->removed.nr;
6081	cfs_rq->removed.util_avg += se->avg.util_avg;
6082	cfs_rq->removed.load_avg += se->avg.load_avg;
6083	cfs_rq->removed.runnable_avg += se->avg.runnable_avg;
6084	raw_spin_unlock_irqrestore(&cfs_rq->removed.lock, flags);
6085	}
6086
6087	static inline unsigned long cfs_rq_runnable_avg(struct cfs_rq *cfs_rq)
6088	{
6089	return cfs_rq->avg.runnable_avg;
6090	}
6091
6092	static inline unsigned long cfs_rq_load_avg(struct cfs_rq *cfs_rq)
6093	{
6094	return cfs_rq->avg.load_avg;
6095	}
6096
6097	static int sched_balance_newidle(struct rq this_rq, struct rq_flags rf)
6098	__must_hold(__rq_lockp(this_rq));
6099
6100	static inline unsigned long task_util(struct task_struct *p)
6101	{
6102	return READ_ONCE(p->se.avg.util_avg);
6103	}
6104
6105	static inline unsigned long _task_util_est(struct task_struct *p)
6106	{
6107	return READ_ONCE(p->se.avg.util_est) & ~UTIL_AVG_UNCHANGED;
6108	}
6109
6110	static inline unsigned long task_util_est(struct task_struct *p)
6111	{
6112	return max(task_util(p), _task_util_est(p));
6113	}
6114
6115	static inline void util_est_enqueue(struct cfs_rq *cfs_rq,
6116	struct task_struct *p)
6117	{
6118	unsigned int enqueued;
6119
6120	if (!sched_feat(UTIL_EST))
6121	return;
6122
6123	/* Update root cfs_rq's estimated utilization */
6124	enqueued = cfs_rq->avg.util_est;
6125	enqueued += _task_util_est(p);
6126	WRITE_ONCE(cfs_rq->avg.util_est, enqueued);
6127
6128	trace_sched_util_est_cfs_tp(cfs_rq);
6129	}
6130
6131	static inline void util_est_dequeue(struct cfs_rq *cfs_rq,
6132	struct task_struct *p)
6133	{
6134	unsigned int enqueued;
6135
6136	if (!sched_feat(UTIL_EST))
6137	return;
6138
6139	/* Update root cfs_rq's estimated utilization */
6140	enqueued = cfs_rq->avg.util_est;
6141	enqueued -= min_t(unsigned int, enqueued, _task_util_est(p));
6142	WRITE_ONCE(cfs_rq->avg.util_est, enqueued);
6143
6144	trace_sched_util_est_cfs_tp(cfs_rq);
6145	}
6146
6147	static inline unsigned long get_actual_cpu_capacity(int cpu)
6148	{
6149	unsigned long capacity = arch_scale_cpu_capacity(cpu);
6150
6151	capacity -= max(hw_load_avg(cpu_rq(cpu)), cpufreq_get_pressure(cpu));
6152
6153	return capacity;
6154	}
6155
6156	static inline int util_fits_cpu(unsigned long util,
6157	unsigned long uclamp_min,
6158	unsigned long uclamp_max,
6159	int cpu)
6160	{
6161	unsigned long capacity = capacity_of(cpu);
6162	unsigned long capacity_orig;
6163	bool fits, uclamp_max_fits;
6164
6165	/*
6166	* Check if the real util fits without any uclamp boost/cap applied.
6167	*/
6168	fits = fits_capacity(util, capacity);
6169
6170	if (!uclamp_is_used())
6171	return fits;
6172
6173	/*
6174	* We must use arch_scale_cpu_capacity() for comparing against uclamp_min and
6175	* uclamp_max. We only care about capacity pressure (by using
6176	* capacity_of()) for comparing against the real util.
6177	*
6178	* If a task is boosted to 1024 for example, we don't want a tiny
6179	* pressure to skew the check whether it fits a CPU or not.
6180	*
6181	* Similarly if a task is capped to arch_scale_cpu_capacity(little_cpu), it
6182	* should fit a little cpu even if there's some pressure.
6183	*
6184	* Only exception is for HW or cpufreq pressure since it has a direct impact
6185	* on available OPP of the system.
6186	*
6187	* We honour it for uclamp_min only as a drop in performance level
6188	* could result in not getting the requested minimum performance level.
6189	*
6190	* For uclamp_max, we can tolerate a drop in performance level as the
6191	* goal is to cap the task. So it's okay if it's getting less.
6192	*/
6193	capacity_orig = arch_scale_cpu_capacity(cpu);
6194
6195	/*
6196	* We want to force a task to fit a cpu as implied by uclamp_max.
6197	* But we do have some corner cases to cater for..
6198	*
6199	*
6200	* C=z
6201	* \| ___
6202	* \| C=y \| \|
6203	* \|_ _ _ _ _ _ _ _ _ ___ _ _ _ \| _ \| _ _ _ _ _ uclamp_max
6204	* \| C=x \| \| \| \|
6205	* \| ___ \| \| \| \|
6206	* \| \| \| \| \| \| \| (util somewhere in this region)
6207	* \| \| \| \| \| \| \|
6208	* \| \| \| \| \| \| \|
6209	* +----------------------------------------
6210	* CPU0 CPU1 CPU2
6211	*
6212	* In the above example if a task is capped to a specific performance
6213	* point, y, then when:
6214	*
6215	* * util = 80% of x then it does not fit on CPU0 and should migrate
6216	* to CPU1
6217	* * util = 80% of y then it is forced to fit on CPU1 to honour
6218	* uclamp_max request.
6219	*
6220	* which is what we're enforcing here. A task always fits if
6221	* uclamp_max <= capacity_orig. But when uclamp_max > capacity_orig,
6222	* the normal upmigration rules should withhold still.
6223	*
6224	* Only exception is when we are on max capacity, then we need to be
6225	* careful not to block overutilized state. This is so because:
6226	*
6227	* 1. There's no concept of capping at max_capacity! We can't go
6228	* beyond this performance level anyway.
6229	* 2. The system is being saturated when we're operating near
6230	* max capacity, it doesn't make sense to block overutilized.
6231	*/
6232	uclamp_max_fits = (capacity_orig == SCHED_CAPACITY_SCALE) && (uclamp_max == SCHED_CAPACITY_SCALE);
6233	uclamp_max_fits = !uclamp_max_fits && (uclamp_max <= capacity_orig);
6234	fits = fits \|\| uclamp_max_fits;
6235
6236	/*
6237	*
6238	* C=z
6239	* \| ___ (region a, capped, util >= uclamp_max)
6240	* \| C=y \| \|
6241	* \|_ _ _ _ _ _ _ _ _ ___ _ _ _ \| _ \| _ _ _ _ _ uclamp_max
6242	* \| C=x \| \| \| \|
6243	* \| ___ \| \| \| \| (region b, uclamp_min <= util <= uclamp_max)
6244	* \|_ _ _\|_ _\|_ _ _ _\| _ \| _ _ _\| _ \| _ _ _ _ _ uclamp_min
6245	* \| \| \| \| \| \| \|
6246	* \| \| \| \| \| \| \| (region c, boosted, util < uclamp_min)
6247	* +----------------------------------------
6248	* CPU0 CPU1 CPU2
6249	*
6250	* a) If util > uclamp_max, then we're capped, we don't care about
6251	* actual fitness value here. We only care if uclamp_max fits
6252	* capacity without taking margin/pressure into account.
6253	* See comment above.
6254	*
6255	* b) If uclamp_min <= util <= uclamp_max, then the normal
6256	* fits_capacity() rules apply. Except we need to ensure that we
6257	* enforce we remain within uclamp_max, see comment above.
6258	*
6259	* c) If util < uclamp_min, then we are boosted. Same as (b) but we
6260	* need to take into account the boosted value fits the CPU without
6261	* taking margin/pressure into account.
6262	*
6263	* Cases (a) and (b) are handled in the 'fits' variable already. We
6264	* just need to consider an extra check for case (c) after ensuring we
6265	* handle the case uclamp_min > uclamp_max.
6266	*/
6267	uclamp_min = min(uclamp_min, uclamp_max);
6268	if (fits && (util < uclamp_min) &&
6269	(uclamp_min > get_actual_cpu_capacity(cpu)))
6270	return -1;
6271
6272	return fits;
6273	}
6274
6275	static inline int task_fits_cpu(struct task_struct *p, int cpu)
6276	{
6277	unsigned long uclamp_min = uclamp_eff_value(p, UCLAMP_MIN);
6278	unsigned long uclamp_max = uclamp_eff_value(p, UCLAMP_MAX);
6279	unsigned long util = task_util_est(p);
6280	/*
6281	* Return true only if the cpu fully fits the task requirements, which
6282	* include the utilization but also the performance hints.
6283	*/
6284	return (util_fits_cpu(util, uclamp_min, uclamp_max, cpu) > 0);
6285	}
6286
6287	static inline void update_misfit_status(struct task_struct p, struct rq rq)
6288	{
6289	int cpu = cpu_of(rq);
6290
6291	if (!sched_asym_cpucap_active())
6292	return;
6293
6294	/*
6295	* Affinity allows us to go somewhere higher? Or are we on biggest
6296	* available CPU already? Or do we fit into this CPU ?
6297	*/
6298	if (!p \|\| (p->nr_cpus_allowed == 1) \|\|
6299	(arch_scale_cpu_capacity(cpu) == p->max_allowed_capacity) \|\|
6300	task_fits_cpu(p, cpu)) {
6301
6302	rq->misfit_task_load = 0;
6303	return;
6304	}
6305
6306	/*
6307	* Make sure that misfit_task_load will not be null even if
6308	* task_h_load() returns 0.
6309	*/
6310	rq->misfit_task_load = max_t(unsigned long, task_h_load(p), 1);
6311	}
6312
6313	void __setparam_fair(struct task_struct p, const struct sched_attr attr)
6314	{
6315	struct sched_entity *se = &p->se;
6316
6317	p->static_prio = NICE_TO_PRIO(attr->sched_nice);
6318	if (attr->sched_runtime) {
6319	se->custom_slice = 1;
6320	se->slice = clamp_t(u64, attr->sched_runtime,
6321	NSEC_PER_MSEC/10, /* HZ=1000 * 10 */
6322	NSEC_PER_MSEC100); / HZ=100 / 10 */
6323	} else {
6324	se->custom_slice = 0;
6325	se->slice = sysctl_sched_base_slice;
6326	}
6327	}
6328
6329	static void
6330	place_entity(struct cfs_rq cfs_rq, struct sched_entity se, int flags)
6331	{
6332	u64 vslice, vruntime = avg_vruntime(cfs_rq);
6333	unsigned int nr_queued = cfs_rq->h_nr_queued;
6334	bool update_zero = false;
6335	s64 lag = 0;
6336
6337	if (!se->custom_slice)
6338	se->slice = sysctl_sched_base_slice;
6339	vslice = calc_delta_fair(se->slice, se);
6340
6341	if (flags & ENQUEUE_QUEUED)
6342	nr_queued -= 1;
6343
6344	/*
6345	* Due to how V is constructed as the weighted average of entities,
6346	* adding tasks with positive lag, or removing tasks with negative lag
6347	* will move 'time' backwards, this can screw around with the lag of
6348	* other tasks.
6349	*
6350	* EEVDF: placement strategy #1 / #2
6351	*/
6352	if (sched_feat(PLACE_LAG) && nr_queued && se->vlag) {
6353	struct sched_entity *curr = cfs_rq->curr;
6354	long load, weight;
6355
6356	lag = se->vlag;
6357
6358	/*
6359	* If we want to place a task and preserve lag, we have to
6360	* consider the effect of the new entity on the weighted
6361	* average and compensate for this, otherwise lag can quickly
6362	* evaporate.
6363	*
6364	* Lag is defined as:
6365	*
6366	* lag_i = S - s_i = w_i * (V - v_i)
6367	*
6368	* To avoid the 'w_i' term all over the place, we only track
6369	* the virtual lag:
6370	*
6371	* vl_i = V - v_i <=> v_i = V - vl_i
6372	*
6373	* And we take V to be the weighted average of all v:
6374	*
6375	* V = (\Sum w_j*v_j) / W
6376	*
6377	* Where W is: \Sum w_j
6378	*
6379	* Then, the weighted average after adding an entity with lag
6380	* vl_i is given by:
6381	*
6382	* V' = (\Sum w_jv_j + w_iv_i) / (W + w_i)
6383	* = (WV + w_i(V - vl_i)) / (W + w_i)
6384	* = (WV + w_iV - w_i*vl_i) / (W + w_i)
6385	* = (V(W + w_i) - w_ivl_i) / (W + w_i)
6386	* = V - w_i*vl_i / (W + w_i)
6387	*
6388	* And the actual lag after adding an entity with vl_i is:
6389	*
6390	* vl'_i = V' - v_i
6391	* = V - w_i*vl_i / (W + w_i) - (V - vl_i)
6392	* = vl_i - w_i*vl_i / (W + w_i)
6393	*
6394	* Which is strictly less than vl_i. So in order to preserve lag
6395	* we should inflate the lag before placement such that the
6396	* effective lag after placement comes out right.
6397	*
6398	* As such, invert the above relation for vl'_i to get the vl_i
6399	* we need to use such that the lag after placement is the lag
6400	* we computed before dequeue.
6401	*
6402	* vl'_i = vl_i - w_i*vl_i / (W + w_i)
6403	* = ((W + w_i)vl_i - w_ivl_i) / (W + w_i)
6404	*
6405	* (W + w_i)vl'_i = (W + w_i)vl_i - w_i*vl_i
6406	* = W*vl_i
6407	*
6408	* vl_i = (W + w_i)*vl'_i / W
6409	*/
6410	load = cfs_rq->sum_weight;
6411	if (curr && curr->on_rq)
6412	load += avg_vruntime_weight(cfs_rq, curr->h_load.weight);
6413	if (WARN_ON_ONCE(!load))
6414	load = 1;
6415
6416	weight = avg_vruntime_weight(cfs_rq, se->h_load.weight);
6417
6418	/*
6419	* Rewrite, because the product (W + w_i)*vl'_i might
6420	* overflow s64:
6421	*
6422	* vl_i = (W + w_i)*vl'_i / W
6423	* = (Wvl'_i + w_ivl'_i) / W
6424	* = vl'_i + w_i*vl'_i/W
6425	*/
6426	lag += div64_long(lag * weight, load);
6427
6428	/*
6429	* A heavy entity (relative to the tree) will pull the
6430	* avg_vruntime close to its vruntime position on enqueue. But
6431	* the zero_vruntime point is only updated at the next
6432	* update_deadline()/place_entity()/update_entity_lag().
6433	*
6434	* Specifically (see the comment near avg_vruntime_weight()):
6435	*
6436	* sum_w_vruntime = \Sum (v_i - v0) * w_i
6437	*
6438	* Note that if v0 is near a light entity, both terms will be
6439	* small for the light entity, while in that case both terms
6440	* are large for the heavy entity, leading to risk of
6441	* overflow.
6442	*
6443	* OTOH if v0 is near the heavy entity, then the difference is
6444	* larger for the light entity, but the factor is small, while
6445	* for the heavy entity the difference is small but the factor
6446	* is large. Avoiding the multiplication overflow.
6447	*/
6448	if (weight > load)
6449	update_zero = true;
6450	}
6451
6452	se->vruntime = vruntime - lag;
6453
6454	if (update_zero)
6455	update_zero_vruntime(cfs_rq, -lag);
6456
6457	if (sched_feat(PLACE_REL_DEADLINE) && se->rel_deadline) {
6458	se->deadline += se->vruntime;
6459	se->rel_deadline = 0;
6460	return;
6461	}
6462
6463	/*
6464	* When joining the competition; the existing tasks will be,
6465	* on average, halfway through their slice, as such start tasks
6466	* off with half a slice to ease into the competition.
6467	*/
6468	if (sched_feat(PLACE_DEADLINE_INITIAL) && (flags & ENQUEUE_INITIAL))
6469	vslice /= 2;
6470
6471	/*
6472	* EEVDF: vd_i = ve_i + r_i/w_i
6473	*/
6474	se->deadline = se->vruntime + vslice;
6475	}
6476
6477	static void check_enqueue_throttle(struct cfs_rq *cfs_rq);
6478	static inline int cfs_rq_throttled(struct cfs_rq *cfs_rq);
6479
6480	static void
6481	enqueue_entity(struct cfs_rq cfs_rq, struct sched_entity se, int flags)
6482	{
6483	/*
6484	* When enqueuing a sched_entity, we must:
6485	* - Update loads to have both entity and cfs_rq synced with now.
6486	* - For group_entity, update its runnable_weight to reflect the new
6487	* h_nr_runnable of its group cfs_rq.
6488	* - For group_entity, update its weight to reflect the new share of
6489	* its group cfs_rq
6490	* - Add its new weight to cfs_rq->load.weight
6491	*/
6492	update_load_avg(cfs_rq, se, UPDATE_TG \| DO_ATTACH);
6493	se_update_runnable(se);
6494	/*
6495	* XXX update_load_avg() above will have attached us to the pelt sum;
6496	* but update_cfs_group() here will re-adjust the weight and have to
6497	* undo/redo all that. Seems wasteful.
6498	*/
6499	update_cfs_group(se);
6500
6501	account_entity_enqueue(cfs_rq, se);
6502
6503	/* Entity has migrated, no longer consider this task hot */
6504	if (flags & ENQUEUE_MIGRATED)
6505	se->exec_start = 0;
6506
6507	check_schedstat_required();
6508	update_stats_enqueue_fair(cfs_rq, se, flags);
6509	se->on_rq = 1;
6510
6511	if (cfs_rq->nr_queued == 1) {
6512	check_enqueue_throttle(cfs_rq);
6513	list_add_leaf_cfs_rq(cfs_rq);
6514	#ifdef CONFIG_CFS_BANDWIDTH
6515	if (cfs_rq->pelt_clock_throttled) {
6516	struct rq *rq = rq_of(cfs_rq);
6517
6518	cfs_rq->throttled_clock_pelt_time += rq_clock_pelt(rq) -
6519	cfs_rq->throttled_clock_pelt;
6520	cfs_rq->pelt_clock_throttled = 0;
6521	}
6522	#endif
6523	}
6524	}
6525
6526	static void set_next_buddy(struct cfs_rq cfs_rq, struct sched_entity se)
6527	{
6528	if (WARN_ON_ONCE(!se->on_rq \|\| se->sched_delayed))
6529	return;
6530	if (se_is_idle(se))
6531	return;
6532	cfs_rq->next = se;
6533	}
6534
6535	static void clear_buddies(struct cfs_rq cfs_rq, struct sched_entity se)
6536	{
6537	if (cfs_rq->next == se)
6538	cfs_rq->next = NULL;
6539	}
6540
6541	static __always_inline void return_cfs_rq_runtime(struct cfs_rq *cfs_rq);
6542
6543	static void set_delayed(struct sched_entity *se)
6544	{
6545	/*
6546	* Delayed se of cfs_rq have no tasks queued on them.
6547	* Do not adjust h_nr_runnable since __dequeue_task()
6548	* will account it for blocked tasks.
6549	*
6550	* This check can be removed because when flat pick
6551	* patches get merged as only task can get delayed,
6552	* same for clear_delayed().
6553	*/
6554	if (!entity_is_task(se)) {
6555	se->sched_delayed = 1;
6556	return;
6557	}
6558
6559	/*
6560	* Drop a task leaving the runnable set.
6561	* Needs to be called before sched_delayed is set.
6562	* clear_delayed() mirrors this after clearing the flag.
6563	*/
6564	pref_llc_running_dec(rq_of(cfs_rq_of(se)), task_of(se));
6565	se->sched_delayed = 1;
6566
6567	for_each_sched_entity(se) {
6568	struct cfs_rq *cfs_rq = cfs_rq_of(se);
6569
6570	cfs_rq->h_nr_runnable--;
6571	}
6572	}
6573
6574	static void clear_delayed(struct sched_entity *se)
6575	{
6576	se->sched_delayed = 0;
6577
6578	/*
6579	* Delayed se of cfs_rq have no tasks queued on them.
6580	* Do not adjust h_nr_runnable since a dequeue has
6581	* already accounted for it or an enqueue of a task
6582	* below it will account for it in enqueue_task_fair().
6583	*/
6584	if (!entity_is_task(se))
6585	return;
6586
6587	/*
6588	* Re-add on wake, after sched_delayed is cleared. On a final delayed
6589	* dequeue account_llc_dequeue() already cleared pref_llc_queued, so
6590	* this does nothing.
6591	*/
6592	pref_llc_running_inc(rq_of(cfs_rq_of(se)), task_of(se));
6593
6594	for_each_sched_entity(se) {
6595	struct cfs_rq *cfs_rq = cfs_rq_of(se);
6596
6597	cfs_rq->h_nr_runnable++;
6598	}
6599	}
6600
6601	static void
6602	dequeue_entity(struct cfs_rq cfs_rq, struct sched_entity se, int flags)
6603	{
6604	int action = UPDATE_TG;
6605
6606	if (entity_is_task(se)) {
6607	if (task_on_rq_migrating(task_of(se)))
6608	action \|= DO_DETACH;
6609
6610	if ((flags & DEQUEUE_SLEEP) && !(flags & DEQUEUE_DELAYED))
6611	action \|= UPDATE_UTIL_EST;
6612	}
6613
6614	/*
6615	* When dequeuing a sched_entity, we must:
6616	* - Update loads to have both entity and cfs_rq synced with now.
6617	* - For group_entity, update its runnable_weight to reflect the new
6618	* h_nr_runnable of its group cfs_rq.
6619	* - Subtract its previous weight from cfs_rq->load.weight.
6620	* - For group entity, update its weight to reflect the new share
6621	* of its group cfs_rq.
6622	*/
6623	update_load_avg(cfs_rq, se, action);
6624	se_update_runnable(se);
6625
6626	update_stats_dequeue_fair(cfs_rq, se, flags);
6627
6628	se->on_rq = 0;
6629	account_entity_dequeue(cfs_rq, se);
6630
6631	/* return excess runtime on last dequeue */
6632	return_cfs_rq_runtime(cfs_rq);
6633
6634	update_cfs_group(se);
6635
6636	if (cfs_rq->nr_queued == 0) {
6637	update_idle_cfs_rq_clock_pelt(cfs_rq);
6638	#ifdef CONFIG_CFS_BANDWIDTH
6639	if (throttled_hierarchy(cfs_rq)) {
6640	struct rq *rq = rq_of(cfs_rq);
6641
6642	list_del_leaf_cfs_rq(cfs_rq);
6643	cfs_rq->throttled_clock_pelt = rq_clock_pelt(rq);
6644	cfs_rq->pelt_clock_throttled = 1;
6645	}
6646	#endif
6647	}
6648	}
6649
6650	static void
6651	set_next_entity(struct cfs_rq cfs_rq, struct sched_entity se)
6652	{
6653	/* 'current' is not kept within the tree. */
6654	if (se->on_rq) {
6655	/*
6656	* Any task has to be enqueued before it get to execute on
6657	* a CPU. So account for the time it spent waiting on the
6658	* runqueue.
6659	*/
6660	update_stats_wait_end_fair(cfs_rq, se);
6661	update_load_avg(cfs_rq, se, UPDATE_TG);
6662	}
6663
6664	update_stats_curr_start(cfs_rq, se);
6665	WARN_ON_ONCE(cfs_rq->h_curr);
6666	cfs_rq->h_curr = se;
6667
6668	/*
6669	* Track our maximum slice length, if the CPU's load is at
6670	* least twice that of our own weight (i.e. don't track it
6671	* when there are only lesser-weight tasks around):
6672	*/
6673	if (schedstat_enabled() &&
6674	rq_of(cfs_rq)->cfs.load.weight >= 2*se->load.weight) {
6675	struct sched_statistics *stats;
6676
6677	stats = __schedstats_from_se(se);
6678	__schedstat_set(stats->slice_max,
6679	max((u64)stats->slice_max,
6680	se->sum_exec_runtime - se->prev_sum_exec_runtime));
6681	}
6682
6683	se->prev_sum_exec_runtime = se->sum_exec_runtime;
6684	}
6685
6686	static bool __dequeue_task(struct rq rq, struct task_struct p, int flags);
6687
6688	static struct sched_entity *
6689	pick_next_entity(struct rq *rq, bool protect)
6690	{
6691	struct cfs_rq *cfs_rq = &rq->cfs;
6692	struct sched_entity *se;
6693
6694	se = pick_eevdf(cfs_rq, protect);
6695	if (se->sched_delayed) {
6696	__dequeue_task(rq, task_of(se), DEQUEUE_SLEEP \| DEQUEUE_DELAYED);
6697	/*
6698	* Must not reference @se again, see __block_task().
6699	*/
6700	return NULL;
6701	}
6702	return se;
6703	}
6704
6705	static void put_prev_entity(struct cfs_rq cfs_rq, struct sched_entity prev)
6706	{
6707	/*
6708	* If still on the runqueue then deactivate_task()
6709	* was not called and update_curr() has to be done:
6710	*/
6711	if (prev->on_rq)
6712	update_curr(cfs_rq);
6713
6714	if (prev->on_rq) {
6715	update_stats_wait_start_fair(cfs_rq, prev);
6716	/* in !on_rq case, update occurred at dequeue */
6717	update_load_avg(cfs_rq, prev, 0);
6718	}
6719	WARN_ON_ONCE(cfs_rq->h_curr != prev);
6720	cfs_rq->h_curr = NULL;
6721	}
6722
6723	static void
6724	entity_tick(struct cfs_rq cfs_rq, struct sched_entity curr, int queued)
6725	{
6726	/*
6727	* Update run-time statistics of the 'current'.
6728	*/
6729	update_curr(cfs_rq);
6730
6731	/*
6732	* Ensure that runnable average is periodically updated.
6733	*/
6734	update_load_avg(cfs_rq, curr, UPDATE_TG);
6735	update_cfs_group(curr);
6736
6737	#ifdef CONFIG_SCHED_HRTICK
6738	/*
6739	* queued ticks are scheduled to match the slice, so don't bother
6740	* validating it and just reschedule.
6741	*/
6742	if (queued) {
6743	resched_curr(rq_of(cfs_rq));
6744	return;
6745	}
6746	#endif
6747	}
6748
6749
6750	/**************************************************
6751	* CFS bandwidth control machinery
6752	*/
6753
6754	#ifdef CONFIG_CFS_BANDWIDTH
6755
6756	#ifdef CONFIG_JUMP_LABEL
6757	static struct static_key __cfs_bandwidth_used;
6758
6759	static inline bool cfs_bandwidth_used(void)
6760	{
6761	return static_key_false(&__cfs_bandwidth_used);
6762	}
6763
6764	void cfs_bandwidth_usage_inc(void)
6765	{
6766	static_key_slow_inc_cpuslocked(&__cfs_bandwidth_used);
6767	}
6768
6769	void cfs_bandwidth_usage_dec(void)
6770	{
6771	static_key_slow_dec_cpuslocked(&__cfs_bandwidth_used);
6772	}
6773	#else /* !CONFIG_JUMP_LABEL: */
6774	static bool cfs_bandwidth_used(void)
6775	{
6776	return true;
6777	}
6778
6779	void cfs_bandwidth_usage_inc(void) {}
6780	void cfs_bandwidth_usage_dec(void) {}
6781	#endif /* !CONFIG_JUMP_LABEL */
6782
6783	static inline u64 sched_cfs_bandwidth_slice(void)
6784	{
6785	return (u64)sysctl_sched_cfs_bandwidth_slice * NSEC_PER_USEC;
6786	}
6787
6788	/*
6789	* Replenish runtime according to assigned quota. We use sched_clock_cpu
6790	* directly instead of rq->clock to avoid adding additional synchronization
6791	* around rq->lock.
6792	*
6793	* requires cfs_b->lock
6794	*/
6795	void __refill_cfs_bandwidth_runtime(struct cfs_bandwidth *cfs_b)
6796	{
6797	s64 runtime;
6798
6799	if (unlikely(cfs_b->quota == RUNTIME_INF))
6800	return;
6801
6802	cfs_b->runtime += cfs_b->quota;
6803	runtime = cfs_b->runtime_snap - cfs_b->runtime;
6804	if (runtime > 0) {
6805	cfs_b->burst_time += runtime;
6806	cfs_b->nr_burst++;
6807	}
6808
6809	cfs_b->runtime = min(cfs_b->runtime, cfs_b->quota + cfs_b->burst);
6810	cfs_b->runtime_snap = cfs_b->runtime;
6811	}
6812
6813	static inline struct cfs_bandwidth tg_cfs_bandwidth(struct task_group tg)
6814	{
6815	return &tg->cfs_bandwidth;
6816	}
6817
6818	/* returns 0 on failure to allocate runtime */
6819	static int __assign_cfs_rq_runtime(struct cfs_bandwidth *cfs_b,
6820	struct cfs_rq *cfs_rq, u64 target_runtime)
6821	{
6822	u64 min_amount, amount = 0;
6823
6824	lockdep_assert_held(&cfs_b->lock);
6825
6826	/* note: this is a positive sum as runtime_remaining <= 0 */
6827	min_amount = target_runtime - cfs_rq->runtime_remaining;
6828
6829	if (cfs_b->quota == RUNTIME_INF)
6830	amount = min_amount;
6831	else {
6832	start_cfs_bandwidth(cfs_b);
6833
6834	if (cfs_b->runtime > 0) {
6835	amount = min(cfs_b->runtime, min_amount);
6836	cfs_b->runtime -= amount;
6837	cfs_b->idle = 0;
6838	}
6839	}
6840
6841	cfs_rq->runtime_remaining += amount;
6842
6843	return cfs_rq->runtime_remaining > 0;
6844	}
6845
6846	static bool throttle_cfs_rq(struct cfs_rq *cfs_rq);
6847
6848	static bool __account_cfs_rq_runtime(struct cfs_rq *cfs_rq, u64 delta_exec)
6849	{
6850	/* dock delta_exec before expiring quota (as it could span periods) */
6851	cfs_rq->runtime_remaining -= delta_exec;
6852
6853	if (likely(cfs_rq->runtime_remaining > 0))
6854	return false;
6855
6856	if (cfs_rq->throttled)
6857	return true;
6858	/*
6859	* throttle_cfs_rq() will try to extend the runtime first
6860	* before throttling the hierarchy.
6861	*/
6862	return throttle_cfs_rq(cfs_rq);
6863	}
6864
6865	static __always_inline
6866	bool account_cfs_rq_runtime(struct cfs_rq *cfs_rq, u64 delta_exec)
6867	{
6868	if (!cfs_bandwidth_used() \|\| !cfs_rq->runtime_enabled)
6869	return false;
6870
6871	return __account_cfs_rq_runtime(cfs_rq, delta_exec);
6872	}
6873
6874	static inline int cfs_rq_throttled(struct cfs_rq *cfs_rq)
6875	{
6876	return cfs_bandwidth_used() && cfs_rq->throttled;
6877	}
6878
6879	static inline bool cfs_rq_pelt_clock_throttled(struct cfs_rq *cfs_rq)
6880	{
6881	return cfs_bandwidth_used() && cfs_rq->pelt_clock_throttled;
6882	}
6883
6884	/* check whether cfs_rq, or any parent, is throttled */
6885	static inline int throttled_hierarchy(struct cfs_rq *cfs_rq)
6886	{
6887	return cfs_bandwidth_used() && cfs_rq->throttle_count;
6888	}
6889
6890	static inline int lb_throttled_hierarchy(struct task_struct *p, int dst_cpu)
6891	{
6892	return throttled_hierarchy(tg_cfs_rq(task_group(p), dst_cpu));
6893	}
6894
6895	static inline bool task_is_throttled(struct task_struct *p)
6896	{
6897	return cfs_bandwidth_used() && p->throttled;
6898	}
6899
6900	static bool dequeue_task_fair(struct rq rq, struct task_struct p, int flags);
6901	static void throttle_cfs_rq_work(struct callback_head *work)
6902	{
6903	struct task_struct *p = container_of(work, struct task_struct, sched_throttle_work);
6904	struct sched_entity *se;
6905	struct cfs_rq *cfs_rq;
6906	struct rq *rq;
6907
6908	WARN_ON_ONCE(p != current);
6909	p->sched_throttle_work.next = &p->sched_throttle_work;
6910
6911	/*
6912	* If task is exiting, then there won't be a return to userspace, so we
6913	* don't have to bother with any of this.
6914	*/
6915	if ((p->flags & PF_EXITING))
6916	return;
6917
6918	scoped_guard(task_rq_lock, p) {
6919	se = &p->se;
6920	cfs_rq = cfs_rq_of(se);
6921
6922	/* Raced, forget */
6923	if (p->sched_class != &fair_sched_class)
6924	return;
6925
6926	/*
6927	* If not in limbo, then either replenish has happened or this
6928	* task got migrated out of the throttled cfs_rq, move along.
6929	*/
6930	if (!cfs_rq->throttle_count)
6931	return;
6932	rq = scope.rq;
6933	update_rq_clock(rq);
6934	WARN_ON_ONCE(p->throttled \|\| !list_empty(&p->throttle_node));
6935	dequeue_task_fair(rq, p, DEQUEUE_SLEEP \| DEQUEUE_THROTTLE);
6936	list_add(&p->throttle_node, &cfs_rq->throttled_limbo_list);
6937	/*
6938	* Must not set throttled before dequeue or dequeue will
6939	* mistakenly regard this task as an already throttled one.
6940	*/
6941	p->throttled = true;
6942	resched_curr(rq);
6943	}
6944	}
6945
6946	void init_cfs_throttle_work(struct task_struct *p)
6947	{
6948	init_task_work(&p->sched_throttle_work, throttle_cfs_rq_work);
6949	/* Protect against double add, see throttle_cfs_rq() and throttle_cfs_rq_work() */
6950	p->sched_throttle_work.next = &p->sched_throttle_work;
6951	INIT_LIST_HEAD(&p->throttle_node);
6952	}
6953
6954	/*
6955	* Task is throttled and someone wants to dequeue it again:
6956	* it could be sched/core when core needs to do things like
6957	* task affinity change, task group change, task sched class
6958	* change etc. and in these cases, DEQUEUE_SLEEP is not set;
6959	* or the task is blocked after throttled due to freezer etc.
6960	* and in these cases, DEQUEUE_SLEEP is set.
6961	*/
6962	static void detach_task_cfs_rq(struct task_struct *p);
6963	static void dequeue_throttled_task(struct task_struct *p, int flags)
6964	{
6965	WARN_ON_ONCE(p->se.on_rq);
6966	list_del_init(&p->throttle_node);
6967
6968	/* task blocked after throttled */
6969	if (flags & DEQUEUE_SLEEP) {
6970	p->throttled = false;
6971	return;
6972	}
6973
6974	/*
6975	* task is migrating off its old cfs_rq, detach
6976	* the task's load from its old cfs_rq.
6977	*/
6978	if (task_on_rq_migrating(p))
6979	detach_task_cfs_rq(p);
6980	}
6981
6982	static bool enqueue_throttled_task(struct task_struct *p)
6983	{
6984	struct cfs_rq *cfs_rq = cfs_rq_of(&p->se);
6985
6986	/* @p should have gone through dequeue_throttled_task() first */
6987	WARN_ON_ONCE(!list_empty(&p->throttle_node));
6988
6989	/*
6990	* If the throttled task @p is enqueued to a throttled cfs_rq,
6991	* take the fast path by directly putting the task on the
6992	* target cfs_rq's limbo list.
6993	*
6994	* Do not do that when @p is current because the following race can
6995	* cause @p's group_node to be incorectly re-insterted in its rq's
6996	* cfs_tasks list, despite being throttled:
6997	*
6998	* cpuX cpuY
6999	* p ret2user
7000	* throttle_cfs_rq_work() sched_move_task(p)
7001	* LOCK task_rq_lock
7002	* dequeue_task_fair(p)
7003	* UNLOCK task_rq_lock
7004	* LOCK task_rq_lock
7005	* task_current_donor(p) == true
7006	* task_on_rq_queued(p) == true
7007	* dequeue_task(p)
7008	* put_prev_task(p)
7009	* sched_change_group()
7010	* enqueue_task(p) -> p's new cfs_rq
7011	* is throttled, go
7012	* fast path and skip
7013	* actual enqueue
7014	* set_next_task(p)
7015	* list_move(&se->group_node, &rq->cfs_tasks); // bug
7016	* schedule()
7017	*
7018	* In the above race case, @p current cfs_rq is in the same rq as
7019	* its previous cfs_rq because sched_move_task() only moves a task
7020	* to a different group from the same rq, so we can use its current
7021	* cfs_rq to derive rq and test if the task is current.
7022	*/
7023	if (throttled_hierarchy(cfs_rq) &&
7024	!task_current_donor(rq_of(cfs_rq), p)) {
7025	list_add(&p->throttle_node, &cfs_rq->throttled_limbo_list);
7026	return true;
7027	}
7028
7029	/* we can't take the fast path, do an actual enqueue*/
7030	p->throttled = false;
7031	return false;
7032	}
7033
7034	static void enqueue_task_fair(struct rq rq, struct task_struct p, int flags);
7035	static int tg_unthrottle_up(struct task_group tg, void data)
7036	{
7037	struct rq *rq = data;
7038	struct cfs_rq *cfs_rq = tg_cfs_rq(tg, cpu_of(rq));
7039	struct task_struct p, tmp;
7040	LIST_HEAD(throttled_tasks);
7041
7042	/*
7043	* If cfs_rq->curr is set, the cfs_rq might not have caught up
7044	* since the last clock update. Do it now before we begin
7045	* queueing task onto it to save the need for unnecessarily
7046	* unthrottle the hierarchy for this cfs_rq to be throttled
7047	* right back again.
7048	*/
7049	update_curr(cfs_rq);
7050
7051	if (--cfs_rq->throttle_count)
7052	return 0;
7053
7054	if (cfs_rq->pelt_clock_throttled) {
7055	cfs_rq->throttled_clock_pelt_time += rq_clock_pelt(rq) -
7056	cfs_rq->throttled_clock_pelt;
7057	cfs_rq->pelt_clock_throttled = 0;
7058	}
7059
7060	if (cfs_rq->throttled_clock_self) {
7061	u64 delta = rq_clock(rq) - cfs_rq->throttled_clock_self;
7062
7063	cfs_rq->throttled_clock_self = 0;
7064
7065	if (WARN_ON_ONCE((s64)delta < 0))
7066	delta = 0;
7067
7068	cfs_rq->throttled_clock_self_time += delta;
7069	}
7070
7071	/*
7072	* Move the tasks to a local list since an update_curr() during
7073	* enqueue_task_fair() can throttle a higher cfs_rq, and it can
7074	* see the "throttled_limbo_list" being non-empty in
7075	* tg_throttle_down() if throttle_count turned 0 above.
7076	*/
7077	list_splice_init(&cfs_rq->throttled_limbo_list, &throttled_tasks);
7078
7079	/* Re-enqueue the tasks that have been throttled at this level. */
7080	list_for_each_entry_safe(p, tmp, &throttled_tasks, throttle_node) {
7081	/*
7082	* Back to being throttled! Break out and put the remaining
7083	* tasks back onto the limbo_list to prevent running them
7084	* unnecessarily.
7085	*/
7086	if (cfs_rq->throttle_count)
7087	break;
7088
7089	list_del_init(&p->throttle_node);
7090	p->throttled = false;
7091	enqueue_task_fair(rq, p, ENQUEUE_WAKEUP);
7092	}
7093
7094	list_splice(&throttled_tasks, &cfs_rq->throttled_limbo_list);
7095
7096	/* Add cfs_rq with load or one or more already running entities to the list */
7097	if (!cfs_rq_is_decayed(cfs_rq))
7098	list_add_leaf_cfs_rq(cfs_rq);
7099
7100	return 0;
7101	}
7102
7103	static inline bool task_has_throttle_work(struct task_struct *p)
7104	{
7105	return p->sched_throttle_work.next != &p->sched_throttle_work;
7106	}
7107
7108	static inline void task_throttle_setup_work(struct task_struct *p)
7109	{
7110	if (task_has_throttle_work(p))
7111	return;
7112
7113	/*
7114	* Kthreads and exiting tasks don't return to userspace, so adding the
7115	* work is pointless
7116	*/
7117	if ((p->flags & (PF_EXITING \| PF_KTHREAD)))
7118	return;
7119
7120	task_work_add(p, &p->sched_throttle_work, TWA_RESUME);
7121	}
7122
7123	static void record_throttle_clock(struct cfs_rq *cfs_rq)
7124	{
7125	struct rq *rq = rq_of(cfs_rq);
7126
7127	if (cfs_rq_throttled(cfs_rq) && !cfs_rq->throttled_clock)
7128	cfs_rq->throttled_clock = rq_clock(rq);
7129
7130	if (!cfs_rq->throttled_clock_self)
7131	cfs_rq->throttled_clock_self = rq_clock(rq);
7132	}
7133
7134	static int tg_throttle_down(struct task_group tg, void data)
7135	{
7136	struct rq *rq = data;
7137	struct cfs_rq *cfs_rq = tg_cfs_rq(tg, cpu_of(rq));
7138
7139	if (cfs_rq->throttle_count++)
7140	return 0;
7141
7142	/*
7143	* For cfs_rqs that still have entities enqueued, PELT clock
7144	* stop happens at dequeue time when all entities are dequeued.
7145	*/
7146	if (!cfs_rq->nr_queued) {
7147	list_del_leaf_cfs_rq(cfs_rq);
7148	cfs_rq->throttled_clock_pelt = rq_clock_pelt(rq);
7149	cfs_rq->pelt_clock_throttled = 1;
7150	}
7151
7152	WARN_ON_ONCE(cfs_rq->throttled_clock_self);
7153	WARN_ON_ONCE(!list_empty(&cfs_rq->throttled_limbo_list));
7154	return 0;
7155	}
7156
7157	static bool throttle_cfs_rq(struct cfs_rq *cfs_rq)
7158	{
7159	struct cfs_bandwidth *cfs_b = tg_cfs_bandwidth(cfs_rq->tg);
7160	struct sched_entity *curr = cfs_rq->h_curr;
7161	struct rq *rq = rq_of(cfs_rq);
7162
7163	scoped_guard(raw_spinlock, &cfs_b->lock) {
7164	u64 target_runtime = 1;
7165
7166	/*
7167	* If cfs_rq->h_curr is still runnable, we are here from an
7168	* update_curr(). Request sysctl_sched_cfs_bandwidth_slice
7169	* worth of bandwidth to continue running.
7170	*
7171	* If the curr is not runnable, just request enough bandwidth
7172	* to be runnable next time the pick selects this cfs_rq.
7173	*/
7174	if (curr && curr->on_rq)
7175	target_runtime = sched_cfs_bandwidth_slice();
7176
7177	/*
7178	* Check if We have raced with bandwidth becoming available. If
7179	* we actually throttled the timer might not unthrottle us for
7180	* an entire period. We additionally needed to make sure that
7181	* any subsequent check_cfs_rq_runtime calls agree not to
7182	* throttle us, as we may commit to do cfs put_prev+pick_next,
7183	* so we ask for 1ns of runtime rather than just check cfs_b.
7184	*
7185	* This will start the period timer if necessary.
7186	*/
7187	if (__assign_cfs_rq_runtime(cfs_b, cfs_rq, target_runtime))
7188	return false;
7189
7190	/*
7191	* No bandwidth available; Add ourselves on the list to be
7192	* unthrottled later.
7193	*/
7194	list_add_tail_rcu(&cfs_rq->throttled_list,
7195	&cfs_b->throttled_cfs_rq);
7196	}
7197
7198	/* freeze hierarchy runnable averages while throttled */
7199	scoped_guard(rcu)
7200	walk_tg_tree_from(cfs_rq->tg, tg_throttle_down, tg_nop, (void *)rq);
7201
7202	/*
7203	* Note: distribution will already see us throttled via the
7204	* throttled-list. rq->lock protects completion.
7205	*/
7206	cfs_rq->throttled = 1;
7207	WARN_ON_ONCE(cfs_rq->throttled_clock);
7208
7209	/*
7210	* If current hierarchy was throttled, add throttle work to the
7211	* current donor. In case of proxy-execution, the execution
7212	* context cannot exit to the userspace while holding a mutex
7213	* and the rule of throttle deferral to only throttle the
7214	* throttled context at exit to userspace is still preserved.
7215	*/
7216	if (curr && curr->on_rq)
7217	task_throttle_setup_work(rq->donor);
7218
7219	return true;
7220	}
7221
7222	void unthrottle_cfs_rq(struct cfs_rq *cfs_rq)
7223	{
7224	struct rq *rq = rq_of(cfs_rq);
7225	struct cfs_bandwidth *cfs_b = tg_cfs_bandwidth(cfs_rq->tg);
7226	struct sched_entity *se = cfs_rq_se(cfs_rq);
7227
7228	/*
7229	* It's possible we are called with runtime_remaining < 0 due to things
7230	* like async unthrottled us with a positive runtime_remaining but other
7231	* still running entities consumed those runtime before we reached here.
7232	*
7233	* We can't unthrottle this cfs_rq without any runtime remaining because
7234	* any enqueue in tg_unthrottle_up() will immediately trigger a throttle,
7235	* which is not supposed to happen on unthrottle path.
7236	*
7237	* Catch up on the remaining runtime since last clock update before
7238	* checking runtime remaining.
7239	*/
7240	update_curr(cfs_rq);
7241	if (cfs_rq->runtime_enabled && cfs_rq->runtime_remaining <= 0)
7242	return;
7243
7244	cfs_rq->throttled = 0;
7245
7246	scoped_guard(raw_spinlock, &cfs_b->lock) {
7247	list_del_rcu(&cfs_rq->throttled_list);
7248
7249	if (!cfs_rq->throttled_clock)
7250	break;
7251
7252	cfs_b->throttled_time += rq_clock(rq) - cfs_rq->throttled_clock;
7253	cfs_rq->throttled_clock = 0;
7254	}
7255
7256	/* update hierarchical throttle state */
7257	walk_tg_tree_from(cfs_rq->tg, tg_nop, tg_unthrottle_up, (void *)rq);
7258
7259	if (!cfs_rq->load.weight) {
7260	if (!cfs_rq->on_list)
7261	return;
7262	/*
7263	* Nothing to run but something to decay (on_list)?
7264	* Complete the branch.
7265	*/
7266	for_each_sched_entity(se) {
7267	if (list_add_leaf_cfs_rq(cfs_rq_of(se)))
7268	break;
7269	}
7270	}
7271
7272	assert_list_leaf_cfs_rq(rq);
7273
7274	/* Determine whether we need to wake up potentially idle CPU: */
7275	if (rq->curr == rq->idle && rq->cfs.h_nr_queued)
7276	resched_curr(rq);
7277	}
7278
7279	static void __cfsb_csd_unthrottle(void *arg)
7280	{
7281	struct cfs_rq cursor, tmp;
7282	struct rq *rq = arg;
7283
7284	guard(rq_lock)(rq);
7285
7286	/*
7287	* Iterating over the list can trigger several call to
7288	* update_rq_clock() in unthrottle_cfs_rq().
7289	* Do it once and skip the potential next ones.
7290	*/
7291	update_rq_clock(rq);
7292	rq_clock_start_loop_update(rq);
7293
7294	/*
7295	* Since we hold rq lock we're safe from concurrent manipulation of
7296	* the CSD list. However, this RCU critical section annotates the
7297	* fact that we pair with sched_free_group_rcu(), so that we cannot
7298	* race with group being freed in the window between removing it
7299	* from the list and advancing to the next entry in the list.
7300	*/
7301	guard(rcu)();
7302
7303	list_for_each_entry_safe(cursor, tmp, &rq->cfsb_csd_list,
7304	throttled_csd_list) {
7305	list_del_init(&cursor->throttled_csd_list);
7306
7307	if (cfs_rq_throttled(cursor))
7308	unthrottle_cfs_rq(cursor);
7309	}
7310
7311	rq_clock_stop_loop_update(rq);
7312	}
7313
7314	static inline void __unthrottle_cfs_rq_async(struct cfs_rq *cfs_rq)
7315	{
7316	struct rq *rq = rq_of(cfs_rq);
7317	bool first;
7318
7319	if (rq == this_rq()) {
7320	update_rq_clock(rq);
7321	unthrottle_cfs_rq(cfs_rq);
7322	return;
7323	}
7324
7325	/* Already enqueued */
7326	if (WARN_ON_ONCE(!list_empty(&cfs_rq->throttled_csd_list)))
7327	return;
7328
7329	first = list_empty(&rq->cfsb_csd_list);
7330	list_add_tail(&cfs_rq->throttled_csd_list, &rq->cfsb_csd_list);
7331	if (first)
7332	smp_call_function_single_async(cpu_of(rq), &rq->cfsb_csd);
7333	}
7334
7335	static void unthrottle_cfs_rq_async(struct cfs_rq *cfs_rq)
7336	{
7337	lockdep_assert_rq_held(rq_of(cfs_rq));
7338
7339	if (WARN_ON_ONCE(!cfs_rq_throttled(cfs_rq) \|\|
7340	cfs_rq->runtime_remaining <= 0))
7341	return;
7342
7343	__unthrottle_cfs_rq_async(cfs_rq);
7344	}
7345
7346	static bool distribute_cfs_runtime(struct cfs_bandwidth *cfs_b)
7347	{
7348	bool throttled = false, unthrottle_local = false;
7349	int this_cpu = smp_processor_id();
7350	u64 runtime, remaining = 1;
7351	struct cfs_rq *cfs_rq;
7352	struct rq *rq;
7353
7354	guard(rcu)();
7355
7356	list_for_each_entry_rcu(cfs_rq, &cfs_b->throttled_cfs_rq,
7357	throttled_list) {
7358	rq = rq_of(cfs_rq);
7359
7360	if (!remaining) {
7361	throttled = true;
7362	break;
7363	}
7364
7365	guard(rq_lock_irqsave)(rq);
7366
7367	if (!cfs_rq_throttled(cfs_rq))
7368	continue;
7369
7370	/* Already queued for async unthrottle */
7371	if (!list_empty(&cfs_rq->throttled_csd_list))
7372	continue;
7373
7374	if (cfs_rq->h_curr) {
7375	update_rq_clock(rq);
7376	update_curr(cfs_rq);
7377	}
7378
7379	/* By the above checks, this should never be true */
7380	WARN_ON_ONCE(cfs_rq->runtime_remaining > 0);
7381
7382	scoped_guard(raw_spinlock, &cfs_b->lock) {
7383	runtime = -cfs_rq->runtime_remaining + 1;
7384	if (runtime > cfs_b->runtime)
7385	runtime = cfs_b->runtime;
7386	cfs_b->runtime -= runtime;
7387	remaining = cfs_b->runtime;
7388	}
7389
7390	cfs_rq->runtime_remaining += runtime;
7391
7392	/*
7393	* Ran out of bandwidth during distribution!
7394	* Indicate throttled entities and break early.
7395	*/
7396	if (cfs_rq->runtime_remaining <= 0) {
7397	throttled = true;
7398	break;
7399	}
7400
7401	/* we check whether we're throttled above */
7402	if (cpu_of(rq) != this_cpu) {
7403	unthrottle_cfs_rq_async(cfs_rq);
7404	continue;
7405	}
7406
7407	/*
7408	* Allow a parallel async unthrottle to unthrottle
7409	* this cfs_rq too via __cfsb_csd_unthrottle().
7410	* If we are first, do it ourselves at the end and
7411	* save on an IPI from remote CPUs.
7412	*/
7413	unthrottle_local = list_empty(&rq->cfsb_csd_list);
7414	list_add_tail(&cfs_rq->throttled_csd_list, &rq->cfsb_csd_list);
7415	}
7416
7417	if (unthrottle_local) {
7418	/*
7419	* Protect against an IPI that is also trying to flush
7420	* the unthrottled cfs_rq(s) from this CPU's csd_list.
7421	*/
7422	scoped_guard(irqsave)
7423	__cfsb_csd_unthrottle(cpu_rq(this_cpu));
7424	}
7425
7426	return throttled;
7427	}
7428
7429	/*
7430	* Responsible for refilling a task_group's bandwidth and unthrottling its
7431	* cfs_rqs as appropriate. If there has been no activity within the last
7432	* period the timer is deactivated until scheduling resumes; cfs_b->idle is
7433	* used to track this state.
7434	*/
7435	static int do_sched_cfs_period_timer(struct cfs_bandwidth *cfs_b, int overrun, unsigned long flags)
7436	__must_hold(&cfs_b->lock)
7437	{
7438	int throttled;
7439
7440	/* no need to continue the timer with no bandwidth constraint */
7441	if (cfs_b->quota == RUNTIME_INF)
7442	goto out_deactivate;
7443
7444	throttled = !list_empty(&cfs_b->throttled_cfs_rq);
7445	cfs_b->nr_periods += overrun;
7446
7447	/* Refill extra burst quota even if cfs_b->idle */
7448	__refill_cfs_bandwidth_runtime(cfs_b);
7449
7450	/*
7451	* idle depends on !throttled (for the case of a large deficit), and if
7452	* we're going inactive then everything else can be deferred
7453	*/
7454	if (cfs_b->idle && !throttled)
7455	goto out_deactivate;
7456
7457	if (!throttled) {
7458	/* mark as potentially idle for the upcoming period */
7459	cfs_b->idle = 1;
7460	return 0;
7461	}
7462
7463	/* account preceding periods in which throttling occurred */
7464	cfs_b->nr_throttled += overrun;
7465
7466	/*
7467	* This check is repeated as we release cfs_b->lock while we unthrottle.
7468	*/
7469	while (throttled && cfs_b->runtime > 0) {
7470	raw_spin_unlock_irqrestore(&cfs_b->lock, flags);
7471	/* we can't nest cfs_b->lock while distributing bandwidth */
7472	throttled = distribute_cfs_runtime(cfs_b);
7473	raw_spin_lock_irqsave(&cfs_b->lock, flags);
7474	}
7475
7476	/*
7477	* While we are ensured activity in the period following an
7478	* unthrottle, this also covers the case in which the new bandwidth is
7479	* insufficient to cover the existing bandwidth deficit. (Forcing the
7480	* timer to remain active while there are any throttled entities.)
7481	*/
7482	cfs_b->idle = 0;
7483
7484	return 0;
7485
7486	out_deactivate:
7487	return 1;
7488	}
7489
7490	/* a cfs_rq won't donate quota below this amount */
7491	static const u64 min_cfs_rq_runtime = 1 * NSEC_PER_MSEC;
7492	/* minimum remaining period time to redistribute slack quota */
7493	static const u64 min_bandwidth_expiration = 2 * NSEC_PER_MSEC;
7494	/* how long we wait to gather additional slack before distributing */
7495	static const u64 cfs_bandwidth_slack_period = 5 * NSEC_PER_MSEC;
7496
7497	/*
7498	* Are we near the end of the current quota period?
7499	*
7500	* Requires cfs_b->lock for hrtimer_expires_remaining to be safe against the
7501	* hrtimer base being cleared by hrtimer_start. In the case of
7502	* migrate_hrtimers, base is never cleared, so we are fine.
7503	*/
7504	static int runtime_refresh_within(struct cfs_bandwidth *cfs_b, u64 min_expire)
7505	{
7506	struct hrtimer *refresh_timer = &cfs_b->period_timer;
7507	s64 remaining;
7508
7509	/* if the call-back is running a quota refresh is already occurring */
7510	if (hrtimer_callback_running(refresh_timer))
7511	return 1;
7512
7513	/* is a quota refresh about to occur? */
7514	remaining = ktime_to_ns(hrtimer_expires_remaining(refresh_timer));
7515	if (remaining < (s64)min_expire)
7516	return 1;
7517
7518	return 0;
7519	}
7520
7521	static void start_cfs_slack_bandwidth(struct cfs_bandwidth *cfs_b)
7522	{
7523	u64 min_left = cfs_bandwidth_slack_period + min_bandwidth_expiration;
7524
7525	/* if there's a quota refresh soon don't bother with slack */
7526	if (runtime_refresh_within(cfs_b, min_left))
7527	return;
7528
7529	/* don't push forwards an existing deferred unthrottle */
7530	if (cfs_b->slack_started)
7531	return;
7532	cfs_b->slack_started = true;
7533
7534	hrtimer_start(&cfs_b->slack_timer,
7535	ns_to_ktime(cfs_bandwidth_slack_period),
7536	HRTIMER_MODE_REL);
7537	}
7538
7539	/* we know any runtime found here is valid as update_curr() precedes return */
7540	static void __return_cfs_rq_runtime(struct cfs_rq *cfs_rq)
7541	{
7542	struct cfs_bandwidth *cfs_b = tg_cfs_bandwidth(cfs_rq->tg);
7543	s64 slack_runtime = cfs_rq->runtime_remaining - min_cfs_rq_runtime;
7544
7545	if (slack_runtime <= 0)
7546	return;
7547
7548	guard(raw_spinlock)(&cfs_b->lock);
7549
7550	if (cfs_b->quota != RUNTIME_INF) {
7551	cfs_b->runtime += slack_runtime;
7552
7553	/* we are under rq->lock, defer unthrottling using a timer */
7554	if (cfs_b->runtime > sched_cfs_bandwidth_slice() &&
7555	!list_empty(&cfs_b->throttled_cfs_rq))
7556	start_cfs_slack_bandwidth(cfs_b);
7557	}
7558
7559	/* even if it's not valid for return we don't want to try again */
7560	cfs_rq->runtime_remaining -= slack_runtime;
7561	}
7562
7563	static __always_inline void return_cfs_rq_runtime(struct cfs_rq *cfs_rq)
7564	{
7565	if (!cfs_bandwidth_used())
7566	return;
7567
7568	if (!cfs_rq->runtime_enabled \|\| cfs_rq->nr_queued)
7569	return;
7570
7571	__return_cfs_rq_runtime(cfs_rq);
7572	}
7573
7574	/*
7575	* This is done with a timer (instead of inline with bandwidth return) since
7576	* it's necessary to juggle rq->locks to unthrottle their respective cfs_rqs.
7577	*/
7578	static void do_sched_cfs_slack_timer(struct cfs_bandwidth *cfs_b)
7579	{
7580	/* confirm we're still not at a refresh boundary */
7581	scoped_guard(raw_spinlock_irqsave, &cfs_b->lock) {
7582	u64 runtime = 0, slice = sched_cfs_bandwidth_slice();
7583
7584	cfs_b->slack_started = false;
7585
7586	if (runtime_refresh_within(cfs_b, min_bandwidth_expiration))
7587	return;
7588
7589	if (cfs_b->quota != RUNTIME_INF && cfs_b->runtime > slice)
7590	runtime = cfs_b->runtime;
7591
7592	if (!runtime)
7593	return;
7594	}
7595
7596	distribute_cfs_runtime(cfs_b);
7597	}
7598
7599	/*
7600	* When a group wakes up we want to make sure that its quota is not already
7601	* expired/exceeded, otherwise it may be allowed to steal additional ticks of
7602	* runtime as update_curr() throttling can not trigger until it's on-rq.
7603	*/
7604	static void check_enqueue_throttle(struct cfs_rq *cfs_rq)
7605	{
7606	if (!cfs_bandwidth_used())
7607	return;
7608
7609	/* an active group must be handled by the update_curr() path */
7610	if (!cfs_rq->runtime_enabled \|\| cfs_rq->h_curr)
7611	return;
7612
7613	/* ensure the group is not already throttled */
7614	if (cfs_rq_throttled(cfs_rq))
7615	return;
7616
7617	/* update runtime allocation */
7618	account_cfs_rq_runtime(cfs_rq, 0);
7619	}
7620
7621	static void sync_throttle(struct task_group *tg, int cpu)
7622	{
7623	struct cfs_rq pcfs_rq, cfs_rq;
7624
7625	if (!cfs_bandwidth_used())
7626	return;
7627
7628	if (!tg->parent)
7629	return;
7630
7631	cfs_rq = tg_cfs_rq(tg, cpu);
7632	pcfs_rq = tg_cfs_rq(tg->parent, cpu);
7633
7634	cfs_rq->throttle_count = pcfs_rq->throttle_count;
7635	cfs_rq->throttled_clock_pelt = rq_clock_pelt(cpu_rq(cpu));
7636
7637	/*
7638	* It is not enough to sync the "pelt_clock_throttled" indicator
7639	* with the parent cfs_rq when the hierarchy is not queued.
7640	* Always join a throttled hierarchy with PELT clock throttled
7641	* and leaf it to the first enqueue, or distribution to
7642	* unthrottle the PELT clock.
7643	*/
7644	if (cfs_rq->throttle_count)
7645	cfs_rq->pelt_clock_throttled = 1;
7646	}
7647
7648	static enum hrtimer_restart sched_cfs_slack_timer(struct hrtimer *timer)
7649	{
7650	struct cfs_bandwidth *cfs_b =
7651	container_of(timer, struct cfs_bandwidth, slack_timer);
7652
7653	do_sched_cfs_slack_timer(cfs_b);
7654
7655	return HRTIMER_NORESTART;
7656	}
7657
7658	static enum hrtimer_restart sched_cfs_period_timer(struct hrtimer *timer)
7659	{
7660	struct cfs_bandwidth *cfs_b =
7661	container_of(timer, struct cfs_bandwidth, period_timer);
7662	int overrun;
7663	int idle = 0;
7664	int count = 0;
7665
7666	CLASS(raw_spinlock_irqsave, cfsb_guard)(&cfs_b->lock);
7667
7668	for (;;) {
7669	overrun = hrtimer_forward_now(timer, cfs_b->period);
7670	if (!overrun)
7671	break;
7672
7673	idle = do_sched_cfs_period_timer(cfs_b, overrun, cfsb_guard.flags);
7674
7675	if (++count > 3) {
7676	u64 new, old = ktime_to_ns(cfs_b->period);
7677
7678	/*
7679	* Grow period by a factor of 2 to avoid losing precision.
7680	* Precision loss in the quota/period ratio can cause __cfs_schedulable
7681	* to fail.
7682	*/
7683	new = old * 2;
7684	if (new < max_bw_quota_period_us * NSEC_PER_USEC) {
7685	cfs_b->period = ns_to_ktime(new);
7686	cfs_b->quota *= 2;
7687	cfs_b->burst *= 2;
7688
7689	pr_warn_ratelimited(
7690	"cfs_period_timer[cpu%d]: period too short, scaling up (new cfs_period_us = %lld, cfs_quota_us = %lld)\n",
7691	smp_processor_id(),
7692	div_u64(new, NSEC_PER_USEC),
7693	div_u64(cfs_b->quota, NSEC_PER_USEC));
7694	} else {
7695	pr_warn_ratelimited(
7696	"cfs_period_timer[cpu%d]: period too short, but cannot scale up without losing precision (cfs_period_us = %lld, cfs_quota_us = %lld)\n",
7697	smp_processor_id(),
7698	div_u64(old, NSEC_PER_USEC),
7699	div_u64(cfs_b->quota, NSEC_PER_USEC));
7700	}
7701
7702	/* reset count so we don't come right back in here */
7703	count = 0;
7704	}
7705	}
7706
7707	if (idle) {
7708	cfs_b->period_active = 0;
7709	return HRTIMER_NORESTART;
7710	}
7711
7712	return HRTIMER_RESTART;
7713	}
7714
7715	void init_cfs_bandwidth(struct cfs_bandwidth cfs_b, struct cfs_bandwidth parent)
7716	{
7717	raw_spin_lock_init(&cfs_b->lock);
7718	cfs_b->runtime = 0;
7719	cfs_b->quota = RUNTIME_INF;
7720	cfs_b->period = us_to_ktime(default_bw_period_us());
7721	cfs_b->burst = 0;
7722	cfs_b->hierarchical_quota = parent ? parent->hierarchical_quota : RUNTIME_INF;
7723
7724	INIT_LIST_HEAD(&cfs_b->throttled_cfs_rq);
7725	hrtimer_setup(&cfs_b->period_timer, sched_cfs_period_timer, CLOCK_MONOTONIC,
7726	HRTIMER_MODE_ABS_PINNED);
7727
7728	/* Add a random offset so that timers interleave */
7729	hrtimer_set_expires(&cfs_b->period_timer,
7730	get_random_u32_below(cfs_b->period));
7731	hrtimer_setup(&cfs_b->slack_timer, sched_cfs_slack_timer, CLOCK_MONOTONIC,
7732	HRTIMER_MODE_REL);
7733	cfs_b->slack_started = false;
7734	}
7735
7736	static void init_cfs_rq_runtime(struct cfs_rq *cfs_rq)
7737	{
7738	cfs_rq->runtime_enabled = 0;
7739	INIT_LIST_HEAD(&cfs_rq->throttled_list);
7740	INIT_LIST_HEAD(&cfs_rq->throttled_csd_list);
7741	INIT_LIST_HEAD(&cfs_rq->throttled_limbo_list);
7742	}
7743
7744	void start_cfs_bandwidth(struct cfs_bandwidth *cfs_b)
7745	{
7746	lockdep_assert_held(&cfs_b->lock);
7747
7748	if (cfs_b->period_active)
7749	return;
7750
7751	cfs_b->period_active = 1;
7752	hrtimer_forward_now(&cfs_b->period_timer, cfs_b->period);
7753	hrtimer_start_expires(&cfs_b->period_timer, HRTIMER_MODE_ABS_PINNED);
7754	}
7755
7756	static void destroy_cfs_bandwidth(struct cfs_bandwidth *cfs_b)
7757	{
7758	int __maybe_unused i;
7759
7760	/* init_cfs_bandwidth() was not called */
7761	if (!cfs_b->throttled_cfs_rq.next)
7762	return;
7763
7764	hrtimer_cancel(&cfs_b->period_timer);
7765	hrtimer_cancel(&cfs_b->slack_timer);
7766
7767	/*
7768	* It is possible that we still have some cfs_rq's pending on a CSD
7769	* list, though this race is very rare. In order for this to occur, we
7770	* must have raced with the last task leaving the group while there
7771	* exist throttled cfs_rq(s), and the period_timer must have queued the
7772	* CSD item but the remote cpu has not yet processed it. To handle this,
7773	* we can simply flush all pending CSD work inline here. We're
7774	* guaranteed at this point that no additional cfs_rq of this group can
7775	* join a CSD list.
7776	*/
7777	for_each_possible_cpu(i) {
7778	struct rq *rq = cpu_rq(i);
7779
7780	if (list_empty(&rq->cfsb_csd_list))
7781	continue;
7782
7783	scoped_guard(irqsave)
7784	__cfsb_csd_unthrottle(rq);
7785	}
7786	}
7787
7788	/*
7789	* Both these CPU hotplug callbacks race against unregister_fair_sched_group()
7790	*
7791	* The race is harmless, since modifying bandwidth settings of unhooked group
7792	* bits doesn't do much.
7793	*/
7794
7795	/* cpu online callback */
7796	static void __maybe_unused update_runtime_enabled(struct rq *rq)
7797	{
7798	struct task_group *tg;
7799
7800	lockdep_assert_rq_held(rq);
7801
7802	guard(rcu)();
7803
7804	list_for_each_entry_rcu(tg, &task_groups, list) {
7805	struct cfs_bandwidth *cfs_b = &tg->cfs_bandwidth;
7806	struct cfs_rq *cfs_rq = tg_cfs_rq(tg, cpu_of(rq));
7807
7808	scoped_guard(raw_spinlock, &cfs_b->lock)
7809	cfs_rq->runtime_enabled = cfs_b->quota != RUNTIME_INF;
7810	}
7811	}
7812
7813	/* cpu offline callback */
7814	static void __maybe_unused unthrottle_offline_cfs_rqs(struct rq *rq)
7815	{
7816	struct task_group *tg;
7817
7818	lockdep_assert_rq_held(rq);
7819
7820	// Do not unthrottle for an active CPU
7821	if (cpumask_test_cpu(cpu_of(rq), cpu_active_mask))
7822	return;
7823
7824	/*
7825	* The rq clock has already been updated in the
7826	* set_rq_offline(), so we should skip updating
7827	* the rq clock again in unthrottle_cfs_rq().
7828	*/
7829	rq_clock_start_loop_update(rq);
7830
7831	guard(rcu)();
7832
7833	list_for_each_entry_rcu(tg, &task_groups, list) {
7834	struct cfs_rq *cfs_rq = tg_cfs_rq(tg, cpu_of(rq));
7835
7836	if (!cfs_rq->runtime_enabled)
7837	continue;
7838
7839	/*
7840	* Offline rq is schedulable till CPU is completely disabled
7841	* in take_cpu_down(), so we prevent new cfs throttling here.
7842	*/
7843	cfs_rq->runtime_enabled = 0;
7844
7845	if (!cfs_rq_throttled(cfs_rq))
7846	continue;
7847
7848	/*
7849	* clock_task is not advancing so we just need to make sure
7850	* there's some valid quota amount
7851	*/
7852	cfs_rq->runtime_remaining = 1;
7853	unthrottle_cfs_rq(cfs_rq);
7854	}
7855
7856	rq_clock_stop_loop_update(rq);
7857	}
7858
7859	bool cfs_task_bw_constrained(struct task_struct *p)
7860	{
7861	struct cfs_rq *cfs_rq = task_cfs_rq(p);
7862
7863	if (!cfs_bandwidth_used())
7864	return false;
7865
7866	if (cfs_rq->runtime_enabled \|\|
7867	tg_cfs_bandwidth(cfs_rq->tg)->hierarchical_quota != RUNTIME_INF)
7868	return true;
7869
7870	return false;
7871	}
7872
7873	#ifdef CONFIG_NO_HZ_FULL
7874	/* called from pick_next_task_fair() */
7875	static void sched_fair_update_stop_tick(struct rq rq, struct task_struct p)
7876	{
7877	int cpu = cpu_of(rq);
7878
7879	if (!cfs_bandwidth_used())
7880	return;
7881
7882	if (!tick_nohz_full_cpu(cpu))
7883	return;
7884
7885	if (rq->nr_running != 1)
7886	return;
7887
7888	/*
7889	* We know there is only one task runnable and we've just picked it. The
7890	* normal enqueue path will have cleared TICK_DEP_BIT_SCHED if we will
7891	* be otherwise able to stop the tick. Just need to check if we are using
7892	* bandwidth control.
7893	*/
7894	if (cfs_task_bw_constrained(p))
7895	tick_nohz_dep_set_cpu(cpu, TICK_DEP_BIT_SCHED);
7896	}
7897	#endif /* CONFIG_NO_HZ_FULL */
7898
7899	#else /* !CONFIG_CFS_BANDWIDTH: */
7900
7901	static bool account_cfs_rq_runtime(struct cfs_rq *cfs_rq, u64 delta_exec) { return false; }
7902	static void check_enqueue_throttle(struct cfs_rq *cfs_rq) {}
7903	static inline void sync_throttle(struct task_group *tg, int cpu) {}
7904	static __always_inline void return_cfs_rq_runtime(struct cfs_rq *cfs_rq) {}
7905	static void task_throttle_setup_work(struct task_struct *p) {}
7906	static bool task_is_throttled(struct task_struct *p) { return false; }
7907	static void dequeue_throttled_task(struct task_struct *p, int flags) {}
7908	static bool enqueue_throttled_task(struct task_struct *p) { return false; }
7909	static void record_throttle_clock(struct cfs_rq *cfs_rq) {}
7910
7911	static inline int cfs_rq_throttled(struct cfs_rq *cfs_rq)
7912	{
7913	return 0;
7914	}
7915
7916	static inline bool cfs_rq_pelt_clock_throttled(struct cfs_rq *cfs_rq)
7917	{
7918	return false;
7919	}
7920
7921	static inline int throttled_hierarchy(struct cfs_rq *cfs_rq)
7922	{
7923	return 0;
7924	}
7925
7926	static inline int lb_throttled_hierarchy(struct task_struct *p, int dst_cpu)
7927	{
7928	return 0;
7929	}
7930
7931	#ifdef CONFIG_FAIR_GROUP_SCHED
7932	void init_cfs_bandwidth(struct cfs_bandwidth cfs_b, struct cfs_bandwidth parent) {}
7933	static void init_cfs_rq_runtime(struct cfs_rq *cfs_rq) {}
7934	#endif
7935
7936	static inline struct cfs_bandwidth tg_cfs_bandwidth(struct task_group tg)
7937	{
7938	return NULL;
7939	}
7940	static inline void destroy_cfs_bandwidth(struct cfs_bandwidth *cfs_b) {}
7941	static inline void update_runtime_enabled(struct rq *rq) {}
7942	static inline void unthrottle_offline_cfs_rqs(struct rq *rq) {}
7943	#ifdef CONFIG_CGROUP_SCHED
7944	bool cfs_task_bw_constrained(struct task_struct *p)
7945	{
7946	return false;
7947	}
7948	#endif
7949	#endif /* !CONFIG_CFS_BANDWIDTH */
7950
7951	#if !defined(CONFIG_CFS_BANDWIDTH) \|\| !defined(CONFIG_NO_HZ_FULL)
7952	static inline void sched_fair_update_stop_tick(struct rq rq, struct task_struct p) {}
7953	#endif
7954
7955	/**************************************************
7956	* CFS operations on tasks:
7957	*/
7958
7959	#ifdef CONFIG_SCHED_HRTICK
7960	static void hrtick_start_fair(struct rq rq, struct task_struct p)
7961	{
7962	struct sched_entity *se = &p->se;
7963	unsigned long scale = 1024;
7964	unsigned long util = 0;
7965	u64 vdelta;
7966	u64 delta;
7967
7968	WARN_ON_ONCE(task_rq(p) != rq);
7969
7970	if (rq->cfs.h_nr_queued <= 1)
7971	return;
7972
7973	/*
7974	* Compute time until virtual deadline
7975	*/
7976	vdelta = se->deadline - se->vruntime;
7977	if ((s64)vdelta < 0) {
7978	if (task_current_donor(rq, p))
7979	resched_curr(rq);
7980	return;
7981	}
7982	delta = (se->h_load.weight * vdelta) / NICE_0_LOAD;
7983
7984	/*
7985	* Correct for instantaneous load of other classes.
7986	*/
7987	util += cpu_util_irq(rq);
7988	if (util && util < 1024) {
7989	scale *= 1024;
7990	scale /= (1024 - util);
7991	}
7992
7993	hrtick_start(rq, (scale * delta) / 1024);
7994	}
7995
7996	/*
7997	* Called on enqueue to start the hrtick when h_nr_queued becomes more than 1.
7998	*/
7999	static void hrtick_update(struct rq *rq)
8000	{
8001	struct task_struct *donor = rq->donor;
8002
8003	if (!hrtick_enabled_fair(rq) \|\| donor->sched_class != &fair_sched_class)
8004	return;
8005
8006	if (hrtick_active(rq))
8007	return;
8008
8009	hrtick_start_fair(rq, donor);
8010	}
8011	#else /* !CONFIG_SCHED_HRTICK: */
8012	static inline void
8013	hrtick_start_fair(struct rq rq, struct task_struct p)
8014	{
8015	}
8016
8017	static inline void hrtick_update(struct rq *rq)
8018	{
8019	}
8020	#endif /* !CONFIG_SCHED_HRTICK */
8021
8022	static inline bool cpu_overutilized(int cpu)
8023	{
8024	unsigned long rq_util_max;
8025
8026	if (!sched_energy_enabled())
8027	return false;
8028
8029	rq_util_max = uclamp_rq_get(cpu_rq(cpu), UCLAMP_MAX);
8030
8031	/* Return true only if the utilization doesn't fit CPU's capacity */
8032	return !util_fits_cpu(cpu_util_cfs(cpu), 0, rq_util_max, cpu);
8033	}
8034
8035	/*
8036	* overutilized value make sense only if EAS is enabled
8037	*/
8038	static inline bool is_rd_overutilized(struct root_domain *rd)
8039	{
8040	return !sched_energy_enabled() \|\| READ_ONCE(rd->overutilized);
8041	}
8042
8043	static inline void set_rd_overutilized(struct root_domain *rd, bool flag)
8044	{
8045	if (!sched_energy_enabled())
8046	return;
8047
8048	WRITE_ONCE(rd->overutilized, flag);
8049	trace_sched_overutilized_tp(rd, flag);
8050	}
8051
8052	static inline void check_update_overutilized_status(struct rq *rq)
8053	{
8054	/*
8055	* overutilized field is used for load balancing decisions only
8056	* if energy aware scheduler is being used
8057	*/
8058
8059	if (!is_rd_overutilized(rq->rd) && cpu_overutilized(rq->cpu))
8060	set_rd_overutilized(rq->rd, 1);
8061	}
8062
8063	/* Runqueue only has SCHED_IDLE tasks enqueued */
8064	static int sched_idle_rq(struct rq *rq)
8065	{
8066	return unlikely(rq->nr_running == rq->cfs.h_nr_idle &&
8067	rq->nr_running);
8068	}
8069
8070	static int choose_sched_idle_rq(struct rq rq, struct task_struct p)
8071	{
8072	return sched_idle_rq(rq) && !task_has_idle_policy(p);
8073	}
8074
8075	static int choose_idle_cpu(int cpu, struct task_struct *p)
8076	{
8077	return available_idle_cpu(cpu) \|\|
8078	choose_sched_idle_rq(cpu_rq(cpu), p);
8079	}
8080
8081	static void
8082	requeue_delayed_entity(struct cfs_rq cfs_rq, struct sched_entity se)
8083	{
8084	/*
8085	* se->sched_delayed should imply: se->on_rq == 1.
8086	* Because a delayed entity is one that is still on
8087	* the runqueue competing until elegibility.
8088	*/
8089	WARN_ON_ONCE(!se->sched_delayed);
8090	WARN_ON_ONCE(!se->on_rq);
8091
8092	if (update_entity_lag(cfs_rq, se)) {
8093	cfs_rq->h_nr_queued--;
8094	if (se != cfs_rq->curr)
8095	__dequeue_entity(cfs_rq, se);
8096	place_entity(cfs_rq, se, 0);
8097	if (se != cfs_rq->curr)
8098	__enqueue_entity(cfs_rq, se);
8099	cfs_rq->h_nr_queued++;
8100	}
8101
8102	update_load_avg(cfs_rq, se, 0);
8103	clear_delayed(se);
8104	}
8105
8106	static unsigned long enqueue_hierarchy(struct task_struct *p, int flags)
8107	{
8108	unsigned long weight = NICE_0_LOAD;
8109	int task_new = !(flags & ENQUEUE_WAKEUP);
8110	struct sched_entity *se = &p->se;
8111	int h_nr_idle = task_has_idle_policy(p);
8112	int h_nr_runnable = 1;
8113
8114	if (task_new && se->sched_delayed)
8115	h_nr_runnable = 0;
8116
8117	for_each_sched_entity(se) {
8118	struct cfs_rq *cfs_rq = cfs_rq_of(se);
8119
8120	update_curr(cfs_rq);
8121
8122	if (!se->on_rq) {
8123	enqueue_entity(cfs_rq, se, flags);
8124	} else {
8125	update_load_avg(cfs_rq, se, UPDATE_TG);
8126	se_update_runnable(se);
8127	update_cfs_group(se);
8128	}
8129
8130	cfs_rq->h_nr_runnable += h_nr_runnable;
8131	cfs_rq->h_nr_queued++;
8132	cfs_rq->h_nr_idle += h_nr_idle;
8133
8134	if (cfs_rq_is_idle(cfs_rq))
8135	h_nr_idle = 1;
8136
8137	weight = __calc_prop_weight(cfs_rq, se, weight);
8138
8139	flags = ENQUEUE_WAKEUP;
8140	}
8141
8142	return weight;
8143	}
8144
8145	/* Update curr's vruntime before placing entity or updating lag */
8146	static inline void update_curr_eevdf(struct cfs_rq *cfs_rq)
8147	{
8148	if (!cfs_rq->curr)
8149	return;
8150
8151	update_curr(cfs_rq_of(cfs_rq->curr));
8152	}
8153
8154	/*
8155	* The enqueue_task method is called before nr_running is
8156	* increased. Here we update the fair scheduling stats and
8157	* then put the task into the rbtree:
8158	*/
8159	static void
8160	enqueue_task_fair(struct rq rq, struct task_struct p, int flags)
8161	{
8162	int rq_h_nr_queued = rq->cfs.h_nr_queued;
8163	int task_new = !(flags & ENQUEUE_WAKEUP);
8164	struct sched_entity *se = &p->se;
8165	struct cfs_rq *cfs_rq = &rq->cfs;
8166	unsigned long weight;
8167	bool curr;
8168
8169	if (task_is_throttled(p) && enqueue_throttled_task(p))
8170	return;
8171
8172	/*
8173	* The code below (indirectly) updates schedutil which looks at
8174	* the cfs_rq utilization to select a frequency.
8175	* Let's add the task's estimated utilization to the cfs_rq's
8176	* estimated utilization, before we update schedutil.
8177	*/
8178	if (!p->se.sched_delayed \|\| (flags & ENQUEUE_DELAYED))
8179	util_est_enqueue(cfs_rq, p);
8180
8181	update_curr_eevdf(cfs_rq);
8182
8183	if (flags & ENQUEUE_DELAYED) {
8184	requeue_delayed_entity(cfs_rq, se);
8185	return;
8186	}
8187
8188	/*
8189	* If in_iowait is set, the code below may not trigger any cpufreq
8190	* utilization updates, so do it here explicitly with the IOWAIT flag
8191	* passed.
8192	*/
8193	if (p->in_iowait)
8194	cpufreq_update_util(rq, SCHED_CPUFREQ_IOWAIT);
8195
8196	curr = (cfs_rq->curr == se);
8197	/*
8198	* This can actually happen with CFS_BANDWIDTH, a task can be throttled
8199	* and dequeued, unthrottled from another CPU and then enqueued again
8200	* before schedule().
8201	*/
8202	if (curr)
8203	place_entity(cfs_rq, se, flags);
8204
8205	if (se->on_rq && se->sched_delayed)
8206	requeue_delayed_entity(cfs_rq, se);
8207
8208	weight = enqueue_hierarchy(p, flags);
8209
8210	if (!curr) {
8211	reweight_eevdf(cfs_rq, se, weight, false);
8212	place_entity(cfs_rq, se, flags \| ENQUEUE_QUEUED);
8213	__enqueue_entity(cfs_rq, se);
8214	}
8215
8216	if (!rq_h_nr_queued && rq->cfs.h_nr_queued)
8217	dl_server_start(&rq->fair_server);
8218
8219	/* At this point se is NULL and we are at root level*/
8220	add_nr_running(rq, 1);
8221
8222	/*
8223	* Since new tasks are assigned an initial util_avg equal to
8224	* half of the spare capacity of their CPU, tiny tasks have the
8225	* ability to cross the overutilized threshold, which will
8226	* result in the load balancer ruining all the task placement
8227	* done by EAS. As a way to mitigate that effect, do not account
8228	* for the first enqueue operation of new tasks during the
8229	* overutilized flag detection.
8230	*
8231	* A better way of solving this problem would be to wait for
8232	* the PELT signals of tasks to converge before taking them
8233	* into account, but that is not straightforward to implement,
8234	* and the following generally works well enough in practice.
8235	*/
8236	if (!task_new)
8237	check_update_overutilized_status(rq);
8238
8239	assert_list_leaf_cfs_rq(rq);
8240
8241	hrtick_update(rq);
8242	}
8243
8244	static void dequeue_hierarchy(struct task_struct *p, int flags)
8245	{
8246	struct sched_entity *se = &p->se;
8247	bool task_sleep = flags & DEQUEUE_SLEEP;
8248	bool task_delayed = flags & DEQUEUE_DELAYED;
8249	bool task_throttled = flags & DEQUEUE_THROTTLE;
8250	int h_nr_runnable = 0;
8251	int h_nr_idle = task_has_idle_policy(p);
8252	bool dequeue = true;
8253
8254	if (task_sleep \|\| task_delayed \|\| !se->sched_delayed)
8255	h_nr_runnable = 1;
8256
8257	for_each_sched_entity(se) {
8258	struct cfs_rq *cfs_rq = cfs_rq_of(se);
8259
8260	update_curr(cfs_rq);
8261
8262	if (dequeue) {
8263	dequeue_entity(cfs_rq, se, flags);
8264	/* Don't dequeue parent if it has other entities besides us */
8265	if (cfs_rq->load.weight)
8266	dequeue = false;
8267	} else {
8268	update_load_avg(cfs_rq, se, UPDATE_TG);
8269	se_update_runnable(se);
8270	update_cfs_group(se);
8271	}
8272
8273	cfs_rq->h_nr_runnable -= h_nr_runnable;
8274	cfs_rq->h_nr_queued--;
8275	cfs_rq->h_nr_idle -= h_nr_idle;
8276
8277	if (cfs_rq_is_idle(cfs_rq))
8278	h_nr_idle = 1;
8279
8280	if (throttled_hierarchy(cfs_rq) && task_throttled)
8281	record_throttle_clock(cfs_rq);
8282
8283	flags \|= DEQUEUE_SLEEP;
8284	flags &= ~(DEQUEUE_DELAYED \| DEQUEUE_SPECIAL);
8285	}
8286	}
8287
8288	/*
8289	* The part of dequeue_task_fair() that is needed to dequeue delayed tasks.
8290	*
8291	* Returns:
8292	* true - dequeued
8293	* false - delayed
8294	*/
8295	static bool __dequeue_task(struct rq rq, struct task_struct p, int flags)
8296	{
8297	struct sched_entity *se = &p->se;
8298	struct cfs_rq *cfs_rq = &rq->cfs;
8299	bool was_sched_idle = sched_idle_rq(rq);
8300	bool task_sleep = flags & DEQUEUE_SLEEP;
8301	bool task_delayed = flags & DEQUEUE_DELAYED;
8302
8303	clear_buddies(cfs_rq, se);
8304
8305	update_curr_eevdf(cfs_rq);
8306	update_entity_lag(cfs_rq, se);
8307
8308	if (flags & DEQUEUE_DELAYED) {
8309	WARN_ON_ONCE(!se->sched_delayed);
8310	} else {
8311	bool delay = task_sleep;
8312	/*
8313	* DELAY_DEQUEUE relies on spurious wakeups, special task
8314	* states must not suffer spurious wakeups, excempt them.
8315	*/
8316	if (flags & (DEQUEUE_SPECIAL \| DEQUEUE_THROTTLE))
8317	delay = false;
8318
8319	WARN_ON_ONCE(delay && se->sched_delayed);
8320
8321	if (sched_feat(DELAY_DEQUEUE) && delay &&
8322	!entity_eligible(cfs_rq, se)) {
8323	update_load_avg(cfs_rq_of(se), se, UPDATE_UTIL_EST);
8324	set_delayed(se);
8325	return false;
8326	}
8327	}
8328
8329	dequeue_hierarchy(p, flags);
8330
8331	if (sched_feat(PLACE_REL_DEADLINE) && !task_sleep) {
8332	se->deadline -= se->vruntime;
8333	se->rel_deadline = 1;
8334	}
8335	if (se != cfs_rq->curr)
8336	__dequeue_entity(cfs_rq, se);
8337
8338	sub_nr_running(rq, 1);
8339
8340	/* balance early to pull high priority tasks */
8341	if (unlikely(!was_sched_idle && sched_idle_rq(rq)))
8342	rq->next_balance = jiffies;
8343
8344	if (task_delayed) {
8345	clear_delayed(se);
8346
8347	WARN_ON_ONCE(!task_sleep);
8348	WARN_ON_ONCE(p->on_rq != 1);
8349
8350	/*
8351	* Fix-up what block_task() skipped.
8352	*
8353	* Must be last, @p might not be valid after this.
8354	*/
8355	__block_task(rq, p);
8356	}
8357
8358	return true;
8359	}
8360
8361	/*
8362	* The dequeue_task method is called before nr_running is
8363	* decreased. We remove the task from the rbtree and
8364	* update the fair scheduling stats:
8365	*/
8366	static bool dequeue_task_fair(struct rq rq, struct task_struct p, int flags)
8367	{
8368	if (task_is_throttled(p)) {
8369	dequeue_throttled_task(p, flags);
8370	return true;
8371	}
8372
8373	if (!p->se.sched_delayed)
8374	util_est_dequeue(&rq->cfs, p);
8375
8376	if (!__dequeue_task(rq, p, flags))
8377	return false;
8378
8379	/*
8380	* Must not reference @p after __dequeue_task(DEQUEUE_DELAYED).
8381	*/
8382	return true;
8383	}
8384
8385	static inline unsigned int cfs_h_nr_delayed(struct rq *rq)
8386	{
8387	return (rq->cfs.h_nr_queued - rq->cfs.h_nr_runnable);
8388	}
8389
8390	/* Working cpumask for: sched_balance_rq(), sched_balance_newidle(). */
8391	static DEFINE_PER_CPU(cpumask_var_t, load_balance_mask);
8392	static DEFINE_PER_CPU(cpumask_var_t, select_rq_mask);
8393	static DEFINE_PER_CPU(cpumask_var_t, should_we_balance_tmpmask);
8394
8395	#ifdef CONFIG_NO_HZ_COMMON
8396
8397	static struct {
8398	cpumask_var_t idle_cpus_mask;
8399	int has_blocked_load; /* Idle CPUS has blocked load */
8400	int needs_update; /* Newly idle CPUs need their next_balance collated */
8401	unsigned long next_balance; /* in jiffy units */
8402	unsigned long next_blocked; /* Next update of blocked load in jiffies */
8403	} nohz ____cacheline_aligned;
8404
8405	#endif /* CONFIG_NO_HZ_COMMON */
8406
8407	static unsigned long cpu_load(struct rq *rq)
8408	{
8409	return cfs_rq_load_avg(&rq->cfs);
8410	}
8411
8412	/*
8413	* cpu_load_without - compute CPU load without any contributions from *p
8414	* @cpu: the CPU which load is requested
8415	* @p: the task which load should be discounted
8416	*
8417	* The load of a CPU is defined by the load of tasks currently enqueued on that
8418	* CPU as well as tasks which are currently sleeping after an execution on that
8419	* CPU.
8420	*
8421	* This method returns the load of the specified CPU by discounting the load of
8422	* the specified task, whenever the task is currently contributing to the CPU
8423	* load.
8424	*/
8425	static unsigned long cpu_load_without(struct rq rq, struct task_struct p)
8426	{
8427	struct cfs_rq *cfs_rq;
8428	unsigned int load;
8429
8430	/* Task has no contribution or is new */
8431	if (cpu_of(rq) != task_cpu(p) \|\| !READ_ONCE(p->se.avg.last_update_time))
8432	return cpu_load(rq);
8433
8434	cfs_rq = &rq->cfs;
8435	load = READ_ONCE(cfs_rq->avg.load_avg);
8436
8437	/* Discount task's util from CPU's util */
8438	lsub_positive(&load, task_h_load(p));
8439
8440	return load;
8441	}
8442
8443	static unsigned long cpu_runnable(struct rq *rq)
8444	{
8445	return cfs_rq_runnable_avg(&rq->cfs);
8446	}
8447
8448	static unsigned long cpu_runnable_without(struct rq rq, struct task_struct p)
8449	{
8450	struct cfs_rq *cfs_rq;
8451	unsigned int runnable;
8452
8453	/* Task has no contribution or is new */
8454	if (cpu_of(rq) != task_cpu(p) \|\| !READ_ONCE(p->se.avg.last_update_time))
8455	return cpu_runnable(rq);
8456
8457	cfs_rq = &rq->cfs;
8458	runnable = READ_ONCE(cfs_rq->avg.runnable_avg);
8459
8460	/* Discount task's runnable from CPU's runnable */
8461	lsub_positive(&runnable, p->se.avg.runnable_avg);
8462
8463	return runnable;
8464	}
8465
8466	static unsigned long capacity_of(int cpu)
8467	{
8468	return cpu_rq(cpu)->cpu_capacity;
8469	}
8470
8471	static void record_wakee(struct task_struct *p)
8472	{
8473	/*
8474	* Only decay a single time; tasks that have less then 1 wakeup per
8475	* jiffy will not have built up many flips.
8476	*/
8477	if (time_after(jiffies, current->wakee_flip_decay_ts + HZ)) {
8478	current->wakee_flips >>= 1;
8479	current->wakee_flip_decay_ts = jiffies;
8480	}
8481
8482	if (current->last_wakee != p) {
8483	current->last_wakee = p;
8484	current->wakee_flips++;
8485	}
8486	}
8487
8488	/*
8489	* Detect M:N waker/wakee relationships via a switching-frequency heuristic.
8490	*
8491	* A waker of many should wake a different task than the one last awakened
8492	* at a frequency roughly N times higher than one of its wakees.
8493	*
8494	* In order to determine whether we should let the load spread vs consolidating
8495	* to shared cache, we look for a minimum 'flip' frequency of llc_size in one
8496	* partner, and a factor of lls_size higher frequency in the other.
8497	*
8498	* With both conditions met, we can be relatively sure that the relationship is
8499	* non-monogamous, with partner count exceeding socket size.
8500	*
8501	* Waker/wakee being client/server, worker/dispatcher, interrupt source or
8502	* whatever is irrelevant, spread criteria is apparent partner count exceeds
8503	* socket size.
8504	*/
8505	static int wake_wide(struct task_struct *p)
8506	{
8507	unsigned int master = current->wakee_flips;
8508	unsigned int slave = p->wakee_flips;
8509	int factor = __this_cpu_read(sd_llc_size);
8510
8511	if (master < slave)
8512	swap(master, slave);
8513	if (slave < factor \|\| master < slave * factor)
8514	return 0;
8515	return 1;
8516	}
8517
8518	/*
8519	* The purpose of wake_affine() is to quickly determine on which CPU we can run
8520	* soonest. For the purpose of speed we only consider the waking and previous
8521	* CPU.
8522	*
8523	* wake_affine_idle() - only considers 'now', it check if the waking CPU is
8524	* cache-affine and is (or will be) idle.
8525	*
8526	* wake_affine_weight() - considers the weight to reflect the average
8527	* scheduling latency of the CPUs. This seems to work
8528	* for the overloaded case.
8529	*/
8530	static int
8531	wake_affine_idle(int this_cpu, int prev_cpu, int sync)
8532	{
8533	/*
8534	* If this_cpu is idle, it implies the wakeup is from interrupt
8535	* context. Only allow the move if cache is shared. Otherwise an
8536	* interrupt intensive workload could force all tasks onto one
8537	* node depending on the IO topology or IRQ affinity settings.
8538	*
8539	* If the prev_cpu is idle and cache affine then avoid a migration.
8540	* There is no guarantee that the cache hot data from an interrupt
8541	* is more important than cache hot data on the prev_cpu and from
8542	* a cpufreq perspective, it's better to have higher utilisation
8543	* on one CPU.
8544	*/
8545	if (available_idle_cpu(this_cpu) && cpus_share_cache(this_cpu, prev_cpu))
8546	return available_idle_cpu(prev_cpu) ? prev_cpu : this_cpu;
8547
8548	if (sync) {
8549	struct rq *rq = cpu_rq(this_cpu);
8550
8551	if ((rq->nr_running - cfs_h_nr_delayed(rq)) == 1)
8552	return this_cpu;
8553	}
8554
8555	if (available_idle_cpu(prev_cpu))
8556	return prev_cpu;
8557
8558	return nr_cpumask_bits;
8559	}
8560
8561	static int
8562	wake_affine_weight(struct sched_domain sd, struct task_struct p,
8563	int this_cpu, int prev_cpu, int sync)
8564	{
8565	s64 this_eff_load, prev_eff_load;
8566	unsigned long task_load;
8567
8568	this_eff_load = cpu_load(cpu_rq(this_cpu));
8569
8570	if (sync) {
8571	unsigned long current_load = task_h_load(current);
8572
8573	if (current_load > this_eff_load)
8574	return this_cpu;
8575
8576	this_eff_load -= current_load;
8577	}
8578
8579	task_load = task_h_load(p);
8580
8581	this_eff_load += task_load;
8582	if (sched_feat(WA_BIAS))
8583	this_eff_load *= 100;
8584	this_eff_load *= capacity_of(prev_cpu);
8585
8586	prev_eff_load = cpu_load(cpu_rq(prev_cpu));
8587	prev_eff_load -= task_load;
8588	if (sched_feat(WA_BIAS))
8589	prev_eff_load *= 100 + (sd->imbalance_pct - 100) / 2;
8590	prev_eff_load *= capacity_of(this_cpu);
8591
8592	/*
8593	* If sync, adjust the weight of prev_eff_load such that if
8594	* prev_eff == this_eff that select_idle_sibling() will consider
8595	* stacking the wakee on top of the waker if no other CPU is
8596	* idle.
8597	*/
8598	if (sync)
8599	prev_eff_load += 1;
8600
8601	return this_eff_load < prev_eff_load ? this_cpu : nr_cpumask_bits;
8602	}
8603
8604	static int wake_affine(struct sched_domain sd, struct task_struct p,
8605	int this_cpu, int prev_cpu, int sync)
8606	{
8607	int target = nr_cpumask_bits;
8608
8609	if (sched_feat(WA_IDLE))
8610	target = wake_affine_idle(this_cpu, prev_cpu, sync);
8611
8612	if (sched_feat(WA_WEIGHT) && target == nr_cpumask_bits)
8613	target = wake_affine_weight(sd, p, this_cpu, prev_cpu, sync);
8614
8615	schedstat_inc(p->stats.nr_wakeups_affine_attempts);
8616	if (target != this_cpu)
8617	return prev_cpu;
8618
8619	schedstat_inc(sd->ttwu_move_affine);
8620	schedstat_inc(p->stats.nr_wakeups_affine);
8621	return target;
8622	}
8623
8624	static struct sched_group *
8625	sched_balance_find_dst_group(struct sched_domain sd, struct task_struct p, int this_cpu);
8626
8627	/*
8628	* sched_balance_find_dst_group_cpu - find the idlest CPU among the CPUs in the group.
8629	*/
8630	static int
8631	sched_balance_find_dst_group_cpu(struct sched_group group, struct task_struct p, int this_cpu)
8632	{
8633	unsigned long load, min_load = ULONG_MAX;
8634	u64 min_exit_latency = U64_MAX;
8635	unsigned int nr_candidates = 0;
8636	int least_loaded_cpu = this_cpu;
8637	int shallowest_idle_cpu = -1;
8638	int i;
8639
8640	/* Check if we have any choice: */
8641	if (group->group_weight == 1)
8642	return cpumask_first(sched_group_span(group));
8643
8644	/* Traverse only the allowed CPUs */
8645	for_each_cpu_and(i, sched_group_span(group), p->cpus_ptr) {
8646	struct rq *rq = cpu_rq(i);
8647
8648	if (!sched_core_cookie_match(rq, p))
8649	continue;
8650
8651	if (choose_sched_idle_rq(rq, p))
8652	return i;
8653
8654	if (available_idle_cpu(i)) {
8655	struct cpuidle_state *idle = idle_get_state(rq);
8656	u64 exit_latency = idle ? idle->exit_latency : U64_MAX;
8657
8658	if (shallowest_idle_cpu == -1 \|\| exit_latency < min_exit_latency) {
8659	min_exit_latency = exit_latency;
8660	shallowest_idle_cpu = i;
8661	nr_candidates = 1;
8662	} else if (exit_latency == min_exit_latency) {
8663	nr_candidates++;
8664	if (!reciprocal_scale(sched_rng(), nr_candidates))
8665	shallowest_idle_cpu = i;
8666	}
8667	} else if (shallowest_idle_cpu == -1) {
8668	load = cpu_load(cpu_rq(i));
8669	if (load < min_load) {
8670	min_load = load;
8671	least_loaded_cpu = i;
8672	}
8673	}
8674	}
8675
8676	return shallowest_idle_cpu != -1 ? shallowest_idle_cpu : least_loaded_cpu;
8677	}
8678
8679	static inline int sched_balance_find_dst_cpu(struct sched_domain sd, struct task_struct p,
8680	int cpu, int prev_cpu, int sd_flag)
8681	{
8682	int new_cpu = cpu;
8683
8684	if (!cpumask_intersects(sched_domain_span(sd), p->cpus_ptr))
8685	return prev_cpu;
8686
8687	/*
8688	* We need task's util for cpu_util_without, sync it up to
8689	* prev_cpu's last_update_time.
8690	*/
8691	if (!(sd_flag & SD_BALANCE_FORK))
8692	sync_entity_load_avg(&p->se);
8693
8694	while (sd) {
8695	struct sched_group *group;
8696	struct sched_domain *tmp;
8697	int weight;
8698
8699	if (!(sd->flags & sd_flag)) {
8700	sd = sd->child;
8701	continue;
8702	}
8703
8704	group = sched_balance_find_dst_group(sd, p, cpu);
8705	if (!group) {
8706	sd = sd->child;
8707	continue;
8708	}
8709
8710	new_cpu = sched_balance_find_dst_group_cpu(group, p, cpu);
8711	if (new_cpu == cpu) {
8712	/* Now try balancing at a lower domain level of 'cpu': */
8713	sd = sd->child;
8714	continue;
8715	}
8716
8717	/* Now try balancing at a lower domain level of 'new_cpu': */
8718	cpu = new_cpu;
8719	weight = sd->span_weight;
8720	sd = NULL;
8721	for_each_domain(cpu, tmp) {
8722	if (weight <= tmp->span_weight)
8723	break;
8724	if (tmp->flags & sd_flag)
8725	sd = tmp;
8726	}
8727	}
8728
8729	return new_cpu;
8730	}
8731
8732	static inline int __select_idle_cpu(int cpu, struct task_struct *p)
8733	{
8734	if (choose_idle_cpu(cpu, p) && sched_cpu_cookie_match(cpu_rq(cpu), p))
8735	return cpu;
8736
8737	return -1;
8738	}
8739
8740	DEFINE_STATIC_KEY_FALSE(sched_smt_present);
8741	EXPORT_SYMBOL_GPL(sched_smt_present);
8742
8743	static inline void set_idle_cores(int cpu, int val)
8744	{
8745	struct sched_domain_shared *sds;
8746
8747	sds = rcu_dereference_all(per_cpu(sd_balance_shared, cpu));
8748	if (sds)
8749	WRITE_ONCE(sds->has_idle_cores, val);
8750	}
8751
8752	static inline bool test_idle_cores(int cpu)
8753	{
8754	struct sched_domain_shared *sds;
8755
8756	sds = rcu_dereference_all(per_cpu(sd_balance_shared, cpu));
8757	if (sds)
8758	return READ_ONCE(sds->has_idle_cores);
8759
8760	return false;
8761	}
8762
8763	/*
8764	* Redirect a CPU to a higher-priority available sibling in its SMT domain,
8765	* subject to task affinity.
8766	*/
8767	static inline int select_idle_smt_cpu(struct task_struct *p, int cpu)
8768	{
8769	struct sched_domain *sd;
8770	int best = cpu;
8771	int sibling;
8772
8773	if (!sched_smt_active())
8774	return cpu;
8775
8776	sd = rcu_dereference_all(cpu_rq(cpu)->sd);
8777	if (!sd \|\| !(sd->flags & SD_SHARE_CPUCAPACITY) \|\|
8778	!(sd->flags & SD_ASYM_PACKING))
8779	return cpu;
8780
8781	for_each_cpu_and(sibling, sched_domain_span(sd), p->cpus_ptr) {
8782	if (sibling == best \|\| !choose_idle_cpu(sibling, p))
8783	continue;
8784
8785	if (sched_asym_prefer(sibling, best))
8786	best = sibling;
8787	}
8788
8789	return best;
8790	}
8791
8792	/*
8793	* Scans the local SMT mask to see if the entire core is idle, and records this
8794	* information in sd_balance_shared->has_idle_cores.
8795	*
8796	* Since SMT siblings share all cache levels, inspecting this limited remote
8797	* state should be fairly cheap.
8798	*/
8799	void __update_idle_core(struct rq *rq)
8800	{
8801	int core = cpu_of(rq);
8802	int cpu;
8803
8804	rcu_read_lock();
8805	if (test_idle_cores(core))
8806	goto unlock;
8807
8808	for_each_cpu(cpu, cpu_smt_mask(core)) {
8809	if (cpu == core)
8810	continue;
8811
8812	if (!available_idle_cpu(cpu))
8813	goto unlock;
8814	}
8815
8816	set_idle_cores(core, 1);
8817	unlock:
8818	rcu_read_unlock();
8819	}
8820
8821	/*
8822	* Scan the entire LLC domain for idle cores; this dynamically switches off if
8823	* there are no idle cores left in the system; tracked through
8824	* sd_balance_shared->has_idle_cores and enabled through update_idle_core()
8825	* above.
8826	*/
8827	static int select_idle_core(struct task_struct p, int core, struct cpumask cpus, int *idle_cpu)
8828	{
8829	bool idle = true;
8830	int cpu;
8831
8832	for_each_cpu(cpu, cpu_smt_mask(core)) {
8833	if (!available_idle_cpu(cpu)) {
8834	idle = false;
8835	if (*idle_cpu == -1) {
8836	if (choose_sched_idle_rq(cpu_rq(cpu), p) &&
8837	cpumask_test_cpu(cpu, cpus)) {
8838	*idle_cpu = cpu;
8839	break;
8840	}
8841	continue;
8842	}
8843	break;
8844	}
8845	if (*idle_cpu == -1 && cpumask_test_cpu(cpu, cpus))
8846	*idle_cpu = cpu;
8847	}
8848
8849	if (idle)
8850	return core;
8851
8852	cpumask_andnot(cpus, cpus, cpu_smt_mask(core));
8853	return -1;
8854	}
8855
8856	/*
8857	* Scan the local SMT mask for idle CPUs.
8858	*/
8859	static int select_idle_smt(struct task_struct p, struct sched_domain sd, int target)
8860	{
8861	int cpu;
8862
8863	for_each_cpu_and(cpu, cpu_smt_mask(target), p->cpus_ptr) {
8864	if (cpu == target)
8865	continue;
8866	/*
8867	* Check if the CPU is in the LLC scheduling domain of @target.
8868	* Due to isolcpus, there is no guarantee that all the siblings are in the domain.
8869	*/
8870	if (!cpumask_test_cpu(cpu, sched_domain_span(sd)))
8871	continue;
8872	if (choose_idle_cpu(cpu, p))
8873	return cpu;
8874	}
8875
8876	return -1;
8877	}
8878
8879	/*
8880	* Scan the LLC domain for idle CPUs; this is dynamically regulated by
8881	* comparing the average scan cost (tracked in sd->avg_scan_cost) against the
8882	* average idle time for this rq (as found in rq->avg_idle).
8883	*/
8884	static int select_idle_cpu(struct task_struct p, struct sched_domain sd, bool has_idle_core, int target)
8885	{
8886	struct cpumask *cpus = this_cpu_cpumask_var_ptr(select_rq_mask);
8887	int i, cpu, idle_cpu = -1, nr = INT_MAX;
8888
8889	if (sched_feat(SIS_UTIL) && sd->shared) {
8890	/*
8891	* Increment because !--nr is the condition to stop scan.
8892	*
8893	* Since "sd" is "sd_llc" for target CPU dereferenced in the
8894	* caller, it is safe to directly dereference "sd->shared".
8895	* Topology bits always ensure it assigned for "sd_llc" abd it
8896	* cannot disappear as long as we have a RCU protected
8897	* reference to one the associated "sd" here.
8898	*/
8899	nr = READ_ONCE(sd->shared->nr_idle_scan) + 1;
8900	/* overloaded LLC is unlikely to have idle cpu/core */
8901	if (nr == 1)
8902	return -1;
8903	}
8904
8905	if (!cpumask_and(cpus, sched_domain_span(sd), p->cpus_ptr))
8906	return -1;
8907
8908	if (static_branch_unlikely(&sched_cluster_active)) {
8909	struct sched_group *sg = sd->groups;
8910
8911	if (sg->flags & SD_CLUSTER) {
8912	for_each_cpu_wrap(cpu, sched_group_span(sg), target + 1) {
8913	if (!cpumask_test_cpu(cpu, cpus))
8914	continue;
8915
8916	if (has_idle_core) {
8917	i = select_idle_core(p, cpu, cpus, &idle_cpu);
8918	if ((unsigned int)i < nr_cpumask_bits)
8919	return i;
8920	} else {
8921	if (--nr <= 0)
8922	return -1;
8923	idle_cpu = __select_idle_cpu(cpu, p);
8924	if ((unsigned int)idle_cpu < nr_cpumask_bits)
8925	return idle_cpu;
8926	}
8927	}
8928	cpumask_andnot(cpus, cpus, sched_group_span(sg));
8929	}
8930	}
8931
8932	for_each_cpu_wrap(cpu, cpus, target + 1) {
8933	if (has_idle_core) {
8934	i = select_idle_core(p, cpu, cpus, &idle_cpu);
8935	if ((unsigned int)i < nr_cpumask_bits)
8936	return i;
8937
8938	} else {
8939	if (--nr <= 0)
8940	return -1;
8941	idle_cpu = __select_idle_cpu(cpu, p);
8942	if ((unsigned int)idle_cpu < nr_cpumask_bits)
8943	break;
8944	}
8945	}
8946
8947	if (has_idle_core)
8948	set_idle_cores(target, false);
8949
8950	return idle_cpu;
8951	}
8952
8953	/*
8954	* Idle-capacity scan converts util_fits_cpu() outcomes into preference ranks,
8955	* where lower values indicate a better fit - see select_idle_capacity().
8956	*
8957	* A CPU that both fits the task and sits on a fully-idle SMT core is returned
8958	* immediately and is never assigned one of these ranks. On !SMT every CPU is
8959	* its own "core", so the early return covers all fits-and-idle cases and the
8960	* core-tier ranks below become unreachable.
8961	*
8962	* Rank Val Tier Meaning
8963	* ------------------------------ --- ------ ---------------------------
8964	* ASYM_IDLE_UCLAMP_MISFIT -4 core Idle core; capacity fits
8965	* util but uclamp_min misses.
8966	* ASYM_IDLE_COMPLETE_MISFIT -3 core Idle core; capacity does
8967	* not fit. Still beats every
8968	* thread-tier rank: a busy
8969	* sibling cuts effective
8970	* capacity more than a
8971	* misfit hurts a quiet core.
8972	* ASYM_IDLE_THREAD_FITS -2 thread Busy SMT sibling; capacity
8973	* fits util + uclamp.
8974	* ASYM_IDLE_THREAD_UCLAMP_MISFIT -1 thread Busy SMT sibling; capacity
8975	* fits but uclamp_min misses
8976	* (native util_fits_cpu()
8977	* return value).
8978	* ASYM_IDLE_THREAD_MISFIT 0 thread Busy SMT sibling; capacity
8979	* does not fit.
8980	*
8981	* ASYM_IDLE_CORE_BIAS (-3) is an offset, not a state. On an idle core,
8982	* fits += ASYM_IDLE_CORE_BIAS rebases thread-tier ranks into the core tier:
8983	*
8984	* ASYM_IDLE_THREAD_UCLAMP_MISFIT (-1) + BIAS -> ASYM_IDLE_UCLAMP_MISFIT (-4)
8985	* ASYM_IDLE_THREAD_MISFIT (0) + BIAS -> ASYM_IDLE_COMPLETE_MISFIT (-3)
8986	*
8987	* ASYM_IDLE_THREAD_FITS (-2) is never rebased because a fully-fitting idle-core
8988	* candidate early-returns from select_idle_capacity().
8989	*/
8990	enum asym_fits_state {
8991	ASYM_IDLE_UCLAMP_MISFIT = -4,
8992	ASYM_IDLE_COMPLETE_MISFIT,
8993	ASYM_IDLE_THREAD_FITS,
8994	ASYM_IDLE_THREAD_UCLAMP_MISFIT,
8995	ASYM_IDLE_THREAD_MISFIT,
8996
8997	/* util_fits_cpu() bias for idle core */
8998	ASYM_IDLE_CORE_BIAS = -3,
8999	};
9000
9001	/*
9002	* Scan the asym_capacity domain for idle CPUs; pick the first idle one on which
9003	* the task fits. If no CPU is big enough, but there are idle ones, try to
9004	* maximize capacity.
9005	*/
9006	static int
9007	select_idle_capacity(struct task_struct p, struct sched_domain sd, int target)
9008	{
9009	/*
9010	* On !SMT systems, has_idle_core is always false and preferred_core
9011	* is always true (CPU == core), so the SMT preference logic below
9012	* collapses to the plain capacity scan.
9013	*/
9014	bool has_idle_core = sched_smt_active() && test_idle_cores(target);
9015	unsigned long task_util, util_min, util_max, best_cap = 0;
9016	int fits, best_fits = ASYM_IDLE_THREAD_MISFIT;
9017	int cpu, best_cpu = -1;
9018	struct cpumask *cpus;
9019	int nr = INT_MAX;
9020
9021	cpus = this_cpu_cpumask_var_ptr(select_rq_mask);
9022	cpumask_and(cpus, sched_domain_span(sd), p->cpus_ptr);
9023
9024	task_util = task_util_est(p);
9025	util_min = uclamp_eff_value(p, UCLAMP_MIN);
9026	util_max = uclamp_eff_value(p, UCLAMP_MAX);
9027
9028	if (sched_feat(SIS_UTIL) && sd->shared) {
9029	/*
9030	* Same nr_idle_scan hint as select_idle_cpu(), nr only limits
9031	* the scan when not preferring an idle core.
9032	*/
9033	nr = READ_ONCE(sd->shared->nr_idle_scan) + 1;
9034	/* overloaded domain is unlikely to have idle cpu/core */
9035	if (nr == 1)
9036	return -1;
9037	}
9038
9039	for_each_cpu_wrap(cpu, cpus, target) {
9040	bool preferred_core = !has_idle_core \|\| is_core_idle(cpu);
9041	unsigned long cpu_cap = capacity_of(cpu);
9042
9043	/*
9044	* Stop when the nr_idle_scan is exhausted (mirrors
9045	* select_idle_cpu() logic).
9046	*/
9047	if (!has_idle_core && --nr <= 0)
9048	return best_cpu;
9049
9050	if (!choose_idle_cpu(cpu, p))
9051	continue;
9052
9053	fits = util_fits_cpu(task_util, util_min, util_max, cpu);
9054
9055	/*
9056	* Perfect fit: capacity satisfies util + uclamp and the CPU
9057	* sits on a fully-idle SMT core, this is a !SMT system, or
9058	* there is no idle core to find.
9059	* Short-circuit the rank-based selection and return
9060	* immediately.
9061	*/
9062	if (fits > 0 && preferred_core)
9063	return cpu;
9064	/*
9065	* Only the min performance hint (i.e. uclamp_min) doesn't fit.
9066	* Look for the CPU with best capacity.
9067	*/
9068	else if (fits < 0)
9069	cpu_cap = get_actual_cpu_capacity(cpu);
9070	/*
9071	* fits > 0 implies we are not on a preferred core, but the util
9072	* fits CPU capacity. Set fits to ASYM_IDLE_THREAD_FITS
9073	* so the effective range becomes
9074	* [ASYM_IDLE_THREAD_FITS, ASYM_IDLE_THREAD_MISFIT], where:
9075	* ASYM_IDLE_THREAD_MISFIT - does not fit
9076	* ASYM_IDLE_THREAD_UCLAMP_MISFIT - fits with the exception of UCLAMP_MIN
9077	* ASYM_IDLE_THREAD_FITS - fits with the exception of preferred_core
9078	*/
9079	else if (fits > 0)
9080	fits = ASYM_IDLE_THREAD_FITS;
9081
9082	/*
9083	* If we are on a preferred core, translate the range of fits
9084	* of [ASYM_IDLE_THREAD_UCLAMP_MISFIT, ASYM_IDLE_THREAD_MISFIT] to
9085	* [ASYM_IDLE_UCLAMP_MISFIT, ASYM_IDLE_COMPLETE_MISFIT].
9086	* This ensures that an idle core is always given priority over
9087	* (partially) busy core.
9088	*
9089	* A fully fitting idle core would have returned early and hence
9090	* fits > 0 for preferred_core need not be dealt with.
9091	*/
9092	if (preferred_core)
9093	fits += ASYM_IDLE_CORE_BIAS;
9094
9095	/*
9096	* First, select CPU which fits better (lower is more preferred).
9097	* Then, select the one with best capacity at same level.
9098	*/
9099	if ((fits < best_fits) \|\|
9100	((fits == best_fits) && (cpu_cap > best_cap))) {
9101	best_cap = cpu_cap;
9102	best_cpu = cpu;
9103	best_fits = fits;
9104	}
9105	}
9106
9107	/*
9108	* A value in the [ASYM_IDLE_UCLAMP_MISFIT, ASYM_IDLE_COMPLETE_MISFIT]
9109	* range means the chosen CPU is in a fully idle SMT core. Values above
9110	* ASYM_IDLE_COMPLETE_MISFIT mean we never ranked such a CPU best.
9111	*
9112	* The asym-capacity wakeup path returns from select_idle_sibling()
9113	* after this function and never runs select_idle_cpu(), so the usual
9114	* select_idle_cpu() tail that clears idle cores must live here when the
9115	* idle-core preference did not win.
9116	*/
9117	if (has_idle_core && best_fits > ASYM_IDLE_COMPLETE_MISFIT)
9118	set_idle_cores(target, false);
9119
9120	return best_cpu;
9121	}
9122
9123	static inline bool asym_fits_cpu(unsigned long util,
9124	unsigned long util_min,
9125	unsigned long util_max,
9126	int cpu)
9127	{
9128	if (sched_asym_cpucap_active()) {
9129	/*
9130	* Return true only if the cpu fully fits the task requirements
9131	* which include the utilization and the performance hints.
9132	*
9133	* When SMT is active, also require that the core has no busy
9134	* siblings.
9135	*
9136	* Note: gating on is_core_idle() also makes the early-bailout
9137	* candidates in select_idle_sibling() (target, prev,
9138	* recent_used_cpu) idle-core-aware on ASYM+SMT, which the
9139	* NO_ASYM path does not do.
9140	*/
9141	return (!sched_smt_active() \|\| is_core_idle(cpu)) &&
9142	(util_fits_cpu(util, util_min, util_max, cpu) > 0);
9143	}
9144
9145	return true;
9146	}
9147
9148	/*
9149	* Try and locate an idle core/thread in the LLC cache domain.
9150	*/
9151	static int select_idle_sibling(struct task_struct *p, int prev, int target)
9152	{
9153	bool has_idle_core = false;
9154	struct sched_domain *sd;
9155	unsigned long task_util, util_min, util_max;
9156	int i, recent_used_cpu, prev_aff = -1;
9157
9158	/*
9159	* On asymmetric system, update task utilization because we will check
9160	* that the task fits with CPU's capacity.
9161	*/
9162	if (sched_asym_cpucap_active()) {
9163	sync_entity_load_avg(&p->se);
9164	task_util = task_util_est(p);
9165	util_min = uclamp_eff_value(p, UCLAMP_MIN);
9166	util_max = uclamp_eff_value(p, UCLAMP_MAX);
9167	}
9168
9169	/*
9170	* per-cpu select_rq_mask usage
9171	*/
9172	lockdep_assert_irqs_disabled();
9173
9174	if (choose_idle_cpu(target, p) &&
9175	asym_fits_cpu(task_util, util_min, util_max, target))
9176	goto select_smt_priority;
9177
9178	/*
9179	* If the previous CPU is cache affine and idle, don't be stupid:
9180	*/
9181	if (prev != target && cpus_share_cache(prev, target) &&
9182	choose_idle_cpu(prev, p) &&
9183	asym_fits_cpu(task_util, util_min, util_max, prev)) {
9184
9185	if (!static_branch_unlikely(&sched_cluster_active) \|\|
9186	cpus_share_resources(prev, target)) {
9187	target = prev;
9188	goto select_smt_priority;
9189	}
9190
9191	prev_aff = prev;
9192	}
9193
9194	/*
9195	* Allow a per-cpu kthread to stack with the wakee if the
9196	* kworker thread and the tasks previous CPUs are the same.
9197	* The assumption is that the wakee queued work for the
9198	* per-cpu kthread that is now complete and the wakeup is
9199	* essentially a sync wakeup. An obvious example of this
9200	* pattern is IO completions.
9201	*/
9202	if (is_per_cpu_kthread(current) &&
9203	in_task() &&
9204	prev == smp_processor_id() &&
9205	this_rq()->nr_running <= 1 &&
9206	asym_fits_cpu(task_util, util_min, util_max, prev)) {
9207	target = prev;
9208	goto select_smt_priority;
9209	}
9210
9211	/* Check a recently used CPU as a potential idle candidate: */
9212	recent_used_cpu = p->recent_used_cpu;
9213	p->recent_used_cpu = prev;
9214	if (recent_used_cpu != prev &&
9215	recent_used_cpu != target &&
9216	cpus_share_cache(recent_used_cpu, target) &&
9217	choose_idle_cpu(recent_used_cpu, p) &&
9218	cpumask_test_cpu(recent_used_cpu, p->cpus_ptr) &&
9219	asym_fits_cpu(task_util, util_min, util_max, recent_used_cpu)) {
9220
9221	if (!static_branch_unlikely(&sched_cluster_active) \|\|
9222	cpus_share_resources(recent_used_cpu, target)) {
9223	target = recent_used_cpu;
9224	goto select_smt_priority;
9225	}
9226
9227	} else {
9228	recent_used_cpu = -1;
9229	}
9230
9231	/*
9232	* For asymmetric CPU capacity systems, our domain of interest is
9233	* sd_asym_cpucapacity rather than sd_llc.
9234	*/
9235	if (sched_asym_cpucap_active()) {
9236	sd = rcu_dereference_all(per_cpu(sd_asym_cpucapacity, target));
9237	/*
9238	* On an asymmetric CPU capacity system where an exclusive
9239	* cpuset defines a symmetric island (i.e. one unique
9240	* capacity_orig value through the cpuset), the key will be set
9241	* but the CPUs within that cpuset will not have a domain with
9242	* SD_ASYM_CPUCAPACITY. These should follow the usual symmetric
9243	* capacity path.
9244	*/
9245	if (sd) {
9246	i = select_idle_capacity(p, sd, target);
9247	if ((unsigned int)i < nr_cpumask_bits) {
9248	target = i;
9249	goto select_smt_priority;
9250	}
9251	return target;
9252	}
9253	}
9254
9255	sd = rcu_dereference_all(per_cpu(sd_llc, target));
9256	if (!sd)
9257	return target;
9258
9259	if (sched_smt_active()) {
9260	has_idle_core = test_idle_cores(target);
9261
9262	if (!has_idle_core && cpus_share_cache(prev, target)) {
9263	i = select_idle_smt(p, sd, prev);
9264	if ((unsigned int)i < nr_cpumask_bits) {
9265	target = i;
9266	goto select_smt_priority;
9267	}
9268	}
9269	}
9270
9271	i = select_idle_cpu(p, sd, has_idle_core, target);
9272	if ((unsigned int)i < nr_cpumask_bits) {
9273	target = i;
9274	goto select_smt_priority;
9275	}
9276
9277	/*
9278	* For cluster machines which have lower sharing cache like L2 or
9279	* LLC Tag, we tend to find an idle CPU in the target's cluster
9280	* first. But prev_cpu or recent_used_cpu may also be a good candidate,
9281	* use them if possible when no idle CPU found in select_idle_cpu().
9282	*/
9283	if ((unsigned int)prev_aff < nr_cpumask_bits) {
9284	target = prev_aff;
9285	goto select_smt_priority;
9286	}
9287	if ((unsigned int)recent_used_cpu < nr_cpumask_bits) {
9288	target = recent_used_cpu;
9289	goto select_smt_priority;
9290	}
9291
9292	return target;
9293
9294	select_smt_priority:
9295	return select_idle_smt_cpu(p, target);
9296	}
9297
9298	/**
9299	* cpu_util() - Estimates the amount of CPU capacity used by CFS tasks.
9300	* @cpu: the CPU to get the utilization for
9301	* @p: task for which the CPU utilization should be predicted or NULL
9302	* @dst_cpu: CPU @p migrates to, -1 if @p moves from @cpu or @p == NULL
9303	* @boost: 1 to enable boosting, otherwise 0
9304	*
9305	* The unit of the return value must be the same as the one of CPU capacity
9306	* so that CPU utilization can be compared with CPU capacity.
9307	*
9308	* CPU utilization is the sum of running time of runnable tasks plus the
9309	* recent utilization of currently non-runnable tasks on that CPU.
9310	* It represents the amount of CPU capacity currently used by CFS tasks in
9311	* the range [0..max CPU capacity] with max CPU capacity being the CPU
9312	* capacity at f_max.
9313	*
9314	* The estimated CPU utilization is defined as the maximum between CPU
9315	* utilization and sum of the estimated utilization of the currently
9316	* runnable tasks on that CPU. It preserves a utilization "snapshot" of
9317	* previously-executed tasks, which helps better deduce how busy a CPU will
9318	* be when a long-sleeping task wakes up. The contribution to CPU utilization
9319	* of such a task would be significantly decayed at this point of time.
9320	*
9321	* Boosted CPU utilization is defined as max(CPU runnable, CPU utilization).
9322	* CPU contention for CFS tasks can be detected by CPU runnable > CPU
9323	* utilization. Boosting is implemented in cpu_util() so that internal
9324	* users (e.g. EAS) can use it next to external users (e.g. schedutil),
9325	* latter via cpu_util_cfs_boost().
9326	*
9327	* CPU utilization can be higher than the current CPU capacity
9328	* (f_curr/f_max * max CPU capacity) or even the max CPU capacity because
9329	* of rounding errors as well as task migrations or wakeups of new tasks.
9330	* CPU utilization has to be capped to fit into the [0..max CPU capacity]
9331	* range. Otherwise a group of CPUs (CPU0 util = 121% + CPU1 util = 80%)
9332	* could be seen as over-utilized even though CPU1 has 20% of spare CPU
9333	* capacity. CPU utilization is allowed to overshoot current CPU capacity
9334	* though since this is useful for predicting the CPU capacity required
9335	* after task migrations (scheduler-driven DVFS).
9336	*
9337	* Return: (Boosted) (estimated) utilization for the specified CPU.
9338	*/
9339	static unsigned long
9340	cpu_util(int cpu, struct task_struct *p, int dst_cpu, int boost)
9341	{
9342	bool add_task = p && task_cpu(p) != cpu && dst_cpu == cpu;
9343	bool sub_task = p && task_cpu(p) == cpu && dst_cpu != cpu;
9344	struct cfs_rq *cfs_rq = &cpu_rq(cpu)->cfs;
9345	unsigned long util = READ_ONCE(cfs_rq->avg.util_avg);
9346	unsigned long runnable;
9347
9348	/*
9349	* If @dst_cpu is -1 or @p migrates from @cpu to @dst_cpu remove its
9350	* contribution. If @p migrates from another CPU to @cpu add its
9351	* contribution. In all the other cases @cpu is not impacted by the
9352	* migration so its util_avg is already correct.
9353	*/
9354	if (add_task)
9355	util += task_util(p);
9356	else if (sub_task)
9357	lsub_positive(&util, task_util(p));
9358
9359	if (boost) {
9360	runnable = READ_ONCE(cfs_rq->avg.runnable_avg);
9361	if (add_task)
9362	runnable += READ_ONCE(p->se.avg.runnable_avg);
9363	else if (sub_task)
9364	lsub_positive(&runnable,
9365	READ_ONCE(p->se.avg.runnable_avg));
9366	util = max(util, runnable);
9367	}
9368
9369	if (sched_feat(UTIL_EST)) {
9370	unsigned long util_est;
9371
9372	util_est = READ_ONCE(cfs_rq->avg.util_est);
9373
9374	/*
9375	* During wake-up @p isn't enqueued yet and doesn't contribute
9376	* to any cpu_rq(cpu)->cfs.avg.util_est.
9377	* If @dst_cpu == @cpu add it to "simulate" cpu_util after @p
9378	* has been enqueued.
9379	*
9380	* During exec (@dst_cpu = -1) @p is enqueued and does
9381	* contribute to cpu_rq(cpu)->cfs.util_est.
9382	* Remove it to "simulate" cpu_util without @p's contribution.
9383	*
9384	* Despite the task_on_rq_queued(@p) check there is still a
9385	* small window for a possible race when an exec
9386	* select_task_rq_fair() races with LB's detach_task().
9387	*
9388	* detach_task()
9389	* deactivate_task()
9390	* p->on_rq = TASK_ON_RQ_MIGRATING;
9391	* -------------------------------- A
9392	* dequeue_task() \
9393	* dequeue_task_fair() + Race Time
9394	* util_est_dequeue() /
9395	* -------------------------------- B
9396	*
9397	* The additional check "current == p" is required to further
9398	* reduce the race window.
9399	*/
9400	if (dst_cpu == cpu)
9401	util_est += _task_util_est(p);
9402	else if (p && unlikely(task_on_rq_queued(p) \|\| current == p))
9403	lsub_positive(&util_est, _task_util_est(p));
9404
9405	util = max(util, util_est);
9406	}
9407
9408	return min(util, arch_scale_cpu_capacity(cpu));
9409	}
9410
9411	unsigned long cpu_util_cfs(int cpu)
9412	{
9413	return cpu_util(cpu, NULL, -1, 0);
9414	}
9415
9416	unsigned long cpu_util_cfs_boost(int cpu)
9417	{
9418	return cpu_util(cpu, NULL, -1, 1);
9419	}
9420
9421	/*
9422	* cpu_util_without: compute cpu utilization without any contributions from *p
9423	* @cpu: the CPU which utilization is requested
9424	* @p: the task which utilization should be discounted
9425	*
9426	* The utilization of a CPU is defined by the utilization of tasks currently
9427	* enqueued on that CPU as well as tasks which are currently sleeping after an
9428	* execution on that CPU.
9429	*
9430	* This method returns the utilization of the specified CPU by discounting the
9431	* utilization of the specified task, whenever the task is currently
9432	* contributing to the CPU utilization.
9433	*/
9434	static unsigned long cpu_util_without(int cpu, struct task_struct *p)
9435	{
9436	/* Task has no contribution or is new */
9437	if (cpu != task_cpu(p) \|\| !READ_ONCE(p->se.avg.last_update_time))
9438	p = NULL;
9439
9440	return cpu_util(cpu, p, -1, 0);
9441	}
9442
9443	/*
9444	* This function computes an effective utilization for the given CPU, to be
9445	* used for frequency selection given the linear relation: f = u * f_max.
9446	*
9447	* The scheduler tracks the following metrics:
9448	*
9449	* cpu_util_{cfs,rt,dl,irq}()
9450	* cpu_bw_dl()
9451	*
9452	* Where the cfs,rt and dl util numbers are tracked with the same metric and
9453	* synchronized windows and are thus directly comparable.
9454	*
9455	* The cfs,rt,dl utilization are the running times measured with rq->clock_task
9456	* which excludes things like IRQ and steal-time. These latter are then accrued
9457	* in the IRQ utilization.
9458	*
9459	* The DL bandwidth number OTOH is not a measured metric but a value computed
9460	* based on the task model parameters and gives the minimal utilization
9461	* required to meet deadlines.
9462	*/
9463	unsigned long effective_cpu_util(int cpu, unsigned long util_cfs,
9464	unsigned long *min,
9465	unsigned long *max)
9466	{
9467	unsigned long util, irq, scale;
9468	struct rq *rq = cpu_rq(cpu);
9469
9470	scale = arch_scale_cpu_capacity(cpu);
9471
9472	/*
9473	* Early check to see if IRQ/steal time saturates the CPU, can be
9474	* because of inaccuracies in how we track these -- see
9475	* update_irq_load_avg().
9476	*/
9477	irq = cpu_util_irq(rq);
9478	if (unlikely(irq >= scale)) {
9479	if (min)
9480	*min = scale;
9481	if (max)
9482	*max = scale;
9483	return scale;
9484	}
9485
9486	if (min) {
9487	/*
9488	* The minimum utilization returns the highest level between:
9489	* - the computed DL bandwidth needed with the IRQ pressure which
9490	* steals time to the deadline task.
9491	* - The minimum performance requirement for CFS and/or RT.
9492	*/
9493	*min = max(irq + cpu_bw_dl(rq), uclamp_rq_get(rq, UCLAMP_MIN));
9494
9495	/*
9496	* When an RT task is runnable and uclamp is not used, we must
9497	* ensure that the task will run at maximum compute capacity.
9498	*/
9499	if (!uclamp_is_used() && rt_rq_is_runnable(&rq->rt))
9500	min = max(min, scale);
9501	}
9502
9503	/*
9504	* Because the time spend on RT/DL tasks is visible as 'lost' time to
9505	* CFS tasks and we use the same metric to track the effective
9506	* utilization (PELT windows are synchronized) we can directly add them
9507	* to obtain the CPU's actual utilization.
9508	*/
9509	util = util_cfs + cpu_util_rt(rq);
9510	util += cpu_util_dl(rq);
9511
9512	/*
9513	* The maximum hint is a soft bandwidth requirement, which can be lower
9514	* than the actual utilization because of uclamp_max requirements.
9515	*/
9516	if (max)
9517	*max = min(scale, uclamp_rq_get(rq, UCLAMP_MAX));
9518
9519	if (util >= scale)
9520	return scale;
9521
9522	/*
9523	* There is still idle time; further improve the number by using the
9524	* IRQ metric. Because IRQ/steal time is hidden from the task clock we
9525	* need to scale the task numbers:
9526	*
9527	* max - irq
9528	* U' = irq + --------- * U
9529	* max
9530	*/
9531	util = scale_irq_capacity(util, irq, scale);
9532	util += irq;
9533
9534	return min(scale, util);
9535	}
9536
9537	unsigned long sched_cpu_util(int cpu)
9538	{
9539	return effective_cpu_util(cpu, cpu_util_cfs(cpu), NULL, NULL);
9540	}
9541
9542	/*
9543	* energy_env - Utilization landscape for energy estimation.
9544	* @task_busy_time: Utilization contribution by the task for which we test the
9545	* placement. Given by eenv_task_busy_time().
9546	* @pd_busy_time: Utilization of the whole perf domain without the task
9547	* contribution. Given by eenv_pd_busy_time().
9548	* @cpu_cap: Maximum CPU capacity for the perf domain.
9549	* @pd_cap: Entire perf domain capacity. (pd->nr_cpus * cpu_cap).
9550	*/
9551	struct energy_env {
9552	unsigned long task_busy_time;
9553	unsigned long pd_busy_time;
9554	unsigned long cpu_cap;
9555	unsigned long pd_cap;
9556	};
9557
9558	/*
9559	* Compute the task busy time for compute_energy(). This time cannot be
9560	* injected directly into effective_cpu_util() because of the IRQ scaling.
9561	* The latter only makes sense with the most recent CPUs where the task has
9562	* run.
9563	*/
9564	static inline void eenv_task_busy_time(struct energy_env *eenv,
9565	struct task_struct *p, int prev_cpu)
9566	{
9567	unsigned long busy_time, max_cap = arch_scale_cpu_capacity(prev_cpu);
9568	unsigned long irq = cpu_util_irq(cpu_rq(prev_cpu));
9569
9570	if (unlikely(irq >= max_cap))
9571	busy_time = max_cap;
9572	else
9573	busy_time = scale_irq_capacity(task_util_est(p), irq, max_cap);
9574
9575	eenv->task_busy_time = busy_time;
9576	}
9577
9578	/*
9579	* Compute the perf_domain (PD) busy time for compute_energy(). Based on the
9580	* utilization for each @pd_cpus, it however doesn't take into account
9581	* clamping since the ratio (utilization / cpu_capacity) is already enough to
9582	* scale the EM reported power consumption at the (eventually clamped)
9583	* cpu_capacity.
9584	*
9585	* The contribution of the task @p for which we want to estimate the
9586	* energy cost is removed (by cpu_util()) and must be calculated
9587	* separately (see eenv_task_busy_time). This ensures:
9588	*
9589	* - A stable PD utilization, no matter which CPU of that PD we want to place
9590	* the task on.
9591	*
9592	* - A fair comparison between CPUs as the task contribution (task_util())
9593	* will always be the same no matter which CPU utilization we rely on
9594	* (util_avg or util_est).
9595	*
9596	* Set @eenv busy time for the PD that spans @pd_cpus. This busy time can't
9597	* exceed @eenv->pd_cap.
9598	*/
9599	static inline void eenv_pd_busy_time(struct energy_env *eenv,
9600	struct cpumask *pd_cpus,
9601	struct task_struct *p)
9602	{
9603	unsigned long busy_time = 0;
9604	int cpu;
9605
9606	for_each_cpu(cpu, pd_cpus) {
9607	unsigned long util = cpu_util(cpu, p, -1, 0);
9608
9609	busy_time += effective_cpu_util(cpu, util, NULL, NULL);
9610	}
9611
9612	eenv->pd_busy_time = min(eenv->pd_cap, busy_time);
9613	}
9614
9615	/*
9616	* Compute the maximum utilization for compute_energy() when the task @p
9617	* is placed on the cpu @dst_cpu.
9618	*
9619	* Returns the maximum utilization among @eenv->cpus. This utilization can't
9620	* exceed @eenv->cpu_cap.
9621	*/
9622	static inline unsigned long
9623	eenv_pd_max_util(struct energy_env eenv, struct cpumask pd_cpus,
9624	struct task_struct *p, int dst_cpu)
9625	{
9626	unsigned long max_util = 0;
9627	int cpu;
9628
9629	for_each_cpu(cpu, pd_cpus) {
9630	struct task_struct *tsk = (cpu == dst_cpu) ? p : NULL;
9631	unsigned long util = cpu_util(cpu, p, dst_cpu, 1);
9632	unsigned long eff_util, min, max;
9633
9634	/*
9635	* Performance domain frequency: utilization clamping
9636	* must be considered since it affects the selection
9637	* of the performance domain frequency.
9638	* NOTE: in case RT tasks are running, by default the min
9639	* utilization can be max OPP.
9640	*/
9641	eff_util = effective_cpu_util(cpu, util, &min, &max);
9642
9643	/* Task's uclamp can modify min and max value */
9644	if (tsk && uclamp_is_used()) {
9645	min = max(min, uclamp_eff_value(p, UCLAMP_MIN));
9646
9647	/*
9648	* If there is no active max uclamp constraint,
9649	* directly use task's one, otherwise keep max.
9650	*/
9651	if (uclamp_rq_is_idle(cpu_rq(cpu)))
9652	max = uclamp_eff_value(p, UCLAMP_MAX);
9653	else
9654	max = max(max, uclamp_eff_value(p, UCLAMP_MAX));
9655	}
9656
9657	eff_util = sugov_effective_cpu_perf(cpu, eff_util, min, max);
9658	max_util = max(max_util, eff_util);
9659	}
9660
9661	return min(max_util, eenv->cpu_cap);
9662	}
9663
9664	/*
9665	* compute_energy(): Use the Energy Model to estimate the energy that @pd would
9666	* consume for a given utilization landscape @eenv. When @dst_cpu < 0, the task
9667	* contribution is ignored.
9668	*/
9669	static inline unsigned long
9670	compute_energy(struct energy_env eenv, struct perf_domain pd,
9671	struct cpumask pd_cpus, struct task_struct p, int dst_cpu)
9672	{
9673	unsigned long max_util = eenv_pd_max_util(eenv, pd_cpus, p, dst_cpu);
9674	unsigned long busy_time = eenv->pd_busy_time;
9675	unsigned long energy;
9676
9677	if (dst_cpu >= 0)
9678	busy_time = min(eenv->pd_cap, busy_time + eenv->task_busy_time);
9679
9680	energy = em_cpu_energy(pd->em_pd, max_util, busy_time, eenv->cpu_cap);
9681
9682	trace_sched_compute_energy_tp(p, dst_cpu, energy, max_util, busy_time);
9683
9684	return energy;
9685	}
9686
9687	/*
9688	* find_energy_efficient_cpu(): Find most energy-efficient target CPU for the
9689	* waking task. find_energy_efficient_cpu() looks for the CPU with maximum
9690	* spare capacity in each performance domain and uses it as a potential
9691	* candidate to execute the task. Then, it uses the Energy Model to figure
9692	* out which of the CPU candidates is the most energy-efficient.
9693	*
9694	* The rationale for this heuristic is as follows. In a performance domain,
9695	* all the most energy efficient CPU candidates (according to the Energy
9696	* Model) are those for which we'll request a low frequency. When there are
9697	* several CPUs for which the frequency request will be the same, we don't
9698	* have enough data to break the tie between them, because the Energy Model
9699	* only includes active power costs. With this model, if we assume that
9700	* frequency requests follow utilization (e.g. using schedutil), the CPU with
9701	* the maximum spare capacity in a performance domain is guaranteed to be among
9702	* the best candidates of the performance domain.
9703	*
9704	* In practice, it could be preferable from an energy standpoint to pack
9705	* small tasks on a CPU in order to let other CPUs go in deeper idle states,
9706	* but that could also hurt our chances to go cluster idle, and we have no
9707	* ways to tell with the current Energy Model if this is actually a good
9708	* idea or not. So, find_energy_efficient_cpu() basically favors
9709	* cluster-packing, and spreading inside a cluster. That should at least be
9710	* a good thing for latency, and this is consistent with the idea that most
9711	* of the energy savings of EAS come from the asymmetry of the system, and
9712	* not so much from breaking the tie between identical CPUs. That's also the
9713	* reason why EAS is enabled in the topology code only for systems where
9714	* SD_ASYM_CPUCAPACITY is set.
9715	*
9716	* NOTE: Forkees are not accepted in the energy-aware wake-up path because
9717	* they don't have any useful utilization data yet and it's not possible to
9718	* forecast their impact on energy consumption. Consequently, they will be
9719	* placed by sched_balance_find_dst_cpu() on the least loaded CPU, which might turn out
9720	* to be energy-inefficient in some use-cases. The alternative would be to
9721	* bias new tasks towards specific types of CPUs first, or to try to infer
9722	* their util_avg from the parent task, but those heuristics could hurt
9723	* other use-cases too. So, until someone finds a better way to solve this,
9724	* let's keep things simple by re-using the existing slow path.
9725	*/
9726	static int find_energy_efficient_cpu(struct task_struct *p, int prev_cpu)
9727	{
9728	struct cpumask *cpus = this_cpu_cpumask_var_ptr(select_rq_mask);
9729	unsigned long prev_delta = ULONG_MAX, best_delta = ULONG_MAX;
9730	unsigned long p_util_min = uclamp_is_used() ? uclamp_eff_value(p, UCLAMP_MIN) : 0;
9731	unsigned long p_util_max = uclamp_is_used() ? uclamp_eff_value(p, UCLAMP_MAX) : 1024;
9732	struct root_domain *rd = this_rq()->rd;
9733	int cpu, best_energy_cpu, target = -1;
9734	int prev_fits = -1, best_fits = -1;
9735	unsigned long best_actual_cap = 0;
9736	unsigned long prev_actual_cap = 0;
9737	struct sched_domain *sd;
9738	struct perf_domain *pd;
9739	struct energy_env eenv;
9740
9741	pd = rcu_dereference_all(rd->pd);
9742	if (!pd)
9743	return target;
9744
9745	/*
9746	* Energy-aware wake-up happens on the lowest sched_domain starting
9747	* from sd_asym_cpucapacity spanning over this_cpu and prev_cpu.
9748	*/
9749	sd = rcu_dereference_all(*this_cpu_ptr(&sd_asym_cpucapacity));
9750	while (sd && !cpumask_test_cpu(prev_cpu, sched_domain_span(sd)))
9751	sd = sd->parent;
9752	if (!sd)
9753	return target;
9754
9755	target = prev_cpu;
9756
9757	sync_entity_load_avg(&p->se);
9758	if (!task_util_est(p) && p_util_min == 0)
9759	return target;
9760
9761	eenv_task_busy_time(&eenv, p, prev_cpu);
9762
9763	for (; pd; pd = pd->next) {
9764	unsigned long util_min = p_util_min, util_max = p_util_max;
9765	unsigned long cpu_cap, cpu_actual_cap, util;
9766	long prev_spare_cap = -1, max_spare_cap = -1;
9767	unsigned long rq_util_min, rq_util_max;
9768	unsigned long cur_delta, base_energy;
9769	int max_spare_cap_cpu = -1;
9770	int fits, max_fits = -1;
9771
9772	if (!cpumask_and(cpus, perf_domain_span(pd), cpu_online_mask))
9773	continue;
9774
9775	/* Account external pressure for the energy estimation */
9776	cpu = cpumask_first(cpus);
9777	cpu_actual_cap = get_actual_cpu_capacity(cpu);
9778
9779	eenv.cpu_cap = cpu_actual_cap;
9780	eenv.pd_cap = 0;
9781
9782	for_each_cpu(cpu, cpus) {
9783	struct rq *rq = cpu_rq(cpu);
9784
9785	eenv.pd_cap += cpu_actual_cap;
9786
9787	if (!cpumask_test_cpu(cpu, sched_domain_span(sd)))
9788	continue;
9789
9790	if (!cpumask_test_cpu(cpu, p->cpus_ptr))
9791	continue;
9792
9793	util = cpu_util(cpu, p, cpu, 0);
9794	cpu_cap = capacity_of(cpu);
9795
9796	/*
9797	* Skip CPUs that cannot satisfy the capacity request.
9798	* IOW, placing the task there would make the CPU
9799	* overutilized. Take uclamp into account to see how
9800	* much capacity we can get out of the CPU; this is
9801	* aligned with sched_cpu_util().
9802	*/
9803	if (uclamp_is_used() && !uclamp_rq_is_idle(rq)) {
9804	/*
9805	* Open code uclamp_rq_util_with() except for
9806	* the clamp() part. I.e.: apply max aggregation
9807	* only. util_fits_cpu() logic requires to
9808	* operate on non clamped util but must use the
9809	* max-aggregated uclamp_{min, max}.
9810	*/
9811	rq_util_min = uclamp_rq_get(rq, UCLAMP_MIN);
9812	rq_util_max = uclamp_rq_get(rq, UCLAMP_MAX);
9813
9814	util_min = max(rq_util_min, p_util_min);
9815	util_max = max(rq_util_max, p_util_max);
9816	}
9817
9818	fits = util_fits_cpu(util, util_min, util_max, cpu);
9819	if (!fits)
9820	continue;
9821
9822	lsub_positive(&cpu_cap, util);
9823
9824	if (cpu == prev_cpu) {
9825	/* Always use prev_cpu as a candidate. */
9826	prev_spare_cap = cpu_cap;
9827	prev_fits = fits;
9828	} else if ((fits > max_fits) \|\|
9829	((fits == max_fits) && ((long)cpu_cap > max_spare_cap))) {
9830	/*
9831	* Find the CPU with the maximum spare capacity
9832	* among the remaining CPUs in the performance
9833	* domain.
9834	*/
9835	max_spare_cap = cpu_cap;
9836	max_spare_cap_cpu = cpu;
9837	max_fits = fits;
9838	}
9839	}
9840
9841	if (max_spare_cap_cpu < 0 && prev_spare_cap < 0)
9842	continue;
9843
9844	eenv_pd_busy_time(&eenv, cpus, p);
9845	/* Compute the 'base' energy of the pd, without @p */
9846	base_energy = compute_energy(&eenv, pd, cpus, p, -1);
9847
9848	/* Evaluate the energy impact of using prev_cpu. */
9849	if (prev_spare_cap > -1) {
9850	prev_delta = compute_energy(&eenv, pd, cpus, p,
9851	prev_cpu);
9852	/* CPU utilization has changed */
9853	if (prev_delta < base_energy)
9854	return target;
9855	prev_delta -= base_energy;
9856	prev_actual_cap = cpu_actual_cap;
9857	best_delta = min(best_delta, prev_delta);
9858	}
9859
9860	/* Evaluate the energy impact of using max_spare_cap_cpu. */
9861	if (max_spare_cap_cpu >= 0 && max_spare_cap > prev_spare_cap) {
9862	/* Current best energy cpu fits better */
9863	if (max_fits < best_fits)
9864	continue;
9865
9866	/*
9867	* Both don't fit performance hint (i.e. uclamp_min)
9868	* but best energy cpu has better capacity.
9869	*/
9870	if ((max_fits < 0) &&
9871	(cpu_actual_cap <= best_actual_cap))
9872	continue;
9873
9874	cur_delta = compute_energy(&eenv, pd, cpus, p,
9875	max_spare_cap_cpu);
9876	/* CPU utilization has changed */
9877	if (cur_delta < base_energy)
9878	return target;
9879	cur_delta -= base_energy;
9880
9881	/*
9882	* Both fit for the task but best energy cpu has lower
9883	* energy impact.
9884	*/
9885	if ((max_fits > 0) && (best_fits > 0) &&
9886	(cur_delta >= best_delta))
9887	continue;
9888
9889	best_delta = cur_delta;
9890	best_energy_cpu = max_spare_cap_cpu;
9891	best_fits = max_fits;
9892	best_actual_cap = cpu_actual_cap;
9893	}
9894	}
9895
9896	if ((best_fits > prev_fits) \|\|
9897	((best_fits > 0) && (best_delta < prev_delta)) \|\|
9898	((best_fits < 0) && (best_actual_cap > prev_actual_cap)))
9899	target = best_energy_cpu;
9900
9901	return target;
9902	}
9903
9904	/*
9905	* select_task_rq_fair: Select target runqueue for the waking task in domains
9906	* that have the relevant SD flag set. In practice, this is SD_BALANCE_WAKE,
9907	* SD_BALANCE_FORK, or SD_BALANCE_EXEC.
9908	*
9909	* Balances load by selecting the idlest CPU in the idlest group, or under
9910	* certain conditions an idle sibling CPU if the domain has SD_WAKE_AFFINE set.
9911	*
9912	* Returns the target CPU number.
9913	*/
9914	static int
9915	select_task_rq_fair(struct task_struct *p, int prev_cpu, int wake_flags)
9916	{
9917	int sync = (wake_flags & WF_SYNC) && !(current->flags & PF_EXITING);
9918	struct sched_domain tmp, sd = NULL;
9919	int cpu = smp_processor_id();
9920	int new_cpu = prev_cpu;
9921	int want_affine = 0;
9922	/* SD_flags and WF_flags share the first nibble */
9923	int sd_flag = wake_flags & 0xF;
9924
9925	/*
9926	* required for stable ->cpus_allowed
9927	*/
9928	lockdep_assert_held(&p->pi_lock);
9929	if (wake_flags & WF_TTWU) {
9930	record_wakee(p);
9931
9932	if ((wake_flags & WF_CURRENT_CPU) &&
9933	cpumask_test_cpu(cpu, p->cpus_ptr))
9934	return cpu;
9935
9936	if (!is_rd_overutilized(this_rq()->rd)) {
9937	new_cpu = find_energy_efficient_cpu(p, prev_cpu);
9938	if (new_cpu >= 0)
9939	return new_cpu;
9940	new_cpu = prev_cpu;
9941	}
9942
9943	want_affine = !wake_wide(p) && cpumask_test_cpu(cpu, p->cpus_ptr);
9944	}
9945
9946	for_each_domain(cpu, tmp) {
9947	/*
9948	* If both 'cpu' and 'prev_cpu' are part of this domain,
9949	* cpu is a valid SD_WAKE_AFFINE target.
9950	*/
9951	if (want_affine && (tmp->flags & SD_WAKE_AFFINE) &&
9952	cpumask_test_cpu(prev_cpu, sched_domain_span(tmp))) {
9953	if (cpu != prev_cpu)
9954	new_cpu = wake_affine(tmp, p, cpu, prev_cpu, sync);
9955
9956	sd = NULL; /* Prefer wake_affine over balance flags */
9957	break;
9958	}
9959
9960	/*
9961	* Usually only true for WF_EXEC and WF_FORK, as sched_domains
9962	* usually do not have SD_BALANCE_WAKE set. That means wakeup
9963	* will usually go to the fast path.
9964	*/
9965	if (tmp->flags & sd_flag)
9966	sd = tmp;
9967	else if (!want_affine)
9968	break;
9969	}
9970
9971	/* Slow path */
9972	if (unlikely(sd)) {
9973	new_cpu = sched_balance_find_dst_cpu(sd, p, cpu, prev_cpu, sd_flag);
9974	return select_idle_smt_cpu(p, new_cpu);
9975	}
9976
9977	/* Fast path */
9978	if (wake_flags & WF_TTWU)
9979	return select_idle_sibling(p, prev_cpu, new_cpu);
9980
9981	return new_cpu;
9982	}
9983
9984	/*
9985	* Called immediately before a task is migrated to a new CPU; task_cpu(p) and
9986	* cfs_rq_of(p) references at time of call are still valid and identify the
9987	* previous CPU. The caller guarantees p->pi_lock or task_rq(p)->lock is held.
9988	*/
9989	static void migrate_task_rq_fair(struct task_struct *p, int new_cpu)
9990	{
9991	struct sched_entity *se = &p->se;
9992
9993	if (!task_on_rq_migrating(p)) {
9994	remove_entity_load_avg(se);
9995
9996	/*
9997	* Here, the task's PELT values have been updated according to
9998	* the current rq's clock. But if that clock hasn't been
9999	* updated in a while, a substantial idle time will be missed,
10000	* leading to an inflation after wake-up on the new rq.
10001	*
10002	* Estimate the missing time from the cfs_rq last_update_time
10003	* and update sched_avg to improve the PELT continuity after
10004	* migration.
10005	*/
10006	migrate_se_pelt_lag(se);
10007	}
10008
10009	/* Tell new CPU we are migrated */
10010	se->avg.last_update_time = 0;
10011
10012	update_scan_period(p, new_cpu);
10013	}
10014
10015	static void task_dead_fair(struct task_struct *p)
10016	{
10017	struct sched_entity *se = &p->se;
10018	remove_entity_load_avg(se);
10019	}
10020
10021	/*
10022	* Set the max capacity the task is allowed to run at for misfit detection.
10023	*/
10024	static void set_task_max_allowed_capacity(struct task_struct *p)
10025	{
10026	struct asym_cap_data *entry;
10027
10028	if (!sched_asym_cpucap_active())
10029	return;
10030
10031	rcu_read_lock();
10032	list_for_each_entry_rcu(entry, &asym_cap_list, link) {
10033	cpumask_t *cpumask;
10034
10035	cpumask = cpu_capacity_span(entry);
10036	if (!cpumask_intersects(p->cpus_ptr, cpumask))
10037	continue;
10038
10039	p->max_allowed_capacity = entry->capacity;
10040	break;
10041	}
10042	rcu_read_unlock();
10043	}
10044
10045	static void set_cpus_allowed_fair(struct task_struct p, struct affinity_context ctx)
10046	{
10047	set_cpus_allowed_common(p, ctx);
10048	set_task_max_allowed_capacity(p);
10049	}
10050
10051	enum preempt_wakeup_action {
10052	PREEMPT_WAKEUP_NONE, /* No preemption. */
10053	PREEMPT_WAKEUP_SHORT, /* Ignore slice protection. */
10054	PREEMPT_WAKEUP_PICK, /* Let pick_eevdf() decide. */
10055	PREEMPT_WAKEUP_RESCHED, /* Force reschedule. */
10056	};
10057
10058	static inline bool set_preempt_buddy(struct cfs_rq cfs_rq, struct sched_entity pse)
10059	{
10060	/*
10061	* Keep existing buddy if the deadline is sooner than pse.
10062	* The older buddy may be cache cold and completely unrelated
10063	* to the current wakeup but that is unpredictable where as
10064	* obeying the deadline is more in line with EEVDF objectives.
10065	*/
10066	if (cfs_rq->next && entity_before(cfs_rq->next, pse))
10067	return false;
10068
10069	set_next_buddy(cfs_rq, pse);
10070	return true;
10071	}
10072
10073	static inline bool set_short_buddy(struct cfs_rq cfs_rq, struct sched_entity pse)
10074	{
10075	if (cfs_rq->next) {
10076	if (cfs_rq->next->slice < pse->slice)
10077	return false;
10078
10079	if (cfs_rq->next->slice == pse->slice &&
10080	entity_before(cfs_rq->next, pse))
10081	return false;
10082	}
10083
10084	set_next_buddy(cfs_rq, pse);
10085	return true;
10086	}
10087
10088	/*
10089	* WF_SYNC\|WF_TTWU indicates the waker expects to sleep but it is not
10090	* strictly enforced because the hint is either misunderstood or
10091	* multiple tasks must be woken up.
10092	*/
10093	static inline enum preempt_wakeup_action
10094	preempt_sync(struct rq *rq, int wake_flags,
10095	struct sched_entity pse, struct sched_entity se)
10096	{
10097	u64 threshold, delta;
10098
10099	/*
10100	* WF_SYNC without WF_TTWU is not expected so warn if it happens even
10101	* though it is likely harmless.
10102	*/
10103	WARN_ON_ONCE(!(wake_flags & WF_TTWU));
10104
10105	threshold = sysctl_sched_migration_cost;
10106	delta = rq_clock_task(rq) - se->exec_start;
10107	if ((s64)delta < 0)
10108	delta = 0;
10109
10110	/*
10111	* WF_RQ_SELECTED implies the tasks are stacking on a CPU when they
10112	* could run on other CPUs. Reduce the threshold before preemption is
10113	* allowed to an arbitrary lower value as it is more likely (but not
10114	* guaranteed) the waker requires the wakee to finish.
10115	*/
10116	if (wake_flags & WF_RQ_SELECTED)
10117	threshold >>= 2;
10118
10119	/*
10120	* As WF_SYNC is not strictly obeyed, allow some runtime for batch
10121	* wakeups to be issued.
10122	*/
10123	if (entity_before(pse, se) && delta >= threshold)
10124	return PREEMPT_WAKEUP_RESCHED;
10125
10126	return PREEMPT_WAKEUP_NONE;
10127	}
10128
10129	/*
10130	* Preempt the current task with a newly woken task if needed:
10131	*/
10132	static void wakeup_preempt_fair(struct rq rq, struct task_struct p, int wake_flags)
10133	{
10134	enum preempt_wakeup_action preempt_action = PREEMPT_WAKEUP_PICK;
10135	struct task_struct *donor = rq->donor;
10136	struct sched_entity nse, se = &donor->se, *pse = &p->se;
10137	struct cfs_rq *cfs_rq = &rq->cfs;
10138	int cse_is_idle, pse_is_idle;
10139
10140	/*
10141	* XXX Getting preempted by higher class, try and find idle CPU?
10142	*/
10143	if (p->sched_class != &fair_sched_class \|\|
10144	donor->sched_class != &fair_sched_class)
10145	return;
10146
10147	if (unlikely(se == pse))
10148	return;
10149
10150	/*
10151	* This is possible from callers such as attach_tasks(), in which we
10152	* unconditionally wakeup_preempt() after an enqueue (which may have
10153	* lead to a throttle). This both saves work and prevents false
10154	* next-buddy nomination below.
10155	*/
10156	if (task_is_throttled(p))
10157	return;
10158
10159	/*
10160	* We can come here with TIF_NEED_RESCHED already set from new task
10161	* wake up path.
10162	*
10163	* Note: this also catches the edge-case of curr being in a throttled
10164	* group (e.g. via set_curr_task), since update_curr() (in the
10165	* enqueue of curr) will have resulted in resched being set. This
10166	* prevents us from potentially nominating it as a false LAST_BUDDY
10167	* below.
10168	*/
10169	if (!sched_feat(PREEMPT_SHORT) && test_tsk_need_resched(rq->curr))
10170	return;
10171
10172	if (!sched_feat(WAKEUP_PREEMPTION))
10173	return;
10174
10175	WARN_ON_ONCE(!pse);
10176
10177	cse_is_idle = se_is_idle(se);
10178	pse_is_idle = se_is_idle(pse);
10179
10180	nse = se;
10181	/*
10182	* Preempt an idle entity in favor of a non-idle entity (and don't preempt
10183	* in the inverse case).
10184	*/
10185	if (cse_is_idle && !pse_is_idle)
10186	goto preempt;
10187
10188	update_curr_fair(rq);
10189
10190	if (cse_is_idle != pse_is_idle)
10191	goto update;
10192
10193	/*
10194	* BATCH and IDLE tasks do not preempt others.
10195	*/
10196	if (unlikely(!normal_policy(p->policy)))
10197	goto update;
10198
10199	/*
10200	* Do not preempt for tasks that are sched_delayed as it would violate
10201	* EEVDF to forcibly queue an ineligible task.
10202	*/
10203	if (pse->sched_delayed)
10204	goto update;
10205
10206	/*
10207	* If @p has a shorter slice than current and @p is eligible, override
10208	* current's slice protection in order to allow preemption.
10209	*/
10210	if (sched_feat(PREEMPT_SHORT) && (pse->slice < se->slice)) {
10211	preempt_action = PREEMPT_WAKEUP_SHORT;
10212	goto pick;
10213	}
10214
10215	/*
10216	* Ignore wakee preemption on WF_FORK as it is less likely that
10217	* there is shared data as exec often follow fork.
10218	*/
10219	if (wake_flags & WF_FORK)
10220	goto update;
10221
10222	/* Prefer picking wakee soon if appropriate. */
10223	if (sched_feat(NEXT_BUDDY) && set_preempt_buddy(cfs_rq, pse)) {
10224	/*
10225	* Decide whether to obey WF_SYNC hint for a new buddy. Old
10226	* buddies are ignored as they may not be relevant to the
10227	* waker and less likely to be cache hot.
10228	*/
10229	if (wake_flags & WF_SYNC)
10230	preempt_action = preempt_sync(rq, wake_flags, pse, se);
10231	}
10232
10233	switch (preempt_action) {
10234	case PREEMPT_WAKEUP_NONE:
10235	return;
10236	case PREEMPT_WAKEUP_RESCHED:
10237	goto preempt;
10238	case PREEMPT_WAKEUP_SHORT:
10239	fallthrough;
10240	case PREEMPT_WAKEUP_PICK:
10241	break;
10242	}
10243
10244	pick:
10245	if (cfs_rq->h_nr_queued) {
10246	nse = pick_next_entity(rq, preempt_action != PREEMPT_WAKEUP_SHORT);
10247	if (unlikely(!nse))
10248	goto pick;
10249
10250	/* If @p has become the most eligible task, force preemption */
10251	if (nse == pse)
10252	goto preempt;
10253	}
10254
10255	/*
10256	* If @p is eligible but not the next task to run then cancel protection
10257	* to prevent large scheduling latency
10258	*/
10259	if (preempt_action == PREEMPT_WAKEUP_SHORT && entity_eligible(cfs_rq, pse))
10260	goto preempt;
10261	update:
10262	if (sched_feat(RUN_TO_PARITY))
10263	update_protect_slice(cfs_rq, se);
10264
10265	return;
10266
10267	preempt:
10268	cancel_protect_slice(se);
10269
10270	if (preempt_action == PREEMPT_WAKEUP_SHORT)
10271	set_short_buddy(cfs_rq, pse);
10272
10273	resched_curr_lazy(rq);
10274	}
10275
10276	struct task_struct pick_task_fair(struct rq rq, struct rq_flags *rf)
10277	__must_hold(__rq_lockp(rq))
10278	{
10279	struct cfs_rq *cfs_rq = &rq->cfs;
10280	struct sched_entity *se;
10281	struct task_struct *p;
10282	int new_tasks;
10283
10284	again:
10285	if (!cfs_rq->h_nr_queued)
10286	goto idle;
10287
10288	/* Might not have done put_prev_entity() */
10289	if (cfs_rq->curr && cfs_rq->curr->on_rq)
10290	update_curr_eevdf(cfs_rq);
10291
10292	se = pick_next_entity(rq, true);
10293	if (!se)
10294	goto again;
10295
10296	p = task_of(se);
10297	return p;
10298
10299	idle:
10300	if (sched_core_enabled(rq))
10301	return NULL;
10302
10303	new_tasks = sched_balance_newidle(rq, rf);
10304	if (new_tasks < 0)
10305	return RETRY_TASK;
10306	if (new_tasks > 0)
10307	goto again;
10308	return NULL;
10309	}
10310
10311	static struct task_struct *
10312	fair_server_pick_task(struct sched_dl_entity dl_se, struct rq_flags rf)
10313	__must_hold(__rq_lockp(dl_se->rq))
10314	{
10315	return pick_task_fair(dl_se->rq, rf);
10316	}
10317
10318	void fair_server_init(struct rq *rq)
10319	{
10320	struct sched_dl_entity *dl_se = &rq->fair_server;
10321
10322	init_dl_entity(dl_se);
10323
10324	dl_server_init(dl_se, rq, fair_server_pick_task);
10325	}
10326
10327	/*
10328	* Account for a descheduled task:
10329	*/
10330	static void put_prev_task_fair(struct rq rq, struct task_struct prev, struct task_struct *next)
10331	{
10332	struct sched_entity *se = &prev->se;
10333	struct cfs_rq *cfs_rq = &rq->cfs;
10334	struct sched_entity *nse = NULL;
10335
10336	#ifdef CONFIG_FAIR_GROUP_SCHED
10337	if (next && next->sched_class == &fair_sched_class)
10338	nse = &next->se;
10339	#endif
10340
10341	while (se) {
10342	cfs_rq = cfs_rq_of(se);
10343	if (!nse \|\| cfs_rq->h_curr)
10344	put_prev_entity(cfs_rq, se);
10345	#ifdef CONFIG_FAIR_GROUP_SCHED
10346	if (nse) {
10347	if (is_same_group(se, nse))
10348	break;
10349
10350	int d = nse->depth - se->depth;
10351	if (d >= 0) {
10352	/* nse has equal or greater depth, ascend */
10353	nse = parent_entity(nse);
10354	/* if nse is the deeper, do not ascend se */
10355	if (d > 0)
10356	continue;
10357	}
10358	}
10359	#endif
10360	se = parent_entity(se);
10361	}
10362
10363	/* Put 'current' back into the tree. */
10364	cfs_rq = &rq->cfs;
10365	se = &prev->se;
10366	WARN_ON_ONCE(cfs_rq->curr != se);
10367	cfs_rq->curr = NULL;
10368	if (se->on_rq)
10369	__enqueue_entity(cfs_rq, se);
10370	}
10371
10372	/*
10373	* sched_yield() is very simple
10374	*/
10375	static void yield_task_fair(struct rq *rq)
10376	{
10377	struct task_struct *curr = rq->donor;
10378	struct sched_entity *se = &curr->se;
10379	struct cfs_rq *cfs_rq = &rq->cfs;
10380
10381	/*
10382	* Are we the only task in the tree?
10383	*/
10384	if (unlikely(rq->nr_running == 1))
10385	return;
10386
10387	clear_buddies(cfs_rq, se);
10388
10389	update_rq_clock(rq);
10390	/*
10391	* Update run-time statistics of the 'current'.
10392	*/
10393	update_curr_eevdf(cfs_rq);
10394	/*
10395	* Tell update_rq_clock() that we've just updated,
10396	* so we don't do microscopic update in schedule()
10397	* and double the fastpath cost.
10398	*/
10399	rq_clock_skip_update(rq);
10400
10401	/*
10402	* Forfeit the remaining vruntime, only if the entity is eligible. This
10403	* condition is necessary because in core scheduling we prefer to run
10404	* ineligible tasks rather than force idling. If this happens we may
10405	* end up in a loop where the core scheduler picks the yielding task,
10406	* which yields immediately again; without the condition the vruntime
10407	* ends up quickly running away.
10408	*/
10409	if (entity_eligible(cfs_rq, se)) {
10410	se->vruntime = se->deadline;
10411	update_deadline(cfs_rq, se);
10412	}
10413	}
10414
10415	static bool yield_to_task_fair(struct rq rq, struct task_struct p)
10416	{
10417	struct sched_entity *se = &p->se;
10418
10419	/* !se->on_rq also covers throttled task */
10420	if (!se->on_rq \|\| se->sched_delayed)
10421	return false;
10422
10423	/* Tell the scheduler that we'd really like se to run next. */
10424	set_next_buddy(&task_rq(p)->cfs, se);
10425
10426	yield_task_fair(rq);
10427
10428	return true;
10429	}
10430
10431	/**************************************************
10432	* Fair scheduling class load-balancing methods.
10433	*
10434	* BASICS
10435	*
10436	* The purpose of load-balancing is to achieve the same basic fairness the
10437	* per-CPU scheduler provides, namely provide a proportional amount of compute
10438	* time to each task. This is expressed in the following equation:
10439	*
10440	* W_i,n/P_i == W_j,n/P_j for all i,j (1)
10441	*
10442	* Where W_i,n is the n-th weight average for CPU i. The instantaneous weight
10443	* W_i,0 is defined as:
10444	*
10445	* W_i,0 = \Sum_j w_i,j (2)
10446	*
10447	* Where w_i,j is the weight of the j-th runnable task on CPU i. This weight
10448	* is derived from the nice value as per sched_prio_to_weight[].
10449	*
10450	* The weight average is an exponential decay average of the instantaneous
10451	* weight:
10452	*
10453	* W'_i,n = (2^n - 1) / 2^n * W_i,n + 1 / 2^n * W_i,0 (3)
10454	*
10455	* C_i is the compute capacity of CPU i, typically it is the
10456	* fraction of 'recent' time available for SCHED_OTHER task execution. But it
10457	* can also include other factors [XXX].
10458	*
10459	* To achieve this balance we define a measure of imbalance which follows
10460	* directly from (1):
10461	*
10462	* imb_i,j = max{ avg(W/C), W_i/C_i } - min{ avg(W/C), W_j/C_j } (4)
10463	*
10464	* We them move tasks around to minimize the imbalance. In the continuous
10465	* function space it is obvious this converges, in the discrete case we get
10466	* a few fun cases generally called infeasible weight scenarios.
10467	*
10468	* [XXX expand on:
10469	* - infeasible weights;
10470	* - local vs global optima in the discrete case. ]
10471	*
10472	*
10473	* SCHED DOMAINS
10474	*
10475	* In order to solve the imbalance equation (4), and avoid the obvious O(n^2)
10476	* for all i,j solution, we create a tree of CPUs that follows the hardware
10477	* topology where each level pairs two lower groups (or better). This results
10478	* in O(log n) layers. Furthermore we reduce the number of CPUs going up the
10479	* tree to only the first of the previous level and we decrease the frequency
10480	* of load-balance at each level inversely proportional to the number of CPUs in
10481	* the groups.
10482	*
10483	* This yields:
10484	*
10485	* log_2 n 1 n
10486	* \Sum { --- * --- * 2^i } = O(n) (5)
10487	* i = 0 2^i 2^i
10488	* `- size of each group
10489	* \| \| `- number of CPUs doing load-balance
10490	* \| `- freq
10491	* `- sum over all levels
10492	*
10493	* Coupled with a limit on how many tasks we can migrate every balance pass,
10494	* this makes (5) the runtime complexity of the balancer.
10495	*
10496	* An important property here is that each CPU is still (indirectly) connected
10497	* to every other CPU in at most O(log n) steps:
10498	*
10499	* The adjacency matrix of the resulting graph is given by:
10500	*
10501	* log_2 n
10502	* A_i,j = \Union (i % 2^k == 0) && i / 2^(k+1) == j / 2^(k+1) (6)
10503	* k = 0
10504	*
10505	* And you'll find that:
10506	*
10507	* A^(log_2 n)_i,j != 0 for all i,j (7)
10508	*
10509	* Showing there's indeed a path between every CPU in at most O(log n) steps.
10510	* The task movement gives a factor of O(m), giving a convergence complexity
10511	* of:
10512	*
10513	* O(nm log n), n := nr_cpus, m := nr_tasks (8)
10514	*
10515	*
10516	* WORK CONSERVING
10517	*
10518	* In order to avoid CPUs going idle while there's still work to do, new idle
10519	* balancing is more aggressive and has the newly idle CPU iterate up the domain
10520	* tree itself instead of relying on other CPUs to bring it work.
10521	*
10522	* This adds some complexity to both (5) and (8) but it reduces the total idle
10523	* time.
10524	*
10525	* [XXX more?]
10526	*
10527	*
10528	* CGROUPS
10529	*
10530	* Cgroups make a horror show out of (2), instead of a simple sum we get:
10531	*
10532	* s_k,i
10533	* W_i,0 = \Sum_j \Prod_k w_k * ----- (9)
10534	* S_k
10535	*
10536	* Where
10537	*
10538	* s_k,i = \Sum_j w_i,j,k and S_k = \Sum_i s_k,i (10)
10539	*
10540	* w_i,j,k is the weight of the j-th runnable task in the k-th cgroup on CPU i.
10541	*
10542	* The big problem is S_k, its a global sum needed to compute a local (W_i)
10543	* property.
10544	*
10545	* [XXX write more on how we solve this.. _after_ merging pjt's patches that
10546	* rewrite all of this once again.]
10547	*/
10548
10549	static unsigned long __read_mostly max_load_balance_interval = HZ/10;
10550
10551	enum fbq_type { regular, remote, all };
10552
10553	/*
10554	* 'group_type' describes the group of CPUs at the moment of load balancing.
10555	*
10556	* The enum is ordered by pulling priority, with the group with lowest priority
10557	* first so the group_type can simply be compared when selecting the busiest
10558	* group. See update_sd_pick_busiest().
10559	*/
10560	enum group_type {
10561	/* The group has spare capacity that can be used to run more tasks. */
10562	group_has_spare = 0,
10563	/*
10564	* The group is fully used and the tasks don't compete for more CPU
10565	* cycles. Nevertheless, some tasks might wait before running.
10566	*/
10567	group_fully_busy,
10568	/*
10569	* One task doesn't fit with CPU's capacity and must be migrated to a
10570	* more powerful CPU.
10571	*/
10572	group_misfit_task,
10573	/*
10574	* Balance SMT group that's fully busy. Can benefit from migration
10575	* a task on SMT with busy sibling to another CPU on idle core.
10576	*/
10577	group_smt_balance,
10578	/*
10579	* SD_ASYM_PACKING only: One local CPU with higher capacity is available,
10580	* and the task should be migrated to it instead of running on the
10581	* current CPU.
10582	*/
10583	group_asym_packing,
10584	/*
10585	* The tasks' affinity constraints previously prevented the scheduler
10586	* from balancing the load across the system.
10587	*/
10588	group_imbalanced,
10589	/*
10590	* There are tasks running on non-preferred LLC, possible to move
10591	* them to their preferred LLC without creating too much imbalance.
10592	* The priority of group_llc_balance is lower than that of
10593	* group_overloaded and higher than that of all other group types.
10594	* This is because group_llc_balance may exacerbate load imbalance.
10595	* If the LLC balancing attempt fails, the nr_balance_failed
10596	* mechanism will trigger other group types to rebalance the load.
10597	*/
10598	group_llc_balance,
10599	/*
10600	* The CPU is overloaded and can't provide expected CPU cycles to all
10601	* tasks.
10602	*/
10603	group_overloaded
10604	};
10605
10606	enum migration_type {
10607	migrate_load = 0,
10608	migrate_util,
10609	migrate_task,
10610	migrate_misfit,
10611	migrate_llc_task
10612	};
10613
10614	#define LBF_ALL_PINNED 0x01
10615	#define LBF_NEED_BREAK 0x02
10616	#define LBF_DST_PINNED 0x04
10617	#define LBF_SOME_PINNED 0x08
10618	#define LBF_ACTIVE_LB 0x10
10619	#define LBF_LLC_PINNED 0x20
10620	#define LBF_ACTIVE_LB_LLC 0x40
10621
10622	struct lb_env {
10623	struct sched_domain *sd;
10624
10625	struct rq *src_rq;
10626	int src_cpu;
10627
10628	int dst_cpu;
10629	struct rq *dst_rq;
10630	bool dst_core_idle;
10631
10632	struct cpumask *dst_grpmask;
10633	int new_dst_cpu;
10634	enum cpu_idle_type idle;
10635	long imbalance;
10636	/* The set of CPUs under consideration for load-balancing */
10637	struct cpumask *cpus;
10638
10639	unsigned int flags;
10640
10641	unsigned int loop;
10642	unsigned int loop_break;
10643	unsigned int loop_max;
10644
10645	enum fbq_type fbq_type;
10646	enum migration_type migration_type;
10647	struct list_head tasks;
10648	};
10649
10650	/*
10651	* Is this task likely cache-hot:
10652	*/
10653	static int task_hot(struct task_struct p, struct lb_env env)
10654	{
10655	s64 delta;
10656
10657	lockdep_assert_rq_held(env->src_rq);
10658
10659	if (p->sched_class != &fair_sched_class)
10660	return 0;
10661
10662	if (unlikely(task_has_idle_policy(p)))
10663	return 0;
10664
10665	/* SMT siblings share cache */
10666	if (env->sd->flags & SD_SHARE_CPUCAPACITY)
10667	return 0;
10668
10669	/*
10670	* Buddy candidates are cache hot:
10671	*/
10672	if (sched_feat(CACHE_HOT_BUDDY) && env->dst_rq->nr_running &&
10673	(&p->se == cfs_rq_of(&p->se)->next))
10674	return 1;
10675
10676	if (sysctl_sched_migration_cost == -1)
10677	return 1;
10678
10679	/*
10680	* Don't migrate task if the task's cookie does not match
10681	* with the destination CPU's core cookie.
10682	*/
10683	if (!sched_core_cookie_match(cpu_rq(env->dst_cpu), p))
10684	return 1;
10685
10686	if (sysctl_sched_migration_cost == 0)
10687	return 0;
10688
10689	delta = rq_clock_task(env->src_rq) - p->se.exec_start;
10690
10691	return delta < (s64)sysctl_sched_migration_cost;
10692	}
10693
10694	#ifdef CONFIG_NUMA_BALANCING
10695	/*
10696	* Returns a positive value, if task migration degrades locality.
10697	* Returns 0, if task migration is not affected by locality.
10698	* Returns a negative value, if task migration improves locality i.e migration preferred.
10699	*/
10700	static long migrate_degrades_locality(struct task_struct p, struct lb_env env)
10701	{
10702	struct numa_group *numa_group = rcu_dereference_all(p->numa_group);
10703	unsigned long src_weight, dst_weight;
10704	int src_nid, dst_nid, dist;
10705
10706	if (!static_branch_likely(&sched_numa_balancing))
10707	return 0;
10708
10709	if (!p->numa_faults \|\| !(env->sd->flags & SD_NUMA))
10710	return 0;
10711
10712	src_nid = cpu_to_node(env->src_cpu);
10713	dst_nid = cpu_to_node(env->dst_cpu);
10714
10715	if (src_nid == dst_nid)
10716	return 0;
10717
10718	/* Migrating away from the preferred node is always bad. */
10719	if (src_nid == p->numa_preferred_nid) {
10720	if (env->src_rq->nr_running > env->src_rq->nr_preferred_running)
10721	return 1;
10722	else
10723	return 0;
10724	}
10725
10726	/* Encourage migration to the preferred node. */
10727	if (dst_nid == p->numa_preferred_nid)
10728	return -1;
10729
10730	/* Leaving a core idle is often worse than degrading locality. */
10731	if (env->idle == CPU_IDLE)
10732	return 0;
10733
10734	dist = node_distance(src_nid, dst_nid);
10735	if (numa_group) {
10736	src_weight = group_weight(p, src_nid, dist);
10737	dst_weight = group_weight(p, dst_nid, dist);
10738	} else {
10739	src_weight = task_weight(p, src_nid, dist);
10740	dst_weight = task_weight(p, dst_nid, dist);
10741	}
10742
10743	return src_weight - dst_weight;
10744	}
10745
10746	#else /* !CONFIG_NUMA_BALANCING: */
10747	static inline long migrate_degrades_locality(struct task_struct *p,
10748	struct lb_env *env)
10749	{
10750	return 0;
10751	}
10752	#endif /* !CONFIG_NUMA_BALANCING */
10753
10754	/*
10755	* Check whether the task is ineligible on the destination cpu
10756	*
10757	* When the PLACE_LAG scheduling feature is enabled and
10758	* dst_cfs_rq->nr_queued is greater than 1, if the task
10759	* is ineligible, it will also be ineligible when
10760	* it is migrated to the destination cpu.
10761	*/
10762	static inline int task_is_ineligible_on_dst_cpu(struct task_struct *p, int dest_cpu)
10763	{
10764	struct cfs_rq *dst_cfs_rq = &cpu_rq(dest_cpu)->cfs;
10765
10766	if (sched_feat(PLACE_LAG) && dst_cfs_rq->h_nr_queued &&
10767	!entity_eligible(&task_rq(p)->cfs, &p->se))
10768	return 1;
10769
10770	return 0;
10771	}
10772
10773	#ifdef CONFIG_SCHED_CACHE
10774	/*
10775	* The margin used when comparing LLC utilization with CPU capacity.
10776	* It determines the LLC load level where active LLC aggregation is
10777	* done.
10778	* Derived from fits_capacity().
10779	*
10780	* (default: ~50%, tunable via debugfs)
10781	*/
10782	static bool fits_llc_capacity(unsigned long util, unsigned long max)
10783	{
10784	u32 aggr_pct = llc_overaggr_pct;
10785
10786	/*
10787	* For single core systems, raise the aggregation
10788	* threshold to accommodate more tasks.
10789	*/
10790	if (cpu_smt_num_threads == 1)
10791	aggr_pct = (aggr_pct * 3 / 2);
10792
10793	return util * 100 < max * aggr_pct;
10794	}
10795
10796	/*
10797	* The margin used when comparing utilization.
10798	* is 'util1' noticeably greater than 'util2'
10799	* Derived from capacity_greater().
10800	* Bias is in perentage.
10801	*/
10802	/* Allows dst util to be bigger than src util by up to bias percent */
10803	#define util_greater(util1, util2) \
10804	((util1) * 100 > (util2) * (100 + llc_imb_pct))
10805
10806	static __maybe_unused bool get_llc_stats(int cpu, unsigned long *util,
10807	unsigned long *cap)
10808	{
10809	struct sched_domain_shared *sd_share;
10810
10811	sd_share = rcu_dereference_all(per_cpu(sd_llc_shared, cpu));
10812	if (!sd_share)
10813	return false;
10814
10815	*util = READ_ONCE(sd_share->util_avg);
10816	*cap = READ_ONCE(sd_share->capacity);
10817
10818	return true;
10819	}
10820
10821	/*
10822	* Decision matrix according to the LLC utilization. To
10823	* decide whether we can do task aggregation across LLC.
10824	*
10825	* By default, 50% is the threshold for treating the LLC
10826	* as busy. The reason for choosing 50% is to avoid saturation
10827	* of SMT-2, and it is also a safe cutoff for other SMT-n
10828	* platforms. SMT-1 has higher threshold because it is
10829	* supposed to accommodate more tasks, see fits_llc_capacity().
10830	*
10831	* 20% is the utilization imbalance percentage to decide
10832	* if the preferred LLC is busier than the non-preferred LLC.
10833	* 20 is a little higher than the LLC domain's imbalance_pct
10834	* 17. The hysteresis is used to avoid task bouncing between the
10835	* preferred LLC and the non-preferred LLC, and it will
10836	* be turned into tunable debugfs.
10837	*
10838	* 1. moving towards the preferred LLC, dst is the preferred
10839	* LLC, src is not.
10840	*
10841	* src \ dst 30% 40% 50% 60%
10842	* 30% Y Y Y N
10843	* 40% Y Y Y Y
10844	* 50% Y Y G G
10845	* 60% Y Y G G
10846	*
10847	* 2. moving out of the preferred LLC, src is the preferred
10848	* LLC, dst is not:
10849	*
10850	* src \ dst 30% 40% 50% 60%
10851	* 30% N N N N
10852	* 40% N N N N
10853	* 50% N N G G
10854	* 60% Y N G G
10855	*
10856	* src : src_util
10857	* dst : dst_util
10858	* Y : Yes, migrate
10859	* N : No, do not migrate
10860	* G : let the Generic load balance to even the load.
10861	*
10862	* The intention is that if both LLCs are quite busy, cache aware
10863	* load balance should not be performed, and generic load balance
10864	* should take effect. However, if one is busy and the other is not,
10865	* the preferred LLC capacity(50%) and imbalance criteria(20%) should
10866	* be considered to determine whether LLC aggregation should be
10867	* performed to bias the load towards the preferred LLC.
10868	*/
10869
10870	/* migration decision, 3 states are orthogonal. */
10871	enum llc_mig {
10872	mig_forbid = 0, /* N: Don't migrate task, respect LLC preference */
10873	mig_llc, /* Y: Do LLC preference based migration */
10874	mig_unrestricted /* G: Don't restrict generic load balance migration */
10875	};
10876
10877	/*
10878	* Check if task can be moved from the source LLC to the
10879	* destination LLC without breaking cache aware preferrence.
10880	* src_cpu and dst_cpu are arbitrary CPUs within the source
10881	* and destination LLCs, respectively.
10882	*/
10883	static enum llc_mig can_migrate_llc(int src_cpu, int dst_cpu,
10884	unsigned long tsk_util,
10885	bool to_pref)
10886	{
10887	unsigned long src_util, dst_util, src_cap, dst_cap;
10888
10889	if (!get_llc_stats(src_cpu, &src_util, &src_cap) \|\|
10890	!get_llc_stats(dst_cpu, &dst_util, &dst_cap))
10891	return mig_unrestricted;
10892
10893	src_util = src_util < tsk_util ? 0 : src_util - tsk_util;
10894	dst_util = dst_util + tsk_util;
10895
10896	if (!fits_llc_capacity(dst_util, dst_cap) &&
10897	!fits_llc_capacity(src_util, src_cap))
10898	return mig_unrestricted;
10899
10900	if (to_pref) {
10901	/*
10902	* Don't migrate if we will get preferred LLC too
10903	* heavily loaded and if the dest is much busier
10904	* than the src, in which case migration will
10905	* increase the imbalance too much.
10906	*/
10907	if (!fits_llc_capacity(dst_util, dst_cap) &&
10908	util_greater(dst_util, src_util))
10909	return mig_forbid;
10910	} else {
10911	/*
10912	* Don't migrate if we will leave preferred LLC
10913	* too idle, or if this migration leads to the
10914	* non-preferred LLC falls within sysctl_aggr_imb percent
10915	* of preferred LLC, leading to migration again
10916	* back to preferred LLC.
10917	*/
10918	if (fits_llc_capacity(src_util, src_cap) \|\|
10919	!util_greater(src_util, dst_util))
10920	return mig_forbid;
10921	}
10922	return mig_llc;
10923	}
10924
10925	static inline bool task_misfits_asym_cpu(struct lb_env env, struct task_struct p)
10926	{
10927	/*
10928	* On asymmetric CPU capacity domains, do not let cache-aware
10929	* balancing pull the task onto a destination CPU that cannot
10930	* accommodate it. Doing so would turn the task into a misfit on
10931	* the destination, trading a cache-locality gain for a capacity
10932	* loss. If the task already does not fit its source CPU, the move
10933	* cannot make things worse, so let the LLC preference decide.
10934	*/
10935	if ((env->sd->flags & SD_ASYM_CPUCAPACITY) && p &&
10936	!task_fits_cpu(p, env->dst_cpu) &&
10937	task_fits_cpu(p, env->src_cpu))
10938	return true;
10939
10940	return false;
10941	}
10942
10943	/*
10944	* Check if task p can migrate from source LLC to
10945	* destination LLC in terms of cache aware load balance.
10946	*/
10947	static enum llc_mig can_migrate_llc_task(struct lb_env *env,
10948	struct task_struct *p)
10949	{
10950	struct sched_cache_group *grp;
10951	bool to_pref;
10952	int cpu, src_cpu, dst_cpu;
10953
10954	if (task_misfits_asym_cpu(env, p))
10955	return mig_forbid;
10956
10957	src_cpu = env->src_cpu;
10958	dst_cpu = env->dst_cpu;
10959	grp = rcu_dereference_all(p->sched_cache_grp);
10960	if (!grp)
10961	return mig_unrestricted;
10962
10963	cpu = READ_ONCE(grp->cpu);
10964	if (cpu < 0 \|\| cpus_share_cache(src_cpu, dst_cpu))
10965	return mig_unrestricted;
10966
10967	/* skip cache aware load balance for too many threads */
10968	if (invalid_llc_nr(grp, p, dst_cpu) \|\|
10969	exceed_llc_capacity(grp, dst_cpu)) {
10970	if (READ_ONCE(grp->cpu) != -1)
10971	WRITE_ONCE(grp->cpu, -1);
10972	return mig_unrestricted;
10973	}
10974
10975	if (cpus_share_cache(dst_cpu, cpu))
10976	to_pref = true;
10977	else if (cpus_share_cache(src_cpu, cpu))
10978	to_pref = false;
10979	else
10980	return mig_unrestricted;
10981
10982	return can_migrate_llc(src_cpu, dst_cpu,
10983	task_util(p), to_pref);
10984	}
10985
10986	/*
10987	* Check if active load balance breaks LLC locality in
10988	* terms of cache aware load balance. The load level and
10989	* imbalance do not warrant breaking LLC preference per
10990	* the can_migrate_llc() policy. Here, the benefit of
10991	* LLC locality outweighs the power efficiency gained from
10992	* migrating the only runnable task away.
10993	*/
10994	static inline bool
10995	alb_break_llc(struct lb_env *env)
10996	{
10997	if (!sched_cache_enabled())
10998	return false;
10999
11000	if (cpus_share_cache(env->src_cpu, env->dst_cpu))
11001	return false;
11002	/*
11003	* All tasks prefer to stay on their current CPU.
11004	* Do not pull a task from its preferred CPU if:
11005	* 1. It is the only task running and does not exceed
11006	* imbalance allowance; OR
11007	* 2. Migrating it away from its preferred LLC would violate
11008	* the cache-aware scheduling policy.
11009	*/
11010	if (env->src_rq->nr_pref_llc_running &&
11011	env->src_rq->nr_pref_llc_running == env->src_rq->cfs.h_nr_runnable) {
11012	unsigned long util = 0;
11013	struct task_struct *cur;
11014
11015	/*
11016	* Migrating misfit tasks from current CPU
11017	* to CPU with a better fit.
11018	* Prioritize that over LLC preference.
11019	*/
11020	if (env->migration_type == migrate_misfit)
11021	return false;
11022
11023	if (env->src_rq->nr_running <= 1)
11024	return true;
11025
11026	cur = rcu_dereference_all(env->src_rq->curr);
11027	if (cur && cur->sched_class == &fair_sched_class)
11028	util = task_util(cur);
11029
11030	if (task_misfits_asym_cpu(env, cur) \|\|
11031	can_migrate_llc(env->src_cpu, env->dst_cpu,
11032	util, false) == mig_forbid)
11033	return true;
11034	}
11035
11036	return false;
11037	}
11038
11039	/*
11040	* Returns true if p's preferred LLC does not match the destination CPU
11041	* under migrate_llc_task semantics. Passive LB passes migrate_llc_task
11042	* in env->migration_type, while active LB carries LBF_ACTIVE_LB_LLC in
11043	* env->flags to avoid overwriting env->migration_type.
11044	*/
11045	static inline bool
11046	migrate_llc_task_wrong_dst(struct task_struct p, struct lb_env env)
11047	{
11048	return sched_cache_enabled() &&
11049	(env->migration_type == migrate_llc_task \|\|
11050	env->flags & LBF_ACTIVE_LB_LLC) &&
11051	READ_ONCE(p->preferred_llc) != llc_id(env->dst_cpu);
11052	}
11053
11054	/*
11055	* Check if migrating task p from env->src_cpu to
11056	* env->dst_cpu breaks LLC localiy.
11057	*/
11058	static bool migrate_degrades_llc(struct task_struct p, struct lb_env env)
11059	{
11060	if (!sched_cache_enabled())
11061	return false;
11062
11063	if (task_has_sched_core(p))
11064	return false;
11065	/*
11066	* Skip over tasks that would degrade LLC locality;
11067	* only when nr_balanced_failed is sufficiently high do we
11068	* ignore this constraint.
11069	*
11070	* Threshold of cache_nice_tries is set to 1 higher
11071	* than nr_balance_failed to avoid excessive task
11072	* migration at the same time.
11073	*/
11074	if (env->sd->nr_balance_failed >= env->sd->cache_nice_tries + 1)
11075	return false;
11076
11077	/*
11078	* We know the env->src_cpu has some tasks prefer to
11079	* run on env->dst_cpu, skip the tasks do not prefer
11080	* env->dst_cpu, and find the one that prefers.
11081	*/
11082	if (migrate_llc_task_wrong_dst(p, env))
11083	return true;
11084
11085	if (can_migrate_llc_task(env, p) != mig_forbid)
11086	return false;
11087
11088	return true;
11089	}
11090
11091	#else
11092	static inline bool get_llc_stats(int cpu, unsigned long *util,
11093	unsigned long *cap)
11094	{
11095	return false;
11096	}
11097
11098	static inline bool
11099	alb_break_llc(struct lb_env *env)
11100	{
11101	return false;
11102	}
11103
11104	static inline bool
11105	migrate_llc_task_wrong_dst(struct task_struct p, struct lb_env env)
11106	{
11107	return false;
11108	}
11109
11110	static inline bool
11111	migrate_degrades_llc(struct task_struct p, struct lb_env env)
11112	{
11113	return false;
11114	}
11115	#endif
11116	/*
11117	* can_migrate_task - may task p from runqueue rq be migrated to this_cpu?
11118	*/
11119	static
11120	int can_migrate_task(struct task_struct p, struct lb_env env)
11121	{
11122	long degrades, hot;
11123
11124	lockdep_assert_rq_held(env->src_rq);
11125	if (p->sched_task_hot)
11126	p->sched_task_hot = 0;
11127
11128	/*
11129	* We do not migrate tasks that are:
11130	* 1) delayed dequeued unless we migrate load, or
11131	* 2) target cfs_rq is in throttled hierarchy, or
11132	* 3) cannot be migrated to this CPU due to cpus_ptr, or
11133	* 4) running (obviously), or
11134	* 5) are cache-hot on their current CPU, or
11135	* 6) are blocked on mutexes (if SCHED_PROXY_EXEC is enabled)
11136	*/
11137	if ((p->se.sched_delayed) && (env->migration_type != migrate_load))
11138	return 0;
11139
11140	if (lb_throttled_hierarchy(p, env->dst_cpu))
11141	return 0;
11142
11143	/*
11144	* We want to prioritize the migration of eligible tasks.
11145	* For ineligible tasks we soft-limit them and only allow
11146	* them to migrate when nr_balance_failed is non-zero to
11147	* avoid load-balancing trying very hard to balance the load.
11148	*/
11149	if (!env->sd->nr_balance_failed &&
11150	task_is_ineligible_on_dst_cpu(p, env->dst_cpu))
11151	return 0;
11152
11153	/* Disregard percpu kthreads; they are where they need to be. */
11154	if (kthread_is_per_cpu(p))
11155	return 0;
11156
11157	if (task_is_blocked(p))
11158	return 0;
11159
11160	if (!cpumask_test_cpu(env->dst_cpu, p->cpus_ptr)) {
11161	int cpu;
11162
11163	schedstat_inc(p->stats.nr_failed_migrations_affine);
11164
11165	env->flags \|= LBF_SOME_PINNED;
11166
11167	/*
11168	* Remember if this task can be migrated to any other CPU in
11169	* our sched_group. We may want to revisit it if we couldn't
11170	* meet load balance goals by pulling other tasks on src_cpu.
11171	*
11172	* Avoid computing new_dst_cpu
11173	* - for NEWLY_IDLE
11174	* - if we have already computed one in current iteration
11175	* - if it's an active balance
11176	*/
11177	if (env->idle == CPU_NEWLY_IDLE \|\|
11178	env->flags & (LBF_DST_PINNED \| LBF_ACTIVE_LB))
11179	return 0;
11180
11181	/* Prevent to re-select dst_cpu via env's CPUs: */
11182	cpu = cpumask_first_and_and(env->dst_grpmask, env->cpus, p->cpus_ptr);
11183
11184	if (cpu < nr_cpu_ids) {
11185	env->flags \|= LBF_DST_PINNED;
11186	env->new_dst_cpu = cpu;
11187	}
11188
11189	return 0;
11190	}
11191
11192	/* Record that we found at least one task that could run on dst_cpu */
11193	env->flags &= ~LBF_ALL_PINNED;
11194
11195	if (task_on_cpu(env->src_rq, p) \|\|
11196	task_current_donor(env->src_rq, p)) {
11197	schedstat_inc(p->stats.nr_failed_migrations_running);
11198	return 0;
11199	}
11200
11201	/*
11202	* Aggressive migration if:
11203	* 1) active balance
11204	* 2) destination numa is preferred
11205	* 3) task is cache cold, or
11206	* 4) too many balance attempts have failed.
11207	*/
11208	if (env->flags & LBF_ACTIVE_LB)
11209	return !migrate_llc_task_wrong_dst(p, env);
11210
11211	degrades = migrate_degrades_locality(p, env);
11212	if (!degrades) {
11213	/*
11214	* If the NUMA locality is not broken,
11215	* further check if migration would hurt
11216	* LLC locality.
11217	*/
11218	if (migrate_degrades_llc(p, env)) {
11219	/*
11220	* If regular load balancing fails to pull a task
11221	* due to LLC locality, this is expected behavior
11222	* and we set LBF_LLC_PINNED so we don't increase
11223	* nr_balance_failed unecessarily.
11224	*/
11225	if (env->migration_type != migrate_llc_task)
11226	env->flags \|= LBF_LLC_PINNED;
11227
11228	return 0;
11229	}
11230
11231	hot = task_hot(p, env);
11232	} else {
11233	hot = degrades > 0;
11234	}
11235
11236	if (!hot \|\| env->sd->nr_balance_failed > env->sd->cache_nice_tries) {
11237	if (hot)
11238	p->sched_task_hot = 1;
11239	return 1;
11240	}
11241
11242	schedstat_inc(p->stats.nr_failed_migrations_hot);
11243	return 0;
11244	}
11245
11246	/*
11247	* detach_task() -- detach the task for the migration specified in env
11248	*/
11249	static void detach_task(struct task_struct p, struct lb_env env)
11250	{
11251	lockdep_assert_rq_held(env->src_rq);
11252
11253	if (p->sched_task_hot) {
11254	p->sched_task_hot = 0;
11255	schedstat_inc(env->sd->lb_hot_gained[env->idle]);
11256	schedstat_inc(p->stats.nr_forced_migrations);
11257	}
11258
11259	WARN_ON(task_current(env->src_rq, p));
11260	WARN_ON(task_current_donor(env->src_rq, p));
11261
11262	deactivate_task(env->src_rq, p, DEQUEUE_NOCLOCK);
11263	set_task_cpu(p, env->dst_cpu);
11264	}
11265
11266	/*
11267	* detach_one_task() -- tries to dequeue exactly one task from env->src_rq, as
11268	* part of active balancing operations within "domain".
11269	*
11270	* Returns a task if successful and NULL otherwise.
11271	*/
11272	static struct task_struct detach_one_task(struct lb_env env)
11273	{
11274	struct task_struct *p;
11275
11276	lockdep_assert_rq_held(env->src_rq);
11277
11278	list_for_each_entry_reverse(p,
11279	&env->src_rq->cfs_tasks, se.group_node) {
11280	if (!can_migrate_task(p, env))
11281	continue;
11282
11283	detach_task(p, env);
11284
11285	/*
11286	* Right now, this is only the second place where
11287	* lb_gained[env->idle] is updated (other is detach_tasks)
11288	* so we can safely collect stats here rather than
11289	* inside detach_tasks().
11290	*/
11291	schedstat_inc(env->sd->lb_gained[env->idle]);
11292	return p;
11293	}
11294	return NULL;
11295	}
11296
11297	/*
11298	* detach_tasks() -- tries to detach up to imbalance load/util/tasks from
11299	* busiest_rq, as part of a balancing operation within domain "sd".
11300	*
11301	* Returns number of detached tasks if successful and 0 otherwise.
11302	*/
11303	static int detach_tasks(struct lb_env *env)
11304	{
11305	struct list_head *tasks = &env->src_rq->cfs_tasks;
11306	unsigned long util, load;
11307	struct task_struct *p;
11308	int detached = 0;
11309
11310	lockdep_assert_rq_held(env->src_rq);
11311
11312	/*
11313	* Source run queue has been emptied by another CPU, clear
11314	* LBF_ALL_PINNED flag as we will not test any task.
11315	*/
11316	if (env->src_rq->nr_running <= 1) {
11317	env->flags &= ~LBF_ALL_PINNED;
11318	return 0;
11319	}
11320
11321	if (env->imbalance <= 0)
11322	return 0;
11323
11324	while (!list_empty(tasks)) {
11325	/*
11326	* We don't want to steal all, otherwise we may be treated likewise,
11327	* which could at worst lead to a livelock crash.
11328	*/
11329	if (env->idle && env->src_rq->nr_running <= 1)
11330	break;
11331
11332	env->loop++;
11333	/* We've more or less seen every task there is, call it quits */
11334	if (env->loop > env->loop_max)
11335	break;
11336
11337	/* take a breather every nr_migrate tasks */
11338	if (env->loop > env->loop_break) {
11339	env->loop_break += SCHED_NR_MIGRATE_BREAK;
11340	env->flags \|= LBF_NEED_BREAK;
11341	break;
11342	}
11343
11344	p = list_last_entry(tasks, struct task_struct, se.group_node);
11345
11346	if (!can_migrate_task(p, env))
11347	goto next;
11348
11349	switch (env->migration_type) {
11350	case migrate_load:
11351	/*
11352	* Depending of the number of CPUs and tasks and the
11353	* cgroup hierarchy, task_h_load() can return a null
11354	* value. Make sure that env->imbalance decreases
11355	* otherwise detach_tasks() will stop only after
11356	* detaching up to loop_max tasks.
11357	*/
11358	load = max_t(unsigned long, task_h_load(p), 1);
11359
11360	if (sched_feat(LB_MIN) &&
11361	load < 16 && !env->sd->nr_balance_failed)
11362	goto next;
11363
11364	/*
11365	* Make sure that we don't migrate too much load.
11366	* Nevertheless, let relax the constraint if
11367	* scheduler fails to find a good waiting task to
11368	* migrate.
11369	*/
11370	if (shr_bound(load, env->sd->nr_balance_failed) > env->imbalance)
11371	goto next;
11372
11373	env->imbalance -= load;
11374	break;
11375
11376	case migrate_util:
11377	util = task_util_est(p);
11378
11379	if (shr_bound(util, env->sd->nr_balance_failed) > env->imbalance)
11380	goto next;
11381
11382	env->imbalance -= util;
11383	break;
11384
11385	case migrate_task:
11386	env->imbalance--;
11387	break;
11388
11389	case migrate_misfit:
11390	/* This is not a misfit task */
11391	if (task_fits_cpu(p, env->src_cpu))
11392	goto next;
11393
11394	env->imbalance = 0;
11395	break;
11396
11397	case migrate_llc_task:
11398	env->imbalance--;
11399	break;
11400	}
11401
11402	detach_task(p, env);
11403	list_add(&p->se.group_node, &env->tasks);
11404
11405	detached++;
11406
11407	#ifdef CONFIG_PREEMPTION
11408	/*
11409	* NEWIDLE balancing is a source of latency, so preemptible
11410	* kernels will stop after the first task is detached to minimize
11411	* the critical section.
11412	*/
11413	if (env->idle == CPU_NEWLY_IDLE)
11414	break;
11415	#endif
11416
11417	/*
11418	* We only want to steal up to the prescribed amount of
11419	* load/util/tasks.
11420	*/
11421	if (env->imbalance <= 0)
11422	break;
11423
11424	continue;
11425	next:
11426	if (p->sched_task_hot)
11427	schedstat_inc(p->stats.nr_failed_migrations_hot);
11428
11429	list_move(&p->se.group_node, tasks);
11430	}
11431
11432	/*
11433	* Right now, this is one of only two places we collect this stat
11434	* so we can safely collect detach_one_task() stats here rather
11435	* than inside detach_one_task().
11436	*/
11437	schedstat_add(env->sd->lb_gained[env->idle], detached);
11438
11439	return detached;
11440	}
11441
11442	/*
11443	* attach_tasks() -- attaches all tasks detached by detach_tasks() to their
11444	* new rq.
11445	*/
11446	static void attach_tasks(struct lb_env *env)
11447	{
11448	__attach_tasks(env->dst_rq, &env->tasks);
11449	}
11450
11451	#ifdef CONFIG_NO_HZ_COMMON
11452	static inline bool cfs_rq_has_blocked_load(struct cfs_rq *cfs_rq)
11453	{
11454	if (cfs_rq->avg.load_avg)
11455	return true;
11456
11457	if (cfs_rq->avg.util_avg)
11458	return true;
11459
11460	return false;
11461	}
11462
11463	static inline bool others_have_blocked(struct rq *rq)
11464	{
11465	if (cpu_util_rt(rq))
11466	return true;
11467
11468	if (cpu_util_dl(rq))
11469	return true;
11470
11471	if (hw_load_avg(rq))
11472	return true;
11473
11474	if (cpu_util_irq(rq))
11475	return true;
11476
11477	return false;
11478	}
11479
11480	static inline void update_blocked_load_tick(struct rq *rq)
11481	{
11482	WRITE_ONCE(rq->last_blocked_load_update_tick, jiffies);
11483	}
11484
11485	static inline void update_has_blocked_load_status(struct rq *rq, bool has_blocked_load)
11486	{
11487	if (!has_blocked_load)
11488	rq->has_blocked_load = 0;
11489	}
11490	#else /* !CONFIG_NO_HZ_COMMON: */
11491	static inline bool cfs_rq_has_blocked_load(struct cfs_rq *cfs_rq) { return false; }
11492	static inline bool others_have_blocked(struct rq *rq) { return false; }
11493	static inline void update_blocked_load_tick(struct rq *rq) {}
11494	static inline void update_has_blocked_load_status(struct rq *rq, bool has_blocked_load) {}
11495	#endif /* !CONFIG_NO_HZ_COMMON */
11496
11497	static bool __update_blocked_others(struct rq rq, bool done)
11498	{
11499	bool updated;
11500
11501	/*
11502	* update_load_avg() can call cpufreq_update_util(). Make sure that RT,
11503	* DL and IRQ signals have been updated before updating CFS.
11504	*/
11505	updated = update_other_load_avgs(rq);
11506
11507	if (others_have_blocked(rq))
11508	*done = false;
11509
11510	return updated;
11511	}
11512
11513	#ifdef CONFIG_FAIR_GROUP_SCHED
11514
11515	static bool __update_blocked_fair(struct rq rq, bool done)
11516	{
11517	struct cfs_rq cfs_rq, pos;
11518	bool decayed = false;
11519
11520	/*
11521	* Iterates the task_group tree in a bottom up fashion, see
11522	* list_add_leaf_cfs_rq() for details.
11523	*/
11524	for_each_leaf_cfs_rq_safe(rq, cfs_rq, pos) {
11525	struct sched_entity *se;
11526
11527	if (update_cfs_rq_load_avg(cfs_rq_clock_pelt(cfs_rq), cfs_rq)) {
11528	update_tg_load_avg(cfs_rq);
11529
11530	if (cfs_rq->nr_queued == 0)
11531	update_idle_cfs_rq_clock_pelt(cfs_rq);
11532
11533	if (cfs_rq == &rq->cfs)
11534	decayed = true;
11535	}
11536
11537	/* Propagate pending load changes to the parent, if any: */
11538	se = cfs_rq_se(cfs_rq);
11539	if (se && !skip_blocked_update(se))
11540	update_load_avg(cfs_rq_of(se), se, UPDATE_TG);
11541
11542	/*
11543	* There can be a lot of idle CPU cgroups. Don't let fully
11544	* decayed cfs_rqs linger on the list.
11545	*/
11546	if (cfs_rq_is_decayed(cfs_rq))
11547	list_del_leaf_cfs_rq(cfs_rq);
11548
11549	/* Don't need periodic decay once load/util_avg are null */
11550	if (cfs_rq_has_blocked_load(cfs_rq))
11551	*done = false;
11552	}
11553
11554	return decayed;
11555	}
11556
11557	/*
11558	* Compute the hierarchical load factor for cfs_rq and all its ascendants.
11559	* This needs to be done in a top-down fashion because the load of a child
11560	* group is a fraction of its parents load.
11561	*/
11562	static void update_cfs_rq_h_load(struct cfs_rq *cfs_rq)
11563	{
11564	struct sched_entity *se = cfs_rq_se(cfs_rq);
11565	unsigned long now = jiffies;
11566	unsigned long load;
11567
11568	if (cfs_rq->last_h_load_update == now)
11569	return;
11570
11571	WRITE_ONCE(cfs_rq->h_load_next, NULL);
11572	for_each_sched_entity(se) {
11573	cfs_rq = cfs_rq_of(se);
11574	WRITE_ONCE(cfs_rq->h_load_next, se);
11575	if (cfs_rq->last_h_load_update == now)
11576	break;
11577	}
11578
11579	if (!se) {
11580	cfs_rq->h_load = cfs_rq_load_avg(cfs_rq);
11581	cfs_rq->last_h_load_update = now;
11582	}
11583
11584	while ((se = READ_ONCE(cfs_rq->h_load_next)) != NULL) {
11585	load = cfs_rq->h_load;
11586	load = div64_ul(load * se->avg.load_avg,
11587	cfs_rq_load_avg(cfs_rq) + 1);
11588	cfs_rq = group_cfs_rq(se);
11589	cfs_rq->h_load = load;
11590	cfs_rq->last_h_load_update = now;
11591	}
11592	}
11593
11594	static unsigned long task_h_load(struct task_struct *p)
11595	{
11596	struct cfs_rq *cfs_rq = task_cfs_rq(p);
11597
11598	update_cfs_rq_h_load(cfs_rq);
11599	return div64_ul(p->se.avg.load_avg * cfs_rq->h_load,
11600	cfs_rq_load_avg(cfs_rq) + 1);
11601	}
11602	#else /* !CONFIG_FAIR_GROUP_SCHED: */
11603	static bool __update_blocked_fair(struct rq rq, bool done)
11604	{
11605	struct cfs_rq *cfs_rq = &rq->cfs;
11606	bool decayed;
11607
11608	decayed = update_cfs_rq_load_avg(cfs_rq_clock_pelt(cfs_rq), cfs_rq);
11609	if (cfs_rq_has_blocked_load(cfs_rq))
11610	*done = false;
11611
11612	return decayed;
11613	}
11614
11615	static unsigned long task_h_load(struct task_struct *p)
11616	{
11617	return p->se.avg.load_avg;
11618	}
11619	#endif /* !CONFIG_FAIR_GROUP_SCHED */
11620
11621	static void __sched_balance_update_blocked_averages(struct rq *rq)
11622	{
11623	bool decayed = false, done = true;
11624
11625	update_blocked_load_tick(rq);
11626
11627	decayed \|= __update_blocked_others(rq, &done);
11628	decayed \|= __update_blocked_fair(rq, &done);
11629
11630	update_has_blocked_load_status(rq, !done);
11631	if (decayed)
11632	cpufreq_update_util(rq, 0);
11633	}
11634
11635	static void sched_balance_update_blocked_averages(int cpu)
11636	{
11637	struct rq *rq = cpu_rq(cpu);
11638
11639	guard(rq_lock_irqsave)(rq);
11640	update_rq_clock(rq);
11641	__sched_balance_update_blocked_averages(rq);
11642	}
11643
11644	/******** Helpers for sched_balance_find_src_group **********************/
11645
11646	/*
11647	* sg_lb_stats - stats of a sched_group required for load-balancing:
11648	*/
11649	struct sg_lb_stats {
11650	unsigned long avg_load; /* Avg load over the CPUs of the group */
11651	unsigned long group_load; /* Total load over the CPUs of the group */
11652	unsigned long group_capacity; /* Capacity over the CPUs of the group */
11653	unsigned long group_util; /* Total utilization over the CPUs of the group */
11654	unsigned long group_runnable; /* Total runnable time over the CPUs of the group */
11655	unsigned int sum_nr_running; /* Nr of all tasks running in the group */
11656	unsigned int sum_h_nr_running; /* Nr of CFS tasks running in the group */
11657	unsigned int idle_cpus; /* Nr of idle CPUs in the group */
11658	unsigned int group_weight;
11659	enum group_type group_type;
11660	unsigned int group_asym_packing; /* Tasks should be moved to preferred CPU */
11661	unsigned int group_smt_balance; /* Task on busy SMT be moved */
11662	unsigned int group_llc_balance; /* Tasks should be moved to preferred LLC */
11663	unsigned long group_misfit_task_load; /* A CPU has a task too big for its capacity */
11664	unsigned int group_overutilized; /* At least one CPU is overutilized in the group */
11665	#ifdef CONFIG_NUMA_BALANCING
11666	unsigned int nr_numa_running;
11667	unsigned int nr_preferred_running;
11668	#endif
11669	#ifdef CONFIG_SCHED_CACHE
11670	unsigned int nr_pref_dst_llc;
11671	#endif
11672	};
11673
11674	/*
11675	* sd_lb_stats - stats of a sched_domain required for load-balancing:
11676	*/
11677	struct sd_lb_stats {
11678	struct sched_group busiest; / Busiest group in this sd */
11679	struct sched_group local; / Local group in this sd */
11680	unsigned long total_load; /* Total load of all groups in sd */
11681	unsigned long total_capacity; /* Total capacity of all groups in sd */
11682	unsigned long avg_load; /* Average load across all groups in sd */
11683	unsigned int prefer_sibling; /* Tasks should go to sibling first */
11684
11685	struct sg_lb_stats busiest_stat; /* Statistics of the busiest group */
11686	struct sg_lb_stats local_stat; /* Statistics of the local group */
11687	};
11688
11689	static inline void init_sd_lb_stats(struct sd_lb_stats *sds)
11690	{
11691	/*
11692	* Skimp on the clearing to avoid duplicate work. We can avoid clearing
11693	* local_stat because update_sg_lb_stats() does a full clear/assignment.
11694	* We must however set busiest_stat::group_type and
11695	* busiest_stat::idle_cpus to the worst busiest group because
11696	* update_sd_pick_busiest() reads these before assignment.
11697	*/
11698	*sds = (struct sd_lb_stats){
11699	.busiest = NULL,
11700	.local = NULL,
11701	.total_load = 0UL,
11702	.total_capacity = 0UL,
11703	.busiest_stat = {
11704	.idle_cpus = UINT_MAX,
11705	.group_type = group_has_spare,
11706	},
11707	};
11708	}
11709
11710	static unsigned long scale_rt_capacity(int cpu)
11711	{
11712	unsigned long max = get_actual_cpu_capacity(cpu);
11713	struct rq *rq = cpu_rq(cpu);
11714	unsigned long used, free;
11715	unsigned long irq;
11716
11717	irq = cpu_util_irq(rq);
11718
11719	if (unlikely(irq >= max))
11720	return 1;
11721
11722	/*
11723	* avg_rt.util_avg and avg_dl.util_avg track binary signals
11724	* (running and not running) with weights 0 and 1024 respectively.
11725	*/
11726	used = cpu_util_rt(rq);
11727	used += cpu_util_dl(rq);
11728
11729	if (unlikely(used >= max))
11730	return 1;
11731
11732	free = max - used;
11733
11734	return scale_irq_capacity(free, irq, max);
11735	}
11736
11737	static void update_cpu_capacity(struct sched_domain *sd, int cpu)
11738	{
11739	unsigned long capacity = scale_rt_capacity(cpu);
11740	struct sched_group *sdg = sd->groups;
11741
11742	if (!capacity)
11743	capacity = 1;
11744
11745	cpu_rq(cpu)->cpu_capacity = capacity;
11746	trace_sched_cpu_capacity_tp(cpu_rq(cpu));
11747
11748	sdg->sgc->capacity = capacity;
11749	sdg->sgc->min_capacity = capacity;
11750	sdg->sgc->max_capacity = capacity;
11751	}
11752
11753	void update_group_capacity(struct sched_domain *sd, int cpu)
11754	{
11755	struct sched_domain *child = sd->child;
11756	struct sched_group group, sdg = sd->groups;
11757	unsigned long capacity, min_capacity, max_capacity;
11758	unsigned long interval;
11759
11760	interval = msecs_to_jiffies(sd->balance_interval);
11761	interval = clamp(interval, 1UL, max_load_balance_interval);
11762	sdg->sgc->next_update = jiffies + interval;
11763
11764	if (!child) {
11765	update_cpu_capacity(sd, cpu);
11766	return;
11767	}
11768
11769	capacity = 0;
11770	min_capacity = ULONG_MAX;
11771	max_capacity = 0;
11772
11773	if (child->flags & SD_NUMA) {
11774	/*
11775	* SD_NUMA domains cannot assume that child groups
11776	* span the current group.
11777	*/
11778
11779	for_each_cpu(cpu, sched_group_span(sdg)) {
11780	unsigned long cpu_cap = capacity_of(cpu);
11781
11782	capacity += cpu_cap;
11783	min_capacity = min(cpu_cap, min_capacity);
11784	max_capacity = max(cpu_cap, max_capacity);
11785	}
11786	} else {
11787	/*
11788	* !SD_NUMA domains can assume that child groups
11789	* span the current group.
11790	*/
11791
11792	group = child->groups;
11793	do {
11794	struct sched_group_capacity *sgc = group->sgc;
11795
11796	capacity += sgc->capacity;
11797	min_capacity = min(sgc->min_capacity, min_capacity);
11798	max_capacity = max(sgc->max_capacity, max_capacity);
11799	group = group->next;
11800	} while (group != child->groups);
11801	}
11802
11803	sdg->sgc->capacity = capacity;
11804	sdg->sgc->min_capacity = min_capacity;
11805	sdg->sgc->max_capacity = max_capacity;
11806	}
11807
11808	/*
11809	* Check whether the capacity of the rq has been noticeably reduced by side
11810	* activity. The imbalance_pct is used for the threshold.
11811	* Return true is the capacity is reduced
11812	*/
11813	static inline int
11814	check_cpu_capacity(struct rq rq, struct sched_domain sd)
11815	{
11816	return ((rq->cpu_capacity * sd->imbalance_pct) <
11817	(arch_scale_cpu_capacity(cpu_of(rq)) * 100));
11818	}
11819
11820	/* Check if the rq has a misfit task */
11821	static inline bool check_misfit_status(struct rq *rq)
11822	{
11823	return rq->misfit_task_load;
11824	}
11825
11826	/*
11827	* Group imbalance indicates (and tries to solve) the problem where balancing
11828	* groups is inadequate due to ->cpus_ptr constraints.
11829	*
11830	* Imagine a situation of two groups of 4 CPUs each and 4 tasks each with a
11831	* cpumask covering 1 CPU of the first group and 3 CPUs of the second group.
11832	* Something like:
11833	*
11834	* { 0 1 2 3 } { 4 5 6 7 }
11835	* * * * *
11836	*
11837	* If we were to balance group-wise we'd place two tasks in the first group and
11838	* two tasks in the second group. Clearly this is undesired as it will overload
11839	* cpu 3 and leave one of the CPUs in the second group unused.
11840	*
11841	* The current solution to this issue is detecting the skew in the first group
11842	* by noticing the lower domain failed to reach balance and had difficulty
11843	* moving tasks due to affinity constraints.
11844	*
11845	* When this is so detected; this group becomes a candidate for busiest; see
11846	* update_sd_pick_busiest(). And calculate_imbalance() and
11847	* sched_balance_find_src_group() avoid some of the usual balance conditions to allow it
11848	* to create an effective group imbalance.
11849	*
11850	* This is a somewhat tricky proposition since the next run might not find the
11851	* group imbalance and decide the groups need to be balanced again. A most
11852	* subtle and fragile situation.
11853	*/
11854
11855	static inline int sg_imbalanced(struct sched_group *group)
11856	{
11857	return group->sgc->imbalance;
11858	}
11859
11860	/*
11861	* group_has_capacity returns true if the group has spare capacity that could
11862	* be used by some tasks.
11863	* We consider that a group has spare capacity if the number of task is
11864	* smaller than the number of CPUs or if the utilization is lower than the
11865	* available capacity for CFS tasks.
11866	* For the latter, we use a threshold to stabilize the state, to take into
11867	* account the variance of the tasks' load and to return true if the available
11868	* capacity in meaningful for the load balancer.
11869	* As an example, an available capacity of 1% can appear but it doesn't make
11870	* any benefit for the load balance.
11871	*/
11872	static inline bool
11873	group_has_capacity(unsigned int imbalance_pct, struct sg_lb_stats *sgs)
11874	{
11875	if (sgs->sum_nr_running < sgs->group_weight)
11876	return true;
11877
11878	if ((sgs->group_capacity * imbalance_pct) <
11879	(sgs->group_runnable * 100))
11880	return false;
11881
11882	if ((sgs->group_capacity * 100) >
11883	(sgs->group_util * imbalance_pct))
11884	return true;
11885
11886	return false;
11887	}
11888
11889	/*
11890	* group_is_overloaded returns true if the group has more tasks than it can
11891	* handle.
11892	* group_is_overloaded is not equals to !group_has_capacity because a group
11893	* with the exact right number of tasks, has no more spare capacity but is not
11894	* overloaded so both group_has_capacity and group_is_overloaded return
11895	* false.
11896	*/
11897	static inline bool
11898	group_is_overloaded(unsigned int imbalance_pct, struct sg_lb_stats *sgs)
11899	{
11900	/*
11901	* With EAS and uclamp, 1 CPU in the group must be overutilized to
11902	* consider the group overloaded.
11903	*/
11904	if (sched_energy_enabled() && !sgs->group_overutilized)
11905	return false;
11906
11907	if (sgs->sum_nr_running <= sgs->group_weight)
11908	return false;
11909
11910	if ((sgs->group_capacity * 100) <
11911	(sgs->group_util * imbalance_pct))
11912	return true;
11913
11914	if ((sgs->group_capacity * imbalance_pct) <
11915	(sgs->group_runnable * 100))
11916	return true;
11917
11918	return false;
11919	}
11920
11921	static inline enum
11922	group_type group_classify(unsigned int imbalance_pct,
11923	struct sched_group *group,
11924	struct sg_lb_stats *sgs)
11925	{
11926	if (group_is_overloaded(imbalance_pct, sgs))
11927	return group_overloaded;
11928
11929	if (sgs->group_llc_balance)
11930	return group_llc_balance;
11931
11932	if (sg_imbalanced(group))
11933	return group_imbalanced;
11934
11935	if (sgs->group_asym_packing)
11936	return group_asym_packing;
11937
11938	if (sgs->group_smt_balance)
11939	return group_smt_balance;
11940
11941	if (sgs->group_misfit_task_load)
11942	return group_misfit_task;
11943
11944	if (!group_has_capacity(imbalance_pct, sgs))
11945	return group_fully_busy;
11946
11947	return group_has_spare;
11948	}
11949
11950	/**
11951	* sched_use_asym_prio - Check whether asym_packing priority must be used
11952	* @sd: The scheduling domain of the load balancing
11953	* @cpu: A CPU
11954	*
11955	* Always use CPU priority when balancing load between SMT siblings. When
11956	* balancing load between cores, it is not sufficient that @cpu is idle. Only
11957	* use CPU priority if the whole core is idle.
11958	*
11959	* Returns: True if the priority of @cpu must be followed. False otherwise.
11960	*/
11961	static bool sched_use_asym_prio(struct sched_domain *sd, int cpu)
11962	{
11963	if (!(sd->flags & SD_ASYM_PACKING))
11964	return false;
11965
11966	if (!sched_smt_active())
11967	return true;
11968
11969	return sd->flags & SD_SHARE_CPUCAPACITY \|\| is_core_idle(cpu);
11970	}
11971
11972	static inline bool sched_asym(struct sched_domain *sd, int dst_cpu, int src_cpu)
11973	{
11974	/*
11975	* First check if @dst_cpu can do asym_packing load balance. Only do it
11976	* if it has higher priority than @src_cpu.
11977	*/
11978	return sched_use_asym_prio(sd, dst_cpu) &&
11979	sched_asym_prefer(dst_cpu, src_cpu);
11980	}
11981
11982	/**
11983	* sched_group_asym - Check if the destination CPU can do asym_packing balance
11984	* @env: The load balancing environment
11985	* @sgs: Load-balancing statistics of the candidate busiest group
11986	* @group: The candidate busiest group
11987	*
11988	* @env::dst_cpu can do asym_packing if it has higher priority than the
11989	* preferred CPU of @group.
11990	*
11991	* Return: true if @env::dst_cpu can do with asym_packing load balance. False
11992	* otherwise.
11993	*/
11994	static inline bool
11995	sched_group_asym(struct lb_env env, struct sg_lb_stats sgs, struct sched_group *group)
11996	{
11997	/*
11998	* CPU priorities do not make sense for SMT cores with more than one
11999	* busy sibling.
12000	*/
12001	if ((group->flags & SD_SHARE_CPUCAPACITY) &&
12002	(sgs->group_weight - sgs->idle_cpus != 1))
12003	return false;
12004
12005	return sched_asym(env->sd, env->dst_cpu, READ_ONCE(group->asym_prefer_cpu));
12006	}
12007
12008	/* One group has more than one SMT CPU while the other group does not */
12009	static inline bool smt_vs_nonsmt_groups(struct sched_group *sg1,
12010	struct sched_group *sg2)
12011	{
12012	if (!sg1 \|\| !sg2)
12013	return false;
12014
12015	return (sg1->flags & SD_SHARE_CPUCAPACITY) !=
12016	(sg2->flags & SD_SHARE_CPUCAPACITY);
12017	}
12018
12019	static inline bool smt_balance(struct lb_env env, struct sg_lb_stats sgs,
12020	struct sched_group *group)
12021	{
12022	if (!env->idle)
12023	return false;
12024
12025	/*
12026	* For SMT source group, it is better to move a task
12027	* to a CPU that doesn't have multiple tasks sharing its CPU capacity.
12028	* Note that if a group has a single SMT, SD_SHARE_CPUCAPACITY
12029	* will not be on.
12030	*/
12031	if (group->flags & SD_SHARE_CPUCAPACITY &&
12032	sgs->sum_h_nr_running > 1)
12033	return true;
12034
12035	return false;
12036	}
12037
12038	static inline long sibling_imbalance(struct lb_env *env,
12039	struct sd_lb_stats *sds,
12040	struct sg_lb_stats *busiest,
12041	struct sg_lb_stats *local)
12042	{
12043	int ncores_busiest, ncores_local;
12044	long imbalance;
12045
12046	if (!env->idle \|\| !busiest->sum_nr_running)
12047	return 0;
12048
12049	ncores_busiest = sds->busiest->cores;
12050	ncores_local = sds->local->cores;
12051
12052	if (ncores_busiest == ncores_local) {
12053	imbalance = busiest->sum_nr_running;
12054	lsub_positive(&imbalance, local->sum_nr_running);
12055	return imbalance;
12056	}
12057
12058	/* Balance such that nr_running/ncores ratio are same on both groups */
12059	imbalance = ncores_local * busiest->sum_nr_running;
12060	lsub_positive(&imbalance, ncores_busiest * local->sum_nr_running);
12061	/* Normalize imbalance and do rounding on normalization */
12062	imbalance = 2 * imbalance + ncores_local + ncores_busiest;
12063	imbalance /= ncores_local + ncores_busiest;
12064
12065	/* Take advantage of resource in an empty sched group */
12066	if (imbalance <= 1 && local->sum_nr_running == 0 &&
12067	busiest->sum_nr_running > 1)
12068	imbalance = 2;
12069
12070	return imbalance;
12071	}
12072
12073	static inline bool
12074	sched_reduced_capacity(struct rq rq, struct sched_domain sd)
12075	{
12076	/*
12077	* When there is more than 1 task, the group_overloaded case already
12078	* takes care of cpu with reduced capacity
12079	*/
12080	if (rq->cfs.h_nr_runnable != 1)
12081	return false;
12082
12083	return check_cpu_capacity(rq, sd);
12084	}
12085
12086	#ifdef CONFIG_SCHED_CACHE
12087	/*
12088	* Record the statistics for this scheduler group for later
12089	* use. These values guide load balancing on aggregating tasks
12090	* to a LLC.
12091	*/
12092	static void record_sg_llc_stats(struct lb_env *env,
12093	struct sg_lb_stats *sgs,
12094	struct sched_group *group)
12095	{
12096	struct sched_domain_shared *sd_share;
12097	int cpu;
12098
12099	if (!sched_cache_enabled() \|\| env->idle == CPU_NEWLY_IDLE)
12100	return;
12101
12102	/* Only care about sched domain spanning multiple LLCs */
12103	if (env->sd->child != rcu_dereference_all(per_cpu(sd_llc, env->dst_cpu)))
12104	return;
12105
12106	/*
12107	* A redo repeats stats published on the first pass. Hotplug can
12108	* still leave env->cpus partial, so util would not match capacity.
12109	*/
12110	if ((env->flags & LBF_ALL_PINNED) \|\|
12111	!cpumask_subset(sched_group_span(group), env->cpus))
12112	return;
12113
12114	/*
12115	* At this point we know this group spans a LLC domain.
12116	* Record the statistic of this group in its corresponding
12117	* shared LLC domain.
12118	* Note: sd_share cannot be obtained via sd->child->shared,
12119	* because the latter refers to the domain that covers the
12120	* local group. Instead, sd_share should be located using
12121	* the first CPU of the LLC group.
12122	*/
12123	cpu = cpumask_first(sched_group_span(group));
12124	sd_share = rcu_dereference_all(per_cpu(sd_llc_shared, cpu));
12125	if (!sd_share)
12126	return;
12127
12128	if (READ_ONCE(sd_share->util_avg) != sgs->group_util)
12129	WRITE_ONCE(sd_share->util_avg, sgs->group_util);
12130
12131	if (unlikely(READ_ONCE(sd_share->capacity) != sgs->group_capacity))
12132	WRITE_ONCE(sd_share->capacity, sgs->group_capacity);
12133	}
12134
12135	/*
12136	* Do LLC balance on sched group that contains LLC, and have tasks preferring
12137	* to run on LLC in idle dst_cpu.
12138	*/
12139	static inline bool llc_balance(struct lb_env env, struct sg_lb_stats sgs,
12140	struct sched_group *group)
12141	{
12142	if (!sched_cache_enabled())
12143	return false;
12144
12145	if (env->sd->flags & SD_SHARE_LLC)
12146	return false;
12147
12148	/*
12149	* On asymmetric domains, group_misfit_task_load
12150	* should be prioritized to move tasks to CPU that fit them
12151	* over aggregating tasks to their preferred LLC.
12152	*/
12153	if ((env->sd->flags & SD_ASYM_CPUCAPACITY) &&
12154	sgs->group_misfit_task_load)
12155	return false;
12156
12157	/*
12158	* Skip cache aware tagging if nr_balanced_failed is sufficiently high.
12159	* Threshold of cache_nice_tries is set to 1 higher than nr_balance_failed
12160	* to avoid excessive task migration at the same time.
12161	*/
12162	if (env->sd->nr_balance_failed >= env->sd->cache_nice_tries + 1)
12163	return false;
12164
12165	if (sgs->nr_pref_dst_llc &&
12166	can_migrate_llc(cpumask_first(sched_group_span(group)),
12167	env->dst_cpu, 0, true) == mig_llc)
12168	return true;
12169
12170	return false;
12171	}
12172
12173	static bool update_llc_busiest(struct lb_env *env,
12174	struct sg_lb_stats *busiest,
12175	struct sg_lb_stats *sgs)
12176	{
12177	/*
12178	* There are more tasks that want to run on dst_cpu's LLC.
12179	*/
12180	return sgs->nr_pref_dst_llc > busiest->nr_pref_dst_llc;
12181	}
12182	#else
12183	static inline void record_sg_llc_stats(struct lb_env env, struct sg_lb_stats sgs,
12184	struct sched_group *group)
12185	{
12186	}
12187
12188	static inline bool llc_balance(struct lb_env env, struct sg_lb_stats sgs,
12189	struct sched_group *group)
12190	{
12191	return false;
12192	}
12193
12194	static bool update_llc_busiest(struct lb_env *env,
12195	struct sg_lb_stats *busiest,
12196	struct sg_lb_stats *sgs)
12197	{
12198	return false;
12199	}
12200	#endif
12201
12202	/**
12203	* update_sg_lb_stats - Update sched_group's statistics for load balancing.
12204	* @env: The load balancing environment.
12205	* @sds: Load-balancing data with statistics of the local group.
12206	* @group: sched_group whose statistics are to be updated.
12207	* @sgs: variable to hold the statistics for this group.
12208	* @sg_overloaded: sched_group is overloaded
12209	*/
12210	static inline void update_sg_lb_stats(struct lb_env *env,
12211	struct sd_lb_stats *sds,
12212	struct sched_group *group,
12213	struct sg_lb_stats *sgs,
12214	bool *sg_overloaded)
12215	{
12216	int i, nr_running, local_group, sd_flags = env->sd->flags;
12217	bool balancing_at_rd = !env->sd->parent;
12218
12219	memset(sgs, 0, sizeof(*sgs));
12220
12221	local_group = group == sds->local;
12222
12223	for_each_cpu_and(i, sched_group_span(group), env->cpus) {
12224	struct rq *rq = cpu_rq(i);
12225	unsigned long load = cpu_load(rq);
12226
12227	sgs->group_load += load;
12228	sgs->group_util += cpu_util_cfs(i);
12229	sgs->group_runnable += cpu_runnable(rq);
12230	sgs->sum_h_nr_running += rq->cfs.h_nr_runnable;
12231
12232	nr_running = rq->nr_running;
12233	sgs->sum_nr_running += nr_running;
12234
12235	if (cpu_overutilized(i))
12236	sgs->group_overutilized = 1;
12237
12238	#ifdef CONFIG_SCHED_CACHE
12239	if (sched_cache_enabled()) {
12240	struct sched_domain *sd_tmp;
12241	int dst_llc;
12242
12243	dst_llc = llc_id(env->dst_cpu);
12244	if (llc_id(i) != dst_llc) {
12245	sd_tmp = rcu_dereference_all(rq->sd);
12246	if (sd_tmp && (unsigned int)dst_llc < sd_tmp->llc_max)
12247	sgs->nr_pref_dst_llc += sd_tmp->llc_counts[dst_llc];
12248	}
12249	}
12250	#endif
12251
12252	/*
12253	* No need to call idle_cpu() if nr_running is not 0
12254	*/
12255	if (!nr_running && idle_cpu(i)) {
12256	sgs->idle_cpus++;
12257	/* Idle cpu can't have misfit task */
12258	continue;
12259	}
12260
12261	/* Overload indicator is only updated at root domain */
12262	if (balancing_at_rd && nr_running > 1)
12263	*sg_overloaded = 1;
12264
12265	#ifdef CONFIG_NUMA_BALANCING
12266	/* Only fbq_classify_group() uses this to classify NUMA groups */
12267	if (sd_flags & SD_NUMA) {
12268	sgs->nr_numa_running += rq->nr_numa_running;
12269	sgs->nr_preferred_running += rq->nr_preferred_running;
12270	}
12271	#endif
12272	if (local_group)
12273	continue;
12274
12275	if (sd_flags & SD_ASYM_CPUCAPACITY) {
12276	if (rq->misfit_task_load) {
12277	/*
12278	* Always mark the root domain overloaded so big
12279	* CPUs can pick up misfit tasks via newly idle
12280	* balance.
12281	*/
12282	if (balancing_at_rd)
12283	*sg_overloaded = 1;
12284
12285	/*
12286	* Only account misfit load if @dst_cpu can
12287	* help; otherwise, the group may be classified
12288	* as misfit_task and update_sd_pick_busiest()
12289	* will skip it.
12290	*/
12291	if (capacity_greater(capacity_of(env->dst_cpu),
12292	group->sgc->max_capacity) &&
12293	(sgs->group_misfit_task_load < rq->misfit_task_load))
12294	sgs->group_misfit_task_load = rq->misfit_task_load;
12295	}
12296	} else if (env->idle && sched_reduced_capacity(rq, env->sd)) {
12297	/* Check for a task running on a CPU with reduced capacity */
12298	if (sgs->group_misfit_task_load < load)
12299	sgs->group_misfit_task_load = load;
12300	}
12301	}
12302
12303	sgs->group_capacity = group->sgc->capacity;
12304
12305	sgs->group_weight = group->group_weight;
12306
12307	if (!local_group) {
12308	/* Check if dst CPU is idle and preferred to this group */
12309	if (env->idle && sgs->sum_h_nr_running &&
12310	sched_group_asym(env, sgs, group))
12311	sgs->group_asym_packing = 1;
12312
12313	/* Check for loaded SMT group to be balanced to dst CPU */
12314	if (smt_balance(env, sgs, group))
12315	sgs->group_smt_balance = 1;
12316
12317	/* Check for tasks in this group can be moved to their preferred LLC */
12318	if (llc_balance(env, sgs, group))
12319	sgs->group_llc_balance = 1;
12320	}
12321
12322	sgs->group_type = group_classify(env->sd->imbalance_pct, group, sgs);
12323
12324	record_sg_llc_stats(env, sgs, group);
12325	/* Computing avg_load makes sense only when group is overloaded */
12326	if (sgs->group_type == group_overloaded)
12327	sgs->avg_load = (sgs->group_load * SCHED_CAPACITY_SCALE) /
12328	sgs->group_capacity;
12329	}
12330
12331	/**
12332	* update_sd_pick_busiest - return 1 on busiest group
12333	* @env: The load balancing environment.
12334	* @sds: sched_domain statistics
12335	* @sg: sched_group candidate to be checked for being the busiest
12336	* @sgs: sched_group statistics
12337	*
12338	* Determine if @sg is a busier group than the previously selected
12339	* busiest group.
12340	*
12341	* Return: %true if @sg is a busier group than the previously selected
12342	* busiest group. %false otherwise.
12343	*/
12344	static bool update_sd_pick_busiest(struct lb_env *env,
12345	struct sd_lb_stats *sds,
12346	struct sched_group *sg,
12347	struct sg_lb_stats *sgs)
12348	{
12349	struct sg_lb_stats *busiest = &sds->busiest_stat;
12350
12351	/* Make sure that there is at least one task to pull */
12352	if (!sgs->sum_h_nr_running)
12353	return false;
12354
12355	/*
12356	* Don't try to pull misfit tasks we can't help.
12357	* We can use max_capacity here as reduction in capacity on some
12358	* CPUs in the group should either be possible to resolve
12359	* internally or be covered by avg_load imbalance (eventually).
12360	*
12361	* When SMT is active, only pull a misfit to dst_cpu if it is on a
12362	* fully idle core; otherwise the effective capacity of the core is
12363	* reduced and we may not actually provide more capacity than the
12364	* source.
12365	*/
12366	if ((env->sd->flags & SD_ASYM_CPUCAPACITY) &&
12367	(sgs->group_type == group_misfit_task) &&
12368	(!env->dst_core_idle \|\|
12369	!capacity_greater(capacity_of(env->dst_cpu), sg->sgc->max_capacity) \|\|
12370	sds->local_stat.group_type != group_has_spare))
12371	return false;
12372
12373	/*
12374	* Candidate sg has no more than one task per CPU and has higher
12375	* per-CPU capacity. Migrating tasks to less capable CPUs may harm
12376	* throughput. Maximize throughput, power/energy consequences are not
12377	* considered.
12378	*/
12379	if ((env->sd->flags & SD_ASYM_CPUCAPACITY) &&
12380	(sgs->group_type <= group_fully_busy) &&
12381	(capacity_greater(sg->sgc->min_capacity, capacity_of(env->dst_cpu))))
12382	return false;
12383
12384	if (sgs->group_type > busiest->group_type)
12385	return true;
12386
12387	if (sgs->group_type < busiest->group_type)
12388	return false;
12389
12390	/*
12391	* The candidate and the current busiest group are the same type of
12392	* group. Let check which one is the busiest according to the type.
12393	*/
12394
12395	switch (sgs->group_type) {
12396	case group_overloaded:
12397	/* Select the overloaded group with highest avg_load. */
12398	return sgs->avg_load > busiest->avg_load;
12399
12400	case group_llc_balance:
12401	/* Select the group with most tasks preferring dst LLC */
12402	return update_llc_busiest(env, busiest, sgs);
12403
12404	case group_imbalanced:
12405	/*
12406	* Select the 1st imbalanced group as we don't have any way to
12407	* choose one more than another.
12408	*/
12409	return false;
12410
12411	case group_asym_packing:
12412	/* Prefer to move from lowest priority CPU's work */
12413	return sched_asym_prefer(READ_ONCE(sds->busiest->asym_prefer_cpu),
12414	READ_ONCE(sg->asym_prefer_cpu));
12415
12416	case group_misfit_task:
12417	/*
12418	* If we have more than one misfit sg go with the biggest
12419	* misfit.
12420	*/
12421	return sgs->group_misfit_task_load > busiest->group_misfit_task_load;
12422
12423	case group_smt_balance:
12424	/*
12425	* Check if we have spare CPUs on either SMT group to
12426	* choose has spare or fully busy handling.
12427	*/
12428	if (sgs->idle_cpus != 0 \|\| busiest->idle_cpus != 0)
12429	goto has_spare;
12430
12431	fallthrough;
12432
12433	case group_fully_busy:
12434	/*
12435	* Select the fully busy group with highest avg_load. In
12436	* theory, there is no need to pull task from such kind of
12437	* group because tasks have all compute capacity that they need
12438	* but we can still improve the overall throughput by reducing
12439	* contention when accessing shared HW resources.
12440	*
12441	* XXX for now avg_load is not computed and always 0 so we
12442	* select the 1st one, except if @sg is composed of SMT
12443	* siblings.
12444	*/
12445
12446	if (sgs->avg_load < busiest->avg_load)
12447	return false;
12448
12449	if (sgs->avg_load == busiest->avg_load) {
12450	/*
12451	* SMT sched groups need more help than non-SMT groups.
12452	* If @sg happens to also be SMT, either choice is good.
12453	*/
12454	if (sds->busiest->flags & SD_SHARE_CPUCAPACITY)
12455	return false;
12456	}
12457
12458	break;
12459
12460	case group_has_spare:
12461	/*
12462	* Do not pick sg with SMT CPUs over sg with pure CPUs,
12463	* as we do not want to pull task off SMT core with one task
12464	* and make the core idle.
12465	*/
12466	if (smt_vs_nonsmt_groups(sds->busiest, sg)) {
12467	if (sg->flags & SD_SHARE_CPUCAPACITY && sgs->sum_h_nr_running <= 1)
12468	return false;
12469	else
12470	return true;
12471	}
12472	has_spare:
12473
12474	/*
12475	* Select not overloaded group with lowest number of idle CPUs
12476	* and highest number of running tasks. We could also compare
12477	* the spare capacity which is more stable but it can end up
12478	* that the group has less spare capacity but finally more idle
12479	* CPUs which means less opportunity to pull tasks.
12480	*/
12481	if (sgs->idle_cpus > busiest->idle_cpus)
12482	return false;
12483	else if ((sgs->idle_cpus == busiest->idle_cpus) &&
12484	(sgs->sum_nr_running <= busiest->sum_nr_running))
12485	return false;
12486
12487	break;
12488	}
12489
12490	return true;
12491	}
12492
12493	#ifdef CONFIG_NUMA_BALANCING
12494	static inline enum fbq_type fbq_classify_group(struct sg_lb_stats *sgs)
12495	{
12496	if (sgs->sum_h_nr_running > sgs->nr_numa_running)
12497	return regular;
12498	if (sgs->sum_h_nr_running > sgs->nr_preferred_running)
12499	return remote;
12500	return all;
12501	}
12502
12503	static inline enum fbq_type fbq_classify_rq(struct rq *rq)
12504	{
12505	if (rq->nr_running > rq->nr_numa_running)
12506	return regular;
12507	if (rq->nr_running > rq->nr_preferred_running)
12508	return remote;
12509	return all;
12510	}
12511	#else /* !CONFIG_NUMA_BALANCING: */
12512	static inline enum fbq_type fbq_classify_group(struct sg_lb_stats *sgs)
12513	{
12514	return all;
12515	}
12516
12517	static inline enum fbq_type fbq_classify_rq(struct rq *rq)
12518	{
12519	return regular;
12520	}
12521	#endif /* !CONFIG_NUMA_BALANCING */
12522
12523
12524	struct sg_lb_stats;
12525
12526	/*
12527	* task_running_on_cpu - return 1 if @p is running on @cpu.
12528	*/
12529
12530	static unsigned int task_running_on_cpu(int cpu, struct task_struct *p)
12531	{
12532	/* Task has no contribution or is new */
12533	if (cpu != task_cpu(p) \|\| !READ_ONCE(p->se.avg.last_update_time))
12534	return 0;
12535
12536	if (task_on_rq_queued(p))
12537	return 1;
12538
12539	return 0;
12540	}
12541
12542	/**
12543	* idle_cpu_without - would a given CPU be idle without p ?
12544	* @cpu: the processor on which idleness is tested.
12545	* @p: task which should be ignored.
12546	*
12547	* Return: 1 if the CPU would be idle. 0 otherwise.
12548	*/
12549	static int idle_cpu_without(int cpu, struct task_struct *p)
12550	{
12551	struct rq *rq = cpu_rq(cpu);
12552
12553	if (rq->curr != rq->idle && rq->curr != p)
12554	return 0;
12555
12556	/*
12557	* rq->nr_running can't be used but an updated version without the
12558	* impact of p on cpu must be used instead. The updated nr_running
12559	* be computed and tested before calling idle_cpu_without().
12560	*/
12561
12562	if (rq->ttwu_pending)
12563	return 0;
12564
12565	return 1;
12566	}
12567
12568	/*
12569	* update_sg_wakeup_stats - Update sched_group's statistics for wakeup.
12570	* @sd: The sched_domain level to look for idlest group.
12571	* @group: sched_group whose statistics are to be updated.
12572	* @sgs: variable to hold the statistics for this group.
12573	* @p: The task for which we look for the idlest group/CPU.
12574	*/
12575	static inline void update_sg_wakeup_stats(struct sched_domain *sd,
12576	struct sched_group *group,
12577	struct sg_lb_stats *sgs,
12578	struct task_struct *p)
12579	{
12580	int i, nr_running;
12581
12582	memset(sgs, 0, sizeof(*sgs));
12583
12584	/* Assume that task can't fit any CPU of the group */
12585	if (sd->flags & SD_ASYM_CPUCAPACITY)
12586	sgs->group_misfit_task_load = 1;
12587
12588	for_each_cpu_and(i, sched_group_span(group), p->cpus_ptr) {
12589	struct rq *rq = cpu_rq(i);
12590	unsigned int local;
12591
12592	sgs->group_load += cpu_load_without(rq, p);
12593	sgs->group_util += cpu_util_without(i, p);
12594	sgs->group_runnable += cpu_runnable_without(rq, p);
12595	local = task_running_on_cpu(i, p);
12596	sgs->sum_h_nr_running += rq->cfs.h_nr_runnable - local;
12597
12598	nr_running = rq->nr_running - local;
12599	sgs->sum_nr_running += nr_running;
12600
12601	/*
12602	* No need to call idle_cpu_without() if nr_running is not 0
12603	*/
12604	if (!nr_running && idle_cpu_without(i, p))
12605	sgs->idle_cpus++;
12606
12607	/* Check if task fits in the CPU */
12608	if (sd->flags & SD_ASYM_CPUCAPACITY &&
12609	sgs->group_misfit_task_load &&
12610	task_fits_cpu(p, i))
12611	sgs->group_misfit_task_load = 0;
12612
12613	}
12614
12615	sgs->group_capacity = group->sgc->capacity;
12616
12617	sgs->group_weight = group->group_weight;
12618
12619	sgs->group_type = group_classify(sd->imbalance_pct, group, sgs);
12620
12621	/*
12622	* Computing avg_load makes sense only when group is fully busy or
12623	* overloaded
12624	*/
12625	if (sgs->group_type == group_fully_busy \|\|
12626	sgs->group_type == group_overloaded)
12627	sgs->avg_load = (sgs->group_load * SCHED_CAPACITY_SCALE) /
12628	sgs->group_capacity;
12629	}
12630
12631	static bool update_pick_idlest(struct sched_group *idlest,
12632	struct sg_lb_stats *idlest_sgs,
12633	struct sched_group *group,
12634	struct sg_lb_stats *sgs)
12635	{
12636	if (sgs->group_type < idlest_sgs->group_type)
12637	return true;
12638
12639	if (sgs->group_type > idlest_sgs->group_type)
12640	return false;
12641
12642	/*
12643	* The candidate and the current idlest group are the same type of
12644	* group. Let check which one is the idlest according to the type.
12645	*/
12646
12647	switch (sgs->group_type) {
12648	case group_overloaded:
12649	case group_fully_busy:
12650	/* Select the group with lowest avg_load. */
12651	if (idlest_sgs->avg_load <= sgs->avg_load)
12652	return false;
12653	break;
12654
12655	case group_llc_balance:
12656	case group_imbalanced:
12657	case group_asym_packing:
12658	case group_smt_balance:
12659	/* Those types are not used in the slow wakeup path */
12660	return false;
12661
12662	case group_misfit_task:
12663	/* Select group with the highest max capacity */
12664	if (idlest->sgc->max_capacity >= group->sgc->max_capacity)
12665	return false;
12666	break;
12667
12668	case group_has_spare:
12669	/* Select group with most idle CPUs */
12670	if (idlest_sgs->idle_cpus > sgs->idle_cpus)
12671	return false;
12672
12673	/* Select group with lowest group_util */
12674	if (idlest_sgs->idle_cpus == sgs->idle_cpus &&
12675	idlest_sgs->group_util <= sgs->group_util)
12676	return false;
12677
12678	break;
12679	}
12680
12681	return true;
12682	}
12683
12684	/*
12685	* sched_balance_find_dst_group() finds and returns the least busy CPU group within the
12686	* domain.
12687	*
12688	* Assumes p is allowed on at least one CPU in sd.
12689	*/
12690	static struct sched_group *
12691	sched_balance_find_dst_group(struct sched_domain sd, struct task_struct p, int this_cpu)
12692	{
12693	struct sched_group idlest = NULL, local = NULL, *group = sd->groups;
12694	struct sg_lb_stats local_sgs, tmp_sgs;
12695	struct sg_lb_stats *sgs;
12696	unsigned long imbalance;
12697	struct sg_lb_stats idlest_sgs = {
12698	.avg_load = UINT_MAX,
12699	.group_type = group_overloaded,
12700	};
12701
12702	do {
12703	int local_group;
12704
12705	/* Skip over this group if it has no CPUs allowed */
12706	if (!cpumask_intersects(sched_group_span(group),
12707	p->cpus_ptr))
12708	continue;
12709
12710	/* Skip over this group if no cookie matched */
12711	if (!sched_group_cookie_match(cpu_rq(this_cpu), p, group))
12712	continue;
12713
12714	local_group = cpumask_test_cpu(this_cpu,
12715	sched_group_span(group));
12716
12717	if (local_group) {
12718	sgs = &local_sgs;
12719	local = group;
12720	} else {
12721	sgs = &tmp_sgs;
12722	}
12723
12724	update_sg_wakeup_stats(sd, group, sgs, p);
12725
12726	if (!local_group && update_pick_idlest(idlest, &idlest_sgs, group, sgs)) {
12727	idlest = group;
12728	idlest_sgs = *sgs;
12729	}
12730
12731	} while (group = group->next, group != sd->groups);
12732
12733
12734	/* There is no idlest group to push tasks to */
12735	if (!idlest)
12736	return NULL;
12737
12738	/* The local group has been skipped because of CPU affinity */
12739	if (!local)
12740	return idlest;
12741
12742	/*
12743	* If the local group is idler than the selected idlest group
12744	* don't try and push the task.
12745	*/
12746	if (local_sgs.group_type < idlest_sgs.group_type)
12747	return NULL;
12748
12749	/*
12750	* If the local group is busier than the selected idlest group
12751	* try and push the task.
12752	*/
12753	if (local_sgs.group_type > idlest_sgs.group_type)
12754	return idlest;
12755
12756	switch (local_sgs.group_type) {
12757	case group_overloaded:
12758	case group_fully_busy:
12759
12760	/* Calculate allowed imbalance based on load */
12761	imbalance = scale_load_down(NICE_0_LOAD) *
12762	(sd->imbalance_pct-100) / 100;
12763
12764	/*
12765	* When comparing groups across NUMA domains, it's possible for
12766	* the local domain to be very lightly loaded relative to the
12767	* remote domains but "imbalance" skews the comparison making
12768	* remote CPUs look much more favourable. When considering
12769	* cross-domain, add imbalance to the load on the remote node
12770	* and consider staying local.
12771	*/
12772
12773	if ((sd->flags & SD_NUMA) &&
12774	((idlest_sgs.avg_load + imbalance) >= local_sgs.avg_load))
12775	return NULL;
12776
12777	/*
12778	* If the local group is less loaded than the selected
12779	* idlest group don't try and push any tasks.
12780	*/
12781	if (idlest_sgs.avg_load >= (local_sgs.avg_load + imbalance))
12782	return NULL;
12783
12784	if (100 * local_sgs.avg_load <= sd->imbalance_pct * idlest_sgs.avg_load)
12785	return NULL;
12786	break;
12787
12788	case group_llc_balance:
12789	case group_imbalanced:
12790	case group_asym_packing:
12791	case group_smt_balance:
12792	/* Those type are not used in the slow wakeup path */
12793	return NULL;
12794
12795	case group_misfit_task:
12796	/* Select group with the highest max capacity */
12797	if (local->sgc->max_capacity >= idlest->sgc->max_capacity)
12798	return NULL;
12799	break;
12800
12801	case group_has_spare:
12802	#ifdef CONFIG_NUMA
12803	if (sd->flags & SD_NUMA) {
12804	int imb_numa_nr = sd->imb_numa_nr;
12805	#ifdef CONFIG_NUMA_BALANCING
12806	int idlest_cpu;
12807	/*
12808	* If there is spare capacity at NUMA, try to select
12809	* the preferred node
12810	*/
12811	if (cpu_to_node(this_cpu) == p->numa_preferred_nid)
12812	return NULL;
12813
12814	idlest_cpu = cpumask_first(sched_group_span(idlest));
12815	if (cpu_to_node(idlest_cpu) == p->numa_preferred_nid)
12816	return idlest;
12817	#endif /* CONFIG_NUMA_BALANCING */
12818	/*
12819	* Otherwise, keep the task close to the wakeup source
12820	* and improve locality if the number of running tasks
12821	* would remain below threshold where an imbalance is
12822	* allowed while accounting for the possibility the
12823	* task is pinned to a subset of CPUs. If there is a
12824	* real need of migration, periodic load balance will
12825	* take care of it.
12826	*/
12827	if (p->nr_cpus_allowed != NR_CPUS) {
12828	unsigned int w = cpumask_weight_and(p->cpus_ptr,
12829	sched_group_span(local));
12830	imb_numa_nr = min(w, sd->imb_numa_nr);
12831	}
12832
12833	imbalance = abs(local_sgs.idle_cpus - idlest_sgs.idle_cpus);
12834	if (!adjust_numa_imbalance(imbalance,
12835	local_sgs.sum_nr_running + 1,
12836	imb_numa_nr)) {
12837	return NULL;
12838	}
12839	}
12840	#endif /* CONFIG_NUMA */
12841
12842	/*
12843	* Select group with highest number of idle CPUs. We could also
12844	* compare the utilization which is more stable but it can end
12845	* up that the group has less spare capacity but finally more
12846	* idle CPUs which means more opportunity to run task.
12847	*/
12848	if (local_sgs.idle_cpus >= idlest_sgs.idle_cpus)
12849	return NULL;
12850	break;
12851	}
12852
12853	return idlest;
12854	}
12855
12856	static void update_idle_cpu_scan(struct lb_env *env,
12857	unsigned long sum_util)
12858	{
12859	struct sched_domain_shared *sd_share;
12860	struct sched_domain *sd = env->sd;
12861	int llc_weight, pct;
12862	u64 x, y, tmp;
12863	/*
12864	* Update the number of CPUs to scan in LLC domain, which could
12865	* be used as a hint in select_idle_cpu(). The update of sd_share
12866	* could be expensive because it is within a shared cache line.
12867	* So the write of this hint only occurs during periodic load
12868	* balancing, rather than CPU_NEWLY_IDLE, because the latter
12869	* can fire way more frequently than the former.
12870	*/
12871	if (!sched_feat(SIS_UTIL) \|\| env->idle == CPU_NEWLY_IDLE)
12872	return;
12873
12874	sd_share = sd->shared;
12875	if (!sd_share)
12876	return;
12877
12878	/*
12879	* The number of CPUs to search drops as sum_util increases, when
12880	* sum_util hits 85% or above, the scan stops.
12881	* The reason to choose 85% as the threshold is because this is the
12882	* imbalance_pct(117) when a LLC sched group is overloaded.
12883	*
12884	* let y = SCHED_CAPACITY_SCALE - p * x^2 [1]
12885	* and y'= y / SCHED_CAPACITY_SCALE
12886	*
12887	* x is the ratio of sum_util compared to the CPU capacity:
12888	* x = sum_util / (llc_weight * SCHED_CAPACITY_SCALE)
12889	* y' is the ratio of CPUs to be scanned in the LLC domain,
12890	* and the number of CPUs to scan is calculated by:
12891	*
12892	* nr_scan = llc_weight * y' [2]
12893	*
12894	* When x hits the threshold of overloaded, AKA, when
12895	* x = 100 / pct, y drops to 0. According to [1],
12896	* p should be SCHED_CAPACITY_SCALE * pct^2 / 10000
12897	*
12898	* Scale x by SCHED_CAPACITY_SCALE:
12899	* x' = sum_util / llc_weight; [3]
12900	*
12901	* and finally [1] becomes:
12902	* y = SCHED_CAPACITY_SCALE -
12903	* x'^2 * pct^2 / (10000 * SCHED_CAPACITY_SCALE) [4]
12904	*
12905	*/
12906	/* equation [3] */
12907	x = sum_util;
12908	llc_weight = sd->span_weight;
12909	do_div(x, llc_weight);
12910
12911	/* equation [4] */
12912	pct = sd->imbalance_pct;
12913	tmp = x * x * pct * pct;
12914	do_div(tmp, 10000 * SCHED_CAPACITY_SCALE);
12915	tmp = min_t(long, tmp, SCHED_CAPACITY_SCALE);
12916	y = SCHED_CAPACITY_SCALE - tmp;
12917
12918	/* equation [2] */
12919	y *= llc_weight;
12920	do_div(y, SCHED_CAPACITY_SCALE);
12921	if ((int)y != sd_share->nr_idle_scan)
12922	WRITE_ONCE(sd_share->nr_idle_scan, (int)y);
12923	}
12924
12925	/**
12926	* update_sd_lb_stats - Update sched_domain's statistics for load balancing.
12927	* @env: The load balancing environment.
12928	* @sds: variable to hold the statistics for this sched_domain.
12929	*/
12930
12931	static inline void update_sd_lb_stats(struct lb_env env, struct sd_lb_stats sds)
12932	{
12933	struct sched_group *sg = env->sd->groups;
12934	struct sg_lb_stats *local = &sds->local_stat;
12935	struct sg_lb_stats tmp_sgs;
12936	unsigned long sum_util = 0;
12937	bool sg_overloaded = 0, sg_overutilized = 0;
12938
12939	env->dst_core_idle = !sched_smt_active() \|\| is_core_idle(env->dst_cpu);
12940
12941	do {
12942	struct sg_lb_stats *sgs = &tmp_sgs;
12943	int local_group;
12944
12945	local_group = cpumask_test_cpu(env->dst_cpu, sched_group_span(sg));
12946	if (local_group) {
12947	sds->local = sg;
12948	sgs = local;
12949
12950	if (env->idle != CPU_NEWLY_IDLE \|\|
12951	time_after_eq(jiffies, sg->sgc->next_update))
12952	update_group_capacity(env->sd, env->dst_cpu);
12953	}
12954
12955	update_sg_lb_stats(env, sds, sg, sgs, &sg_overloaded);
12956
12957	if (!local_group && update_sd_pick_busiest(env, sds, sg, sgs)) {
12958	sds->busiest = sg;
12959	sds->busiest_stat = *sgs;
12960	}
12961
12962	sg_overutilized \|= sgs->group_overutilized;
12963
12964	/* Now, start updating sd_lb_stats */
12965	sds->total_load += sgs->group_load;
12966	sds->total_capacity += sgs->group_capacity;
12967
12968	sum_util += sgs->group_util;
12969	sg = sg->next;
12970	} while (sg != env->sd->groups);
12971
12972	/*
12973	* Indicate that the child domain of the busiest group prefers tasks
12974	* go to a child's sibling domains first. NB the flags of a sched group
12975	* are those of the child domain.
12976	*/
12977	if (sds->busiest)
12978	sds->prefer_sibling = !!(sds->busiest->flags & SD_PREFER_SIBLING);
12979
12980
12981	if (env->sd->flags & SD_NUMA)
12982	env->fbq_type = fbq_classify_group(&sds->busiest_stat);
12983
12984	if (!env->sd->parent) {
12985	/* update overload indicator if we are at root domain */
12986	set_rd_overloaded(env->dst_rq->rd, sg_overloaded);
12987
12988	/* Update over-utilization (tipping point, U >= 0) indicator */
12989	set_rd_overutilized(env->dst_rq->rd, sg_overutilized);
12990	} else if (sg_overutilized) {
12991	set_rd_overutilized(env->dst_rq->rd, sg_overutilized);
12992	}
12993
12994	update_idle_cpu_scan(env, sum_util);
12995	}
12996
12997	/**
12998	* calculate_imbalance - Calculate the amount of imbalance present within the
12999	* groups of a given sched_domain during load balance.
13000	* @env: load balance environment
13001	* @sds: statistics of the sched_domain whose imbalance is to be calculated.
13002	*/
13003	static inline void calculate_imbalance(struct lb_env env, struct sd_lb_stats sds)
13004	{
13005	struct sg_lb_stats local, busiest;
13006
13007	local = &sds->local_stat;
13008	busiest = &sds->busiest_stat;
13009
13010	if (busiest->group_type == group_misfit_task) {
13011	if (env->sd->flags & SD_ASYM_CPUCAPACITY) {
13012	/* Set imbalance to allow misfit tasks to be balanced. */
13013	env->migration_type = migrate_misfit;
13014	env->imbalance = 1;
13015	} else {
13016	/*
13017	* Set load imbalance to allow moving task from cpu
13018	* with reduced capacity.
13019	*/
13020	env->migration_type = migrate_load;
13021	env->imbalance = busiest->group_misfit_task_load;
13022	}
13023	return;
13024	}
13025
13026	if (busiest->group_type == group_asym_packing) {
13027	/*
13028	* In case of asym capacity, we will try to migrate all load to
13029	* the preferred CPU.
13030	*/
13031	env->migration_type = migrate_task;
13032	env->imbalance = busiest->sum_h_nr_running;
13033	return;
13034	}
13035
13036	if (busiest->group_type == group_smt_balance) {
13037	/* Reduce number of tasks sharing CPU capacity */
13038	env->migration_type = migrate_task;
13039	env->imbalance = 1;
13040	return;
13041	}
13042
13043	#ifdef CONFIG_SCHED_CACHE
13044	if (busiest->group_type == group_llc_balance) {
13045	/* Move a task that prefer local LLC */
13046	env->migration_type = migrate_llc_task;
13047	env->imbalance = 1;
13048	return;
13049	}
13050	#endif
13051
13052	if (busiest->group_type == group_imbalanced) {
13053	/*
13054	* In the group_imb case we cannot rely on group-wide averages
13055	* to ensure CPU-load equilibrium, try to move any task to fix
13056	* the imbalance. The next load balance will take care of
13057	* balancing back the system.
13058	*/
13059	env->migration_type = migrate_task;
13060	env->imbalance = 1;
13061	return;
13062	}
13063
13064	/*
13065	* Try to use spare capacity of local group without overloading it or
13066	* emptying busiest.
13067	*/
13068	if (local->group_type == group_has_spare) {
13069	if ((busiest->group_type > group_fully_busy) &&
13070	!(env->sd->flags & SD_SHARE_LLC)) {
13071	/*
13072	* If busiest is overloaded, try to fill spare
13073	* capacity. This might end up creating spare capacity
13074	* in busiest or busiest still being overloaded but
13075	* there is no simple way to directly compute the
13076	* amount of load to migrate in order to balance the
13077	* system.
13078	*/
13079	env->migration_type = migrate_util;
13080	env->imbalance = max(local->group_capacity, local->group_util) -
13081	local->group_util;
13082
13083	/*
13084	* In some cases, the group's utilization is max or even
13085	* higher than capacity because of migrations but the
13086	* local CPU is (newly) idle. There is at least one
13087	* waiting task in this overloaded busiest group. Let's
13088	* try to pull it.
13089	*/
13090	if (env->idle && env->imbalance == 0) {
13091	env->migration_type = migrate_task;
13092	env->imbalance = 1;
13093	}
13094
13095	return;
13096	}
13097
13098	if (busiest->group_weight == 1 \|\| sds->prefer_sibling) {
13099	/*
13100	* When prefer sibling, evenly spread running tasks on
13101	* groups.
13102	*/
13103	env->migration_type = migrate_task;
13104	env->imbalance = sibling_imbalance(env, sds, busiest, local);
13105	} else {
13106
13107	/*
13108	* If there is no overload, we just want to even the number of
13109	* idle CPUs.
13110	*/
13111	env->migration_type = migrate_task;
13112	env->imbalance = max_t(long, 0,
13113	(local->idle_cpus - busiest->idle_cpus));
13114	}
13115
13116	#ifdef CONFIG_NUMA
13117	/* Consider allowing a small imbalance between NUMA groups */
13118	if (env->sd->flags & SD_NUMA) {
13119	env->imbalance = adjust_numa_imbalance(env->imbalance,
13120	local->sum_nr_running + 1,
13121	env->sd->imb_numa_nr);
13122	}
13123	#endif
13124
13125	/* Number of tasks to move to restore balance */
13126	env->imbalance >>= 1;
13127
13128	return;
13129	}
13130
13131	/*
13132	* Local is fully busy but has to take more load to relieve the
13133	* busiest group
13134	*/
13135	if (local->group_type < group_overloaded) {
13136	/*
13137	* Local will become overloaded so the avg_load metrics are
13138	* finally needed.
13139	*/
13140
13141	local->avg_load = (local->group_load * SCHED_CAPACITY_SCALE) /
13142	local->group_capacity;
13143
13144	/*
13145	* If the local group is more loaded than the selected
13146	* busiest group don't try to pull any tasks.
13147	*/
13148	if (local->avg_load >= busiest->avg_load) {
13149	env->imbalance = 0;
13150	return;
13151	}
13152
13153	sds->avg_load = (sds->total_load * SCHED_CAPACITY_SCALE) /
13154	sds->total_capacity;
13155
13156	/*
13157	* If the local group is more loaded than the average system
13158	* load, don't try to pull any tasks.
13159	*/
13160	if (local->avg_load >= sds->avg_load) {
13161	env->imbalance = 0;
13162	return;
13163	}
13164
13165	}
13166
13167	/*
13168	* Both group are or will become overloaded and we're trying to get all
13169	* the CPUs to the average_load, so we don't want to push ourselves
13170	* above the average load, nor do we wish to reduce the max loaded CPU
13171	* below the average load. At the same time, we also don't want to
13172	* reduce the group load below the group capacity. Thus we look for
13173	* the minimum possible imbalance.
13174	*/
13175	env->migration_type = migrate_load;
13176	env->imbalance = min(
13177	(busiest->avg_load - sds->avg_load) * busiest->group_capacity,
13178	(sds->avg_load - local->avg_load) * local->group_capacity
13179	) / SCHED_CAPACITY_SCALE;
13180	}
13181
13182	/***** sched_balance_find_src_group() helpers end here *******************/
13183
13184	/*
13185	* Decision matrix according to the local and busiest group type:
13186	*
13187	* busiest \ local has_spare fully_busy misfit asym imbalanced overloaded
13188	* has_spare nr_idle balanced N/A N/A balanced balanced
13189	* fully_busy nr_idle nr_idle N/A N/A balanced balanced
13190	* misfit_task force N/A N/A N/A N/A N/A
13191	* asym_packing force force N/A N/A force force
13192	* imbalanced force force N/A N/A force force
13193	* overloaded force force N/A N/A force avg_load
13194	*
13195	* N/A : Not Applicable because already filtered while updating
13196	* statistics.
13197	* balanced : The system is balanced for these 2 groups.
13198	* force : Calculate the imbalance as load migration is probably needed.
13199	* avg_load : Only if imbalance is significant enough.
13200	* nr_idle : dst_cpu is not busy and the number of idle CPUs is quite
13201	* different in groups.
13202	*/
13203
13204	/**
13205	* sched_balance_find_src_group - Returns the busiest group within the sched_domain
13206	* if there is an imbalance.
13207	* @env: The load balancing environment.
13208	*
13209	* Also calculates the amount of runnable load which should be moved
13210	* to restore balance.
13211	*
13212	* Return: - The busiest group if imbalance exists.
13213	*/
13214	static struct sched_group sched_balance_find_src_group(struct lb_env env)
13215	{
13216	struct sg_lb_stats local, busiest;
13217	struct sd_lb_stats sds;
13218
13219	init_sd_lb_stats(&sds);
13220
13221	/*
13222	* Compute the various statistics relevant for load balancing at
13223	* this level.
13224	*/
13225	update_sd_lb_stats(env, &sds);
13226
13227	/* There is no busy sibling group to pull tasks from */
13228	if (!sds.busiest)
13229	goto out_balanced;
13230
13231	busiest = &sds.busiest_stat;
13232
13233	/* Misfit tasks should be dealt with regardless of the avg load */
13234	if (busiest->group_type == group_misfit_task)
13235	goto force_balance;
13236
13237	if (!is_rd_overutilized(env->dst_rq->rd) &&
13238	rcu_dereference_all(env->dst_rq->rd->pd))
13239	goto out_balanced;
13240
13241	/* ASYM feature bypasses nice load balance check */
13242	if (busiest->group_type == group_asym_packing)
13243	goto force_balance;
13244
13245	/*
13246	* If the busiest group is imbalanced the below checks don't
13247	* work because they assume all things are equal, which typically
13248	* isn't true due to cpus_ptr constraints and the like.
13249	*/
13250	if (busiest->group_type == group_imbalanced)
13251	goto force_balance;
13252
13253	local = &sds.local_stat;
13254	/*
13255	* If the local group is busier than the selected busiest group
13256	* don't try and pull any tasks.
13257	*/
13258	if (local->group_type > busiest->group_type)
13259	goto out_balanced;
13260
13261	/*
13262	* When groups are overloaded, use the avg_load to ensure fairness
13263	* between tasks.
13264	*/
13265	if (local->group_type == group_overloaded) {
13266	/*
13267	* If the local group is more loaded than the selected
13268	* busiest group don't try to pull any tasks.
13269	*/
13270	if (local->avg_load >= busiest->avg_load)
13271	goto out_balanced;
13272
13273	/* XXX broken for overlapping NUMA groups */
13274	sds.avg_load = (sds.total_load * SCHED_CAPACITY_SCALE) /
13275	sds.total_capacity;
13276
13277	/*
13278	* Don't pull any tasks if this group is already above the
13279	* domain average load.
13280	*/
13281	if (local->avg_load >= sds.avg_load)
13282	goto out_balanced;
13283
13284	/*
13285	* If the busiest group is more loaded, use imbalance_pct to be
13286	* conservative.
13287	*/
13288	if (100 * busiest->avg_load <=
13289	env->sd->imbalance_pct * local->avg_load)
13290	goto out_balanced;
13291	}
13292
13293	/*
13294	* Try to move all excess tasks to a sibling domain of the busiest
13295	* group's child domain.
13296	*/
13297	if (sds.prefer_sibling && local->group_type == group_has_spare &&
13298	(busiest->group_type == group_llc_balance \|\|
13299	sibling_imbalance(env, &sds, busiest, local) > 1))
13300	goto force_balance;
13301
13302	if (busiest->group_type != group_overloaded) {
13303	if (!env->idle) {
13304	/*
13305	* If the busiest group is not overloaded (and as a
13306	* result the local one too) but this CPU is already
13307	* busy, let another idle CPU try to pull task.
13308	*/
13309	goto out_balanced;
13310	}
13311
13312	if (busiest->group_type == group_smt_balance &&
13313	smt_vs_nonsmt_groups(sds.local, sds.busiest)) {
13314	/* Let non SMT CPU pull from SMT CPU sharing with sibling */
13315	goto force_balance;
13316	}
13317
13318	if (busiest->group_weight > 1 &&
13319	local->idle_cpus <= (busiest->idle_cpus + 1)) {
13320	/*
13321	* If the busiest group is not overloaded
13322	* and there is no imbalance between this and busiest
13323	* group wrt idle CPUs, it is balanced. The imbalance
13324	* becomes significant if the diff is greater than 1
13325	* otherwise we might end up to just move the imbalance
13326	* on another group. Of course this applies only if
13327	* there is more than 1 CPU per group.
13328	*/
13329	goto out_balanced;
13330	}
13331
13332	if (busiest->sum_h_nr_running == 1) {
13333	/*
13334	* busiest doesn't have any tasks waiting to run
13335	*/
13336	goto out_balanced;
13337	}
13338	}
13339
13340	force_balance:
13341	/* Looks like there is an imbalance. Compute it */
13342	calculate_imbalance(env, &sds);
13343	return env->imbalance ? sds.busiest : NULL;
13344
13345	out_balanced:
13346	env->imbalance = 0;
13347	return NULL;
13348	}
13349
13350	/*
13351	* sched_balance_find_src_rq - find the busiest runqueue among the CPUs in the group.
13352	*/
13353	static struct rq sched_balance_find_src_rq(struct lb_env env,
13354	struct sched_group *group)
13355	{
13356	struct rq busiest = NULL, rq;
13357	unsigned long busiest_util = 0, busiest_load = 0, busiest_capacity = 1;
13358	unsigned int __maybe_unused busiest_pref_llc = 0;
13359	struct sched_domain __maybe_unused *sd_tmp;
13360	unsigned int busiest_nr = 0;
13361	int __maybe_unused dst_llc;
13362	int i;
13363
13364	for_each_cpu_and(i, sched_group_span(group), env->cpus) {
13365	unsigned long capacity, load, util;
13366	unsigned int nr_running;
13367	enum fbq_type rt;
13368
13369	rq = cpu_rq(i);
13370	rt = fbq_classify_rq(rq);
13371
13372	/*
13373	* We classify groups/runqueues into three groups:
13374	* - regular: there are !numa tasks
13375	* - remote: there are numa tasks that run on the 'wrong' node
13376	* - all: there is no distinction
13377	*
13378	* In order to avoid migrating ideally placed numa tasks,
13379	* ignore those when there's better options.
13380	*
13381	* If we ignore the actual busiest queue to migrate another
13382	* task, the next balance pass can still reduce the busiest
13383	* queue by moving tasks around inside the node.
13384	*
13385	* If we cannot move enough load due to this classification
13386	* the next pass will adjust the group classification and
13387	* allow migration of more tasks.
13388	*
13389	* Both cases only affect the total convergence complexity.
13390	*/
13391	if (rt > env->fbq_type)
13392	continue;
13393
13394	nr_running = rq->cfs.h_nr_runnable;
13395	if (!nr_running)
13396	continue;
13397
13398	capacity = capacity_of(i);
13399
13400	/*
13401	* For ASYM_CPUCAPACITY domains, don't pick a CPU that could
13402	* eventually lead to active_balancing high->low capacity.
13403	* Higher per-CPU capacity is considered better than balancing
13404	* average load.
13405	*/
13406	if (env->sd->flags & SD_ASYM_CPUCAPACITY &&
13407	nr_running == 1) {
13408	bool cluster_equal_cap = static_branch_unlikely(&sched_cluster_active) &&
13409	(get_actual_cpu_capacity(env->dst_cpu) ==
13410	get_actual_cpu_capacity(i));
13411	bool smt_degraded_cap = sched_smt_active() && !is_core_idle(i);
13412
13413	/*
13414	* Busy SMT siblings reduce the capacity of CPU @i. Do
13415	* not skip it in this case.
13416	*
13417	* CONFIG_SCHED_CLUSTER requires balancing load across
13418	* clusters of identical capacity, accounting for
13419	* hardware and cpufreq pressure.
13420	*/
13421	if (!smt_degraded_cap && !cluster_equal_cap &&
13422	!capacity_greater(capacity_of(env->dst_cpu), capacity))
13423	continue;
13424	}
13425
13426	/*
13427	* Make sure we only pull tasks from a CPU of lower priority
13428	* when balancing between SMT siblings.
13429	*
13430	* If balancing between cores, let lower priority CPUs help
13431	* SMT cores with more than one busy sibling.
13432	*/
13433	if (sched_asym(env->sd, i, env->dst_cpu) && nr_running == 1)
13434	continue;
13435
13436	switch (env->migration_type) {
13437	case migrate_load:
13438	/*
13439	* When comparing with load imbalance, use cpu_load()
13440	* which is not scaled with the CPU capacity.
13441	*/
13442	load = cpu_load(rq);
13443
13444	if (nr_running == 1 && load > env->imbalance &&
13445	!check_cpu_capacity(rq, env->sd))
13446	break;
13447
13448	/*
13449	* For the load comparisons with the other CPUs,
13450	* consider the cpu_load() scaled with the CPU
13451	* capacity, so that the load can be moved away
13452	* from the CPU that is potentially running at a
13453	* lower capacity.
13454	*
13455	* Thus we're looking for max(load_i / capacity_i),
13456	* crosswise multiplication to rid ourselves of the
13457	* division works out to:
13458	* load_i * capacity_j > load_j * capacity_i;
13459	* where j is our previous maximum.
13460	*/
13461	if (load * busiest_capacity > busiest_load * capacity) {
13462	busiest_load = load;
13463	busiest_capacity = capacity;
13464	busiest = rq;
13465	}
13466	break;
13467
13468	case migrate_util:
13469	util = cpu_util_cfs_boost(i);
13470
13471	/*
13472	* Don't try to pull utilization from a CPU with one
13473	* running task. Whatever its utilization, we will fail
13474	* detach the task.
13475	*/
13476	if (nr_running <= 1)
13477	continue;
13478
13479	if (busiest_util < util) {
13480	busiest_util = util;
13481	busiest = rq;
13482	}
13483	break;
13484
13485	case migrate_task:
13486	if (busiest_nr < nr_running) {
13487	busiest_nr = nr_running;
13488	busiest = rq;
13489	}
13490	break;
13491
13492	case migrate_misfit:
13493	/*
13494	* For ASYM_CPUCAPACITY domains with misfit tasks we
13495	* simply seek the "biggest" misfit task.
13496	*/
13497	if (rq->misfit_task_load > busiest_load) {
13498	busiest_load = rq->misfit_task_load;
13499	busiest = rq;
13500	}
13501
13502	break;
13503
13504	case migrate_llc_task:
13505	#ifdef CONFIG_SCHED_CACHE
13506	sd_tmp = rcu_dereference_all(rq->sd);
13507	dst_llc = llc_id(env->dst_cpu);
13508
13509	if (sd_tmp && (unsigned)dst_llc < sd_tmp->llc_max) {
13510	unsigned int this_pref_llc =
13511	sd_tmp->llc_counts[dst_llc];
13512
13513	if (busiest_pref_llc < this_pref_llc) {
13514	busiest_pref_llc = this_pref_llc;
13515	busiest = rq;
13516	}
13517	}
13518	#endif
13519	break;
13520
13521	}
13522	}
13523
13524	return busiest;
13525	}
13526
13527	/*
13528	* Max backoff if we encounter pinned tasks. Pretty arbitrary value, but
13529	* so long as it is large enough.
13530	*/
13531	#define MAX_PINNED_INTERVAL 512
13532
13533	static inline bool
13534	asym_active_balance(struct lb_env *env)
13535	{
13536	/*
13537	* ASYM_PACKING needs to force migrate tasks from busy but lower
13538	* priority CPUs in order to pack all tasks in the highest priority
13539	* CPUs. When done between cores, do it only if the whole core if the
13540	* whole core is idle.
13541	*
13542	* If @env::src_cpu is an SMT core with busy siblings, let
13543	* the lower priority @env::dst_cpu help it. Do not follow
13544	* CPU priority.
13545	*/
13546	return env->idle && sched_use_asym_prio(env->sd, env->dst_cpu) &&
13547	(sched_asym_prefer(env->dst_cpu, env->src_cpu) \|\|
13548	!sched_use_asym_prio(env->sd, env->src_cpu));
13549	}
13550
13551	static inline bool
13552	imbalanced_active_balance(struct lb_env *env)
13553	{
13554	struct sched_domain *sd = env->sd;
13555
13556	/*
13557	* The imbalanced case includes the case of pinned tasks preventing a fair
13558	* distribution of the load on the system but also the even distribution of the
13559	* threads on a system with spare capacity
13560	*/
13561	if ((env->migration_type == migrate_task) &&
13562	(sd->nr_balance_failed > sd->cache_nice_tries+2))
13563	return 1;
13564
13565	return 0;
13566	}
13567
13568	static int need_active_balance(struct lb_env *env)
13569	{
13570	struct sched_domain *sd = env->sd;
13571
13572	if (alb_break_llc(env))
13573	return 0;
13574
13575	if (asym_active_balance(env))
13576	return 1;
13577
13578	if (imbalanced_active_balance(env))
13579	return 1;
13580
13581	/*
13582	* The dst_cpu is idle and the src_cpu CPU has only 1 CFS task.
13583	* It's worth migrating the task if the src_cpu's capacity is reduced
13584	* because of other sched_class or IRQs if more capacity stays
13585	* available on dst_cpu.
13586	*/
13587	if (env->idle &&
13588	(env->src_rq->cfs.h_nr_runnable == 1)) {
13589	if ((check_cpu_capacity(env->src_rq, sd)) &&
13590	(capacity_of(env->src_cpu)sd->imbalance_pct < capacity_of(env->dst_cpu)100))
13591	return 1;
13592	}
13593
13594	if (env->migration_type == migrate_misfit \|\|
13595	env->migration_type == migrate_llc_task)
13596	return 1;
13597
13598	return 0;
13599	}
13600
13601	static int active_load_balance_cpu_stop(void *data);
13602	static int active_load_balance_llc_cpu_stop(void *data);
13603
13604	/*
13605	* migration_type is checked elsewhere to decide migration policy, so
13606	* it shouldn't be repurposed just to flag an LLC-directed active
13607	* balance across the stopper. Pick the callback here instead.
13608	*/
13609	static inline cpu_stop_fn_t alb_stop_fn(struct lb_env *env)
13610	{
13611	if (env->migration_type == migrate_llc_task)
13612	return active_load_balance_llc_cpu_stop;
13613
13614	return active_load_balance_cpu_stop;
13615	}
13616
13617	static int should_we_balance(struct lb_env *env)
13618	{
13619	struct cpumask *swb_cpus = this_cpu_cpumask_var_ptr(should_we_balance_tmpmask);
13620	struct sched_group *sg = env->sd->groups;
13621	int cpu, idle_smt = -1;
13622
13623	/*
13624	* Ensure the balancing environment is consistent; can happen
13625	* when the softirq triggers 'during' hotplug.
13626	*/
13627	if (!cpumask_test_cpu(env->dst_cpu, env->cpus))
13628	return 0;
13629
13630	/*
13631	* In the newly idle case, we will allow all the CPUs
13632	* to do the newly idle load balance.
13633	*
13634	* However, we bail out if we already have tasks or a wakeup pending,
13635	* to optimize wakeup latency.
13636	*/
13637	if (env->idle == CPU_NEWLY_IDLE) {
13638	if (env->dst_rq->nr_running > 0 \|\| env->dst_rq->ttwu_pending)
13639	return 0;
13640	return 1;
13641	}
13642
13643	cpumask_copy(swb_cpus, group_balance_mask(sg));
13644	/* Try to find first idle CPU */
13645	for_each_cpu_and(cpu, swb_cpus, env->cpus) {
13646	if (!idle_cpu(cpu))
13647	continue;
13648
13649	/*
13650	* Don't balance to idle SMT in busy core right away when
13651	* balancing cores, but remember the first idle SMT CPU for
13652	* later consideration. Find CPU on an idle core first.
13653	*/
13654	if (sched_smt_active() &&
13655	!(env->sd->flags & SD_SHARE_CPUCAPACITY) &&
13656	!is_core_idle(cpu)) {
13657	if (idle_smt == -1)
13658	idle_smt = cpu;
13659	/*
13660	* If the core is not idle, and first SMT sibling which is
13661	* idle has been found, then its not needed to check other
13662	* SMT siblings for idleness:
13663	*/
13664	cpumask_andnot(swb_cpus, swb_cpus, cpu_smt_mask(cpu));
13665	continue;
13666	}
13667
13668	/*
13669	* Are we the first idle core in a non-SMT domain or higher,
13670	* or the first idle CPU in a SMT domain?
13671	*/
13672	return cpu == env->dst_cpu;
13673	}
13674
13675	/* Are we the first idle CPU with busy siblings? */
13676	if (idle_smt != -1)
13677	return idle_smt == env->dst_cpu;
13678
13679	/* Are we the first CPU of this group ? */
13680	return group_balance_cpu(sg) == env->dst_cpu;
13681	}
13682
13683	static void update_lb_imbalance_stat(struct lb_env env, struct sched_domain sd,
13684	enum cpu_idle_type idle)
13685	{
13686	if (!schedstat_enabled())
13687	return;
13688
13689	switch (env->migration_type) {
13690	case migrate_load:
13691	__schedstat_add(sd->lb_imbalance_load[idle], env->imbalance);
13692	break;
13693	case migrate_util:
13694	__schedstat_add(sd->lb_imbalance_util[idle], env->imbalance);
13695	break;
13696	case migrate_task:
13697	__schedstat_add(sd->lb_imbalance_task[idle], env->imbalance);
13698	break;
13699	case migrate_misfit:
13700	__schedstat_add(sd->lb_imbalance_misfit[idle], env->imbalance);
13701	break;
13702	case migrate_llc_task:
13703	break;
13704	}
13705	}
13706
13707	/*
13708	* This flag serializes load-balancing passes over large domains
13709	* (above the NODE topology level) - only one load-balancing instance
13710	* may run at a time, to reduce overhead on very large systems with
13711	* lots of CPUs and large NUMA distances.
13712	*
13713	* - Note that load-balancing passes triggered while another one
13714	* is executing are skipped and not re-tried.
13715	*
13716	* - Also note that this does not serialize rebalance_domains()
13717	* execution, as non-SD_SERIALIZE domains will still be
13718	* load-balanced in parallel.
13719	*/
13720	static atomic_t sched_balance_running = ATOMIC_INIT(0);
13721
13722	/*
13723	* Check this_cpu to ensure it is balanced within domain. Attempt to move
13724	* tasks if there is an imbalance.
13725	*/
13726	static int sched_balance_rq(int this_cpu, struct rq *this_rq,
13727	struct sched_domain *sd, enum cpu_idle_type idle,
13728	int *continue_balancing)
13729	{
13730	int ld_moved, cur_ld_moved, active_balance = 0;
13731	struct sched_domain *sd_parent = sd->parent;
13732	struct sched_group *group;
13733	struct rq *busiest;
13734	struct rq_flags rf;
13735	struct cpumask *cpus = this_cpu_cpumask_var_ptr(load_balance_mask);
13736	struct lb_env env = {
13737	.sd = sd,
13738	.dst_cpu = this_cpu,
13739	.dst_rq = this_rq,
13740	.dst_grpmask = group_balance_mask(sd->groups),
13741	.idle = idle,
13742	.loop_break = SCHED_NR_MIGRATE_BREAK,
13743	.cpus = cpus,
13744	.fbq_type = all,
13745	.tasks = LIST_HEAD_INIT(env.tasks),
13746	};
13747	bool need_unlock = false;
13748
13749	cpumask_and(cpus, sched_domain_span(sd), cpu_preferred_mask);
13750
13751	schedstat_inc(sd->lb_count[idle]);
13752
13753	redo:
13754	if (!should_we_balance(&env)) {
13755	*continue_balancing = 0;
13756	goto out_balanced;
13757	}
13758
13759	if (!need_unlock && (sd->flags & SD_SERIALIZE)) {
13760	int zero = 0;
13761	if (!atomic_try_cmpxchg_acquire(&sched_balance_running, &zero, 1))
13762	goto out_balanced;
13763
13764	need_unlock = true;
13765	}
13766
13767	group = sched_balance_find_src_group(&env);
13768	if (!group) {
13769	schedstat_inc(sd->lb_nobusyg[idle]);
13770	goto out_balanced;
13771	}
13772
13773	busiest = sched_balance_find_src_rq(&env, group);
13774	if (!busiest) {
13775	schedstat_inc(sd->lb_nobusyq[idle]);
13776	goto out_balanced;
13777	}
13778
13779	WARN_ON_ONCE(busiest == env.dst_rq);
13780
13781	update_lb_imbalance_stat(&env, sd, idle);
13782
13783	env.src_cpu = busiest->cpu;
13784	env.src_rq = busiest;
13785
13786	ld_moved = 0;
13787	/* Clear this flag as soon as we find a pullable task */
13788	env.flags \|= LBF_ALL_PINNED;
13789	if (busiest->nr_running > 1) {
13790	/*
13791	* Attempt to move tasks. If sched_balance_find_src_group has found
13792	* an imbalance but busiest->nr_running <= 1, the group is
13793	* still unbalanced. ld_moved simply stays zero, so it is
13794	* correctly treated as an imbalance.
13795	*/
13796	env.loop_max = min(sysctl_sched_nr_migrate, busiest->nr_running);
13797
13798	more_balance:
13799	rq_lock_irqsave(busiest, &rf);
13800	update_rq_clock(busiest);
13801
13802	/*
13803	* cur_ld_moved - load moved in current iteration
13804	* ld_moved - cumulative load moved across iterations
13805	*/
13806	cur_ld_moved = detach_tasks(&env);
13807
13808	/*
13809	* We've detached some tasks from busiest_rq. Every
13810	* task is masked "TASK_ON_RQ_MIGRATING", so we can safely
13811	* unlock busiest->lock, and we are able to be sure
13812	* that nobody can manipulate the tasks in parallel.
13813	* See task_rq_lock() family for the details.
13814	*/
13815
13816	rq_unlock(busiest, &rf);
13817
13818	if (cur_ld_moved) {
13819	attach_tasks(&env);
13820	ld_moved += cur_ld_moved;
13821	}
13822
13823	local_irq_restore(rf.flags);
13824
13825	if (env.flags & LBF_NEED_BREAK) {
13826	env.flags &= ~LBF_NEED_BREAK;
13827	goto more_balance;
13828	}
13829
13830	/*
13831	* Revisit (affine) tasks on src_cpu that couldn't be moved to
13832	* us and move them to an alternate dst_cpu in our sched_group
13833	* where they can run. The upper limit on how many times we
13834	* iterate on same src_cpu is dependent on number of CPUs in our
13835	* sched_group.
13836	*
13837	* This changes load balance semantics a bit on who can move
13838	* load to a given_cpu. In addition to the given_cpu itself
13839	* (or a ilb_cpu acting on its behalf where given_cpu is
13840	* nohz-idle), we now have balance_cpu in a position to move
13841	* load to given_cpu. In rare situations, this may cause
13842	* conflicts (balance_cpu and given_cpu/ilb_cpu deciding
13843	* _independently_ and at _same_ time to move some load to
13844	* given_cpu) causing excess load to be moved to given_cpu.
13845	* This however should not happen so much in practice and
13846	* moreover subsequent load balance cycles should correct the
13847	* excess load moved.
13848	*/
13849	if ((env.flags & LBF_DST_PINNED) && env.imbalance > 0) {
13850
13851	/* Prevent to re-select dst_cpu via env's CPUs */
13852	__cpumask_clear_cpu(env.dst_cpu, env.cpus);
13853
13854	env.dst_rq = cpu_rq(env.new_dst_cpu);
13855	env.dst_cpu = env.new_dst_cpu;
13856	env.flags &= ~LBF_DST_PINNED;
13857	env.loop = 0;
13858	env.loop_break = SCHED_NR_MIGRATE_BREAK;
13859
13860	/*
13861	* Go back to "more_balance" rather than "redo" since we
13862	* need to continue with same src_cpu.
13863	*/
13864	goto more_balance;
13865	}
13866
13867	/*
13868	* We failed to reach balance because of affinity.
13869	*/
13870	if (sd_parent) {
13871	int *group_imbalance = &sd_parent->groups->sgc->imbalance;
13872
13873	if ((env.flags & LBF_SOME_PINNED) && env.imbalance > 0)
13874	*group_imbalance = 1;
13875	}
13876
13877	/* All tasks on this runqueue were pinned by CPU affinity */
13878	if (unlikely(env.flags & LBF_ALL_PINNED)) {
13879	__cpumask_clear_cpu(cpu_of(busiest), cpus);
13880	/*
13881	* Attempting to continue load balancing at the current
13882	* sched_domain level only makes sense if there are
13883	* active CPUs remaining as possible busiest CPUs to
13884	* pull load from which are not contained within the
13885	* destination group that is receiving any migrated
13886	* load.
13887	*/
13888	if (!cpumask_subset(cpus, env.dst_grpmask)) {
13889	env.loop = 0;
13890	env.loop_break = SCHED_NR_MIGRATE_BREAK;
13891	goto redo;
13892	}
13893	goto out_all_pinned;
13894	}
13895	}
13896
13897	if (ld_moved) {
13898	sd->nr_balance_failed = 0;
13899	goto out_unbalanced;
13900	}
13901
13902	schedstat_inc(sd->lb_failed[idle]);
13903	/*
13904	* Increment the failure counter only on periodic balance.
13905	* We do not want newidle balance, which can be very
13906	* frequent, pollute the failure counter causing
13907	* excessive cache_hot migrations and active balances.
13908	*
13909	* Similarly for migration_misfit which is not related to
13910	* load/util migration, don't pollute nr_balance_failed.
13911	*
13912	* The same for cache aware scheduling's allowance for
13913	* load imbalance. If regular load balance does not
13914	* migrate task due to LLC locality, it is a expected
13915	* behavior and don't pollute nr_balance_failed.
13916	* See can_migrate_task().
13917	*/
13918	if (idle != CPU_NEWLY_IDLE &&
13919	env.migration_type != migrate_misfit &&
13920	!(env.flags & LBF_LLC_PINNED))
13921	sd->nr_balance_failed++;
13922
13923	if (!need_active_balance(&env))
13924	goto out_unbalanced;
13925
13926	scoped_guard (raw_spin_rq_lock_irqsave, busiest) {
13927	/*
13928	* Don't kick the active_load_balance_cpu_stop,
13929	* if the curr task on busiest CPU can't be
13930	* moved to this_cpu:
13931	*/
13932	if (!cpumask_test_cpu(this_cpu, busiest->curr->cpus_ptr))
13933	goto out_one_pinned;
13934
13935	/* Record that we found at least one task that could run on this_cpu */
13936	env.flags &= ~LBF_ALL_PINNED;
13937
13938	/*
13939	* ->active_balance synchronizes accesses to
13940	* ->active_balance_work. Once set, it's cleared
13941	* only after active load balance is finished.
13942	*/
13943	if (busiest->active_balance)
13944	goto out_unbalanced;
13945
13946	/*
13947	* @busiest dropped its rq_lock in the middle of
13948	* scheduling out its ->curr task (->on_rq := 0), no
13949	* need to forcefully punt it away with active balance.
13950	*/
13951	if (!busiest->curr->on_rq)
13952	goto out_unbalanced;
13953
13954	busiest->active_balance = 1;
13955	busiest->push_cpu = this_cpu;
13956	active_balance = 1;
13957	preempt_disable();
13958	}
13959	if (active_balance) {
13960	stop_one_cpu_nowait(cpu_of(busiest),
13961	alb_stop_fn(&env), busiest,
13962	&busiest->active_balance_work);
13963	}
13964	preempt_enable();
13965
13966	out_unbalanced:
13967	/* We were unbalanced, so reset the balancing interval */
13968	sd->balance_interval = sd->min_interval;
13969	goto out;
13970
13971	out_balanced:
13972	/*
13973	* We reach balance although we may have faced some affinity
13974	* constraints. Clear the imbalance flag only if other tasks got
13975	* a chance to move and fix the imbalance.
13976	*/
13977	if (sd_parent && !(env.flags & LBF_ALL_PINNED)) {
13978	int *group_imbalance = &sd_parent->groups->sgc->imbalance;
13979
13980	if (*group_imbalance)
13981	*group_imbalance = 0;
13982	}
13983
13984	out_all_pinned:
13985	/*
13986	* We reach balance because all tasks are pinned at this level so
13987	* we can't migrate them. Let the imbalance flag set so parent level
13988	* can try to migrate them.
13989	*/
13990	schedstat_inc(sd->lb_balanced[idle]);
13991
13992	sd->nr_balance_failed = 0;
13993
13994	out_one_pinned:
13995	ld_moved = 0;
13996
13997	/*
13998	* sched_balance_newidle() disregards balance intervals, so we could
13999	* repeatedly reach this code, which would lead to balance_interval
14000	* skyrocketing in a short amount of time. Skip the balance_interval
14001	* increase logic to avoid that.
14002	*
14003	* Similarly misfit migration which is not necessarily an indication of
14004	* the system being busy and requires lb to backoff to let it settle
14005	* down.
14006	*/
14007	if (env.idle == CPU_NEWLY_IDLE \|\|
14008	env.migration_type == migrate_misfit)
14009	goto out;
14010
14011	/* tune up the balancing interval */
14012	if ((env.flags & LBF_ALL_PINNED &&
14013	sd->balance_interval < MAX_PINNED_INTERVAL) \|\|
14014	sd->balance_interval < sd->max_interval)
14015	sd->balance_interval *= 2;
14016	out:
14017	if (need_unlock)
14018	atomic_set_release(&sched_balance_running, 0);
14019
14020	return ld_moved;
14021	}
14022
14023	static inline unsigned long
14024	get_sd_balance_interval(struct sched_domain *sd, int cpu_busy)
14025	{
14026	unsigned long interval = sd->balance_interval;
14027
14028	if (cpu_busy)
14029	interval *= sd->busy_factor;
14030
14031	/* scale ms to jiffies */
14032	interval = msecs_to_jiffies(interval);
14033
14034	/*
14035	* Reduce likelihood of busy balancing at higher domains racing with
14036	* balancing at lower domains by preventing their balancing periods
14037	* from being multiples of each other.
14038	*/
14039	if (cpu_busy)
14040	interval -= 1;
14041
14042	interval = clamp(interval, 1UL, max_load_balance_interval);
14043
14044	return interval;
14045	}
14046
14047	static inline void
14048	update_next_balance(struct sched_domain sd, unsigned long next_balance)
14049	{
14050	unsigned long interval, next;
14051
14052	/* used by idle balance, so cpu_busy = 0 */
14053	interval = get_sd_balance_interval(sd, 0);
14054	next = sd->last_balance + interval;
14055
14056	if (time_after(*next_balance, next))
14057	*next_balance = next;
14058	}
14059
14060	/*
14061	* active_load_balance_cpu_stop is run by the CPU stopper. It pushes
14062	* running tasks off the busiest CPU onto idle CPUs. It requires at
14063	* least 1 task to be running on each physical CPU where possible, and
14064	* avoids physical / logical imbalances.
14065	*/
14066	static int __active_load_balance_cpu_stop(void *data, unsigned int lb_flags)
14067	{
14068	struct rq *busiest_rq = data;
14069	int busiest_cpu = cpu_of(busiest_rq);
14070	int target_cpu = busiest_rq->push_cpu;
14071	struct rq *target_rq = cpu_rq(target_cpu);
14072	struct sched_domain *sd;
14073	struct task_struct *p = NULL;
14074	struct rq_flags rf;
14075
14076	rq_lock_irq(busiest_rq, &rf);
14077	/*
14078	* Between queueing the stop-work and running it is a hole in which
14079	* CPUs can become inactive. We should not move tasks from or to
14080	* inactive CPUs.
14081	*/
14082	if (!cpu_active(busiest_cpu) \|\| !cpu_active(target_cpu))
14083	goto out_unlock;
14084
14085	/* Make sure the requested CPU hasn't gone down in the meantime: */
14086	if (unlikely(busiest_cpu != smp_processor_id() \|\|
14087	!busiest_rq->active_balance))
14088	goto out_unlock;
14089
14090	/* Is there any task to move? */
14091	if (busiest_rq->nr_running <= 1)
14092	goto out_unlock;
14093
14094	/*
14095	* This condition is "impossible", if it occurs
14096	* we need to fix it. Originally reported by
14097	* Bjorn Helgaas on a 128-CPU setup.
14098	*/
14099	WARN_ON_ONCE(busiest_rq == target_rq);
14100
14101	/* Search for an sd spanning us and the target CPU. */
14102	rcu_read_lock();
14103	for_each_domain(target_cpu, sd) {
14104	if (cpumask_test_cpu(busiest_cpu, sched_domain_span(sd)))
14105	break;
14106	}
14107
14108	if (likely(sd)) {
14109	struct lb_env env = {
14110	.sd = sd,
14111	.dst_cpu = target_cpu,
14112	.dst_rq = target_rq,
14113	.src_cpu = busiest_rq->cpu,
14114	.src_rq = busiest_rq,
14115	.idle = CPU_IDLE,
14116	.flags = LBF_ACTIVE_LB \| lb_flags,
14117	};
14118
14119	schedstat_inc(sd->alb_count);
14120	update_rq_clock(busiest_rq);
14121
14122	p = detach_one_task(&env);
14123	if (p) {
14124	schedstat_inc(sd->alb_pushed);
14125	/* Active balancing done, reset the failure counter. */
14126	sd->nr_balance_failed = 0;
14127	} else {
14128	schedstat_inc(sd->alb_failed);
14129	}
14130	}
14131	rcu_read_unlock();
14132	out_unlock:
14133	busiest_rq->active_balance = 0;
14134	rq_unlock(busiest_rq, &rf);
14135
14136	if (p)
14137	attach_one_task(target_rq, p);
14138
14139	local_irq_enable();
14140
14141	return 0;
14142	}
14143
14144	static int active_load_balance_cpu_stop(void *data)
14145	{
14146	return __active_load_balance_cpu_stop(data, 0);
14147	}
14148
14149	static int active_load_balance_llc_cpu_stop(void *data)
14150	{
14151	return __active_load_balance_cpu_stop(data, LBF_ACTIVE_LB_LLC);
14152	}
14153
14154	/*
14155	* Scale the max sched_balance_rq interval with the number of CPUs in the system.
14156	* This trades load-balance latency on larger machines for less cross talk.
14157	*/
14158	void update_max_interval(void)
14159	{
14160	max_load_balance_interval = HZ*num_online_cpus()/10;
14161	}
14162
14163	static inline void update_newidle_stats(struct sched_domain *sd, unsigned int success)
14164	{
14165	sd->newidle_call++;
14166	sd->newidle_success += success;
14167
14168	if (sd->newidle_call >= 1024) {
14169	u64 now = sched_clock();
14170	s64 delta = now - sd->newidle_stamp;
14171	sd->newidle_stamp = now;
14172	int ratio = 0;
14173
14174	if (delta < 0)
14175	delta = 0;
14176
14177	if (sched_feat(NI_RATE)) {
14178	/*
14179	* ratio delta freq
14180	*
14181	* 1024 - 4 s - 128 Hz
14182	* 512 - 2 s - 256 Hz
14183	* 256 - 1 s - 512 Hz
14184	* 128 - .5 s - 1024 Hz
14185	* 64 - .25 s - 2048 Hz
14186	*/
14187	ratio = delta >> 22;
14188	}
14189
14190	ratio += sd->newidle_success;
14191
14192	sd->newidle_ratio = min(1024, ratio);
14193	sd->newidle_call /= 2;
14194	sd->newidle_success /= 2;
14195	}
14196	}
14197
14198	static inline bool
14199	update_newidle_cost(struct sched_domain *sd, u64 cost, unsigned int success)
14200	{
14201	unsigned long next_decay = sd->last_decay_max_lb_cost + HZ;
14202	unsigned long now = jiffies;
14203
14204	if (cost)
14205	update_newidle_stats(sd, success);
14206
14207	if (cost > sd->max_newidle_lb_cost) {
14208	/*
14209	* Track max cost of a domain to make sure to not delay the
14210	* next wakeup on the CPU.
14211	*/
14212	sd->max_newidle_lb_cost = cost;
14213	sd->last_decay_max_lb_cost = now;
14214
14215	} else if (time_after(now, next_decay)) {
14216	/*
14217	* Decay the newidle max times by ~1% per second to ensure that
14218	* it is not outdated and the current max cost is actually
14219	* shorter.
14220	*/
14221	sd->max_newidle_lb_cost = (sd->max_newidle_lb_cost * 253) / 256;
14222	sd->last_decay_max_lb_cost = now;
14223	return true;
14224	}
14225
14226	return false;
14227	}
14228
14229	/*
14230	* It checks each scheduling domain to see if it is due to be balanced,
14231	* and initiates a balancing operation if so.
14232	*
14233	* Balancing parameters are set up in init_sched_domains.
14234	*/
14235	static void sched_balance_domains(struct rq *rq, enum cpu_idle_type idle)
14236	{
14237	int continue_balancing = 1;
14238	int cpu = rq->cpu;
14239	int busy = idle != CPU_IDLE && !sched_idle_rq(rq);
14240	unsigned long interval;
14241	struct sched_domain *sd;
14242	/* Earliest time when we have to do rebalance again */
14243	unsigned long next_balance = jiffies + 60*HZ;
14244	int update_next_balance = 0;
14245	int need_decay = 0;
14246	u64 max_cost = 0;
14247
14248	rcu_read_lock();
14249	for_each_domain(cpu, sd) {
14250	/*
14251	* Decay the newidle max times here because this is a regular
14252	* visit to all the domains.
14253	*/
14254	need_decay = update_newidle_cost(sd, 0, 0);
14255	max_cost += sd->max_newidle_lb_cost;
14256
14257	/*
14258	* Stop the load balance at this level. There is another
14259	* CPU in our sched group which is doing load balancing more
14260	* actively.
14261	*/
14262	if (!continue_balancing) {
14263	if (need_decay)
14264	continue;
14265	break;
14266	}
14267
14268	interval = get_sd_balance_interval(sd, busy);
14269	if (time_after_eq(jiffies, sd->last_balance + interval)) {
14270	if (sched_balance_rq(cpu, rq, sd, idle, &continue_balancing)) {
14271	/*
14272	* The LBF_DST_PINNED logic could have changed
14273	* env->dst_cpu, so we can't know our idle
14274	* state even if we migrated tasks. Update it.
14275	*/
14276	idle = idle_cpu(cpu);
14277	busy = !idle && !sched_idle_rq(rq);
14278	}
14279	sd->last_balance = jiffies;
14280	interval = get_sd_balance_interval(sd, busy);
14281	}
14282	if (time_after(next_balance, sd->last_balance + interval)) {
14283	next_balance = sd->last_balance + interval;
14284	update_next_balance = 1;
14285	}
14286	}
14287	if (need_decay) {
14288	/*
14289	* Ensure the rq-wide value also decays but keep it at a
14290	* reasonable floor to avoid funnies with rq->avg_idle.
14291	*/
14292	rq->max_idle_balance_cost =
14293	max((u64)sysctl_sched_migration_cost, max_cost);
14294	}
14295	rcu_read_unlock();
14296
14297	/*
14298	* next_balance will be updated only when there is a need.
14299	* When the cpu is attached to null domain for ex, it will not be
14300	* updated.
14301	*/
14302	if (likely(update_next_balance))
14303	rq->next_balance = next_balance;
14304
14305	}
14306
14307	static inline int on_null_domain(struct rq *rq)
14308	{
14309	return unlikely(!rcu_dereference_sched(rq->sd));
14310	}
14311
14312	#ifdef CONFIG_NO_HZ_COMMON
14313	/*
14314	* NOHZ idle load balancing (ILB) details:
14315	*
14316	* - When one of the busy CPUs notices that there may be an idle rebalancing
14317	* needed, they will kick the idle load balancer, which then does idle
14318	* load balancing for all the idle CPUs.
14319	*/
14320	static inline int find_new_ilb(void)
14321	{
14322	struct cpumask *ilb_cpus;
14323	int ilb_cpu, fallback = -1;
14324
14325	lockdep_assert_irqs_disabled();
14326
14327	/*
14328	* Reuse the per-CPU select_rq_mask, which is protected from concurrent
14329	* use on this CPU by having interrupts disabled.
14330	*/
14331	ilb_cpus = this_cpu_cpumask_var_ptr(select_rq_mask);
14332	cpumask_and(ilb_cpus, nohz.idle_cpus_mask,
14333	housekeeping_cpumask(HK_TYPE_KERNEL_NOISE));
14334
14335	for_each_cpu(ilb_cpu, ilb_cpus) {
14336	if (!idle_cpu(ilb_cpu)) {
14337	/*
14338	* Once an idle fallback exists, a busy CPU proves that
14339	* this core cannot be fully idle. Skip its siblings.
14340	*/
14341	if (sched_smt_active() && fallback >= 0)
14342	cpumask_andnot(ilb_cpus, ilb_cpus, cpu_smt_mask(ilb_cpu));
14343	continue;
14344	}
14345
14346	/*
14347	* Running the idle load balancer on an idle sibling of a busy
14348	* SMT core can reduce the capacity available to its sibling. Prefer
14349	* a CPU whose entire core is idle, but retain the first idle CPU as
14350	* a fallback so idle balancing can still make progress when no fully
14351	* idle core exists.
14352	*/
14353	if (sched_smt_active() && !is_core_idle(ilb_cpu)) {
14354	if (fallback < 0)
14355	fallback = ilb_cpu;
14356
14357	/*
14358	* The core is not idle, so there is no need to check
14359	* any of its other SMT siblings.
14360	*/
14361	cpumask_andnot(ilb_cpus, ilb_cpus,
14362	cpu_smt_mask(ilb_cpu));
14363	continue;
14364	}
14365
14366	return ilb_cpu;
14367	}
14368
14369	return fallback;
14370	}
14371
14372	/*
14373	* Kick a CPU to do the NOHZ balancing, if it is time for it, via a cross-CPU
14374	* SMP function call (IPI).
14375	*
14376	* Prefer a CPU on a fully idle core in the HK_TYPE_KERNEL_NOISE housekeeping
14377	* set. Fall back to the first idle CPU when no fully idle core exists.
14378	*/
14379	static void kick_ilb(unsigned int flags)
14380	{
14381	int ilb_cpu;
14382
14383	/*
14384	* Increase nohz.next_balance only when if full ilb is triggered but
14385	* not if we only update stats.
14386	*/
14387	if (flags & NOHZ_BALANCE_KICK)
14388	nohz.next_balance = jiffies+1;
14389
14390	ilb_cpu = find_new_ilb();
14391	if (ilb_cpu < 0)
14392	return;
14393
14394	/*
14395	* Don't bother if no new NOHZ balance work items for ilb_cpu,
14396	* i.e. all bits in flags are already set in ilb_cpu.
14397	*/
14398	if ((atomic_read(nohz_flags(ilb_cpu)) & flags) == flags)
14399	return;
14400
14401	/*
14402	* Access to rq::nohz_csd is serialized by NOHZ_KICK_MASK; he who sets
14403	* the first flag owns it; cleared by nohz_csd_func().
14404	*/
14405	flags = atomic_fetch_or(flags, nohz_flags(ilb_cpu));
14406	if (flags & NOHZ_KICK_MASK)
14407	return;
14408
14409	/*
14410	* This way we generate an IPI on the target CPU which
14411	* is idle, and the softirq performing NOHZ idle load balancing
14412	* will be run before returning from the IPI.
14413	*/
14414	smp_call_function_single_async(ilb_cpu, &cpu_rq(ilb_cpu)->nohz_csd);
14415	}
14416
14417	/*
14418	* Current decision point for kicking the idle load balancer in the presence
14419	* of idle CPUs in the system.
14420	*/
14421	static void nohz_balancer_kick(struct rq *rq)
14422	{
14423	unsigned long now = jiffies;
14424	struct sched_domain_shared *sds;
14425	struct sched_domain *sd;
14426	int nr_busy, i, cpu = rq->cpu;
14427	unsigned int flags = 0;
14428
14429	if (unlikely(rq->idle_balance))
14430	return;
14431
14432	/*
14433	* We may be recently in ticked or tickless idle mode. At the first
14434	* busy tick after returning from idle, we will update the busy stats.
14435	*/
14436	nohz_balance_exit_idle(rq);
14437
14438	if (READ_ONCE(nohz.has_blocked_load) &&
14439	time_after(now, READ_ONCE(nohz.next_blocked)))
14440	flags = NOHZ_STATS_KICK;
14441
14442	/*
14443	* Most of the time system is not 100% busy. i.e nohz.nr_cpus > 0
14444	* Skip the read if time is not due.
14445	*
14446	* If none are in tickless mode, there maybe a narrow window
14447	* (28 jiffies, HZ=1000) where flags maybe set and kick_ilb called.
14448	* But idle load balancing is not done as find_new_ilb fails.
14449	* That's very rare. So read nohz.nr_cpus only if time is due.
14450	*/
14451	if (time_before(now, nohz.next_balance))
14452	goto out;
14453
14454	/*
14455	* None are in tickless mode and hence no need for NOHZ idle load
14456	* balancing
14457	*/
14458	if (unlikely(cpumask_empty(nohz.idle_cpus_mask)))
14459	return;
14460
14461	if (rq->nr_running >= 2) {
14462	flags = NOHZ_STATS_KICK \| NOHZ_BALANCE_KICK;
14463	goto out;
14464	}
14465
14466	sd = rcu_dereference_all(rq->sd);
14467	if (sd) {
14468	/*
14469	* If there's a runnable CFS task and the current CPU has reduced
14470	* capacity, kick the ILB to see if there's a better CPU to run on:
14471	*/
14472	if (rq->cfs.h_nr_runnable >= 1 && check_cpu_capacity(rq, sd)) {
14473	flags \|= NOHZ_STATS_KICK \| NOHZ_BALANCE_KICK;
14474	goto out;
14475	}
14476	}
14477
14478	sd = rcu_dereference_all(per_cpu(sd_asym_packing, cpu));
14479	if (sd) {
14480	/*
14481	* When ASYM_PACKING; see if there's a more preferred CPU
14482	* currently idle; in which case, kick the ILB to move tasks
14483	* around.
14484	*
14485	* When balancing between cores, all the SMT siblings of the
14486	* preferred CPU must be idle.
14487	*/
14488	for_each_cpu_and(i, sched_domain_span(sd), nohz.idle_cpus_mask) {
14489	if (sched_asym(sd, i, cpu)) {
14490	flags \|= NOHZ_STATS_KICK \| NOHZ_BALANCE_KICK;
14491	goto out;
14492	}
14493	}
14494	}
14495
14496	sd = rcu_dereference_all(per_cpu(sd_asym_cpucapacity, cpu));
14497	if (sd) {
14498	/*
14499	* When ASYM_CPUCAPACITY; see if there's a higher capacity CPU
14500	* to run the misfit task on.
14501	*/
14502	if (check_misfit_status(rq))
14503	flags \|= NOHZ_STATS_KICK \| NOHZ_BALANCE_KICK;
14504
14505	/*
14506	* For asymmetric systems, we do not want to nicely balance
14507	* cache use, instead we want to embrace asymmetry and only
14508	* ensure tasks have enough CPU capacity.
14509	*
14510	* Skip the LLC logic because it's not relevant in that case.
14511	*/
14512	goto out;
14513	}
14514
14515	sds = rcu_dereference_all(per_cpu(sd_balance_shared, cpu));
14516	if (sds) {
14517	/*
14518	* If there is an imbalance between LLC domains (IOW we could
14519	* increase the overall cache utilization), we need a less-loaded LLC
14520	* domain to pull some load from. Likewise, we may need to spread
14521	* load within the current LLC domain (e.g. packed SMT cores but
14522	* other CPUs are idle). We can't really know from here how busy
14523	* the others are - so just get a NOHZ balance going if it looks
14524	* like this LLC domain has tasks we could move.
14525	*/
14526	nr_busy = atomic_read(&sds->nr_busy_cpus);
14527	if (nr_busy > 1)
14528	flags \|= NOHZ_STATS_KICK \| NOHZ_BALANCE_KICK;
14529	}
14530	out:
14531	if (READ_ONCE(nohz.needs_update))
14532	flags \|= NOHZ_NEXT_KICK;
14533
14534	if (flags)
14535	kick_ilb(flags);
14536	}
14537
14538	static void set_cpu_sd_state_busy(int cpu)
14539	{
14540	struct sched_domain *sd;
14541	sd = rcu_dereference_all(per_cpu(sd_llc, cpu));
14542
14543	/*
14544	* sd->nohz_idle only pairs with nr_busy_cpus on sd->shared; if this
14545	* domain has no shared object there is nothing to clear or account.
14546	*/
14547	if (!sd \|\| !sd->shared \|\| !sd->nohz_idle)
14548	return;
14549	sd->nohz_idle = 0;
14550
14551	atomic_inc(&sd->shared->nr_busy_cpus);
14552	}
14553
14554	void nohz_balance_exit_idle(struct rq *rq)
14555	{
14556	WARN_ON_ONCE(rq != this_rq());
14557
14558	if (likely(!rq->nohz_tick_stopped))
14559	return;
14560
14561	rq->nohz_tick_stopped = 0;
14562	cpumask_clear_cpu(rq->cpu, nohz.idle_cpus_mask);
14563
14564	set_cpu_sd_state_busy(rq->cpu);
14565	}
14566
14567	static void set_cpu_sd_state_idle(int cpu)
14568	{
14569	struct sched_domain *sd;
14570	sd = rcu_dereference_all(per_cpu(sd_llc, cpu));
14571
14572	/* See set_cpu_sd_state_busy(): nohz_idle is only used with sd->shared. */
14573	if (!sd \|\| !sd->shared \|\| sd->nohz_idle)
14574	return;
14575	sd->nohz_idle = 1;
14576
14577	atomic_dec(&sd->shared->nr_busy_cpus);
14578	}
14579
14580	/*
14581	* This routine will record that the CPU is going idle with tick stopped.
14582	* This info will be used in performing idle load balancing in the future.
14583	*/
14584	void nohz_balance_enter_idle(int cpu)
14585	{
14586	struct rq *rq = cpu_rq(cpu);
14587
14588	WARN_ON_ONCE(cpu != smp_processor_id());
14589
14590	/* If this CPU is going down, then nothing needs to be done: */
14591	if (!cpu_active(cpu))
14592	return;
14593
14594	/*
14595	* Can be set safely without rq->lock held
14596	* If a clear happens, it will have evaluated last additions because
14597	* rq->lock is held during the check and the clear
14598	*/
14599	rq->has_blocked_load = 1;
14600
14601	/*
14602	* The tick is still stopped but load could have been added in the
14603	* meantime. We set the nohz.has_blocked_load flag to trig a check of the
14604	* *_avg. The CPU is already part of nohz.idle_cpus_mask so the clear
14605	* of nohz.has_blocked_load can only happen after checking the new load
14606	*/
14607	if (rq->nohz_tick_stopped)
14608	goto out;
14609
14610	/* If we're a completely isolated CPU, we don't play: */
14611	if (on_null_domain(rq))
14612	return;
14613
14614	rq->nohz_tick_stopped = 1;
14615
14616	cpumask_set_cpu(cpu, nohz.idle_cpus_mask);
14617
14618	/*
14619	* Ensures that if nohz_idle_balance() fails to observe our
14620	* @idle_cpus_mask store, it must observe the @has_blocked_load
14621	* and @needs_update stores.
14622	*/
14623	smp_mb__after_atomic();
14624
14625	set_cpu_sd_state_idle(cpu);
14626
14627	WRITE_ONCE(nohz.needs_update, 1);
14628	out:
14629	/*
14630	* Each time a cpu enter idle, we assume that it has blocked load and
14631	* enable the periodic update of the load of idle CPUs
14632	*/
14633	WRITE_ONCE(nohz.has_blocked_load, 1);
14634	}
14635
14636	static bool update_nohz_stats(struct rq *rq)
14637	{
14638	unsigned int cpu = rq->cpu;
14639
14640	if (!rq->has_blocked_load)
14641	return false;
14642
14643	if (!cpumask_test_cpu(cpu, nohz.idle_cpus_mask))
14644	return false;
14645
14646	if (!time_after(jiffies, READ_ONCE(rq->last_blocked_load_update_tick)))
14647	return true;
14648
14649	sched_balance_update_blocked_averages(cpu);
14650
14651	return rq->has_blocked_load;
14652	}
14653
14654	/*
14655	* Internal function that runs load balance for all idle CPUs. The load balance
14656	* can be a simple update of blocked load or a complete load balance with
14657	* tasks movement depending of flags.
14658	*/
14659	static void _nohz_idle_balance(struct rq *this_rq, unsigned int flags)
14660	{
14661	/* Earliest time when we have to do rebalance again */
14662	unsigned long now = jiffies;
14663	unsigned long next_balance = now + 60*HZ;
14664	bool has_blocked_load = false;
14665	int update_next_balance = 0;
14666	int this_cpu = this_rq->cpu;
14667	int balance_cpu;
14668	struct rq *rq;
14669
14670	WARN_ON_ONCE((flags & NOHZ_KICK_MASK) == NOHZ_BALANCE_KICK);
14671
14672	/*
14673	* We assume there will be no idle load after this update and clear
14674	* the has_blocked_load flag. If a cpu enters idle in the mean time, it will
14675	* set the has_blocked_load flag and trigger another update of idle load.
14676	* Because a cpu that becomes idle, is added to idle_cpus_mask before
14677	* setting the flag, we are sure to not clear the state and not
14678	* check the load of an idle cpu.
14679	*
14680	* Same applies to idle_cpus_mask vs needs_update.
14681	*/
14682	if (flags & NOHZ_STATS_KICK)
14683	WRITE_ONCE(nohz.has_blocked_load, 0);
14684	if (flags & NOHZ_NEXT_KICK)
14685	WRITE_ONCE(nohz.needs_update, 0);
14686
14687	/*
14688	* Ensures that if we miss the CPU, we must see the has_blocked_load
14689	* store from nohz_balance_enter_idle().
14690	*/
14691	smp_mb();
14692
14693	/*
14694	* Start with the next CPU after this_cpu so we will end with this_cpu and let a
14695	* chance for other idle cpu to pull load.
14696	*/
14697	for_each_cpu_wrap(balance_cpu, nohz.idle_cpus_mask, this_cpu+1) {
14698	if (!idle_cpu(balance_cpu))
14699	continue;
14700
14701	/*
14702	* If this CPU gets work to do, stop the load balancing
14703	* work being done for other CPUs. Next load
14704	* balancing owner will pick it up.
14705	*/
14706	if (!idle_cpu(this_cpu) && need_resched()) {
14707	if (flags & NOHZ_STATS_KICK)
14708	has_blocked_load = true;
14709	if (flags & NOHZ_NEXT_KICK)
14710	WRITE_ONCE(nohz.needs_update, 1);
14711	goto abort;
14712	}
14713
14714	rq = cpu_rq(balance_cpu);
14715
14716	if (flags & NOHZ_STATS_KICK)
14717	has_blocked_load \|= update_nohz_stats(rq);
14718
14719	/*
14720	* If time for next balance is due,
14721	* do the balance.
14722	*/
14723	if (time_after_eq(jiffies, rq->next_balance)) {
14724	struct rq_flags rf;
14725
14726	rq_lock_irqsave(rq, &rf);
14727	update_rq_clock(rq);
14728	rq_unlock_irqrestore(rq, &rf);
14729
14730	if (flags & NOHZ_BALANCE_KICK)
14731	sched_balance_domains(rq, CPU_IDLE);
14732	}
14733
14734	if (time_after(next_balance, rq->next_balance)) {
14735	next_balance = rq->next_balance;
14736	update_next_balance = 1;
14737	}
14738	}
14739
14740	/*
14741	* next_balance will be updated only when there is a need.
14742	* When the CPU is attached to null domain for ex, it will not be
14743	* updated.
14744	*/
14745	if (likely(update_next_balance))
14746	nohz.next_balance = next_balance;
14747
14748	if (flags & NOHZ_STATS_KICK)
14749	WRITE_ONCE(nohz.next_blocked,
14750	now + msecs_to_jiffies(LOAD_AVG_PERIOD));
14751
14752	abort:
14753	/* There is still blocked load, enable periodic update */
14754	if (has_blocked_load)
14755	WRITE_ONCE(nohz.has_blocked_load, 1);
14756	}
14757
14758	/*
14759	* In CONFIG_NO_HZ_COMMON case, the idle balance kickee will do the
14760	* rebalancing for all the CPUs for whom scheduler ticks are stopped.
14761	*/
14762	static bool nohz_idle_balance(struct rq *this_rq, enum cpu_idle_type idle)
14763	{
14764	unsigned int flags = this_rq->nohz_idle_balance;
14765
14766	if (!flags)
14767	return false;
14768
14769	this_rq->nohz_idle_balance = 0;
14770
14771	if (idle != CPU_IDLE)
14772	return false;
14773
14774	_nohz_idle_balance(this_rq, flags);
14775
14776	return true;
14777	}
14778
14779	/*
14780	* Check if we need to directly run the ILB for updating blocked load before
14781	* entering idle state. Here we run ILB directly without issuing IPIs.
14782	*
14783	* Note that when this function is called, the tick may not yet be stopped on
14784	* this CPU yet. nohz.idle_cpus_mask is updated only when tick is stopped and
14785	* cleared on the next busy tick. In other words, nohz.idle_cpus_mask updates
14786	* don't align with CPUs enter/exit idle to avoid bottlenecks due to high idle
14787	* entry/exit rate (usec). So it is possible that _nohz_idle_balance() is
14788	* called from this function on (this) CPU that's not yet in the mask. That's
14789	* OK because the goal of nohz_run_idle_balance() is to run ILB only for
14790	* updating the blocked load of already idle CPUs without waking up one of
14791	* those idle CPUs and outside the preempt disable / IRQ off phase of the local
14792	* cpu about to enter idle, because it can take a long time.
14793	*/
14794	void nohz_run_idle_balance(int cpu)
14795	{
14796	unsigned int flags;
14797
14798	flags = atomic_fetch_andnot(NOHZ_NEWILB_KICK, nohz_flags(cpu));
14799
14800	/*
14801	* Update the blocked load only if no SCHED_SOFTIRQ is about to happen
14802	* (i.e. NOHZ_STATS_KICK set) and will do the same.
14803	*/
14804	if ((flags == NOHZ_NEWILB_KICK) && !need_resched())
14805	_nohz_idle_balance(cpu_rq(cpu), NOHZ_STATS_KICK);
14806	}
14807
14808	static void nohz_newidle_balance(struct rq *this_rq)
14809	{
14810	int this_cpu = this_rq->cpu;
14811
14812	/* Will wake up very soon. No time for doing anything else*/
14813	if (this_rq->avg_idle < sysctl_sched_migration_cost)
14814	return;
14815
14816	/* Don't need to update blocked load of idle CPUs*/
14817	if (!READ_ONCE(nohz.has_blocked_load) \|\|
14818	time_before(jiffies, READ_ONCE(nohz.next_blocked)))
14819	return;
14820
14821	/*
14822	* Set the need to trigger ILB in order to update blocked load
14823	* before entering idle state.
14824	*/
14825	atomic_or(NOHZ_NEWILB_KICK, nohz_flags(this_cpu));
14826	}
14827
14828	#else /* !CONFIG_NO_HZ_COMMON: */
14829	static inline void nohz_balancer_kick(struct rq *rq) { }
14830
14831	static inline bool nohz_idle_balance(struct rq *this_rq, enum cpu_idle_type idle)
14832	{
14833	return false;
14834	}
14835
14836	static inline void nohz_newidle_balance(struct rq *this_rq) { }
14837	#endif /* !CONFIG_NO_HZ_COMMON */
14838
14839	/*
14840	* sched_balance_newidle is called by schedule() if this_cpu is about to become
14841	* idle. Attempts to pull tasks from other CPUs.
14842	*
14843	* Returns:
14844	* < 0 - we released the lock and there are !fair tasks present
14845	* 0 - failed, no new tasks
14846	* > 0 - success, new (fair) tasks present
14847	*/
14848	static int sched_balance_newidle(struct rq this_rq, struct rq_flags rf)
14849	__must_hold(__rq_lockp(this_rq))
14850	{
14851	unsigned long next_balance = jiffies + HZ;
14852	int this_cpu = this_rq->cpu;
14853	int continue_balancing = 1;
14854	u64 t0, t1, curr_cost = 0;
14855	struct sched_domain *sd;
14856	int pulled_task = 0;
14857
14858	update_misfit_status(NULL, this_rq);
14859
14860	/*
14861	* There is a task waiting to run. No need to search for one.
14862	* Return 0; the task will be enqueued when switching to idle.
14863	*/
14864	if (this_rq->ttwu_pending)
14865	return 0;
14866
14867	/*
14868	* We must set idle_stamp _before_ calling sched_balance_rq()
14869	* for CPU_NEWLY_IDLE, such that we measure the this duration
14870	* as idle time.
14871	*/
14872	this_rq->idle_stamp = rq_clock(this_rq);
14873
14874	/* Do not pull tasks towards !preferred CPUs */
14875	if (!cpu_preferred(this_cpu))
14876	return 0;
14877
14878	/*
14879	* This is OK, because current is on_cpu, which avoids it being picked
14880	* for load-balance and preemption/IRQs are still disabled avoiding
14881	* further scheduler activity on it and we're being very careful to
14882	* re-start the picking loop.
14883	*/
14884	rq_unpin_lock(this_rq, rf);
14885
14886	sd = rcu_dereference_sched_domain(this_rq->sd);
14887	if (!sd)
14888	goto out;
14889
14890	if (!get_rd_overloaded(this_rq->rd) \|\|
14891	this_rq->avg_idle < sd->max_newidle_lb_cost) {
14892
14893	update_next_balance(sd, &next_balance);
14894	goto out;
14895	}
14896
14897	/*
14898	* Include sched_balance_update_blocked_averages() in the cost
14899	* calculation because it can be quite costly -- this ensures we skip
14900	* it when avg_idle gets to be very low.
14901	*/
14902	t0 = sched_clock_cpu(this_cpu);
14903	__sched_balance_update_blocked_averages(this_rq);
14904
14905	rq_modified_begin(this_rq, &fair_sched_class);
14906	raw_spin_rq_unlock(this_rq);
14907
14908	for_each_domain(this_cpu, sd) {
14909	u64 domain_cost;
14910
14911	update_next_balance(sd, &next_balance);
14912
14913	if (this_rq->avg_idle < curr_cost + sd->max_newidle_lb_cost)
14914	break;
14915
14916	if (sd->flags & SD_BALANCE_NEWIDLE) {
14917	unsigned int weight = 1;
14918
14919	if (sched_feat(NI_RANDOM) && sd->newidle_ratio < 1024) {
14920	/*
14921	* Throw a 1k sided dice; and only run
14922	* newidle_balance according to the success
14923	* rate.
14924	*/
14925	u32 d1k = sched_rng() % 1024;
14926	weight = 1 + sd->newidle_ratio;
14927	if (d1k > weight) {
14928	update_newidle_stats(sd, 0);
14929	continue;
14930	}
14931	weight = (1024 + weight/2) / weight;
14932	}
14933
14934	pulled_task = sched_balance_rq(this_cpu, this_rq,
14935	sd, CPU_NEWLY_IDLE,
14936	&continue_balancing);
14937
14938	t1 = sched_clock_cpu(this_cpu);
14939	domain_cost = t1 - t0;
14940	curr_cost += domain_cost;
14941	t0 = t1;
14942
14943	/*
14944	* Track max cost of a domain to make sure to not delay the
14945	* next wakeup on the CPU.
14946	*/
14947	update_newidle_cost(sd, domain_cost, weight * !!pulled_task);
14948	}
14949
14950	/*
14951	* Stop searching for tasks to pull if there are
14952	* now runnable tasks on this rq.
14953	*/
14954	if (pulled_task \|\| !continue_balancing)
14955	break;
14956	}
14957
14958	raw_spin_rq_lock(this_rq);
14959
14960	if (curr_cost > this_rq->max_idle_balance_cost)
14961	this_rq->max_idle_balance_cost = curr_cost;
14962
14963	/*
14964	* While browsing the domains, we released the rq lock, a task could
14965	* have been enqueued in the meantime. Since we're not going idle,
14966	* pretend we pulled a task.
14967	*/
14968	if (this_rq->cfs.h_nr_queued && !pulled_task)
14969	pulled_task = 1;
14970
14971	/* If a higher prio class was modified, restart the pick */
14972	if (rq_modified_above(this_rq, &fair_sched_class))
14973	pulled_task = -1;
14974
14975	out:
14976	/* Move the next balance forward */
14977	if (time_after(this_rq->next_balance, next_balance))
14978	this_rq->next_balance = next_balance;
14979
14980	if (pulled_task)
14981	this_rq->idle_stamp = 0;
14982	else
14983	nohz_newidle_balance(this_rq);
14984
14985	rq_repin_lock(this_rq, rf);
14986
14987	return pulled_task;
14988	}
14989
14990	/*
14991	* This softirq handler is triggered via SCHED_SOFTIRQ from two places:
14992	*
14993	* - directly from the local sched_tick() for periodic load balancing
14994	*
14995	* - indirectly from a remote sched_tick() for NOHZ idle balancing
14996	* through the SMP cross-call nohz_csd_func()
14997	*/
14998	static __latent_entropy void sched_balance_softirq(void)
14999	{
15000	struct rq *this_rq = this_rq();
15001	enum cpu_idle_type idle = this_rq->idle_balance;
15002	/*
15003	* If this CPU has a pending NOHZ_BALANCE_KICK, then do the
15004	* balancing on behalf of the other idle CPUs whose ticks are
15005	* stopped. Do nohz_idle_balance before sched_balance_domains to
15006	* give the idle CPUs a chance to load balance. Else we may
15007	* load balance only within the local sched_domain hierarchy
15008	* and abort nohz_idle_balance altogether if we pull some load.
15009	*/
15010	if (nohz_idle_balance(this_rq, idle))
15011	return;
15012
15013	/* normal load balance */
15014	sched_balance_update_blocked_averages(this_rq->cpu);
15015	sched_balance_domains(this_rq, idle);
15016	}
15017
15018	/*
15019	* Trigger the SCHED_SOFTIRQ if it is time to do periodic load balancing.
15020	*/
15021	void sched_balance_trigger(struct rq *rq)
15022	{
15023	/*
15024	* Don't need to rebalance while attached to NULL domain or
15025	* runqueue CPU is not active
15026	*/
15027	if (unlikely(on_null_domain(rq) \|\| !cpu_active(cpu_of(rq))))
15028	return;
15029
15030	if (time_after_eq(jiffies, rq->next_balance))
15031	raise_softirq(SCHED_SOFTIRQ);
15032
15033	nohz_balancer_kick(rq);
15034	}
15035
15036	static void rq_online_fair(struct rq *rq)
15037	{
15038	update_sysctl();
15039
15040	update_runtime_enabled(rq);
15041	}
15042
15043	static void rq_offline_fair(struct rq *rq)
15044	{
15045	update_sysctl();
15046
15047	/* Ensure any throttled groups are reachable by pick_next_task */
15048	unthrottle_offline_cfs_rqs(rq);
15049
15050	/* Ensure that we remove rq contribution to group share: */
15051	clear_tg_offline_cfs_rqs(rq);
15052	}
15053
15054	#ifdef CONFIG_SCHED_CORE
15055	static inline bool
15056	__entity_slice_used(struct sched_entity *se, int min_nr_tasks)
15057	{
15058	u64 rtime = se->sum_exec_runtime - se->prev_sum_exec_runtime;
15059	u64 slice = se->slice;
15060
15061	return (rtime * min_nr_tasks > slice);
15062	}
15063
15064	#define MIN_NR_TASKS_DURING_FORCEIDLE 2
15065	static inline void task_tick_core(struct rq rq, struct task_struct curr)
15066	{
15067	if (!sched_core_enabled(rq))
15068	return;
15069
15070	/*
15071	* If runqueue has only one task which used up its slice and
15072	* if the sibling is forced idle, then trigger schedule to
15073	* give forced idle task a chance.
15074	*
15075	* __entity_slice_used() considers only this active rq and it gets the
15076	* whole slice. But during force idle, we have siblings acting
15077	* like a single runqueue and hence we need to consider runnable
15078	* tasks on this CPU and the forced idle CPU. Ideally, we should
15079	* go through the forced idle rq, but that would be a perf hit.
15080	* We can assume that the forced idle CPU has at least
15081	* MIN_NR_TASKS_DURING_FORCEIDLE - 1 tasks and use that to check
15082	* if we need to give up the CPU.
15083	*/
15084	if (rq->core->core_forceidle_count && rq->cfs.h_nr_queued == 1 &&
15085	__entity_slice_used(&curr->se, MIN_NR_TASKS_DURING_FORCEIDLE))
15086	resched_curr(rq);
15087	}
15088
15089	/*
15090	* Consider any infeasible weight scenario. Take for instance two tasks,
15091	* each bound to their respective sibling, one with weight 1 and one with
15092	* weight 2. Then the lower weight task will run ahead of the higher weight
15093	* task without bound.
15094	*
15095	* This utterly destroys the concept of a shared time base.
15096	*
15097	* Remember; all this is about a proportionally fair scheduling, where each
15098	* tasks receives:
15099	*
15100	* w_i
15101	* dt_i = ---------- dt (1)
15102	* \Sum_j w_j
15103	*
15104	* which we do by tracking a virtual time, s_i:
15105	*
15106	* 1
15107	* s_i = --- d[t]_i (2)
15108	* w_i
15109	*
15110	* Where d[t] is a delta of discrete time, while dt is an infinitesimal.
15111	* The immediate corollary is that the ideal schedule S, where (2) to use
15112	* an infinitesimal delta, is:
15113	*
15114	* 1
15115	* S = ---------- dt (3)
15116	* \Sum_i w_i
15117	*
15118	* From which we can define the lag, or deviation from the ideal, as:
15119	*
15120	* lag(i) = S - s_i (4)
15121	*
15122	* And since the one and only purpose is to approximate S, we get that:
15123	*
15124	* \Sum_i w_i lag(i) := 0 (5)
15125	*
15126	* If this were not so, we no longer converge to S, and we can no longer
15127	* claim our scheduler has any of the properties we derive from S. This is
15128	* exactly what you did above, you broke it!
15129	*
15130	*
15131	* Let's continue for a while though; to see if there is anything useful to
15132	* be learned. We can combine (1)-(3) or (4)-(5) and express S in s_i:
15133	*
15134	* \Sum_i w_i s_i
15135	* S = -------------- (6)
15136	* \Sum_i w_i
15137	*
15138	* Which gives us a way to compute S, given our s_i. Now, if you've read
15139	* our code, you know that we do not in fact do this, the reason for this
15140	* is two-fold. Firstly, computing S in that way requires a 64bit division
15141	* for every time we'd use it (see 12), and secondly, this only describes
15142	* the steady-state, it doesn't handle dynamics.
15143	*
15144	* Anyway, in (6): s_i -> x + (s_i - x), to get:
15145	*
15146	* \Sum_i w_i (s_i - x)
15147	* S - x = -------------------- (7)
15148	* \Sum_i w_i
15149	*
15150	* Which shows that S and s_i transform alike (which makes perfect sense
15151	* given that S is basically the (weighted) average of s_i).
15152	*
15153	* So the thing to remember is that the above is strictly UP. It is
15154	* possible to generalize to multiple runqueues -- however it gets really
15155	* yuck when you have to add affinity support, as illustrated by our very
15156	* first counter-example.
15157	*
15158	* Luckily I think we can avoid needing a full multi-queue variant for
15159	* core-scheduling (or load-balancing). The crucial observation is that we
15160	* only actually need this comparison in the presence of forced-idle; only
15161	* then do we need to tell if the stalled rq has higher priority over the
15162	* other.
15163	*
15164	* [XXX assumes SMT2; better consider the more general case, I suspect
15165	* it'll work out because our comparison is always between 2 rqs and the
15166	* answer is only interesting if one of them is forced-idle]
15167	*
15168	* And (under assumption of SMT2) when there is forced-idle, there is only
15169	* a single queue, so everything works like normal.
15170	*
15171	* Let, for our runqueue 'k':
15172	*
15173	* T_k = \Sum_i w_i s_i
15174	* W_k = \Sum_i w_i ; for all i of k (8)
15175	*
15176	* Then we can write (6) like:
15177	*
15178	* T_k
15179	* S_k = --- (9)
15180	* W_k
15181	*
15182	* From which immediately follows that:
15183	*
15184	* T_k + T_l
15185	* S_k+l = --------- (10)
15186	* W_k + W_l
15187	*
15188	* On which we can define a combined lag:
15189	*
15190	* lag_k+l(i) := S_k+l - s_i (11)
15191	*
15192	* And that gives us the tools to compare tasks across a combined runqueue.
15193	*
15194	*
15195	* Combined this gives the following:
15196	*
15197	* a) when a runqueue enters force-idle, sync it against it's sibling rq(s)
15198	* using (7); this only requires storing single 'time'-stamps.
15199	*
15200	* b) when comparing tasks between 2 runqueues of which one is forced-idle,
15201	* compare the combined lag, per (11).
15202	*
15203	* Now, of course cgroups (I so hate them) make this more interesting in
15204	* that a) seems to suggest we need to iterate all cgroup on a CPU at such
15205	* boundaries, but I think we can avoid that. The force-idle is for the
15206	* whole CPU, all it's rqs. So we can mark it in the root and lazily
15207	* propagate downward on demand.
15208	*/
15209
15210	/*
15211	* So this sync is basically a relative reset of S to 0.
15212	*
15213	* So with 2 queues, when one goes idle, we drop them both to 0 and one
15214	* then increases due to not being idle, and the idle one builds up lag to
15215	* get re-elected. So far so simple, right?
15216	*
15217	* When there's 3, we can have the situation where 2 run and one is idle,
15218	* we sync to 0 and let the idle one build up lag to get re-election. Now
15219	* suppose another one also drops idle. At this point dropping all to 0
15220	* again would destroy the built-up lag from the queue that was already
15221	* idle, not good.
15222	*
15223	* So instead of syncing everything, we can:
15224	*
15225	* less := !((s64)(s_a - s_b) <= 0)
15226	*
15227	* (v_a - S_a) - (v_b - S_b) == v_a - v_b - S_a + S_b
15228	* == v_a - (v_b - S_a + S_b)
15229	*
15230	* IOW, we can recast the (lag) comparison to a one-sided difference.
15231	* So if then, instead of syncing the whole queue, sync the idle queue
15232	* against the active queue with S_a + S_b at the point where we sync.
15233	*
15234	* (XXX consider the implication of living in a cyclic group: N / 2^n N)
15235	*
15236	* This gives us means of syncing single queues against the active queue,
15237	* and for already idle queues to preserve their build-up lag.
15238	*
15239	* Of course, then we get the situation where there's 2 active and one
15240	* going idle, who do we pick to sync against? Theory would have us sync
15241	* against the combined S, but as we've already demonstrated, there is no
15242	* such thing in infeasible weight scenarios.
15243	*
15244	* One thing I've considered; and this is where that core_active rudiment
15245	* came from, is having active queues sync up between themselves after
15246	* every tick. This limits the observed divergence due to the work
15247	* conservancy.
15248	*
15249	* On top of that, we can improve upon things by employing (10) here.
15250	*/
15251
15252	/*
15253	* se_fi_update - Update the cfs_rq->zero_vruntime_fi in a CFS hierarchy if needed.
15254	*/
15255	static void se_fi_update(const struct sched_entity *se, unsigned int fi_seq,
15256	bool forceidle)
15257	{
15258	for_each_sched_entity(se) {
15259	struct cfs_rq *cfs_rq = cfs_rq_of(se);
15260
15261	if (forceidle) {
15262	if (cfs_rq->forceidle_seq == fi_seq)
15263	break;
15264	cfs_rq->forceidle_seq = fi_seq;
15265	}
15266
15267	cfs_rq->zero_vruntime_fi = cfs_rq->zero_vruntime;
15268	}
15269	}
15270
15271	void task_vruntime_update(struct rq rq, struct task_struct p, bool in_fi)
15272	{
15273	struct sched_entity *se = &p->se;
15274
15275	if (p->sched_class != &fair_sched_class)
15276	return;
15277
15278	se_fi_update(se, rq->core->core_forceidle_seq, in_fi);
15279	}
15280
15281	bool cfs_prio_less(const struct task_struct a, const struct task_struct b,
15282	bool in_fi)
15283	{
15284	struct rq *rq = task_rq(a);
15285	const struct sched_entity *sea = &a->se;
15286	const struct sched_entity *seb = &b->se;
15287	struct cfs_rq *cfs_rqa;
15288	struct cfs_rq *cfs_rqb;
15289	s64 delta;
15290
15291	WARN_ON_ONCE(task_rq(b)->core != rq->core);
15292
15293	cfs_rqa = &task_rq(a)->cfs;
15294	cfs_rqb = &task_rq(b)->cfs;
15295
15296	/*
15297	* Find delta after normalizing se's vruntime with its cfs_rq's
15298	* zero_vruntime_fi, which would have been updated in prior calls
15299	* to se_fi_update().
15300	*/
15301	delta = vruntime_op(sea->vruntime, "-", seb->vruntime) +
15302	vruntime_op(cfs_rqb->zero_vruntime_fi, "-", cfs_rqa->zero_vruntime_fi);
15303
15304	return delta > 0;
15305	}
15306
15307	static int task_is_throttled_fair(struct task_struct *p, int cpu)
15308	{
15309	struct cfs_rq *cfs_rq;
15310
15311	#ifdef CONFIG_FAIR_GROUP_SCHED
15312	cfs_rq = tg_cfs_rq(task_group(p), cpu);
15313	#else
15314	cfs_rq = &cpu_rq(cpu)->cfs;
15315	#endif
15316	return throttled_hierarchy(cfs_rq);
15317	}
15318	#else /* !CONFIG_SCHED_CORE: */
15319	static inline void task_tick_core(struct rq rq, struct task_struct curr) {}
15320	#endif /* !CONFIG_SCHED_CORE */
15321
15322	/*
15323	* scheduler tick hitting a task of our scheduling class.
15324	*
15325	* NOTE: This function can be called remotely by the tick offload that
15326	* goes along full dynticks. Therefore no local assumption can be made
15327	* and everything must be accessed through the @rq and @curr passed in
15328	* parameters.
15329	*/
15330	static void task_tick_fair(struct rq rq, struct task_struct curr, int queued)
15331	{
15332	struct sched_entity *se = &curr->se;
15333
15334	if (se->on_rq) {
15335	unsigned long weight = NICE_0_LOAD;
15336	struct cfs_rq *cfs_rq;
15337
15338	for_each_sched_entity(se) {
15339	cfs_rq = cfs_rq_of(se);
15340	entity_tick(cfs_rq, se, queued);
15341
15342	weight = __calc_prop_weight(cfs_rq, se, weight);
15343	}
15344
15345	se = &curr->se;
15346	reweight_eevdf(cfs_rq, se, weight, se->on_rq);
15347	}
15348
15349	if (queued)
15350	return;
15351
15352	if (static_branch_unlikely(&sched_numa_balancing))
15353	task_tick_numa(rq, curr);
15354
15355	task_tick_cache(rq, curr);
15356
15357	update_misfit_status(curr, rq);
15358	check_update_overutilized_status(task_rq(curr));
15359
15360	task_tick_core(rq, curr);
15361	}
15362
15363	/*
15364	* called on fork with the child task as argument from the parent's context
15365	* - child not yet on the tasklist
15366	* - preemption disabled
15367	*/
15368	static void task_fork_fair(struct task_struct *p)
15369	{
15370	set_task_max_allowed_capacity(p);
15371	}
15372
15373	/*
15374	* Priority of the task has changed. Check to see if we preempt
15375	* the current task.
15376	*/
15377	static void
15378	prio_changed_fair(struct rq rq, struct task_struct p, u64 oldprio)
15379	{
15380	if (!task_on_rq_queued(p))
15381	return;
15382
15383	if (p->prio == oldprio)
15384	return;
15385
15386	if (rq->cfs.h_nr_queued == 1)
15387	return;
15388
15389	/*
15390	* Reschedule if we are currently running on this runqueue and
15391	* our priority decreased, or if we are not currently running on
15392	* this runqueue and our priority is higher than the current's
15393	*/
15394	if (task_current_donor(rq, p)) {
15395	if (p->prio > oldprio)
15396	resched_curr(rq);
15397	} else {
15398	wakeup_preempt(rq, p, 0);
15399	}
15400	}
15401
15402	#ifdef CONFIG_FAIR_GROUP_SCHED
15403	/*
15404	* Propagate the changes of the sched_entity across the tg tree to make it
15405	* visible to the root
15406	*/
15407	static void propagate_entity_cfs_rq(struct sched_entity *se)
15408	{
15409	struct cfs_rq *cfs_rq = cfs_rq_of(se);
15410
15411	/*
15412	* If a task gets attached to this cfs_rq and before being queued,
15413	* it gets migrated to another CPU due to reasons like affinity
15414	* change, make sure this cfs_rq stays on leaf cfs_rq list to have
15415	* that removed load decayed or it can cause faireness problem.
15416	*/
15417	if (!cfs_rq_pelt_clock_throttled(cfs_rq))
15418	list_add_leaf_cfs_rq(cfs_rq);
15419
15420	/* Start to propagate at parent */
15421	se = se->parent;
15422
15423	for_each_sched_entity(se) {
15424	cfs_rq = cfs_rq_of(se);
15425
15426	update_load_avg(cfs_rq, se, UPDATE_TG);
15427
15428	if (!cfs_rq_pelt_clock_throttled(cfs_rq))
15429	list_add_leaf_cfs_rq(cfs_rq);
15430	}
15431
15432	assert_list_leaf_cfs_rq(rq_of(cfs_rq));
15433	}
15434	#else /* !CONFIG_FAIR_GROUP_SCHED: */
15435	static void propagate_entity_cfs_rq(struct sched_entity *se) { }
15436	#endif /* !CONFIG_FAIR_GROUP_SCHED */
15437
15438	static void detach_entity_cfs_rq(struct sched_entity *se)
15439	{
15440	struct cfs_rq *cfs_rq = cfs_rq_of(se);
15441
15442	/*
15443	* In case the task sched_avg hasn't been attached:
15444	* - A forked task which hasn't been woken up by wake_up_new_task().
15445	* - A task which has been woken up by try_to_wake_up() but is
15446	* waiting for actually being woken up by sched_ttwu_pending().
15447	*/
15448	if (!se->avg.last_update_time)
15449	return;
15450
15451	/* Catch up with the cfs_rq and remove our load when we leave */
15452	update_load_avg(cfs_rq, se, 0);
15453	detach_entity_load_avg(cfs_rq, se);
15454	update_tg_load_avg(cfs_rq);
15455	propagate_entity_cfs_rq(se);
15456	}
15457
15458	static void attach_entity_cfs_rq(struct sched_entity *se)
15459	{
15460	struct cfs_rq *cfs_rq = cfs_rq_of(se);
15461
15462	/* Synchronize entity with its cfs_rq */
15463	update_load_avg(cfs_rq, se, sched_feat(ATTACH_AGE_LOAD) ? 0 : SKIP_AGE_LOAD);
15464	attach_entity_load_avg(cfs_rq, se);
15465	update_tg_load_avg(cfs_rq);
15466	propagate_entity_cfs_rq(se);
15467	}
15468
15469	static void detach_task_cfs_rq(struct task_struct *p)
15470	{
15471	struct sched_entity *se = &p->se;
15472
15473	detach_entity_cfs_rq(se);
15474	}
15475
15476	static void attach_task_cfs_rq(struct task_struct *p)
15477	{
15478	struct sched_entity *se = &p->se;
15479
15480	attach_entity_cfs_rq(se);
15481	}
15482
15483	static void switching_from_fair(struct rq rq, struct task_struct p)
15484	{
15485	if (p->se.sched_delayed)
15486	dequeue_task(rq, p, DEQUEUE_SLEEP \| DEQUEUE_DELAYED \| DEQUEUE_NOCLOCK);
15487	}
15488
15489	static void switched_from_fair(struct rq rq, struct task_struct p)
15490	{
15491	detach_task_cfs_rq(p);
15492	}
15493
15494	static void switched_to_fair(struct rq rq, struct task_struct p)
15495	{
15496	WARN_ON_ONCE(p->se.sched_delayed);
15497
15498	attach_task_cfs_rq(p);
15499
15500	set_task_max_allowed_capacity(p);
15501
15502	if (task_on_rq_queued(p)) {
15503	/*
15504	* We were most likely switched from sched_rt, so
15505	* kick off the schedule if running, otherwise just see
15506	* if we can still preempt the current task.
15507	*/
15508	if (task_current_donor(rq, p))
15509	resched_curr(rq);
15510	else
15511	wakeup_preempt(rq, p, 0);
15512	}
15513	}
15514
15515	static void set_next_task_fair(struct rq rq, struct task_struct p, enum snt_e type)
15516	{
15517	struct sched_entity *se = &p->se;
15518	struct cfs_rq *cfs_rq = &rq->cfs;
15519	unsigned long weight = NICE_0_LOAD;
15520	bool first = type == SNT_PICK;
15521	bool throttled = false;
15522	bool on_rq = se->on_rq;
15523
15524	if (type == SNT_REPICK)
15525	goto repick;
15526
15527	clear_buddies(cfs_rq, se);
15528
15529	if (on_rq)
15530	__dequeue_entity(cfs_rq, se);
15531
15532	for_each_sched_entity(se) {
15533	cfs_rq = cfs_rq_of(se);
15534
15535	if (!IS_ENABLED(CONFIG_FAIR_GROUP_SCHED) \|\|
15536	!first \|\| !cfs_rq->h_curr)
15537	set_next_entity(cfs_rq, se);
15538
15539	/* ensure bandwidth has been allocated on our new cfs_rq */
15540	throttled \|= account_cfs_rq_runtime(cfs_rq, 0);
15541
15542	if (on_rq)
15543	weight = __calc_prop_weight(cfs_rq, se, weight);
15544	}
15545
15546	if (throttled)
15547	task_throttle_setup_work(p);
15548
15549	se = &p->se;
15550	cfs_rq->curr = se;
15551
15552	if (on_rq) {
15553	reweight_eevdf(cfs_rq, se, weight, se->on_rq);
15554	if (first)
15555	set_protect_slice(cfs_rq, se);
15556	}
15557
15558	if (task_on_rq_queued(p)) {
15559	/*
15560	* Move the next running task to the front of the list, so our
15561	* cfs_tasks list becomes MRU one.
15562	*/
15563	list_move(&se->group_node, &rq->cfs_tasks);
15564	}
15565	if (!first)
15566	return;
15567
15568	WARN_ON_ONCE(se->sched_delayed);
15569
15570	update_misfit_status(p, rq);
15571	sched_fair_update_stop_tick(rq, p);
15572
15573	repick:
15574	/*
15575	* A same-task repick skips put_prev_task_fair(), but
15576	* pick_task_fair() refreshed the entity hrtick_start_fair() reads
15577	* before selecting it again. rq->cfs.curr identifies that entity,
15578	* including with group scheduling.
15579	*/
15580	if (hrtick_enabled_fair(rq))
15581	hrtick_start_fair(rq, p);
15582	}
15583
15584	void init_cfs_rq(struct cfs_rq *cfs_rq)
15585	{
15586	cfs_rq->tasks_timeline = RB_ROOT_CACHED;
15587	cfs_rq->zero_vruntime = (u64)(-(1LL << 20));
15588	raw_spin_lock_init(&cfs_rq->removed.lock);
15589	}
15590
15591	#ifdef CONFIG_FAIR_GROUP_SCHED
15592	static void task_change_group_fair(struct task_struct *p)
15593	{
15594	/*
15595	* We couldn't detach or attach a forked task which
15596	* hasn't been woken up by wake_up_new_task().
15597	*/
15598	if (READ_ONCE(p->__state) == TASK_NEW)
15599	return;
15600
15601	detach_task_cfs_rq(p);
15602
15603	/* Tell se's cfs_rq has been changed -- migrated */
15604	p->se.avg.last_update_time = 0;
15605	set_task_rq(p, task_cpu(p));
15606	attach_task_cfs_rq(p);
15607	}
15608
15609	void free_fair_sched_group(struct task_group *tg)
15610	{
15611	free_percpu(tg->cfs_rq);
15612	}
15613
15614	int alloc_fair_sched_group(struct task_group tg, struct task_group parent)
15615	{
15616	struct cfs_tg_state __percpu *state;
15617	struct sched_entity *se;
15618	struct cfs_rq *cfs_rq;
15619	int i;
15620
15621	state = alloc_percpu_gfp(struct cfs_tg_state, GFP_KERNEL);
15622	if (!state)
15623	goto err;
15624
15625	tg->cfs_rq = &state->cfs_rq;
15626	tg->shares = NICE_0_LOAD;
15627
15628	init_cfs_bandwidth(tg_cfs_bandwidth(tg), tg_cfs_bandwidth(parent));
15629
15630	for_each_possible_cpu(i) {
15631	cfs_rq = tg_cfs_rq(tg, i);
15632	if (!cfs_rq)
15633	goto err;
15634
15635	se = tg_se(tg, i);
15636	init_cfs_rq(cfs_rq);
15637	init_tg_cfs_entry(tg, cfs_rq, se, i, tg_se(parent, i));
15638	init_entity_runnable_average(se);
15639	}
15640
15641	return 1;
15642
15643	err:
15644	return 0;
15645	}
15646
15647	void online_fair_sched_group(struct task_group *tg)
15648	{
15649	struct sched_entity *se;
15650	struct rq_flags rf;
15651	struct rq *rq;
15652	int i;
15653
15654	for_each_possible_cpu(i) {
15655	rq = cpu_rq(i);
15656	se = tg_se(tg, i);
15657	rq_lock_irq(rq, &rf);
15658	update_rq_clock(rq);
15659	attach_entity_cfs_rq(se);
15660	sync_throttle(tg, i);
15661	rq_unlock_irq(rq, &rf);
15662	}
15663	}
15664
15665	void unregister_fair_sched_group(struct task_group *tg)
15666	{
15667	int cpu;
15668
15669	destroy_cfs_bandwidth(tg_cfs_bandwidth(tg));
15670
15671	for_each_possible_cpu(cpu) {
15672	struct cfs_rq *cfs_rq = tg_cfs_rq(tg, cpu);
15673	struct sched_entity *se = tg_se(tg, cpu);
15674	struct rq *rq = cpu_rq(cpu);
15675
15676	if (se)
15677	remove_entity_load_avg(se);
15678
15679	/*
15680	* Only empty task groups can be destroyed; so we can speculatively
15681	* check on_list without danger of it being re-added.
15682	*/
15683	if (cfs_rq->on_list) {
15684	guard(rq_lock_irqsave)(rq);
15685	list_del_leaf_cfs_rq(cfs_rq);
15686	}
15687	}
15688	}
15689
15690	void init_tg_cfs_entry(struct task_group tg, struct cfs_rq cfs_rq,
15691	struct sched_entity *se, int cpu,
15692	struct sched_entity *parent)
15693	{
15694	struct rq *rq = cpu_rq(cpu);
15695
15696	cfs_rq->tg = tg;
15697	cfs_rq->rq = rq;
15698	init_cfs_rq_runtime(cfs_rq);
15699
15700	/* se could be NULL for root_task_group */
15701	if (!se)
15702	return;
15703
15704	if (!parent) {
15705	se->cfs_rq = &rq->cfs;
15706	se->depth = 0;
15707	} else {
15708	se->cfs_rq = parent->my_q;
15709	se->depth = parent->depth + 1;
15710	}
15711
15712	se->my_q = cfs_rq;
15713	/* guarantee group entities always have weight */
15714	update_load_set(&se->load, NICE_0_LOAD);
15715	se->parent = parent;
15716	}
15717
15718	static DEFINE_MUTEX(shares_mutex);
15719
15720	static int __sched_group_set_shares(struct task_group *tg, unsigned long shares)
15721	{
15722	int i;
15723
15724	lockdep_assert_held(&shares_mutex);
15725
15726	/*
15727	* We can't change the weight of the root cgroup.
15728	*/
15729	if (is_root_task_group(tg))
15730	return -EINVAL;
15731
15732	shares = clamp(shares, scale_load(MIN_SHARES), scale_load(MAX_SHARES));
15733
15734	if (tg->shares == shares)
15735	return 0;
15736
15737	tg->shares = shares;
15738	for_each_possible_cpu(i) {
15739	struct rq *rq = cpu_rq(i);
15740	struct sched_entity *se = tg_se(tg, i);
15741	struct rq_flags rf;
15742
15743	/* Propagate contribution to hierarchy */
15744	rq_lock_irqsave(rq, &rf);
15745	update_rq_clock(rq);
15746	for_each_sched_entity(se) {
15747	update_load_avg(cfs_rq_of(se), se, UPDATE_TG);
15748	update_cfs_group(se);
15749	}
15750	rq_unlock_irqrestore(rq, &rf);
15751	}
15752
15753	return 0;
15754	}
15755
15756	int sched_group_set_shares(struct task_group *tg, unsigned long shares)
15757	{
15758	int ret;
15759
15760	mutex_lock(&shares_mutex);
15761	if (tg_is_idle(tg))
15762	ret = -EINVAL;
15763	else
15764	ret = __sched_group_set_shares(tg, shares);
15765	mutex_unlock(&shares_mutex);
15766
15767	return ret;
15768	}
15769
15770	int sched_group_set_idle(struct task_group *tg, long idle)
15771	{
15772	int i;
15773
15774	if (tg == &root_task_group)
15775	return -EINVAL;
15776
15777	if (idle < 0 \|\| idle > 1)
15778	return -EINVAL;
15779
15780	mutex_lock(&shares_mutex);
15781
15782	if (tg->idle == idle) {
15783	mutex_unlock(&shares_mutex);
15784	return 0;
15785	}
15786
15787	tg->idle = idle;
15788
15789	for_each_possible_cpu(i) {
15790	struct rq *rq = cpu_rq(i);
15791	struct sched_entity *se = tg_se(tg, i);
15792	struct cfs_rq *grp_cfs_rq = tg_cfs_rq(tg, i);
15793	bool was_idle = cfs_rq_is_idle(grp_cfs_rq);
15794	long idle_task_delta;
15795	struct rq_flags rf;
15796
15797	rq_lock_irqsave(rq, &rf);
15798
15799	grp_cfs_rq->idle = idle;
15800	if (WARN_ON_ONCE(was_idle == cfs_rq_is_idle(grp_cfs_rq)))
15801	goto next_cpu;
15802
15803	idle_task_delta = grp_cfs_rq->h_nr_queued -
15804	grp_cfs_rq->h_nr_idle;
15805	if (!cfs_rq_is_idle(grp_cfs_rq))
15806	idle_task_delta *= -1;
15807
15808	for_each_sched_entity(se) {
15809	struct cfs_rq *cfs_rq = cfs_rq_of(se);
15810
15811	if (!se->on_rq)
15812	break;
15813
15814	cfs_rq->h_nr_idle += idle_task_delta;
15815
15816	/* Already accounted at parent level and above. */
15817	if (cfs_rq_is_idle(cfs_rq))
15818	break;
15819	}
15820
15821	next_cpu:
15822	rq_unlock_irqrestore(rq, &rf);
15823	}
15824
15825	/* Idle groups have minimum weight. */
15826	if (tg_is_idle(tg))
15827	__sched_group_set_shares(tg, scale_load(WEIGHT_IDLEPRIO));
15828	else
15829	__sched_group_set_shares(tg, NICE_0_LOAD);
15830
15831	mutex_unlock(&shares_mutex);
15832	return 0;
15833	}
15834
15835	#endif /* CONFIG_FAIR_GROUP_SCHED */
15836
15837
15838	static unsigned int get_rr_interval_fair(struct rq rq, struct task_struct task)
15839	{
15840	struct sched_entity *se = &task->se;
15841	unsigned int rr_interval = 0;
15842
15843	/*
15844	* Time slice is 0 for SCHED_OTHER tasks that are on an otherwise
15845	* idle runqueue:
15846	*/
15847	if (rq->cfs.load.weight)
15848	rr_interval = NS_TO_JIFFIES(se->slice);
15849
15850	return rr_interval;
15851	}
15852
15853	/*
15854	* All the scheduling class methods:
15855	*/
15856	DEFINE_SCHED_CLASS(fair) = {
15857	.enqueue_task = enqueue_task_fair,
15858	.dequeue_task = dequeue_task_fair,
15859	.yield_task = yield_task_fair,
15860	.yield_to_task = yield_to_task_fair,
15861
15862	.wakeup_preempt = wakeup_preempt_fair,
15863
15864	.pick_task = pick_task_fair,
15865	.put_prev_task = put_prev_task_fair,
15866	.set_next_task = set_next_task_fair,
15867
15868	.select_task_rq = select_task_rq_fair,
15869	.migrate_task_rq = migrate_task_rq_fair,
15870
15871	.rq_online = rq_online_fair,
15872	.rq_offline = rq_offline_fair,
15873
15874	.task_dead = task_dead_fair,
15875	.set_cpus_allowed = set_cpus_allowed_fair,
15876
15877	.task_tick = task_tick_fair,
15878	.task_fork = task_fork_fair,
15879
15880	.reweight_task = reweight_task_fair,
15881	.prio_changed = prio_changed_fair,
15882	.switching_from = switching_from_fair,
15883	.switched_from = switched_from_fair,
15884	.switched_to = switched_to_fair,
15885
15886	.get_rr_interval = get_rr_interval_fair,
15887
15888	.update_curr = update_curr_fair,
15889
15890	#ifdef CONFIG_FAIR_GROUP_SCHED
15891	.task_change_group = task_change_group_fair,
15892	#endif
15893
15894	#ifdef CONFIG_SCHED_CORE
15895	.task_is_throttled = task_is_throttled_fair,
15896	#endif
15897
15898	#ifdef CONFIG_UCLAMP_TASK
15899	.uclamp_enabled = 1,
15900	#endif
15901	};
15902
15903	void print_cfs_stats(struct seq_file *m, int cpu)
15904	{
15905	struct cfs_rq cfs_rq, pos;
15906
15907	rcu_read_lock();
15908	for_each_leaf_cfs_rq_safe(cpu_rq(cpu), cfs_rq, pos)
15909	print_cfs_rq(m, cpu, cfs_rq);
15910	rcu_read_unlock();
15911	}
15912
15913	#ifdef CONFIG_NUMA_BALANCING
15914	void show_numa_stats(struct task_struct p, struct seq_file m)
15915	{
15916	int node;
15917	unsigned long tsf = 0, tpf = 0, gsf = 0, gpf = 0;
15918	struct numa_group *ng;
15919
15920	rcu_read_lock();
15921	ng = rcu_dereference_all(p->numa_group);
15922	for_each_online_node(node) {
15923	if (p->numa_faults) {
15924	tsf = p->numa_faults[task_faults_idx(NUMA_MEM, node, 0)];
15925	tpf = p->numa_faults[task_faults_idx(NUMA_MEM, node, 1)];
15926	}
15927	if (ng) {
15928	gsf = ng->faults[task_faults_idx(NUMA_MEM, node, 0)];
15929	gpf = ng->faults[task_faults_idx(NUMA_MEM, node, 1)];
15930	}
15931	print_numa_stats(m, node, tsf, tpf, gsf, gpf);
15932	}
15933	rcu_read_unlock();
15934	}
15935	#endif /* CONFIG_NUMA_BALANCING */
15936
15937	__init void init_sched_fair_class(void)
15938	{
15939	int i;
15940
15941	for_each_possible_cpu(i) {
15942	zalloc_cpumask_var_node(&per_cpu(load_balance_mask, i), GFP_KERNEL, cpu_to_node(i));
15943	zalloc_cpumask_var_node(&per_cpu(select_rq_mask, i), GFP_KERNEL, cpu_to_node(i));
15944	zalloc_cpumask_var_node(&per_cpu(should_we_balance_tmpmask, i),
15945	GFP_KERNEL, cpu_to_node(i));
15946
15947	#ifdef CONFIG_CFS_BANDWIDTH
15948	INIT_CSD(&cpu_rq(i)->cfsb_csd, __cfsb_csd_unthrottle, cpu_rq(i));
15949	INIT_LIST_HEAD(&cpu_rq(i)->cfsb_csd_list);
15950	#endif
15951	}
15952
15953	open_softirq(SCHED_SOFTIRQ, sched_balance_softirq);
15954
15955	#ifdef CONFIG_NO_HZ_COMMON
15956	nohz.next_balance = jiffies;
15957	nohz.next_blocked = jiffies;
15958	zalloc_cpumask_var(&nohz.idle_cpus_mask, GFP_NOWAIT);
15959	#endif
15960	}