linux - kernel/workqueue.c

8005 lines · 224.0 KiB

1	// SPDX-License-Identifier: GPL-2.0-only
2	/*
3	* kernel/workqueue.c - generic async execution with shared worker pool
4	*
5	* Copyright (C) 2002 Ingo Molnar
6	*
7	* Derived from the taskqueue/keventd code by:
8	* David Woodhouse <[email protected]>
9	* Andrew Morton
10	* Kai Petzke <[email protected]>
11	* Theodore Ts'o <[email protected]>
12	*
13	* Made to use alloc_percpu by Christoph Lameter.
14	*
15	* Copyright (C) 2010 SUSE Linux Products GmbH
16	* Copyright (C) 2010 Tejun Heo <[email protected]>
17	*
18	* This is the generic async execution mechanism. Work items as are
19	* executed in process context. The worker pool is shared and
20	* automatically managed. There are two worker pools for each CPU (one for
21	* normal work items and the other for high priority ones) and some extra
22	* pools for workqueues which are not bound to any specific CPU - the
23	* number of these backing pools is dynamic.
24	*
25	* Please read Documentation/core-api/workqueue.rst for details.
26	*/
27
28	#include <linux/export.h>
29	#include <linux/kernel.h>
30	#include <linux/sched.h>
31	#include <linux/init.h>
32	#include <linux/interrupt.h>
33	#include <linux/signal.h>
34	#include <linux/completion.h>
35	#include <linux/workqueue.h>
36	#include <linux/slab.h>
37	#include <linux/cpu.h>
38	#include <linux/notifier.h>
39	#include <linux/kthread.h>
40	#include <linux/hardirq.h>
41	#include <linux/mempolicy.h>
42	#include <linux/freezer.h>
43	#include <linux/debug_locks.h>
44	#include <linux/lockdep.h>
45	#include <linux/idr.h>
46	#include <linux/jhash.h>
47	#include <linux/hashtable.h>
48	#include <linux/rculist.h>
49	#include <linux/nodemask.h>
50	#include <linux/moduleparam.h>
51	#include <linux/uaccess.h>
52	#include <linux/sched/isolation.h>
53	#include <linux/sched/debug.h>
54	#include <linux/nmi.h>
55	#include <linux/kvm_para.h>
56	#include <linux/delay.h>
57	#include <linux/irq_work.h>
58
59	#include "workqueue_internal.h"
60
61	enum worker_pool_flags {
62	/*
63	* worker_pool flags
64	*
65	* A bound pool is either associated or disassociated with its CPU.
66	* While associated (!DISASSOCIATED), all workers are bound to the
67	* CPU and none has %WORKER_UNBOUND set and concurrency management
68	* is in effect.
69	*
70	* While DISASSOCIATED, the cpu may be offline and all workers have
71	* %WORKER_UNBOUND set and concurrency management disabled, and may
72	* be executing on any CPU. The pool behaves as an unbound one.
73	*
74	* Note that DISASSOCIATED should be flipped only while holding
75	* wq_pool_attach_mutex to avoid changing binding state while
76	* worker_attach_to_pool() is in progress.
77	*
78	* As there can only be one concurrent BH execution context per CPU, a
79	* BH pool is per-CPU and always DISASSOCIATED.
80	*/
81	POOL_BH = 1 << 0, /* is a BH pool */
82	POOL_MANAGER_ACTIVE = 1 << 1, /* being managed */
83	POOL_DISASSOCIATED = 1 << 2, /* cpu can't serve workers */
84	POOL_BH_DRAINING = 1 << 3, /* draining after CPU offline */
85	};
86
87	enum worker_flags {
88	/* worker flags */
89	WORKER_DIE = 1 << 1, /* die die die */
90	WORKER_IDLE = 1 << 2, /* is idle */
91	WORKER_PREP = 1 << 3, /* preparing to run works */
92	WORKER_CPU_INTENSIVE = 1 << 6, /* cpu intensive */
93	WORKER_UNBOUND = 1 << 7, /* worker is unbound */
94	WORKER_REBOUND = 1 << 8, /* worker was rebound */
95
96	WORKER_NOT_RUNNING = WORKER_PREP \| WORKER_CPU_INTENSIVE \|
97	WORKER_UNBOUND \| WORKER_REBOUND,
98	};
99
100	enum work_cancel_flags {
101	WORK_CANCEL_DELAYED = 1 << 0, /* canceling a delayed_work */
102	WORK_CANCEL_DISABLE = 1 << 1, /* canceling to disable */
103	};
104
105	enum wq_internal_consts {
106	NR_STD_WORKER_POOLS = 2, /* # standard pools per cpu */
107
108	UNBOUND_POOL_HASH_ORDER = 6, /* hashed by pool->attrs */
109	BUSY_WORKER_HASH_ORDER = 6, /* 64 pointers */
110
111	MAX_IDLE_WORKERS_RATIO = 4, /* 1/4 of busy can be idle */
112	IDLE_WORKER_TIMEOUT = 300 * HZ, /* keep idle ones for 5 mins */
113
114	MAYDAY_INITIAL_TIMEOUT = HZ / 100 >= 2 ? HZ / 100 : 2,
115	/* call for help after 10ms
116	(min two ticks) */
117	MAYDAY_INTERVAL = HZ / 10, /* and then every 100ms */
118	CREATE_COOLDOWN = HZ, /* time to breath after fail */
119
120	/*
121	* Rescue workers are used only on emergencies and shared by
122	* all cpus. Give MIN_NICE.
123	*/
124	RESCUER_NICE_LEVEL = MIN_NICE,
125	HIGHPRI_NICE_LEVEL = MIN_NICE,
126
127	WQ_NAME_LEN = 32,
128	WORKER_ID_LEN = 10 + WQ_NAME_LEN, /* "kworker/R-" + WQ_NAME_LEN */
129	};
130
131	/*
132	* We don't want to trap softirq for too long. See MAX_SOFTIRQ_TIME and
133	* MAX_SOFTIRQ_RESTART in kernel/softirq.c. These are macros because
134	* msecs_to_jiffies() can't be an initializer.
135	*/
136	#define BH_WORKER_JIFFIES msecs_to_jiffies(2)
137	#define BH_WORKER_RESTARTS 10
138
139	/*
140	* Structure fields follow one of the following exclusion rules.
141	*
142	* I: Modifiable by initialization/destruction paths and read-only for
143	* everyone else.
144	*
145	* P: Preemption protected. Disabling preemption is enough and should
146	* only be modified and accessed from the local cpu.
147	*
148	* L: pool->lock protected. Access with pool->lock held.
149	*
150	* LN: pool->lock and wq_node_nr_active->lock protected for writes. Either for
151	* reads.
152	*
153	* K: Only modified by worker while holding pool->lock. Can be safely read by
154	* self, while holding pool->lock or from IRQ context if %current is the
155	* kworker.
156	*
157	* S: Only modified by worker self.
158	*
159	* A: wq_pool_attach_mutex protected.
160	*
161	* PL: wq_pool_mutex protected.
162	*
163	* PR: wq_pool_mutex protected for writes. RCU protected for reads.
164	*
165	* PW: wq_pool_mutex and wq->mutex protected for writes. Either for reads.
166	*
167	* PWR: wq_pool_mutex and wq->mutex protected for writes. Either or
168	* RCU for reads.
169	*
170	* WQ: wq->mutex protected.
171	*
172	* WR: wq->mutex protected for writes. RCU protected for reads.
173	*
174	* WO: wq->mutex protected for writes. Updated with WRITE_ONCE() and can be read
175	* with READ_ONCE() without locking.
176	*
177	* MD: wq_mayday_lock protected.
178	*
179	* WD: Used internally by the watchdog.
180	*/
181
182	/* struct worker is defined in workqueue_internal.h */
183
184	struct worker_pool {
185	raw_spinlock_t lock; /* the pool lock */
186	int cpu; /* I: the associated cpu */
187	int node; /* I: the associated node ID */
188	int id; /* I: pool ID */
189	unsigned int flags; /* L: flags */
190
191	unsigned long watchdog_ts; /* L: watchdog timestamp */
192	bool cpu_stall; /* WD: stalled cpu bound pool */
193
194	/*
195	* The counter is incremented in a process context on the associated CPU
196	* w/ preemption disabled, and decremented or reset in the same context
197	* but w/ pool->lock held. The readers grab pool->lock and are
198	* guaranteed to see if the counter reached zero.
199	*/
200	int nr_running;
201
202	struct list_head worklist; /* L: list of pending works */
203
204	int nr_workers; /* L: total number of workers */
205	int nr_idle; /* L: currently idle workers */
206
207	struct list_head idle_list; /* L: list of idle workers */
208	struct timer_list idle_timer; /* L: worker idle timeout */
209	struct work_struct idle_cull_work; /* L: worker idle cleanup */
210
211	struct timer_list mayday_timer; /* L: SOS timer for workers */
212
213	/* a workers is either on busy_hash or idle_list, or the manager */
214	DECLARE_HASHTABLE(busy_hash, BUSY_WORKER_HASH_ORDER);
215	/* L: hash of busy workers */
216
217	struct worker manager; / L: purely informational */
218	struct list_head workers; /* A: attached workers */
219	struct list_head dying_workers; /* A: workers about to die */
220	struct completion detach_completion; / all workers detached */
221
222	struct ida worker_ida; /* worker IDs for task name */
223
224	struct workqueue_attrs attrs; / I: worker attributes */
225	struct hlist_node hash_node; /* PL: unbound_pool_hash node */
226	int refcnt; /* PL: refcnt for unbound pools */
227
228	/*
229	* Destruction of pool is RCU protected to allow dereferences
230	* from get_work_pool().
231	*/
232	struct rcu_head rcu;
233	};
234
235	/*
236	* Per-pool_workqueue statistics. These can be monitored using
237	* tools/workqueue/wq_monitor.py.
238	*/
239	enum pool_workqueue_stats {
240	PWQ_STAT_STARTED, /* work items started execution */
241	PWQ_STAT_COMPLETED, /* work items completed execution */
242	PWQ_STAT_CPU_TIME, /* total CPU time consumed */
243	PWQ_STAT_CPU_INTENSIVE, /* wq_cpu_intensive_thresh_us violations */
244	PWQ_STAT_CM_WAKEUP, /* concurrency-management worker wakeups */
245	PWQ_STAT_REPATRIATED, /* unbound workers brought back into scope */
246	PWQ_STAT_MAYDAY, /* maydays to rescuer */
247	PWQ_STAT_RESCUED, /* linked work items executed by rescuer */
248
249	PWQ_NR_STATS,
250	};
251
252	/*
253	* The per-pool workqueue. While queued, bits below WORK_PWQ_SHIFT
254	* of work_struct->data are used for flags and the remaining high bits
255	* point to the pwq; thus, pwqs need to be aligned at two's power of the
256	* number of flag bits.
257	*/
258	struct pool_workqueue {
259	struct worker_pool pool; / I: the associated pool */
260	struct workqueue_struct wq; / I: the owning workqueue */
261	int work_color; /* L: current color */
262	int flush_color; /* L: flushing color */
263	int refcnt; /* L: reference count */
264	int nr_in_flight[WORK_NR_COLORS];
265	/* L: nr of in_flight works */
266	bool plugged; /* L: execution suspended */
267
268	/*
269	* nr_active management and WORK_STRUCT_INACTIVE:
270	*
271	* When pwq->nr_active >= max_active, new work item is queued to
272	* pwq->inactive_works instead of pool->worklist and marked with
273	* WORK_STRUCT_INACTIVE.
274	*
275	* All work items marked with WORK_STRUCT_INACTIVE do not participate in
276	* nr_active and all work items in pwq->inactive_works are marked with
277	* WORK_STRUCT_INACTIVE. But not all WORK_STRUCT_INACTIVE work items are
278	* in pwq->inactive_works. Some of them are ready to run in
279	* pool->worklist or worker->scheduled. Those work itmes are only struct
280	* wq_barrier which is used for flush_work() and should not participate
281	* in nr_active. For non-barrier work item, it is marked with
282	* WORK_STRUCT_INACTIVE iff it is in pwq->inactive_works.
283	*/
284	int nr_active; /* L: nr of active works */
285	struct list_head inactive_works; /* L: inactive works */
286	struct list_head pending_node; /* LN: node on wq_node_nr_active->pending_pwqs */
287	struct list_head pwqs_node; /* WR: node on wq->pwqs */
288	struct list_head mayday_node; /* MD: node on wq->maydays */
289
290	u64 stats[PWQ_NR_STATS];
291
292	/*
293	* Release of unbound pwq is punted to a kthread_worker. See put_pwq()
294	* and pwq_release_workfn() for details. pool_workqueue itself is also
295	* RCU protected so that the first pwq can be determined without
296	* grabbing wq->mutex.
297	*/
298	struct kthread_work release_work;
299	struct rcu_head rcu;
300	} __aligned(1 << WORK_STRUCT_PWQ_SHIFT);
301
302	/*
303	* Structure used to wait for workqueue flush.
304	*/
305	struct wq_flusher {
306	struct list_head list; /* WQ: list of flushers */
307	int flush_color; /* WQ: flush color waiting for */
308	struct completion done; /* flush completion */
309	};
310
311	struct wq_device;
312
313	/*
314	* Unlike in a per-cpu workqueue where max_active limits its concurrency level
315	* on each CPU, in an unbound workqueue, max_active applies to the whole system.
316	* As sharing a single nr_active across multiple sockets can be very expensive,
317	* the counting and enforcement is per NUMA node.
318	*
319	* The following struct is used to enforce per-node max_active. When a pwq wants
320	* to start executing a work item, it should increment ->nr using
321	* tryinc_node_nr_active(). If acquisition fails due to ->nr already being over
322	* ->max, the pwq is queued on ->pending_pwqs. As in-flight work items finish
323	* and decrement ->nr, node_activate_pending_pwq() activates the pending pwqs in
324	* round-robin order.
325	*/
326	struct wq_node_nr_active {
327	int max; /* per-node max_active */
328	atomic_t nr; /* per-node nr_active */
329	raw_spinlock_t lock; /* nests inside pool locks */
330	struct list_head pending_pwqs; /* LN: pwqs with inactive works */
331	};
332
333	/*
334	* The externally visible workqueue. It relays the issued work items to
335	* the appropriate worker_pool through its pool_workqueues.
336	*/
337	struct workqueue_struct {
338	struct list_head pwqs; /* WR: all pwqs of this wq */
339	struct list_head list; /* PR: list of all workqueues */
340
341	struct mutex mutex; /* protects this wq */
342	int work_color; /* WQ: current work color */
343	int flush_color; /* WQ: current flush color */
344	atomic_t nr_pwqs_to_flush; /* flush in progress */
345	struct wq_flusher first_flusher; / WQ: first flusher */
346	struct list_head flusher_queue; /* WQ: flush waiters */
347	struct list_head flusher_overflow; /* WQ: flush overflow list */
348
349	struct list_head maydays; /* MD: pwqs requesting rescue */
350	struct worker rescuer; / MD: rescue worker */
351
352	int nr_drainers; /* WQ: drain in progress */
353
354	/* See alloc_workqueue() function comment for info on min/max_active */
355	int max_active; /* WO: max active works */
356	int min_active; /* WO: min active works */
357	int saved_max_active; /* WQ: saved max_active */
358	int saved_min_active; /* WQ: saved min_active */
359
360	struct workqueue_attrs unbound_attrs; / PW: only for unbound wqs */
361	struct pool_workqueue __rcu dfl_pwq; / PW: only for unbound wqs */
362
363	#ifdef CONFIG_SYSFS
364	struct wq_device wq_dev; / I: for sysfs interface */
365	#endif
366	#ifdef CONFIG_LOCKDEP
367	char *lock_name;
368	struct lock_class_key key;
369	struct lockdep_map lockdep_map;
370	#endif
371	char name[WQ_NAME_LEN]; /* I: workqueue name */
372
373	/*
374	* Destruction of workqueue_struct is RCU protected to allow walking
375	* the workqueues list without grabbing wq_pool_mutex.
376	* This is used to dump all workqueues from sysrq.
377	*/
378	struct rcu_head rcu;
379
380	/* hot fields used during command issue, aligned to cacheline */
381	unsigned int flags ____cacheline_aligned; /* WQ: WQ_* flags */
382	struct pool_workqueue __percpu __rcu *cpu_pwq; / I: per-cpu pwqs */
383	struct wq_node_nr_active node_nr_active[]; / I: per-node nr_active */
384	};
385
386	/*
387	* Each pod type describes how CPUs should be grouped for unbound workqueues.
388	* See the comment above workqueue_attrs->affn_scope.
389	*/
390	struct wq_pod_type {
391	int nr_pods; /* number of pods */
392	cpumask_var_t pod_cpus; / pod -> cpus */
393	int pod_node; / pod -> node */
394	int cpu_pod; / cpu -> pod */
395	};
396
397	struct work_offq_data {
398	u32 pool_id;
399	u32 disable;
400	u32 flags;
401	};
402
403	static const char *wq_affn_names[WQ_AFFN_NR_TYPES] = {
404	[WQ_AFFN_DFL] = "default",
405	[WQ_AFFN_CPU] = "cpu",
406	[WQ_AFFN_SMT] = "smt",
407	[WQ_AFFN_CACHE] = "cache",
408	[WQ_AFFN_NUMA] = "numa",
409	[WQ_AFFN_SYSTEM] = "system",
410	};
411
412	/*
413	* Per-cpu work items which run for longer than the following threshold are
414	* automatically considered CPU intensive and excluded from concurrency
415	* management to prevent them from noticeably delaying other per-cpu work items.
416	* ULONG_MAX indicates that the user hasn't overridden it with a boot parameter.
417	* The actual value is initialized in wq_cpu_intensive_thresh_init().
418	*/
419	static unsigned long wq_cpu_intensive_thresh_us = ULONG_MAX;
420	module_param_named(cpu_intensive_thresh_us, wq_cpu_intensive_thresh_us, ulong, 0644);
421	#ifdef CONFIG_WQ_CPU_INTENSIVE_REPORT
422	static unsigned int wq_cpu_intensive_warning_thresh = 4;
423	module_param_named(cpu_intensive_warning_thresh, wq_cpu_intensive_warning_thresh, uint, 0644);
424	#endif
425
426	/* see the comment above the definition of WQ_POWER_EFFICIENT */
427	static bool wq_power_efficient = IS_ENABLED(CONFIG_WQ_POWER_EFFICIENT_DEFAULT);
428	module_param_named(power_efficient, wq_power_efficient, bool, 0444);
429
430	static bool wq_online; /* can kworkers be created yet? */
431	static bool wq_topo_initialized __read_mostly = false;
432
433	static struct kmem_cache *pwq_cache;
434
435	static struct wq_pod_type wq_pod_types[WQ_AFFN_NR_TYPES];
436	static enum wq_affn_scope wq_affn_dfl = WQ_AFFN_CACHE;
437
438	/* buf for wq_update_unbound_pod_attrs(), protected by CPU hotplug exclusion */
439	static struct workqueue_attrs *wq_update_pod_attrs_buf;
440
441	static DEFINE_MUTEX(wq_pool_mutex); /* protects pools and workqueues list */
442	static DEFINE_MUTEX(wq_pool_attach_mutex); /* protects worker attach/detach */
443	static DEFINE_RAW_SPINLOCK(wq_mayday_lock); /* protects wq->maydays list */
444	/* wait for manager to go away */
445	static struct rcuwait manager_wait = __RCUWAIT_INITIALIZER(manager_wait);
446
447	static LIST_HEAD(workqueues); /* PR: list of all workqueues */
448	static bool workqueue_freezing; /* PL: have wqs started freezing? */
449
450	/* PL&A: allowable cpus for unbound wqs and work items */
451	static cpumask_var_t wq_unbound_cpumask;
452
453	/* PL: user requested unbound cpumask via sysfs */
454	static cpumask_var_t wq_requested_unbound_cpumask;
455
456	/* PL: isolated cpumask to be excluded from unbound cpumask */
457	static cpumask_var_t wq_isolated_cpumask;
458
459	/* for further constrain wq_unbound_cpumask by cmdline parameter*/
460	static struct cpumask wq_cmdline_cpumask __initdata;
461
462	/* CPU where unbound work was last round robin scheduled from this CPU */
463	static DEFINE_PER_CPU(int, wq_rr_cpu_last);
464
465	/*
466	* Local execution of unbound work items is no longer guaranteed. The
467	* following always forces round-robin CPU selection on unbound work items
468	* to uncover usages which depend on it.
469	*/
470	#ifdef CONFIG_DEBUG_WQ_FORCE_RR_CPU
471	static bool wq_debug_force_rr_cpu = true;
472	#else
473	static bool wq_debug_force_rr_cpu = false;
474	#endif
475	module_param_named(debug_force_rr_cpu, wq_debug_force_rr_cpu, bool, 0644);
476
477	/* to raise softirq for the BH worker pools on other CPUs */
478	static DEFINE_PER_CPU_SHARED_ALIGNED(struct irq_work [NR_STD_WORKER_POOLS],
479	bh_pool_irq_works);
480
481	/* the BH worker pools */
482	static DEFINE_PER_CPU_SHARED_ALIGNED(struct worker_pool [NR_STD_WORKER_POOLS],
483	bh_worker_pools);
484
485	/* the per-cpu worker pools */
486	static DEFINE_PER_CPU_SHARED_ALIGNED(struct worker_pool [NR_STD_WORKER_POOLS],
487	cpu_worker_pools);
488
489	static DEFINE_IDR(worker_pool_idr); /* PR: idr of all pools */
490
491	/* PL: hash of all unbound pools keyed by pool->attrs */
492	static DEFINE_HASHTABLE(unbound_pool_hash, UNBOUND_POOL_HASH_ORDER);
493
494	/* I: attributes used when instantiating standard unbound pools on demand */
495	static struct workqueue_attrs *unbound_std_wq_attrs[NR_STD_WORKER_POOLS];
496
497	/* I: attributes used when instantiating ordered pools on demand */
498	static struct workqueue_attrs *ordered_wq_attrs[NR_STD_WORKER_POOLS];
499
500	/*
501	* I: kthread_worker to release pwq's. pwq release needs to be bounced to a
502	* process context while holding a pool lock. Bounce to a dedicated kthread
503	* worker to avoid A-A deadlocks.
504	*/
505	static struct kthread_worker *pwq_release_worker __ro_after_init;
506
507	struct workqueue_struct *system_wq __ro_after_init;
508	EXPORT_SYMBOL(system_wq);
509	struct workqueue_struct *system_highpri_wq __ro_after_init;
510	EXPORT_SYMBOL_GPL(system_highpri_wq);
511	struct workqueue_struct *system_long_wq __ro_after_init;
512	EXPORT_SYMBOL_GPL(system_long_wq);
513	struct workqueue_struct *system_unbound_wq __ro_after_init;
514	EXPORT_SYMBOL_GPL(system_unbound_wq);
515	struct workqueue_struct *system_freezable_wq __ro_after_init;
516	EXPORT_SYMBOL_GPL(system_freezable_wq);
517	struct workqueue_struct *system_power_efficient_wq __ro_after_init;
518	EXPORT_SYMBOL_GPL(system_power_efficient_wq);
519	struct workqueue_struct *system_freezable_power_efficient_wq __ro_after_init;
520	EXPORT_SYMBOL_GPL(system_freezable_power_efficient_wq);
521	struct workqueue_struct *system_bh_wq;
522	EXPORT_SYMBOL_GPL(system_bh_wq);
523	struct workqueue_struct *system_bh_highpri_wq;
524	EXPORT_SYMBOL_GPL(system_bh_highpri_wq);
525
526	static int worker_thread(void *__worker);
527	static void workqueue_sysfs_unregister(struct workqueue_struct *wq);
528	static void show_pwq(struct pool_workqueue *pwq);
529	static void show_one_worker_pool(struct worker_pool *pool);
530
531	#define CREATE_TRACE_POINTS
532	#include <trace/events/workqueue.h>
533
534	#define assert_rcu_or_pool_mutex() \
535	RCU_LOCKDEP_WARN(!rcu_read_lock_any_held() && \
536	!lockdep_is_held(&wq_pool_mutex), \
537	"RCU or wq_pool_mutex should be held")
538
539	#define assert_rcu_or_wq_mutex_or_pool_mutex(wq) \
540	RCU_LOCKDEP_WARN(!rcu_read_lock_any_held() && \
541	!lockdep_is_held(&wq->mutex) && \
542	!lockdep_is_held(&wq_pool_mutex), \
543	"RCU, wq->mutex or wq_pool_mutex should be held")
544
545	#define for_each_bh_worker_pool(pool, cpu) \
546	for ((pool) = &per_cpu(bh_worker_pools, cpu)[0]; \
547	(pool) < &per_cpu(bh_worker_pools, cpu)[NR_STD_WORKER_POOLS]; \
548	(pool)++)
549
550	#define for_each_cpu_worker_pool(pool, cpu) \
551	for ((pool) = &per_cpu(cpu_worker_pools, cpu)[0]; \
552	(pool) < &per_cpu(cpu_worker_pools, cpu)[NR_STD_WORKER_POOLS]; \
553	(pool)++)
554
555	/**
556	* for_each_pool - iterate through all worker_pools in the system
557	* @pool: iteration cursor
558	* @pi: integer used for iteration
559	*
560	* This must be called either with wq_pool_mutex held or RCU read
561	* locked. If the pool needs to be used beyond the locking in effect, the
562	* caller is responsible for guaranteeing that the pool stays online.
563	*
564	* The if/else clause exists only for the lockdep assertion and can be
565	* ignored.
566	*/
567	#define for_each_pool(pool, pi) \
568	idr_for_each_entry(&worker_pool_idr, pool, pi) \
569	if (({ assert_rcu_or_pool_mutex(); false; })) { } \
570	else
571
572	/**
573	* for_each_pool_worker - iterate through all workers of a worker_pool
574	* @worker: iteration cursor
575	* @pool: worker_pool to iterate workers of
576	*
577	* This must be called with wq_pool_attach_mutex.
578	*
579	* The if/else clause exists only for the lockdep assertion and can be
580	* ignored.
581	*/
582	#define for_each_pool_worker(worker, pool) \
583	list_for_each_entry((worker), &(pool)->workers, node) \
584	if (({ lockdep_assert_held(&wq_pool_attach_mutex); false; })) { } \
585	else
586
587	/**
588	* for_each_pwq - iterate through all pool_workqueues of the specified workqueue
589	* @pwq: iteration cursor
590	* @wq: the target workqueue
591	*
592	* This must be called either with wq->mutex held or RCU read locked.
593	* If the pwq needs to be used beyond the locking in effect, the caller is
594	* responsible for guaranteeing that the pwq stays online.
595	*
596	* The if/else clause exists only for the lockdep assertion and can be
597	* ignored.
598	*/
599	#define for_each_pwq(pwq, wq) \
600	list_for_each_entry_rcu((pwq), &(wq)->pwqs, pwqs_node, \
601	lockdep_is_held(&(wq->mutex)))
602
603	#ifdef CONFIG_DEBUG_OBJECTS_WORK
604
605	static const struct debug_obj_descr work_debug_descr;
606
607	static void work_debug_hint(void addr)
608	{
609	return ((struct work_struct *) addr)->func;
610	}
611
612	static bool work_is_static_object(void *addr)
613	{
614	struct work_struct *work = addr;
615
616	return test_bit(WORK_STRUCT_STATIC_BIT, work_data_bits(work));
617	}
618
619	/*
620	* fixup_init is called when:
621	* - an active object is initialized
622	*/
623	static bool work_fixup_init(void *addr, enum debug_obj_state state)
624	{
625	struct work_struct *work = addr;
626
627	switch (state) {
628	case ODEBUG_STATE_ACTIVE:
629	cancel_work_sync(work);
630	debug_object_init(work, &work_debug_descr);
631	return true;
632	default:
633	return false;
634	}
635	}
636
637	/*
638	* fixup_free is called when:
639	* - an active object is freed
640	*/
641	static bool work_fixup_free(void *addr, enum debug_obj_state state)
642	{
643	struct work_struct *work = addr;
644
645	switch (state) {
646	case ODEBUG_STATE_ACTIVE:
647	cancel_work_sync(work);
648	debug_object_free(work, &work_debug_descr);
649	return true;
650	default:
651	return false;
652	}
653	}
654
655	static const struct debug_obj_descr work_debug_descr = {
656	.name = "work_struct",
657	.debug_hint = work_debug_hint,
658	.is_static_object = work_is_static_object,
659	.fixup_init = work_fixup_init,
660	.fixup_free = work_fixup_free,
661	};
662
663	static inline void debug_work_activate(struct work_struct *work)
664	{
665	debug_object_activate(work, &work_debug_descr);
666	}
667
668	static inline void debug_work_deactivate(struct work_struct *work)
669	{
670	debug_object_deactivate(work, &work_debug_descr);
671	}
672
673	void __init_work(struct work_struct *work, int onstack)
674	{
675	if (onstack)
676	debug_object_init_on_stack(work, &work_debug_descr);
677	else
678	debug_object_init(work, &work_debug_descr);
679	}
680	EXPORT_SYMBOL_GPL(__init_work);
681
682	void destroy_work_on_stack(struct work_struct *work)
683	{
684	debug_object_free(work, &work_debug_descr);
685	}
686	EXPORT_SYMBOL_GPL(destroy_work_on_stack);
687
688	void destroy_delayed_work_on_stack(struct delayed_work *work)
689	{
690	destroy_timer_on_stack(&work->timer);
691	debug_object_free(&work->work, &work_debug_descr);
692	}
693	EXPORT_SYMBOL_GPL(destroy_delayed_work_on_stack);
694
695	#else
696	static inline void debug_work_activate(struct work_struct *work) { }
697	static inline void debug_work_deactivate(struct work_struct *work) { }
698	#endif
699
700	/**
701	* worker_pool_assign_id - allocate ID and assign it to @pool
702	* @pool: the pool pointer of interest
703	*
704	* Returns 0 if ID in [0, WORK_OFFQ_POOL_NONE) is allocated and assigned
705	* successfully, -errno on failure.
706	*/
707	static int worker_pool_assign_id(struct worker_pool *pool)
708	{
709	int ret;
710
711	lockdep_assert_held(&wq_pool_mutex);
712
713	ret = idr_alloc(&worker_pool_idr, pool, 0, WORK_OFFQ_POOL_NONE,
714	GFP_KERNEL);
715	if (ret >= 0) {
716	pool->id = ret;
717	return 0;
718	}
719	return ret;
720	}
721
722	static struct pool_workqueue __rcu **
723	unbound_pwq_slot(struct workqueue_struct *wq, int cpu)
724	{
725	if (cpu >= 0)
726	return per_cpu_ptr(wq->cpu_pwq, cpu);
727	else
728	return &wq->dfl_pwq;
729	}
730
731	/* @cpu < 0 for dfl_pwq */
732	static struct pool_workqueue unbound_pwq(struct workqueue_struct wq, int cpu)
733	{
734	return rcu_dereference_check(*unbound_pwq_slot(wq, cpu),
735	lockdep_is_held(&wq_pool_mutex) \|\|
736	lockdep_is_held(&wq->mutex));
737	}
738
739	/**
740	* unbound_effective_cpumask - effective cpumask of an unbound workqueue
741	* @wq: workqueue of interest
742	*
743	* @wq->unbound_attrs->cpumask contains the cpumask requested by the user which
744	* is masked with wq_unbound_cpumask to determine the effective cpumask. The
745	* default pwq is always mapped to the pool with the current effective cpumask.
746	*/
747	static struct cpumask unbound_effective_cpumask(struct workqueue_struct wq)
748	{
749	return unbound_pwq(wq, -1)->pool->attrs->__pod_cpumask;
750	}
751
752	static unsigned int work_color_to_flags(int color)
753	{
754	return color << WORK_STRUCT_COLOR_SHIFT;
755	}
756
757	static int get_work_color(unsigned long work_data)
758	{
759	return (work_data >> WORK_STRUCT_COLOR_SHIFT) &
760	((1 << WORK_STRUCT_COLOR_BITS) - 1);
761	}
762
763	static int work_next_color(int color)
764	{
765	return (color + 1) % WORK_NR_COLORS;
766	}
767
768	static unsigned long pool_offq_flags(struct worker_pool *pool)
769	{
770	return (pool->flags & POOL_BH) ? WORK_OFFQ_BH : 0;
771	}
772
773	/*
774	* While queued, %WORK_STRUCT_PWQ is set and non flag bits of a work's data
775	* contain the pointer to the queued pwq. Once execution starts, the flag
776	* is cleared and the high bits contain OFFQ flags and pool ID.
777	*
778	* set_work_pwq(), set_work_pool_and_clear_pending() and mark_work_canceling()
779	* can be used to set the pwq, pool or clear work->data. These functions should
780	* only be called while the work is owned - ie. while the PENDING bit is set.
781	*
782	* get_work_pool() and get_work_pwq() can be used to obtain the pool or pwq
783	* corresponding to a work. Pool is available once the work has been
784	* queued anywhere after initialization until it is sync canceled. pwq is
785	* available only while the work item is queued.
786	*/
787	static inline void set_work_data(struct work_struct *work, unsigned long data)
788	{
789	WARN_ON_ONCE(!work_pending(work));
790	atomic_long_set(&work->data, data \| work_static(work));
791	}
792
793	static void set_work_pwq(struct work_struct work, struct pool_workqueue pwq,
794	unsigned long flags)
795	{
796	set_work_data(work, (unsigned long)pwq \| WORK_STRUCT_PENDING \|
797	WORK_STRUCT_PWQ \| flags);
798	}
799
800	static void set_work_pool_and_keep_pending(struct work_struct *work,
801	int pool_id, unsigned long flags)
802	{
803	set_work_data(work, ((unsigned long)pool_id << WORK_OFFQ_POOL_SHIFT) \|
804	WORK_STRUCT_PENDING \| flags);
805	}
806
807	static void set_work_pool_and_clear_pending(struct work_struct *work,
808	int pool_id, unsigned long flags)
809	{
810	/*
811	* The following wmb is paired with the implied mb in
812	* test_and_set_bit(PENDING) and ensures all updates to @work made
813	* here are visible to and precede any updates by the next PENDING
814	* owner.
815	*/
816	smp_wmb();
817	set_work_data(work, ((unsigned long)pool_id << WORK_OFFQ_POOL_SHIFT) \|
818	flags);
819	/*
820	* The following mb guarantees that previous clear of a PENDING bit
821	* will not be reordered with any speculative LOADS or STORES from
822	* work->current_func, which is executed afterwards. This possible
823	* reordering can lead to a missed execution on attempt to queue
824	* the same @work. E.g. consider this case:
825	*
826	* CPU#0 CPU#1
827	* ---------------------------- --------------------------------
828	*
829	* 1 STORE event_indicated
830	* 2 queue_work_on() {
831	* 3 test_and_set_bit(PENDING)
832	* 4 } set_..._and_clear_pending() {
833	* 5 set_work_data() # clear bit
834	* 6 smp_mb()
835	* 7 work->current_func() {
836	* 8 LOAD event_indicated
837	* }
838	*
839	* Without an explicit full barrier speculative LOAD on line 8 can
840	* be executed before CPU#0 does STORE on line 1. If that happens,
841	* CPU#0 observes the PENDING bit is still set and new execution of
842	* a @work is not queued in a hope, that CPU#1 will eventually
843	* finish the queued @work. Meanwhile CPU#1 does not see
844	* event_indicated is set, because speculative LOAD was executed
845	* before actual STORE.
846	*/
847	smp_mb();
848	}
849
850	static inline struct pool_workqueue *work_struct_pwq(unsigned long data)
851	{
852	return (struct pool_workqueue *)(data & WORK_STRUCT_PWQ_MASK);
853	}
854
855	static struct pool_workqueue get_work_pwq(struct work_struct work)
856	{
857	unsigned long data = atomic_long_read(&work->data);
858
859	if (data & WORK_STRUCT_PWQ)
860	return work_struct_pwq(data);
861	else
862	return NULL;
863	}
864
865	/**
866	* get_work_pool - return the worker_pool a given work was associated with
867	* @work: the work item of interest
868	*
869	* Pools are created and destroyed under wq_pool_mutex, and allows read
870	* access under RCU read lock. As such, this function should be
871	* called under wq_pool_mutex or inside of a rcu_read_lock() region.
872	*
873	* All fields of the returned pool are accessible as long as the above
874	* mentioned locking is in effect. If the returned pool needs to be used
875	* beyond the critical section, the caller is responsible for ensuring the
876	* returned pool is and stays online.
877	*
878	* Return: The worker_pool @work was last associated with. %NULL if none.
879	*/
880	static struct worker_pool get_work_pool(struct work_struct work)
881	{
882	unsigned long data = atomic_long_read(&work->data);
883	int pool_id;
884
885	assert_rcu_or_pool_mutex();
886
887	if (data & WORK_STRUCT_PWQ)
888	return work_struct_pwq(data)->pool;
889
890	pool_id = data >> WORK_OFFQ_POOL_SHIFT;
891	if (pool_id == WORK_OFFQ_POOL_NONE)
892	return NULL;
893
894	return idr_find(&worker_pool_idr, pool_id);
895	}
896
897	static unsigned long shift_and_mask(unsigned long v, u32 shift, u32 bits)
898	{
899	return (v >> shift) & ((1 << bits) - 1);
900	}
901
902	static void work_offqd_unpack(struct work_offq_data *offqd, unsigned long data)
903	{
904	WARN_ON_ONCE(data & WORK_STRUCT_PWQ);
905
906	offqd->pool_id = shift_and_mask(data, WORK_OFFQ_POOL_SHIFT,
907	WORK_OFFQ_POOL_BITS);
908	offqd->disable = shift_and_mask(data, WORK_OFFQ_DISABLE_SHIFT,
909	WORK_OFFQ_DISABLE_BITS);
910	offqd->flags = data & WORK_OFFQ_FLAG_MASK;
911	}
912
913	static unsigned long work_offqd_pack_flags(struct work_offq_data *offqd)
914	{
915	return ((unsigned long)offqd->disable << WORK_OFFQ_DISABLE_SHIFT) \|
916	((unsigned long)offqd->flags);
917	}
918
919	/*
920	* Policy functions. These define the policies on how the global worker
921	* pools are managed. Unless noted otherwise, these functions assume that
922	* they're being called with pool->lock held.
923	*/
924
925	/*
926	* Need to wake up a worker? Called from anything but currently
927	* running workers.
928	*
929	* Note that, because unbound workers never contribute to nr_running, this
930	* function will always return %true for unbound pools as long as the
931	* worklist isn't empty.
932	*/
933	static bool need_more_worker(struct worker_pool *pool)
934	{
935	return !list_empty(&pool->worklist) && !pool->nr_running;
936	}
937
938	/* Can I start working? Called from busy but !running workers. */
939	static bool may_start_working(struct worker_pool *pool)
940	{
941	return pool->nr_idle;
942	}
943
944	/* Do I need to keep working? Called from currently running workers. */
945	static bool keep_working(struct worker_pool *pool)
946	{
947	return !list_empty(&pool->worklist) && (pool->nr_running <= 1);
948	}
949
950	/* Do we need a new worker? Called from manager. */
951	static bool need_to_create_worker(struct worker_pool *pool)
952	{
953	return need_more_worker(pool) && !may_start_working(pool);
954	}
955
956	/* Do we have too many workers and should some go away? */
957	static bool too_many_workers(struct worker_pool *pool)
958	{
959	bool managing = pool->flags & POOL_MANAGER_ACTIVE;
960	int nr_idle = pool->nr_idle + managing; /* manager is considered idle */
961	int nr_busy = pool->nr_workers - nr_idle;
962
963	return nr_idle > 2 && (nr_idle - 2) * MAX_IDLE_WORKERS_RATIO >= nr_busy;
964	}
965
966	/**
967	* worker_set_flags - set worker flags and adjust nr_running accordingly
968	* @worker: self
969	* @flags: flags to set
970	*
971	* Set @flags in @worker->flags and adjust nr_running accordingly.
972	*/
973	static inline void worker_set_flags(struct worker *worker, unsigned int flags)
974	{
975	struct worker_pool *pool = worker->pool;
976
977	lockdep_assert_held(&pool->lock);
978
979	/* If transitioning into NOT_RUNNING, adjust nr_running. */
980	if ((flags & WORKER_NOT_RUNNING) &&
981	!(worker->flags & WORKER_NOT_RUNNING)) {
982	pool->nr_running--;
983	}
984
985	worker->flags \|= flags;
986	}
987
988	/**
989	* worker_clr_flags - clear worker flags and adjust nr_running accordingly
990	* @worker: self
991	* @flags: flags to clear
992	*
993	* Clear @flags in @worker->flags and adjust nr_running accordingly.
994	*/
995	static inline void worker_clr_flags(struct worker *worker, unsigned int flags)
996	{
997	struct worker_pool *pool = worker->pool;
998	unsigned int oflags = worker->flags;
999
1000	lockdep_assert_held(&pool->lock);
1001
1002	worker->flags &= ~flags;
1003
1004	/*
1005	* If transitioning out of NOT_RUNNING, increment nr_running. Note
1006	* that the nested NOT_RUNNING is not a noop. NOT_RUNNING is mask
1007	* of multiple flags, not a single flag.
1008	*/
1009	if ((flags & WORKER_NOT_RUNNING) && (oflags & WORKER_NOT_RUNNING))
1010	if (!(worker->flags & WORKER_NOT_RUNNING))
1011	pool->nr_running++;
1012	}
1013
1014	/* Return the first idle worker. Called with pool->lock held. */
1015	static struct worker first_idle_worker(struct worker_pool pool)
1016	{
1017	if (unlikely(list_empty(&pool->idle_list)))
1018	return NULL;
1019
1020	return list_first_entry(&pool->idle_list, struct worker, entry);
1021	}
1022
1023	/**
1024	* worker_enter_idle - enter idle state
1025	* @worker: worker which is entering idle state
1026	*
1027	* @worker is entering idle state. Update stats and idle timer if
1028	* necessary.
1029	*
1030	* LOCKING:
1031	* raw_spin_lock_irq(pool->lock).
1032	*/
1033	static void worker_enter_idle(struct worker *worker)
1034	{
1035	struct worker_pool *pool = worker->pool;
1036
1037	if (WARN_ON_ONCE(worker->flags & WORKER_IDLE) \|\|
1038	WARN_ON_ONCE(!list_empty(&worker->entry) &&
1039	(worker->hentry.next \|\| worker->hentry.pprev)))
1040	return;
1041
1042	/* can't use worker_set_flags(), also called from create_worker() */
1043	worker->flags \|= WORKER_IDLE;
1044	pool->nr_idle++;
1045	worker->last_active = jiffies;
1046
1047	/* idle_list is LIFO */
1048	list_add(&worker->entry, &pool->idle_list);
1049
1050	if (too_many_workers(pool) && !timer_pending(&pool->idle_timer))
1051	mod_timer(&pool->idle_timer, jiffies + IDLE_WORKER_TIMEOUT);
1052
1053	/* Sanity check nr_running. */
1054	WARN_ON_ONCE(pool->nr_workers == pool->nr_idle && pool->nr_running);
1055	}
1056
1057	/**
1058	* worker_leave_idle - leave idle state
1059	* @worker: worker which is leaving idle state
1060	*
1061	* @worker is leaving idle state. Update stats.
1062	*
1063	* LOCKING:
1064	* raw_spin_lock_irq(pool->lock).
1065	*/
1066	static void worker_leave_idle(struct worker *worker)
1067	{
1068	struct worker_pool *pool = worker->pool;
1069
1070	if (WARN_ON_ONCE(!(worker->flags & WORKER_IDLE)))
1071	return;
1072	worker_clr_flags(worker, WORKER_IDLE);
1073	pool->nr_idle--;
1074	list_del_init(&worker->entry);
1075	}
1076
1077	/**
1078	* find_worker_executing_work - find worker which is executing a work
1079	* @pool: pool of interest
1080	* @work: work to find worker for
1081	*
1082	* Find a worker which is executing @work on @pool by searching
1083	* @pool->busy_hash which is keyed by the address of @work. For a worker
1084	* to match, its current execution should match the address of @work and
1085	* its work function. This is to avoid unwanted dependency between
1086	* unrelated work executions through a work item being recycled while still
1087	* being executed.
1088	*
1089	* This is a bit tricky. A work item may be freed once its execution
1090	* starts and nothing prevents the freed area from being recycled for
1091	* another work item. If the same work item address ends up being reused
1092	* before the original execution finishes, workqueue will identify the
1093	* recycled work item as currently executing and make it wait until the
1094	* current execution finishes, introducing an unwanted dependency.
1095	*
1096	* This function checks the work item address and work function to avoid
1097	* false positives. Note that this isn't complete as one may construct a
1098	* work function which can introduce dependency onto itself through a
1099	* recycled work item. Well, if somebody wants to shoot oneself in the
1100	* foot that badly, there's only so much we can do, and if such deadlock
1101	* actually occurs, it should be easy to locate the culprit work function.
1102	*
1103	* CONTEXT:
1104	* raw_spin_lock_irq(pool->lock).
1105	*
1106	* Return:
1107	* Pointer to worker which is executing @work if found, %NULL
1108	* otherwise.
1109	*/
1110	static struct worker find_worker_executing_work(struct worker_pool pool,
1111	struct work_struct *work)
1112	{
1113	struct worker *worker;
1114
1115	hash_for_each_possible(pool->busy_hash, worker, hentry,
1116	(unsigned long)work)
1117	if (worker->current_work == work &&
1118	worker->current_func == work->func)
1119	return worker;
1120
1121	return NULL;
1122	}
1123
1124	/**
1125	* move_linked_works - move linked works to a list
1126	* @work: start of series of works to be scheduled
1127	* @head: target list to append @work to
1128	* @nextp: out parameter for nested worklist walking
1129	*
1130	* Schedule linked works starting from @work to @head. Work series to be
1131	* scheduled starts at @work and includes any consecutive work with
1132	* WORK_STRUCT_LINKED set in its predecessor. See assign_work() for details on
1133	* @nextp.
1134	*
1135	* CONTEXT:
1136	* raw_spin_lock_irq(pool->lock).
1137	*/
1138	static void move_linked_works(struct work_struct work, struct list_head head,
1139	struct work_struct **nextp)
1140	{
1141	struct work_struct *n;
1142
1143	/*
1144	* Linked worklist will always end before the end of the list,
1145	* use NULL for list head.
1146	*/
1147	list_for_each_entry_safe_from(work, n, NULL, entry) {
1148	list_move_tail(&work->entry, head);
1149	if (!(*work_data_bits(work) & WORK_STRUCT_LINKED))
1150	break;
1151	}
1152
1153	/*
1154	* If we're already inside safe list traversal and have moved
1155	* multiple works to the scheduled queue, the next position
1156	* needs to be updated.
1157	*/
1158	if (nextp)
1159	*nextp = n;
1160	}
1161
1162	/**
1163	* assign_work - assign a work item and its linked work items to a worker
1164	* @work: work to assign
1165	* @worker: worker to assign to
1166	* @nextp: out parameter for nested worklist walking
1167	*
1168	* Assign @work and its linked work items to @worker. If @work is already being
1169	* executed by another worker in the same pool, it'll be punted there.
1170	*
1171	* If @nextp is not NULL, it's updated to point to the next work of the last
1172	* scheduled work. This allows assign_work() to be nested inside
1173	* list_for_each_entry_safe().
1174	*
1175	* Returns %true if @work was successfully assigned to @worker. %false if @work
1176	* was punted to another worker already executing it.
1177	*/
1178	static bool assign_work(struct work_struct work, struct worker worker,
1179	struct work_struct **nextp)
1180	{
1181	struct worker_pool *pool = worker->pool;
1182	struct worker *collision;
1183
1184	lockdep_assert_held(&pool->lock);
1185
1186	/*
1187	* A single work shouldn't be executed concurrently by multiple workers.
1188	* __queue_work() ensures that @work doesn't jump to a different pool
1189	* while still running in the previous pool. Here, we should ensure that
1190	* @work is not executed concurrently by multiple workers from the same
1191	* pool. Check whether anyone is already processing the work. If so,
1192	* defer the work to the currently executing one.
1193	*/
1194	collision = find_worker_executing_work(pool, work);
1195	if (unlikely(collision)) {
1196	move_linked_works(work, &collision->scheduled, nextp);
1197	return false;
1198	}
1199
1200	move_linked_works(work, &worker->scheduled, nextp);
1201	return true;
1202	}
1203
1204	static struct irq_work bh_pool_irq_work(struct worker_pool pool)
1205	{
1206	int high = pool->attrs->nice == HIGHPRI_NICE_LEVEL ? 1 : 0;
1207
1208	return &per_cpu(bh_pool_irq_works, pool->cpu)[high];
1209	}
1210
1211	static void kick_bh_pool(struct worker_pool *pool)
1212	{
1213	#ifdef CONFIG_SMP
1214	/* see drain_dead_softirq_workfn() for BH_DRAINING */
1215	if (unlikely(pool->cpu != smp_processor_id() &&
1216	!(pool->flags & POOL_BH_DRAINING))) {
1217	irq_work_queue_on(bh_pool_irq_work(pool), pool->cpu);
1218	return;
1219	}
1220	#endif
1221	if (pool->attrs->nice == HIGHPRI_NICE_LEVEL)
1222	raise_softirq_irqoff(HI_SOFTIRQ);
1223	else
1224	raise_softirq_irqoff(TASKLET_SOFTIRQ);
1225	}
1226
1227	/**
1228	* kick_pool - wake up an idle worker if necessary
1229	* @pool: pool to kick
1230	*
1231	* @pool may have pending work items. Wake up worker if necessary. Returns
1232	* whether a worker was woken up.
1233	*/
1234	static bool kick_pool(struct worker_pool *pool)
1235	{
1236	struct worker *worker = first_idle_worker(pool);
1237	struct task_struct *p;
1238
1239	lockdep_assert_held(&pool->lock);
1240
1241	if (!need_more_worker(pool) \|\| !worker)
1242	return false;
1243
1244	if (pool->flags & POOL_BH) {
1245	kick_bh_pool(pool);
1246	return true;
1247	}
1248
1249	p = worker->task;
1250
1251	#ifdef CONFIG_SMP
1252	/*
1253	* Idle @worker is about to execute @work and waking up provides an
1254	* opportunity to migrate @worker at a lower cost by setting the task's
1255	* wake_cpu field. Let's see if we want to move @worker to improve
1256	* execution locality.
1257	*
1258	* We're waking the worker that went idle the latest and there's some
1259	* chance that @worker is marked idle but hasn't gone off CPU yet. If
1260	* so, setting the wake_cpu won't do anything. As this is a best-effort
1261	* optimization and the race window is narrow, let's leave as-is for
1262	* now. If this becomes pronounced, we can skip over workers which are
1263	* still on cpu when picking an idle worker.
1264	*
1265	* If @pool has non-strict affinity, @worker might have ended up outside
1266	* its affinity scope. Repatriate.
1267	*/
1268	if (!pool->attrs->affn_strict &&
1269	!cpumask_test_cpu(p->wake_cpu, pool->attrs->__pod_cpumask)) {
1270	struct work_struct *work = list_first_entry(&pool->worklist,
1271	struct work_struct, entry);
1272	int wake_cpu = cpumask_any_and_distribute(pool->attrs->__pod_cpumask,
1273	cpu_online_mask);
1274	if (wake_cpu < nr_cpu_ids) {
1275	p->wake_cpu = wake_cpu;
1276	get_work_pwq(work)->stats[PWQ_STAT_REPATRIATED]++;
1277	}
1278	}
1279	#endif
1280	wake_up_process(p);
1281	return true;
1282	}
1283
1284	#ifdef CONFIG_WQ_CPU_INTENSIVE_REPORT
1285
1286	/*
1287	* Concurrency-managed per-cpu work items that hog CPU for longer than
1288	* wq_cpu_intensive_thresh_us trigger the automatic CPU_INTENSIVE mechanism,
1289	* which prevents them from stalling other concurrency-managed work items. If a
1290	* work function keeps triggering this mechanism, it's likely that the work item
1291	* should be using an unbound workqueue instead.
1292	*
1293	* wq_cpu_intensive_report() tracks work functions which trigger such conditions
1294	* and report them so that they can be examined and converted to use unbound
1295	* workqueues as appropriate. To avoid flooding the console, each violating work
1296	* function is tracked and reported with exponential backoff.
1297	*/
1298	#define WCI_MAX_ENTS 128
1299
1300	struct wci_ent {
1301	work_func_t func;
1302	atomic64_t cnt;
1303	struct hlist_node hash_node;
1304	};
1305
1306	static struct wci_ent wci_ents[WCI_MAX_ENTS];
1307	static int wci_nr_ents;
1308	static DEFINE_RAW_SPINLOCK(wci_lock);
1309	static DEFINE_HASHTABLE(wci_hash, ilog2(WCI_MAX_ENTS));
1310
1311	static struct wci_ent *wci_find_ent(work_func_t func)
1312	{
1313	struct wci_ent *ent;
1314
1315	hash_for_each_possible_rcu(wci_hash, ent, hash_node,
1316	(unsigned long)func) {
1317	if (ent->func == func)
1318	return ent;
1319	}
1320	return NULL;
1321	}
1322
1323	static void wq_cpu_intensive_report(work_func_t func)
1324	{
1325	struct wci_ent *ent;
1326
1327	restart:
1328	ent = wci_find_ent(func);
1329	if (ent) {
1330	u64 cnt;
1331
1332	/*
1333	* Start reporting from the warning_thresh and back off
1334	* exponentially.
1335	*/
1336	cnt = atomic64_inc_return_relaxed(&ent->cnt);
1337	if (wq_cpu_intensive_warning_thresh &&
1338	cnt >= wq_cpu_intensive_warning_thresh &&
1339	is_power_of_2(cnt + 1 - wq_cpu_intensive_warning_thresh))
1340	printk_deferred(KERN_WARNING "workqueue: %ps hogged CPU for >%luus %llu times, consider switching to WQ_UNBOUND\n",
1341	ent->func, wq_cpu_intensive_thresh_us,
1342	atomic64_read(&ent->cnt));
1343	return;
1344	}
1345
1346	/*
1347	* @func is a new violation. Allocate a new entry for it. If wcn_ents[]
1348	* is exhausted, something went really wrong and we probably made enough
1349	* noise already.
1350	*/
1351	if (wci_nr_ents >= WCI_MAX_ENTS)
1352	return;
1353
1354	raw_spin_lock(&wci_lock);
1355
1356	if (wci_nr_ents >= WCI_MAX_ENTS) {
1357	raw_spin_unlock(&wci_lock);
1358	return;
1359	}
1360
1361	if (wci_find_ent(func)) {
1362	raw_spin_unlock(&wci_lock);
1363	goto restart;
1364	}
1365
1366	ent = &wci_ents[wci_nr_ents++];
1367	ent->func = func;
1368	atomic64_set(&ent->cnt, 0);
1369	hash_add_rcu(wci_hash, &ent->hash_node, (unsigned long)func);
1370
1371	raw_spin_unlock(&wci_lock);
1372
1373	goto restart;
1374	}
1375
1376	#else /* CONFIG_WQ_CPU_INTENSIVE_REPORT */
1377	static void wq_cpu_intensive_report(work_func_t func) {}
1378	#endif /* CONFIG_WQ_CPU_INTENSIVE_REPORT */
1379
1380	/**
1381	* wq_worker_running - a worker is running again
1382	* @task: task waking up
1383	*
1384	* This function is called when a worker returns from schedule()
1385	*/
1386	void wq_worker_running(struct task_struct *task)
1387	{
1388	struct worker *worker = kthread_data(task);
1389
1390	if (!READ_ONCE(worker->sleeping))
1391	return;
1392
1393	/*
1394	* If preempted by unbind_workers() between the WORKER_NOT_RUNNING check
1395	* and the nr_running increment below, we may ruin the nr_running reset
1396	* and leave with an unexpected pool->nr_running == 1 on the newly unbound
1397	* pool. Protect against such race.
1398	*/
1399	preempt_disable();
1400	if (!(worker->flags & WORKER_NOT_RUNNING))
1401	worker->pool->nr_running++;
1402	preempt_enable();
1403
1404	/*
1405	* CPU intensive auto-detection cares about how long a work item hogged
1406	* CPU without sleeping. Reset the starting timestamp on wakeup.
1407	*/
1408	worker->current_at = worker->task->se.sum_exec_runtime;
1409
1410	WRITE_ONCE(worker->sleeping, 0);
1411	}
1412
1413	/**
1414	* wq_worker_sleeping - a worker is going to sleep
1415	* @task: task going to sleep
1416	*
1417	* This function is called from schedule() when a busy worker is
1418	* going to sleep.
1419	*/
1420	void wq_worker_sleeping(struct task_struct *task)
1421	{
1422	struct worker *worker = kthread_data(task);
1423	struct worker_pool *pool;
1424
1425	/*
1426	* Rescuers, which may not have all the fields set up like normal
1427	* workers, also reach here, let's not access anything before
1428	* checking NOT_RUNNING.
1429	*/
1430	if (worker->flags & WORKER_NOT_RUNNING)
1431	return;
1432
1433	pool = worker->pool;
1434
1435	/* Return if preempted before wq_worker_running() was reached */
1436	if (READ_ONCE(worker->sleeping))
1437	return;
1438
1439	WRITE_ONCE(worker->sleeping, 1);
1440	raw_spin_lock_irq(&pool->lock);
1441
1442	/*
1443	* Recheck in case unbind_workers() preempted us. We don't
1444	* want to decrement nr_running after the worker is unbound
1445	* and nr_running has been reset.
1446	*/
1447	if (worker->flags & WORKER_NOT_RUNNING) {
1448	raw_spin_unlock_irq(&pool->lock);
1449	return;
1450	}
1451
1452	pool->nr_running--;
1453	if (kick_pool(pool))
1454	worker->current_pwq->stats[PWQ_STAT_CM_WAKEUP]++;
1455
1456	raw_spin_unlock_irq(&pool->lock);
1457	}
1458
1459	/**
1460	* wq_worker_tick - a scheduler tick occurred while a kworker is running
1461	* @task: task currently running
1462	*
1463	* Called from sched_tick(). We're in the IRQ context and the current
1464	* worker's fields which follow the 'K' locking rule can be accessed safely.
1465	*/
1466	void wq_worker_tick(struct task_struct *task)
1467	{
1468	struct worker *worker = kthread_data(task);
1469	struct pool_workqueue *pwq = worker->current_pwq;
1470	struct worker_pool *pool = worker->pool;
1471
1472	if (!pwq)
1473	return;
1474
1475	pwq->stats[PWQ_STAT_CPU_TIME] += TICK_USEC;
1476
1477	if (!wq_cpu_intensive_thresh_us)
1478	return;
1479
1480	/*
1481	* If the current worker is concurrency managed and hogged the CPU for
1482	* longer than wq_cpu_intensive_thresh_us, it's automatically marked
1483	* CPU_INTENSIVE to avoid stalling other concurrency-managed work items.
1484	*
1485	* Set @worker->sleeping means that @worker is in the process of
1486	* switching out voluntarily and won't be contributing to
1487	* @pool->nr_running until it wakes up. As wq_worker_sleeping() also
1488	* decrements ->nr_running, setting CPU_INTENSIVE here can lead to
1489	* double decrements. The task is releasing the CPU anyway. Let's skip.
1490	* We probably want to make this prettier in the future.
1491	*/
1492	if ((worker->flags & WORKER_NOT_RUNNING) \|\| READ_ONCE(worker->sleeping) \|\|
1493	worker->task->se.sum_exec_runtime - worker->current_at <
1494	wq_cpu_intensive_thresh_us * NSEC_PER_USEC)
1495	return;
1496
1497	raw_spin_lock(&pool->lock);
1498
1499	worker_set_flags(worker, WORKER_CPU_INTENSIVE);
1500	wq_cpu_intensive_report(worker->current_func);
1501	pwq->stats[PWQ_STAT_CPU_INTENSIVE]++;
1502
1503	if (kick_pool(pool))
1504	pwq->stats[PWQ_STAT_CM_WAKEUP]++;
1505
1506	raw_spin_unlock(&pool->lock);
1507	}
1508
1509	/**
1510	* wq_worker_last_func - retrieve worker's last work function
1511	* @task: Task to retrieve last work function of.
1512	*
1513	* Determine the last function a worker executed. This is called from
1514	* the scheduler to get a worker's last known identity.
1515	*
1516	* CONTEXT:
1517	* raw_spin_lock_irq(rq->lock)
1518	*
1519	* This function is called during schedule() when a kworker is going
1520	* to sleep. It's used by psi to identify aggregation workers during
1521	* dequeuing, to allow periodic aggregation to shut-off when that
1522	* worker is the last task in the system or cgroup to go to sleep.
1523	*
1524	* As this function doesn't involve any workqueue-related locking, it
1525	* only returns stable values when called from inside the scheduler's
1526	* queuing and dequeuing paths, when @task, which must be a kworker,
1527	* is guaranteed to not be processing any works.
1528	*
1529	* Return:
1530	* The last work function %current executed as a worker, NULL if it
1531	* hasn't executed any work yet.
1532	*/
1533	work_func_t wq_worker_last_func(struct task_struct *task)
1534	{
1535	struct worker *worker = kthread_data(task);
1536
1537	return worker->last_func;
1538	}
1539
1540	/**
1541	* wq_node_nr_active - Determine wq_node_nr_active to use
1542	* @wq: workqueue of interest
1543	* @node: NUMA node, can be %NUMA_NO_NODE
1544	*
1545	* Determine wq_node_nr_active to use for @wq on @node. Returns:
1546	*
1547	* - %NULL for per-cpu workqueues as they don't need to use shared nr_active.
1548	*
1549	* - node_nr_active[nr_node_ids] if @node is %NUMA_NO_NODE.
1550	*
1551	* - Otherwise, node_nr_active[@node].
1552	*/
1553	static struct wq_node_nr_active wq_node_nr_active(struct workqueue_struct wq,
1554	int node)
1555	{
1556	if (!(wq->flags & WQ_UNBOUND))
1557	return NULL;
1558
1559	if (node == NUMA_NO_NODE)
1560	node = nr_node_ids;
1561
1562	return wq->node_nr_active[node];
1563	}
1564
1565	/**
1566	* wq_update_node_max_active - Update per-node max_actives to use
1567	* @wq: workqueue to update
1568	* @off_cpu: CPU that's going down, -1 if a CPU is not going down
1569	*
1570	* Update @wq->node_nr_active[]->max. @wq must be unbound. max_active is
1571	* distributed among nodes according to the proportions of numbers of online
1572	* cpus. The result is always between @wq->min_active and max_active.
1573	*/
1574	static void wq_update_node_max_active(struct workqueue_struct *wq, int off_cpu)
1575	{
1576	struct cpumask *effective = unbound_effective_cpumask(wq);
1577	int min_active = READ_ONCE(wq->min_active);
1578	int max_active = READ_ONCE(wq->max_active);
1579	int total_cpus, node;
1580
1581	lockdep_assert_held(&wq->mutex);
1582
1583	if (!wq_topo_initialized)
1584	return;
1585
1586	if (off_cpu >= 0 && !cpumask_test_cpu(off_cpu, effective))
1587	off_cpu = -1;
1588
1589	total_cpus = cpumask_weight_and(effective, cpu_online_mask);
1590	if (off_cpu >= 0)
1591	total_cpus--;
1592
1593	/* If all CPUs of the wq get offline, use the default values */
1594	if (unlikely(!total_cpus)) {
1595	for_each_node(node)
1596	wq_node_nr_active(wq, node)->max = min_active;
1597
1598	wq_node_nr_active(wq, NUMA_NO_NODE)->max = max_active;
1599	return;
1600	}
1601
1602	for_each_node(node) {
1603	int node_cpus;
1604
1605	node_cpus = cpumask_weight_and(effective, cpumask_of_node(node));
1606	if (off_cpu >= 0 && cpu_to_node(off_cpu) == node)
1607	node_cpus--;
1608
1609	wq_node_nr_active(wq, node)->max =
1610	clamp(DIV_ROUND_UP(max_active * node_cpus, total_cpus),
1611	min_active, max_active);
1612	}
1613
1614	wq_node_nr_active(wq, NUMA_NO_NODE)->max = max_active;
1615	}
1616
1617	/**
1618	* get_pwq - get an extra reference on the specified pool_workqueue
1619	* @pwq: pool_workqueue to get
1620	*
1621	* Obtain an extra reference on @pwq. The caller should guarantee that
1622	* @pwq has positive refcnt and be holding the matching pool->lock.
1623	*/
1624	static void get_pwq(struct pool_workqueue *pwq)
1625	{
1626	lockdep_assert_held(&pwq->pool->lock);
1627	WARN_ON_ONCE(pwq->refcnt <= 0);
1628	pwq->refcnt++;
1629	}
1630
1631	/**
1632	* put_pwq - put a pool_workqueue reference
1633	* @pwq: pool_workqueue to put
1634	*
1635	* Drop a reference of @pwq. If its refcnt reaches zero, schedule its
1636	* destruction. The caller should be holding the matching pool->lock.
1637	*/
1638	static void put_pwq(struct pool_workqueue *pwq)
1639	{
1640	lockdep_assert_held(&pwq->pool->lock);
1641	if (likely(--pwq->refcnt))
1642	return;
1643	/*
1644	* @pwq can't be released under pool->lock, bounce to a dedicated
1645	* kthread_worker to avoid A-A deadlocks.
1646	*/
1647	kthread_queue_work(pwq_release_worker, &pwq->release_work);
1648	}
1649
1650	/**
1651	* put_pwq_unlocked - put_pwq() with surrounding pool lock/unlock
1652	* @pwq: pool_workqueue to put (can be %NULL)
1653	*
1654	* put_pwq() with locking. This function also allows %NULL @pwq.
1655	*/
1656	static void put_pwq_unlocked(struct pool_workqueue *pwq)
1657	{
1658	if (pwq) {
1659	/*
1660	* As both pwqs and pools are RCU protected, the
1661	* following lock operations are safe.
1662	*/
1663	raw_spin_lock_irq(&pwq->pool->lock);
1664	put_pwq(pwq);
1665	raw_spin_unlock_irq(&pwq->pool->lock);
1666	}
1667	}
1668
1669	static bool pwq_is_empty(struct pool_workqueue *pwq)
1670	{
1671	return !pwq->nr_active && list_empty(&pwq->inactive_works);
1672	}
1673
1674	static void __pwq_activate_work(struct pool_workqueue *pwq,
1675	struct work_struct *work)
1676	{
1677	unsigned long *wdb = work_data_bits(work);
1678
1679	WARN_ON_ONCE(!(*wdb & WORK_STRUCT_INACTIVE));
1680	trace_workqueue_activate_work(work);
1681	if (list_empty(&pwq->pool->worklist))
1682	pwq->pool->watchdog_ts = jiffies;
1683	move_linked_works(work, &pwq->pool->worklist, NULL);
1684	__clear_bit(WORK_STRUCT_INACTIVE_BIT, wdb);
1685	}
1686
1687	/**
1688	* pwq_activate_work - Activate a work item if inactive
1689	* @pwq: pool_workqueue @work belongs to
1690	* @work: work item to activate
1691	*
1692	* Returns %true if activated. %false if already active.
1693	*/
1694	static bool pwq_activate_work(struct pool_workqueue *pwq,
1695	struct work_struct *work)
1696	{
1697	struct worker_pool *pool = pwq->pool;
1698	struct wq_node_nr_active *nna;
1699
1700	lockdep_assert_held(&pool->lock);
1701
1702	if (!(*work_data_bits(work) & WORK_STRUCT_INACTIVE))
1703	return false;
1704
1705	nna = wq_node_nr_active(pwq->wq, pool->node);
1706	if (nna)
1707	atomic_inc(&nna->nr);
1708
1709	pwq->nr_active++;
1710	__pwq_activate_work(pwq, work);
1711	return true;
1712	}
1713
1714	static bool tryinc_node_nr_active(struct wq_node_nr_active *nna)
1715	{
1716	int max = READ_ONCE(nna->max);
1717
1718	while (true) {
1719	int old, tmp;
1720
1721	old = atomic_read(&nna->nr);
1722	if (old >= max)
1723	return false;
1724	tmp = atomic_cmpxchg_relaxed(&nna->nr, old, old + 1);
1725	if (tmp == old)
1726	return true;
1727	}
1728	}
1729
1730	/**
1731	* pwq_tryinc_nr_active - Try to increment nr_active for a pwq
1732	* @pwq: pool_workqueue of interest
1733	* @fill: max_active may have increased, try to increase concurrency level
1734	*
1735	* Try to increment nr_active for @pwq. Returns %true if an nr_active count is
1736	* successfully obtained. %false otherwise.
1737	*/
1738	static bool pwq_tryinc_nr_active(struct pool_workqueue *pwq, bool fill)
1739	{
1740	struct workqueue_struct *wq = pwq->wq;
1741	struct worker_pool *pool = pwq->pool;
1742	struct wq_node_nr_active *nna = wq_node_nr_active(wq, pool->node);
1743	bool obtained = false;
1744
1745	lockdep_assert_held(&pool->lock);
1746
1747	if (!nna) {
1748	/* BH or per-cpu workqueue, pwq->nr_active is sufficient */
1749	obtained = pwq->nr_active < READ_ONCE(wq->max_active);
1750	goto out;
1751	}
1752
1753	if (unlikely(pwq->plugged))
1754	return false;
1755
1756	/*
1757	* Unbound workqueue uses per-node shared nr_active $nna. If @pwq is
1758	* already waiting on $nna, pwq_dec_nr_active() will maintain the
1759	* concurrency level. Don't jump the line.
1760	*
1761	* We need to ignore the pending test after max_active has increased as
1762	* pwq_dec_nr_active() can only maintain the concurrency level but not
1763	* increase it. This is indicated by @fill.
1764	*/
1765	if (!list_empty(&pwq->pending_node) && likely(!fill))
1766	goto out;
1767
1768	obtained = tryinc_node_nr_active(nna);
1769	if (obtained)
1770	goto out;
1771
1772	/*
1773	* Lockless acquisition failed. Lock, add ourself to $nna->pending_pwqs
1774	* and try again. The smp_mb() is paired with the implied memory barrier
1775	* of atomic_dec_return() in pwq_dec_nr_active() to ensure that either
1776	* we see the decremented $nna->nr or they see non-empty
1777	* $nna->pending_pwqs.
1778	*/
1779	raw_spin_lock(&nna->lock);
1780
1781	if (list_empty(&pwq->pending_node))
1782	list_add_tail(&pwq->pending_node, &nna->pending_pwqs);
1783	else if (likely(!fill))
1784	goto out_unlock;
1785
1786	smp_mb();
1787
1788	obtained = tryinc_node_nr_active(nna);
1789
1790	/*
1791	* If @fill, @pwq might have already been pending. Being spuriously
1792	* pending in cold paths doesn't affect anything. Let's leave it be.
1793	*/
1794	if (obtained && likely(!fill))
1795	list_del_init(&pwq->pending_node);
1796
1797	out_unlock:
1798	raw_spin_unlock(&nna->lock);
1799	out:
1800	if (obtained)
1801	pwq->nr_active++;
1802	return obtained;
1803	}
1804
1805	/**
1806	* pwq_activate_first_inactive - Activate the first inactive work item on a pwq
1807	* @pwq: pool_workqueue of interest
1808	* @fill: max_active may have increased, try to increase concurrency level
1809	*
1810	* Activate the first inactive work item of @pwq if available and allowed by
1811	* max_active limit.
1812	*
1813	* Returns %true if an inactive work item has been activated. %false if no
1814	* inactive work item is found or max_active limit is reached.
1815	*/
1816	static bool pwq_activate_first_inactive(struct pool_workqueue *pwq, bool fill)
1817	{
1818	struct work_struct *work =
1819	list_first_entry_or_null(&pwq->inactive_works,
1820	struct work_struct, entry);
1821
1822	if (work && pwq_tryinc_nr_active(pwq, fill)) {
1823	__pwq_activate_work(pwq, work);
1824	return true;
1825	} else {
1826	return false;
1827	}
1828	}
1829
1830	/**
1831	* unplug_oldest_pwq - unplug the oldest pool_workqueue
1832	* @wq: workqueue_struct where its oldest pwq is to be unplugged
1833	*
1834	* This function should only be called for ordered workqueues where only the
1835	* oldest pwq is unplugged, the others are plugged to suspend execution to
1836	* ensure proper work item ordering::
1837	*
1838	* dfl_pwq --------------+ [P] - plugged
1839	* \|
1840	* v
1841	* pwqs -> A -> B [P] -> C [P] (newest)
1842	* \| \| \|
1843	* 1 3 5
1844	* \| \| \|
1845	* 2 4 6
1846	*
1847	* When the oldest pwq is drained and removed, this function should be called
1848	* to unplug the next oldest one to start its work item execution. Note that
1849	* pwq's are linked into wq->pwqs with the oldest first, so the first one in
1850	* the list is the oldest.
1851	*/
1852	static void unplug_oldest_pwq(struct workqueue_struct *wq)
1853	{
1854	struct pool_workqueue *pwq;
1855
1856	lockdep_assert_held(&wq->mutex);
1857
1858	/* Caller should make sure that pwqs isn't empty before calling */
1859	pwq = list_first_entry_or_null(&wq->pwqs, struct pool_workqueue,
1860	pwqs_node);
1861	raw_spin_lock_irq(&pwq->pool->lock);
1862	if (pwq->plugged) {
1863	pwq->plugged = false;
1864	if (pwq_activate_first_inactive(pwq, true))
1865	kick_pool(pwq->pool);
1866	}
1867	raw_spin_unlock_irq(&pwq->pool->lock);
1868	}
1869
1870	/**
1871	* node_activate_pending_pwq - Activate a pending pwq on a wq_node_nr_active
1872	* @nna: wq_node_nr_active to activate a pending pwq for
1873	* @caller_pool: worker_pool the caller is locking
1874	*
1875	* Activate a pwq in @nna->pending_pwqs. Called with @caller_pool locked.
1876	* @caller_pool may be unlocked and relocked to lock other worker_pools.
1877	*/
1878	static void node_activate_pending_pwq(struct wq_node_nr_active *nna,
1879	struct worker_pool *caller_pool)
1880	{
1881	struct worker_pool *locked_pool = caller_pool;
1882	struct pool_workqueue *pwq;
1883	struct work_struct *work;
1884
1885	lockdep_assert_held(&caller_pool->lock);
1886
1887	raw_spin_lock(&nna->lock);
1888	retry:
1889	pwq = list_first_entry_or_null(&nna->pending_pwqs,
1890	struct pool_workqueue, pending_node);
1891	if (!pwq)
1892	goto out_unlock;
1893
1894	/*
1895	* If @pwq is for a different pool than @locked_pool, we need to lock
1896	* @pwq->pool->lock. Let's trylock first. If unsuccessful, do the unlock
1897	* / lock dance. For that, we also need to release @nna->lock as it's
1898	* nested inside pool locks.
1899	*/
1900	if (pwq->pool != locked_pool) {
1901	raw_spin_unlock(&locked_pool->lock);
1902	locked_pool = pwq->pool;
1903	if (!raw_spin_trylock(&locked_pool->lock)) {
1904	raw_spin_unlock(&nna->lock);
1905	raw_spin_lock(&locked_pool->lock);
1906	raw_spin_lock(&nna->lock);
1907	goto retry;
1908	}
1909	}
1910
1911	/*
1912	* $pwq may not have any inactive work items due to e.g. cancellations.
1913	* Drop it from pending_pwqs and see if there's another one.
1914	*/
1915	work = list_first_entry_or_null(&pwq->inactive_works,
1916	struct work_struct, entry);
1917	if (!work) {
1918	list_del_init(&pwq->pending_node);
1919	goto retry;
1920	}
1921
1922	/*
1923	* Acquire an nr_active count and activate the inactive work item. If
1924	* $pwq still has inactive work items, rotate it to the end of the
1925	* pending_pwqs so that we round-robin through them. This means that
1926	* inactive work items are not activated in queueing order which is fine
1927	* given that there has never been any ordering across different pwqs.
1928	*/
1929	if (likely(tryinc_node_nr_active(nna))) {
1930	pwq->nr_active++;
1931	__pwq_activate_work(pwq, work);
1932
1933	if (list_empty(&pwq->inactive_works))
1934	list_del_init(&pwq->pending_node);
1935	else
1936	list_move_tail(&pwq->pending_node, &nna->pending_pwqs);
1937
1938	/* if activating a foreign pool, make sure it's running */
1939	if (pwq->pool != caller_pool)
1940	kick_pool(pwq->pool);
1941	}
1942
1943	out_unlock:
1944	raw_spin_unlock(&nna->lock);
1945	if (locked_pool != caller_pool) {
1946	raw_spin_unlock(&locked_pool->lock);
1947	raw_spin_lock(&caller_pool->lock);
1948	}
1949	}
1950
1951	/**
1952	* pwq_dec_nr_active - Retire an active count
1953	* @pwq: pool_workqueue of interest
1954	*
1955	* Decrement @pwq's nr_active and try to activate the first inactive work item.
1956	* For unbound workqueues, this function may temporarily drop @pwq->pool->lock.
1957	*/
1958	static void pwq_dec_nr_active(struct pool_workqueue *pwq)
1959	{
1960	struct worker_pool *pool = pwq->pool;
1961	struct wq_node_nr_active *nna = wq_node_nr_active(pwq->wq, pool->node);
1962
1963	lockdep_assert_held(&pool->lock);
1964
1965	/*
1966	* @pwq->nr_active should be decremented for both percpu and unbound
1967	* workqueues.
1968	*/
1969	pwq->nr_active--;
1970
1971	/*
1972	* For a percpu workqueue, it's simple. Just need to kick the first
1973	* inactive work item on @pwq itself.
1974	*/
1975	if (!nna) {
1976	pwq_activate_first_inactive(pwq, false);
1977	return;
1978	}
1979
1980	/*
1981	* If @pwq is for an unbound workqueue, it's more complicated because
1982	* multiple pwqs and pools may be sharing the nr_active count. When a
1983	* pwq needs to wait for an nr_active count, it puts itself on
1984	* $nna->pending_pwqs. The following atomic_dec_return()'s implied
1985	* memory barrier is paired with smp_mb() in pwq_tryinc_nr_active() to
1986	* guarantee that either we see non-empty pending_pwqs or they see
1987	* decremented $nna->nr.
1988	*
1989	* $nna->max may change as CPUs come online/offline and @pwq->wq's
1990	* max_active gets updated. However, it is guaranteed to be equal to or
1991	* larger than @pwq->wq->min_active which is above zero unless freezing.
1992	* This maintains the forward progress guarantee.
1993	*/
1994	if (atomic_dec_return(&nna->nr) >= READ_ONCE(nna->max))
1995	return;
1996
1997	if (!list_empty(&nna->pending_pwqs))
1998	node_activate_pending_pwq(nna, pool);
1999	}
2000
2001	/**
2002	* pwq_dec_nr_in_flight - decrement pwq's nr_in_flight
2003	* @pwq: pwq of interest
2004	* @work_data: work_data of work which left the queue
2005	*
2006	* A work either has completed or is removed from pending queue,
2007	* decrement nr_in_flight of its pwq and handle workqueue flushing.
2008	*
2009	* NOTE:
2010	* For unbound workqueues, this function may temporarily drop @pwq->pool->lock
2011	* and thus should be called after all other state updates for the in-flight
2012	* work item is complete.
2013	*
2014	* CONTEXT:
2015	* raw_spin_lock_irq(pool->lock).
2016	*/
2017	static void pwq_dec_nr_in_flight(struct pool_workqueue *pwq, unsigned long work_data)
2018	{
2019	int color = get_work_color(work_data);
2020
2021	if (!(work_data & WORK_STRUCT_INACTIVE))
2022	pwq_dec_nr_active(pwq);
2023
2024	pwq->nr_in_flight[color]--;
2025
2026	/* is flush in progress and are we at the flushing tip? */
2027	if (likely(pwq->flush_color != color))
2028	goto out_put;
2029
2030	/* are there still in-flight works? */
2031	if (pwq->nr_in_flight[color])
2032	goto out_put;
2033
2034	/* this pwq is done, clear flush_color */
2035	pwq->flush_color = -1;
2036
2037	/*
2038	* If this was the last pwq, wake up the first flusher. It
2039	* will handle the rest.
2040	*/
2041	if (atomic_dec_and_test(&pwq->wq->nr_pwqs_to_flush))
2042	complete(&pwq->wq->first_flusher->done);
2043	out_put:
2044	put_pwq(pwq);
2045	}
2046
2047	/**
2048	* try_to_grab_pending - steal work item from worklist and disable irq
2049	* @work: work item to steal
2050	* @cflags: %WORK_CANCEL_ flags
2051	* @irq_flags: place to store irq state
2052	*
2053	* Try to grab PENDING bit of @work. This function can handle @work in any
2054	* stable state - idle, on timer or on worklist.
2055	*
2056	* Return:
2057	*
2058	* ======== ================================================================
2059	* 1 if @work was pending and we successfully stole PENDING
2060	* 0 if @work was idle and we claimed PENDING
2061	* -EAGAIN if PENDING couldn't be grabbed at the moment, safe to busy-retry
2062	* ======== ================================================================
2063	*
2064	* Note:
2065	* On >= 0 return, the caller owns @work's PENDING bit. To avoid getting
2066	* interrupted while holding PENDING and @work off queue, irq must be
2067	* disabled on entry. This, combined with delayed_work->timer being
2068	* irqsafe, ensures that we return -EAGAIN for finite short period of time.
2069	*
2070	* On successful return, >= 0, irq is disabled and the caller is
2071	* responsible for releasing it using local_irq_restore(*@irq_flags).
2072	*
2073	* This function is safe to call from any context including IRQ handler.
2074	*/
2075	static int try_to_grab_pending(struct work_struct *work, u32 cflags,
2076	unsigned long *irq_flags)
2077	{
2078	struct worker_pool *pool;
2079	struct pool_workqueue *pwq;
2080
2081	local_irq_save(*irq_flags);
2082
2083	/* try to steal the timer if it exists */
2084	if (cflags & WORK_CANCEL_DELAYED) {
2085	struct delayed_work *dwork = to_delayed_work(work);
2086
2087	/*
2088	* dwork->timer is irqsafe. If del_timer() fails, it's
2089	* guaranteed that the timer is not queued anywhere and not
2090	* running on the local CPU.
2091	*/
2092	if (likely(del_timer(&dwork->timer)))
2093	return 1;
2094	}
2095
2096	/* try to claim PENDING the normal way */
2097	if (!test_and_set_bit(WORK_STRUCT_PENDING_BIT, work_data_bits(work)))
2098	return 0;
2099
2100	rcu_read_lock();
2101	/*
2102	* The queueing is in progress, or it is already queued. Try to
2103	* steal it from ->worklist without clearing WORK_STRUCT_PENDING.
2104	*/
2105	pool = get_work_pool(work);
2106	if (!pool)
2107	goto fail;
2108
2109	raw_spin_lock(&pool->lock);
2110	/*
2111	* work->data is guaranteed to point to pwq only while the work
2112	* item is queued on pwq->wq, and both updating work->data to point
2113	* to pwq on queueing and to pool on dequeueing are done under
2114	* pwq->pool->lock. This in turn guarantees that, if work->data
2115	* points to pwq which is associated with a locked pool, the work
2116	* item is currently queued on that pool.
2117	*/
2118	pwq = get_work_pwq(work);
2119	if (pwq && pwq->pool == pool) {
2120	unsigned long work_data;
2121
2122	debug_work_deactivate(work);
2123
2124	/*
2125	* A cancelable inactive work item must be in the
2126	* pwq->inactive_works since a queued barrier can't be
2127	* canceled (see the comments in insert_wq_barrier()).
2128	*
2129	* An inactive work item cannot be grabbed directly because
2130	* it might have linked barrier work items which, if left
2131	* on the inactive_works list, will confuse pwq->nr_active
2132	* management later on and cause stall. Make sure the work
2133	* item is activated before grabbing.
2134	*/
2135	pwq_activate_work(pwq, work);
2136
2137	list_del_init(&work->entry);
2138
2139	/*
2140	* work->data points to pwq iff queued. Let's point to pool. As
2141	* this destroys work->data needed by the next step, stash it.
2142	*/
2143	work_data = *work_data_bits(work);
2144	set_work_pool_and_keep_pending(work, pool->id,
2145	pool_offq_flags(pool));
2146
2147	/* must be the last step, see the function comment */
2148	pwq_dec_nr_in_flight(pwq, work_data);
2149
2150	raw_spin_unlock(&pool->lock);
2151	rcu_read_unlock();
2152	return 1;
2153	}
2154	raw_spin_unlock(&pool->lock);
2155	fail:
2156	rcu_read_unlock();
2157	local_irq_restore(*irq_flags);
2158	return -EAGAIN;
2159	}
2160
2161	/**
2162	* work_grab_pending - steal work item from worklist and disable irq
2163	* @work: work item to steal
2164	* @cflags: %WORK_CANCEL_ flags
2165	* @irq_flags: place to store IRQ state
2166	*
2167	* Grab PENDING bit of @work. @work can be in any stable state - idle, on timer
2168	* or on worklist.
2169	*
2170	* Can be called from any context. IRQ is disabled on return with IRQ state
2171	* stored in *@irq_flags. The caller is responsible for re-enabling it using
2172	* local_irq_restore().
2173	*
2174	* Returns %true if @work was pending. %false if idle.
2175	*/
2176	static bool work_grab_pending(struct work_struct *work, u32 cflags,
2177	unsigned long *irq_flags)
2178	{
2179	int ret;
2180
2181	while (true) {
2182	ret = try_to_grab_pending(work, cflags, irq_flags);
2183	if (ret >= 0)
2184	return ret;
2185	cpu_relax();
2186	}
2187	}
2188
2189	/**
2190	* insert_work - insert a work into a pool
2191	* @pwq: pwq @work belongs to
2192	* @work: work to insert
2193	* @head: insertion point
2194	* @extra_flags: extra WORK_STRUCT_* flags to set
2195	*
2196	* Insert @work which belongs to @pwq after @head. @extra_flags is or'd to
2197	* work_struct flags.
2198	*
2199	* CONTEXT:
2200	* raw_spin_lock_irq(pool->lock).
2201	*/
2202	static void insert_work(struct pool_workqueue pwq, struct work_struct work,
2203	struct list_head *head, unsigned int extra_flags)
2204	{
2205	debug_work_activate(work);
2206
2207	/* record the work call stack in order to print it in KASAN reports */
2208	kasan_record_aux_stack_noalloc(work);
2209
2210	/* we own @work, set data and link */
2211	set_work_pwq(work, pwq, extra_flags);
2212	list_add_tail(&work->entry, head);
2213	get_pwq(pwq);
2214	}
2215
2216	/*
2217	* Test whether @work is being queued from another work executing on the
2218	* same workqueue.
2219	*/
2220	static bool is_chained_work(struct workqueue_struct *wq)
2221	{
2222	struct worker *worker;
2223
2224	worker = current_wq_worker();
2225	/*
2226	* Return %true iff I'm a worker executing a work item on @wq. If
2227	* I'm @worker, it's safe to dereference it without locking.
2228	*/
2229	return worker && worker->current_pwq->wq == wq;
2230	}
2231
2232	/*
2233	* When queueing an unbound work item to a wq, prefer local CPU if allowed
2234	* by wq_unbound_cpumask. Otherwise, round robin among the allowed ones to
2235	* avoid perturbing sensitive tasks.
2236	*/
2237	static int wq_select_unbound_cpu(int cpu)
2238	{
2239	int new_cpu;
2240
2241	if (likely(!wq_debug_force_rr_cpu)) {
2242	if (cpumask_test_cpu(cpu, wq_unbound_cpumask))
2243	return cpu;
2244	} else {
2245	pr_warn_once("workqueue: round-robin CPU selection forced, expect performance impact\n");
2246	}
2247
2248	new_cpu = __this_cpu_read(wq_rr_cpu_last);
2249	new_cpu = cpumask_next_and(new_cpu, wq_unbound_cpumask, cpu_online_mask);
2250	if (unlikely(new_cpu >= nr_cpu_ids)) {
2251	new_cpu = cpumask_first_and(wq_unbound_cpumask, cpu_online_mask);
2252	if (unlikely(new_cpu >= nr_cpu_ids))
2253	return cpu;
2254	}
2255	__this_cpu_write(wq_rr_cpu_last, new_cpu);
2256
2257	return new_cpu;
2258	}
2259
2260	static void __queue_work(int cpu, struct workqueue_struct *wq,
2261	struct work_struct *work)
2262	{
2263	struct pool_workqueue *pwq;
2264	struct worker_pool last_pool, pool;
2265	unsigned int work_flags;
2266	unsigned int req_cpu = cpu;
2267
2268	/*
2269	* While a work item is PENDING && off queue, a task trying to
2270	* steal the PENDING will busy-loop waiting for it to either get
2271	* queued or lose PENDING. Grabbing PENDING and queueing should
2272	* happen with IRQ disabled.
2273	*/
2274	lockdep_assert_irqs_disabled();
2275
2276	/*
2277	* For a draining wq, only works from the same workqueue are
2278	* allowed. The __WQ_DESTROYING helps to spot the issue that
2279	* queues a new work item to a wq after destroy_workqueue(wq).
2280	*/
2281	if (unlikely(wq->flags & (__WQ_DESTROYING \| __WQ_DRAINING) &&
2282	WARN_ON_ONCE(!is_chained_work(wq))))
2283	return;
2284	rcu_read_lock();
2285	retry:
2286	/* pwq which will be used unless @work is executing elsewhere */
2287	if (req_cpu == WORK_CPU_UNBOUND) {
2288	if (wq->flags & WQ_UNBOUND)
2289	cpu = wq_select_unbound_cpu(raw_smp_processor_id());
2290	else
2291	cpu = raw_smp_processor_id();
2292	}
2293
2294	pwq = rcu_dereference(*per_cpu_ptr(wq->cpu_pwq, cpu));
2295	pool = pwq->pool;
2296
2297	/*
2298	* If @work was previously on a different pool, it might still be
2299	* running there, in which case the work needs to be queued on that
2300	* pool to guarantee non-reentrancy.
2301	*/
2302	last_pool = get_work_pool(work);
2303	if (last_pool && last_pool != pool) {
2304	struct worker *worker;
2305
2306	raw_spin_lock(&last_pool->lock);
2307
2308	worker = find_worker_executing_work(last_pool, work);
2309
2310	if (worker && worker->current_pwq->wq == wq) {
2311	pwq = worker->current_pwq;
2312	pool = pwq->pool;
2313	WARN_ON_ONCE(pool != last_pool);
2314	} else {
2315	/* meh... not running there, queue here */
2316	raw_spin_unlock(&last_pool->lock);
2317	raw_spin_lock(&pool->lock);
2318	}
2319	} else {
2320	raw_spin_lock(&pool->lock);
2321	}
2322
2323	/*
2324	* pwq is determined and locked. For unbound pools, we could have raced
2325	* with pwq release and it could already be dead. If its refcnt is zero,
2326	* repeat pwq selection. Note that unbound pwqs never die without
2327	* another pwq replacing it in cpu_pwq or while work items are executing
2328	* on it, so the retrying is guaranteed to make forward-progress.
2329	*/
2330	if (unlikely(!pwq->refcnt)) {
2331	if (wq->flags & WQ_UNBOUND) {
2332	raw_spin_unlock(&pool->lock);
2333	cpu_relax();
2334	goto retry;
2335	}
2336	/* oops */
2337	WARN_ONCE(true, "workqueue: per-cpu pwq for %s on cpu%d has 0 refcnt",
2338	wq->name, cpu);
2339	}
2340
2341	/* pwq determined, queue */
2342	trace_workqueue_queue_work(req_cpu, pwq, work);
2343
2344	if (WARN_ON(!list_empty(&work->entry)))
2345	goto out;
2346
2347	pwq->nr_in_flight[pwq->work_color]++;
2348	work_flags = work_color_to_flags(pwq->work_color);
2349
2350	/*
2351	* Limit the number of concurrently active work items to max_active.
2352	* @work must also queue behind existing inactive work items to maintain
2353	* ordering when max_active changes. See wq_adjust_max_active().
2354	*/
2355	if (list_empty(&pwq->inactive_works) && pwq_tryinc_nr_active(pwq, false)) {
2356	if (list_empty(&pool->worklist))
2357	pool->watchdog_ts = jiffies;
2358
2359	trace_workqueue_activate_work(work);
2360	insert_work(pwq, work, &pool->worklist, work_flags);
2361	kick_pool(pool);
2362	} else {
2363	work_flags \|= WORK_STRUCT_INACTIVE;
2364	insert_work(pwq, work, &pwq->inactive_works, work_flags);
2365	}
2366
2367	out:
2368	raw_spin_unlock(&pool->lock);
2369	rcu_read_unlock();
2370	}
2371
2372	static bool clear_pending_if_disabled(struct work_struct *work)
2373	{
2374	unsigned long data = *work_data_bits(work);
2375	struct work_offq_data offqd;
2376
2377	if (likely((data & WORK_STRUCT_PWQ) \|\|
2378	!(data & WORK_OFFQ_DISABLE_MASK)))
2379	return false;
2380
2381	work_offqd_unpack(&offqd, data);
2382	set_work_pool_and_clear_pending(work, offqd.pool_id,
2383	work_offqd_pack_flags(&offqd));
2384	return true;
2385	}
2386
2387	/**
2388	* queue_work_on - queue work on specific cpu
2389	* @cpu: CPU number to execute work on
2390	* @wq: workqueue to use
2391	* @work: work to queue
2392	*
2393	* We queue the work to a specific CPU, the caller must ensure it
2394	* can't go away. Callers that fail to ensure that the specified
2395	* CPU cannot go away will execute on a randomly chosen CPU.
2396	* But note well that callers specifying a CPU that never has been
2397	* online will get a splat.
2398	*
2399	* Return: %false if @work was already on a queue, %true otherwise.
2400	*/
2401	bool queue_work_on(int cpu, struct workqueue_struct *wq,
2402	struct work_struct *work)
2403	{
2404	bool ret = false;
2405	unsigned long irq_flags;
2406
2407	local_irq_save(irq_flags);
2408
2409	if (!test_and_set_bit(WORK_STRUCT_PENDING_BIT, work_data_bits(work)) &&
2410	!clear_pending_if_disabled(work)) {
2411	__queue_work(cpu, wq, work);
2412	ret = true;
2413	}
2414
2415	local_irq_restore(irq_flags);
2416	return ret;
2417	}
2418	EXPORT_SYMBOL(queue_work_on);
2419
2420	/**
2421	* select_numa_node_cpu - Select a CPU based on NUMA node
2422	* @node: NUMA node ID that we want to select a CPU from
2423	*
2424	* This function will attempt to find a "random" cpu available on a given
2425	* node. If there are no CPUs available on the given node it will return
2426	* WORK_CPU_UNBOUND indicating that we should just schedule to any
2427	* available CPU if we need to schedule this work.
2428	*/
2429	static int select_numa_node_cpu(int node)
2430	{
2431	int cpu;
2432
2433	/* Delay binding to CPU if node is not valid or online */
2434	if (node < 0 \|\| node >= MAX_NUMNODES \|\| !node_online(node))
2435	return WORK_CPU_UNBOUND;
2436
2437	/* Use local node/cpu if we are already there */
2438	cpu = raw_smp_processor_id();
2439	if (node == cpu_to_node(cpu))
2440	return cpu;
2441
2442	/* Use "random" otherwise know as "first" online CPU of node */
2443	cpu = cpumask_any_and(cpumask_of_node(node), cpu_online_mask);
2444
2445	/* If CPU is valid return that, otherwise just defer */
2446	return cpu < nr_cpu_ids ? cpu : WORK_CPU_UNBOUND;
2447	}
2448
2449	/**
2450	* queue_work_node - queue work on a "random" cpu for a given NUMA node
2451	* @node: NUMA node that we are targeting the work for
2452	* @wq: workqueue to use
2453	* @work: work to queue
2454	*
2455	* We queue the work to a "random" CPU within a given NUMA node. The basic
2456	* idea here is to provide a way to somehow associate work with a given
2457	* NUMA node.
2458	*
2459	* This function will only make a best effort attempt at getting this onto
2460	* the right NUMA node. If no node is requested or the requested node is
2461	* offline then we just fall back to standard queue_work behavior.
2462	*
2463	* Currently the "random" CPU ends up being the first available CPU in the
2464	* intersection of cpu_online_mask and the cpumask of the node, unless we
2465	* are running on the node. In that case we just use the current CPU.
2466	*
2467	* Return: %false if @work was already on a queue, %true otherwise.
2468	*/
2469	bool queue_work_node(int node, struct workqueue_struct *wq,
2470	struct work_struct *work)
2471	{
2472	unsigned long irq_flags;
2473	bool ret = false;
2474
2475	/*
2476	* This current implementation is specific to unbound workqueues.
2477	* Specifically we only return the first available CPU for a given
2478	* node instead of cycling through individual CPUs within the node.
2479	*
2480	* If this is used with a per-cpu workqueue then the logic in
2481	* workqueue_select_cpu_near would need to be updated to allow for
2482	* some round robin type logic.
2483	*/
2484	WARN_ON_ONCE(!(wq->flags & WQ_UNBOUND));
2485
2486	local_irq_save(irq_flags);
2487
2488	if (!test_and_set_bit(WORK_STRUCT_PENDING_BIT, work_data_bits(work)) &&
2489	!clear_pending_if_disabled(work)) {
2490	int cpu = select_numa_node_cpu(node);
2491
2492	__queue_work(cpu, wq, work);
2493	ret = true;
2494	}
2495
2496	local_irq_restore(irq_flags);
2497	return ret;
2498	}
2499	EXPORT_SYMBOL_GPL(queue_work_node);
2500
2501	void delayed_work_timer_fn(struct timer_list *t)
2502	{
2503	struct delayed_work *dwork = from_timer(dwork, t, timer);
2504
2505	/* should have been called from irqsafe timer with irq already off */
2506	__queue_work(dwork->cpu, dwork->wq, &dwork->work);
2507	}
2508	EXPORT_SYMBOL(delayed_work_timer_fn);
2509
2510	static void __queue_delayed_work(int cpu, struct workqueue_struct *wq,
2511	struct delayed_work *dwork, unsigned long delay)
2512	{
2513	struct timer_list *timer = &dwork->timer;
2514	struct work_struct *work = &dwork->work;
2515
2516	WARN_ON_ONCE(!wq);
2517	WARN_ON_ONCE(timer->function != delayed_work_timer_fn);
2518	WARN_ON_ONCE(timer_pending(timer));
2519	WARN_ON_ONCE(!list_empty(&work->entry));
2520
2521	/*
2522	* If @delay is 0, queue @dwork->work immediately. This is for
2523	* both optimization and correctness. The earliest @timer can
2524	* expire is on the closest next tick and delayed_work users depend
2525	* on that there's no such delay when @delay is 0.
2526	*/
2527	if (!delay) {
2528	__queue_work(cpu, wq, &dwork->work);
2529	return;
2530	}
2531
2532	dwork->wq = wq;
2533	dwork->cpu = cpu;
2534	timer->expires = jiffies + delay;
2535
2536	if (housekeeping_enabled(HK_TYPE_TIMER)) {
2537	/* If the current cpu is a housekeeping cpu, use it. */
2538	cpu = smp_processor_id();
2539	if (!housekeeping_test_cpu(cpu, HK_TYPE_TIMER))
2540	cpu = housekeeping_any_cpu(HK_TYPE_TIMER);
2541	add_timer_on(timer, cpu);
2542	} else {
2543	if (likely(cpu == WORK_CPU_UNBOUND))
2544	add_timer_global(timer);
2545	else
2546	add_timer_on(timer, cpu);
2547	}
2548	}
2549
2550	/**
2551	* queue_delayed_work_on - queue work on specific CPU after delay
2552	* @cpu: CPU number to execute work on
2553	* @wq: workqueue to use
2554	* @dwork: work to queue
2555	* @delay: number of jiffies to wait before queueing
2556	*
2557	* Return: %false if @work was already on a queue, %true otherwise. If
2558	* @delay is zero and @dwork is idle, it will be scheduled for immediate
2559	* execution.
2560	*/
2561	bool queue_delayed_work_on(int cpu, struct workqueue_struct *wq,
2562	struct delayed_work *dwork, unsigned long delay)
2563	{
2564	struct work_struct *work = &dwork->work;
2565	bool ret = false;
2566	unsigned long irq_flags;
2567
2568	/* read the comment in __queue_work() */
2569	local_irq_save(irq_flags);
2570
2571	if (!test_and_set_bit(WORK_STRUCT_PENDING_BIT, work_data_bits(work)) &&
2572	!clear_pending_if_disabled(work)) {
2573	__queue_delayed_work(cpu, wq, dwork, delay);
2574	ret = true;
2575	}
2576
2577	local_irq_restore(irq_flags);
2578	return ret;
2579	}
2580	EXPORT_SYMBOL(queue_delayed_work_on);
2581
2582	/**
2583	* mod_delayed_work_on - modify delay of or queue a delayed work on specific CPU
2584	* @cpu: CPU number to execute work on
2585	* @wq: workqueue to use
2586	* @dwork: work to queue
2587	* @delay: number of jiffies to wait before queueing
2588	*
2589	* If @dwork is idle, equivalent to queue_delayed_work_on(); otherwise,
2590	* modify @dwork's timer so that it expires after @delay. If @delay is
2591	* zero, @work is guaranteed to be scheduled immediately regardless of its
2592	* current state.
2593	*
2594	* Return: %false if @dwork was idle and queued, %true if @dwork was
2595	* pending and its timer was modified.
2596	*
2597	* This function is safe to call from any context including IRQ handler.
2598	* See try_to_grab_pending() for details.
2599	*/
2600	bool mod_delayed_work_on(int cpu, struct workqueue_struct *wq,
2601	struct delayed_work *dwork, unsigned long delay)
2602	{
2603	unsigned long irq_flags;
2604	bool ret;
2605
2606	ret = work_grab_pending(&dwork->work, WORK_CANCEL_DELAYED, &irq_flags);
2607
2608	if (!clear_pending_if_disabled(&dwork->work))
2609	__queue_delayed_work(cpu, wq, dwork, delay);
2610
2611	local_irq_restore(irq_flags);
2612	return ret;
2613	}
2614	EXPORT_SYMBOL_GPL(mod_delayed_work_on);
2615
2616	static void rcu_work_rcufn(struct rcu_head *rcu)
2617	{
2618	struct rcu_work *rwork = container_of(rcu, struct rcu_work, rcu);
2619
2620	/* read the comment in __queue_work() */
2621	local_irq_disable();
2622	__queue_work(WORK_CPU_UNBOUND, rwork->wq, &rwork->work);
2623	local_irq_enable();
2624	}
2625
2626	/**
2627	* queue_rcu_work - queue work after a RCU grace period
2628	* @wq: workqueue to use
2629	* @rwork: work to queue
2630	*
2631	* Return: %false if @rwork was already pending, %true otherwise. Note
2632	* that a full RCU grace period is guaranteed only after a %true return.
2633	* While @rwork is guaranteed to be executed after a %false return, the
2634	* execution may happen before a full RCU grace period has passed.
2635	*/
2636	bool queue_rcu_work(struct workqueue_struct wq, struct rcu_work rwork)
2637	{
2638	struct work_struct *work = &rwork->work;
2639
2640	/*
2641	* rcu_work can't be canceled or disabled. Warn if the user reached
2642	* inside @rwork and disabled the inner work.
2643	*/
2644	if (!test_and_set_bit(WORK_STRUCT_PENDING_BIT, work_data_bits(work)) &&
2645	!WARN_ON_ONCE(clear_pending_if_disabled(work))) {
2646	rwork->wq = wq;
2647	call_rcu_hurry(&rwork->rcu, rcu_work_rcufn);
2648	return true;
2649	}
2650
2651	return false;
2652	}
2653	EXPORT_SYMBOL(queue_rcu_work);
2654
2655	static struct worker *alloc_worker(int node)
2656	{
2657	struct worker *worker;
2658
2659	worker = kzalloc_node(sizeof(*worker), GFP_KERNEL, node);
2660	if (worker) {
2661	INIT_LIST_HEAD(&worker->entry);
2662	INIT_LIST_HEAD(&worker->scheduled);
2663	INIT_LIST_HEAD(&worker->node);
2664	/* on creation a worker is in !idle && prep state */
2665	worker->flags = WORKER_PREP;
2666	}
2667	return worker;
2668	}
2669
2670	static cpumask_t pool_allowed_cpus(struct worker_pool pool)
2671	{
2672	if (pool->cpu < 0 && pool->attrs->affn_strict)
2673	return pool->attrs->__pod_cpumask;
2674	else
2675	return pool->attrs->cpumask;
2676	}
2677
2678	/**
2679	* worker_attach_to_pool() - attach a worker to a pool
2680	* @worker: worker to be attached
2681	* @pool: the target pool
2682	*
2683	* Attach @worker to @pool. Once attached, the %WORKER_UNBOUND flag and
2684	* cpu-binding of @worker are kept coordinated with the pool across
2685	* cpu-[un]hotplugs.
2686	*/
2687	static void worker_attach_to_pool(struct worker *worker,
2688	struct worker_pool *pool)
2689	{
2690	mutex_lock(&wq_pool_attach_mutex);
2691
2692	/*
2693	* The wq_pool_attach_mutex ensures %POOL_DISASSOCIATED remains stable
2694	* across this function. See the comments above the flag definition for
2695	* details. BH workers are, while per-CPU, always DISASSOCIATED.
2696	*/
2697	if (pool->flags & POOL_DISASSOCIATED) {
2698	worker->flags \|= WORKER_UNBOUND;
2699	} else {
2700	WARN_ON_ONCE(pool->flags & POOL_BH);
2701	kthread_set_per_cpu(worker->task, pool->cpu);
2702	}
2703
2704	if (worker->rescue_wq)
2705	set_cpus_allowed_ptr(worker->task, pool_allowed_cpus(pool));
2706
2707	list_add_tail(&worker->node, &pool->workers);
2708	worker->pool = pool;
2709
2710	mutex_unlock(&wq_pool_attach_mutex);
2711	}
2712
2713	/**
2714	* worker_detach_from_pool() - detach a worker from its pool
2715	* @worker: worker which is attached to its pool
2716	*
2717	* Undo the attaching which had been done in worker_attach_to_pool(). The
2718	* caller worker shouldn't access to the pool after detached except it has
2719	* other reference to the pool.
2720	*/
2721	static void worker_detach_from_pool(struct worker *worker)
2722	{
2723	struct worker_pool *pool = worker->pool;
2724	struct completion *detach_completion = NULL;
2725
2726	/* there is one permanent BH worker per CPU which should never detach */
2727	WARN_ON_ONCE(pool->flags & POOL_BH);
2728
2729	mutex_lock(&wq_pool_attach_mutex);
2730
2731	kthread_set_per_cpu(worker->task, -1);
2732	list_del(&worker->node);
2733	worker->pool = NULL;
2734
2735	if (list_empty(&pool->workers) && list_empty(&pool->dying_workers))
2736	detach_completion = pool->detach_completion;
2737	mutex_unlock(&wq_pool_attach_mutex);
2738
2739	/* clear leftover flags without pool->lock after it is detached */
2740	worker->flags &= ~(WORKER_UNBOUND \| WORKER_REBOUND);
2741
2742	if (detach_completion)
2743	complete(detach_completion);
2744	}
2745
2746	static int format_worker_id(char buf, size_t size, struct worker worker,
2747	struct worker_pool *pool)
2748	{
2749	if (worker->rescue_wq)
2750	return scnprintf(buf, size, "kworker/R-%s",
2751	worker->rescue_wq->name);
2752
2753	if (pool) {
2754	if (pool->cpu >= 0)
2755	return scnprintf(buf, size, "kworker/%d:%d%s",
2756	pool->cpu, worker->id,
2757	pool->attrs->nice < 0 ? "H" : "");
2758	else
2759	return scnprintf(buf, size, "kworker/u%d:%d",
2760	pool->id, worker->id);
2761	} else {
2762	return scnprintf(buf, size, "kworker/dying");
2763	}
2764	}
2765
2766	/**
2767	* create_worker - create a new workqueue worker
2768	* @pool: pool the new worker will belong to
2769	*
2770	* Create and start a new worker which is attached to @pool.
2771	*
2772	* CONTEXT:
2773	* Might sleep. Does GFP_KERNEL allocations.
2774	*
2775	* Return:
2776	* Pointer to the newly created worker.
2777	*/
2778	static struct worker create_worker(struct worker_pool pool)
2779	{
2780	struct worker *worker;
2781	int id;
2782
2783	/* ID is needed to determine kthread name */
2784	id = ida_alloc(&pool->worker_ida, GFP_KERNEL);
2785	if (id < 0) {
2786	pr_err_once("workqueue: Failed to allocate a worker ID: %pe\n",
2787	ERR_PTR(id));
2788	return NULL;
2789	}
2790
2791	worker = alloc_worker(pool->node);
2792	if (!worker) {
2793	pr_err_once("workqueue: Failed to allocate a worker\n");
2794	goto fail;
2795	}
2796
2797	worker->id = id;
2798
2799	if (!(pool->flags & POOL_BH)) {
2800	char id_buf[WORKER_ID_LEN];
2801
2802	format_worker_id(id_buf, sizeof(id_buf), worker, pool);
2803	worker->task = kthread_create_on_node(worker_thread, worker,
2804	pool->node, "%s", id_buf);
2805	if (IS_ERR(worker->task)) {
2806	if (PTR_ERR(worker->task) == -EINTR) {
2807	pr_err("workqueue: Interrupted when creating a worker thread \"%s\"\n",
2808	id_buf);
2809	} else {
2810	pr_err_once("workqueue: Failed to create a worker thread: %pe",
2811	worker->task);
2812	}
2813	goto fail;
2814	}
2815
2816	set_user_nice(worker->task, pool->attrs->nice);
2817	kthread_bind_mask(worker->task, pool_allowed_cpus(pool));
2818	}
2819
2820	/* successful, attach the worker to the pool */
2821	worker_attach_to_pool(worker, pool);
2822
2823	/* start the newly created worker */
2824	raw_spin_lock_irq(&pool->lock);
2825
2826	worker->pool->nr_workers++;
2827	worker_enter_idle(worker);
2828
2829	/*
2830	* @worker is waiting on a completion in kthread() and will trigger hung
2831	* check if not woken up soon. As kick_pool() is noop if @pool is empty,
2832	* wake it up explicitly.
2833	*/
2834	if (worker->task)
2835	wake_up_process(worker->task);
2836
2837	raw_spin_unlock_irq(&pool->lock);
2838
2839	return worker;
2840
2841	fail:
2842	ida_free(&pool->worker_ida, id);
2843	kfree(worker);
2844	return NULL;
2845	}
2846
2847	static void unbind_worker(struct worker *worker)
2848	{
2849	lockdep_assert_held(&wq_pool_attach_mutex);
2850
2851	kthread_set_per_cpu(worker->task, -1);
2852	if (cpumask_intersects(wq_unbound_cpumask, cpu_active_mask))
2853	WARN_ON_ONCE(set_cpus_allowed_ptr(worker->task, wq_unbound_cpumask) < 0);
2854	else
2855	WARN_ON_ONCE(set_cpus_allowed_ptr(worker->task, cpu_possible_mask) < 0);
2856	}
2857
2858	static void wake_dying_workers(struct list_head *cull_list)
2859	{
2860	struct worker worker, tmp;
2861
2862	list_for_each_entry_safe(worker, tmp, cull_list, entry) {
2863	list_del_init(&worker->entry);
2864	unbind_worker(worker);
2865	/*
2866	* If the worker was somehow already running, then it had to be
2867	* in pool->idle_list when set_worker_dying() happened or we
2868	* wouldn't have gotten here.
2869	*
2870	* Thus, the worker must either have observed the WORKER_DIE
2871	* flag, or have set its state to TASK_IDLE. Either way, the
2872	* below will be observed by the worker and is safe to do
2873	* outside of pool->lock.
2874	*/
2875	wake_up_process(worker->task);
2876	}
2877	}
2878
2879	/**
2880	* set_worker_dying - Tag a worker for destruction
2881	* @worker: worker to be destroyed
2882	* @list: transfer worker away from its pool->idle_list and into list
2883	*
2884	* Tag @worker for destruction and adjust @pool stats accordingly. The worker
2885	* should be idle.
2886	*
2887	* CONTEXT:
2888	* raw_spin_lock_irq(pool->lock).
2889	*/
2890	static void set_worker_dying(struct worker worker, struct list_head list)
2891	{
2892	struct worker_pool *pool = worker->pool;
2893
2894	lockdep_assert_held(&pool->lock);
2895	lockdep_assert_held(&wq_pool_attach_mutex);
2896
2897	/* sanity check frenzy */
2898	if (WARN_ON(worker->current_work) \|\|
2899	WARN_ON(!list_empty(&worker->scheduled)) \|\|
2900	WARN_ON(!(worker->flags & WORKER_IDLE)))
2901	return;
2902
2903	pool->nr_workers--;
2904	pool->nr_idle--;
2905
2906	worker->flags \|= WORKER_DIE;
2907
2908	list_move(&worker->entry, list);
2909	list_move(&worker->node, &pool->dying_workers);
2910	}
2911
2912	/**
2913	* idle_worker_timeout - check if some idle workers can now be deleted.
2914	* @t: The pool's idle_timer that just expired
2915	*
2916	* The timer is armed in worker_enter_idle(). Note that it isn't disarmed in
2917	* worker_leave_idle(), as a worker flicking between idle and active while its
2918	* pool is at the too_many_workers() tipping point would cause too much timer
2919	* housekeeping overhead. Since IDLE_WORKER_TIMEOUT is long enough, we just let
2920	* it expire and re-evaluate things from there.
2921	*/
2922	static void idle_worker_timeout(struct timer_list *t)
2923	{
2924	struct worker_pool *pool = from_timer(pool, t, idle_timer);
2925	bool do_cull = false;
2926
2927	if (work_pending(&pool->idle_cull_work))
2928	return;
2929
2930	raw_spin_lock_irq(&pool->lock);
2931
2932	if (too_many_workers(pool)) {
2933	struct worker *worker;
2934	unsigned long expires;
2935
2936	/* idle_list is kept in LIFO order, check the last one */
2937	worker = list_last_entry(&pool->idle_list, struct worker, entry);
2938	expires = worker->last_active + IDLE_WORKER_TIMEOUT;
2939	do_cull = !time_before(jiffies, expires);
2940
2941	if (!do_cull)
2942	mod_timer(&pool->idle_timer, expires);
2943	}
2944	raw_spin_unlock_irq(&pool->lock);
2945
2946	if (do_cull)
2947	queue_work(system_unbound_wq, &pool->idle_cull_work);
2948	}
2949
2950	/**
2951	* idle_cull_fn - cull workers that have been idle for too long.
2952	* @work: the pool's work for handling these idle workers
2953	*
2954	* This goes through a pool's idle workers and gets rid of those that have been
2955	* idle for at least IDLE_WORKER_TIMEOUT seconds.
2956	*
2957	* We don't want to disturb isolated CPUs because of a pcpu kworker being
2958	* culled, so this also resets worker affinity. This requires a sleepable
2959	* context, hence the split between timer callback and work item.
2960	*/
2961	static void idle_cull_fn(struct work_struct *work)
2962	{
2963	struct worker_pool *pool = container_of(work, struct worker_pool, idle_cull_work);
2964	LIST_HEAD(cull_list);
2965
2966	/*
2967	* Grabbing wq_pool_attach_mutex here ensures an already-running worker
2968	* cannot proceed beyong worker_detach_from_pool() in its self-destruct
2969	* path. This is required as a previously-preempted worker could run after
2970	* set_worker_dying() has happened but before wake_dying_workers() did.
2971	*/
2972	mutex_lock(&wq_pool_attach_mutex);
2973	raw_spin_lock_irq(&pool->lock);
2974
2975	while (too_many_workers(pool)) {
2976	struct worker *worker;
2977	unsigned long expires;
2978
2979	worker = list_last_entry(&pool->idle_list, struct worker, entry);
2980	expires = worker->last_active + IDLE_WORKER_TIMEOUT;
2981
2982	if (time_before(jiffies, expires)) {
2983	mod_timer(&pool->idle_timer, expires);
2984	break;
2985	}
2986
2987	set_worker_dying(worker, &cull_list);
2988	}
2989
2990	raw_spin_unlock_irq(&pool->lock);
2991	wake_dying_workers(&cull_list);
2992	mutex_unlock(&wq_pool_attach_mutex);
2993	}
2994
2995	static void send_mayday(struct work_struct *work)
2996	{
2997	struct pool_workqueue *pwq = get_work_pwq(work);
2998	struct workqueue_struct *wq = pwq->wq;
2999
3000	lockdep_assert_held(&wq_mayday_lock);
3001
3002	if (!wq->rescuer)
3003	return;
3004
3005	/* mayday mayday mayday */
3006	if (list_empty(&pwq->mayday_node)) {
3007	/*
3008	* If @pwq is for an unbound wq, its base ref may be put at
3009	* any time due to an attribute change. Pin @pwq until the
3010	* rescuer is done with it.
3011	*/
3012	get_pwq(pwq);
3013	list_add_tail(&pwq->mayday_node, &wq->maydays);
3014	wake_up_process(wq->rescuer->task);
3015	pwq->stats[PWQ_STAT_MAYDAY]++;
3016	}
3017	}
3018
3019	static void pool_mayday_timeout(struct timer_list *t)
3020	{
3021	struct worker_pool *pool = from_timer(pool, t, mayday_timer);
3022	struct work_struct *work;
3023
3024	raw_spin_lock_irq(&pool->lock);
3025	raw_spin_lock(&wq_mayday_lock); /* for wq->maydays */
3026
3027	if (need_to_create_worker(pool)) {
3028	/*
3029	* We've been trying to create a new worker but
3030	* haven't been successful. We might be hitting an
3031	* allocation deadlock. Send distress signals to
3032	* rescuers.
3033	*/
3034	list_for_each_entry(work, &pool->worklist, entry)
3035	send_mayday(work);
3036	}
3037
3038	raw_spin_unlock(&wq_mayday_lock);
3039	raw_spin_unlock_irq(&pool->lock);
3040
3041	mod_timer(&pool->mayday_timer, jiffies + MAYDAY_INTERVAL);
3042	}
3043
3044	/**
3045	* maybe_create_worker - create a new worker if necessary
3046	* @pool: pool to create a new worker for
3047	*
3048	* Create a new worker for @pool if necessary. @pool is guaranteed to
3049	* have at least one idle worker on return from this function. If
3050	* creating a new worker takes longer than MAYDAY_INTERVAL, mayday is
3051	* sent to all rescuers with works scheduled on @pool to resolve
3052	* possible allocation deadlock.
3053	*
3054	* On return, need_to_create_worker() is guaranteed to be %false and
3055	* may_start_working() %true.
3056	*
3057	* LOCKING:
3058	* raw_spin_lock_irq(pool->lock) which may be released and regrabbed
3059	* multiple times. Does GFP_KERNEL allocations. Called only from
3060	* manager.
3061	*/
3062	static void maybe_create_worker(struct worker_pool *pool)
3063	__releases(&pool->lock)
3064	__acquires(&pool->lock)
3065	{
3066	restart:
3067	raw_spin_unlock_irq(&pool->lock);
3068
3069	/* if we don't make progress in MAYDAY_INITIAL_TIMEOUT, call for help */
3070	mod_timer(&pool->mayday_timer, jiffies + MAYDAY_INITIAL_TIMEOUT);
3071
3072	while (true) {
3073	if (create_worker(pool) \|\| !need_to_create_worker(pool))
3074	break;
3075
3076	schedule_timeout_interruptible(CREATE_COOLDOWN);
3077
3078	if (!need_to_create_worker(pool))
3079	break;
3080	}
3081
3082	del_timer_sync(&pool->mayday_timer);
3083	raw_spin_lock_irq(&pool->lock);
3084	/*
3085	* This is necessary even after a new worker was just successfully
3086	* created as @pool->lock was dropped and the new worker might have
3087	* already become busy.
3088	*/
3089	if (need_to_create_worker(pool))
3090	goto restart;
3091	}
3092
3093	/**
3094	* manage_workers - manage worker pool
3095	* @worker: self
3096	*
3097	* Assume the manager role and manage the worker pool @worker belongs
3098	* to. At any given time, there can be only zero or one manager per
3099	* pool. The exclusion is handled automatically by this function.
3100	*
3101	* The caller can safely start processing works on false return. On
3102	* true return, it's guaranteed that need_to_create_worker() is false
3103	* and may_start_working() is true.
3104	*
3105	* CONTEXT:
3106	* raw_spin_lock_irq(pool->lock) which may be released and regrabbed
3107	* multiple times. Does GFP_KERNEL allocations.
3108	*
3109	* Return:
3110	* %false if the pool doesn't need management and the caller can safely
3111	* start processing works, %true if management function was performed and
3112	* the conditions that the caller verified before calling the function may
3113	* no longer be true.
3114	*/
3115	static bool manage_workers(struct worker *worker)
3116	{
3117	struct worker_pool *pool = worker->pool;
3118
3119	if (pool->flags & POOL_MANAGER_ACTIVE)
3120	return false;
3121
3122	pool->flags \|= POOL_MANAGER_ACTIVE;
3123	pool->manager = worker;
3124
3125	maybe_create_worker(pool);
3126
3127	pool->manager = NULL;
3128	pool->flags &= ~POOL_MANAGER_ACTIVE;
3129	rcuwait_wake_up(&manager_wait);
3130	return true;
3131	}
3132
3133	/**
3134	* process_one_work - process single work
3135	* @worker: self
3136	* @work: work to process
3137	*
3138	* Process @work. This function contains all the logics necessary to
3139	* process a single work including synchronization against and
3140	* interaction with other workers on the same cpu, queueing and
3141	* flushing. As long as context requirement is met, any worker can
3142	* call this function to process a work.
3143	*
3144	* CONTEXT:
3145	* raw_spin_lock_irq(pool->lock) which is released and regrabbed.
3146	*/
3147	static void process_one_work(struct worker worker, struct work_struct work)
3148	__releases(&pool->lock)
3149	__acquires(&pool->lock)
3150	{
3151	struct pool_workqueue *pwq = get_work_pwq(work);
3152	struct worker_pool *pool = worker->pool;
3153	unsigned long work_data;
3154	int lockdep_start_depth, rcu_start_depth;
3155	bool bh_draining = pool->flags & POOL_BH_DRAINING;
3156	#ifdef CONFIG_LOCKDEP
3157	/*
3158	* It is permissible to free the struct work_struct from
3159	* inside the function that is called from it, this we need to
3160	* take into account for lockdep too. To avoid bogus "held
3161	* lock freed" warnings as well as problems when looking into
3162	* work->lockdep_map, make a copy and use that here.
3163	*/
3164	struct lockdep_map lockdep_map;
3165
3166	lockdep_copy_map(&lockdep_map, &work->lockdep_map);
3167	#endif
3168	/* ensure we're on the correct CPU */
3169	WARN_ON_ONCE(!(pool->flags & POOL_DISASSOCIATED) &&
3170	raw_smp_processor_id() != pool->cpu);
3171
3172	/* claim and dequeue */
3173	debug_work_deactivate(work);
3174	hash_add(pool->busy_hash, &worker->hentry, (unsigned long)work);
3175	worker->current_work = work;
3176	worker->current_func = work->func;
3177	worker->current_pwq = pwq;
3178	if (worker->task)
3179	worker->current_at = worker->task->se.sum_exec_runtime;
3180	work_data = *work_data_bits(work);
3181	worker->current_color = get_work_color(work_data);
3182
3183	/*
3184	* Record wq name for cmdline and debug reporting, may get
3185	* overridden through set_worker_desc().
3186	*/
3187	strscpy(worker->desc, pwq->wq->name, WORKER_DESC_LEN);
3188
3189	list_del_init(&work->entry);
3190
3191	/*
3192	* CPU intensive works don't participate in concurrency management.
3193	* They're the scheduler's responsibility. This takes @worker out
3194	* of concurrency management and the next code block will chain
3195	* execution of the pending work items.
3196	*/
3197	if (unlikely(pwq->wq->flags & WQ_CPU_INTENSIVE))
3198	worker_set_flags(worker, WORKER_CPU_INTENSIVE);
3199
3200	/*
3201	* Kick @pool if necessary. It's always noop for per-cpu worker pools
3202	* since nr_running would always be >= 1 at this point. This is used to
3203	* chain execution of the pending work items for WORKER_NOT_RUNNING
3204	* workers such as the UNBOUND and CPU_INTENSIVE ones.
3205	*/
3206	kick_pool(pool);
3207
3208	/*
3209	* Record the last pool and clear PENDING which should be the last
3210	* update to @work. Also, do this inside @pool->lock so that
3211	* PENDING and queued state changes happen together while IRQ is
3212	* disabled.
3213	*/
3214	set_work_pool_and_clear_pending(work, pool->id, pool_offq_flags(pool));
3215
3216	pwq->stats[PWQ_STAT_STARTED]++;
3217	raw_spin_unlock_irq(&pool->lock);
3218
3219	rcu_start_depth = rcu_preempt_depth();
3220	lockdep_start_depth = lockdep_depth(current);
3221	/* see drain_dead_softirq_workfn() */
3222	if (!bh_draining)
3223	lock_map_acquire(&pwq->wq->lockdep_map);
3224	lock_map_acquire(&lockdep_map);
3225	/*
3226	* Strictly speaking we should mark the invariant state without holding
3227	* any locks, that is, before these two lock_map_acquire()'s.
3228	*
3229	* However, that would result in:
3230	*
3231	* A(W1)
3232	* WFC(C)
3233	* A(W1)
3234	* C(C)
3235	*
3236	* Which would create W1->C->W1 dependencies, even though there is no
3237	* actual deadlock possible. There are two solutions, using a
3238	* read-recursive acquire on the work(queue) 'locks', but this will then
3239	* hit the lockdep limitation on recursive locks, or simply discard
3240	* these locks.
3241	*
3242	* AFAICT there is no possible deadlock scenario between the
3243	* flush_work() and complete() primitives (except for single-threaded
3244	* workqueues), so hiding them isn't a problem.
3245	*/
3246	lockdep_invariant_state(true);
3247	trace_workqueue_execute_start(work);
3248	worker->current_func(work);
3249	/*
3250	* While we must be careful to not use "work" after this, the trace
3251	* point will only record its address.
3252	*/
3253	trace_workqueue_execute_end(work, worker->current_func);
3254	pwq->stats[PWQ_STAT_COMPLETED]++;
3255	lock_map_release(&lockdep_map);
3256	if (!bh_draining)
3257	lock_map_release(&pwq->wq->lockdep_map);
3258
3259	if (unlikely((worker->task && in_atomic()) \|\|
3260	lockdep_depth(current) != lockdep_start_depth \|\|
3261	rcu_preempt_depth() != rcu_start_depth)) {
3262	pr_err("BUG: workqueue leaked atomic, lock or RCU: %s[%d]\n"
3263	" preempt=0x%08x lock=%d->%d RCU=%d->%d workfn=%ps\n",
3264	current->comm, task_pid_nr(current), preempt_count(),
3265	lockdep_start_depth, lockdep_depth(current),
3266	rcu_start_depth, rcu_preempt_depth(),
3267	worker->current_func);
3268	debug_show_held_locks(current);
3269	dump_stack();
3270	}
3271
3272	/*
3273	* The following prevents a kworker from hogging CPU on !PREEMPTION
3274	* kernels, where a requeueing work item waiting for something to
3275	* happen could deadlock with stop_machine as such work item could
3276	* indefinitely requeue itself while all other CPUs are trapped in
3277	* stop_machine. At the same time, report a quiescent RCU state so
3278	* the same condition doesn't freeze RCU.
3279	*/
3280	if (worker->task)
3281	cond_resched();
3282
3283	raw_spin_lock_irq(&pool->lock);
3284
3285	/*
3286	* In addition to %WQ_CPU_INTENSIVE, @worker may also have been marked
3287	* CPU intensive by wq_worker_tick() if @work hogged CPU longer than
3288	* wq_cpu_intensive_thresh_us. Clear it.
3289	*/
3290	worker_clr_flags(worker, WORKER_CPU_INTENSIVE);
3291
3292	/* tag the worker for identification in schedule() */
3293	worker->last_func = worker->current_func;
3294
3295	/* we're done with it, release */
3296	hash_del(&worker->hentry);
3297	worker->current_work = NULL;
3298	worker->current_func = NULL;
3299	worker->current_pwq = NULL;
3300	worker->current_color = INT_MAX;
3301
3302	/* must be the last step, see the function comment */
3303	pwq_dec_nr_in_flight(pwq, work_data);
3304	}
3305
3306	/**
3307	* process_scheduled_works - process scheduled works
3308	* @worker: self
3309	*
3310	* Process all scheduled works. Please note that the scheduled list
3311	* may change while processing a work, so this function repeatedly
3312	* fetches a work from the top and executes it.
3313	*
3314	* CONTEXT:
3315	* raw_spin_lock_irq(pool->lock) which may be released and regrabbed
3316	* multiple times.
3317	*/
3318	static void process_scheduled_works(struct worker *worker)
3319	{
3320	struct work_struct *work;
3321	bool first = true;
3322
3323	while ((work = list_first_entry_or_null(&worker->scheduled,
3324	struct work_struct, entry))) {
3325	if (first) {
3326	worker->pool->watchdog_ts = jiffies;
3327	first = false;
3328	}
3329	process_one_work(worker, work);
3330	}
3331	}
3332
3333	static void set_pf_worker(bool val)
3334	{
3335	mutex_lock(&wq_pool_attach_mutex);
3336	if (val)
3337	current->flags \|= PF_WQ_WORKER;
3338	else
3339	current->flags &= ~PF_WQ_WORKER;
3340	mutex_unlock(&wq_pool_attach_mutex);
3341	}
3342
3343	/**
3344	* worker_thread - the worker thread function
3345	* @__worker: self
3346	*
3347	* The worker thread function. All workers belong to a worker_pool -
3348	* either a per-cpu one or dynamic unbound one. These workers process all
3349	* work items regardless of their specific target workqueue. The only
3350	* exception is work items which belong to workqueues with a rescuer which
3351	* will be explained in rescuer_thread().
3352	*
3353	* Return: 0
3354	*/
3355	static int worker_thread(void *__worker)
3356	{
3357	struct worker *worker = __worker;
3358	struct worker_pool *pool = worker->pool;
3359
3360	/* tell the scheduler that this is a workqueue worker */
3361	set_pf_worker(true);
3362	woke_up:
3363	raw_spin_lock_irq(&pool->lock);
3364
3365	/* am I supposed to die? */
3366	if (unlikely(worker->flags & WORKER_DIE)) {
3367	raw_spin_unlock_irq(&pool->lock);
3368	set_pf_worker(false);
3369
3370	ida_free(&pool->worker_ida, worker->id);
3371	worker_detach_from_pool(worker);
3372	WARN_ON_ONCE(!list_empty(&worker->entry));
3373	kfree(worker);
3374	return 0;
3375	}
3376
3377	worker_leave_idle(worker);
3378	recheck:
3379	/* no more worker necessary? */
3380	if (!need_more_worker(pool))
3381	goto sleep;
3382
3383	/* do we need to manage? */
3384	if (unlikely(!may_start_working(pool)) && manage_workers(worker))
3385	goto recheck;
3386
3387	/*
3388	* ->scheduled list can only be filled while a worker is
3389	* preparing to process a work or actually processing it.
3390	* Make sure nobody diddled with it while I was sleeping.
3391	*/
3392	WARN_ON_ONCE(!list_empty(&worker->scheduled));
3393
3394	/*
3395	* Finish PREP stage. We're guaranteed to have at least one idle
3396	* worker or that someone else has already assumed the manager
3397	* role. This is where @worker starts participating in concurrency
3398	* management if applicable and concurrency management is restored
3399	* after being rebound. See rebind_workers() for details.
3400	*/
3401	worker_clr_flags(worker, WORKER_PREP \| WORKER_REBOUND);
3402
3403	do {
3404	struct work_struct *work =
3405	list_first_entry(&pool->worklist,
3406	struct work_struct, entry);
3407
3408	if (assign_work(work, worker, NULL))
3409	process_scheduled_works(worker);
3410	} while (keep_working(pool));
3411
3412	worker_set_flags(worker, WORKER_PREP);
3413	sleep:
3414	/*
3415	* pool->lock is held and there's no work to process and no need to
3416	* manage, sleep. Workers are woken up only while holding
3417	* pool->lock or from local cpu, so setting the current state
3418	* before releasing pool->lock is enough to prevent losing any
3419	* event.
3420	*/
3421	worker_enter_idle(worker);
3422	__set_current_state(TASK_IDLE);
3423	raw_spin_unlock_irq(&pool->lock);
3424	schedule();
3425	goto woke_up;
3426	}
3427
3428	/**
3429	* rescuer_thread - the rescuer thread function
3430	* @__rescuer: self
3431	*
3432	* Workqueue rescuer thread function. There's one rescuer for each
3433	* workqueue which has WQ_MEM_RECLAIM set.
3434	*
3435	* Regular work processing on a pool may block trying to create a new
3436	* worker which uses GFP_KERNEL allocation which has slight chance of
3437	* developing into deadlock if some works currently on the same queue
3438	* need to be processed to satisfy the GFP_KERNEL allocation. This is
3439	* the problem rescuer solves.
3440	*
3441	* When such condition is possible, the pool summons rescuers of all
3442	* workqueues which have works queued on the pool and let them process
3443	* those works so that forward progress can be guaranteed.
3444	*
3445	* This should happen rarely.
3446	*
3447	* Return: 0
3448	*/
3449	static int rescuer_thread(void *__rescuer)
3450	{
3451	struct worker *rescuer = __rescuer;
3452	struct workqueue_struct *wq = rescuer->rescue_wq;
3453	bool should_stop;
3454
3455	set_user_nice(current, RESCUER_NICE_LEVEL);
3456
3457	/*
3458	* Mark rescuer as worker too. As WORKER_PREP is never cleared, it
3459	* doesn't participate in concurrency management.
3460	*/
3461	set_pf_worker(true);
3462	repeat:
3463	set_current_state(TASK_IDLE);
3464
3465	/*
3466	* By the time the rescuer is requested to stop, the workqueue
3467	* shouldn't have any work pending, but @wq->maydays may still have
3468	* pwq(s) queued. This can happen by non-rescuer workers consuming
3469	* all the work items before the rescuer got to them. Go through
3470	* @wq->maydays processing before acting on should_stop so that the
3471	* list is always empty on exit.
3472	*/
3473	should_stop = kthread_should_stop();
3474
3475	/* see whether any pwq is asking for help */
3476	raw_spin_lock_irq(&wq_mayday_lock);
3477
3478	while (!list_empty(&wq->maydays)) {
3479	struct pool_workqueue *pwq = list_first_entry(&wq->maydays,
3480	struct pool_workqueue, mayday_node);
3481	struct worker_pool *pool = pwq->pool;
3482	struct work_struct work, n;
3483
3484	__set_current_state(TASK_RUNNING);
3485	list_del_init(&pwq->mayday_node);
3486
3487	raw_spin_unlock_irq(&wq_mayday_lock);
3488
3489	worker_attach_to_pool(rescuer, pool);
3490
3491	raw_spin_lock_irq(&pool->lock);
3492
3493	/*
3494	* Slurp in all works issued via this workqueue and
3495	* process'em.
3496	*/
3497	WARN_ON_ONCE(!list_empty(&rescuer->scheduled));
3498	list_for_each_entry_safe(work, n, &pool->worklist, entry) {
3499	if (get_work_pwq(work) == pwq &&
3500	assign_work(work, rescuer, &n))
3501	pwq->stats[PWQ_STAT_RESCUED]++;
3502	}
3503
3504	if (!list_empty(&rescuer->scheduled)) {
3505	process_scheduled_works(rescuer);
3506
3507	/*
3508	* The above execution of rescued work items could
3509	* have created more to rescue through
3510	* pwq_activate_first_inactive() or chained
3511	* queueing. Let's put @pwq back on mayday list so
3512	* that such back-to-back work items, which may be
3513	* being used to relieve memory pressure, don't
3514	* incur MAYDAY_INTERVAL delay inbetween.
3515	*/
3516	if (pwq->nr_active && need_to_create_worker(pool)) {
3517	raw_spin_lock(&wq_mayday_lock);
3518	/*
3519	* Queue iff we aren't racing destruction
3520	* and somebody else hasn't queued it already.
3521	*/
3522	if (wq->rescuer && list_empty(&pwq->mayday_node)) {
3523	get_pwq(pwq);
3524	list_add_tail(&pwq->mayday_node, &wq->maydays);
3525	}
3526	raw_spin_unlock(&wq_mayday_lock);
3527	}
3528	}
3529
3530	/*
3531	* Put the reference grabbed by send_mayday(). @pool won't
3532	* go away while we're still attached to it.
3533	*/
3534	put_pwq(pwq);
3535
3536	/*
3537	* Leave this pool. Notify regular workers; otherwise, we end up
3538	* with 0 concurrency and stalling the execution.
3539	*/
3540	kick_pool(pool);
3541
3542	raw_spin_unlock_irq(&pool->lock);
3543
3544	worker_detach_from_pool(rescuer);
3545
3546	raw_spin_lock_irq(&wq_mayday_lock);
3547	}
3548
3549	raw_spin_unlock_irq(&wq_mayday_lock);
3550
3551	if (should_stop) {
3552	__set_current_state(TASK_RUNNING);
3553	set_pf_worker(false);
3554	return 0;
3555	}
3556
3557	/* rescuers should never participate in concurrency management */
3558	WARN_ON_ONCE(!(rescuer->flags & WORKER_NOT_RUNNING));
3559	schedule();
3560	goto repeat;
3561	}
3562
3563	static void bh_worker(struct worker *worker)
3564	{
3565	struct worker_pool *pool = worker->pool;
3566	int nr_restarts = BH_WORKER_RESTARTS;
3567	unsigned long end = jiffies + BH_WORKER_JIFFIES;
3568
3569	raw_spin_lock_irq(&pool->lock);
3570	worker_leave_idle(worker);
3571
3572	/*
3573	* This function follows the structure of worker_thread(). See there for
3574	* explanations on each step.
3575	*/
3576	if (!need_more_worker(pool))
3577	goto done;
3578
3579	WARN_ON_ONCE(!list_empty(&worker->scheduled));
3580	worker_clr_flags(worker, WORKER_PREP \| WORKER_REBOUND);
3581
3582	do {
3583	struct work_struct *work =
3584	list_first_entry(&pool->worklist,
3585	struct work_struct, entry);
3586
3587	if (assign_work(work, worker, NULL))
3588	process_scheduled_works(worker);
3589	} while (keep_working(pool) &&
3590	--nr_restarts && time_before(jiffies, end));
3591
3592	worker_set_flags(worker, WORKER_PREP);
3593	done:
3594	worker_enter_idle(worker);
3595	kick_pool(pool);
3596	raw_spin_unlock_irq(&pool->lock);
3597	}
3598
3599	/*
3600	* TODO: Convert all tasklet users to workqueue and use softirq directly.
3601	*
3602	* This is currently called from tasklet[_hi]action() and thus is also called
3603	* whenever there are tasklets to run. Let's do an early exit if there's nothing
3604	* queued. Once conversion from tasklet is complete, the need_more_worker() test
3605	* can be dropped.
3606	*
3607	* After full conversion, we'll add worker->softirq_action, directly use the
3608	* softirq action and obtain the worker pointer from the softirq_action pointer.
3609	*/
3610	void workqueue_softirq_action(bool highpri)
3611	{
3612	struct worker_pool *pool =
3613	&per_cpu(bh_worker_pools, smp_processor_id())[highpri];
3614	if (need_more_worker(pool))
3615	bh_worker(list_first_entry(&pool->workers, struct worker, node));
3616	}
3617
3618	struct wq_drain_dead_softirq_work {
3619	struct work_struct work;
3620	struct worker_pool *pool;
3621	struct completion done;
3622	};
3623
3624	static void drain_dead_softirq_workfn(struct work_struct *work)
3625	{
3626	struct wq_drain_dead_softirq_work *dead_work =
3627	container_of(work, struct wq_drain_dead_softirq_work, work);
3628	struct worker_pool *pool = dead_work->pool;
3629	bool repeat;
3630
3631	/*
3632	* @pool's CPU is dead and we want to execute its still pending work
3633	* items from this BH work item which is running on a different CPU. As
3634	* its CPU is dead, @pool can't be kicked and, as work execution path
3635	* will be nested, a lockdep annotation needs to be suppressed. Mark
3636	* @pool with %POOL_BH_DRAINING for the special treatments.
3637	*/
3638	raw_spin_lock_irq(&pool->lock);
3639	pool->flags \|= POOL_BH_DRAINING;
3640	raw_spin_unlock_irq(&pool->lock);
3641
3642	bh_worker(list_first_entry(&pool->workers, struct worker, node));
3643
3644	raw_spin_lock_irq(&pool->lock);
3645	pool->flags &= ~POOL_BH_DRAINING;
3646	repeat = need_more_worker(pool);
3647	raw_spin_unlock_irq(&pool->lock);
3648
3649	/*
3650	* bh_worker() might hit consecutive execution limit and bail. If there
3651	* still are pending work items, reschedule self and return so that we
3652	* don't hog this CPU's BH.
3653	*/
3654	if (repeat) {
3655	if (pool->attrs->nice == HIGHPRI_NICE_LEVEL)
3656	queue_work(system_bh_highpri_wq, work);
3657	else
3658	queue_work(system_bh_wq, work);
3659	} else {
3660	complete(&dead_work->done);
3661	}
3662	}
3663
3664	/*
3665	* @cpu is dead. Drain the remaining BH work items on the current CPU. It's
3666	* possible to allocate dead_work per CPU and avoid flushing. However, then we
3667	* have to worry about draining overlapping with CPU coming back online or
3668	* nesting (one CPU's dead_work queued on another CPU which is also dead and so
3669	* on). Let's keep it simple and drain them synchronously. These are BH work
3670	* items which shouldn't be requeued on the same pool. Shouldn't take long.
3671	*/
3672	void workqueue_softirq_dead(unsigned int cpu)
3673	{
3674	int i;
3675
3676	for (i = 0; i < NR_STD_WORKER_POOLS; i++) {
3677	struct worker_pool *pool = &per_cpu(bh_worker_pools, cpu)[i];
3678	struct wq_drain_dead_softirq_work dead_work;
3679
3680	if (!need_more_worker(pool))
3681	continue;
3682
3683	INIT_WORK_ONSTACK(&dead_work.work, drain_dead_softirq_workfn);
3684	dead_work.pool = pool;
3685	init_completion(&dead_work.done);
3686
3687	if (pool->attrs->nice == HIGHPRI_NICE_LEVEL)
3688	queue_work(system_bh_highpri_wq, &dead_work.work);
3689	else
3690	queue_work(system_bh_wq, &dead_work.work);
3691
3692	wait_for_completion(&dead_work.done);
3693	destroy_work_on_stack(&dead_work.work);
3694	}
3695	}
3696
3697	/**
3698	* check_flush_dependency - check for flush dependency sanity
3699	* @target_wq: workqueue being flushed
3700	* @target_work: work item being flushed (NULL for workqueue flushes)
3701	*
3702	* %current is trying to flush the whole @target_wq or @target_work on it.
3703	* If @target_wq doesn't have %WQ_MEM_RECLAIM, verify that %current is not
3704	* reclaiming memory or running on a workqueue which doesn't have
3705	* %WQ_MEM_RECLAIM as that can break forward-progress guarantee leading to
3706	* a deadlock.
3707	*/
3708	static void check_flush_dependency(struct workqueue_struct *target_wq,
3709	struct work_struct *target_work)
3710	{
3711	work_func_t target_func = target_work ? target_work->func : NULL;
3712	struct worker *worker;
3713
3714	if (target_wq->flags & WQ_MEM_RECLAIM)
3715	return;
3716
3717	worker = current_wq_worker();
3718
3719	WARN_ONCE(current->flags & PF_MEMALLOC,
3720	"workqueue: PF_MEMALLOC task %d(%s) is flushing !WQ_MEM_RECLAIM %s:%ps",
3721	current->pid, current->comm, target_wq->name, target_func);
3722	WARN_ONCE(worker && ((worker->current_pwq->wq->flags &
3723	(WQ_MEM_RECLAIM \| __WQ_LEGACY)) == WQ_MEM_RECLAIM),
3724	"workqueue: WQ_MEM_RECLAIM %s:%ps is flushing !WQ_MEM_RECLAIM %s:%ps",
3725	worker->current_pwq->wq->name, worker->current_func,
3726	target_wq->name, target_func);
3727	}
3728
3729	struct wq_barrier {
3730	struct work_struct work;
3731	struct completion done;
3732	struct task_struct task; / purely informational */
3733	};
3734
3735	static void wq_barrier_func(struct work_struct *work)
3736	{
3737	struct wq_barrier *barr = container_of(work, struct wq_barrier, work);
3738	complete(&barr->done);
3739	}
3740
3741	/**
3742	* insert_wq_barrier - insert a barrier work
3743	* @pwq: pwq to insert barrier into
3744	* @barr: wq_barrier to insert
3745	* @target: target work to attach @barr to
3746	* @worker: worker currently executing @target, NULL if @target is not executing
3747	*
3748	* @barr is linked to @target such that @barr is completed only after
3749	* @target finishes execution. Please note that the ordering
3750	* guarantee is observed only with respect to @target and on the local
3751	* cpu.
3752	*
3753	* Currently, a queued barrier can't be canceled. This is because
3754	* try_to_grab_pending() can't determine whether the work to be
3755	* grabbed is at the head of the queue and thus can't clear LINKED
3756	* flag of the previous work while there must be a valid next work
3757	* after a work with LINKED flag set.
3758	*
3759	* Note that when @worker is non-NULL, @target may be modified
3760	* underneath us, so we can't reliably determine pwq from @target.
3761	*
3762	* CONTEXT:
3763	* raw_spin_lock_irq(pool->lock).
3764	*/
3765	static void insert_wq_barrier(struct pool_workqueue *pwq,
3766	struct wq_barrier *barr,
3767	struct work_struct target, struct worker worker)
3768	{
3769	static __maybe_unused struct lock_class_key bh_key, thr_key;
3770	unsigned int work_flags = 0;
3771	unsigned int work_color;
3772	struct list_head *head;
3773
3774	/*
3775	* debugobject calls are safe here even with pool->lock locked
3776	* as we know for sure that this will not trigger any of the
3777	* checks and call back into the fixup functions where we
3778	* might deadlock.
3779	*
3780	* BH and threaded workqueues need separate lockdep keys to avoid
3781	* spuriously triggering "inconsistent {SOFTIRQ-ON-W} -> {IN-SOFTIRQ-W}
3782	* usage".
3783	*/
3784	INIT_WORK_ONSTACK_KEY(&barr->work, wq_barrier_func,
3785	(pwq->wq->flags & WQ_BH) ? &bh_key : &thr_key);
3786	__set_bit(WORK_STRUCT_PENDING_BIT, work_data_bits(&barr->work));
3787
3788	init_completion_map(&barr->done, &target->lockdep_map);
3789
3790	barr->task = current;
3791
3792	/* The barrier work item does not participate in nr_active. */
3793	work_flags \|= WORK_STRUCT_INACTIVE;
3794
3795	/*
3796	* If @target is currently being executed, schedule the
3797	* barrier to the worker; otherwise, put it after @target.
3798	*/
3799	if (worker) {
3800	head = worker->scheduled.next;
3801	work_color = worker->current_color;
3802	} else {
3803	unsigned long *bits = work_data_bits(target);
3804
3805	head = target->entry.next;
3806	/* there can already be other linked works, inherit and set */
3807	work_flags \|= *bits & WORK_STRUCT_LINKED;
3808	work_color = get_work_color(*bits);
3809	__set_bit(WORK_STRUCT_LINKED_BIT, bits);
3810	}
3811
3812	pwq->nr_in_flight[work_color]++;
3813	work_flags \|= work_color_to_flags(work_color);
3814
3815	insert_work(pwq, &barr->work, head, work_flags);
3816	}
3817
3818	/**
3819	* flush_workqueue_prep_pwqs - prepare pwqs for workqueue flushing
3820	* @wq: workqueue being flushed
3821	* @flush_color: new flush color, < 0 for no-op
3822	* @work_color: new work color, < 0 for no-op
3823	*
3824	* Prepare pwqs for workqueue flushing.
3825	*
3826	* If @flush_color is non-negative, flush_color on all pwqs should be
3827	* -1. If no pwq has in-flight commands at the specified color, all
3828	* pwq->flush_color's stay at -1 and %false is returned. If any pwq
3829	* has in flight commands, its pwq->flush_color is set to
3830	* @flush_color, @wq->nr_pwqs_to_flush is updated accordingly, pwq
3831	* wakeup logic is armed and %true is returned.
3832	*
3833	* The caller should have initialized @wq->first_flusher prior to
3834	* calling this function with non-negative @flush_color. If
3835	* @flush_color is negative, no flush color update is done and %false
3836	* is returned.
3837	*
3838	* If @work_color is non-negative, all pwqs should have the same
3839	* work_color which is previous to @work_color and all will be
3840	* advanced to @work_color.
3841	*
3842	* CONTEXT:
3843	* mutex_lock(wq->mutex).
3844	*
3845	* Return:
3846	* %true if @flush_color >= 0 and there's something to flush. %false
3847	* otherwise.
3848	*/
3849	static bool flush_workqueue_prep_pwqs(struct workqueue_struct *wq,
3850	int flush_color, int work_color)
3851	{
3852	bool wait = false;
3853	struct pool_workqueue *pwq;
3854
3855	if (flush_color >= 0) {
3856	WARN_ON_ONCE(atomic_read(&wq->nr_pwqs_to_flush));
3857	atomic_set(&wq->nr_pwqs_to_flush, 1);
3858	}
3859
3860	for_each_pwq(pwq, wq) {
3861	struct worker_pool *pool = pwq->pool;
3862
3863	raw_spin_lock_irq(&pool->lock);
3864
3865	if (flush_color >= 0) {
3866	WARN_ON_ONCE(pwq->flush_color != -1);
3867
3868	if (pwq->nr_in_flight[flush_color]) {
3869	pwq->flush_color = flush_color;
3870	atomic_inc(&wq->nr_pwqs_to_flush);
3871	wait = true;
3872	}
3873	}
3874
3875	if (work_color >= 0) {
3876	WARN_ON_ONCE(work_color != work_next_color(pwq->work_color));
3877	pwq->work_color = work_color;
3878	}
3879
3880	raw_spin_unlock_irq(&pool->lock);
3881	}
3882
3883	if (flush_color >= 0 && atomic_dec_and_test(&wq->nr_pwqs_to_flush))
3884	complete(&wq->first_flusher->done);
3885
3886	return wait;
3887	}
3888
3889	static void touch_wq_lockdep_map(struct workqueue_struct *wq)
3890	{
3891	#ifdef CONFIG_LOCKDEP
3892	if (wq->flags & WQ_BH)
3893	local_bh_disable();
3894
3895	lock_map_acquire(&wq->lockdep_map);
3896	lock_map_release(&wq->lockdep_map);
3897
3898	if (wq->flags & WQ_BH)
3899	local_bh_enable();
3900	#endif
3901	}
3902
3903	static void touch_work_lockdep_map(struct work_struct *work,
3904	struct workqueue_struct *wq)
3905	{
3906	#ifdef CONFIG_LOCKDEP
3907	if (wq->flags & WQ_BH)
3908	local_bh_disable();
3909
3910	lock_map_acquire(&work->lockdep_map);
3911	lock_map_release(&work->lockdep_map);
3912
3913	if (wq->flags & WQ_BH)
3914	local_bh_enable();
3915	#endif
3916	}
3917
3918	/**
3919	* __flush_workqueue - ensure that any scheduled work has run to completion.
3920	* @wq: workqueue to flush
3921	*
3922	* This function sleeps until all work items which were queued on entry
3923	* have finished execution, but it is not livelocked by new incoming ones.
3924	*/
3925	void __flush_workqueue(struct workqueue_struct *wq)
3926	{
3927	struct wq_flusher this_flusher = {
3928	.list = LIST_HEAD_INIT(this_flusher.list),
3929	.flush_color = -1,
3930	.done = COMPLETION_INITIALIZER_ONSTACK_MAP(this_flusher.done, wq->lockdep_map),
3931	};
3932	int next_color;
3933
3934	if (WARN_ON(!wq_online))
3935	return;
3936
3937	touch_wq_lockdep_map(wq);
3938
3939	mutex_lock(&wq->mutex);
3940
3941	/*
3942	* Start-to-wait phase
3943	*/
3944	next_color = work_next_color(wq->work_color);
3945
3946	if (next_color != wq->flush_color) {
3947	/*
3948	* Color space is not full. The current work_color
3949	* becomes our flush_color and work_color is advanced
3950	* by one.
3951	*/
3952	WARN_ON_ONCE(!list_empty(&wq->flusher_overflow));
3953	this_flusher.flush_color = wq->work_color;
3954	wq->work_color = next_color;
3955
3956	if (!wq->first_flusher) {
3957	/* no flush in progress, become the first flusher */
3958	WARN_ON_ONCE(wq->flush_color != this_flusher.flush_color);
3959
3960	wq->first_flusher = &this_flusher;
3961
3962	if (!flush_workqueue_prep_pwqs(wq, wq->flush_color,
3963	wq->work_color)) {
3964	/* nothing to flush, done */
3965	wq->flush_color = next_color;
3966	wq->first_flusher = NULL;
3967	goto out_unlock;
3968	}
3969	} else {
3970	/* wait in queue */
3971	WARN_ON_ONCE(wq->flush_color == this_flusher.flush_color);
3972	list_add_tail(&this_flusher.list, &wq->flusher_queue);
3973	flush_workqueue_prep_pwqs(wq, -1, wq->work_color);
3974	}
3975	} else {
3976	/*
3977	* Oops, color space is full, wait on overflow queue.
3978	* The next flush completion will assign us
3979	* flush_color and transfer to flusher_queue.
3980	*/
3981	list_add_tail(&this_flusher.list, &wq->flusher_overflow);
3982	}
3983
3984	check_flush_dependency(wq, NULL);
3985
3986	mutex_unlock(&wq->mutex);
3987
3988	wait_for_completion(&this_flusher.done);
3989
3990	/*
3991	* Wake-up-and-cascade phase
3992	*
3993	* First flushers are responsible for cascading flushes and
3994	* handling overflow. Non-first flushers can simply return.
3995	*/
3996	if (READ_ONCE(wq->first_flusher) != &this_flusher)
3997	return;
3998
3999	mutex_lock(&wq->mutex);
4000
4001	/* we might have raced, check again with mutex held */
4002	if (wq->first_flusher != &this_flusher)
4003	goto out_unlock;
4004
4005	WRITE_ONCE(wq->first_flusher, NULL);
4006
4007	WARN_ON_ONCE(!list_empty(&this_flusher.list));
4008	WARN_ON_ONCE(wq->flush_color != this_flusher.flush_color);
4009
4010	while (true) {
4011	struct wq_flusher next, tmp;
4012
4013	/* complete all the flushers sharing the current flush color */
4014	list_for_each_entry_safe(next, tmp, &wq->flusher_queue, list) {
4015	if (next->flush_color != wq->flush_color)
4016	break;
4017	list_del_init(&next->list);
4018	complete(&next->done);
4019	}
4020
4021	WARN_ON_ONCE(!list_empty(&wq->flusher_overflow) &&
4022	wq->flush_color != work_next_color(wq->work_color));
4023
4024	/* this flush_color is finished, advance by one */
4025	wq->flush_color = work_next_color(wq->flush_color);
4026
4027	/* one color has been freed, handle overflow queue */
4028	if (!list_empty(&wq->flusher_overflow)) {
4029	/*
4030	* Assign the same color to all overflowed
4031	* flushers, advance work_color and append to
4032	* flusher_queue. This is the start-to-wait
4033	* phase for these overflowed flushers.
4034	*/
4035	list_for_each_entry(tmp, &wq->flusher_overflow, list)
4036	tmp->flush_color = wq->work_color;
4037
4038	wq->work_color = work_next_color(wq->work_color);
4039
4040	list_splice_tail_init(&wq->flusher_overflow,
4041	&wq->flusher_queue);
4042	flush_workqueue_prep_pwqs(wq, -1, wq->work_color);
4043	}
4044
4045	if (list_empty(&wq->flusher_queue)) {
4046	WARN_ON_ONCE(wq->flush_color != wq->work_color);
4047	break;
4048	}
4049
4050	/*
4051	* Need to flush more colors. Make the next flusher
4052	* the new first flusher and arm pwqs.
4053	*/
4054	WARN_ON_ONCE(wq->flush_color == wq->work_color);
4055	WARN_ON_ONCE(wq->flush_color != next->flush_color);
4056
4057	list_del_init(&next->list);
4058	wq->first_flusher = next;
4059
4060	if (flush_workqueue_prep_pwqs(wq, wq->flush_color, -1))
4061	break;
4062
4063	/*
4064	* Meh... this color is already done, clear first
4065	* flusher and repeat cascading.
4066	*/
4067	wq->first_flusher = NULL;
4068	}
4069
4070	out_unlock:
4071	mutex_unlock(&wq->mutex);
4072	}
4073	EXPORT_SYMBOL(__flush_workqueue);
4074
4075	/**
4076	* drain_workqueue - drain a workqueue
4077	* @wq: workqueue to drain
4078	*
4079	* Wait until the workqueue becomes empty. While draining is in progress,
4080	* only chain queueing is allowed. IOW, only currently pending or running
4081	* work items on @wq can queue further work items on it. @wq is flushed
4082	* repeatedly until it becomes empty. The number of flushing is determined
4083	* by the depth of chaining and should be relatively short. Whine if it
4084	* takes too long.
4085	*/
4086	void drain_workqueue(struct workqueue_struct *wq)
4087	{
4088	unsigned int flush_cnt = 0;
4089	struct pool_workqueue *pwq;
4090
4091	/*
4092	* __queue_work() needs to test whether there are drainers, is much
4093	* hotter than drain_workqueue() and already looks at @wq->flags.
4094	* Use __WQ_DRAINING so that queue doesn't have to check nr_drainers.
4095	*/
4096	mutex_lock(&wq->mutex);
4097	if (!wq->nr_drainers++)
4098	wq->flags \|= __WQ_DRAINING;
4099	mutex_unlock(&wq->mutex);
4100	reflush:
4101	__flush_workqueue(wq);
4102
4103	mutex_lock(&wq->mutex);
4104
4105	for_each_pwq(pwq, wq) {
4106	bool drained;
4107
4108	raw_spin_lock_irq(&pwq->pool->lock);
4109	drained = pwq_is_empty(pwq);
4110	raw_spin_unlock_irq(&pwq->pool->lock);
4111
4112	if (drained)
4113	continue;
4114
4115	if (++flush_cnt == 10 \|\|
4116	(flush_cnt % 100 == 0 && flush_cnt <= 1000))
4117	pr_warn("workqueue %s: %s() isn't complete after %u tries\n",
4118	wq->name, __func__, flush_cnt);
4119
4120	mutex_unlock(&wq->mutex);
4121	goto reflush;
4122	}
4123
4124	if (!--wq->nr_drainers)
4125	wq->flags &= ~__WQ_DRAINING;
4126	mutex_unlock(&wq->mutex);
4127	}
4128	EXPORT_SYMBOL_GPL(drain_workqueue);
4129
4130	static bool start_flush_work(struct work_struct work, struct wq_barrier barr,
4131	bool from_cancel)
4132	{
4133	struct worker *worker = NULL;
4134	struct worker_pool *pool;
4135	struct pool_workqueue *pwq;
4136	struct workqueue_struct *wq;
4137
4138	rcu_read_lock();
4139	pool = get_work_pool(work);
4140	if (!pool) {
4141	rcu_read_unlock();
4142	return false;
4143	}
4144
4145	raw_spin_lock_irq(&pool->lock);
4146	/* see the comment in try_to_grab_pending() with the same code */
4147	pwq = get_work_pwq(work);
4148	if (pwq) {
4149	if (unlikely(pwq->pool != pool))
4150	goto already_gone;
4151	} else {
4152	worker = find_worker_executing_work(pool, work);
4153	if (!worker)
4154	goto already_gone;
4155	pwq = worker->current_pwq;
4156	}
4157
4158	wq = pwq->wq;
4159	check_flush_dependency(wq, work);
4160
4161	insert_wq_barrier(pwq, barr, work, worker);
4162	raw_spin_unlock_irq(&pool->lock);
4163
4164	touch_work_lockdep_map(work, wq);
4165
4166	/*
4167	* Force a lock recursion deadlock when using flush_work() inside a
4168	* single-threaded or rescuer equipped workqueue.
4169	*
4170	* For single threaded workqueues the deadlock happens when the work
4171	* is after the work issuing the flush_work(). For rescuer equipped
4172	* workqueues the deadlock happens when the rescuer stalls, blocking
4173	* forward progress.
4174	*/
4175	if (!from_cancel && (wq->saved_max_active == 1 \|\| wq->rescuer))
4176	touch_wq_lockdep_map(wq);
4177
4178	rcu_read_unlock();
4179	return true;
4180	already_gone:
4181	raw_spin_unlock_irq(&pool->lock);
4182	rcu_read_unlock();
4183	return false;
4184	}
4185
4186	static bool __flush_work(struct work_struct *work, bool from_cancel)
4187	{
4188	struct wq_barrier barr;
4189	unsigned long data;
4190
4191	if (WARN_ON(!wq_online))
4192	return false;
4193
4194	if (WARN_ON(!work->func))
4195	return false;
4196
4197	if (!start_flush_work(work, &barr, from_cancel))
4198	return false;
4199
4200	/*
4201	* start_flush_work() returned %true. If @from_cancel is set, we know
4202	* that @work must have been executing during start_flush_work() and
4203	* can't currently be queued. Its data must contain OFFQ bits. If @work
4204	* was queued on a BH workqueue, we also know that it was running in the
4205	* BH context and thus can be busy-waited.
4206	*/
4207	data = *work_data_bits(work);
4208	if (from_cancel &&
4209	!WARN_ON_ONCE(data & WORK_STRUCT_PWQ) && (data & WORK_OFFQ_BH)) {
4210	/*
4211	* On RT, prevent a live lock when %current preempted soft
4212	* interrupt processing or prevents ksoftirqd from running by
4213	* keeping flipping BH. If the BH work item runs on a different
4214	* CPU then this has no effect other than doing the BH
4215	* disable/enable dance for nothing. This is copied from
4216	* kernel/softirq.c::tasklet_unlock_spin_wait().
4217	*/
4218	while (!try_wait_for_completion(&barr.done)) {
4219	if (IS_ENABLED(CONFIG_PREEMPT_RT)) {
4220	local_bh_disable();
4221	local_bh_enable();
4222	} else {
4223	cpu_relax();
4224	}
4225	}
4226	} else {
4227	wait_for_completion(&barr.done);
4228	}
4229
4230	destroy_work_on_stack(&barr.work);
4231	return true;
4232	}
4233
4234	/**
4235	* flush_work - wait for a work to finish executing the last queueing instance
4236	* @work: the work to flush
4237	*
4238	* Wait until @work has finished execution. @work is guaranteed to be idle
4239	* on return if it hasn't been requeued since flush started.
4240	*
4241	* Return:
4242	* %true if flush_work() waited for the work to finish execution,
4243	* %false if it was already idle.
4244	*/
4245	bool flush_work(struct work_struct *work)
4246	{
4247	might_sleep();
4248	return __flush_work(work, false);
4249	}
4250	EXPORT_SYMBOL_GPL(flush_work);
4251
4252	/**
4253	* flush_delayed_work - wait for a dwork to finish executing the last queueing
4254	* @dwork: the delayed work to flush
4255	*
4256	* Delayed timer is cancelled and the pending work is queued for
4257	* immediate execution. Like flush_work(), this function only
4258	* considers the last queueing instance of @dwork.
4259	*
4260	* Return:
4261	* %true if flush_work() waited for the work to finish execution,
4262	* %false if it was already idle.
4263	*/
4264	bool flush_delayed_work(struct delayed_work *dwork)
4265	{
4266	local_irq_disable();
4267	if (del_timer_sync(&dwork->timer))
4268	__queue_work(dwork->cpu, dwork->wq, &dwork->work);
4269	local_irq_enable();
4270	return flush_work(&dwork->work);
4271	}
4272	EXPORT_SYMBOL(flush_delayed_work);
4273
4274	/**
4275	* flush_rcu_work - wait for a rwork to finish executing the last queueing
4276	* @rwork: the rcu work to flush
4277	*
4278	* Return:
4279	* %true if flush_rcu_work() waited for the work to finish execution,
4280	* %false if it was already idle.
4281	*/
4282	bool flush_rcu_work(struct rcu_work *rwork)
4283	{
4284	if (test_bit(WORK_STRUCT_PENDING_BIT, work_data_bits(&rwork->work))) {
4285	rcu_barrier();
4286	flush_work(&rwork->work);
4287	return true;
4288	} else {
4289	return flush_work(&rwork->work);
4290	}
4291	}
4292	EXPORT_SYMBOL(flush_rcu_work);
4293
4294	static void work_offqd_disable(struct work_offq_data *offqd)
4295	{
4296	const unsigned long max = (1lu << WORK_OFFQ_DISABLE_BITS) - 1;
4297
4298	if (likely(offqd->disable < max))
4299	offqd->disable++;
4300	else
4301	WARN_ONCE(true, "workqueue: work disable count overflowed\n");
4302	}
4303
4304	static void work_offqd_enable(struct work_offq_data *offqd)
4305	{
4306	if (likely(offqd->disable > 0))
4307	offqd->disable--;
4308	else
4309	WARN_ONCE(true, "workqueue: work disable count underflowed\n");
4310	}
4311
4312	static bool __cancel_work(struct work_struct *work, u32 cflags)
4313	{
4314	struct work_offq_data offqd;
4315	unsigned long irq_flags;
4316	int ret;
4317
4318	ret = work_grab_pending(work, cflags, &irq_flags);
4319
4320	work_offqd_unpack(&offqd, *work_data_bits(work));
4321
4322	if (cflags & WORK_CANCEL_DISABLE)
4323	work_offqd_disable(&offqd);
4324
4325	set_work_pool_and_clear_pending(work, offqd.pool_id,
4326	work_offqd_pack_flags(&offqd));
4327	local_irq_restore(irq_flags);
4328	return ret;
4329	}
4330
4331	static bool __cancel_work_sync(struct work_struct *work, u32 cflags)
4332	{
4333	bool ret;
4334
4335	ret = __cancel_work(work, cflags \| WORK_CANCEL_DISABLE);
4336
4337	if (*work_data_bits(work) & WORK_OFFQ_BH)
4338	WARN_ON_ONCE(in_hardirq());
4339	else
4340	might_sleep();
4341
4342	/*
4343	* Skip __flush_work() during early boot when we know that @work isn't
4344	* executing. This allows canceling during early boot.
4345	*/
4346	if (wq_online)
4347	__flush_work(work, true);
4348
4349	if (!(cflags & WORK_CANCEL_DISABLE))
4350	enable_work(work);
4351
4352	return ret;
4353	}
4354
4355	/*
4356	* See cancel_delayed_work()
4357	*/
4358	bool cancel_work(struct work_struct *work)
4359	{
4360	return __cancel_work(work, 0);
4361	}
4362	EXPORT_SYMBOL(cancel_work);
4363
4364	/**
4365	* cancel_work_sync - cancel a work and wait for it to finish
4366	* @work: the work to cancel
4367	*
4368	* Cancel @work and wait for its execution to finish. This function can be used
4369	* even if the work re-queues itself or migrates to another workqueue. On return
4370	* from this function, @work is guaranteed to be not pending or executing on any
4371	* CPU as long as there aren't racing enqueues.
4372	*
4373	* cancel_work_sync(&delayed_work->work) must not be used for delayed_work's.
4374	* Use cancel_delayed_work_sync() instead.
4375	*
4376	* Must be called from a sleepable context if @work was last queued on a non-BH
4377	* workqueue. Can also be called from non-hardirq atomic contexts including BH
4378	* if @work was last queued on a BH workqueue.
4379	*
4380	* Returns %true if @work was pending, %false otherwise.
4381	*/
4382	bool cancel_work_sync(struct work_struct *work)
4383	{
4384	return __cancel_work_sync(work, 0);
4385	}
4386	EXPORT_SYMBOL_GPL(cancel_work_sync);
4387
4388	/**
4389	* cancel_delayed_work - cancel a delayed work
4390	* @dwork: delayed_work to cancel
4391	*
4392	* Kill off a pending delayed_work.
4393	*
4394	* Return: %true if @dwork was pending and canceled; %false if it wasn't
4395	* pending.
4396	*
4397	* Note:
4398	* The work callback function may still be running on return, unless
4399	* it returns %true and the work doesn't re-arm itself. Explicitly flush or
4400	* use cancel_delayed_work_sync() to wait on it.
4401	*
4402	* This function is safe to call from any context including IRQ handler.
4403	*/
4404	bool cancel_delayed_work(struct delayed_work *dwork)
4405	{
4406	return __cancel_work(&dwork->work, WORK_CANCEL_DELAYED);
4407	}
4408	EXPORT_SYMBOL(cancel_delayed_work);
4409
4410	/**
4411	* cancel_delayed_work_sync - cancel a delayed work and wait for it to finish
4412	* @dwork: the delayed work cancel
4413	*
4414	* This is cancel_work_sync() for delayed works.
4415	*
4416	* Return:
4417	* %true if @dwork was pending, %false otherwise.
4418	*/
4419	bool cancel_delayed_work_sync(struct delayed_work *dwork)
4420	{
4421	return __cancel_work_sync(&dwork->work, WORK_CANCEL_DELAYED);
4422	}
4423	EXPORT_SYMBOL(cancel_delayed_work_sync);
4424
4425	/**
4426	* disable_work - Disable and cancel a work item
4427	* @work: work item to disable
4428	*
4429	* Disable @work by incrementing its disable count and cancel it if currently
4430	* pending. As long as the disable count is non-zero, any attempt to queue @work
4431	* will fail and return %false. The maximum supported disable depth is 2 to the
4432	* power of %WORK_OFFQ_DISABLE_BITS, currently 65536.
4433	*
4434	* Can be called from any context. Returns %true if @work was pending, %false
4435	* otherwise.
4436	*/
4437	bool disable_work(struct work_struct *work)
4438	{
4439	return __cancel_work(work, WORK_CANCEL_DISABLE);
4440	}
4441	EXPORT_SYMBOL_GPL(disable_work);
4442
4443	/**
4444	* disable_work_sync - Disable, cancel and drain a work item
4445	* @work: work item to disable
4446	*
4447	* Similar to disable_work() but also wait for @work to finish if currently
4448	* executing.
4449	*
4450	* Must be called from a sleepable context if @work was last queued on a non-BH
4451	* workqueue. Can also be called from non-hardirq atomic contexts including BH
4452	* if @work was last queued on a BH workqueue.
4453	*
4454	* Returns %true if @work was pending, %false otherwise.
4455	*/
4456	bool disable_work_sync(struct work_struct *work)
4457	{
4458	return __cancel_work_sync(work, WORK_CANCEL_DISABLE);
4459	}
4460	EXPORT_SYMBOL_GPL(disable_work_sync);
4461
4462	/**
4463	* enable_work - Enable a work item
4464	* @work: work item to enable
4465	*
4466	* Undo disable_work[_sync]() by decrementing @work's disable count. @work can
4467	* only be queued if its disable count is 0.
4468	*
4469	* Can be called from any context. Returns %true if the disable count reached 0.
4470	* Otherwise, %false.
4471	*/
4472	bool enable_work(struct work_struct *work)
4473	{
4474	struct work_offq_data offqd;
4475	unsigned long irq_flags;
4476
4477	work_grab_pending(work, 0, &irq_flags);
4478
4479	work_offqd_unpack(&offqd, *work_data_bits(work));
4480	work_offqd_enable(&offqd);
4481	set_work_pool_and_clear_pending(work, offqd.pool_id,
4482	work_offqd_pack_flags(&offqd));
4483	local_irq_restore(irq_flags);
4484
4485	return !offqd.disable;
4486	}
4487	EXPORT_SYMBOL_GPL(enable_work);
4488
4489	/**
4490	* disable_delayed_work - Disable and cancel a delayed work item
4491	* @dwork: delayed work item to disable
4492	*
4493	* disable_work() for delayed work items.
4494	*/
4495	bool disable_delayed_work(struct delayed_work *dwork)
4496	{
4497	return __cancel_work(&dwork->work,
4498	WORK_CANCEL_DELAYED \| WORK_CANCEL_DISABLE);
4499	}
4500	EXPORT_SYMBOL_GPL(disable_delayed_work);
4501
4502	/**
4503	* disable_delayed_work_sync - Disable, cancel and drain a delayed work item
4504	* @dwork: delayed work item to disable
4505	*
4506	* disable_work_sync() for delayed work items.
4507	*/
4508	bool disable_delayed_work_sync(struct delayed_work *dwork)
4509	{
4510	return __cancel_work_sync(&dwork->work,
4511	WORK_CANCEL_DELAYED \| WORK_CANCEL_DISABLE);
4512	}
4513	EXPORT_SYMBOL_GPL(disable_delayed_work_sync);
4514
4515	/**
4516	* enable_delayed_work - Enable a delayed work item
4517	* @dwork: delayed work item to enable
4518	*
4519	* enable_work() for delayed work items.
4520	*/
4521	bool enable_delayed_work(struct delayed_work *dwork)
4522	{
4523	return enable_work(&dwork->work);
4524	}
4525	EXPORT_SYMBOL_GPL(enable_delayed_work);
4526
4527	/**
4528	* schedule_on_each_cpu - execute a function synchronously on each online CPU
4529	* @func: the function to call
4530	*
4531	* schedule_on_each_cpu() executes @func on each online CPU using the
4532	* system workqueue and blocks until all CPUs have completed.
4533	* schedule_on_each_cpu() is very slow.
4534	*
4535	* Return:
4536	* 0 on success, -errno on failure.
4537	*/
4538	int schedule_on_each_cpu(work_func_t func)
4539	{
4540	int cpu;
4541	struct work_struct __percpu *works;
4542
4543	works = alloc_percpu(struct work_struct);
4544	if (!works)
4545	return -ENOMEM;
4546
4547	cpus_read_lock();
4548
4549	for_each_online_cpu(cpu) {
4550	struct work_struct *work = per_cpu_ptr(works, cpu);
4551
4552	INIT_WORK(work, func);
4553	schedule_work_on(cpu, work);
4554	}
4555
4556	for_each_online_cpu(cpu)
4557	flush_work(per_cpu_ptr(works, cpu));
4558
4559	cpus_read_unlock();
4560	free_percpu(works);
4561	return 0;
4562	}
4563
4564	/**
4565	* execute_in_process_context - reliably execute the routine with user context
4566	* @fn: the function to execute
4567	* @ew: guaranteed storage for the execute work structure (must
4568	* be available when the work executes)
4569	*
4570	* Executes the function immediately if process context is available,
4571	* otherwise schedules the function for delayed execution.
4572	*
4573	* Return: 0 - function was executed
4574	* 1 - function was scheduled for execution
4575	*/
4576	int execute_in_process_context(work_func_t fn, struct execute_work *ew)
4577	{
4578	if (!in_interrupt()) {
4579	fn(&ew->work);
4580	return 0;
4581	}
4582
4583	INIT_WORK(&ew->work, fn);
4584	schedule_work(&ew->work);
4585
4586	return 1;
4587	}
4588	EXPORT_SYMBOL_GPL(execute_in_process_context);
4589
4590	/**
4591	* free_workqueue_attrs - free a workqueue_attrs
4592	* @attrs: workqueue_attrs to free
4593	*
4594	* Undo alloc_workqueue_attrs().
4595	*/
4596	void free_workqueue_attrs(struct workqueue_attrs *attrs)
4597	{
4598	if (attrs) {
4599	free_cpumask_var(attrs->cpumask);
4600	free_cpumask_var(attrs->__pod_cpumask);
4601	kfree(attrs);
4602	}
4603	}
4604
4605	/**
4606	* alloc_workqueue_attrs - allocate a workqueue_attrs
4607	*
4608	* Allocate a new workqueue_attrs, initialize with default settings and
4609	* return it.
4610	*
4611	* Return: The allocated new workqueue_attr on success. %NULL on failure.
4612	*/
4613	struct workqueue_attrs *alloc_workqueue_attrs(void)
4614	{
4615	struct workqueue_attrs *attrs;
4616
4617	attrs = kzalloc(sizeof(*attrs), GFP_KERNEL);
4618	if (!attrs)
4619	goto fail;
4620	if (!alloc_cpumask_var(&attrs->cpumask, GFP_KERNEL))
4621	goto fail;
4622	if (!alloc_cpumask_var(&attrs->__pod_cpumask, GFP_KERNEL))
4623	goto fail;
4624
4625	cpumask_copy(attrs->cpumask, cpu_possible_mask);
4626	attrs->affn_scope = WQ_AFFN_DFL;
4627	return attrs;
4628	fail:
4629	free_workqueue_attrs(attrs);
4630	return NULL;
4631	}
4632
4633	static void copy_workqueue_attrs(struct workqueue_attrs *to,
4634	const struct workqueue_attrs *from)
4635	{
4636	to->nice = from->nice;
4637	cpumask_copy(to->cpumask, from->cpumask);
4638	cpumask_copy(to->__pod_cpumask, from->__pod_cpumask);
4639	to->affn_strict = from->affn_strict;
4640
4641	/*
4642	* Unlike hash and equality test, copying shouldn't ignore wq-only
4643	* fields as copying is used for both pool and wq attrs. Instead,
4644	* get_unbound_pool() explicitly clears the fields.
4645	*/
4646	to->affn_scope = from->affn_scope;
4647	to->ordered = from->ordered;
4648	}
4649
4650	/*
4651	* Some attrs fields are workqueue-only. Clear them for worker_pool's. See the
4652	* comments in 'struct workqueue_attrs' definition.
4653	*/
4654	static void wqattrs_clear_for_pool(struct workqueue_attrs *attrs)
4655	{
4656	attrs->affn_scope = WQ_AFFN_NR_TYPES;
4657	attrs->ordered = false;
4658	if (attrs->affn_strict)
4659	cpumask_copy(attrs->cpumask, cpu_possible_mask);
4660	}
4661
4662	/* hash value of the content of @attr */
4663	static u32 wqattrs_hash(const struct workqueue_attrs *attrs)
4664	{
4665	u32 hash = 0;
4666
4667	hash = jhash_1word(attrs->nice, hash);
4668	hash = jhash_1word(attrs->affn_strict, hash);
4669	hash = jhash(cpumask_bits(attrs->__pod_cpumask),
4670	BITS_TO_LONGS(nr_cpumask_bits) * sizeof(long), hash);
4671	if (!attrs->affn_strict)
4672	hash = jhash(cpumask_bits(attrs->cpumask),
4673	BITS_TO_LONGS(nr_cpumask_bits) * sizeof(long), hash);
4674	return hash;
4675	}
4676
4677	/* content equality test */
4678	static bool wqattrs_equal(const struct workqueue_attrs *a,
4679	const struct workqueue_attrs *b)
4680	{
4681	if (a->nice != b->nice)
4682	return false;
4683	if (a->affn_strict != b->affn_strict)
4684	return false;
4685	if (!cpumask_equal(a->__pod_cpumask, b->__pod_cpumask))
4686	return false;
4687	if (!a->affn_strict && !cpumask_equal(a->cpumask, b->cpumask))
4688	return false;
4689	return true;
4690	}
4691
4692	/* Update @attrs with actually available CPUs */
4693	static void wqattrs_actualize_cpumask(struct workqueue_attrs *attrs,
4694	const cpumask_t *unbound_cpumask)
4695	{
4696	/*
4697	* Calculate the effective CPU mask of @attrs given @unbound_cpumask. If
4698	* @attrs->cpumask doesn't overlap with @unbound_cpumask, we fallback to
4699	* @unbound_cpumask.
4700	*/
4701	cpumask_and(attrs->cpumask, attrs->cpumask, unbound_cpumask);
4702	if (unlikely(cpumask_empty(attrs->cpumask)))
4703	cpumask_copy(attrs->cpumask, unbound_cpumask);
4704	}
4705
4706	/* find wq_pod_type to use for @attrs */
4707	static const struct wq_pod_type *
4708	wqattrs_pod_type(const struct workqueue_attrs *attrs)
4709	{
4710	enum wq_affn_scope scope;
4711	struct wq_pod_type *pt;
4712
4713	/* to synchronize access to wq_affn_dfl */
4714	lockdep_assert_held(&wq_pool_mutex);
4715
4716	if (attrs->affn_scope == WQ_AFFN_DFL)
4717	scope = wq_affn_dfl;
4718	else
4719	scope = attrs->affn_scope;
4720
4721	pt = &wq_pod_types[scope];
4722
4723	if (!WARN_ON_ONCE(attrs->affn_scope == WQ_AFFN_NR_TYPES) &&
4724	likely(pt->nr_pods))
4725	return pt;
4726
4727	/*
4728	* Before workqueue_init_topology(), only SYSTEM is available which is
4729	* initialized in workqueue_init_early().
4730	*/
4731	pt = &wq_pod_types[WQ_AFFN_SYSTEM];
4732	BUG_ON(!pt->nr_pods);
4733	return pt;
4734	}
4735
4736	/**
4737	* init_worker_pool - initialize a newly zalloc'd worker_pool
4738	* @pool: worker_pool to initialize
4739	*
4740	* Initialize a newly zalloc'd @pool. It also allocates @pool->attrs.
4741	*
4742	* Return: 0 on success, -errno on failure. Even on failure, all fields
4743	* inside @pool proper are initialized and put_unbound_pool() can be called
4744	* on @pool safely to release it.
4745	*/
4746	static int init_worker_pool(struct worker_pool *pool)
4747	{
4748	raw_spin_lock_init(&pool->lock);
4749	pool->id = -1;
4750	pool->cpu = -1;
4751	pool->node = NUMA_NO_NODE;
4752	pool->flags \|= POOL_DISASSOCIATED;
4753	pool->watchdog_ts = jiffies;
4754	INIT_LIST_HEAD(&pool->worklist);
4755	INIT_LIST_HEAD(&pool->idle_list);
4756	hash_init(pool->busy_hash);
4757
4758	timer_setup(&pool->idle_timer, idle_worker_timeout, TIMER_DEFERRABLE);
4759	INIT_WORK(&pool->idle_cull_work, idle_cull_fn);
4760
4761	timer_setup(&pool->mayday_timer, pool_mayday_timeout, 0);
4762
4763	INIT_LIST_HEAD(&pool->workers);
4764	INIT_LIST_HEAD(&pool->dying_workers);
4765
4766	ida_init(&pool->worker_ida);
4767	INIT_HLIST_NODE(&pool->hash_node);
4768	pool->refcnt = 1;
4769
4770	/* shouldn't fail above this point */
4771	pool->attrs = alloc_workqueue_attrs();
4772	if (!pool->attrs)
4773	return -ENOMEM;
4774
4775	wqattrs_clear_for_pool(pool->attrs);
4776
4777	return 0;
4778	}
4779
4780	#ifdef CONFIG_LOCKDEP
4781	static void wq_init_lockdep(struct workqueue_struct *wq)
4782	{
4783	char *lock_name;
4784
4785	lockdep_register_key(&wq->key);
4786	lock_name = kasprintf(GFP_KERNEL, "%s%s", "(wq_completion)", wq->name);
4787	if (!lock_name)
4788	lock_name = wq->name;
4789
4790	wq->lock_name = lock_name;
4791	lockdep_init_map(&wq->lockdep_map, lock_name, &wq->key, 0);
4792	}
4793
4794	static void wq_unregister_lockdep(struct workqueue_struct *wq)
4795	{
4796	lockdep_unregister_key(&wq->key);
4797	}
4798
4799	static void wq_free_lockdep(struct workqueue_struct *wq)
4800	{
4801	if (wq->lock_name != wq->name)
4802	kfree(wq->lock_name);
4803	}
4804	#else
4805	static void wq_init_lockdep(struct workqueue_struct *wq)
4806	{
4807	}
4808
4809	static void wq_unregister_lockdep(struct workqueue_struct *wq)
4810	{
4811	}
4812
4813	static void wq_free_lockdep(struct workqueue_struct *wq)
4814	{
4815	}
4816	#endif
4817
4818	static void free_node_nr_active(struct wq_node_nr_active **nna_ar)
4819	{
4820	int node;
4821
4822	for_each_node(node) {
4823	kfree(nna_ar[node]);
4824	nna_ar[node] = NULL;
4825	}
4826
4827	kfree(nna_ar[nr_node_ids]);
4828	nna_ar[nr_node_ids] = NULL;
4829	}
4830
4831	static void init_node_nr_active(struct wq_node_nr_active *nna)
4832	{
4833	nna->max = WQ_DFL_MIN_ACTIVE;
4834	atomic_set(&nna->nr, 0);
4835	raw_spin_lock_init(&nna->lock);
4836	INIT_LIST_HEAD(&nna->pending_pwqs);
4837	}
4838
4839	/*
4840	* Each node's nr_active counter will be accessed mostly from its own node and
4841	* should be allocated in the node.
4842	*/
4843	static int alloc_node_nr_active(struct wq_node_nr_active **nna_ar)
4844	{
4845	struct wq_node_nr_active *nna;
4846	int node;
4847
4848	for_each_node(node) {
4849	nna = kzalloc_node(sizeof(*nna), GFP_KERNEL, node);
4850	if (!nna)
4851	goto err_free;
4852	init_node_nr_active(nna);
4853	nna_ar[node] = nna;
4854	}
4855
4856	/* [nr_node_ids] is used as the fallback */
4857	nna = kzalloc_node(sizeof(*nna), GFP_KERNEL, NUMA_NO_NODE);
4858	if (!nna)
4859	goto err_free;
4860	init_node_nr_active(nna);
4861	nna_ar[nr_node_ids] = nna;
4862
4863	return 0;
4864
4865	err_free:
4866	free_node_nr_active(nna_ar);
4867	return -ENOMEM;
4868	}
4869
4870	static void rcu_free_wq(struct rcu_head *rcu)
4871	{
4872	struct workqueue_struct *wq =
4873	container_of(rcu, struct workqueue_struct, rcu);
4874
4875	if (wq->flags & WQ_UNBOUND)
4876	free_node_nr_active(wq->node_nr_active);
4877
4878	wq_free_lockdep(wq);
4879	free_percpu(wq->cpu_pwq);
4880	free_workqueue_attrs(wq->unbound_attrs);
4881	kfree(wq);
4882	}
4883
4884	static void rcu_free_pool(struct rcu_head *rcu)
4885	{
4886	struct worker_pool *pool = container_of(rcu, struct worker_pool, rcu);
4887
4888	ida_destroy(&pool->worker_ida);
4889	free_workqueue_attrs(pool->attrs);
4890	kfree(pool);
4891	}
4892
4893	/**
4894	* put_unbound_pool - put a worker_pool
4895	* @pool: worker_pool to put
4896	*
4897	* Put @pool. If its refcnt reaches zero, it gets destroyed in RCU
4898	* safe manner. get_unbound_pool() calls this function on its failure path
4899	* and this function should be able to release pools which went through,
4900	* successfully or not, init_worker_pool().
4901	*
4902	* Should be called with wq_pool_mutex held.
4903	*/
4904	static void put_unbound_pool(struct worker_pool *pool)
4905	{
4906	DECLARE_COMPLETION_ONSTACK(detach_completion);
4907	struct worker *worker;
4908	LIST_HEAD(cull_list);
4909
4910	lockdep_assert_held(&wq_pool_mutex);
4911
4912	if (--pool->refcnt)
4913	return;
4914
4915	/* sanity checks */
4916	if (WARN_ON(!(pool->cpu < 0)) \|\|
4917	WARN_ON(!list_empty(&pool->worklist)))
4918	return;
4919
4920	/* release id and unhash */
4921	if (pool->id >= 0)
4922	idr_remove(&worker_pool_idr, pool->id);
4923	hash_del(&pool->hash_node);
4924
4925	/*
4926	* Become the manager and destroy all workers. This prevents
4927	* @pool's workers from blocking on attach_mutex. We're the last
4928	* manager and @pool gets freed with the flag set.
4929	*
4930	* Having a concurrent manager is quite unlikely to happen as we can
4931	* only get here with
4932	* pwq->refcnt == pool->refcnt == 0
4933	* which implies no work queued to the pool, which implies no worker can
4934	* become the manager. However a worker could have taken the role of
4935	* manager before the refcnts dropped to 0, since maybe_create_worker()
4936	* drops pool->lock
4937	*/
4938	while (true) {
4939	rcuwait_wait_event(&manager_wait,
4940	!(pool->flags & POOL_MANAGER_ACTIVE),
4941	TASK_UNINTERRUPTIBLE);
4942
4943	mutex_lock(&wq_pool_attach_mutex);
4944	raw_spin_lock_irq(&pool->lock);
4945	if (!(pool->flags & POOL_MANAGER_ACTIVE)) {
4946	pool->flags \|= POOL_MANAGER_ACTIVE;
4947	break;
4948	}
4949	raw_spin_unlock_irq(&pool->lock);
4950	mutex_unlock(&wq_pool_attach_mutex);
4951	}
4952
4953	while ((worker = first_idle_worker(pool)))
4954	set_worker_dying(worker, &cull_list);
4955	WARN_ON(pool->nr_workers \|\| pool->nr_idle);
4956	raw_spin_unlock_irq(&pool->lock);
4957
4958	wake_dying_workers(&cull_list);
4959
4960	if (!list_empty(&pool->workers) \|\| !list_empty(&pool->dying_workers))
4961	pool->detach_completion = &detach_completion;
4962	mutex_unlock(&wq_pool_attach_mutex);
4963
4964	if (pool->detach_completion)
4965	wait_for_completion(pool->detach_completion);
4966
4967	/* shut down the timers */
4968	del_timer_sync(&pool->idle_timer);
4969	cancel_work_sync(&pool->idle_cull_work);
4970	del_timer_sync(&pool->mayday_timer);
4971
4972	/* RCU protected to allow dereferences from get_work_pool() */
4973	call_rcu(&pool->rcu, rcu_free_pool);
4974	}
4975
4976	/**
4977	* get_unbound_pool - get a worker_pool with the specified attributes
4978	* @attrs: the attributes of the worker_pool to get
4979	*
4980	* Obtain a worker_pool which has the same attributes as @attrs, bump the
4981	* reference count and return it. If there already is a matching
4982	* worker_pool, it will be used; otherwise, this function attempts to
4983	* create a new one.
4984	*
4985	* Should be called with wq_pool_mutex held.
4986	*
4987	* Return: On success, a worker_pool with the same attributes as @attrs.
4988	* On failure, %NULL.
4989	*/
4990	static struct worker_pool get_unbound_pool(const struct workqueue_attrs attrs)
4991	{
4992	struct wq_pod_type *pt = &wq_pod_types[WQ_AFFN_NUMA];
4993	u32 hash = wqattrs_hash(attrs);
4994	struct worker_pool *pool;
4995	int pod, node = NUMA_NO_NODE;
4996
4997	lockdep_assert_held(&wq_pool_mutex);
4998
4999	/* do we already have a matching pool? */
5000	hash_for_each_possible(unbound_pool_hash, pool, hash_node, hash) {
5001	if (wqattrs_equal(pool->attrs, attrs)) {
5002	pool->refcnt++;
5003	return pool;
5004	}
5005	}
5006
5007	/* If __pod_cpumask is contained inside a NUMA pod, that's our node */
5008	for (pod = 0; pod < pt->nr_pods; pod++) {
5009	if (cpumask_subset(attrs->__pod_cpumask, pt->pod_cpus[pod])) {
5010	node = pt->pod_node[pod];
5011	break;
5012	}
5013	}
5014
5015	/* nope, create a new one */
5016	pool = kzalloc_node(sizeof(*pool), GFP_KERNEL, node);
5017	if (!pool \|\| init_worker_pool(pool) < 0)
5018	goto fail;
5019
5020	pool->node = node;
5021	copy_workqueue_attrs(pool->attrs, attrs);
5022	wqattrs_clear_for_pool(pool->attrs);
5023
5024	if (worker_pool_assign_id(pool) < 0)
5025	goto fail;
5026
5027	/* create and start the initial worker */
5028	if (wq_online && !create_worker(pool))
5029	goto fail;
5030
5031	/* install */
5032	hash_add(unbound_pool_hash, &pool->hash_node, hash);
5033
5034	return pool;
5035	fail:
5036	if (pool)
5037	put_unbound_pool(pool);
5038	return NULL;
5039	}
5040
5041	static void rcu_free_pwq(struct rcu_head *rcu)
5042	{
5043	kmem_cache_free(pwq_cache,
5044	container_of(rcu, struct pool_workqueue, rcu));
5045	}
5046
5047	/*
5048	* Scheduled on pwq_release_worker by put_pwq() when an unbound pwq hits zero
5049	* refcnt and needs to be destroyed.
5050	*/
5051	static void pwq_release_workfn(struct kthread_work *work)
5052	{
5053	struct pool_workqueue *pwq = container_of(work, struct pool_workqueue,
5054	release_work);
5055	struct workqueue_struct *wq = pwq->wq;
5056	struct worker_pool *pool = pwq->pool;
5057	bool is_last = false;
5058
5059	/*
5060	* When @pwq is not linked, it doesn't hold any reference to the
5061	* @wq, and @wq is invalid to access.
5062	*/
5063	if (!list_empty(&pwq->pwqs_node)) {
5064	mutex_lock(&wq->mutex);
5065	list_del_rcu(&pwq->pwqs_node);
5066	is_last = list_empty(&wq->pwqs);
5067
5068	/*
5069	* For ordered workqueue with a plugged dfl_pwq, restart it now.
5070	*/
5071	if (!is_last && (wq->flags & __WQ_ORDERED))
5072	unplug_oldest_pwq(wq);
5073
5074	mutex_unlock(&wq->mutex);
5075	}
5076
5077	if (wq->flags & WQ_UNBOUND) {
5078	mutex_lock(&wq_pool_mutex);
5079	put_unbound_pool(pool);
5080	mutex_unlock(&wq_pool_mutex);
5081	}
5082
5083	if (!list_empty(&pwq->pending_node)) {
5084	struct wq_node_nr_active *nna =
5085	wq_node_nr_active(pwq->wq, pwq->pool->node);
5086
5087	raw_spin_lock_irq(&nna->lock);
5088	list_del_init(&pwq->pending_node);
5089	raw_spin_unlock_irq(&nna->lock);
5090	}
5091
5092	call_rcu(&pwq->rcu, rcu_free_pwq);
5093
5094	/*
5095	* If we're the last pwq going away, @wq is already dead and no one
5096	* is gonna access it anymore. Schedule RCU free.
5097	*/
5098	if (is_last) {
5099	wq_unregister_lockdep(wq);
5100	call_rcu(&wq->rcu, rcu_free_wq);
5101	}
5102	}
5103
5104	/* initialize newly allocated @pwq which is associated with @wq and @pool */
5105	static void init_pwq(struct pool_workqueue pwq, struct workqueue_struct wq,
5106	struct worker_pool *pool)
5107	{
5108	BUG_ON((unsigned long)pwq & ~WORK_STRUCT_PWQ_MASK);
5109
5110	memset(pwq, 0, sizeof(*pwq));
5111
5112	pwq->pool = pool;
5113	pwq->wq = wq;
5114	pwq->flush_color = -1;
5115	pwq->refcnt = 1;
5116	INIT_LIST_HEAD(&pwq->inactive_works);
5117	INIT_LIST_HEAD(&pwq->pending_node);
5118	INIT_LIST_HEAD(&pwq->pwqs_node);
5119	INIT_LIST_HEAD(&pwq->mayday_node);
5120	kthread_init_work(&pwq->release_work, pwq_release_workfn);
5121	}
5122
5123	/* sync @pwq with the current state of its associated wq and link it */
5124	static void link_pwq(struct pool_workqueue *pwq)
5125	{
5126	struct workqueue_struct *wq = pwq->wq;
5127
5128	lockdep_assert_held(&wq->mutex);
5129
5130	/* may be called multiple times, ignore if already linked */
5131	if (!list_empty(&pwq->pwqs_node))
5132	return;
5133
5134	/* set the matching work_color */
5135	pwq->work_color = wq->work_color;
5136
5137	/* link in @pwq */
5138	list_add_tail_rcu(&pwq->pwqs_node, &wq->pwqs);
5139	}
5140
5141	/* obtain a pool matching @attr and create a pwq associating the pool and @wq */
5142	static struct pool_workqueue alloc_unbound_pwq(struct workqueue_struct wq,
5143	const struct workqueue_attrs *attrs)
5144	{
5145	struct worker_pool *pool;
5146	struct pool_workqueue *pwq;
5147
5148	lockdep_assert_held(&wq_pool_mutex);
5149
5150	pool = get_unbound_pool(attrs);
5151	if (!pool)
5152	return NULL;
5153
5154	pwq = kmem_cache_alloc_node(pwq_cache, GFP_KERNEL, pool->node);
5155	if (!pwq) {
5156	put_unbound_pool(pool);
5157	return NULL;
5158	}
5159
5160	init_pwq(pwq, wq, pool);
5161	return pwq;
5162	}
5163
5164	/**
5165	* wq_calc_pod_cpumask - calculate a wq_attrs' cpumask for a pod
5166	* @attrs: the wq_attrs of the default pwq of the target workqueue
5167	* @cpu: the target CPU
5168	* @cpu_going_down: if >= 0, the CPU to consider as offline
5169	*
5170	* Calculate the cpumask a workqueue with @attrs should use on @pod. If
5171	* @cpu_going_down is >= 0, that cpu is considered offline during calculation.
5172	* The result is stored in @attrs->__pod_cpumask.
5173	*
5174	* If pod affinity is not enabled, @attrs->cpumask is always used. If enabled
5175	* and @pod has online CPUs requested by @attrs, the returned cpumask is the
5176	* intersection of the possible CPUs of @pod and @attrs->cpumask.
5177	*
5178	* The caller is responsible for ensuring that the cpumask of @pod stays stable.
5179	*/
5180	static void wq_calc_pod_cpumask(struct workqueue_attrs *attrs, int cpu,
5181	int cpu_going_down)
5182	{
5183	const struct wq_pod_type *pt = wqattrs_pod_type(attrs);
5184	int pod = pt->cpu_pod[cpu];
5185
5186	/* does @pod have any online CPUs @attrs wants? */
5187	cpumask_and(attrs->__pod_cpumask, pt->pod_cpus[pod], attrs->cpumask);
5188	cpumask_and(attrs->__pod_cpumask, attrs->__pod_cpumask, cpu_online_mask);
5189	if (cpu_going_down >= 0)
5190	cpumask_clear_cpu(cpu_going_down, attrs->__pod_cpumask);
5191
5192	if (cpumask_empty(attrs->__pod_cpumask)) {
5193	cpumask_copy(attrs->__pod_cpumask, attrs->cpumask);
5194	return;
5195	}
5196
5197	/* yeap, return possible CPUs in @pod that @attrs wants */
5198	cpumask_and(attrs->__pod_cpumask, attrs->cpumask, pt->pod_cpus[pod]);
5199
5200	if (cpumask_empty(attrs->__pod_cpumask))
5201	pr_warn_once("WARNING: workqueue cpumask: online intersect > "
5202	"possible intersect\n");
5203	}
5204
5205	/* install @pwq into @wq and return the old pwq, @cpu < 0 for dfl_pwq */
5206	static struct pool_workqueue install_unbound_pwq(struct workqueue_struct wq,
5207	int cpu, struct pool_workqueue *pwq)
5208	{
5209	struct pool_workqueue __rcu **slot = unbound_pwq_slot(wq, cpu);
5210	struct pool_workqueue *old_pwq;
5211
5212	lockdep_assert_held(&wq_pool_mutex);
5213	lockdep_assert_held(&wq->mutex);
5214
5215	/* link_pwq() can handle duplicate calls */
5216	link_pwq(pwq);
5217
5218	old_pwq = rcu_access_pointer(*slot);
5219	rcu_assign_pointer(*slot, pwq);
5220	return old_pwq;
5221	}
5222
5223	/* context to store the prepared attrs & pwqs before applying */
5224	struct apply_wqattrs_ctx {
5225	struct workqueue_struct wq; / target workqueue */
5226	struct workqueue_attrs attrs; / attrs to apply */
5227	struct list_head list; /* queued for batching commit */
5228	struct pool_workqueue *dfl_pwq;
5229	struct pool_workqueue *pwq_tbl[];
5230	};
5231
5232	/* free the resources after success or abort */
5233	static void apply_wqattrs_cleanup(struct apply_wqattrs_ctx *ctx)
5234	{
5235	if (ctx) {
5236	int cpu;
5237
5238	for_each_possible_cpu(cpu)
5239	put_pwq_unlocked(ctx->pwq_tbl[cpu]);
5240	put_pwq_unlocked(ctx->dfl_pwq);
5241
5242	free_workqueue_attrs(ctx->attrs);
5243
5244	kfree(ctx);
5245	}
5246	}
5247
5248	/* allocate the attrs and pwqs for later installation */
5249	static struct apply_wqattrs_ctx *
5250	apply_wqattrs_prepare(struct workqueue_struct *wq,
5251	const struct workqueue_attrs *attrs,
5252	const cpumask_var_t unbound_cpumask)
5253	{
5254	struct apply_wqattrs_ctx *ctx;
5255	struct workqueue_attrs *new_attrs;
5256	int cpu;
5257
5258	lockdep_assert_held(&wq_pool_mutex);
5259
5260	if (WARN_ON(attrs->affn_scope < 0 \|\|
5261	attrs->affn_scope >= WQ_AFFN_NR_TYPES))
5262	return ERR_PTR(-EINVAL);
5263
5264	ctx = kzalloc(struct_size(ctx, pwq_tbl, nr_cpu_ids), GFP_KERNEL);
5265
5266	new_attrs = alloc_workqueue_attrs();
5267	if (!ctx \|\| !new_attrs)
5268	goto out_free;
5269
5270	/*
5271	* If something goes wrong during CPU up/down, we'll fall back to
5272	* the default pwq covering whole @attrs->cpumask. Always create
5273	* it even if we don't use it immediately.
5274	*/
5275	copy_workqueue_attrs(new_attrs, attrs);
5276	wqattrs_actualize_cpumask(new_attrs, unbound_cpumask);
5277	cpumask_copy(new_attrs->__pod_cpumask, new_attrs->cpumask);
5278	ctx->dfl_pwq = alloc_unbound_pwq(wq, new_attrs);
5279	if (!ctx->dfl_pwq)
5280	goto out_free;
5281
5282	for_each_possible_cpu(cpu) {
5283	if (new_attrs->ordered) {
5284	ctx->dfl_pwq->refcnt++;
5285	ctx->pwq_tbl[cpu] = ctx->dfl_pwq;
5286	} else {
5287	wq_calc_pod_cpumask(new_attrs, cpu, -1);
5288	ctx->pwq_tbl[cpu] = alloc_unbound_pwq(wq, new_attrs);
5289	if (!ctx->pwq_tbl[cpu])
5290	goto out_free;
5291	}
5292	}
5293
5294	/* save the user configured attrs and sanitize it. */
5295	copy_workqueue_attrs(new_attrs, attrs);
5296	cpumask_and(new_attrs->cpumask, new_attrs->cpumask, cpu_possible_mask);
5297	cpumask_copy(new_attrs->__pod_cpumask, new_attrs->cpumask);
5298	ctx->attrs = new_attrs;
5299
5300	/*
5301	* For initialized ordered workqueues, there should only be one pwq
5302	* (dfl_pwq). Set the plugged flag of ctx->dfl_pwq to suspend execution
5303	* of newly queued work items until execution of older work items in
5304	* the old pwq's have completed.
5305	*/
5306	if ((wq->flags & __WQ_ORDERED) && !list_empty(&wq->pwqs))
5307	ctx->dfl_pwq->plugged = true;
5308
5309	ctx->wq = wq;
5310	return ctx;
5311
5312	out_free:
5313	free_workqueue_attrs(new_attrs);
5314	apply_wqattrs_cleanup(ctx);
5315	return ERR_PTR(-ENOMEM);
5316	}
5317
5318	/* set attrs and install prepared pwqs, @ctx points to old pwqs on return */
5319	static void apply_wqattrs_commit(struct apply_wqattrs_ctx *ctx)
5320	{
5321	int cpu;
5322
5323	/* all pwqs have been created successfully, let's install'em */
5324	mutex_lock(&ctx->wq->mutex);
5325
5326	copy_workqueue_attrs(ctx->wq->unbound_attrs, ctx->attrs);
5327
5328	/* save the previous pwqs and install the new ones */
5329	for_each_possible_cpu(cpu)
5330	ctx->pwq_tbl[cpu] = install_unbound_pwq(ctx->wq, cpu,
5331	ctx->pwq_tbl[cpu]);
5332	ctx->dfl_pwq = install_unbound_pwq(ctx->wq, -1, ctx->dfl_pwq);
5333
5334	/* update node_nr_active->max */
5335	wq_update_node_max_active(ctx->wq, -1);
5336
5337	/* rescuer needs to respect wq cpumask changes */
5338	if (ctx->wq->rescuer)
5339	set_cpus_allowed_ptr(ctx->wq->rescuer->task,
5340	unbound_effective_cpumask(ctx->wq));
5341
5342	mutex_unlock(&ctx->wq->mutex);
5343	}
5344
5345	static int apply_workqueue_attrs_locked(struct workqueue_struct *wq,
5346	const struct workqueue_attrs *attrs)
5347	{
5348	struct apply_wqattrs_ctx *ctx;
5349
5350	/* only unbound workqueues can change attributes */
5351	if (WARN_ON(!(wq->flags & WQ_UNBOUND)))
5352	return -EINVAL;
5353
5354	ctx = apply_wqattrs_prepare(wq, attrs, wq_unbound_cpumask);
5355	if (IS_ERR(ctx))
5356	return PTR_ERR(ctx);
5357
5358	/* the ctx has been prepared successfully, let's commit it */
5359	apply_wqattrs_commit(ctx);
5360	apply_wqattrs_cleanup(ctx);
5361
5362	return 0;
5363	}
5364
5365	/**
5366	* apply_workqueue_attrs - apply new workqueue_attrs to an unbound workqueue
5367	* @wq: the target workqueue
5368	* @attrs: the workqueue_attrs to apply, allocated with alloc_workqueue_attrs()
5369	*
5370	* Apply @attrs to an unbound workqueue @wq. Unless disabled, this function maps
5371	* a separate pwq to each CPU pod with possibles CPUs in @attrs->cpumask so that
5372	* work items are affine to the pod it was issued on. Older pwqs are released as
5373	* in-flight work items finish. Note that a work item which repeatedly requeues
5374	* itself back-to-back will stay on its current pwq.
5375	*
5376	* Performs GFP_KERNEL allocations.
5377	*
5378	* Assumes caller has CPU hotplug read exclusion, i.e. cpus_read_lock().
5379	*
5380	* Return: 0 on success and -errno on failure.
5381	*/
5382	int apply_workqueue_attrs(struct workqueue_struct *wq,
5383	const struct workqueue_attrs *attrs)
5384	{
5385	int ret;
5386
5387	lockdep_assert_cpus_held();
5388
5389	mutex_lock(&wq_pool_mutex);
5390	ret = apply_workqueue_attrs_locked(wq, attrs);
5391	mutex_unlock(&wq_pool_mutex);
5392
5393	return ret;
5394	}
5395
5396	/**
5397	* wq_update_pod - update pod affinity of a wq for CPU hot[un]plug
5398	* @wq: the target workqueue
5399	* @cpu: the CPU to update pool association for
5400	* @hotplug_cpu: the CPU coming up or going down
5401	* @online: whether @cpu is coming up or going down
5402	*
5403	* This function is to be called from %CPU_DOWN_PREPARE, %CPU_ONLINE and
5404	* %CPU_DOWN_FAILED. @cpu is being hot[un]plugged, update pod affinity of
5405	* @wq accordingly.
5406	*
5407	*
5408	* If pod affinity can't be adjusted due to memory allocation failure, it falls
5409	* back to @wq->dfl_pwq which may not be optimal but is always correct.
5410	*
5411	* Note that when the last allowed CPU of a pod goes offline for a workqueue
5412	* with a cpumask spanning multiple pods, the workers which were already
5413	* executing the work items for the workqueue will lose their CPU affinity and
5414	* may execute on any CPU. This is similar to how per-cpu workqueues behave on
5415	* CPU_DOWN. If a workqueue user wants strict affinity, it's the user's
5416	* responsibility to flush the work item from CPU_DOWN_PREPARE.
5417	*/
5418	static void wq_update_pod(struct workqueue_struct *wq, int cpu,
5419	int hotplug_cpu, bool online)
5420	{
5421	int off_cpu = online ? -1 : hotplug_cpu;
5422	struct pool_workqueue old_pwq = NULL, pwq;
5423	struct workqueue_attrs *target_attrs;
5424
5425	lockdep_assert_held(&wq_pool_mutex);
5426
5427	if (!(wq->flags & WQ_UNBOUND) \|\| wq->unbound_attrs->ordered)
5428	return;
5429
5430	/*
5431	* We don't wanna alloc/free wq_attrs for each wq for each CPU.
5432	* Let's use a preallocated one. The following buf is protected by
5433	* CPU hotplug exclusion.
5434	*/
5435	target_attrs = wq_update_pod_attrs_buf;
5436
5437	copy_workqueue_attrs(target_attrs, wq->unbound_attrs);
5438	wqattrs_actualize_cpumask(target_attrs, wq_unbound_cpumask);
5439
5440	/* nothing to do if the target cpumask matches the current pwq */
5441	wq_calc_pod_cpumask(target_attrs, cpu, off_cpu);
5442	if (wqattrs_equal(target_attrs, unbound_pwq(wq, cpu)->pool->attrs))
5443	return;
5444
5445	/* create a new pwq */
5446	pwq = alloc_unbound_pwq(wq, target_attrs);
5447	if (!pwq) {
5448	pr_warn("workqueue: allocation failed while updating CPU pod affinity of \"%s\"\n",
5449	wq->name);
5450	goto use_dfl_pwq;
5451	}
5452
5453	/* Install the new pwq. */
5454	mutex_lock(&wq->mutex);
5455	old_pwq = install_unbound_pwq(wq, cpu, pwq);
5456	goto out_unlock;
5457
5458	use_dfl_pwq:
5459	mutex_lock(&wq->mutex);
5460	pwq = unbound_pwq(wq, -1);
5461	raw_spin_lock_irq(&pwq->pool->lock);
5462	get_pwq(pwq);
5463	raw_spin_unlock_irq(&pwq->pool->lock);
5464	old_pwq = install_unbound_pwq(wq, cpu, pwq);
5465	out_unlock:
5466	mutex_unlock(&wq->mutex);
5467	put_pwq_unlocked(old_pwq);
5468	}
5469
5470	static int alloc_and_link_pwqs(struct workqueue_struct *wq)
5471	{
5472	bool highpri = wq->flags & WQ_HIGHPRI;
5473	int cpu, ret;
5474
5475	wq->cpu_pwq = alloc_percpu(struct pool_workqueue *);
5476	if (!wq->cpu_pwq)
5477	goto enomem;
5478
5479	if (!(wq->flags & WQ_UNBOUND)) {
5480	for_each_possible_cpu(cpu) {
5481	struct pool_workqueue **pwq_p;
5482	struct worker_pool __percpu *pools;
5483	struct worker_pool *pool;
5484
5485	if (wq->flags & WQ_BH)
5486	pools = bh_worker_pools;
5487	else
5488	pools = cpu_worker_pools;
5489
5490	pool = &(per_cpu_ptr(pools, cpu)[highpri]);
5491	pwq_p = per_cpu_ptr(wq->cpu_pwq, cpu);
5492
5493	*pwq_p = kmem_cache_alloc_node(pwq_cache, GFP_KERNEL,
5494	pool->node);
5495	if (!*pwq_p)
5496	goto enomem;
5497
5498	init_pwq(*pwq_p, wq, pool);
5499
5500	mutex_lock(&wq->mutex);
5501	link_pwq(*pwq_p);
5502	mutex_unlock(&wq->mutex);
5503	}
5504	return 0;
5505	}
5506
5507	cpus_read_lock();
5508	if (wq->flags & __WQ_ORDERED) {
5509	struct pool_workqueue *dfl_pwq;
5510
5511	ret = apply_workqueue_attrs(wq, ordered_wq_attrs[highpri]);
5512	/* there should only be single pwq for ordering guarantee */
5513	dfl_pwq = rcu_access_pointer(wq->dfl_pwq);
5514	WARN(!ret && (wq->pwqs.next != &dfl_pwq->pwqs_node \|\|
5515	wq->pwqs.prev != &dfl_pwq->pwqs_node),
5516	"ordering guarantee broken for workqueue %s\n", wq->name);
5517	} else {
5518	ret = apply_workqueue_attrs(wq, unbound_std_wq_attrs[highpri]);
5519	}
5520	cpus_read_unlock();
5521
5522	/* for unbound pwq, flush the pwq_release_worker ensures that the
5523	* pwq_release_workfn() completes before calling kfree(wq).
5524	*/
5525	if (ret)
5526	kthread_flush_worker(pwq_release_worker);
5527
5528	return ret;
5529
5530	enomem:
5531	if (wq->cpu_pwq) {
5532	for_each_possible_cpu(cpu) {
5533	struct pool_workqueue pwq = per_cpu_ptr(wq->cpu_pwq, cpu);
5534
5535	if (pwq)
5536	kmem_cache_free(pwq_cache, pwq);
5537	}
5538	free_percpu(wq->cpu_pwq);
5539	wq->cpu_pwq = NULL;
5540	}
5541	return -ENOMEM;
5542	}
5543
5544	static int wq_clamp_max_active(int max_active, unsigned int flags,
5545	const char *name)
5546	{
5547	if (max_active < 1 \|\| max_active > WQ_MAX_ACTIVE)
5548	pr_warn("workqueue: max_active %d requested for %s is out of range, clamping between %d and %d\n",
5549	max_active, name, 1, WQ_MAX_ACTIVE);
5550
5551	return clamp_val(max_active, 1, WQ_MAX_ACTIVE);
5552	}
5553
5554	/*
5555	* Workqueues which may be used during memory reclaim should have a rescuer
5556	* to guarantee forward progress.
5557	*/
5558	static int init_rescuer(struct workqueue_struct *wq)
5559	{
5560	struct worker *rescuer;
5561	char id_buf[WORKER_ID_LEN];
5562	int ret;
5563
5564	if (!(wq->flags & WQ_MEM_RECLAIM))
5565	return 0;
5566
5567	rescuer = alloc_worker(NUMA_NO_NODE);
5568	if (!rescuer) {
5569	pr_err("workqueue: Failed to allocate a rescuer for wq \"%s\"\n",
5570	wq->name);
5571	return -ENOMEM;
5572	}
5573
5574	rescuer->rescue_wq = wq;
5575	format_worker_id(id_buf, sizeof(id_buf), rescuer, NULL);
5576
5577	rescuer->task = kthread_create(rescuer_thread, rescuer, "%s", id_buf);
5578	if (IS_ERR(rescuer->task)) {
5579	ret = PTR_ERR(rescuer->task);
5580	pr_err("workqueue: Failed to create a rescuer kthread for wq \"%s\": %pe",
5581	wq->name, ERR_PTR(ret));
5582	kfree(rescuer);
5583	return ret;
5584	}
5585
5586	wq->rescuer = rescuer;
5587	if (wq->flags & WQ_UNBOUND)
5588	kthread_bind_mask(rescuer->task, wq_unbound_cpumask);
5589	else
5590	kthread_bind_mask(rescuer->task, cpu_possible_mask);
5591	wake_up_process(rescuer->task);
5592
5593	return 0;
5594	}
5595
5596	/**
5597	* wq_adjust_max_active - update a wq's max_active to the current setting
5598	* @wq: target workqueue
5599	*
5600	* If @wq isn't freezing, set @wq->max_active to the saved_max_active and
5601	* activate inactive work items accordingly. If @wq is freezing, clear
5602	* @wq->max_active to zero.
5603	*/
5604	static void wq_adjust_max_active(struct workqueue_struct *wq)
5605	{
5606	bool activated;
5607	int new_max, new_min;
5608
5609	lockdep_assert_held(&wq->mutex);
5610
5611	if ((wq->flags & WQ_FREEZABLE) && workqueue_freezing) {
5612	new_max = 0;
5613	new_min = 0;
5614	} else {
5615	new_max = wq->saved_max_active;
5616	new_min = wq->saved_min_active;
5617	}
5618
5619	if (wq->max_active == new_max && wq->min_active == new_min)
5620	return;
5621
5622	/*
5623	* Update @wq->max/min_active and then kick inactive work items if more
5624	* active work items are allowed. This doesn't break work item ordering
5625	* because new work items are always queued behind existing inactive
5626	* work items if there are any.
5627	*/
5628	WRITE_ONCE(wq->max_active, new_max);
5629	WRITE_ONCE(wq->min_active, new_min);
5630
5631	if (wq->flags & WQ_UNBOUND)
5632	wq_update_node_max_active(wq, -1);
5633
5634	if (new_max == 0)
5635	return;
5636
5637	/*
5638	* Round-robin through pwq's activating the first inactive work item
5639	* until max_active is filled.
5640	*/
5641	do {
5642	struct pool_workqueue *pwq;
5643
5644	activated = false;
5645	for_each_pwq(pwq, wq) {
5646	unsigned long irq_flags;
5647
5648	/* can be called during early boot w/ irq disabled */
5649	raw_spin_lock_irqsave(&pwq->pool->lock, irq_flags);
5650	if (pwq_activate_first_inactive(pwq, true)) {
5651	activated = true;
5652	kick_pool(pwq->pool);
5653	}
5654	raw_spin_unlock_irqrestore(&pwq->pool->lock, irq_flags);
5655	}
5656	} while (activated);
5657	}
5658
5659	__printf(1, 4)
5660	struct workqueue_struct alloc_workqueue(const char fmt,
5661	unsigned int flags,
5662	int max_active, ...)
5663	{
5664	va_list args;
5665	struct workqueue_struct *wq;
5666	size_t wq_size;
5667	int name_len;
5668
5669	if (flags & WQ_BH) {
5670	if (WARN_ON_ONCE(flags & ~__WQ_BH_ALLOWS))
5671	return NULL;
5672	if (WARN_ON_ONCE(max_active))
5673	return NULL;
5674	}
5675
5676	/* see the comment above the definition of WQ_POWER_EFFICIENT */
5677	if ((flags & WQ_POWER_EFFICIENT) && wq_power_efficient)
5678	flags \|= WQ_UNBOUND;
5679
5680	/* allocate wq and format name */
5681	if (flags & WQ_UNBOUND)
5682	wq_size = struct_size(wq, node_nr_active, nr_node_ids + 1);
5683	else
5684	wq_size = sizeof(*wq);
5685
5686	wq = kzalloc(wq_size, GFP_KERNEL);
5687	if (!wq)
5688	return NULL;
5689
5690	if (flags & WQ_UNBOUND) {
5691	wq->unbound_attrs = alloc_workqueue_attrs();
5692	if (!wq->unbound_attrs)
5693	goto err_free_wq;
5694	}
5695
5696	va_start(args, max_active);
5697	name_len = vsnprintf(wq->name, sizeof(wq->name), fmt, args);
5698	va_end(args);
5699
5700	if (name_len >= WQ_NAME_LEN)
5701	pr_warn_once("workqueue: name exceeds WQ_NAME_LEN. Truncating to: %s\n",
5702	wq->name);
5703
5704	if (flags & WQ_BH) {
5705	/*
5706	* BH workqueues always share a single execution context per CPU
5707	* and don't impose any max_active limit.
5708	*/
5709	max_active = INT_MAX;
5710	} else {
5711	max_active = max_active ?: WQ_DFL_ACTIVE;
5712	max_active = wq_clamp_max_active(max_active, flags, wq->name);
5713	}
5714
5715	/* init wq */
5716	wq->flags = flags;
5717	wq->max_active = max_active;
5718	wq->min_active = min(max_active, WQ_DFL_MIN_ACTIVE);
5719	wq->saved_max_active = wq->max_active;
5720	wq->saved_min_active = wq->min_active;
5721	mutex_init(&wq->mutex);
5722	atomic_set(&wq->nr_pwqs_to_flush, 0);
5723	INIT_LIST_HEAD(&wq->pwqs);
5724	INIT_LIST_HEAD(&wq->flusher_queue);
5725	INIT_LIST_HEAD(&wq->flusher_overflow);
5726	INIT_LIST_HEAD(&wq->maydays);
5727
5728	wq_init_lockdep(wq);
5729	INIT_LIST_HEAD(&wq->list);
5730
5731	if (flags & WQ_UNBOUND) {
5732	if (alloc_node_nr_active(wq->node_nr_active) < 0)
5733	goto err_unreg_lockdep;
5734	}
5735
5736	if (alloc_and_link_pwqs(wq) < 0)
5737	goto err_free_node_nr_active;
5738
5739	if (wq_online && init_rescuer(wq) < 0)
5740	goto err_destroy;
5741
5742	if ((wq->flags & WQ_SYSFS) && workqueue_sysfs_register(wq))
5743	goto err_destroy;
5744
5745	/*
5746	* wq_pool_mutex protects global freeze state and workqueues list.
5747	* Grab it, adjust max_active and add the new @wq to workqueues
5748	* list.
5749	*/
5750	mutex_lock(&wq_pool_mutex);
5751
5752	mutex_lock(&wq->mutex);
5753	wq_adjust_max_active(wq);
5754	mutex_unlock(&wq->mutex);
5755
5756	list_add_tail_rcu(&wq->list, &workqueues);
5757
5758	mutex_unlock(&wq_pool_mutex);
5759
5760	return wq;
5761
5762	err_free_node_nr_active:
5763	if (wq->flags & WQ_UNBOUND)
5764	free_node_nr_active(wq->node_nr_active);
5765	err_unreg_lockdep:
5766	wq_unregister_lockdep(wq);
5767	wq_free_lockdep(wq);
5768	err_free_wq:
5769	free_workqueue_attrs(wq->unbound_attrs);
5770	kfree(wq);
5771	return NULL;
5772	err_destroy:
5773	destroy_workqueue(wq);
5774	return NULL;
5775	}
5776	EXPORT_SYMBOL_GPL(alloc_workqueue);
5777
5778	static bool pwq_busy(struct pool_workqueue *pwq)
5779	{
5780	int i;
5781
5782	for (i = 0; i < WORK_NR_COLORS; i++)
5783	if (pwq->nr_in_flight[i])
5784	return true;
5785
5786	if ((pwq != rcu_access_pointer(pwq->wq->dfl_pwq)) && (pwq->refcnt > 1))
5787	return true;
5788	if (!pwq_is_empty(pwq))
5789	return true;
5790
5791	return false;
5792	}
5793
5794	/**
5795	* destroy_workqueue - safely terminate a workqueue
5796	* @wq: target workqueue
5797	*
5798	* Safely destroy a workqueue. All work currently pending will be done first.
5799	*/
5800	void destroy_workqueue(struct workqueue_struct *wq)
5801	{
5802	struct pool_workqueue *pwq;
5803	int cpu;
5804
5805	/*
5806	* Remove it from sysfs first so that sanity check failure doesn't
5807	* lead to sysfs name conflicts.
5808	*/
5809	workqueue_sysfs_unregister(wq);
5810
5811	/* mark the workqueue destruction is in progress */
5812	mutex_lock(&wq->mutex);
5813	wq->flags \|= __WQ_DESTROYING;
5814	mutex_unlock(&wq->mutex);
5815
5816	/* drain it before proceeding with destruction */
5817	drain_workqueue(wq);
5818
5819	/* kill rescuer, if sanity checks fail, leave it w/o rescuer */
5820	if (wq->rescuer) {
5821	struct worker *rescuer = wq->rescuer;
5822
5823	/* this prevents new queueing */
5824	raw_spin_lock_irq(&wq_mayday_lock);
5825	wq->rescuer = NULL;
5826	raw_spin_unlock_irq(&wq_mayday_lock);
5827
5828	/* rescuer will empty maydays list before exiting */
5829	kthread_stop(rescuer->task);
5830	kfree(rescuer);
5831	}
5832
5833	/*
5834	* Sanity checks - grab all the locks so that we wait for all
5835	* in-flight operations which may do put_pwq().
5836	*/
5837	mutex_lock(&wq_pool_mutex);
5838	mutex_lock(&wq->mutex);
5839	for_each_pwq(pwq, wq) {
5840	raw_spin_lock_irq(&pwq->pool->lock);
5841	if (WARN_ON(pwq_busy(pwq))) {
5842	pr_warn("%s: %s has the following busy pwq\n",
5843	__func__, wq->name);
5844	show_pwq(pwq);
5845	raw_spin_unlock_irq(&pwq->pool->lock);
5846	mutex_unlock(&wq->mutex);
5847	mutex_unlock(&wq_pool_mutex);
5848	show_one_workqueue(wq);
5849	return;
5850	}
5851	raw_spin_unlock_irq(&pwq->pool->lock);
5852	}
5853	mutex_unlock(&wq->mutex);
5854
5855	/*
5856	* wq list is used to freeze wq, remove from list after
5857	* flushing is complete in case freeze races us.
5858	*/
5859	list_del_rcu(&wq->list);
5860	mutex_unlock(&wq_pool_mutex);
5861
5862	/*
5863	* We're the sole accessor of @wq. Directly access cpu_pwq and dfl_pwq
5864	* to put the base refs. @wq will be auto-destroyed from the last
5865	* pwq_put. RCU read lock prevents @wq from going away from under us.
5866	*/
5867	rcu_read_lock();
5868
5869	for_each_possible_cpu(cpu) {
5870	put_pwq_unlocked(unbound_pwq(wq, cpu));
5871	RCU_INIT_POINTER(*unbound_pwq_slot(wq, cpu), NULL);
5872	}
5873
5874	put_pwq_unlocked(unbound_pwq(wq, -1));
5875	RCU_INIT_POINTER(*unbound_pwq_slot(wq, -1), NULL);
5876
5877	rcu_read_unlock();
5878	}
5879	EXPORT_SYMBOL_GPL(destroy_workqueue);
5880
5881	/**
5882	* workqueue_set_max_active - adjust max_active of a workqueue
5883	* @wq: target workqueue
5884	* @max_active: new max_active value.
5885	*
5886	* Set max_active of @wq to @max_active. See the alloc_workqueue() function
5887	* comment.
5888	*
5889	* CONTEXT:
5890	* Don't call from IRQ context.
5891	*/
5892	void workqueue_set_max_active(struct workqueue_struct *wq, int max_active)
5893	{
5894	/* max_active doesn't mean anything for BH workqueues */
5895	if (WARN_ON(wq->flags & WQ_BH))
5896	return;
5897	/* disallow meddling with max_active for ordered workqueues */
5898	if (WARN_ON(wq->flags & __WQ_ORDERED))
5899	return;
5900
5901	max_active = wq_clamp_max_active(max_active, wq->flags, wq->name);
5902
5903	mutex_lock(&wq->mutex);
5904
5905	wq->saved_max_active = max_active;
5906	if (wq->flags & WQ_UNBOUND)
5907	wq->saved_min_active = min(wq->saved_min_active, max_active);
5908
5909	wq_adjust_max_active(wq);
5910
5911	mutex_unlock(&wq->mutex);
5912	}
5913	EXPORT_SYMBOL_GPL(workqueue_set_max_active);
5914
5915	/**
5916	* workqueue_set_min_active - adjust min_active of an unbound workqueue
5917	* @wq: target unbound workqueue
5918	* @min_active: new min_active value
5919	*
5920	* Set min_active of an unbound workqueue. Unlike other types of workqueues, an
5921	* unbound workqueue is not guaranteed to be able to process max_active
5922	* interdependent work items. Instead, an unbound workqueue is guaranteed to be
5923	* able to process min_active number of interdependent work items which is
5924	* %WQ_DFL_MIN_ACTIVE by default.
5925	*
5926	* Use this function to adjust the min_active value between 0 and the current
5927	* max_active.
5928	*/
5929	void workqueue_set_min_active(struct workqueue_struct *wq, int min_active)
5930	{
5931	/* min_active is only meaningful for non-ordered unbound workqueues */
5932	if (WARN_ON((wq->flags & (WQ_BH \| WQ_UNBOUND \| __WQ_ORDERED)) !=
5933	WQ_UNBOUND))
5934	return;
5935
5936	mutex_lock(&wq->mutex);
5937	wq->saved_min_active = clamp(min_active, 0, wq->saved_max_active);
5938	wq_adjust_max_active(wq);
5939	mutex_unlock(&wq->mutex);
5940	}
5941
5942	/**
5943	* current_work - retrieve %current task's work struct
5944	*
5945	* Determine if %current task is a workqueue worker and what it's working on.
5946	* Useful to find out the context that the %current task is running in.
5947	*
5948	* Return: work struct if %current task is a workqueue worker, %NULL otherwise.
5949	*/
5950	struct work_struct *current_work(void)
5951	{
5952	struct worker *worker = current_wq_worker();
5953
5954	return worker ? worker->current_work : NULL;
5955	}
5956	EXPORT_SYMBOL(current_work);
5957
5958	/**
5959	* current_is_workqueue_rescuer - is %current workqueue rescuer?
5960	*
5961	* Determine whether %current is a workqueue rescuer. Can be used from
5962	* work functions to determine whether it's being run off the rescuer task.
5963	*
5964	* Return: %true if %current is a workqueue rescuer. %false otherwise.
5965	*/
5966	bool current_is_workqueue_rescuer(void)
5967	{
5968	struct worker *worker = current_wq_worker();
5969
5970	return worker && worker->rescue_wq;
5971	}
5972
5973	/**
5974	* workqueue_congested - test whether a workqueue is congested
5975	* @cpu: CPU in question
5976	* @wq: target workqueue
5977	*
5978	* Test whether @wq's cpu workqueue for @cpu is congested. There is
5979	* no synchronization around this function and the test result is
5980	* unreliable and only useful as advisory hints or for debugging.
5981	*
5982	* If @cpu is WORK_CPU_UNBOUND, the test is performed on the local CPU.
5983	*
5984	* With the exception of ordered workqueues, all workqueues have per-cpu
5985	* pool_workqueues, each with its own congested state. A workqueue being
5986	* congested on one CPU doesn't mean that the workqueue is contested on any
5987	* other CPUs.
5988	*
5989	* Return:
5990	* %true if congested, %false otherwise.
5991	*/
5992	bool workqueue_congested(int cpu, struct workqueue_struct *wq)
5993	{
5994	struct pool_workqueue *pwq;
5995	bool ret;
5996
5997	rcu_read_lock();
5998	preempt_disable();
5999
6000	if (cpu == WORK_CPU_UNBOUND)
6001	cpu = smp_processor_id();
6002
6003	pwq = *per_cpu_ptr(wq->cpu_pwq, cpu);
6004	ret = !list_empty(&pwq->inactive_works);
6005
6006	preempt_enable();
6007	rcu_read_unlock();
6008
6009	return ret;
6010	}
6011	EXPORT_SYMBOL_GPL(workqueue_congested);
6012
6013	/**
6014	* work_busy - test whether a work is currently pending or running
6015	* @work: the work to be tested
6016	*
6017	* Test whether @work is currently pending or running. There is no
6018	* synchronization around this function and the test result is
6019	* unreliable and only useful as advisory hints or for debugging.
6020	*
6021	* Return:
6022	* OR'd bitmask of WORK_BUSY_* bits.
6023	*/
6024	unsigned int work_busy(struct work_struct *work)
6025	{
6026	struct worker_pool *pool;
6027	unsigned long irq_flags;
6028	unsigned int ret = 0;
6029
6030	if (work_pending(work))
6031	ret \|= WORK_BUSY_PENDING;
6032
6033	rcu_read_lock();
6034	pool = get_work_pool(work);
6035	if (pool) {
6036	raw_spin_lock_irqsave(&pool->lock, irq_flags);
6037	if (find_worker_executing_work(pool, work))
6038	ret \|= WORK_BUSY_RUNNING;
6039	raw_spin_unlock_irqrestore(&pool->lock, irq_flags);
6040	}
6041	rcu_read_unlock();
6042
6043	return ret;
6044	}
6045	EXPORT_SYMBOL_GPL(work_busy);
6046
6047	/**
6048	* set_worker_desc - set description for the current work item
6049	* @fmt: printf-style format string
6050	* @...: arguments for the format string
6051	*
6052	* This function can be called by a running work function to describe what
6053	* the work item is about. If the worker task gets dumped, this
6054	* information will be printed out together to help debugging. The
6055	* description can be at most WORKER_DESC_LEN including the trailing '\0'.
6056	*/
6057	void set_worker_desc(const char *fmt, ...)
6058	{
6059	struct worker *worker = current_wq_worker();
6060	va_list args;
6061
6062	if (worker) {
6063	va_start(args, fmt);
6064	vsnprintf(worker->desc, sizeof(worker->desc), fmt, args);
6065	va_end(args);
6066	}
6067	}
6068	EXPORT_SYMBOL_GPL(set_worker_desc);
6069
6070	/**
6071	* print_worker_info - print out worker information and description
6072	* @log_lvl: the log level to use when printing
6073	* @task: target task
6074	*
6075	* If @task is a worker and currently executing a work item, print out the
6076	* name of the workqueue being serviced and worker description set with
6077	* set_worker_desc() by the currently executing work item.
6078	*
6079	* This function can be safely called on any task as long as the
6080	* task_struct itself is accessible. While safe, this function isn't
6081	* synchronized and may print out mixups or garbages of limited length.
6082	*/
6083	void print_worker_info(const char log_lvl, struct task_struct task)
6084	{
6085	work_func_t *fn = NULL;
6086	char name[WQ_NAME_LEN] = { };
6087	char desc[WORKER_DESC_LEN] = { };
6088	struct pool_workqueue *pwq = NULL;
6089	struct workqueue_struct *wq = NULL;
6090	struct worker *worker;
6091
6092	if (!(task->flags & PF_WQ_WORKER))
6093	return;
6094
6095	/*
6096	* This function is called without any synchronization and @task
6097	* could be in any state. Be careful with dereferences.
6098	*/
6099	worker = kthread_probe_data(task);
6100
6101	/*
6102	* Carefully copy the associated workqueue's workfn, name and desc.
6103	* Keep the original last '\0' in case the original is garbage.
6104	*/
6105	copy_from_kernel_nofault(&fn, &worker->current_func, sizeof(fn));
6106	copy_from_kernel_nofault(&pwq, &worker->current_pwq, sizeof(pwq));
6107	copy_from_kernel_nofault(&wq, &pwq->wq, sizeof(wq));
6108	copy_from_kernel_nofault(name, wq->name, sizeof(name) - 1);
6109	copy_from_kernel_nofault(desc, worker->desc, sizeof(desc) - 1);
6110
6111	if (fn \|\| name[0] \|\| desc[0]) {
6112	printk("%sWorkqueue: %s %ps", log_lvl, name, fn);
6113	if (strcmp(name, desc))
6114	pr_cont(" (%s)", desc);
6115	pr_cont("\n");
6116	}
6117	}
6118
6119	static void pr_cont_pool_info(struct worker_pool *pool)
6120	{
6121	pr_cont(" cpus=%*pbl", nr_cpumask_bits, pool->attrs->cpumask);
6122	if (pool->node != NUMA_NO_NODE)
6123	pr_cont(" node=%d", pool->node);
6124	pr_cont(" flags=0x%x", pool->flags);
6125	if (pool->flags & POOL_BH)
6126	pr_cont(" bh%s",
6127	pool->attrs->nice == HIGHPRI_NICE_LEVEL ? "-hi" : "");
6128	else
6129	pr_cont(" nice=%d", pool->attrs->nice);
6130	}
6131
6132	static void pr_cont_worker_id(struct worker *worker)
6133	{
6134	struct worker_pool *pool = worker->pool;
6135
6136	if (pool->flags & WQ_BH)
6137	pr_cont("bh%s",
6138	pool->attrs->nice == HIGHPRI_NICE_LEVEL ? "-hi" : "");
6139	else
6140	pr_cont("%d%s", task_pid_nr(worker->task),
6141	worker->rescue_wq ? "(RESCUER)" : "");
6142	}
6143
6144	struct pr_cont_work_struct {
6145	bool comma;
6146	work_func_t func;
6147	long ctr;
6148	};
6149
6150	static void pr_cont_work_flush(bool comma, work_func_t func, struct pr_cont_work_struct *pcwsp)
6151	{
6152	if (!pcwsp->ctr)
6153	goto out_record;
6154	if (func == pcwsp->func) {
6155	pcwsp->ctr++;
6156	return;
6157	}
6158	if (pcwsp->ctr == 1)
6159	pr_cont("%s %ps", pcwsp->comma ? "," : "", pcwsp->func);
6160	else
6161	pr_cont("%s %ld*%ps", pcwsp->comma ? "," : "", pcwsp->ctr, pcwsp->func);
6162	pcwsp->ctr = 0;
6163	out_record:
6164	if ((long)func == -1L)
6165	return;
6166	pcwsp->comma = comma;
6167	pcwsp->func = func;
6168	pcwsp->ctr = 1;
6169	}
6170
6171	static void pr_cont_work(bool comma, struct work_struct work, struct pr_cont_work_struct pcwsp)
6172	{
6173	if (work->func == wq_barrier_func) {
6174	struct wq_barrier *barr;
6175
6176	barr = container_of(work, struct wq_barrier, work);
6177
6178	pr_cont_work_flush(comma, (work_func_t)-1, pcwsp);
6179	pr_cont("%s BAR(%d)", comma ? "," : "",
6180	task_pid_nr(barr->task));
6181	} else {
6182	if (!comma)
6183	pr_cont_work_flush(comma, (work_func_t)-1, pcwsp);
6184	pr_cont_work_flush(comma, work->func, pcwsp);
6185	}
6186	}
6187
6188	static void show_pwq(struct pool_workqueue *pwq)
6189	{
6190	struct pr_cont_work_struct pcws = { .ctr = 0, };
6191	struct worker_pool *pool = pwq->pool;
6192	struct work_struct *work;
6193	struct worker *worker;
6194	bool has_in_flight = false, has_pending = false;
6195	int bkt;
6196
6197	pr_info(" pwq %d:", pool->id);
6198	pr_cont_pool_info(pool);
6199
6200	pr_cont(" active=%d refcnt=%d%s\n",
6201	pwq->nr_active, pwq->refcnt,
6202	!list_empty(&pwq->mayday_node) ? " MAYDAY" : "");
6203
6204	hash_for_each(pool->busy_hash, bkt, worker, hentry) {
6205	if (worker->current_pwq == pwq) {
6206	has_in_flight = true;
6207	break;
6208	}
6209	}
6210	if (has_in_flight) {
6211	bool comma = false;
6212
6213	pr_info(" in-flight:");
6214	hash_for_each(pool->busy_hash, bkt, worker, hentry) {
6215	if (worker->current_pwq != pwq)
6216	continue;
6217
6218	pr_cont(" %s", comma ? "," : "");
6219	pr_cont_worker_id(worker);
6220	pr_cont(":%ps", worker->current_func);
6221	list_for_each_entry(work, &worker->scheduled, entry)
6222	pr_cont_work(false, work, &pcws);
6223	pr_cont_work_flush(comma, (work_func_t)-1L, &pcws);
6224	comma = true;
6225	}
6226	pr_cont("\n");
6227	}
6228
6229	list_for_each_entry(work, &pool->worklist, entry) {
6230	if (get_work_pwq(work) == pwq) {
6231	has_pending = true;
6232	break;
6233	}
6234	}
6235	if (has_pending) {
6236	bool comma = false;
6237
6238	pr_info(" pending:");
6239	list_for_each_entry(work, &pool->worklist, entry) {
6240	if (get_work_pwq(work) != pwq)
6241	continue;
6242
6243	pr_cont_work(comma, work, &pcws);
6244	comma = !(*work_data_bits(work) & WORK_STRUCT_LINKED);
6245	}
6246	pr_cont_work_flush(comma, (work_func_t)-1L, &pcws);
6247	pr_cont("\n");
6248	}
6249
6250	if (!list_empty(&pwq->inactive_works)) {
6251	bool comma = false;
6252
6253	pr_info(" inactive:");
6254	list_for_each_entry(work, &pwq->inactive_works, entry) {
6255	pr_cont_work(comma, work, &pcws);
6256	comma = !(*work_data_bits(work) & WORK_STRUCT_LINKED);
6257	}
6258	pr_cont_work_flush(comma, (work_func_t)-1L, &pcws);
6259	pr_cont("\n");
6260	}
6261	}
6262
6263	/**
6264	* show_one_workqueue - dump state of specified workqueue
6265	* @wq: workqueue whose state will be printed
6266	*/
6267	void show_one_workqueue(struct workqueue_struct *wq)
6268	{
6269	struct pool_workqueue *pwq;
6270	bool idle = true;
6271	unsigned long irq_flags;
6272
6273	for_each_pwq(pwq, wq) {
6274	if (!pwq_is_empty(pwq)) {
6275	idle = false;
6276	break;
6277	}
6278	}
6279	if (idle) /* Nothing to print for idle workqueue */
6280	return;
6281
6282	pr_info("workqueue %s: flags=0x%x\n", wq->name, wq->flags);
6283
6284	for_each_pwq(pwq, wq) {
6285	raw_spin_lock_irqsave(&pwq->pool->lock, irq_flags);
6286	if (!pwq_is_empty(pwq)) {
6287	/*
6288	* Defer printing to avoid deadlocks in console
6289	* drivers that queue work while holding locks
6290	* also taken in their write paths.
6291	*/
6292	printk_deferred_enter();
6293	show_pwq(pwq);
6294	printk_deferred_exit();
6295	}
6296	raw_spin_unlock_irqrestore(&pwq->pool->lock, irq_flags);
6297	/*
6298	* We could be printing a lot from atomic context, e.g.
6299	* sysrq-t -> show_all_workqueues(). Avoid triggering
6300	* hard lockup.
6301	*/
6302	touch_nmi_watchdog();
6303	}
6304
6305	}
6306
6307	/**
6308	* show_one_worker_pool - dump state of specified worker pool
6309	* @pool: worker pool whose state will be printed
6310	*/
6311	static void show_one_worker_pool(struct worker_pool *pool)
6312	{
6313	struct worker *worker;
6314	bool first = true;
6315	unsigned long irq_flags;
6316	unsigned long hung = 0;
6317
6318	raw_spin_lock_irqsave(&pool->lock, irq_flags);
6319	if (pool->nr_workers == pool->nr_idle)
6320	goto next_pool;
6321
6322	/* How long the first pending work is waiting for a worker. */
6323	if (!list_empty(&pool->worklist))
6324	hung = jiffies_to_msecs(jiffies - pool->watchdog_ts) / 1000;
6325
6326	/*
6327	* Defer printing to avoid deadlocks in console drivers that
6328	* queue work while holding locks also taken in their write
6329	* paths.
6330	*/
6331	printk_deferred_enter();
6332	pr_info("pool %d:", pool->id);
6333	pr_cont_pool_info(pool);
6334	pr_cont(" hung=%lus workers=%d", hung, pool->nr_workers);
6335	if (pool->manager)
6336	pr_cont(" manager: %d",
6337	task_pid_nr(pool->manager->task));
6338	list_for_each_entry(worker, &pool->idle_list, entry) {
6339	pr_cont(" %s", first ? "idle: " : "");
6340	pr_cont_worker_id(worker);
6341	first = false;
6342	}
6343	pr_cont("\n");
6344	printk_deferred_exit();
6345	next_pool:
6346	raw_spin_unlock_irqrestore(&pool->lock, irq_flags);
6347	/*
6348	* We could be printing a lot from atomic context, e.g.
6349	* sysrq-t -> show_all_workqueues(). Avoid triggering
6350	* hard lockup.
6351	*/
6352	touch_nmi_watchdog();
6353
6354	}
6355
6356	/**
6357	* show_all_workqueues - dump workqueue state
6358	*
6359	* Called from a sysrq handler and prints out all busy workqueues and pools.
6360	*/
6361	void show_all_workqueues(void)
6362	{
6363	struct workqueue_struct *wq;
6364	struct worker_pool *pool;
6365	int pi;
6366
6367	rcu_read_lock();
6368
6369	pr_info("Showing busy workqueues and worker pools:\n");
6370
6371	list_for_each_entry_rcu(wq, &workqueues, list)
6372	show_one_workqueue(wq);
6373
6374	for_each_pool(pool, pi)
6375	show_one_worker_pool(pool);
6376
6377	rcu_read_unlock();
6378	}
6379
6380	/**
6381	* show_freezable_workqueues - dump freezable workqueue state
6382	*
6383	* Called from try_to_freeze_tasks() and prints out all freezable workqueues
6384	* still busy.
6385	*/
6386	void show_freezable_workqueues(void)
6387	{
6388	struct workqueue_struct *wq;
6389
6390	rcu_read_lock();
6391
6392	pr_info("Showing freezable workqueues that are still busy:\n");
6393
6394	list_for_each_entry_rcu(wq, &workqueues, list) {
6395	if (!(wq->flags & WQ_FREEZABLE))
6396	continue;
6397	show_one_workqueue(wq);
6398	}
6399
6400	rcu_read_unlock();
6401	}
6402
6403	/* used to show worker information through /proc/PID/{comm,stat,status} */
6404	void wq_worker_comm(char buf, size_t size, struct task_struct task)
6405	{
6406	/* stabilize PF_WQ_WORKER and worker pool association */
6407	mutex_lock(&wq_pool_attach_mutex);
6408
6409	if (task->flags & PF_WQ_WORKER) {
6410	struct worker *worker = kthread_data(task);
6411	struct worker_pool *pool = worker->pool;
6412	int off;
6413
6414	off = format_worker_id(buf, size, worker, pool);
6415
6416	if (pool) {
6417	raw_spin_lock_irq(&pool->lock);
6418	/*
6419	* ->desc tracks information (wq name or
6420	* set_worker_desc()) for the latest execution. If
6421	* current, prepend '+', otherwise '-'.
6422	*/
6423	if (worker->desc[0] != '\0') {
6424	if (worker->current_work)
6425	scnprintf(buf + off, size - off, "+%s",
6426	worker->desc);
6427	else
6428	scnprintf(buf + off, size - off, "-%s",
6429	worker->desc);
6430	}
6431	raw_spin_unlock_irq(&pool->lock);
6432	}
6433	} else {
6434	strscpy(buf, task->comm, size);
6435	}
6436
6437	mutex_unlock(&wq_pool_attach_mutex);
6438	}
6439
6440	#ifdef CONFIG_SMP
6441
6442	/*
6443	* CPU hotplug.
6444	*
6445	* There are two challenges in supporting CPU hotplug. Firstly, there
6446	* are a lot of assumptions on strong associations among work, pwq and
6447	* pool which make migrating pending and scheduled works very
6448	* difficult to implement without impacting hot paths. Secondly,
6449	* worker pools serve mix of short, long and very long running works making
6450	* blocked draining impractical.
6451	*
6452	* This is solved by allowing the pools to be disassociated from the CPU
6453	* running as an unbound one and allowing it to be reattached later if the
6454	* cpu comes back online.
6455	*/
6456
6457	static void unbind_workers(int cpu)
6458	{
6459	struct worker_pool *pool;
6460	struct worker *worker;
6461
6462	for_each_cpu_worker_pool(pool, cpu) {
6463	mutex_lock(&wq_pool_attach_mutex);
6464	raw_spin_lock_irq(&pool->lock);
6465
6466	/*
6467	* We've blocked all attach/detach operations. Make all workers
6468	* unbound and set DISASSOCIATED. Before this, all workers
6469	* must be on the cpu. After this, they may become diasporas.
6470	* And the preemption disabled section in their sched callbacks
6471	* are guaranteed to see WORKER_UNBOUND since the code here
6472	* is on the same cpu.
6473	*/
6474	for_each_pool_worker(worker, pool)
6475	worker->flags \|= WORKER_UNBOUND;
6476
6477	pool->flags \|= POOL_DISASSOCIATED;
6478
6479	/*
6480	* The handling of nr_running in sched callbacks are disabled
6481	* now. Zap nr_running. After this, nr_running stays zero and
6482	* need_more_worker() and keep_working() are always true as
6483	* long as the worklist is not empty. This pool now behaves as
6484	* an unbound (in terms of concurrency management) pool which
6485	* are served by workers tied to the pool.
6486	*/
6487	pool->nr_running = 0;
6488
6489	/*
6490	* With concurrency management just turned off, a busy
6491	* worker blocking could lead to lengthy stalls. Kick off
6492	* unbound chain execution of currently pending work items.
6493	*/
6494	kick_pool(pool);
6495
6496	raw_spin_unlock_irq(&pool->lock);
6497
6498	for_each_pool_worker(worker, pool)
6499	unbind_worker(worker);
6500
6501	mutex_unlock(&wq_pool_attach_mutex);
6502	}
6503	}
6504
6505	/**
6506	* rebind_workers - rebind all workers of a pool to the associated CPU
6507	* @pool: pool of interest
6508	*
6509	* @pool->cpu is coming online. Rebind all workers to the CPU.
6510	*/
6511	static void rebind_workers(struct worker_pool *pool)
6512	{
6513	struct worker *worker;
6514
6515	lockdep_assert_held(&wq_pool_attach_mutex);
6516
6517	/*
6518	* Restore CPU affinity of all workers. As all idle workers should
6519	* be on the run-queue of the associated CPU before any local
6520	* wake-ups for concurrency management happen, restore CPU affinity
6521	* of all workers first and then clear UNBOUND. As we're called
6522	* from CPU_ONLINE, the following shouldn't fail.
6523	*/
6524	for_each_pool_worker(worker, pool) {
6525	kthread_set_per_cpu(worker->task, pool->cpu);
6526	WARN_ON_ONCE(set_cpus_allowed_ptr(worker->task,
6527	pool_allowed_cpus(pool)) < 0);
6528	}
6529
6530	raw_spin_lock_irq(&pool->lock);
6531
6532	pool->flags &= ~POOL_DISASSOCIATED;
6533
6534	for_each_pool_worker(worker, pool) {
6535	unsigned int worker_flags = worker->flags;
6536
6537	/*
6538	* We want to clear UNBOUND but can't directly call
6539	* worker_clr_flags() or adjust nr_running. Atomically
6540	* replace UNBOUND with another NOT_RUNNING flag REBOUND.
6541	* @worker will clear REBOUND using worker_clr_flags() when
6542	* it initiates the next execution cycle thus restoring
6543	* concurrency management. Note that when or whether
6544	* @worker clears REBOUND doesn't affect correctness.
6545	*
6546	* WRITE_ONCE() is necessary because @worker->flags may be
6547	* tested without holding any lock in
6548	* wq_worker_running(). Without it, NOT_RUNNING test may
6549	* fail incorrectly leading to premature concurrency
6550	* management operations.
6551	*/
6552	WARN_ON_ONCE(!(worker_flags & WORKER_UNBOUND));
6553	worker_flags \|= WORKER_REBOUND;
6554	worker_flags &= ~WORKER_UNBOUND;
6555	WRITE_ONCE(worker->flags, worker_flags);
6556	}
6557
6558	raw_spin_unlock_irq(&pool->lock);
6559	}
6560
6561	/**
6562	* restore_unbound_workers_cpumask - restore cpumask of unbound workers
6563	* @pool: unbound pool of interest
6564	* @cpu: the CPU which is coming up
6565	*
6566	* An unbound pool may end up with a cpumask which doesn't have any online
6567	* CPUs. When a worker of such pool get scheduled, the scheduler resets
6568	* its cpus_allowed. If @cpu is in @pool's cpumask which didn't have any
6569	* online CPU before, cpus_allowed of all its workers should be restored.
6570	*/
6571	static void restore_unbound_workers_cpumask(struct worker_pool *pool, int cpu)
6572	{
6573	static cpumask_t cpumask;
6574	struct worker *worker;
6575
6576	lockdep_assert_held(&wq_pool_attach_mutex);
6577
6578	/* is @cpu allowed for @pool? */
6579	if (!cpumask_test_cpu(cpu, pool->attrs->cpumask))
6580	return;
6581
6582	cpumask_and(&cpumask, pool->attrs->cpumask, cpu_online_mask);
6583
6584	/* as we're called from CPU_ONLINE, the following shouldn't fail */
6585	for_each_pool_worker(worker, pool)
6586	WARN_ON_ONCE(set_cpus_allowed_ptr(worker->task, &cpumask) < 0);
6587	}
6588
6589	int workqueue_prepare_cpu(unsigned int cpu)
6590	{
6591	struct worker_pool *pool;
6592
6593	for_each_cpu_worker_pool(pool, cpu) {
6594	if (pool->nr_workers)
6595	continue;
6596	if (!create_worker(pool))
6597	return -ENOMEM;
6598	}
6599	return 0;
6600	}
6601
6602	int workqueue_online_cpu(unsigned int cpu)
6603	{
6604	struct worker_pool *pool;
6605	struct workqueue_struct *wq;
6606	int pi;
6607
6608	mutex_lock(&wq_pool_mutex);
6609
6610	for_each_pool(pool, pi) {
6611	/* BH pools aren't affected by hotplug */
6612	if (pool->flags & POOL_BH)
6613	continue;
6614
6615	mutex_lock(&wq_pool_attach_mutex);
6616	if (pool->cpu == cpu)
6617	rebind_workers(pool);
6618	else if (pool->cpu < 0)
6619	restore_unbound_workers_cpumask(pool, cpu);
6620	mutex_unlock(&wq_pool_attach_mutex);
6621	}
6622
6623	/* update pod affinity of unbound workqueues */
6624	list_for_each_entry(wq, &workqueues, list) {
6625	struct workqueue_attrs *attrs = wq->unbound_attrs;
6626
6627	if (attrs) {
6628	const struct wq_pod_type *pt = wqattrs_pod_type(attrs);
6629	int tcpu;
6630
6631	for_each_cpu(tcpu, pt->pod_cpus[pt->cpu_pod[cpu]])
6632	wq_update_pod(wq, tcpu, cpu, true);
6633
6634	mutex_lock(&wq->mutex);
6635	wq_update_node_max_active(wq, -1);
6636	mutex_unlock(&wq->mutex);
6637	}
6638	}
6639
6640	mutex_unlock(&wq_pool_mutex);
6641	return 0;
6642	}
6643
6644	int workqueue_offline_cpu(unsigned int cpu)
6645	{
6646	struct workqueue_struct *wq;
6647
6648	/* unbinding per-cpu workers should happen on the local CPU */
6649	if (WARN_ON(cpu != smp_processor_id()))
6650	return -1;
6651
6652	unbind_workers(cpu);
6653
6654	/* update pod affinity of unbound workqueues */
6655	mutex_lock(&wq_pool_mutex);
6656	list_for_each_entry(wq, &workqueues, list) {
6657	struct workqueue_attrs *attrs = wq->unbound_attrs;
6658
6659	if (attrs) {
6660	const struct wq_pod_type *pt = wqattrs_pod_type(attrs);
6661	int tcpu;
6662
6663	for_each_cpu(tcpu, pt->pod_cpus[pt->cpu_pod[cpu]])
6664	wq_update_pod(wq, tcpu, cpu, false);
6665
6666	mutex_lock(&wq->mutex);
6667	wq_update_node_max_active(wq, cpu);
6668	mutex_unlock(&wq->mutex);
6669	}
6670	}
6671	mutex_unlock(&wq_pool_mutex);
6672
6673	return 0;
6674	}
6675
6676	struct work_for_cpu {
6677	struct work_struct work;
6678	long (fn)(void );
6679	void *arg;
6680	long ret;
6681	};
6682
6683	static void work_for_cpu_fn(struct work_struct *work)
6684	{
6685	struct work_for_cpu *wfc = container_of(work, struct work_for_cpu, work);
6686
6687	wfc->ret = wfc->fn(wfc->arg);
6688	}
6689
6690	/**
6691	* work_on_cpu_key - run a function in thread context on a particular cpu
6692	* @cpu: the cpu to run on
6693	* @fn: the function to run
6694	* @arg: the function arg
6695	* @key: The lock class key for lock debugging purposes
6696	*
6697	* It is up to the caller to ensure that the cpu doesn't go offline.
6698	* The caller must not hold any locks which would prevent @fn from completing.
6699	*
6700	* Return: The value @fn returns.
6701	*/
6702	long work_on_cpu_key(int cpu, long (fn)(void ),
6703	void arg, struct lock_class_key key)
6704	{
6705	struct work_for_cpu wfc = { .fn = fn, .arg = arg };
6706
6707	INIT_WORK_ONSTACK_KEY(&wfc.work, work_for_cpu_fn, key);
6708	schedule_work_on(cpu, &wfc.work);
6709	flush_work(&wfc.work);
6710	destroy_work_on_stack(&wfc.work);
6711	return wfc.ret;
6712	}
6713	EXPORT_SYMBOL_GPL(work_on_cpu_key);
6714
6715	/**
6716	* work_on_cpu_safe_key - run a function in thread context on a particular cpu
6717	* @cpu: the cpu to run on
6718	* @fn: the function to run
6719	* @arg: the function argument
6720	* @key: The lock class key for lock debugging purposes
6721	*
6722	* Disables CPU hotplug and calls work_on_cpu(). The caller must not hold
6723	* any locks which would prevent @fn from completing.
6724	*
6725	* Return: The value @fn returns.
6726	*/
6727	long work_on_cpu_safe_key(int cpu, long (fn)(void ),
6728	void arg, struct lock_class_key key)
6729	{
6730	long ret = -ENODEV;
6731
6732	cpus_read_lock();
6733	if (cpu_online(cpu))
6734	ret = work_on_cpu_key(cpu, fn, arg, key);
6735	cpus_read_unlock();
6736	return ret;
6737	}
6738	EXPORT_SYMBOL_GPL(work_on_cpu_safe_key);
6739	#endif /* CONFIG_SMP */
6740
6741	#ifdef CONFIG_FREEZER
6742
6743	/**
6744	* freeze_workqueues_begin - begin freezing workqueues
6745	*
6746	* Start freezing workqueues. After this function returns, all freezable
6747	* workqueues will queue new works to their inactive_works list instead of
6748	* pool->worklist.
6749	*
6750	* CONTEXT:
6751	* Grabs and releases wq_pool_mutex, wq->mutex and pool->lock's.
6752	*/
6753	void freeze_workqueues_begin(void)
6754	{
6755	struct workqueue_struct *wq;
6756
6757	mutex_lock(&wq_pool_mutex);
6758
6759	WARN_ON_ONCE(workqueue_freezing);
6760	workqueue_freezing = true;
6761
6762	list_for_each_entry(wq, &workqueues, list) {
6763	mutex_lock(&wq->mutex);
6764	wq_adjust_max_active(wq);
6765	mutex_unlock(&wq->mutex);
6766	}
6767
6768	mutex_unlock(&wq_pool_mutex);
6769	}
6770
6771	/**
6772	* freeze_workqueues_busy - are freezable workqueues still busy?
6773	*
6774	* Check whether freezing is complete. This function must be called
6775	* between freeze_workqueues_begin() and thaw_workqueues().
6776	*
6777	* CONTEXT:
6778	* Grabs and releases wq_pool_mutex.
6779	*
6780	* Return:
6781	* %true if some freezable workqueues are still busy. %false if freezing
6782	* is complete.
6783	*/
6784	bool freeze_workqueues_busy(void)
6785	{
6786	bool busy = false;
6787	struct workqueue_struct *wq;
6788	struct pool_workqueue *pwq;
6789
6790	mutex_lock(&wq_pool_mutex);
6791
6792	WARN_ON_ONCE(!workqueue_freezing);
6793
6794	list_for_each_entry(wq, &workqueues, list) {
6795	if (!(wq->flags & WQ_FREEZABLE))
6796	continue;
6797	/*
6798	* nr_active is monotonically decreasing. It's safe
6799	* to peek without lock.
6800	*/
6801	rcu_read_lock();
6802	for_each_pwq(pwq, wq) {
6803	WARN_ON_ONCE(pwq->nr_active < 0);
6804	if (pwq->nr_active) {
6805	busy = true;
6806	rcu_read_unlock();
6807	goto out_unlock;
6808	}
6809	}
6810	rcu_read_unlock();
6811	}
6812	out_unlock:
6813	mutex_unlock(&wq_pool_mutex);
6814	return busy;
6815	}
6816
6817	/**
6818	* thaw_workqueues - thaw workqueues
6819	*
6820	* Thaw workqueues. Normal queueing is restored and all collected
6821	* frozen works are transferred to their respective pool worklists.
6822	*
6823	* CONTEXT:
6824	* Grabs and releases wq_pool_mutex, wq->mutex and pool->lock's.
6825	*/
6826	void thaw_workqueues(void)
6827	{
6828	struct workqueue_struct *wq;
6829
6830	mutex_lock(&wq_pool_mutex);
6831
6832	if (!workqueue_freezing)
6833	goto out_unlock;
6834
6835	workqueue_freezing = false;
6836
6837	/* restore max_active and repopulate worklist */
6838	list_for_each_entry(wq, &workqueues, list) {
6839	mutex_lock(&wq->mutex);
6840	wq_adjust_max_active(wq);
6841	mutex_unlock(&wq->mutex);
6842	}
6843
6844	out_unlock:
6845	mutex_unlock(&wq_pool_mutex);
6846	}
6847	#endif /* CONFIG_FREEZER */
6848
6849	static int workqueue_apply_unbound_cpumask(const cpumask_var_t unbound_cpumask)
6850	{
6851	LIST_HEAD(ctxs);
6852	int ret = 0;
6853	struct workqueue_struct *wq;
6854	struct apply_wqattrs_ctx ctx, n;
6855
6856	lockdep_assert_held(&wq_pool_mutex);
6857
6858	list_for_each_entry(wq, &workqueues, list) {
6859	if (!(wq->flags & WQ_UNBOUND) \|\| (wq->flags & __WQ_DESTROYING))
6860	continue;
6861
6862	ctx = apply_wqattrs_prepare(wq, wq->unbound_attrs, unbound_cpumask);
6863	if (IS_ERR(ctx)) {
6864	ret = PTR_ERR(ctx);
6865	break;
6866	}
6867
6868	list_add_tail(&ctx->list, &ctxs);
6869	}
6870
6871	list_for_each_entry_safe(ctx, n, &ctxs, list) {
6872	if (!ret)
6873	apply_wqattrs_commit(ctx);
6874	apply_wqattrs_cleanup(ctx);
6875	}
6876
6877	if (!ret) {
6878	mutex_lock(&wq_pool_attach_mutex);
6879	cpumask_copy(wq_unbound_cpumask, unbound_cpumask);
6880	mutex_unlock(&wq_pool_attach_mutex);
6881	}
6882	return ret;
6883	}
6884
6885	/**
6886	* workqueue_unbound_exclude_cpumask - Exclude given CPUs from unbound cpumask
6887	* @exclude_cpumask: the cpumask to be excluded from wq_unbound_cpumask
6888	*
6889	* This function can be called from cpuset code to provide a set of isolated
6890	* CPUs that should be excluded from wq_unbound_cpumask. The caller must hold
6891	* either cpus_read_lock or cpus_write_lock.
6892	*/
6893	int workqueue_unbound_exclude_cpumask(cpumask_var_t exclude_cpumask)
6894	{
6895	cpumask_var_t cpumask;
6896	int ret = 0;
6897
6898	if (!zalloc_cpumask_var(&cpumask, GFP_KERNEL))
6899	return -ENOMEM;
6900
6901	lockdep_assert_cpus_held();
6902	mutex_lock(&wq_pool_mutex);
6903
6904	/* Save the current isolated cpumask & export it via sysfs */
6905	cpumask_copy(wq_isolated_cpumask, exclude_cpumask);
6906
6907	/*
6908	* If the operation fails, it will fall back to
6909	* wq_requested_unbound_cpumask which is initially set to
6910	* (HK_TYPE_WQ ∩ HK_TYPE_DOMAIN) house keeping mask and rewritten
6911	* by any subsequent write to workqueue/cpumask sysfs file.
6912	*/
6913	if (!cpumask_andnot(cpumask, wq_requested_unbound_cpumask, exclude_cpumask))
6914	cpumask_copy(cpumask, wq_requested_unbound_cpumask);
6915	if (!cpumask_equal(cpumask, wq_unbound_cpumask))
6916	ret = workqueue_apply_unbound_cpumask(cpumask);
6917
6918	mutex_unlock(&wq_pool_mutex);
6919	free_cpumask_var(cpumask);
6920	return ret;
6921	}
6922
6923	static int parse_affn_scope(const char *val)
6924	{
6925	int i;
6926
6927	for (i = 0; i < ARRAY_SIZE(wq_affn_names); i++) {
6928	if (!strncasecmp(val, wq_affn_names[i], strlen(wq_affn_names[i])))
6929	return i;
6930	}
6931	return -EINVAL;
6932	}
6933
6934	static int wq_affn_dfl_set(const char val, const struct kernel_param kp)
6935	{
6936	struct workqueue_struct *wq;
6937	int affn, cpu;
6938
6939	affn = parse_affn_scope(val);
6940	if (affn < 0)
6941	return affn;
6942	if (affn == WQ_AFFN_DFL)
6943	return -EINVAL;
6944
6945	cpus_read_lock();
6946	mutex_lock(&wq_pool_mutex);
6947
6948	wq_affn_dfl = affn;
6949
6950	list_for_each_entry(wq, &workqueues, list) {
6951	for_each_online_cpu(cpu) {
6952	wq_update_pod(wq, cpu, cpu, true);
6953	}
6954	}
6955
6956	mutex_unlock(&wq_pool_mutex);
6957	cpus_read_unlock();
6958
6959	return 0;
6960	}
6961
6962	static int wq_affn_dfl_get(char buffer, const struct kernel_param kp)
6963	{
6964	return scnprintf(buffer, PAGE_SIZE, "%s\n", wq_affn_names[wq_affn_dfl]);
6965	}
6966
6967	static const struct kernel_param_ops wq_affn_dfl_ops = {
6968	.set = wq_affn_dfl_set,
6969	.get = wq_affn_dfl_get,
6970	};
6971
6972	module_param_cb(default_affinity_scope, &wq_affn_dfl_ops, NULL, 0644);
6973
6974	#ifdef CONFIG_SYSFS
6975	/*
6976	* Workqueues with WQ_SYSFS flag set is visible to userland via
6977	* /sys/bus/workqueue/devices/WQ_NAME. All visible workqueues have the
6978	* following attributes.
6979	*
6980	* per_cpu RO bool : whether the workqueue is per-cpu or unbound
6981	* max_active RW int : maximum number of in-flight work items
6982	*
6983	* Unbound workqueues have the following extra attributes.
6984	*
6985	* nice RW int : nice value of the workers
6986	* cpumask RW mask : bitmask of allowed CPUs for the workers
6987	* affinity_scope RW str : worker CPU affinity scope (cache, numa, none)
6988	* affinity_strict RW bool : worker CPU affinity is strict
6989	*/
6990	struct wq_device {
6991	struct workqueue_struct *wq;
6992	struct device dev;
6993	};
6994
6995	static struct workqueue_struct dev_to_wq(struct device dev)
6996	{
6997	struct wq_device *wq_dev = container_of(dev, struct wq_device, dev);
6998
6999	return wq_dev->wq;
7000	}
7001
7002	static ssize_t per_cpu_show(struct device dev, struct device_attribute attr,
7003	char *buf)
7004	{
7005	struct workqueue_struct *wq = dev_to_wq(dev);
7006
7007	return scnprintf(buf, PAGE_SIZE, "%d\n", (bool)!(wq->flags & WQ_UNBOUND));
7008	}
7009	static DEVICE_ATTR_RO(per_cpu);
7010
7011	static ssize_t max_active_show(struct device *dev,
7012	struct device_attribute attr, char buf)
7013	{
7014	struct workqueue_struct *wq = dev_to_wq(dev);
7015
7016	return scnprintf(buf, PAGE_SIZE, "%d\n", wq->saved_max_active);
7017	}
7018
7019	static ssize_t max_active_store(struct device *dev,
7020	struct device_attribute attr, const char buf,
7021	size_t count)
7022	{
7023	struct workqueue_struct *wq = dev_to_wq(dev);
7024	int val;
7025
7026	if (sscanf(buf, "%d", &val) != 1 \|\| val <= 0)
7027	return -EINVAL;
7028
7029	workqueue_set_max_active(wq, val);
7030	return count;
7031	}
7032	static DEVICE_ATTR_RW(max_active);
7033
7034	static struct attribute *wq_sysfs_attrs[] = {
7035	&dev_attr_per_cpu.attr,
7036	&dev_attr_max_active.attr,
7037	NULL,
7038	};
7039	ATTRIBUTE_GROUPS(wq_sysfs);
7040
7041	static void apply_wqattrs_lock(void)
7042	{
7043	/* CPUs should stay stable across pwq creations and installations */
7044	cpus_read_lock();
7045	mutex_lock(&wq_pool_mutex);
7046	}
7047
7048	static void apply_wqattrs_unlock(void)
7049	{
7050	mutex_unlock(&wq_pool_mutex);
7051	cpus_read_unlock();
7052	}
7053
7054	static ssize_t wq_nice_show(struct device dev, struct device_attribute attr,
7055	char *buf)
7056	{
7057	struct workqueue_struct *wq = dev_to_wq(dev);
7058	int written;
7059
7060	mutex_lock(&wq->mutex);
7061	written = scnprintf(buf, PAGE_SIZE, "%d\n", wq->unbound_attrs->nice);
7062	mutex_unlock(&wq->mutex);
7063
7064	return written;
7065	}
7066
7067	/* prepare workqueue_attrs for sysfs store operations */
7068	static struct workqueue_attrs wq_sysfs_prep_attrs(struct workqueue_struct wq)
7069	{
7070	struct workqueue_attrs *attrs;
7071
7072	lockdep_assert_held(&wq_pool_mutex);
7073
7074	attrs = alloc_workqueue_attrs();
7075	if (!attrs)
7076	return NULL;
7077
7078	copy_workqueue_attrs(attrs, wq->unbound_attrs);
7079	return attrs;
7080	}
7081
7082	static ssize_t wq_nice_store(struct device dev, struct device_attribute attr,
7083	const char *buf, size_t count)
7084	{
7085	struct workqueue_struct *wq = dev_to_wq(dev);
7086	struct workqueue_attrs *attrs;
7087	int ret = -ENOMEM;
7088
7089	apply_wqattrs_lock();
7090
7091	attrs = wq_sysfs_prep_attrs(wq);
7092	if (!attrs)
7093	goto out_unlock;
7094
7095	if (sscanf(buf, "%d", &attrs->nice) == 1 &&
7096	attrs->nice >= MIN_NICE && attrs->nice <= MAX_NICE)
7097	ret = apply_workqueue_attrs_locked(wq, attrs);
7098	else
7099	ret = -EINVAL;
7100
7101	out_unlock:
7102	apply_wqattrs_unlock();
7103	free_workqueue_attrs(attrs);
7104	return ret ?: count;
7105	}
7106
7107	static ssize_t wq_cpumask_show(struct device *dev,
7108	struct device_attribute attr, char buf)
7109	{
7110	struct workqueue_struct *wq = dev_to_wq(dev);
7111	int written;
7112
7113	mutex_lock(&wq->mutex);
7114	written = scnprintf(buf, PAGE_SIZE, "%*pb\n",
7115	cpumask_pr_args(wq->unbound_attrs->cpumask));
7116	mutex_unlock(&wq->mutex);
7117	return written;
7118	}
7119
7120	static ssize_t wq_cpumask_store(struct device *dev,
7121	struct device_attribute *attr,
7122	const char *buf, size_t count)
7123	{
7124	struct workqueue_struct *wq = dev_to_wq(dev);
7125	struct workqueue_attrs *attrs;
7126	int ret = -ENOMEM;
7127
7128	apply_wqattrs_lock();
7129
7130	attrs = wq_sysfs_prep_attrs(wq);
7131	if (!attrs)
7132	goto out_unlock;
7133
7134	ret = cpumask_parse(buf, attrs->cpumask);
7135	if (!ret)
7136	ret = apply_workqueue_attrs_locked(wq, attrs);
7137
7138	out_unlock:
7139	apply_wqattrs_unlock();
7140	free_workqueue_attrs(attrs);
7141	return ret ?: count;
7142	}
7143
7144	static ssize_t wq_affn_scope_show(struct device *dev,
7145	struct device_attribute attr, char buf)
7146	{
7147	struct workqueue_struct *wq = dev_to_wq(dev);
7148	int written;
7149
7150	mutex_lock(&wq->mutex);
7151	if (wq->unbound_attrs->affn_scope == WQ_AFFN_DFL)
7152	written = scnprintf(buf, PAGE_SIZE, "%s (%s)\n",
7153	wq_affn_names[WQ_AFFN_DFL],
7154	wq_affn_names[wq_affn_dfl]);
7155	else
7156	written = scnprintf(buf, PAGE_SIZE, "%s\n",
7157	wq_affn_names[wq->unbound_attrs->affn_scope]);
7158	mutex_unlock(&wq->mutex);
7159
7160	return written;
7161	}
7162
7163	static ssize_t wq_affn_scope_store(struct device *dev,
7164	struct device_attribute *attr,
7165	const char *buf, size_t count)
7166	{
7167	struct workqueue_struct *wq = dev_to_wq(dev);
7168	struct workqueue_attrs *attrs;
7169	int affn, ret = -ENOMEM;
7170
7171	affn = parse_affn_scope(buf);
7172	if (affn < 0)
7173	return affn;
7174
7175	apply_wqattrs_lock();
7176	attrs = wq_sysfs_prep_attrs(wq);
7177	if (attrs) {
7178	attrs->affn_scope = affn;
7179	ret = apply_workqueue_attrs_locked(wq, attrs);
7180	}
7181	apply_wqattrs_unlock();
7182	free_workqueue_attrs(attrs);
7183	return ret ?: count;
7184	}
7185
7186	static ssize_t wq_affinity_strict_show(struct device *dev,
7187	struct device_attribute attr, char buf)
7188	{
7189	struct workqueue_struct *wq = dev_to_wq(dev);
7190
7191	return scnprintf(buf, PAGE_SIZE, "%d\n",
7192	wq->unbound_attrs->affn_strict);
7193	}
7194
7195	static ssize_t wq_affinity_strict_store(struct device *dev,
7196	struct device_attribute *attr,
7197	const char *buf, size_t count)
7198	{
7199	struct workqueue_struct *wq = dev_to_wq(dev);
7200	struct workqueue_attrs *attrs;
7201	int v, ret = -ENOMEM;
7202
7203	if (sscanf(buf, "%d", &v) != 1)
7204	return -EINVAL;
7205
7206	apply_wqattrs_lock();
7207	attrs = wq_sysfs_prep_attrs(wq);
7208	if (attrs) {
7209	attrs->affn_strict = (bool)v;
7210	ret = apply_workqueue_attrs_locked(wq, attrs);
7211	}
7212	apply_wqattrs_unlock();
7213	free_workqueue_attrs(attrs);
7214	return ret ?: count;
7215	}
7216
7217	static struct device_attribute wq_sysfs_unbound_attrs[] = {
7218	__ATTR(nice, 0644, wq_nice_show, wq_nice_store),
7219	__ATTR(cpumask, 0644, wq_cpumask_show, wq_cpumask_store),
7220	__ATTR(affinity_scope, 0644, wq_affn_scope_show, wq_affn_scope_store),
7221	__ATTR(affinity_strict, 0644, wq_affinity_strict_show, wq_affinity_strict_store),
7222	__ATTR_NULL,
7223	};
7224
7225	static const struct bus_type wq_subsys = {
7226	.name = "workqueue",
7227	.dev_groups = wq_sysfs_groups,
7228	};
7229
7230	/**
7231	* workqueue_set_unbound_cpumask - Set the low-level unbound cpumask
7232	* @cpumask: the cpumask to set
7233	*
7234	* The low-level workqueues cpumask is a global cpumask that limits
7235	* the affinity of all unbound workqueues. This function check the @cpumask
7236	* and apply it to all unbound workqueues and updates all pwqs of them.
7237	*
7238	* Return: 0 - Success
7239	* -EINVAL - Invalid @cpumask
7240	* -ENOMEM - Failed to allocate memory for attrs or pwqs.
7241	*/
7242	static int workqueue_set_unbound_cpumask(cpumask_var_t cpumask)
7243	{
7244	int ret = -EINVAL;
7245
7246	/*
7247	* Not excluding isolated cpus on purpose.
7248	* If the user wishes to include them, we allow that.
7249	*/
7250	cpumask_and(cpumask, cpumask, cpu_possible_mask);
7251	if (!cpumask_empty(cpumask)) {
7252	apply_wqattrs_lock();
7253	cpumask_copy(wq_requested_unbound_cpumask, cpumask);
7254	if (cpumask_equal(cpumask, wq_unbound_cpumask)) {
7255	ret = 0;
7256	goto out_unlock;
7257	}
7258
7259	ret = workqueue_apply_unbound_cpumask(cpumask);
7260
7261	out_unlock:
7262	apply_wqattrs_unlock();
7263	}
7264
7265	return ret;
7266	}
7267
7268	static ssize_t __wq_cpumask_show(struct device *dev,
7269	struct device_attribute attr, char buf, cpumask_var_t mask)
7270	{
7271	int written;
7272
7273	mutex_lock(&wq_pool_mutex);
7274	written = scnprintf(buf, PAGE_SIZE, "%*pb\n", cpumask_pr_args(mask));
7275	mutex_unlock(&wq_pool_mutex);
7276
7277	return written;
7278	}
7279
7280	static ssize_t cpumask_requested_show(struct device *dev,
7281	struct device_attribute attr, char buf)
7282	{
7283	return __wq_cpumask_show(dev, attr, buf, wq_requested_unbound_cpumask);
7284	}
7285	static DEVICE_ATTR_RO(cpumask_requested);
7286
7287	static ssize_t cpumask_isolated_show(struct device *dev,
7288	struct device_attribute attr, char buf)
7289	{
7290	return __wq_cpumask_show(dev, attr, buf, wq_isolated_cpumask);
7291	}
7292	static DEVICE_ATTR_RO(cpumask_isolated);
7293
7294	static ssize_t cpumask_show(struct device *dev,
7295	struct device_attribute attr, char buf)
7296	{
7297	return __wq_cpumask_show(dev, attr, buf, wq_unbound_cpumask);
7298	}
7299
7300	static ssize_t cpumask_store(struct device *dev,
7301	struct device_attribute attr, const char buf, size_t count)
7302	{
7303	cpumask_var_t cpumask;
7304	int ret;
7305
7306	if (!zalloc_cpumask_var(&cpumask, GFP_KERNEL))
7307	return -ENOMEM;
7308
7309	ret = cpumask_parse(buf, cpumask);
7310	if (!ret)
7311	ret = workqueue_set_unbound_cpumask(cpumask);
7312
7313	free_cpumask_var(cpumask);
7314	return ret ? ret : count;
7315	}
7316	static DEVICE_ATTR_RW(cpumask);
7317
7318	static struct attribute *wq_sysfs_cpumask_attrs[] = {
7319	&dev_attr_cpumask.attr,
7320	&dev_attr_cpumask_requested.attr,
7321	&dev_attr_cpumask_isolated.attr,
7322	NULL,
7323	};
7324	ATTRIBUTE_GROUPS(wq_sysfs_cpumask);
7325
7326	static int __init wq_sysfs_init(void)
7327	{
7328	return subsys_virtual_register(&wq_subsys, wq_sysfs_cpumask_groups);
7329	}
7330	core_initcall(wq_sysfs_init);
7331
7332	static void wq_device_release(struct device *dev)
7333	{
7334	struct wq_device *wq_dev = container_of(dev, struct wq_device, dev);
7335
7336	kfree(wq_dev);
7337	}
7338
7339	/**
7340	* workqueue_sysfs_register - make a workqueue visible in sysfs
7341	* @wq: the workqueue to register
7342	*
7343	* Expose @wq in sysfs under /sys/bus/workqueue/devices.
7344	* alloc_workqueue*() automatically calls this function if WQ_SYSFS is set
7345	* which is the preferred method.
7346	*
7347	* Workqueue user should use this function directly iff it wants to apply
7348	* workqueue_attrs before making the workqueue visible in sysfs; otherwise,
7349	* apply_workqueue_attrs() may race against userland updating the
7350	* attributes.
7351	*
7352	* Return: 0 on success, -errno on failure.
7353	*/
7354	int workqueue_sysfs_register(struct workqueue_struct *wq)
7355	{
7356	struct wq_device *wq_dev;
7357	int ret;
7358
7359	/*
7360	* Adjusting max_active breaks ordering guarantee. Disallow exposing
7361	* ordered workqueues.
7362	*/
7363	if (WARN_ON(wq->flags & __WQ_ORDERED))
7364	return -EINVAL;
7365
7366	wq->wq_dev = wq_dev = kzalloc(sizeof(*wq_dev), GFP_KERNEL);
7367	if (!wq_dev)
7368	return -ENOMEM;
7369
7370	wq_dev->wq = wq;
7371	wq_dev->dev.bus = &wq_subsys;
7372	wq_dev->dev.release = wq_device_release;
7373	dev_set_name(&wq_dev->dev, "%s", wq->name);
7374
7375	/*
7376	* unbound_attrs are created separately. Suppress uevent until
7377	* everything is ready.
7378	*/
7379	dev_set_uevent_suppress(&wq_dev->dev, true);
7380
7381	ret = device_register(&wq_dev->dev);
7382	if (ret) {
7383	put_device(&wq_dev->dev);
7384	wq->wq_dev = NULL;
7385	return ret;
7386	}
7387
7388	if (wq->flags & WQ_UNBOUND) {
7389	struct device_attribute *attr;
7390
7391	for (attr = wq_sysfs_unbound_attrs; attr->attr.name; attr++) {
7392	ret = device_create_file(&wq_dev->dev, attr);
7393	if (ret) {
7394	device_unregister(&wq_dev->dev);
7395	wq->wq_dev = NULL;
7396	return ret;
7397	}
7398	}
7399	}
7400
7401	dev_set_uevent_suppress(&wq_dev->dev, false);
7402	kobject_uevent(&wq_dev->dev.kobj, KOBJ_ADD);
7403	return 0;
7404	}
7405
7406	/**
7407	* workqueue_sysfs_unregister - undo workqueue_sysfs_register()
7408	* @wq: the workqueue to unregister
7409	*
7410	* If @wq is registered to sysfs by workqueue_sysfs_register(), unregister.
7411	*/
7412	static void workqueue_sysfs_unregister(struct workqueue_struct *wq)
7413	{
7414	struct wq_device *wq_dev = wq->wq_dev;
7415
7416	if (!wq->wq_dev)
7417	return;
7418
7419	wq->wq_dev = NULL;
7420	device_unregister(&wq_dev->dev);
7421	}
7422	#else /* CONFIG_SYSFS */
7423	static void workqueue_sysfs_unregister(struct workqueue_struct *wq) { }
7424	#endif /* CONFIG_SYSFS */
7425
7426	/*
7427	* Workqueue watchdog.
7428	*
7429	* Stall may be caused by various bugs - missing WQ_MEM_RECLAIM, illegal
7430	* flush dependency, a concurrency managed work item which stays RUNNING
7431	* indefinitely. Workqueue stalls can be very difficult to debug as the
7432	* usual warning mechanisms don't trigger and internal workqueue state is
7433	* largely opaque.
7434	*
7435	* Workqueue watchdog monitors all worker pools periodically and dumps
7436	* state if some pools failed to make forward progress for a while where
7437	* forward progress is defined as the first item on ->worklist changing.
7438	*
7439	* This mechanism is controlled through the kernel parameter
7440	* "workqueue.watchdog_thresh" which can be updated at runtime through the
7441	* corresponding sysfs parameter file.
7442	*/
7443	#ifdef CONFIG_WQ_WATCHDOG
7444
7445	static unsigned long wq_watchdog_thresh = 30;
7446	static struct timer_list wq_watchdog_timer;
7447
7448	static unsigned long wq_watchdog_touched = INITIAL_JIFFIES;
7449	static DEFINE_PER_CPU(unsigned long, wq_watchdog_touched_cpu) = INITIAL_JIFFIES;
7450
7451	/*
7452	* Show workers that might prevent the processing of pending work items.
7453	* The only candidates are CPU-bound workers in the running state.
7454	* Pending work items should be handled by another idle worker
7455	* in all other situations.
7456	*/
7457	static void show_cpu_pool_hog(struct worker_pool *pool)
7458	{
7459	struct worker *worker;
7460	unsigned long irq_flags;
7461	int bkt;
7462
7463	raw_spin_lock_irqsave(&pool->lock, irq_flags);
7464
7465	hash_for_each(pool->busy_hash, bkt, worker, hentry) {
7466	if (task_is_running(worker->task)) {
7467	/*
7468	* Defer printing to avoid deadlocks in console
7469	* drivers that queue work while holding locks
7470	* also taken in their write paths.
7471	*/
7472	printk_deferred_enter();
7473
7474	pr_info("pool %d:\n", pool->id);
7475	sched_show_task(worker->task);
7476
7477	printk_deferred_exit();
7478	}
7479	}
7480
7481	raw_spin_unlock_irqrestore(&pool->lock, irq_flags);
7482	}
7483
7484	static void show_cpu_pools_hogs(void)
7485	{
7486	struct worker_pool *pool;
7487	int pi;
7488
7489	pr_info("Showing backtraces of running workers in stalled CPU-bound worker pools:\n");
7490
7491	rcu_read_lock();
7492
7493	for_each_pool(pool, pi) {
7494	if (pool->cpu_stall)
7495	show_cpu_pool_hog(pool);
7496
7497	}
7498
7499	rcu_read_unlock();
7500	}
7501
7502	static void wq_watchdog_reset_touched(void)
7503	{
7504	int cpu;
7505
7506	wq_watchdog_touched = jiffies;
7507	for_each_possible_cpu(cpu)
7508	per_cpu(wq_watchdog_touched_cpu, cpu) = jiffies;
7509	}
7510
7511	static void wq_watchdog_timer_fn(struct timer_list *unused)
7512	{
7513	unsigned long thresh = READ_ONCE(wq_watchdog_thresh) * HZ;
7514	bool lockup_detected = false;
7515	bool cpu_pool_stall = false;
7516	unsigned long now = jiffies;
7517	struct worker_pool *pool;
7518	int pi;
7519
7520	if (!thresh)
7521	return;
7522
7523	rcu_read_lock();
7524
7525	for_each_pool(pool, pi) {
7526	unsigned long pool_ts, touched, ts;
7527
7528	pool->cpu_stall = false;
7529	if (list_empty(&pool->worklist))
7530	continue;
7531
7532	/*
7533	* If a virtual machine is stopped by the host it can look to
7534	* the watchdog like a stall.
7535	*/
7536	kvm_check_and_clear_guest_paused();
7537
7538	/* get the latest of pool and touched timestamps */
7539	if (pool->cpu >= 0)
7540	touched = READ_ONCE(per_cpu(wq_watchdog_touched_cpu, pool->cpu));
7541	else
7542	touched = READ_ONCE(wq_watchdog_touched);
7543	pool_ts = READ_ONCE(pool->watchdog_ts);
7544
7545	if (time_after(pool_ts, touched))
7546	ts = pool_ts;
7547	else
7548	ts = touched;
7549
7550	/* did we stall? */
7551	if (time_after(now, ts + thresh)) {
7552	lockup_detected = true;
7553	if (pool->cpu >= 0 && !(pool->flags & POOL_BH)) {
7554	pool->cpu_stall = true;
7555	cpu_pool_stall = true;
7556	}
7557	pr_emerg("BUG: workqueue lockup - pool");
7558	pr_cont_pool_info(pool);
7559	pr_cont(" stuck for %us!\n",
7560	jiffies_to_msecs(now - pool_ts) / 1000);
7561	}
7562
7563
7564	}
7565
7566	rcu_read_unlock();
7567
7568	if (lockup_detected)
7569	show_all_workqueues();
7570
7571	if (cpu_pool_stall)
7572	show_cpu_pools_hogs();
7573
7574	wq_watchdog_reset_touched();
7575	mod_timer(&wq_watchdog_timer, jiffies + thresh);
7576	}
7577
7578	notrace void wq_watchdog_touch(int cpu)
7579	{
7580	if (cpu >= 0)
7581	per_cpu(wq_watchdog_touched_cpu, cpu) = jiffies;
7582
7583	wq_watchdog_touched = jiffies;
7584	}
7585
7586	static void wq_watchdog_set_thresh(unsigned long thresh)
7587	{
7588	wq_watchdog_thresh = 0;
7589	del_timer_sync(&wq_watchdog_timer);
7590
7591	if (thresh) {
7592	wq_watchdog_thresh = thresh;
7593	wq_watchdog_reset_touched();
7594	mod_timer(&wq_watchdog_timer, jiffies + thresh * HZ);
7595	}
7596	}
7597
7598	static int wq_watchdog_param_set_thresh(const char *val,
7599	const struct kernel_param *kp)
7600	{
7601	unsigned long thresh;
7602	int ret;
7603
7604	ret = kstrtoul(val, 0, &thresh);
7605	if (ret)
7606	return ret;
7607
7608	if (system_wq)
7609	wq_watchdog_set_thresh(thresh);
7610	else
7611	wq_watchdog_thresh = thresh;
7612
7613	return 0;
7614	}
7615
7616	static const struct kernel_param_ops wq_watchdog_thresh_ops = {
7617	.set = wq_watchdog_param_set_thresh,
7618	.get = param_get_ulong,
7619	};
7620
7621	module_param_cb(watchdog_thresh, &wq_watchdog_thresh_ops, &wq_watchdog_thresh,
7622	0644);
7623
7624	static void wq_watchdog_init(void)
7625	{
7626	timer_setup(&wq_watchdog_timer, wq_watchdog_timer_fn, TIMER_DEFERRABLE);
7627	wq_watchdog_set_thresh(wq_watchdog_thresh);
7628	}
7629
7630	#else /* CONFIG_WQ_WATCHDOG */
7631
7632	static inline void wq_watchdog_init(void) { }
7633
7634	#endif /* CONFIG_WQ_WATCHDOG */
7635
7636	static void bh_pool_kick_normal(struct irq_work *irq_work)
7637	{
7638	raise_softirq_irqoff(TASKLET_SOFTIRQ);
7639	}
7640
7641	static void bh_pool_kick_highpri(struct irq_work *irq_work)
7642	{
7643	raise_softirq_irqoff(HI_SOFTIRQ);
7644	}
7645
7646	static void __init restrict_unbound_cpumask(const char name, const struct cpumask mask)
7647	{
7648	if (!cpumask_intersects(wq_unbound_cpumask, mask)) {
7649	pr_warn("workqueue: Restricting unbound_cpumask (%pb) with %s (%pb) leaves no CPU, ignoring\n",
7650	cpumask_pr_args(wq_unbound_cpumask), name, cpumask_pr_args(mask));
7651	return;
7652	}
7653
7654	cpumask_and(wq_unbound_cpumask, wq_unbound_cpumask, mask);
7655	}
7656
7657	static void __init init_cpu_worker_pool(struct worker_pool *pool, int cpu, int nice)
7658	{
7659	BUG_ON(init_worker_pool(pool));
7660	pool->cpu = cpu;
7661	cpumask_copy(pool->attrs->cpumask, cpumask_of(cpu));
7662	cpumask_copy(pool->attrs->__pod_cpumask, cpumask_of(cpu));
7663	pool->attrs->nice = nice;
7664	pool->attrs->affn_strict = true;
7665	pool->node = cpu_to_node(cpu);
7666
7667	/* alloc pool ID */
7668	mutex_lock(&wq_pool_mutex);
7669	BUG_ON(worker_pool_assign_id(pool));
7670	mutex_unlock(&wq_pool_mutex);
7671	}
7672
7673	/**
7674	* workqueue_init_early - early init for workqueue subsystem
7675	*
7676	* This is the first step of three-staged workqueue subsystem initialization and
7677	* invoked as soon as the bare basics - memory allocation, cpumasks and idr are
7678	* up. It sets up all the data structures and system workqueues and allows early
7679	* boot code to create workqueues and queue/cancel work items. Actual work item
7680	* execution starts only after kthreads can be created and scheduled right
7681	* before early initcalls.
7682	*/
7683	void __init workqueue_init_early(void)
7684	{
7685	struct wq_pod_type *pt = &wq_pod_types[WQ_AFFN_SYSTEM];
7686	int std_nice[NR_STD_WORKER_POOLS] = { 0, HIGHPRI_NICE_LEVEL };
7687	void (irq_work_fns[2])(struct irq_work ) = { bh_pool_kick_normal,
7688	bh_pool_kick_highpri };
7689	int i, cpu;
7690
7691	BUILD_BUG_ON(__alignof__(struct pool_workqueue) < __alignof__(long long));
7692
7693	BUG_ON(!alloc_cpumask_var(&wq_unbound_cpumask, GFP_KERNEL));
7694	BUG_ON(!alloc_cpumask_var(&wq_requested_unbound_cpumask, GFP_KERNEL));
7695	BUG_ON(!zalloc_cpumask_var(&wq_isolated_cpumask, GFP_KERNEL));
7696
7697	cpumask_copy(wq_unbound_cpumask, cpu_possible_mask);
7698	restrict_unbound_cpumask("HK_TYPE_WQ", housekeeping_cpumask(HK_TYPE_WQ));
7699	restrict_unbound_cpumask("HK_TYPE_DOMAIN", housekeeping_cpumask(HK_TYPE_DOMAIN));
7700	if (!cpumask_empty(&wq_cmdline_cpumask))
7701	restrict_unbound_cpumask("workqueue.unbound_cpus", &wq_cmdline_cpumask);
7702
7703	cpumask_copy(wq_requested_unbound_cpumask, wq_unbound_cpumask);
7704
7705	pwq_cache = KMEM_CACHE(pool_workqueue, SLAB_PANIC);
7706
7707	wq_update_pod_attrs_buf = alloc_workqueue_attrs();
7708	BUG_ON(!wq_update_pod_attrs_buf);
7709
7710	/*
7711	* If nohz_full is enabled, set power efficient workqueue as unbound.
7712	* This allows workqueue items to be moved to HK CPUs.
7713	*/
7714	if (housekeeping_enabled(HK_TYPE_TICK))
7715	wq_power_efficient = true;
7716
7717	/* initialize WQ_AFFN_SYSTEM pods */
7718	pt->pod_cpus = kcalloc(1, sizeof(pt->pod_cpus[0]), GFP_KERNEL);
7719	pt->pod_node = kcalloc(1, sizeof(pt->pod_node[0]), GFP_KERNEL);
7720	pt->cpu_pod = kcalloc(nr_cpu_ids, sizeof(pt->cpu_pod[0]), GFP_KERNEL);
7721	BUG_ON(!pt->pod_cpus \|\| !pt->pod_node \|\| !pt->cpu_pod);
7722
7723	BUG_ON(!zalloc_cpumask_var_node(&pt->pod_cpus[0], GFP_KERNEL, NUMA_NO_NODE));
7724
7725	pt->nr_pods = 1;
7726	cpumask_copy(pt->pod_cpus[0], cpu_possible_mask);
7727	pt->pod_node[0] = NUMA_NO_NODE;
7728	pt->cpu_pod[0] = 0;
7729
7730	/* initialize BH and CPU pools */
7731	for_each_possible_cpu(cpu) {
7732	struct worker_pool *pool;
7733
7734	i = 0;
7735	for_each_bh_worker_pool(pool, cpu) {
7736	init_cpu_worker_pool(pool, cpu, std_nice[i]);
7737	pool->flags \|= POOL_BH;
7738	init_irq_work(bh_pool_irq_work(pool), irq_work_fns[i]);
7739	i++;
7740	}
7741
7742	i = 0;
7743	for_each_cpu_worker_pool(pool, cpu)
7744	init_cpu_worker_pool(pool, cpu, std_nice[i++]);
7745	}
7746
7747	/* create default unbound and ordered wq attrs */
7748	for (i = 0; i < NR_STD_WORKER_POOLS; i++) {
7749	struct workqueue_attrs *attrs;
7750
7751	BUG_ON(!(attrs = alloc_workqueue_attrs()));
7752	attrs->nice = std_nice[i];
7753	unbound_std_wq_attrs[i] = attrs;
7754
7755	/*
7756	* An ordered wq should have only one pwq as ordering is
7757	* guaranteed by max_active which is enforced by pwqs.
7758	*/
7759	BUG_ON(!(attrs = alloc_workqueue_attrs()));
7760	attrs->nice = std_nice[i];
7761	attrs->ordered = true;
7762	ordered_wq_attrs[i] = attrs;
7763	}
7764
7765	system_wq = alloc_workqueue("events", 0, 0);
7766	system_highpri_wq = alloc_workqueue("events_highpri", WQ_HIGHPRI, 0);
7767	system_long_wq = alloc_workqueue("events_long", 0, 0);
7768	system_unbound_wq = alloc_workqueue("events_unbound", WQ_UNBOUND,
7769	WQ_MAX_ACTIVE);
7770	system_freezable_wq = alloc_workqueue("events_freezable",
7771	WQ_FREEZABLE, 0);
7772	system_power_efficient_wq = alloc_workqueue("events_power_efficient",
7773	WQ_POWER_EFFICIENT, 0);
7774	system_freezable_power_efficient_wq = alloc_workqueue("events_freezable_pwr_efficient",
7775	WQ_FREEZABLE \| WQ_POWER_EFFICIENT,
7776	0);
7777	system_bh_wq = alloc_workqueue("events_bh", WQ_BH, 0);
7778	system_bh_highpri_wq = alloc_workqueue("events_bh_highpri",
7779	WQ_BH \| WQ_HIGHPRI, 0);
7780	BUG_ON(!system_wq \|\| !system_highpri_wq \|\| !system_long_wq \|\|
7781	!system_unbound_wq \|\| !system_freezable_wq \|\|
7782	!system_power_efficient_wq \|\|
7783	!system_freezable_power_efficient_wq \|\|
7784	!system_bh_wq \|\| !system_bh_highpri_wq);
7785	}
7786
7787	static void __init wq_cpu_intensive_thresh_init(void)
7788	{
7789	unsigned long thresh;
7790	unsigned long bogo;
7791
7792	pwq_release_worker = kthread_create_worker(0, "pool_workqueue_release");
7793	BUG_ON(IS_ERR(pwq_release_worker));
7794
7795	/* if the user set it to a specific value, keep it */
7796	if (wq_cpu_intensive_thresh_us != ULONG_MAX)
7797	return;
7798
7799	/*
7800	* The default of 10ms is derived from the fact that most modern (as of
7801	* 2023) processors can do a lot in 10ms and that it's just below what
7802	* most consider human-perceivable. However, the kernel also runs on a
7803	* lot slower CPUs including microcontrollers where the threshold is way
7804	* too low.
7805	*
7806	* Let's scale up the threshold upto 1 second if BogoMips is below 4000.
7807	* This is by no means accurate but it doesn't have to be. The mechanism
7808	* is still useful even when the threshold is fully scaled up. Also, as
7809	* the reports would usually be applicable to everyone, some machines
7810	* operating on longer thresholds won't significantly diminish their
7811	* usefulness.
7812	*/
7813	thresh = 10 * USEC_PER_MSEC;
7814
7815	/* see init/calibrate.c for lpj -> BogoMIPS calculation */
7816	bogo = max_t(unsigned long, loops_per_jiffy / 500000 * HZ, 1);
7817	if (bogo < 4000)
7818	thresh = min_t(unsigned long, thresh * 4000 / bogo, USEC_PER_SEC);
7819
7820	pr_debug("wq_cpu_intensive_thresh: lpj=%lu BogoMIPS=%lu thresh_us=%lu\n",
7821	loops_per_jiffy, bogo, thresh);
7822
7823	wq_cpu_intensive_thresh_us = thresh;
7824	}
7825
7826	/**
7827	* workqueue_init - bring workqueue subsystem fully online
7828	*
7829	* This is the second step of three-staged workqueue subsystem initialization
7830	* and invoked as soon as kthreads can be created and scheduled. Workqueues have
7831	* been created and work items queued on them, but there are no kworkers
7832	* executing the work items yet. Populate the worker pools with the initial
7833	* workers and enable future kworker creations.
7834	*/
7835	void __init workqueue_init(void)
7836	{
7837	struct workqueue_struct *wq;
7838	struct worker_pool *pool;
7839	int cpu, bkt;
7840
7841	wq_cpu_intensive_thresh_init();
7842
7843	mutex_lock(&wq_pool_mutex);
7844
7845	/*
7846	* Per-cpu pools created earlier could be missing node hint. Fix them
7847	* up. Also, create a rescuer for workqueues that requested it.
7848	*/
7849	for_each_possible_cpu(cpu) {
7850	for_each_bh_worker_pool(pool, cpu)
7851	pool->node = cpu_to_node(cpu);
7852	for_each_cpu_worker_pool(pool, cpu)
7853	pool->node = cpu_to_node(cpu);
7854	}
7855
7856	list_for_each_entry(wq, &workqueues, list) {
7857	WARN(init_rescuer(wq),
7858	"workqueue: failed to create early rescuer for %s",
7859	wq->name);
7860	}
7861
7862	mutex_unlock(&wq_pool_mutex);
7863
7864	/*
7865	* Create the initial workers. A BH pool has one pseudo worker that
7866	* represents the shared BH execution context and thus doesn't get
7867	* affected by hotplug events. Create the BH pseudo workers for all
7868	* possible CPUs here.
7869	*/
7870	for_each_possible_cpu(cpu)
7871	for_each_bh_worker_pool(pool, cpu)
7872	BUG_ON(!create_worker(pool));
7873
7874	for_each_online_cpu(cpu) {
7875	for_each_cpu_worker_pool(pool, cpu) {
7876	pool->flags &= ~POOL_DISASSOCIATED;
7877	BUG_ON(!create_worker(pool));
7878	}
7879	}
7880
7881	hash_for_each(unbound_pool_hash, bkt, pool, hash_node)
7882	BUG_ON(!create_worker(pool));
7883
7884	wq_online = true;
7885	wq_watchdog_init();
7886	}
7887
7888	/*
7889	* Initialize @pt by first initializing @pt->cpu_pod[] with pod IDs according to
7890	* @cpu_shares_pod(). Each subset of CPUs that share a pod is assigned a unique
7891	* and consecutive pod ID. The rest of @pt is initialized accordingly.
7892	*/
7893	static void __init init_pod_type(struct wq_pod_type *pt,
7894	bool (*cpus_share_pod)(int, int))
7895	{
7896	int cur, pre, cpu, pod;
7897
7898	pt->nr_pods = 0;
7899
7900	/* init @pt->cpu_pod[] according to @cpus_share_pod() */
7901	pt->cpu_pod = kcalloc(nr_cpu_ids, sizeof(pt->cpu_pod[0]), GFP_KERNEL);
7902	BUG_ON(!pt->cpu_pod);
7903
7904	for_each_possible_cpu(cur) {
7905	for_each_possible_cpu(pre) {
7906	if (pre >= cur) {
7907	pt->cpu_pod[cur] = pt->nr_pods++;
7908	break;
7909	}
7910	if (cpus_share_pod(cur, pre)) {
7911	pt->cpu_pod[cur] = pt->cpu_pod[pre];
7912	break;
7913	}
7914	}
7915	}
7916
7917	/* init the rest to match @pt->cpu_pod[] */
7918	pt->pod_cpus = kcalloc(pt->nr_pods, sizeof(pt->pod_cpus[0]), GFP_KERNEL);
7919	pt->pod_node = kcalloc(pt->nr_pods, sizeof(pt->pod_node[0]), GFP_KERNEL);
7920	BUG_ON(!pt->pod_cpus \|\| !pt->pod_node);
7921
7922	for (pod = 0; pod < pt->nr_pods; pod++)
7923	BUG_ON(!zalloc_cpumask_var(&pt->pod_cpus[pod], GFP_KERNEL));
7924
7925	for_each_possible_cpu(cpu) {
7926	cpumask_set_cpu(cpu, pt->pod_cpus[pt->cpu_pod[cpu]]);
7927	pt->pod_node[pt->cpu_pod[cpu]] = cpu_to_node(cpu);
7928	}
7929	}
7930
7931	static bool __init cpus_dont_share(int cpu0, int cpu1)
7932	{
7933	return false;
7934	}
7935
7936	static bool __init cpus_share_smt(int cpu0, int cpu1)
7937	{
7938	#ifdef CONFIG_SCHED_SMT
7939	return cpumask_test_cpu(cpu0, cpu_smt_mask(cpu1));
7940	#else
7941	return false;
7942	#endif
7943	}
7944
7945	static bool __init cpus_share_numa(int cpu0, int cpu1)
7946	{
7947	return cpu_to_node(cpu0) == cpu_to_node(cpu1);
7948	}
7949
7950	/**
7951	* workqueue_init_topology - initialize CPU pods for unbound workqueues
7952	*
7953	* This is the third step of three-staged workqueue subsystem initialization and
7954	* invoked after SMP and topology information are fully initialized. It
7955	* initializes the unbound CPU pods accordingly.
7956	*/
7957	void __init workqueue_init_topology(void)
7958	{
7959	struct workqueue_struct *wq;
7960	int cpu;
7961
7962	init_pod_type(&wq_pod_types[WQ_AFFN_CPU], cpus_dont_share);
7963	init_pod_type(&wq_pod_types[WQ_AFFN_SMT], cpus_share_smt);
7964	init_pod_type(&wq_pod_types[WQ_AFFN_CACHE], cpus_share_cache);
7965	init_pod_type(&wq_pod_types[WQ_AFFN_NUMA], cpus_share_numa);
7966
7967	wq_topo_initialized = true;
7968
7969	mutex_lock(&wq_pool_mutex);
7970
7971	/*
7972	* Workqueues allocated earlier would have all CPUs sharing the default
7973	* worker pool. Explicitly call wq_update_pod() on all workqueue and CPU
7974	* combinations to apply per-pod sharing.
7975	*/
7976	list_for_each_entry(wq, &workqueues, list) {
7977	for_each_online_cpu(cpu)
7978	wq_update_pod(wq, cpu, cpu, true);
7979	if (wq->flags & WQ_UNBOUND) {
7980	mutex_lock(&wq->mutex);
7981	wq_update_node_max_active(wq, -1);
7982	mutex_unlock(&wq->mutex);
7983	}
7984	}
7985
7986	mutex_unlock(&wq_pool_mutex);
7987	}
7988
7989	void __warn_flushing_systemwide_wq(void)
7990	{
7991	pr_warn("WARNING: Flushing system-wide workqueues will be prohibited in near future.\n");
7992	dump_stack();
7993	}
7994	EXPORT_SYMBOL(__warn_flushing_systemwide_wq);
7995
7996	static int __init workqueue_unbound_cpus_setup(char *str)
7997	{
7998	if (cpulist_parse(str, &wq_cmdline_cpumask) < 0) {
7999	cpumask_clear(&wq_cmdline_cpumask);
8000	pr_warn("workqueue.unbound_cpus: incorrect CPU range, using default\n");
8001	}
8002
8003	return 1;
8004	}
8005	__setup("workqueue.unbound_cpus=", workqueue_unbound_cpus_setup);