linux - kernel/workqueue.c

6142 lines · 168.8 KiB

1	// SPDX-License-Identifier: GPL-2.0-only
2	/*
3	* kernel/workqueue.c - generic async execution with shared worker pool
4	*
5	* Copyright (C) 2002 Ingo Molnar
6	*
7	* Derived from the taskqueue/keventd code by:
8	* David Woodhouse <[email protected]>
9	* Andrew Morton
10	* Kai Petzke <[email protected]>
11	* Theodore Ts'o <[email protected]>
12	*
13	* Made to use alloc_percpu by Christoph Lameter.
14	*
15	* Copyright (C) 2010 SUSE Linux Products GmbH
16	* Copyright (C) 2010 Tejun Heo <[email protected]>
17	*
18	* This is the generic async execution mechanism. Work items as are
19	* executed in process context. The worker pool is shared and
20	* automatically managed. There are two worker pools for each CPU (one for
21	* normal work items and the other for high priority ones) and some extra
22	* pools for workqueues which are not bound to any specific CPU - the
23	* number of these backing pools is dynamic.
24	*
25	* Please read Documentation/core-api/workqueue.rst for details.
26	*/
27
28	#include <linux/export.h>
29	#include <linux/kernel.h>
30	#include <linux/sched.h>
31	#include <linux/init.h>
32	#include <linux/signal.h>
33	#include <linux/completion.h>
34	#include <linux/workqueue.h>
35	#include <linux/slab.h>
36	#include <linux/cpu.h>
37	#include <linux/notifier.h>
38	#include <linux/kthread.h>
39	#include <linux/hardirq.h>
40	#include <linux/mempolicy.h>
41	#include <linux/freezer.h>
42	#include <linux/debug_locks.h>
43	#include <linux/lockdep.h>
44	#include <linux/idr.h>
45	#include <linux/jhash.h>
46	#include <linux/hashtable.h>
47	#include <linux/rculist.h>
48	#include <linux/nodemask.h>
49	#include <linux/moduleparam.h>
50	#include <linux/uaccess.h>
51	#include <linux/sched/isolation.h>
52	#include <linux/nmi.h>
53	#include <linux/kvm_para.h>
54
55	#include "workqueue_internal.h"
56
57	enum {
58	/*
59	* worker_pool flags
60	*
61	* A bound pool is either associated or disassociated with its CPU.
62	* While associated (!DISASSOCIATED), all workers are bound to the
63	* CPU and none has %WORKER_UNBOUND set and concurrency management
64	* is in effect.
65	*
66	* While DISASSOCIATED, the cpu may be offline and all workers have
67	* %WORKER_UNBOUND set and concurrency management disabled, and may
68	* be executing on any CPU. The pool behaves as an unbound one.
69	*
70	* Note that DISASSOCIATED should be flipped only while holding
71	* wq_pool_attach_mutex to avoid changing binding state while
72	* worker_attach_to_pool() is in progress.
73	*/
74	POOL_MANAGER_ACTIVE = 1 << 0, /* being managed */
75	POOL_DISASSOCIATED = 1 << 2, /* cpu can't serve workers */
76
77	/* worker flags */
78	WORKER_DIE = 1 << 1, /* die die die */
79	WORKER_IDLE = 1 << 2, /* is idle */
80	WORKER_PREP = 1 << 3, /* preparing to run works */
81	WORKER_CPU_INTENSIVE = 1 << 6, /* cpu intensive */
82	WORKER_UNBOUND = 1 << 7, /* worker is unbound */
83	WORKER_REBOUND = 1 << 8, /* worker was rebound */
84
85	WORKER_NOT_RUNNING = WORKER_PREP \| WORKER_CPU_INTENSIVE \|
86	WORKER_UNBOUND \| WORKER_REBOUND,
87
88	NR_STD_WORKER_POOLS = 2, /* # standard pools per cpu */
89
90	UNBOUND_POOL_HASH_ORDER = 6, /* hashed by pool->attrs */
91	BUSY_WORKER_HASH_ORDER = 6, /* 64 pointers */
92
93	MAX_IDLE_WORKERS_RATIO = 4, /* 1/4 of busy can be idle */
94	IDLE_WORKER_TIMEOUT = 300 * HZ, /* keep idle ones for 5 mins */
95
96	MAYDAY_INITIAL_TIMEOUT = HZ / 100 >= 2 ? HZ / 100 : 2,
97	/* call for help after 10ms
98	(min two ticks) */
99	MAYDAY_INTERVAL = HZ / 10, /* and then every 100ms */
100	CREATE_COOLDOWN = HZ, /* time to breath after fail */
101
102	/*
103	* Rescue workers are used only on emergencies and shared by
104	* all cpus. Give MIN_NICE.
105	*/
106	RESCUER_NICE_LEVEL = MIN_NICE,
107	HIGHPRI_NICE_LEVEL = MIN_NICE,
108
109	WQ_NAME_LEN = 24,
110	};
111
112	/*
113	* Structure fields follow one of the following exclusion rules.
114	*
115	* I: Modifiable by initialization/destruction paths and read-only for
116	* everyone else.
117	*
118	* P: Preemption protected. Disabling preemption is enough and should
119	* only be modified and accessed from the local cpu.
120	*
121	* L: pool->lock protected. Access with pool->lock held.
122	*
123	* X: During normal operation, modification requires pool->lock and should
124	* be done only from local cpu. Either disabling preemption on local
125	* cpu or grabbing pool->lock is enough for read access. If
126	* POOL_DISASSOCIATED is set, it's identical to L.
127	*
128	* A: wq_pool_attach_mutex protected.
129	*
130	* PL: wq_pool_mutex protected.
131	*
132	* PR: wq_pool_mutex protected for writes. RCU protected for reads.
133	*
134	* PW: wq_pool_mutex and wq->mutex protected for writes. Either for reads.
135	*
136	* PWR: wq_pool_mutex and wq->mutex protected for writes. Either or
137	* RCU for reads.
138	*
139	* WQ: wq->mutex protected.
140	*
141	* WR: wq->mutex protected for writes. RCU protected for reads.
142	*
143	* MD: wq_mayday_lock protected.
144	*/
145
146	/* struct worker is defined in workqueue_internal.h */
147
148	struct worker_pool {
149	raw_spinlock_t lock; /* the pool lock */
150	int cpu; /* I: the associated cpu */
151	int node; /* I: the associated node ID */
152	int id; /* I: pool ID */
153	unsigned int flags; /* X: flags */
154
155	unsigned long watchdog_ts; /* L: watchdog timestamp */
156
157	struct list_head worklist; /* L: list of pending works */
158
159	int nr_workers; /* L: total number of workers */
160	int nr_idle; /* L: currently idle workers */
161
162	struct list_head idle_list; /* X: list of idle workers */
163	struct timer_list idle_timer; /* L: worker idle timeout */
164	struct timer_list mayday_timer; /* L: SOS timer for workers */
165
166	/* a workers is either on busy_hash or idle_list, or the manager */
167	DECLARE_HASHTABLE(busy_hash, BUSY_WORKER_HASH_ORDER);
168	/* L: hash of busy workers */
169
170	struct worker manager; / L: purely informational */
171	struct list_head workers; /* A: attached workers */
172	struct completion detach_completion; / all workers detached */
173
174	struct ida worker_ida; /* worker IDs for task name */
175
176	struct workqueue_attrs attrs; / I: worker attributes */
177	struct hlist_node hash_node; /* PL: unbound_pool_hash node */
178	int refcnt; /* PL: refcnt for unbound pools */
179
180	/*
181	* The current concurrency level. As it's likely to be accessed
182	* from other CPUs during try_to_wake_up(), put it in a separate
183	* cacheline.
184	*/
185	atomic_t nr_running ____cacheline_aligned_in_smp;
186
187	/*
188	* Destruction of pool is RCU protected to allow dereferences
189	* from get_work_pool().
190	*/
191	struct rcu_head rcu;
192	} ____cacheline_aligned_in_smp;
193
194	/*
195	* The per-pool workqueue. While queued, the lower WORK_STRUCT_FLAG_BITS
196	* of work_struct->data are used for flags and the remaining high bits
197	* point to the pwq; thus, pwqs need to be aligned at two's power of the
198	* number of flag bits.
199	*/
200	struct pool_workqueue {
201	struct worker_pool pool; / I: the associated pool */
202	struct workqueue_struct wq; / I: the owning workqueue */
203	int work_color; /* L: current color */
204	int flush_color; /* L: flushing color */
205	int refcnt; /* L: reference count */
206	int nr_in_flight[WORK_NR_COLORS];
207	/* L: nr of in_flight works */
208
209	/*
210	* nr_active management and WORK_STRUCT_INACTIVE:
211	*
212	* When pwq->nr_active >= max_active, new work item is queued to
213	* pwq->inactive_works instead of pool->worklist and marked with
214	* WORK_STRUCT_INACTIVE.
215	*
216	* All work items marked with WORK_STRUCT_INACTIVE do not participate
217	* in pwq->nr_active and all work items in pwq->inactive_works are
218	* marked with WORK_STRUCT_INACTIVE. But not all WORK_STRUCT_INACTIVE
219	* work items are in pwq->inactive_works. Some of them are ready to
220	* run in pool->worklist or worker->scheduled. Those work itmes are
221	* only struct wq_barrier which is used for flush_work() and should
222	* not participate in pwq->nr_active. For non-barrier work item, it
223	* is marked with WORK_STRUCT_INACTIVE iff it is in pwq->inactive_works.
224	*/
225	int nr_active; /* L: nr of active works */
226	int max_active; /* L: max active works */
227	struct list_head inactive_works; /* L: inactive works */
228	struct list_head pwqs_node; /* WR: node on wq->pwqs */
229	struct list_head mayday_node; /* MD: node on wq->maydays */
230
231	/*
232	* Release of unbound pwq is punted to system_wq. See put_pwq()
233	* and pwq_unbound_release_workfn() for details. pool_workqueue
234	* itself is also RCU protected so that the first pwq can be
235	* determined without grabbing wq->mutex.
236	*/
237	struct work_struct unbound_release_work;
238	struct rcu_head rcu;
239	} __aligned(1 << WORK_STRUCT_FLAG_BITS);
240
241	/*
242	* Structure used to wait for workqueue flush.
243	*/
244	struct wq_flusher {
245	struct list_head list; /* WQ: list of flushers */
246	int flush_color; /* WQ: flush color waiting for */
247	struct completion done; /* flush completion */
248	};
249
250	struct wq_device;
251
252	/*
253	* The externally visible workqueue. It relays the issued work items to
254	* the appropriate worker_pool through its pool_workqueues.
255	*/
256	struct workqueue_struct {
257	struct list_head pwqs; /* WR: all pwqs of this wq */
258	struct list_head list; /* PR: list of all workqueues */
259
260	struct mutex mutex; /* protects this wq */
261	int work_color; /* WQ: current work color */
262	int flush_color; /* WQ: current flush color */
263	atomic_t nr_pwqs_to_flush; /* flush in progress */
264	struct wq_flusher first_flusher; / WQ: first flusher */
265	struct list_head flusher_queue; /* WQ: flush waiters */
266	struct list_head flusher_overflow; /* WQ: flush overflow list */
267
268	struct list_head maydays; /* MD: pwqs requesting rescue */
269	struct worker rescuer; / MD: rescue worker */
270
271	int nr_drainers; /* WQ: drain in progress */
272	int saved_max_active; /* WQ: saved pwq max_active */
273
274	struct workqueue_attrs unbound_attrs; / PW: only for unbound wqs */
275	struct pool_workqueue dfl_pwq; / PW: only for unbound wqs */
276
277	#ifdef CONFIG_SYSFS
278	struct wq_device wq_dev; / I: for sysfs interface */
279	#endif
280	#ifdef CONFIG_LOCKDEP
281	char *lock_name;
282	struct lock_class_key key;
283	struct lockdep_map lockdep_map;
284	#endif
285	char name[WQ_NAME_LEN]; /* I: workqueue name */
286
287	/*
288	* Destruction of workqueue_struct is RCU protected to allow walking
289	* the workqueues list without grabbing wq_pool_mutex.
290	* This is used to dump all workqueues from sysrq.
291	*/
292	struct rcu_head rcu;
293
294	/* hot fields used during command issue, aligned to cacheline */
295	unsigned int flags ____cacheline_aligned; /* WQ: WQ_* flags */
296	struct pool_workqueue __percpu cpu_pwqs; / I: per-cpu pwqs */
297	struct pool_workqueue __rcu numa_pwq_tbl[]; / PWR: unbound pwqs indexed by node */
298	};
299
300	static struct kmem_cache *pwq_cache;
301
302	static cpumask_var_t *wq_numa_possible_cpumask;
303	/* possible CPUs of each node */
304
305	static bool wq_disable_numa;
306	module_param_named(disable_numa, wq_disable_numa, bool, 0444);
307
308	/* see the comment above the definition of WQ_POWER_EFFICIENT */
309	static bool wq_power_efficient = IS_ENABLED(CONFIG_WQ_POWER_EFFICIENT_DEFAULT);
310	module_param_named(power_efficient, wq_power_efficient, bool, 0444);
311
312	static bool wq_online; /* can kworkers be created yet? */
313
314	static bool wq_numa_enabled; /* unbound NUMA affinity enabled */
315
316	/* buf for wq_update_unbound_numa_attrs(), protected by CPU hotplug exclusion */
317	static struct workqueue_attrs *wq_update_unbound_numa_attrs_buf;
318
319	static DEFINE_MUTEX(wq_pool_mutex); /* protects pools and workqueues list */
320	static DEFINE_MUTEX(wq_pool_attach_mutex); /* protects worker attach/detach */
321	static DEFINE_RAW_SPINLOCK(wq_mayday_lock); /* protects wq->maydays list */
322	/* wait for manager to go away */
323	static struct rcuwait manager_wait = __RCUWAIT_INITIALIZER(manager_wait);
324
325	static LIST_HEAD(workqueues); /* PR: list of all workqueues */
326	static bool workqueue_freezing; /* PL: have wqs started freezing? */
327
328	/* PL: allowable cpus for unbound wqs and work items */
329	static cpumask_var_t wq_unbound_cpumask;
330
331	/* CPU where unbound work was last round robin scheduled from this CPU */
332	static DEFINE_PER_CPU(int, wq_rr_cpu_last);
333
334	/*
335	* Local execution of unbound work items is no longer guaranteed. The
336	* following always forces round-robin CPU selection on unbound work items
337	* to uncover usages which depend on it.
338	*/
339	#ifdef CONFIG_DEBUG_WQ_FORCE_RR_CPU
340	static bool wq_debug_force_rr_cpu = true;
341	#else
342	static bool wq_debug_force_rr_cpu = false;
343	#endif
344	module_param_named(debug_force_rr_cpu, wq_debug_force_rr_cpu, bool, 0644);
345
346	/* the per-cpu worker pools */
347	static DEFINE_PER_CPU_SHARED_ALIGNED(struct worker_pool [NR_STD_WORKER_POOLS], cpu_worker_pools);
348
349	static DEFINE_IDR(worker_pool_idr); /* PR: idr of all pools */
350
351	/* PL: hash of all unbound pools keyed by pool->attrs */
352	static DEFINE_HASHTABLE(unbound_pool_hash, UNBOUND_POOL_HASH_ORDER);
353
354	/* I: attributes used when instantiating standard unbound pools on demand */
355	static struct workqueue_attrs *unbound_std_wq_attrs[NR_STD_WORKER_POOLS];
356
357	/* I: attributes used when instantiating ordered pools on demand */
358	static struct workqueue_attrs *ordered_wq_attrs[NR_STD_WORKER_POOLS];
359
360	struct workqueue_struct *system_wq __read_mostly;
361	EXPORT_SYMBOL(system_wq);
362	struct workqueue_struct *system_highpri_wq __read_mostly;
363	EXPORT_SYMBOL_GPL(system_highpri_wq);
364	struct workqueue_struct *system_long_wq __read_mostly;
365	EXPORT_SYMBOL_GPL(system_long_wq);
366	struct workqueue_struct *system_unbound_wq __read_mostly;
367	EXPORT_SYMBOL_GPL(system_unbound_wq);
368	struct workqueue_struct *system_freezable_wq __read_mostly;
369	EXPORT_SYMBOL_GPL(system_freezable_wq);
370	struct workqueue_struct *system_power_efficient_wq __read_mostly;
371	EXPORT_SYMBOL_GPL(system_power_efficient_wq);
372	struct workqueue_struct *system_freezable_power_efficient_wq __read_mostly;
373	EXPORT_SYMBOL_GPL(system_freezable_power_efficient_wq);
374
375	static int worker_thread(void *__worker);
376	static void workqueue_sysfs_unregister(struct workqueue_struct *wq);
377	static void show_pwq(struct pool_workqueue *pwq);
378	static void show_one_worker_pool(struct worker_pool *pool);
379
380	#define CREATE_TRACE_POINTS
381	#include <trace/events/workqueue.h>
382
383	#define assert_rcu_or_pool_mutex() \
384	RCU_LOCKDEP_WARN(!rcu_read_lock_held() && \
385	!lockdep_is_held(&wq_pool_mutex), \
386	"RCU or wq_pool_mutex should be held")
387
388	#define assert_rcu_or_wq_mutex_or_pool_mutex(wq) \
389	RCU_LOCKDEP_WARN(!rcu_read_lock_held() && \
390	!lockdep_is_held(&wq->mutex) && \
391	!lockdep_is_held(&wq_pool_mutex), \
392	"RCU, wq->mutex or wq_pool_mutex should be held")
393
394	#define for_each_cpu_worker_pool(pool, cpu) \
395	for ((pool) = &per_cpu(cpu_worker_pools, cpu)[0]; \
396	(pool) < &per_cpu(cpu_worker_pools, cpu)[NR_STD_WORKER_POOLS]; \
397	(pool)++)
398
399	/**
400	* for_each_pool - iterate through all worker_pools in the system
401	* @pool: iteration cursor
402	* @pi: integer used for iteration
403	*
404	* This must be called either with wq_pool_mutex held or RCU read
405	* locked. If the pool needs to be used beyond the locking in effect, the
406	* caller is responsible for guaranteeing that the pool stays online.
407	*
408	* The if/else clause exists only for the lockdep assertion and can be
409	* ignored.
410	*/
411	#define for_each_pool(pool, pi) \
412	idr_for_each_entry(&worker_pool_idr, pool, pi) \
413	if (({ assert_rcu_or_pool_mutex(); false; })) { } \
414	else
415
416	/**
417	* for_each_pool_worker - iterate through all workers of a worker_pool
418	* @worker: iteration cursor
419	* @pool: worker_pool to iterate workers of
420	*
421	* This must be called with wq_pool_attach_mutex.
422	*
423	* The if/else clause exists only for the lockdep assertion and can be
424	* ignored.
425	*/
426	#define for_each_pool_worker(worker, pool) \
427	list_for_each_entry((worker), &(pool)->workers, node) \
428	if (({ lockdep_assert_held(&wq_pool_attach_mutex); false; })) { } \
429	else
430
431	/**
432	* for_each_pwq - iterate through all pool_workqueues of the specified workqueue
433	* @pwq: iteration cursor
434	* @wq: the target workqueue
435	*
436	* This must be called either with wq->mutex held or RCU read locked.
437	* If the pwq needs to be used beyond the locking in effect, the caller is
438	* responsible for guaranteeing that the pwq stays online.
439	*
440	* The if/else clause exists only for the lockdep assertion and can be
441	* ignored.
442	*/
443	#define for_each_pwq(pwq, wq) \
444	list_for_each_entry_rcu((pwq), &(wq)->pwqs, pwqs_node, \
445	lockdep_is_held(&(wq->mutex)))
446
447	#ifdef CONFIG_DEBUG_OBJECTS_WORK
448
449	static const struct debug_obj_descr work_debug_descr;
450
451	static void work_debug_hint(void addr)
452	{
453	return ((struct work_struct *) addr)->func;
454	}
455
456	static bool work_is_static_object(void *addr)
457	{
458	struct work_struct *work = addr;
459
460	return test_bit(WORK_STRUCT_STATIC_BIT, work_data_bits(work));
461	}
462
463	/*
464	* fixup_init is called when:
465	* - an active object is initialized
466	*/
467	static bool work_fixup_init(void *addr, enum debug_obj_state state)
468	{
469	struct work_struct *work = addr;
470
471	switch (state) {
472	case ODEBUG_STATE_ACTIVE:
473	cancel_work_sync(work);
474	debug_object_init(work, &work_debug_descr);
475	return true;
476	default:
477	return false;
478	}
479	}
480
481	/*
482	* fixup_free is called when:
483	* - an active object is freed
484	*/
485	static bool work_fixup_free(void *addr, enum debug_obj_state state)
486	{
487	struct work_struct *work = addr;
488
489	switch (state) {
490	case ODEBUG_STATE_ACTIVE:
491	cancel_work_sync(work);
492	debug_object_free(work, &work_debug_descr);
493	return true;
494	default:
495	return false;
496	}
497	}
498
499	static const struct debug_obj_descr work_debug_descr = {
500	.name = "work_struct",
501	.debug_hint = work_debug_hint,
502	.is_static_object = work_is_static_object,
503	.fixup_init = work_fixup_init,
504	.fixup_free = work_fixup_free,
505	};
506
507	static inline void debug_work_activate(struct work_struct *work)
508	{
509	debug_object_activate(work, &work_debug_descr);
510	}
511
512	static inline void debug_work_deactivate(struct work_struct *work)
513	{
514	debug_object_deactivate(work, &work_debug_descr);
515	}
516
517	void __init_work(struct work_struct *work, int onstack)
518	{
519	if (onstack)
520	debug_object_init_on_stack(work, &work_debug_descr);
521	else
522	debug_object_init(work, &work_debug_descr);
523	}
524	EXPORT_SYMBOL_GPL(__init_work);
525
526	void destroy_work_on_stack(struct work_struct *work)
527	{
528	debug_object_free(work, &work_debug_descr);
529	}
530	EXPORT_SYMBOL_GPL(destroy_work_on_stack);
531
532	void destroy_delayed_work_on_stack(struct delayed_work *work)
533	{
534	destroy_timer_on_stack(&work->timer);
535	debug_object_free(&work->work, &work_debug_descr);
536	}
537	EXPORT_SYMBOL_GPL(destroy_delayed_work_on_stack);
538
539	#else
540	static inline void debug_work_activate(struct work_struct *work) { }
541	static inline void debug_work_deactivate(struct work_struct *work) { }
542	#endif
543
544	/**
545	* worker_pool_assign_id - allocate ID and assign it to @pool
546	* @pool: the pool pointer of interest
547	*
548	* Returns 0 if ID in [0, WORK_OFFQ_POOL_NONE) is allocated and assigned
549	* successfully, -errno on failure.
550	*/
551	static int worker_pool_assign_id(struct worker_pool *pool)
552	{
553	int ret;
554
555	lockdep_assert_held(&wq_pool_mutex);
556
557	ret = idr_alloc(&worker_pool_idr, pool, 0, WORK_OFFQ_POOL_NONE,
558	GFP_KERNEL);
559	if (ret >= 0) {
560	pool->id = ret;
561	return 0;
562	}
563	return ret;
564	}
565
566	/**
567	* unbound_pwq_by_node - return the unbound pool_workqueue for the given node
568	* @wq: the target workqueue
569	* @node: the node ID
570	*
571	* This must be called with any of wq_pool_mutex, wq->mutex or RCU
572	* read locked.
573	* If the pwq needs to be used beyond the locking in effect, the caller is
574	* responsible for guaranteeing that the pwq stays online.
575	*
576	* Return: The unbound pool_workqueue for @node.
577	*/
578	static struct pool_workqueue unbound_pwq_by_node(struct workqueue_struct wq,
579	int node)
580	{
581	assert_rcu_or_wq_mutex_or_pool_mutex(wq);
582
583	/*
584	* XXX: @node can be NUMA_NO_NODE if CPU goes offline while a
585	* delayed item is pending. The plan is to keep CPU -> NODE
586	* mapping valid and stable across CPU on/offlines. Once that
587	* happens, this workaround can be removed.
588	*/
589	if (unlikely(node == NUMA_NO_NODE))
590	return wq->dfl_pwq;
591
592	return rcu_dereference_raw(wq->numa_pwq_tbl[node]);
593	}
594
595	static unsigned int work_color_to_flags(int color)
596	{
597	return color << WORK_STRUCT_COLOR_SHIFT;
598	}
599
600	static int get_work_color(unsigned long work_data)
601	{
602	return (work_data >> WORK_STRUCT_COLOR_SHIFT) &
603	((1 << WORK_STRUCT_COLOR_BITS) - 1);
604	}
605
606	static int work_next_color(int color)
607	{
608	return (color + 1) % WORK_NR_COLORS;
609	}
610
611	/*
612	* While queued, %WORK_STRUCT_PWQ is set and non flag bits of a work's data
613	* contain the pointer to the queued pwq. Once execution starts, the flag
614	* is cleared and the high bits contain OFFQ flags and pool ID.
615	*
616	* set_work_pwq(), set_work_pool_and_clear_pending(), mark_work_canceling()
617	* and clear_work_data() can be used to set the pwq, pool or clear
618	* work->data. These functions should only be called while the work is
619	* owned - ie. while the PENDING bit is set.
620	*
621	* get_work_pool() and get_work_pwq() can be used to obtain the pool or pwq
622	* corresponding to a work. Pool is available once the work has been
623	* queued anywhere after initialization until it is sync canceled. pwq is
624	* available only while the work item is queued.
625	*
626	* %WORK_OFFQ_CANCELING is used to mark a work item which is being
627	* canceled. While being canceled, a work item may have its PENDING set
628	* but stay off timer and worklist for arbitrarily long and nobody should
629	* try to steal the PENDING bit.
630	*/
631	static inline void set_work_data(struct work_struct *work, unsigned long data,
632	unsigned long flags)
633	{
634	WARN_ON_ONCE(!work_pending(work));
635	atomic_long_set(&work->data, data \| flags \| work_static(work));
636	}
637
638	static void set_work_pwq(struct work_struct work, struct pool_workqueue pwq,
639	unsigned long extra_flags)
640	{
641	set_work_data(work, (unsigned long)pwq,
642	WORK_STRUCT_PENDING \| WORK_STRUCT_PWQ \| extra_flags);
643	}
644
645	static void set_work_pool_and_keep_pending(struct work_struct *work,
646	int pool_id)
647	{
648	set_work_data(work, (unsigned long)pool_id << WORK_OFFQ_POOL_SHIFT,
649	WORK_STRUCT_PENDING);
650	}
651
652	static void set_work_pool_and_clear_pending(struct work_struct *work,
653	int pool_id)
654	{
655	/*
656	* The following wmb is paired with the implied mb in
657	* test_and_set_bit(PENDING) and ensures all updates to @work made
658	* here are visible to and precede any updates by the next PENDING
659	* owner.
660	*/
661	smp_wmb();
662	set_work_data(work, (unsigned long)pool_id << WORK_OFFQ_POOL_SHIFT, 0);
663	/*
664	* The following mb guarantees that previous clear of a PENDING bit
665	* will not be reordered with any speculative LOADS or STORES from
666	* work->current_func, which is executed afterwards. This possible
667	* reordering can lead to a missed execution on attempt to queue
668	* the same @work. E.g. consider this case:
669	*
670	* CPU#0 CPU#1
671	* ---------------------------- --------------------------------
672	*
673	* 1 STORE event_indicated
674	* 2 queue_work_on() {
675	* 3 test_and_set_bit(PENDING)
676	* 4 } set_..._and_clear_pending() {
677	* 5 set_work_data() # clear bit
678	* 6 smp_mb()
679	* 7 work->current_func() {
680	* 8 LOAD event_indicated
681	* }
682	*
683	* Without an explicit full barrier speculative LOAD on line 8 can
684	* be executed before CPU#0 does STORE on line 1. If that happens,
685	* CPU#0 observes the PENDING bit is still set and new execution of
686	* a @work is not queued in a hope, that CPU#1 will eventually
687	* finish the queued @work. Meanwhile CPU#1 does not see
688	* event_indicated is set, because speculative LOAD was executed
689	* before actual STORE.
690	*/
691	smp_mb();
692	}
693
694	static void clear_work_data(struct work_struct *work)
695	{
696	smp_wmb(); /* see set_work_pool_and_clear_pending() */
697	set_work_data(work, WORK_STRUCT_NO_POOL, 0);
698	}
699
700	static struct pool_workqueue get_work_pwq(struct work_struct work)
701	{
702	unsigned long data = atomic_long_read(&work->data);
703
704	if (data & WORK_STRUCT_PWQ)
705	return (void *)(data & WORK_STRUCT_WQ_DATA_MASK);
706	else
707	return NULL;
708	}
709
710	/**
711	* get_work_pool - return the worker_pool a given work was associated with
712	* @work: the work item of interest
713	*
714	* Pools are created and destroyed under wq_pool_mutex, and allows read
715	* access under RCU read lock. As such, this function should be
716	* called under wq_pool_mutex or inside of a rcu_read_lock() region.
717	*
718	* All fields of the returned pool are accessible as long as the above
719	* mentioned locking is in effect. If the returned pool needs to be used
720	* beyond the critical section, the caller is responsible for ensuring the
721	* returned pool is and stays online.
722	*
723	* Return: The worker_pool @work was last associated with. %NULL if none.
724	*/
725	static struct worker_pool get_work_pool(struct work_struct work)
726	{
727	unsigned long data = atomic_long_read(&work->data);
728	int pool_id;
729
730	assert_rcu_or_pool_mutex();
731
732	if (data & WORK_STRUCT_PWQ)
733	return ((struct pool_workqueue *)
734	(data & WORK_STRUCT_WQ_DATA_MASK))->pool;
735
736	pool_id = data >> WORK_OFFQ_POOL_SHIFT;
737	if (pool_id == WORK_OFFQ_POOL_NONE)
738	return NULL;
739
740	return idr_find(&worker_pool_idr, pool_id);
741	}
742
743	/**
744	* get_work_pool_id - return the worker pool ID a given work is associated with
745	* @work: the work item of interest
746	*
747	* Return: The worker_pool ID @work was last associated with.
748	* %WORK_OFFQ_POOL_NONE if none.
749	*/
750	static int get_work_pool_id(struct work_struct *work)
751	{
752	unsigned long data = atomic_long_read(&work->data);
753
754	if (data & WORK_STRUCT_PWQ)
755	return ((struct pool_workqueue *)
756	(data & WORK_STRUCT_WQ_DATA_MASK))->pool->id;
757
758	return data >> WORK_OFFQ_POOL_SHIFT;
759	}
760
761	static void mark_work_canceling(struct work_struct *work)
762	{
763	unsigned long pool_id = get_work_pool_id(work);
764
765	pool_id <<= WORK_OFFQ_POOL_SHIFT;
766	set_work_data(work, pool_id \| WORK_OFFQ_CANCELING, WORK_STRUCT_PENDING);
767	}
768
769	static bool work_is_canceling(struct work_struct *work)
770	{
771	unsigned long data = atomic_long_read(&work->data);
772
773	return !(data & WORK_STRUCT_PWQ) && (data & WORK_OFFQ_CANCELING);
774	}
775
776	/*
777	* Policy functions. These define the policies on how the global worker
778	* pools are managed. Unless noted otherwise, these functions assume that
779	* they're being called with pool->lock held.
780	*/
781
782	static bool __need_more_worker(struct worker_pool *pool)
783	{
784	return !atomic_read(&pool->nr_running);
785	}
786
787	/*
788	* Need to wake up a worker? Called from anything but currently
789	* running workers.
790	*
791	* Note that, because unbound workers never contribute to nr_running, this
792	* function will always return %true for unbound pools as long as the
793	* worklist isn't empty.
794	*/
795	static bool need_more_worker(struct worker_pool *pool)
796	{
797	return !list_empty(&pool->worklist) && __need_more_worker(pool);
798	}
799
800	/* Can I start working? Called from busy but !running workers. */
801	static bool may_start_working(struct worker_pool *pool)
802	{
803	return pool->nr_idle;
804	}
805
806	/* Do I need to keep working? Called from currently running workers. */
807	static bool keep_working(struct worker_pool *pool)
808	{
809	return !list_empty(&pool->worklist) &&
810	atomic_read(&pool->nr_running) <= 1;
811	}
812
813	/* Do we need a new worker? Called from manager. */
814	static bool need_to_create_worker(struct worker_pool *pool)
815	{
816	return need_more_worker(pool) && !may_start_working(pool);
817	}
818
819	/* Do we have too many workers and should some go away? */
820	static bool too_many_workers(struct worker_pool *pool)
821	{
822	bool managing = pool->flags & POOL_MANAGER_ACTIVE;
823	int nr_idle = pool->nr_idle + managing; /* manager is considered idle */
824	int nr_busy = pool->nr_workers - nr_idle;
825
826	return nr_idle > 2 && (nr_idle - 2) * MAX_IDLE_WORKERS_RATIO >= nr_busy;
827	}
828
829	/*
830	* Wake up functions.
831	*/
832
833	/* Return the first idle worker. Safe with preemption disabled */
834	static struct worker first_idle_worker(struct worker_pool pool)
835	{
836	if (unlikely(list_empty(&pool->idle_list)))
837	return NULL;
838
839	return list_first_entry(&pool->idle_list, struct worker, entry);
840	}
841
842	/**
843	* wake_up_worker - wake up an idle worker
844	* @pool: worker pool to wake worker from
845	*
846	* Wake up the first idle worker of @pool.
847	*
848	* CONTEXT:
849	* raw_spin_lock_irq(pool->lock).
850	*/
851	static void wake_up_worker(struct worker_pool *pool)
852	{
853	struct worker *worker = first_idle_worker(pool);
854
855	if (likely(worker))
856	wake_up_process(worker->task);
857	}
858
859	/**
860	* wq_worker_running - a worker is running again
861	* @task: task waking up
862	*
863	* This function is called when a worker returns from schedule()
864	*/
865	void wq_worker_running(struct task_struct *task)
866	{
867	struct worker *worker = kthread_data(task);
868
869	if (!worker->sleeping)
870	return;
871	if (!(worker->flags & WORKER_NOT_RUNNING))
872	atomic_inc(&worker->pool->nr_running);
873	worker->sleeping = 0;
874	}
875
876	/**
877	* wq_worker_sleeping - a worker is going to sleep
878	* @task: task going to sleep
879	*
880	* This function is called from schedule() when a busy worker is
881	* going to sleep. Preemption needs to be disabled to protect ->sleeping
882	* assignment.
883	*/
884	void wq_worker_sleeping(struct task_struct *task)
885	{
886	struct worker next, worker = kthread_data(task);
887	struct worker_pool *pool;
888
889	/*
890	* Rescuers, which may not have all the fields set up like normal
891	* workers, also reach here, let's not access anything before
892	* checking NOT_RUNNING.
893	*/
894	if (worker->flags & WORKER_NOT_RUNNING)
895	return;
896
897	pool = worker->pool;
898
899	/* Return if preempted before wq_worker_running() was reached */
900	if (worker->sleeping)
901	return;
902
903	worker->sleeping = 1;
904	raw_spin_lock_irq(&pool->lock);
905
906	/*
907	* The counterpart of the following dec_and_test, implied mb,
908	* worklist not empty test sequence is in insert_work().
909	* Please read comment there.
910	*
911	* NOT_RUNNING is clear. This means that we're bound to and
912	* running on the local cpu w/ rq lock held and preemption
913	* disabled, which in turn means that none else could be
914	* manipulating idle_list, so dereferencing idle_list without pool
915	* lock is safe.
916	*/
917	if (atomic_dec_and_test(&pool->nr_running) &&
918	!list_empty(&pool->worklist)) {
919	next = first_idle_worker(pool);
920	if (next)
921	wake_up_process(next->task);
922	}
923	raw_spin_unlock_irq(&pool->lock);
924	}
925
926	/**
927	* wq_worker_last_func - retrieve worker's last work function
928	* @task: Task to retrieve last work function of.
929	*
930	* Determine the last function a worker executed. This is called from
931	* the scheduler to get a worker's last known identity.
932	*
933	* CONTEXT:
934	* raw_spin_lock_irq(rq->lock)
935	*
936	* This function is called during schedule() when a kworker is going
937	* to sleep. It's used by psi to identify aggregation workers during
938	* dequeuing, to allow periodic aggregation to shut-off when that
939	* worker is the last task in the system or cgroup to go to sleep.
940	*
941	* As this function doesn't involve any workqueue-related locking, it
942	* only returns stable values when called from inside the scheduler's
943	* queuing and dequeuing paths, when @task, which must be a kworker,
944	* is guaranteed to not be processing any works.
945	*
946	* Return:
947	* The last work function %current executed as a worker, NULL if it
948	* hasn't executed any work yet.
949	*/
950	work_func_t wq_worker_last_func(struct task_struct *task)
951	{
952	struct worker *worker = kthread_data(task);
953
954	return worker->last_func;
955	}
956
957	/**
958	* worker_set_flags - set worker flags and adjust nr_running accordingly
959	* @worker: self
960	* @flags: flags to set
961	*
962	* Set @flags in @worker->flags and adjust nr_running accordingly.
963	*
964	* CONTEXT:
965	* raw_spin_lock_irq(pool->lock)
966	*/
967	static inline void worker_set_flags(struct worker *worker, unsigned int flags)
968	{
969	struct worker_pool *pool = worker->pool;
970
971	WARN_ON_ONCE(worker->task != current);
972
973	/* If transitioning into NOT_RUNNING, adjust nr_running. */
974	if ((flags & WORKER_NOT_RUNNING) &&
975	!(worker->flags & WORKER_NOT_RUNNING)) {
976	atomic_dec(&pool->nr_running);
977	}
978
979	worker->flags \|= flags;
980	}
981
982	/**
983	* worker_clr_flags - clear worker flags and adjust nr_running accordingly
984	* @worker: self
985	* @flags: flags to clear
986	*
987	* Clear @flags in @worker->flags and adjust nr_running accordingly.
988	*
989	* CONTEXT:
990	* raw_spin_lock_irq(pool->lock)
991	*/
992	static inline void worker_clr_flags(struct worker *worker, unsigned int flags)
993	{
994	struct worker_pool *pool = worker->pool;
995	unsigned int oflags = worker->flags;
996
997	WARN_ON_ONCE(worker->task != current);
998
999	worker->flags &= ~flags;
1000
1001	/*
1002	* If transitioning out of NOT_RUNNING, increment nr_running. Note
1003	* that the nested NOT_RUNNING is not a noop. NOT_RUNNING is mask
1004	* of multiple flags, not a single flag.
1005	*/
1006	if ((flags & WORKER_NOT_RUNNING) && (oflags & WORKER_NOT_RUNNING))
1007	if (!(worker->flags & WORKER_NOT_RUNNING))
1008	atomic_inc(&pool->nr_running);
1009	}
1010
1011	/**
1012	* find_worker_executing_work - find worker which is executing a work
1013	* @pool: pool of interest
1014	* @work: work to find worker for
1015	*
1016	* Find a worker which is executing @work on @pool by searching
1017	* @pool->busy_hash which is keyed by the address of @work. For a worker
1018	* to match, its current execution should match the address of @work and
1019	* its work function. This is to avoid unwanted dependency between
1020	* unrelated work executions through a work item being recycled while still
1021	* being executed.
1022	*
1023	* This is a bit tricky. A work item may be freed once its execution
1024	* starts and nothing prevents the freed area from being recycled for
1025	* another work item. If the same work item address ends up being reused
1026	* before the original execution finishes, workqueue will identify the
1027	* recycled work item as currently executing and make it wait until the
1028	* current execution finishes, introducing an unwanted dependency.
1029	*
1030	* This function checks the work item address and work function to avoid
1031	* false positives. Note that this isn't complete as one may construct a
1032	* work function which can introduce dependency onto itself through a
1033	* recycled work item. Well, if somebody wants to shoot oneself in the
1034	* foot that badly, there's only so much we can do, and if such deadlock
1035	* actually occurs, it should be easy to locate the culprit work function.
1036	*
1037	* CONTEXT:
1038	* raw_spin_lock_irq(pool->lock).
1039	*
1040	* Return:
1041	* Pointer to worker which is executing @work if found, %NULL
1042	* otherwise.
1043	*/
1044	static struct worker find_worker_executing_work(struct worker_pool pool,
1045	struct work_struct *work)
1046	{
1047	struct worker *worker;
1048
1049	hash_for_each_possible(pool->busy_hash, worker, hentry,
1050	(unsigned long)work)
1051	if (worker->current_work == work &&
1052	worker->current_func == work->func)
1053	return worker;
1054
1055	return NULL;
1056	}
1057
1058	/**
1059	* move_linked_works - move linked works to a list
1060	* @work: start of series of works to be scheduled
1061	* @head: target list to append @work to
1062	* @nextp: out parameter for nested worklist walking
1063	*
1064	* Schedule linked works starting from @work to @head. Work series to
1065	* be scheduled starts at @work and includes any consecutive work with
1066	* WORK_STRUCT_LINKED set in its predecessor.
1067	*
1068	* If @nextp is not NULL, it's updated to point to the next work of
1069	* the last scheduled work. This allows move_linked_works() to be
1070	* nested inside outer list_for_each_entry_safe().
1071	*
1072	* CONTEXT:
1073	* raw_spin_lock_irq(pool->lock).
1074	*/
1075	static void move_linked_works(struct work_struct work, struct list_head head,
1076	struct work_struct **nextp)
1077	{
1078	struct work_struct *n;
1079
1080	/*
1081	* Linked worklist will always end before the end of the list,
1082	* use NULL for list head.
1083	*/
1084	list_for_each_entry_safe_from(work, n, NULL, entry) {
1085	list_move_tail(&work->entry, head);
1086	if (!(*work_data_bits(work) & WORK_STRUCT_LINKED))
1087	break;
1088	}
1089
1090	/*
1091	* If we're already inside safe list traversal and have moved
1092	* multiple works to the scheduled queue, the next position
1093	* needs to be updated.
1094	*/
1095	if (nextp)
1096	*nextp = n;
1097	}
1098
1099	/**
1100	* get_pwq - get an extra reference on the specified pool_workqueue
1101	* @pwq: pool_workqueue to get
1102	*
1103	* Obtain an extra reference on @pwq. The caller should guarantee that
1104	* @pwq has positive refcnt and be holding the matching pool->lock.
1105	*/
1106	static void get_pwq(struct pool_workqueue *pwq)
1107	{
1108	lockdep_assert_held(&pwq->pool->lock);
1109	WARN_ON_ONCE(pwq->refcnt <= 0);
1110	pwq->refcnt++;
1111	}
1112
1113	/**
1114	* put_pwq - put a pool_workqueue reference
1115	* @pwq: pool_workqueue to put
1116	*
1117	* Drop a reference of @pwq. If its refcnt reaches zero, schedule its
1118	* destruction. The caller should be holding the matching pool->lock.
1119	*/
1120	static void put_pwq(struct pool_workqueue *pwq)
1121	{
1122	lockdep_assert_held(&pwq->pool->lock);
1123	if (likely(--pwq->refcnt))
1124	return;
1125	if (WARN_ON_ONCE(!(pwq->wq->flags & WQ_UNBOUND)))
1126	return;
1127	/*
1128	* @pwq can't be released under pool->lock, bounce to
1129	* pwq_unbound_release_workfn(). This never recurses on the same
1130	* pool->lock as this path is taken only for unbound workqueues and
1131	* the release work item is scheduled on a per-cpu workqueue. To
1132	* avoid lockdep warning, unbound pool->locks are given lockdep
1133	* subclass of 1 in get_unbound_pool().
1134	*/
1135	schedule_work(&pwq->unbound_release_work);
1136	}
1137
1138	/**
1139	* put_pwq_unlocked - put_pwq() with surrounding pool lock/unlock
1140	* @pwq: pool_workqueue to put (can be %NULL)
1141	*
1142	* put_pwq() with locking. This function also allows %NULL @pwq.
1143	*/
1144	static void put_pwq_unlocked(struct pool_workqueue *pwq)
1145	{
1146	if (pwq) {
1147	/*
1148	* As both pwqs and pools are RCU protected, the
1149	* following lock operations are safe.
1150	*/
1151	raw_spin_lock_irq(&pwq->pool->lock);
1152	put_pwq(pwq);
1153	raw_spin_unlock_irq(&pwq->pool->lock);
1154	}
1155	}
1156
1157	static void pwq_activate_inactive_work(struct work_struct *work)
1158	{
1159	struct pool_workqueue *pwq = get_work_pwq(work);
1160
1161	trace_workqueue_activate_work(work);
1162	if (list_empty(&pwq->pool->worklist))
1163	pwq->pool->watchdog_ts = jiffies;
1164	move_linked_works(work, &pwq->pool->worklist, NULL);
1165	__clear_bit(WORK_STRUCT_INACTIVE_BIT, work_data_bits(work));
1166	pwq->nr_active++;
1167	}
1168
1169	static void pwq_activate_first_inactive(struct pool_workqueue *pwq)
1170	{
1171	struct work_struct *work = list_first_entry(&pwq->inactive_works,
1172	struct work_struct, entry);
1173
1174	pwq_activate_inactive_work(work);
1175	}
1176
1177	/**
1178	* pwq_dec_nr_in_flight - decrement pwq's nr_in_flight
1179	* @pwq: pwq of interest
1180	* @work_data: work_data of work which left the queue
1181	*
1182	* A work either has completed or is removed from pending queue,
1183	* decrement nr_in_flight of its pwq and handle workqueue flushing.
1184	*
1185	* CONTEXT:
1186	* raw_spin_lock_irq(pool->lock).
1187	*/
1188	static void pwq_dec_nr_in_flight(struct pool_workqueue *pwq, unsigned long work_data)
1189	{
1190	int color = get_work_color(work_data);
1191
1192	if (!(work_data & WORK_STRUCT_INACTIVE)) {
1193	pwq->nr_active--;
1194	if (!list_empty(&pwq->inactive_works)) {
1195	/* one down, submit an inactive one */
1196	if (pwq->nr_active < pwq->max_active)
1197	pwq_activate_first_inactive(pwq);
1198	}
1199	}
1200
1201	pwq->nr_in_flight[color]--;
1202
1203	/* is flush in progress and are we at the flushing tip? */
1204	if (likely(pwq->flush_color != color))
1205	goto out_put;
1206
1207	/* are there still in-flight works? */
1208	if (pwq->nr_in_flight[color])
1209	goto out_put;
1210
1211	/* this pwq is done, clear flush_color */
1212	pwq->flush_color = -1;
1213
1214	/*
1215	* If this was the last pwq, wake up the first flusher. It
1216	* will handle the rest.
1217	*/
1218	if (atomic_dec_and_test(&pwq->wq->nr_pwqs_to_flush))
1219	complete(&pwq->wq->first_flusher->done);
1220	out_put:
1221	put_pwq(pwq);
1222	}
1223
1224	/**
1225	* try_to_grab_pending - steal work item from worklist and disable irq
1226	* @work: work item to steal
1227	* @is_dwork: @work is a delayed_work
1228	* @flags: place to store irq state
1229	*
1230	* Try to grab PENDING bit of @work. This function can handle @work in any
1231	* stable state - idle, on timer or on worklist.
1232	*
1233	* Return:
1234	*
1235	* ======== ================================================================
1236	* 1 if @work was pending and we successfully stole PENDING
1237	* 0 if @work was idle and we claimed PENDING
1238	* -EAGAIN if PENDING couldn't be grabbed at the moment, safe to busy-retry
1239	* -ENOENT if someone else is canceling @work, this state may persist
1240	* for arbitrarily long
1241	* ======== ================================================================
1242	*
1243	* Note:
1244	* On >= 0 return, the caller owns @work's PENDING bit. To avoid getting
1245	* interrupted while holding PENDING and @work off queue, irq must be
1246	* disabled on entry. This, combined with delayed_work->timer being
1247	* irqsafe, ensures that we return -EAGAIN for finite short period of time.
1248	*
1249	* On successful return, >= 0, irq is disabled and the caller is
1250	* responsible for releasing it using local_irq_restore(*@flags).
1251	*
1252	* This function is safe to call from any context including IRQ handler.
1253	*/
1254	static int try_to_grab_pending(struct work_struct *work, bool is_dwork,
1255	unsigned long *flags)
1256	{
1257	struct worker_pool *pool;
1258	struct pool_workqueue *pwq;
1259
1260	local_irq_save(*flags);
1261
1262	/* try to steal the timer if it exists */
1263	if (is_dwork) {
1264	struct delayed_work *dwork = to_delayed_work(work);
1265
1266	/*
1267	* dwork->timer is irqsafe. If del_timer() fails, it's
1268	* guaranteed that the timer is not queued anywhere and not
1269	* running on the local CPU.
1270	*/
1271	if (likely(del_timer(&dwork->timer)))
1272	return 1;
1273	}
1274
1275	/* try to claim PENDING the normal way */
1276	if (!test_and_set_bit(WORK_STRUCT_PENDING_BIT, work_data_bits(work)))
1277	return 0;
1278
1279	rcu_read_lock();
1280	/*
1281	* The queueing is in progress, or it is already queued. Try to
1282	* steal it from ->worklist without clearing WORK_STRUCT_PENDING.
1283	*/
1284	pool = get_work_pool(work);
1285	if (!pool)
1286	goto fail;
1287
1288	raw_spin_lock(&pool->lock);
1289	/*
1290	* work->data is guaranteed to point to pwq only while the work
1291	* item is queued on pwq->wq, and both updating work->data to point
1292	* to pwq on queueing and to pool on dequeueing are done under
1293	* pwq->pool->lock. This in turn guarantees that, if work->data
1294	* points to pwq which is associated with a locked pool, the work
1295	* item is currently queued on that pool.
1296	*/
1297	pwq = get_work_pwq(work);
1298	if (pwq && pwq->pool == pool) {
1299	debug_work_deactivate(work);
1300
1301	/*
1302	* A cancelable inactive work item must be in the
1303	* pwq->inactive_works since a queued barrier can't be
1304	* canceled (see the comments in insert_wq_barrier()).
1305	*
1306	* An inactive work item cannot be grabbed directly because
1307	* it might have linked barrier work items which, if left
1308	* on the inactive_works list, will confuse pwq->nr_active
1309	* management later on and cause stall. Make sure the work
1310	* item is activated before grabbing.
1311	*/
1312	if (*work_data_bits(work) & WORK_STRUCT_INACTIVE)
1313	pwq_activate_inactive_work(work);
1314
1315	list_del_init(&work->entry);
1316	pwq_dec_nr_in_flight(pwq, *work_data_bits(work));
1317
1318	/* work->data points to pwq iff queued, point to pool */
1319	set_work_pool_and_keep_pending(work, pool->id);
1320
1321	raw_spin_unlock(&pool->lock);
1322	rcu_read_unlock();
1323	return 1;
1324	}
1325	raw_spin_unlock(&pool->lock);
1326	fail:
1327	rcu_read_unlock();
1328	local_irq_restore(*flags);
1329	if (work_is_canceling(work))
1330	return -ENOENT;
1331	cpu_relax();
1332	return -EAGAIN;
1333	}
1334
1335	/**
1336	* insert_work - insert a work into a pool
1337	* @pwq: pwq @work belongs to
1338	* @work: work to insert
1339	* @head: insertion point
1340	* @extra_flags: extra WORK_STRUCT_* flags to set
1341	*
1342	* Insert @work which belongs to @pwq after @head. @extra_flags is or'd to
1343	* work_struct flags.
1344	*
1345	* CONTEXT:
1346	* raw_spin_lock_irq(pool->lock).
1347	*/
1348	static void insert_work(struct pool_workqueue pwq, struct work_struct work,
1349	struct list_head *head, unsigned int extra_flags)
1350	{
1351	struct worker_pool *pool = pwq->pool;
1352
1353	/* record the work call stack in order to print it in KASAN reports */
1354	kasan_record_aux_stack_noalloc(work);
1355
1356	/* we own @work, set data and link */
1357	set_work_pwq(work, pwq, extra_flags);
1358	list_add_tail(&work->entry, head);
1359	get_pwq(pwq);
1360
1361	/*
1362	* Ensure either wq_worker_sleeping() sees the above
1363	* list_add_tail() or we see zero nr_running to avoid workers lying
1364	* around lazily while there are works to be processed.
1365	*/
1366	smp_mb();
1367
1368	if (__need_more_worker(pool))
1369	wake_up_worker(pool);
1370	}
1371
1372	/*
1373	* Test whether @work is being queued from another work executing on the
1374	* same workqueue.
1375	*/
1376	static bool is_chained_work(struct workqueue_struct *wq)
1377	{
1378	struct worker *worker;
1379
1380	worker = current_wq_worker();
1381	/*
1382	* Return %true iff I'm a worker executing a work item on @wq. If
1383	* I'm @worker, it's safe to dereference it without locking.
1384	*/
1385	return worker && worker->current_pwq->wq == wq;
1386	}
1387
1388	/*
1389	* When queueing an unbound work item to a wq, prefer local CPU if allowed
1390	* by wq_unbound_cpumask. Otherwise, round robin among the allowed ones to
1391	* avoid perturbing sensitive tasks.
1392	*/
1393	static int wq_select_unbound_cpu(int cpu)
1394	{
1395	static bool printed_dbg_warning;
1396	int new_cpu;
1397
1398	if (likely(!wq_debug_force_rr_cpu)) {
1399	if (cpumask_test_cpu(cpu, wq_unbound_cpumask))
1400	return cpu;
1401	} else if (!printed_dbg_warning) {
1402	pr_warn("workqueue: round-robin CPU selection forced, expect performance impact\n");
1403	printed_dbg_warning = true;
1404	}
1405
1406	if (cpumask_empty(wq_unbound_cpumask))
1407	return cpu;
1408
1409	new_cpu = __this_cpu_read(wq_rr_cpu_last);
1410	new_cpu = cpumask_next_and(new_cpu, wq_unbound_cpumask, cpu_online_mask);
1411	if (unlikely(new_cpu >= nr_cpu_ids)) {
1412	new_cpu = cpumask_first_and(wq_unbound_cpumask, cpu_online_mask);
1413	if (unlikely(new_cpu >= nr_cpu_ids))
1414	return cpu;
1415	}
1416	__this_cpu_write(wq_rr_cpu_last, new_cpu);
1417
1418	return new_cpu;
1419	}
1420
1421	static void __queue_work(int cpu, struct workqueue_struct *wq,
1422	struct work_struct *work)
1423	{
1424	struct pool_workqueue *pwq;
1425	struct worker_pool *last_pool;
1426	struct list_head *worklist;
1427	unsigned int work_flags;
1428	unsigned int req_cpu = cpu;
1429
1430	/*
1431	* While a work item is PENDING && off queue, a task trying to
1432	* steal the PENDING will busy-loop waiting for it to either get
1433	* queued or lose PENDING. Grabbing PENDING and queueing should
1434	* happen with IRQ disabled.
1435	*/
1436	lockdep_assert_irqs_disabled();
1437
1438
1439	/* if draining, only works from the same workqueue are allowed */
1440	if (unlikely(wq->flags & __WQ_DRAINING) &&
1441	WARN_ON_ONCE(!is_chained_work(wq)))
1442	return;
1443	rcu_read_lock();
1444	retry:
1445	/* pwq which will be used unless @work is executing elsewhere */
1446	if (wq->flags & WQ_UNBOUND) {
1447	if (req_cpu == WORK_CPU_UNBOUND)
1448	cpu = wq_select_unbound_cpu(raw_smp_processor_id());
1449	pwq = unbound_pwq_by_node(wq, cpu_to_node(cpu));
1450	} else {
1451	if (req_cpu == WORK_CPU_UNBOUND)
1452	cpu = raw_smp_processor_id();
1453	pwq = per_cpu_ptr(wq->cpu_pwqs, cpu);
1454	}
1455
1456	/*
1457	* If @work was previously on a different pool, it might still be
1458	* running there, in which case the work needs to be queued on that
1459	* pool to guarantee non-reentrancy.
1460	*/
1461	last_pool = get_work_pool(work);
1462	if (last_pool && last_pool != pwq->pool) {
1463	struct worker *worker;
1464
1465	raw_spin_lock(&last_pool->lock);
1466
1467	worker = find_worker_executing_work(last_pool, work);
1468
1469	if (worker && worker->current_pwq->wq == wq) {
1470	pwq = worker->current_pwq;
1471	} else {
1472	/* meh... not running there, queue here */
1473	raw_spin_unlock(&last_pool->lock);
1474	raw_spin_lock(&pwq->pool->lock);
1475	}
1476	} else {
1477	raw_spin_lock(&pwq->pool->lock);
1478	}
1479
1480	/*
1481	* pwq is determined and locked. For unbound pools, we could have
1482	* raced with pwq release and it could already be dead. If its
1483	* refcnt is zero, repeat pwq selection. Note that pwqs never die
1484	* without another pwq replacing it in the numa_pwq_tbl or while
1485	* work items are executing on it, so the retrying is guaranteed to
1486	* make forward-progress.
1487	*/
1488	if (unlikely(!pwq->refcnt)) {
1489	if (wq->flags & WQ_UNBOUND) {
1490	raw_spin_unlock(&pwq->pool->lock);
1491	cpu_relax();
1492	goto retry;
1493	}
1494	/* oops */
1495	WARN_ONCE(true, "workqueue: per-cpu pwq for %s on cpu%d has 0 refcnt",
1496	wq->name, cpu);
1497	}
1498
1499	/* pwq determined, queue */
1500	trace_workqueue_queue_work(req_cpu, pwq, work);
1501
1502	if (WARN_ON(!list_empty(&work->entry)))
1503	goto out;
1504
1505	pwq->nr_in_flight[pwq->work_color]++;
1506	work_flags = work_color_to_flags(pwq->work_color);
1507
1508	if (likely(pwq->nr_active < pwq->max_active)) {
1509	trace_workqueue_activate_work(work);
1510	pwq->nr_active++;
1511	worklist = &pwq->pool->worklist;
1512	if (list_empty(worklist))
1513	pwq->pool->watchdog_ts = jiffies;
1514	} else {
1515	work_flags \|= WORK_STRUCT_INACTIVE;
1516	worklist = &pwq->inactive_works;
1517	}
1518
1519	debug_work_activate(work);
1520	insert_work(pwq, work, worklist, work_flags);
1521
1522	out:
1523	raw_spin_unlock(&pwq->pool->lock);
1524	rcu_read_unlock();
1525	}
1526
1527	/**
1528	* queue_work_on - queue work on specific cpu
1529	* @cpu: CPU number to execute work on
1530	* @wq: workqueue to use
1531	* @work: work to queue
1532	*
1533	* We queue the work to a specific CPU, the caller must ensure it
1534	* can't go away.
1535	*
1536	* Return: %false if @work was already on a queue, %true otherwise.
1537	*/
1538	bool queue_work_on(int cpu, struct workqueue_struct *wq,
1539	struct work_struct *work)
1540	{
1541	bool ret = false;
1542	unsigned long flags;
1543
1544	local_irq_save(flags);
1545
1546	if (!test_and_set_bit(WORK_STRUCT_PENDING_BIT, work_data_bits(work))) {
1547	__queue_work(cpu, wq, work);
1548	ret = true;
1549	}
1550
1551	local_irq_restore(flags);
1552	return ret;
1553	}
1554	EXPORT_SYMBOL(queue_work_on);
1555
1556	/**
1557	* workqueue_select_cpu_near - Select a CPU based on NUMA node
1558	* @node: NUMA node ID that we want to select a CPU from
1559	*
1560	* This function will attempt to find a "random" cpu available on a given
1561	* node. If there are no CPUs available on the given node it will return
1562	* WORK_CPU_UNBOUND indicating that we should just schedule to any
1563	* available CPU if we need to schedule this work.
1564	*/
1565	static int workqueue_select_cpu_near(int node)
1566	{
1567	int cpu;
1568
1569	/* No point in doing this if NUMA isn't enabled for workqueues */
1570	if (!wq_numa_enabled)
1571	return WORK_CPU_UNBOUND;
1572
1573	/* Delay binding to CPU if node is not valid or online */
1574	if (node < 0 \|\| node >= MAX_NUMNODES \|\| !node_online(node))
1575	return WORK_CPU_UNBOUND;
1576
1577	/* Use local node/cpu if we are already there */
1578	cpu = raw_smp_processor_id();
1579	if (node == cpu_to_node(cpu))
1580	return cpu;
1581
1582	/* Use "random" otherwise know as "first" online CPU of node */
1583	cpu = cpumask_any_and(cpumask_of_node(node), cpu_online_mask);
1584
1585	/* If CPU is valid return that, otherwise just defer */
1586	return cpu < nr_cpu_ids ? cpu : WORK_CPU_UNBOUND;
1587	}
1588
1589	/**
1590	* queue_work_node - queue work on a "random" cpu for a given NUMA node
1591	* @node: NUMA node that we are targeting the work for
1592	* @wq: workqueue to use
1593	* @work: work to queue
1594	*
1595	* We queue the work to a "random" CPU within a given NUMA node. The basic
1596	* idea here is to provide a way to somehow associate work with a given
1597	* NUMA node.
1598	*
1599	* This function will only make a best effort attempt at getting this onto
1600	* the right NUMA node. If no node is requested or the requested node is
1601	* offline then we just fall back to standard queue_work behavior.
1602	*
1603	* Currently the "random" CPU ends up being the first available CPU in the
1604	* intersection of cpu_online_mask and the cpumask of the node, unless we
1605	* are running on the node. In that case we just use the current CPU.
1606	*
1607	* Return: %false if @work was already on a queue, %true otherwise.
1608	*/
1609	bool queue_work_node(int node, struct workqueue_struct *wq,
1610	struct work_struct *work)
1611	{
1612	unsigned long flags;
1613	bool ret = false;
1614
1615	/*
1616	* This current implementation is specific to unbound workqueues.
1617	* Specifically we only return the first available CPU for a given
1618	* node instead of cycling through individual CPUs within the node.
1619	*
1620	* If this is used with a per-cpu workqueue then the logic in
1621	* workqueue_select_cpu_near would need to be updated to allow for
1622	* some round robin type logic.
1623	*/
1624	WARN_ON_ONCE(!(wq->flags & WQ_UNBOUND));
1625
1626	local_irq_save(flags);
1627
1628	if (!test_and_set_bit(WORK_STRUCT_PENDING_BIT, work_data_bits(work))) {
1629	int cpu = workqueue_select_cpu_near(node);
1630
1631	__queue_work(cpu, wq, work);
1632	ret = true;
1633	}
1634
1635	local_irq_restore(flags);
1636	return ret;
1637	}
1638	EXPORT_SYMBOL_GPL(queue_work_node);
1639
1640	void delayed_work_timer_fn(struct timer_list *t)
1641	{
1642	struct delayed_work *dwork = from_timer(dwork, t, timer);
1643
1644	/* should have been called from irqsafe timer with irq already off */
1645	__queue_work(dwork->cpu, dwork->wq, &dwork->work);
1646	}
1647	EXPORT_SYMBOL(delayed_work_timer_fn);
1648
1649	static void __queue_delayed_work(int cpu, struct workqueue_struct *wq,
1650	struct delayed_work *dwork, unsigned long delay)
1651	{
1652	struct timer_list *timer = &dwork->timer;
1653	struct work_struct *work = &dwork->work;
1654
1655	WARN_ON_ONCE(!wq);
1656	WARN_ON_FUNCTION_MISMATCH(timer->function, delayed_work_timer_fn);
1657	WARN_ON_ONCE(timer_pending(timer));
1658	WARN_ON_ONCE(!list_empty(&work->entry));
1659
1660	/*
1661	* If @delay is 0, queue @dwork->work immediately. This is for
1662	* both optimization and correctness. The earliest @timer can
1663	* expire is on the closest next tick and delayed_work users depend
1664	* on that there's no such delay when @delay is 0.
1665	*/
1666	if (!delay) {
1667	__queue_work(cpu, wq, &dwork->work);
1668	return;
1669	}
1670
1671	dwork->wq = wq;
1672	dwork->cpu = cpu;
1673	timer->expires = jiffies + delay;
1674
1675	if (unlikely(cpu != WORK_CPU_UNBOUND))
1676	add_timer_on(timer, cpu);
1677	else
1678	add_timer(timer);
1679	}
1680
1681	/**
1682	* queue_delayed_work_on - queue work on specific CPU after delay
1683	* @cpu: CPU number to execute work on
1684	* @wq: workqueue to use
1685	* @dwork: work to queue
1686	* @delay: number of jiffies to wait before queueing
1687	*
1688	* Return: %false if @work was already on a queue, %true otherwise. If
1689	* @delay is zero and @dwork is idle, it will be scheduled for immediate
1690	* execution.
1691	*/
1692	bool queue_delayed_work_on(int cpu, struct workqueue_struct *wq,
1693	struct delayed_work *dwork, unsigned long delay)
1694	{
1695	struct work_struct *work = &dwork->work;
1696	bool ret = false;
1697	unsigned long flags;
1698
1699	/* read the comment in __queue_work() */
1700	local_irq_save(flags);
1701
1702	if (!test_and_set_bit(WORK_STRUCT_PENDING_BIT, work_data_bits(work))) {
1703	__queue_delayed_work(cpu, wq, dwork, delay);
1704	ret = true;
1705	}
1706
1707	local_irq_restore(flags);
1708	return ret;
1709	}
1710	EXPORT_SYMBOL(queue_delayed_work_on);
1711
1712	/**
1713	* mod_delayed_work_on - modify delay of or queue a delayed work on specific CPU
1714	* @cpu: CPU number to execute work on
1715	* @wq: workqueue to use
1716	* @dwork: work to queue
1717	* @delay: number of jiffies to wait before queueing
1718	*
1719	* If @dwork is idle, equivalent to queue_delayed_work_on(); otherwise,
1720	* modify @dwork's timer so that it expires after @delay. If @delay is
1721	* zero, @work is guaranteed to be scheduled immediately regardless of its
1722	* current state.
1723	*
1724	* Return: %false if @dwork was idle and queued, %true if @dwork was
1725	* pending and its timer was modified.
1726	*
1727	* This function is safe to call from any context including IRQ handler.
1728	* See try_to_grab_pending() for details.
1729	*/
1730	bool mod_delayed_work_on(int cpu, struct workqueue_struct *wq,
1731	struct delayed_work *dwork, unsigned long delay)
1732	{
1733	unsigned long flags;
1734	int ret;
1735
1736	do {
1737	ret = try_to_grab_pending(&dwork->work, true, &flags);
1738	} while (unlikely(ret == -EAGAIN));
1739
1740	if (likely(ret >= 0)) {
1741	__queue_delayed_work(cpu, wq, dwork, delay);
1742	local_irq_restore(flags);
1743	}
1744
1745	/* -ENOENT from try_to_grab_pending() becomes %true */
1746	return ret;
1747	}
1748	EXPORT_SYMBOL_GPL(mod_delayed_work_on);
1749
1750	static void rcu_work_rcufn(struct rcu_head *rcu)
1751	{
1752	struct rcu_work *rwork = container_of(rcu, struct rcu_work, rcu);
1753
1754	/* read the comment in __queue_work() */
1755	local_irq_disable();
1756	__queue_work(WORK_CPU_UNBOUND, rwork->wq, &rwork->work);
1757	local_irq_enable();
1758	}
1759
1760	/**
1761	* queue_rcu_work - queue work after a RCU grace period
1762	* @wq: workqueue to use
1763	* @rwork: work to queue
1764	*
1765	* Return: %false if @rwork was already pending, %true otherwise. Note
1766	* that a full RCU grace period is guaranteed only after a %true return.
1767	* While @rwork is guaranteed to be executed after a %false return, the
1768	* execution may happen before a full RCU grace period has passed.
1769	*/
1770	bool queue_rcu_work(struct workqueue_struct wq, struct rcu_work rwork)
1771	{
1772	struct work_struct *work = &rwork->work;
1773
1774	if (!test_and_set_bit(WORK_STRUCT_PENDING_BIT, work_data_bits(work))) {
1775	rwork->wq = wq;
1776	call_rcu(&rwork->rcu, rcu_work_rcufn);
1777	return true;
1778	}
1779
1780	return false;
1781	}
1782	EXPORT_SYMBOL(queue_rcu_work);
1783
1784	/**
1785	* worker_enter_idle - enter idle state
1786	* @worker: worker which is entering idle state
1787	*
1788	* @worker is entering idle state. Update stats and idle timer if
1789	* necessary.
1790	*
1791	* LOCKING:
1792	* raw_spin_lock_irq(pool->lock).
1793	*/
1794	static void worker_enter_idle(struct worker *worker)
1795	{
1796	struct worker_pool *pool = worker->pool;
1797
1798	if (WARN_ON_ONCE(worker->flags & WORKER_IDLE) \|\|
1799	WARN_ON_ONCE(!list_empty(&worker->entry) &&
1800	(worker->hentry.next \|\| worker->hentry.pprev)))
1801	return;
1802
1803	/* can't use worker_set_flags(), also called from create_worker() */
1804	worker->flags \|= WORKER_IDLE;
1805	pool->nr_idle++;
1806	worker->last_active = jiffies;
1807
1808	/* idle_list is LIFO */
1809	list_add(&worker->entry, &pool->idle_list);
1810
1811	if (too_many_workers(pool) && !timer_pending(&pool->idle_timer))
1812	mod_timer(&pool->idle_timer, jiffies + IDLE_WORKER_TIMEOUT);
1813
1814	/*
1815	* Sanity check nr_running. Because unbind_workers() releases
1816	* pool->lock between setting %WORKER_UNBOUND and zapping
1817	* nr_running, the warning may trigger spuriously. Check iff
1818	* unbind is not in progress.
1819	*/
1820	WARN_ON_ONCE(!(pool->flags & POOL_DISASSOCIATED) &&
1821	pool->nr_workers == pool->nr_idle &&
1822	atomic_read(&pool->nr_running));
1823	}
1824
1825	/**
1826	* worker_leave_idle - leave idle state
1827	* @worker: worker which is leaving idle state
1828	*
1829	* @worker is leaving idle state. Update stats.
1830	*
1831	* LOCKING:
1832	* raw_spin_lock_irq(pool->lock).
1833	*/
1834	static void worker_leave_idle(struct worker *worker)
1835	{
1836	struct worker_pool *pool = worker->pool;
1837
1838	if (WARN_ON_ONCE(!(worker->flags & WORKER_IDLE)))
1839	return;
1840	worker_clr_flags(worker, WORKER_IDLE);
1841	pool->nr_idle--;
1842	list_del_init(&worker->entry);
1843	}
1844
1845	static struct worker *alloc_worker(int node)
1846	{
1847	struct worker *worker;
1848
1849	worker = kzalloc_node(sizeof(*worker), GFP_KERNEL, node);
1850	if (worker) {
1851	INIT_LIST_HEAD(&worker->entry);
1852	INIT_LIST_HEAD(&worker->scheduled);
1853	INIT_LIST_HEAD(&worker->node);
1854	/* on creation a worker is in !idle && prep state */
1855	worker->flags = WORKER_PREP;
1856	}
1857	return worker;
1858	}
1859
1860	/**
1861	* worker_attach_to_pool() - attach a worker to a pool
1862	* @worker: worker to be attached
1863	* @pool: the target pool
1864	*
1865	* Attach @worker to @pool. Once attached, the %WORKER_UNBOUND flag and
1866	* cpu-binding of @worker are kept coordinated with the pool across
1867	* cpu-[un]hotplugs.
1868	*/
1869	static void worker_attach_to_pool(struct worker *worker,
1870	struct worker_pool *pool)
1871	{
1872	mutex_lock(&wq_pool_attach_mutex);
1873
1874	/*
1875	* The wq_pool_attach_mutex ensures %POOL_DISASSOCIATED remains
1876	* stable across this function. See the comments above the flag
1877	* definition for details.
1878	*/
1879	if (pool->flags & POOL_DISASSOCIATED)
1880	worker->flags \|= WORKER_UNBOUND;
1881	else
1882	kthread_set_per_cpu(worker->task, pool->cpu);
1883
1884	if (worker->rescue_wq)
1885	set_cpus_allowed_ptr(worker->task, pool->attrs->cpumask);
1886
1887	list_add_tail(&worker->node, &pool->workers);
1888	worker->pool = pool;
1889
1890	mutex_unlock(&wq_pool_attach_mutex);
1891	}
1892
1893	/**
1894	* worker_detach_from_pool() - detach a worker from its pool
1895	* @worker: worker which is attached to its pool
1896	*
1897	* Undo the attaching which had been done in worker_attach_to_pool(). The
1898	* caller worker shouldn't access to the pool after detached except it has
1899	* other reference to the pool.
1900	*/
1901	static void worker_detach_from_pool(struct worker *worker)
1902	{
1903	struct worker_pool *pool = worker->pool;
1904	struct completion *detach_completion = NULL;
1905
1906	mutex_lock(&wq_pool_attach_mutex);
1907
1908	kthread_set_per_cpu(worker->task, -1);
1909	list_del(&worker->node);
1910	worker->pool = NULL;
1911
1912	if (list_empty(&pool->workers))
1913	detach_completion = pool->detach_completion;
1914	mutex_unlock(&wq_pool_attach_mutex);
1915
1916	/* clear leftover flags without pool->lock after it is detached */
1917	worker->flags &= ~(WORKER_UNBOUND \| WORKER_REBOUND);
1918
1919	if (detach_completion)
1920	complete(detach_completion);
1921	}
1922
1923	/**
1924	* create_worker - create a new workqueue worker
1925	* @pool: pool the new worker will belong to
1926	*
1927	* Create and start a new worker which is attached to @pool.
1928	*
1929	* CONTEXT:
1930	* Might sleep. Does GFP_KERNEL allocations.
1931	*
1932	* Return:
1933	* Pointer to the newly created worker.
1934	*/
1935	static struct worker create_worker(struct worker_pool pool)
1936	{
1937	struct worker *worker;
1938	int id;
1939	char id_buf[16];
1940
1941	/* ID is needed to determine kthread name */
1942	id = ida_alloc(&pool->worker_ida, GFP_KERNEL);
1943	if (id < 0)
1944	return NULL;
1945
1946	worker = alloc_worker(pool->node);
1947	if (!worker)
1948	goto fail;
1949
1950	worker->id = id;
1951
1952	if (pool->cpu >= 0)
1953	snprintf(id_buf, sizeof(id_buf), "%d:%d%s", pool->cpu, id,
1954	pool->attrs->nice < 0 ? "H" : "");
1955	else
1956	snprintf(id_buf, sizeof(id_buf), "u%d:%d", pool->id, id);
1957
1958	worker->task = kthread_create_on_node(worker_thread, worker, pool->node,
1959	"kworker/%s", id_buf);
1960	if (IS_ERR(worker->task))
1961	goto fail;
1962
1963	set_user_nice(worker->task, pool->attrs->nice);
1964	kthread_bind_mask(worker->task, pool->attrs->cpumask);
1965
1966	/* successful, attach the worker to the pool */
1967	worker_attach_to_pool(worker, pool);
1968
1969	/* start the newly created worker */
1970	raw_spin_lock_irq(&pool->lock);
1971	worker->pool->nr_workers++;
1972	worker_enter_idle(worker);
1973	wake_up_process(worker->task);
1974	raw_spin_unlock_irq(&pool->lock);
1975
1976	return worker;
1977
1978	fail:
1979	ida_free(&pool->worker_ida, id);
1980	kfree(worker);
1981	return NULL;
1982	}
1983
1984	/**
1985	* destroy_worker - destroy a workqueue worker
1986	* @worker: worker to be destroyed
1987	*
1988	* Destroy @worker and adjust @pool stats accordingly. The worker should
1989	* be idle.
1990	*
1991	* CONTEXT:
1992	* raw_spin_lock_irq(pool->lock).
1993	*/
1994	static void destroy_worker(struct worker *worker)
1995	{
1996	struct worker_pool *pool = worker->pool;
1997
1998	lockdep_assert_held(&pool->lock);
1999
2000	/* sanity check frenzy */
2001	if (WARN_ON(worker->current_work) \|\|
2002	WARN_ON(!list_empty(&worker->scheduled)) \|\|
2003	WARN_ON(!(worker->flags & WORKER_IDLE)))
2004	return;
2005
2006	pool->nr_workers--;
2007	pool->nr_idle--;
2008
2009	list_del_init(&worker->entry);
2010	worker->flags \|= WORKER_DIE;
2011	wake_up_process(worker->task);
2012	}
2013
2014	static void idle_worker_timeout(struct timer_list *t)
2015	{
2016	struct worker_pool *pool = from_timer(pool, t, idle_timer);
2017
2018	raw_spin_lock_irq(&pool->lock);
2019
2020	while (too_many_workers(pool)) {
2021	struct worker *worker;
2022	unsigned long expires;
2023
2024	/* idle_list is kept in LIFO order, check the last one */
2025	worker = list_entry(pool->idle_list.prev, struct worker, entry);
2026	expires = worker->last_active + IDLE_WORKER_TIMEOUT;
2027
2028	if (time_before(jiffies, expires)) {
2029	mod_timer(&pool->idle_timer, expires);
2030	break;
2031	}
2032
2033	destroy_worker(worker);
2034	}
2035
2036	raw_spin_unlock_irq(&pool->lock);
2037	}
2038
2039	static void send_mayday(struct work_struct *work)
2040	{
2041	struct pool_workqueue *pwq = get_work_pwq(work);
2042	struct workqueue_struct *wq = pwq->wq;
2043
2044	lockdep_assert_held(&wq_mayday_lock);
2045
2046	if (!wq->rescuer)
2047	return;
2048
2049	/* mayday mayday mayday */
2050	if (list_empty(&pwq->mayday_node)) {
2051	/*
2052	* If @pwq is for an unbound wq, its base ref may be put at
2053	* any time due to an attribute change. Pin @pwq until the
2054	* rescuer is done with it.
2055	*/
2056	get_pwq(pwq);
2057	list_add_tail(&pwq->mayday_node, &wq->maydays);
2058	wake_up_process(wq->rescuer->task);
2059	}
2060	}
2061
2062	static void pool_mayday_timeout(struct timer_list *t)
2063	{
2064	struct worker_pool *pool = from_timer(pool, t, mayday_timer);
2065	struct work_struct *work;
2066
2067	raw_spin_lock_irq(&pool->lock);
2068	raw_spin_lock(&wq_mayday_lock); /* for wq->maydays */
2069
2070	if (need_to_create_worker(pool)) {
2071	/*
2072	* We've been trying to create a new worker but
2073	* haven't been successful. We might be hitting an
2074	* allocation deadlock. Send distress signals to
2075	* rescuers.
2076	*/
2077	list_for_each_entry(work, &pool->worklist, entry)
2078	send_mayday(work);
2079	}
2080
2081	raw_spin_unlock(&wq_mayday_lock);
2082	raw_spin_unlock_irq(&pool->lock);
2083
2084	mod_timer(&pool->mayday_timer, jiffies + MAYDAY_INTERVAL);
2085	}
2086
2087	/**
2088	* maybe_create_worker - create a new worker if necessary
2089	* @pool: pool to create a new worker for
2090	*
2091	* Create a new worker for @pool if necessary. @pool is guaranteed to
2092	* have at least one idle worker on return from this function. If
2093	* creating a new worker takes longer than MAYDAY_INTERVAL, mayday is
2094	* sent to all rescuers with works scheduled on @pool to resolve
2095	* possible allocation deadlock.
2096	*
2097	* On return, need_to_create_worker() is guaranteed to be %false and
2098	* may_start_working() %true.
2099	*
2100	* LOCKING:
2101	* raw_spin_lock_irq(pool->lock) which may be released and regrabbed
2102	* multiple times. Does GFP_KERNEL allocations. Called only from
2103	* manager.
2104	*/
2105	static void maybe_create_worker(struct worker_pool *pool)
2106	__releases(&pool->lock)
2107	__acquires(&pool->lock)
2108	{
2109	restart:
2110	raw_spin_unlock_irq(&pool->lock);
2111
2112	/* if we don't make progress in MAYDAY_INITIAL_TIMEOUT, call for help */
2113	mod_timer(&pool->mayday_timer, jiffies + MAYDAY_INITIAL_TIMEOUT);
2114
2115	while (true) {
2116	if (create_worker(pool) \|\| !need_to_create_worker(pool))
2117	break;
2118
2119	schedule_timeout_interruptible(CREATE_COOLDOWN);
2120
2121	if (!need_to_create_worker(pool))
2122	break;
2123	}
2124
2125	del_timer_sync(&pool->mayday_timer);
2126	raw_spin_lock_irq(&pool->lock);
2127	/*
2128	* This is necessary even after a new worker was just successfully
2129	* created as @pool->lock was dropped and the new worker might have
2130	* already become busy.
2131	*/
2132	if (need_to_create_worker(pool))
2133	goto restart;
2134	}
2135
2136	/**
2137	* manage_workers - manage worker pool
2138	* @worker: self
2139	*
2140	* Assume the manager role and manage the worker pool @worker belongs
2141	* to. At any given time, there can be only zero or one manager per
2142	* pool. The exclusion is handled automatically by this function.
2143	*
2144	* The caller can safely start processing works on false return. On
2145	* true return, it's guaranteed that need_to_create_worker() is false
2146	* and may_start_working() is true.
2147	*
2148	* CONTEXT:
2149	* raw_spin_lock_irq(pool->lock) which may be released and regrabbed
2150	* multiple times. Does GFP_KERNEL allocations.
2151	*
2152	* Return:
2153	* %false if the pool doesn't need management and the caller can safely
2154	* start processing works, %true if management function was performed and
2155	* the conditions that the caller verified before calling the function may
2156	* no longer be true.
2157	*/
2158	static bool manage_workers(struct worker *worker)
2159	{
2160	struct worker_pool *pool = worker->pool;
2161
2162	if (pool->flags & POOL_MANAGER_ACTIVE)
2163	return false;
2164
2165	pool->flags \|= POOL_MANAGER_ACTIVE;
2166	pool->manager = worker;
2167
2168	maybe_create_worker(pool);
2169
2170	pool->manager = NULL;
2171	pool->flags &= ~POOL_MANAGER_ACTIVE;
2172	rcuwait_wake_up(&manager_wait);
2173	return true;
2174	}
2175
2176	/**
2177	* process_one_work - process single work
2178	* @worker: self
2179	* @work: work to process
2180	*
2181	* Process @work. This function contains all the logics necessary to
2182	* process a single work including synchronization against and
2183	* interaction with other workers on the same cpu, queueing and
2184	* flushing. As long as context requirement is met, any worker can
2185	* call this function to process a work.
2186	*
2187	* CONTEXT:
2188	* raw_spin_lock_irq(pool->lock) which is released and regrabbed.
2189	*/
2190	static void process_one_work(struct worker worker, struct work_struct work)
2191	__releases(&pool->lock)
2192	__acquires(&pool->lock)
2193	{
2194	struct pool_workqueue *pwq = get_work_pwq(work);
2195	struct worker_pool *pool = worker->pool;
2196	bool cpu_intensive = pwq->wq->flags & WQ_CPU_INTENSIVE;
2197	unsigned long work_data;
2198	struct worker *collision;
2199	#ifdef CONFIG_LOCKDEP
2200	/*
2201	* It is permissible to free the struct work_struct from
2202	* inside the function that is called from it, this we need to
2203	* take into account for lockdep too. To avoid bogus "held
2204	* lock freed" warnings as well as problems when looking into
2205	* work->lockdep_map, make a copy and use that here.
2206	*/
2207	struct lockdep_map lockdep_map;
2208
2209	lockdep_copy_map(&lockdep_map, &work->lockdep_map);
2210	#endif
2211	/* ensure we're on the correct CPU */
2212	WARN_ON_ONCE(!(pool->flags & POOL_DISASSOCIATED) &&
2213	raw_smp_processor_id() != pool->cpu);
2214
2215	/*
2216	* A single work shouldn't be executed concurrently by
2217	* multiple workers on a single cpu. Check whether anyone is
2218	* already processing the work. If so, defer the work to the
2219	* currently executing one.
2220	*/
2221	collision = find_worker_executing_work(pool, work);
2222	if (unlikely(collision)) {
2223	move_linked_works(work, &collision->scheduled, NULL);
2224	return;
2225	}
2226
2227	/* claim and dequeue */
2228	debug_work_deactivate(work);
2229	hash_add(pool->busy_hash, &worker->hentry, (unsigned long)work);
2230	worker->current_work = work;
2231	worker->current_func = work->func;
2232	worker->current_pwq = pwq;
2233	work_data = *work_data_bits(work);
2234	worker->current_color = get_work_color(work_data);
2235
2236	/*
2237	* Record wq name for cmdline and debug reporting, may get
2238	* overridden through set_worker_desc().
2239	*/
2240	strscpy(worker->desc, pwq->wq->name, WORKER_DESC_LEN);
2241
2242	list_del_init(&work->entry);
2243
2244	/*
2245	* CPU intensive works don't participate in concurrency management.
2246	* They're the scheduler's responsibility. This takes @worker out
2247	* of concurrency management and the next code block will chain
2248	* execution of the pending work items.
2249	*/
2250	if (unlikely(cpu_intensive))
2251	worker_set_flags(worker, WORKER_CPU_INTENSIVE);
2252
2253	/*
2254	* Wake up another worker if necessary. The condition is always
2255	* false for normal per-cpu workers since nr_running would always
2256	* be >= 1 at this point. This is used to chain execution of the
2257	* pending work items for WORKER_NOT_RUNNING workers such as the
2258	* UNBOUND and CPU_INTENSIVE ones.
2259	*/
2260	if (need_more_worker(pool))
2261	wake_up_worker(pool);
2262
2263	/*
2264	* Record the last pool and clear PENDING which should be the last
2265	* update to @work. Also, do this inside @pool->lock so that
2266	* PENDING and queued state changes happen together while IRQ is
2267	* disabled.
2268	*/
2269	set_work_pool_and_clear_pending(work, pool->id);
2270
2271	raw_spin_unlock_irq(&pool->lock);
2272
2273	lock_map_acquire(&pwq->wq->lockdep_map);
2274	lock_map_acquire(&lockdep_map);
2275	/*
2276	* Strictly speaking we should mark the invariant state without holding
2277	* any locks, that is, before these two lock_map_acquire()'s.
2278	*
2279	* However, that would result in:
2280	*
2281	* A(W1)
2282	* WFC(C)
2283	* A(W1)
2284	* C(C)
2285	*
2286	* Which would create W1->C->W1 dependencies, even though there is no
2287	* actual deadlock possible. There are two solutions, using a
2288	* read-recursive acquire on the work(queue) 'locks', but this will then
2289	* hit the lockdep limitation on recursive locks, or simply discard
2290	* these locks.
2291	*
2292	* AFAICT there is no possible deadlock scenario between the
2293	* flush_work() and complete() primitives (except for single-threaded
2294	* workqueues), so hiding them isn't a problem.
2295	*/
2296	lockdep_invariant_state(true);
2297	trace_workqueue_execute_start(work);
2298	worker->current_func(work);
2299	/*
2300	* While we must be careful to not use "work" after this, the trace
2301	* point will only record its address.
2302	*/
2303	trace_workqueue_execute_end(work, worker->current_func);
2304	lock_map_release(&lockdep_map);
2305	lock_map_release(&pwq->wq->lockdep_map);
2306
2307	if (unlikely(in_atomic() \|\| lockdep_depth(current) > 0)) {
2308	pr_err("BUG: workqueue leaked lock or atomic: %s/0x%08x/%d\n"
2309	" last function: %ps\n",
2310	current->comm, preempt_count(), task_pid_nr(current),
2311	worker->current_func);
2312	debug_show_held_locks(current);
2313	dump_stack();
2314	}
2315
2316	/*
2317	* The following prevents a kworker from hogging CPU on !PREEMPTION
2318	* kernels, where a requeueing work item waiting for something to
2319	* happen could deadlock with stop_machine as such work item could
2320	* indefinitely requeue itself while all other CPUs are trapped in
2321	* stop_machine. At the same time, report a quiescent RCU state so
2322	* the same condition doesn't freeze RCU.
2323	*/
2324	cond_resched();
2325
2326	raw_spin_lock_irq(&pool->lock);
2327
2328	/* clear cpu intensive status */
2329	if (unlikely(cpu_intensive))
2330	worker_clr_flags(worker, WORKER_CPU_INTENSIVE);
2331
2332	/* tag the worker for identification in schedule() */
2333	worker->last_func = worker->current_func;
2334
2335	/* we're done with it, release */
2336	hash_del(&worker->hentry);
2337	worker->current_work = NULL;
2338	worker->current_func = NULL;
2339	worker->current_pwq = NULL;
2340	worker->current_color = INT_MAX;
2341	pwq_dec_nr_in_flight(pwq, work_data);
2342	}
2343
2344	/**
2345	* process_scheduled_works - process scheduled works
2346	* @worker: self
2347	*
2348	* Process all scheduled works. Please note that the scheduled list
2349	* may change while processing a work, so this function repeatedly
2350	* fetches a work from the top and executes it.
2351	*
2352	* CONTEXT:
2353	* raw_spin_lock_irq(pool->lock) which may be released and regrabbed
2354	* multiple times.
2355	*/
2356	static void process_scheduled_works(struct worker *worker)
2357	{
2358	while (!list_empty(&worker->scheduled)) {
2359	struct work_struct *work = list_first_entry(&worker->scheduled,
2360	struct work_struct, entry);
2361	process_one_work(worker, work);
2362	}
2363	}
2364
2365	static void set_pf_worker(bool val)
2366	{
2367	mutex_lock(&wq_pool_attach_mutex);
2368	if (val)
2369	current->flags \|= PF_WQ_WORKER;
2370	else
2371	current->flags &= ~PF_WQ_WORKER;
2372	mutex_unlock(&wq_pool_attach_mutex);
2373	}
2374
2375	/**
2376	* worker_thread - the worker thread function
2377	* @__worker: self
2378	*
2379	* The worker thread function. All workers belong to a worker_pool -
2380	* either a per-cpu one or dynamic unbound one. These workers process all
2381	* work items regardless of their specific target workqueue. The only
2382	* exception is work items which belong to workqueues with a rescuer which
2383	* will be explained in rescuer_thread().
2384	*
2385	* Return: 0
2386	*/
2387	static int worker_thread(void *__worker)
2388	{
2389	struct worker *worker = __worker;
2390	struct worker_pool *pool = worker->pool;
2391
2392	/* tell the scheduler that this is a workqueue worker */
2393	set_pf_worker(true);
2394	woke_up:
2395	raw_spin_lock_irq(&pool->lock);
2396
2397	/* am I supposed to die? */
2398	if (unlikely(worker->flags & WORKER_DIE)) {
2399	raw_spin_unlock_irq(&pool->lock);
2400	WARN_ON_ONCE(!list_empty(&worker->entry));
2401	set_pf_worker(false);
2402
2403	set_task_comm(worker->task, "kworker/dying");
2404	ida_free(&pool->worker_ida, worker->id);
2405	worker_detach_from_pool(worker);
2406	kfree(worker);
2407	return 0;
2408	}
2409
2410	worker_leave_idle(worker);
2411	recheck:
2412	/* no more worker necessary? */
2413	if (!need_more_worker(pool))
2414	goto sleep;
2415
2416	/* do we need to manage? */
2417	if (unlikely(!may_start_working(pool)) && manage_workers(worker))
2418	goto recheck;
2419
2420	/*
2421	* ->scheduled list can only be filled while a worker is
2422	* preparing to process a work or actually processing it.
2423	* Make sure nobody diddled with it while I was sleeping.
2424	*/
2425	WARN_ON_ONCE(!list_empty(&worker->scheduled));
2426
2427	/*
2428	* Finish PREP stage. We're guaranteed to have at least one idle
2429	* worker or that someone else has already assumed the manager
2430	* role. This is where @worker starts participating in concurrency
2431	* management if applicable and concurrency management is restored
2432	* after being rebound. See rebind_workers() for details.
2433	*/
2434	worker_clr_flags(worker, WORKER_PREP \| WORKER_REBOUND);
2435
2436	do {
2437	struct work_struct *work =
2438	list_first_entry(&pool->worklist,
2439	struct work_struct, entry);
2440
2441	pool->watchdog_ts = jiffies;
2442
2443	if (likely(!(*work_data_bits(work) & WORK_STRUCT_LINKED))) {
2444	/* optimization path, not strictly necessary */
2445	process_one_work(worker, work);
2446	if (unlikely(!list_empty(&worker->scheduled)))
2447	process_scheduled_works(worker);
2448	} else {
2449	move_linked_works(work, &worker->scheduled, NULL);
2450	process_scheduled_works(worker);
2451	}
2452	} while (keep_working(pool));
2453
2454	worker_set_flags(worker, WORKER_PREP);
2455	sleep:
2456	/*
2457	* pool->lock is held and there's no work to process and no need to
2458	* manage, sleep. Workers are woken up only while holding
2459	* pool->lock or from local cpu, so setting the current state
2460	* before releasing pool->lock is enough to prevent losing any
2461	* event.
2462	*/
2463	worker_enter_idle(worker);
2464	__set_current_state(TASK_IDLE);
2465	raw_spin_unlock_irq(&pool->lock);
2466	schedule();
2467	goto woke_up;
2468	}
2469
2470	/**
2471	* rescuer_thread - the rescuer thread function
2472	* @__rescuer: self
2473	*
2474	* Workqueue rescuer thread function. There's one rescuer for each
2475	* workqueue which has WQ_MEM_RECLAIM set.
2476	*
2477	* Regular work processing on a pool may block trying to create a new
2478	* worker which uses GFP_KERNEL allocation which has slight chance of
2479	* developing into deadlock if some works currently on the same queue
2480	* need to be processed to satisfy the GFP_KERNEL allocation. This is
2481	* the problem rescuer solves.
2482	*
2483	* When such condition is possible, the pool summons rescuers of all
2484	* workqueues which have works queued on the pool and let them process
2485	* those works so that forward progress can be guaranteed.
2486	*
2487	* This should happen rarely.
2488	*
2489	* Return: 0
2490	*/
2491	static int rescuer_thread(void *__rescuer)
2492	{
2493	struct worker *rescuer = __rescuer;
2494	struct workqueue_struct *wq = rescuer->rescue_wq;
2495	struct list_head *scheduled = &rescuer->scheduled;
2496	bool should_stop;
2497
2498	set_user_nice(current, RESCUER_NICE_LEVEL);
2499
2500	/*
2501	* Mark rescuer as worker too. As WORKER_PREP is never cleared, it
2502	* doesn't participate in concurrency management.
2503	*/
2504	set_pf_worker(true);
2505	repeat:
2506	set_current_state(TASK_IDLE);
2507
2508	/*
2509	* By the time the rescuer is requested to stop, the workqueue
2510	* shouldn't have any work pending, but @wq->maydays may still have
2511	* pwq(s) queued. This can happen by non-rescuer workers consuming
2512	* all the work items before the rescuer got to them. Go through
2513	* @wq->maydays processing before acting on should_stop so that the
2514	* list is always empty on exit.
2515	*/
2516	should_stop = kthread_should_stop();
2517
2518	/* see whether any pwq is asking for help */
2519	raw_spin_lock_irq(&wq_mayday_lock);
2520
2521	while (!list_empty(&wq->maydays)) {
2522	struct pool_workqueue *pwq = list_first_entry(&wq->maydays,
2523	struct pool_workqueue, mayday_node);
2524	struct worker_pool *pool = pwq->pool;
2525	struct work_struct work, n;
2526	bool first = true;
2527
2528	__set_current_state(TASK_RUNNING);
2529	list_del_init(&pwq->mayday_node);
2530
2531	raw_spin_unlock_irq(&wq_mayday_lock);
2532
2533	worker_attach_to_pool(rescuer, pool);
2534
2535	raw_spin_lock_irq(&pool->lock);
2536
2537	/*
2538	* Slurp in all works issued via this workqueue and
2539	* process'em.
2540	*/
2541	WARN_ON_ONCE(!list_empty(scheduled));
2542	list_for_each_entry_safe(work, n, &pool->worklist, entry) {
2543	if (get_work_pwq(work) == pwq) {
2544	if (first)
2545	pool->watchdog_ts = jiffies;
2546	move_linked_works(work, scheduled, &n);
2547	}
2548	first = false;
2549	}
2550
2551	if (!list_empty(scheduled)) {
2552	process_scheduled_works(rescuer);
2553
2554	/*
2555	* The above execution of rescued work items could
2556	* have created more to rescue through
2557	* pwq_activate_first_inactive() or chained
2558	* queueing. Let's put @pwq back on mayday list so
2559	* that such back-to-back work items, which may be
2560	* being used to relieve memory pressure, don't
2561	* incur MAYDAY_INTERVAL delay inbetween.
2562	*/
2563	if (pwq->nr_active && need_to_create_worker(pool)) {
2564	raw_spin_lock(&wq_mayday_lock);
2565	/*
2566	* Queue iff we aren't racing destruction
2567	* and somebody else hasn't queued it already.
2568	*/
2569	if (wq->rescuer && list_empty(&pwq->mayday_node)) {
2570	get_pwq(pwq);
2571	list_add_tail(&pwq->mayday_node, &wq->maydays);
2572	}
2573	raw_spin_unlock(&wq_mayday_lock);
2574	}
2575	}
2576
2577	/*
2578	* Put the reference grabbed by send_mayday(). @pool won't
2579	* go away while we're still attached to it.
2580	*/
2581	put_pwq(pwq);
2582
2583	/*
2584	* Leave this pool. If need_more_worker() is %true, notify a
2585	* regular worker; otherwise, we end up with 0 concurrency
2586	* and stalling the execution.
2587	*/
2588	if (need_more_worker(pool))
2589	wake_up_worker(pool);
2590
2591	raw_spin_unlock_irq(&pool->lock);
2592
2593	worker_detach_from_pool(rescuer);
2594
2595	raw_spin_lock_irq(&wq_mayday_lock);
2596	}
2597
2598	raw_spin_unlock_irq(&wq_mayday_lock);
2599
2600	if (should_stop) {
2601	__set_current_state(TASK_RUNNING);
2602	set_pf_worker(false);
2603	return 0;
2604	}
2605
2606	/* rescuers should never participate in concurrency management */
2607	WARN_ON_ONCE(!(rescuer->flags & WORKER_NOT_RUNNING));
2608	schedule();
2609	goto repeat;
2610	}
2611
2612	/**
2613	* check_flush_dependency - check for flush dependency sanity
2614	* @target_wq: workqueue being flushed
2615	* @target_work: work item being flushed (NULL for workqueue flushes)
2616	*
2617	* %current is trying to flush the whole @target_wq or @target_work on it.
2618	* If @target_wq doesn't have %WQ_MEM_RECLAIM, verify that %current is not
2619	* reclaiming memory or running on a workqueue which doesn't have
2620	* %WQ_MEM_RECLAIM as that can break forward-progress guarantee leading to
2621	* a deadlock.
2622	*/
2623	static void check_flush_dependency(struct workqueue_struct *target_wq,
2624	struct work_struct *target_work)
2625	{
2626	work_func_t target_func = target_work ? target_work->func : NULL;
2627	struct worker *worker;
2628
2629	if (target_wq->flags & WQ_MEM_RECLAIM)
2630	return;
2631
2632	worker = current_wq_worker();
2633
2634	WARN_ONCE(current->flags & PF_MEMALLOC,
2635	"workqueue: PF_MEMALLOC task %d(%s) is flushing !WQ_MEM_RECLAIM %s:%ps",
2636	current->pid, current->comm, target_wq->name, target_func);
2637	WARN_ONCE(worker && ((worker->current_pwq->wq->flags &
2638	(WQ_MEM_RECLAIM \| __WQ_LEGACY)) == WQ_MEM_RECLAIM),
2639	"workqueue: WQ_MEM_RECLAIM %s:%ps is flushing !WQ_MEM_RECLAIM %s:%ps",
2640	worker->current_pwq->wq->name, worker->current_func,
2641	target_wq->name, target_func);
2642	}
2643
2644	struct wq_barrier {
2645	struct work_struct work;
2646	struct completion done;
2647	struct task_struct task; / purely informational */
2648	};
2649
2650	static void wq_barrier_func(struct work_struct *work)
2651	{
2652	struct wq_barrier *barr = container_of(work, struct wq_barrier, work);
2653	complete(&barr->done);
2654	}
2655
2656	/**
2657	* insert_wq_barrier - insert a barrier work
2658	* @pwq: pwq to insert barrier into
2659	* @barr: wq_barrier to insert
2660	* @target: target work to attach @barr to
2661	* @worker: worker currently executing @target, NULL if @target is not executing
2662	*
2663	* @barr is linked to @target such that @barr is completed only after
2664	* @target finishes execution. Please note that the ordering
2665	* guarantee is observed only with respect to @target and on the local
2666	* cpu.
2667	*
2668	* Currently, a queued barrier can't be canceled. This is because
2669	* try_to_grab_pending() can't determine whether the work to be
2670	* grabbed is at the head of the queue and thus can't clear LINKED
2671	* flag of the previous work while there must be a valid next work
2672	* after a work with LINKED flag set.
2673	*
2674	* Note that when @worker is non-NULL, @target may be modified
2675	* underneath us, so we can't reliably determine pwq from @target.
2676	*
2677	* CONTEXT:
2678	* raw_spin_lock_irq(pool->lock).
2679	*/
2680	static void insert_wq_barrier(struct pool_workqueue *pwq,
2681	struct wq_barrier *barr,
2682	struct work_struct target, struct worker worker)
2683	{
2684	unsigned int work_flags = 0;
2685	unsigned int work_color;
2686	struct list_head *head;
2687
2688	/*
2689	* debugobject calls are safe here even with pool->lock locked
2690	* as we know for sure that this will not trigger any of the
2691	* checks and call back into the fixup functions where we
2692	* might deadlock.
2693	*/
2694	INIT_WORK_ONSTACK(&barr->work, wq_barrier_func);
2695	__set_bit(WORK_STRUCT_PENDING_BIT, work_data_bits(&barr->work));
2696
2697	init_completion_map(&barr->done, &target->lockdep_map);
2698
2699	barr->task = current;
2700
2701	/* The barrier work item does not participate in pwq->nr_active. */
2702	work_flags \|= WORK_STRUCT_INACTIVE;
2703
2704	/*
2705	* If @target is currently being executed, schedule the
2706	* barrier to the worker; otherwise, put it after @target.
2707	*/
2708	if (worker) {
2709	head = worker->scheduled.next;
2710	work_color = worker->current_color;
2711	} else {
2712	unsigned long *bits = work_data_bits(target);
2713
2714	head = target->entry.next;
2715	/* there can already be other linked works, inherit and set */
2716	work_flags \|= *bits & WORK_STRUCT_LINKED;
2717	work_color = get_work_color(*bits);
2718	__set_bit(WORK_STRUCT_LINKED_BIT, bits);
2719	}
2720
2721	pwq->nr_in_flight[work_color]++;
2722	work_flags \|= work_color_to_flags(work_color);
2723
2724	debug_work_activate(&barr->work);
2725	insert_work(pwq, &barr->work, head, work_flags);
2726	}
2727
2728	/**
2729	* flush_workqueue_prep_pwqs - prepare pwqs for workqueue flushing
2730	* @wq: workqueue being flushed
2731	* @flush_color: new flush color, < 0 for no-op
2732	* @work_color: new work color, < 0 for no-op
2733	*
2734	* Prepare pwqs for workqueue flushing.
2735	*
2736	* If @flush_color is non-negative, flush_color on all pwqs should be
2737	* -1. If no pwq has in-flight commands at the specified color, all
2738	* pwq->flush_color's stay at -1 and %false is returned. If any pwq
2739	* has in flight commands, its pwq->flush_color is set to
2740	* @flush_color, @wq->nr_pwqs_to_flush is updated accordingly, pwq
2741	* wakeup logic is armed and %true is returned.
2742	*
2743	* The caller should have initialized @wq->first_flusher prior to
2744	* calling this function with non-negative @flush_color. If
2745	* @flush_color is negative, no flush color update is done and %false
2746	* is returned.
2747	*
2748	* If @work_color is non-negative, all pwqs should have the same
2749	* work_color which is previous to @work_color and all will be
2750	* advanced to @work_color.
2751	*
2752	* CONTEXT:
2753	* mutex_lock(wq->mutex).
2754	*
2755	* Return:
2756	* %true if @flush_color >= 0 and there's something to flush. %false
2757	* otherwise.
2758	*/
2759	static bool flush_workqueue_prep_pwqs(struct workqueue_struct *wq,
2760	int flush_color, int work_color)
2761	{
2762	bool wait = false;
2763	struct pool_workqueue *pwq;
2764
2765	if (flush_color >= 0) {
2766	WARN_ON_ONCE(atomic_read(&wq->nr_pwqs_to_flush));
2767	atomic_set(&wq->nr_pwqs_to_flush, 1);
2768	}
2769
2770	for_each_pwq(pwq, wq) {
2771	struct worker_pool *pool = pwq->pool;
2772
2773	raw_spin_lock_irq(&pool->lock);
2774
2775	if (flush_color >= 0) {
2776	WARN_ON_ONCE(pwq->flush_color != -1);
2777
2778	if (pwq->nr_in_flight[flush_color]) {
2779	pwq->flush_color = flush_color;
2780	atomic_inc(&wq->nr_pwqs_to_flush);
2781	wait = true;
2782	}
2783	}
2784
2785	if (work_color >= 0) {
2786	WARN_ON_ONCE(work_color != work_next_color(pwq->work_color));
2787	pwq->work_color = work_color;
2788	}
2789
2790	raw_spin_unlock_irq(&pool->lock);
2791	}
2792
2793	if (flush_color >= 0 && atomic_dec_and_test(&wq->nr_pwqs_to_flush))
2794	complete(&wq->first_flusher->done);
2795
2796	return wait;
2797	}
2798
2799	/**
2800	* flush_workqueue - ensure that any scheduled work has run to completion.
2801	* @wq: workqueue to flush
2802	*
2803	* This function sleeps until all work items which were queued on entry
2804	* have finished execution, but it is not livelocked by new incoming ones.
2805	*/
2806	void flush_workqueue(struct workqueue_struct *wq)
2807	{
2808	struct wq_flusher this_flusher = {
2809	.list = LIST_HEAD_INIT(this_flusher.list),
2810	.flush_color = -1,
2811	.done = COMPLETION_INITIALIZER_ONSTACK_MAP(this_flusher.done, wq->lockdep_map),
2812	};
2813	int next_color;
2814
2815	if (WARN_ON(!wq_online))
2816	return;
2817
2818	lock_map_acquire(&wq->lockdep_map);
2819	lock_map_release(&wq->lockdep_map);
2820
2821	mutex_lock(&wq->mutex);
2822
2823	/*
2824	* Start-to-wait phase
2825	*/
2826	next_color = work_next_color(wq->work_color);
2827
2828	if (next_color != wq->flush_color) {
2829	/*
2830	* Color space is not full. The current work_color
2831	* becomes our flush_color and work_color is advanced
2832	* by one.
2833	*/
2834	WARN_ON_ONCE(!list_empty(&wq->flusher_overflow));
2835	this_flusher.flush_color = wq->work_color;
2836	wq->work_color = next_color;
2837
2838	if (!wq->first_flusher) {
2839	/* no flush in progress, become the first flusher */
2840	WARN_ON_ONCE(wq->flush_color != this_flusher.flush_color);
2841
2842	wq->first_flusher = &this_flusher;
2843
2844	if (!flush_workqueue_prep_pwqs(wq, wq->flush_color,
2845	wq->work_color)) {
2846	/* nothing to flush, done */
2847	wq->flush_color = next_color;
2848	wq->first_flusher = NULL;
2849	goto out_unlock;
2850	}
2851	} else {
2852	/* wait in queue */
2853	WARN_ON_ONCE(wq->flush_color == this_flusher.flush_color);
2854	list_add_tail(&this_flusher.list, &wq->flusher_queue);
2855	flush_workqueue_prep_pwqs(wq, -1, wq->work_color);
2856	}
2857	} else {
2858	/*
2859	* Oops, color space is full, wait on overflow queue.
2860	* The next flush completion will assign us
2861	* flush_color and transfer to flusher_queue.
2862	*/
2863	list_add_tail(&this_flusher.list, &wq->flusher_overflow);
2864	}
2865
2866	check_flush_dependency(wq, NULL);
2867
2868	mutex_unlock(&wq->mutex);
2869
2870	wait_for_completion(&this_flusher.done);
2871
2872	/*
2873	* Wake-up-and-cascade phase
2874	*
2875	* First flushers are responsible for cascading flushes and
2876	* handling overflow. Non-first flushers can simply return.
2877	*/
2878	if (READ_ONCE(wq->first_flusher) != &this_flusher)
2879	return;
2880
2881	mutex_lock(&wq->mutex);
2882
2883	/* we might have raced, check again with mutex held */
2884	if (wq->first_flusher != &this_flusher)
2885	goto out_unlock;
2886
2887	WRITE_ONCE(wq->first_flusher, NULL);
2888
2889	WARN_ON_ONCE(!list_empty(&this_flusher.list));
2890	WARN_ON_ONCE(wq->flush_color != this_flusher.flush_color);
2891
2892	while (true) {
2893	struct wq_flusher next, tmp;
2894
2895	/* complete all the flushers sharing the current flush color */
2896	list_for_each_entry_safe(next, tmp, &wq->flusher_queue, list) {
2897	if (next->flush_color != wq->flush_color)
2898	break;
2899	list_del_init(&next->list);
2900	complete(&next->done);
2901	}
2902
2903	WARN_ON_ONCE(!list_empty(&wq->flusher_overflow) &&
2904	wq->flush_color != work_next_color(wq->work_color));
2905
2906	/* this flush_color is finished, advance by one */
2907	wq->flush_color = work_next_color(wq->flush_color);
2908
2909	/* one color has been freed, handle overflow queue */
2910	if (!list_empty(&wq->flusher_overflow)) {
2911	/*
2912	* Assign the same color to all overflowed
2913	* flushers, advance work_color and append to
2914	* flusher_queue. This is the start-to-wait
2915	* phase for these overflowed flushers.
2916	*/
2917	list_for_each_entry(tmp, &wq->flusher_overflow, list)
2918	tmp->flush_color = wq->work_color;
2919
2920	wq->work_color = work_next_color(wq->work_color);
2921
2922	list_splice_tail_init(&wq->flusher_overflow,
2923	&wq->flusher_queue);
2924	flush_workqueue_prep_pwqs(wq, -1, wq->work_color);
2925	}
2926
2927	if (list_empty(&wq->flusher_queue)) {
2928	WARN_ON_ONCE(wq->flush_color != wq->work_color);
2929	break;
2930	}
2931
2932	/*
2933	* Need to flush more colors. Make the next flusher
2934	* the new first flusher and arm pwqs.
2935	*/
2936	WARN_ON_ONCE(wq->flush_color == wq->work_color);
2937	WARN_ON_ONCE(wq->flush_color != next->flush_color);
2938
2939	list_del_init(&next->list);
2940	wq->first_flusher = next;
2941
2942	if (flush_workqueue_prep_pwqs(wq, wq->flush_color, -1))
2943	break;
2944
2945	/*
2946	* Meh... this color is already done, clear first
2947	* flusher and repeat cascading.
2948	*/
2949	wq->first_flusher = NULL;
2950	}
2951
2952	out_unlock:
2953	mutex_unlock(&wq->mutex);
2954	}
2955	EXPORT_SYMBOL(flush_workqueue);
2956
2957	/**
2958	* drain_workqueue - drain a workqueue
2959	* @wq: workqueue to drain
2960	*
2961	* Wait until the workqueue becomes empty. While draining is in progress,
2962	* only chain queueing is allowed. IOW, only currently pending or running
2963	* work items on @wq can queue further work items on it. @wq is flushed
2964	* repeatedly until it becomes empty. The number of flushing is determined
2965	* by the depth of chaining and should be relatively short. Whine if it
2966	* takes too long.
2967	*/
2968	void drain_workqueue(struct workqueue_struct *wq)
2969	{
2970	unsigned int flush_cnt = 0;
2971	struct pool_workqueue *pwq;
2972
2973	/*
2974	* __queue_work() needs to test whether there are drainers, is much
2975	* hotter than drain_workqueue() and already looks at @wq->flags.
2976	* Use __WQ_DRAINING so that queue doesn't have to check nr_drainers.
2977	*/
2978	mutex_lock(&wq->mutex);
2979	if (!wq->nr_drainers++)
2980	wq->flags \|= __WQ_DRAINING;
2981	mutex_unlock(&wq->mutex);
2982	reflush:
2983	flush_workqueue(wq);
2984
2985	mutex_lock(&wq->mutex);
2986
2987	for_each_pwq(pwq, wq) {
2988	bool drained;
2989
2990	raw_spin_lock_irq(&pwq->pool->lock);
2991	drained = !pwq->nr_active && list_empty(&pwq->inactive_works);
2992	raw_spin_unlock_irq(&pwq->pool->lock);
2993
2994	if (drained)
2995	continue;
2996
2997	if (++flush_cnt == 10 \|\|
2998	(flush_cnt % 100 == 0 && flush_cnt <= 1000))
2999	pr_warn("workqueue %s: %s() isn't complete after %u tries\n",
3000	wq->name, __func__, flush_cnt);
3001
3002	mutex_unlock(&wq->mutex);
3003	goto reflush;
3004	}
3005
3006	if (!--wq->nr_drainers)
3007	wq->flags &= ~__WQ_DRAINING;
3008	mutex_unlock(&wq->mutex);
3009	}
3010	EXPORT_SYMBOL_GPL(drain_workqueue);
3011
3012	static bool start_flush_work(struct work_struct work, struct wq_barrier barr,
3013	bool from_cancel)
3014	{
3015	struct worker *worker = NULL;
3016	struct worker_pool *pool;
3017	struct pool_workqueue *pwq;
3018
3019	might_sleep();
3020
3021	rcu_read_lock();
3022	pool = get_work_pool(work);
3023	if (!pool) {
3024	rcu_read_unlock();
3025	return false;
3026	}
3027
3028	raw_spin_lock_irq(&pool->lock);
3029	/* see the comment in try_to_grab_pending() with the same code */
3030	pwq = get_work_pwq(work);
3031	if (pwq) {
3032	if (unlikely(pwq->pool != pool))
3033	goto already_gone;
3034	} else {
3035	worker = find_worker_executing_work(pool, work);
3036	if (!worker)
3037	goto already_gone;
3038	pwq = worker->current_pwq;
3039	}
3040
3041	check_flush_dependency(pwq->wq, work);
3042
3043	insert_wq_barrier(pwq, barr, work, worker);
3044	raw_spin_unlock_irq(&pool->lock);
3045
3046	/*
3047	* Force a lock recursion deadlock when using flush_work() inside a
3048	* single-threaded or rescuer equipped workqueue.
3049	*
3050	* For single threaded workqueues the deadlock happens when the work
3051	* is after the work issuing the flush_work(). For rescuer equipped
3052	* workqueues the deadlock happens when the rescuer stalls, blocking
3053	* forward progress.
3054	*/
3055	if (!from_cancel &&
3056	(pwq->wq->saved_max_active == 1 \|\| pwq->wq->rescuer)) {
3057	lock_map_acquire(&pwq->wq->lockdep_map);
3058	lock_map_release(&pwq->wq->lockdep_map);
3059	}
3060	rcu_read_unlock();
3061	return true;
3062	already_gone:
3063	raw_spin_unlock_irq(&pool->lock);
3064	rcu_read_unlock();
3065	return false;
3066	}
3067
3068	static bool __flush_work(struct work_struct *work, bool from_cancel)
3069	{
3070	struct wq_barrier barr;
3071
3072	if (WARN_ON(!wq_online))
3073	return false;
3074
3075	if (WARN_ON(!work->func))
3076	return false;
3077
3078	if (!from_cancel) {
3079	lock_map_acquire(&work->lockdep_map);
3080	lock_map_release(&work->lockdep_map);
3081	}
3082
3083	if (start_flush_work(work, &barr, from_cancel)) {
3084	wait_for_completion(&barr.done);
3085	destroy_work_on_stack(&barr.work);
3086	return true;
3087	} else {
3088	return false;
3089	}
3090	}
3091
3092	/**
3093	* flush_work - wait for a work to finish executing the last queueing instance
3094	* @work: the work to flush
3095	*
3096	* Wait until @work has finished execution. @work is guaranteed to be idle
3097	* on return if it hasn't been requeued since flush started.
3098	*
3099	* Return:
3100	* %true if flush_work() waited for the work to finish execution,
3101	* %false if it was already idle.
3102	*/
3103	bool flush_work(struct work_struct *work)
3104	{
3105	return __flush_work(work, false);
3106	}
3107	EXPORT_SYMBOL_GPL(flush_work);
3108
3109	struct cwt_wait {
3110	wait_queue_entry_t wait;
3111	struct work_struct *work;
3112	};
3113
3114	static int cwt_wakefn(wait_queue_entry_t wait, unsigned mode, int sync, void key)
3115	{
3116	struct cwt_wait *cwait = container_of(wait, struct cwt_wait, wait);
3117
3118	if (cwait->work != key)
3119	return 0;
3120	return autoremove_wake_function(wait, mode, sync, key);
3121	}
3122
3123	static bool __cancel_work_timer(struct work_struct *work, bool is_dwork)
3124	{
3125	static DECLARE_WAIT_QUEUE_HEAD(cancel_waitq);
3126	unsigned long flags;
3127	int ret;
3128
3129	do {
3130	ret = try_to_grab_pending(work, is_dwork, &flags);
3131	/*
3132	* If someone else is already canceling, wait for it to
3133	* finish. flush_work() doesn't work for PREEMPT_NONE
3134	* because we may get scheduled between @work's completion
3135	* and the other canceling task resuming and clearing
3136	* CANCELING - flush_work() will return false immediately
3137	* as @work is no longer busy, try_to_grab_pending() will
3138	* return -ENOENT as @work is still being canceled and the
3139	* other canceling task won't be able to clear CANCELING as
3140	* we're hogging the CPU.
3141	*
3142	* Let's wait for completion using a waitqueue. As this
3143	* may lead to the thundering herd problem, use a custom
3144	* wake function which matches @work along with exclusive
3145	* wait and wakeup.
3146	*/
3147	if (unlikely(ret == -ENOENT)) {
3148	struct cwt_wait cwait;
3149
3150	init_wait(&cwait.wait);
3151	cwait.wait.func = cwt_wakefn;
3152	cwait.work = work;
3153
3154	prepare_to_wait_exclusive(&cancel_waitq, &cwait.wait,
3155	TASK_UNINTERRUPTIBLE);
3156	if (work_is_canceling(work))
3157	schedule();
3158	finish_wait(&cancel_waitq, &cwait.wait);
3159	}
3160	} while (unlikely(ret < 0));
3161
3162	/* tell other tasks trying to grab @work to back off */
3163	mark_work_canceling(work);
3164	local_irq_restore(flags);
3165
3166	/*
3167	* This allows canceling during early boot. We know that @work
3168	* isn't executing.
3169	*/
3170	if (wq_online)
3171	__flush_work(work, true);
3172
3173	clear_work_data(work);
3174
3175	/*
3176	* Paired with prepare_to_wait() above so that either
3177	* waitqueue_active() is visible here or !work_is_canceling() is
3178	* visible there.
3179	*/
3180	smp_mb();
3181	if (waitqueue_active(&cancel_waitq))
3182	__wake_up(&cancel_waitq, TASK_NORMAL, 1, work);
3183
3184	return ret;
3185	}
3186
3187	/**
3188	* cancel_work_sync - cancel a work and wait for it to finish
3189	* @work: the work to cancel
3190	*
3191	* Cancel @work and wait for its execution to finish. This function
3192	* can be used even if the work re-queues itself or migrates to
3193	* another workqueue. On return from this function, @work is
3194	* guaranteed to be not pending or executing on any CPU.
3195	*
3196	* cancel_work_sync(&delayed_work->work) must not be used for
3197	* delayed_work's. Use cancel_delayed_work_sync() instead.
3198	*
3199	* The caller must ensure that the workqueue on which @work was last
3200	* queued can't be destroyed before this function returns.
3201	*
3202	* Return:
3203	* %true if @work was pending, %false otherwise.
3204	*/
3205	bool cancel_work_sync(struct work_struct *work)
3206	{
3207	return __cancel_work_timer(work, false);
3208	}
3209	EXPORT_SYMBOL_GPL(cancel_work_sync);
3210
3211	/**
3212	* flush_delayed_work - wait for a dwork to finish executing the last queueing
3213	* @dwork: the delayed work to flush
3214	*
3215	* Delayed timer is cancelled and the pending work is queued for
3216	* immediate execution. Like flush_work(), this function only
3217	* considers the last queueing instance of @dwork.
3218	*
3219	* Return:
3220	* %true if flush_work() waited for the work to finish execution,
3221	* %false if it was already idle.
3222	*/
3223	bool flush_delayed_work(struct delayed_work *dwork)
3224	{
3225	local_irq_disable();
3226	if (del_timer_sync(&dwork->timer))
3227	__queue_work(dwork->cpu, dwork->wq, &dwork->work);
3228	local_irq_enable();
3229	return flush_work(&dwork->work);
3230	}
3231	EXPORT_SYMBOL(flush_delayed_work);
3232
3233	/**
3234	* flush_rcu_work - wait for a rwork to finish executing the last queueing
3235	* @rwork: the rcu work to flush
3236	*
3237	* Return:
3238	* %true if flush_rcu_work() waited for the work to finish execution,
3239	* %false if it was already idle.
3240	*/
3241	bool flush_rcu_work(struct rcu_work *rwork)
3242	{
3243	if (test_bit(WORK_STRUCT_PENDING_BIT, work_data_bits(&rwork->work))) {
3244	rcu_barrier();
3245	flush_work(&rwork->work);
3246	return true;
3247	} else {
3248	return flush_work(&rwork->work);
3249	}
3250	}
3251	EXPORT_SYMBOL(flush_rcu_work);
3252
3253	static bool __cancel_work(struct work_struct *work, bool is_dwork)
3254	{
3255	unsigned long flags;
3256	int ret;
3257
3258	do {
3259	ret = try_to_grab_pending(work, is_dwork, &flags);
3260	} while (unlikely(ret == -EAGAIN));
3261
3262	if (unlikely(ret < 0))
3263	return false;
3264
3265	set_work_pool_and_clear_pending(work, get_work_pool_id(work));
3266	local_irq_restore(flags);
3267	return ret;
3268	}
3269
3270	/**
3271	* cancel_delayed_work - cancel a delayed work
3272	* @dwork: delayed_work to cancel
3273	*
3274	* Kill off a pending delayed_work.
3275	*
3276	* Return: %true if @dwork was pending and canceled; %false if it wasn't
3277	* pending.
3278	*
3279	* Note:
3280	* The work callback function may still be running on return, unless
3281	* it returns %true and the work doesn't re-arm itself. Explicitly flush or
3282	* use cancel_delayed_work_sync() to wait on it.
3283	*
3284	* This function is safe to call from any context including IRQ handler.
3285	*/
3286	bool cancel_delayed_work(struct delayed_work *dwork)
3287	{
3288	return __cancel_work(&dwork->work, true);
3289	}
3290	EXPORT_SYMBOL(cancel_delayed_work);
3291
3292	/**
3293	* cancel_delayed_work_sync - cancel a delayed work and wait for it to finish
3294	* @dwork: the delayed work cancel
3295	*
3296	* This is cancel_work_sync() for delayed works.
3297	*
3298	* Return:
3299	* %true if @dwork was pending, %false otherwise.
3300	*/
3301	bool cancel_delayed_work_sync(struct delayed_work *dwork)
3302	{
3303	return __cancel_work_timer(&dwork->work, true);
3304	}
3305	EXPORT_SYMBOL(cancel_delayed_work_sync);
3306
3307	/**
3308	* schedule_on_each_cpu - execute a function synchronously on each online CPU
3309	* @func: the function to call
3310	*
3311	* schedule_on_each_cpu() executes @func on each online CPU using the
3312	* system workqueue and blocks until all CPUs have completed.
3313	* schedule_on_each_cpu() is very slow.
3314	*
3315	* Return:
3316	* 0 on success, -errno on failure.
3317	*/
3318	int schedule_on_each_cpu(work_func_t func)
3319	{
3320	int cpu;
3321	struct work_struct __percpu *works;
3322
3323	works = alloc_percpu(struct work_struct);
3324	if (!works)
3325	return -ENOMEM;
3326
3327	cpus_read_lock();
3328
3329	for_each_online_cpu(cpu) {
3330	struct work_struct *work = per_cpu_ptr(works, cpu);
3331
3332	INIT_WORK(work, func);
3333	schedule_work_on(cpu, work);
3334	}
3335
3336	for_each_online_cpu(cpu)
3337	flush_work(per_cpu_ptr(works, cpu));
3338
3339	cpus_read_unlock();
3340	free_percpu(works);
3341	return 0;
3342	}
3343
3344	/**
3345	* execute_in_process_context - reliably execute the routine with user context
3346	* @fn: the function to execute
3347	* @ew: guaranteed storage for the execute work structure (must
3348	* be available when the work executes)
3349	*
3350	* Executes the function immediately if process context is available,
3351	* otherwise schedules the function for delayed execution.
3352	*
3353	* Return: 0 - function was executed
3354	* 1 - function was scheduled for execution
3355	*/
3356	int execute_in_process_context(work_func_t fn, struct execute_work *ew)
3357	{
3358	if (!in_interrupt()) {
3359	fn(&ew->work);
3360	return 0;
3361	}
3362
3363	INIT_WORK(&ew->work, fn);
3364	schedule_work(&ew->work);
3365
3366	return 1;
3367	}
3368	EXPORT_SYMBOL_GPL(execute_in_process_context);
3369
3370	/**
3371	* free_workqueue_attrs - free a workqueue_attrs
3372	* @attrs: workqueue_attrs to free
3373	*
3374	* Undo alloc_workqueue_attrs().
3375	*/
3376	void free_workqueue_attrs(struct workqueue_attrs *attrs)
3377	{
3378	if (attrs) {
3379	free_cpumask_var(attrs->cpumask);
3380	kfree(attrs);
3381	}
3382	}
3383
3384	/**
3385	* alloc_workqueue_attrs - allocate a workqueue_attrs
3386	*
3387	* Allocate a new workqueue_attrs, initialize with default settings and
3388	* return it.
3389	*
3390	* Return: The allocated new workqueue_attr on success. %NULL on failure.
3391	*/
3392	struct workqueue_attrs *alloc_workqueue_attrs(void)
3393	{
3394	struct workqueue_attrs *attrs;
3395
3396	attrs = kzalloc(sizeof(*attrs), GFP_KERNEL);
3397	if (!attrs)
3398	goto fail;
3399	if (!alloc_cpumask_var(&attrs->cpumask, GFP_KERNEL))
3400	goto fail;
3401
3402	cpumask_copy(attrs->cpumask, cpu_possible_mask);
3403	return attrs;
3404	fail:
3405	free_workqueue_attrs(attrs);
3406	return NULL;
3407	}
3408
3409	static void copy_workqueue_attrs(struct workqueue_attrs *to,
3410	const struct workqueue_attrs *from)
3411	{
3412	to->nice = from->nice;
3413	cpumask_copy(to->cpumask, from->cpumask);
3414	/*
3415	* Unlike hash and equality test, this function doesn't ignore
3416	* ->no_numa as it is used for both pool and wq attrs. Instead,
3417	* get_unbound_pool() explicitly clears ->no_numa after copying.
3418	*/
3419	to->no_numa = from->no_numa;
3420	}
3421
3422	/* hash value of the content of @attr */
3423	static u32 wqattrs_hash(const struct workqueue_attrs *attrs)
3424	{
3425	u32 hash = 0;
3426
3427	hash = jhash_1word(attrs->nice, hash);
3428	hash = jhash(cpumask_bits(attrs->cpumask),
3429	BITS_TO_LONGS(nr_cpumask_bits) * sizeof(long), hash);
3430	return hash;
3431	}
3432
3433	/* content equality test */
3434	static bool wqattrs_equal(const struct workqueue_attrs *a,
3435	const struct workqueue_attrs *b)
3436	{
3437	if (a->nice != b->nice)
3438	return false;
3439	if (!cpumask_equal(a->cpumask, b->cpumask))
3440	return false;
3441	return true;
3442	}
3443
3444	/**
3445	* init_worker_pool - initialize a newly zalloc'd worker_pool
3446	* @pool: worker_pool to initialize
3447	*
3448	* Initialize a newly zalloc'd @pool. It also allocates @pool->attrs.
3449	*
3450	* Return: 0 on success, -errno on failure. Even on failure, all fields
3451	* inside @pool proper are initialized and put_unbound_pool() can be called
3452	* on @pool safely to release it.
3453	*/
3454	static int init_worker_pool(struct worker_pool *pool)
3455	{
3456	raw_spin_lock_init(&pool->lock);
3457	pool->id = -1;
3458	pool->cpu = -1;
3459	pool->node = NUMA_NO_NODE;
3460	pool->flags \|= POOL_DISASSOCIATED;
3461	pool->watchdog_ts = jiffies;
3462	INIT_LIST_HEAD(&pool->worklist);
3463	INIT_LIST_HEAD(&pool->idle_list);
3464	hash_init(pool->busy_hash);
3465
3466	timer_setup(&pool->idle_timer, idle_worker_timeout, TIMER_DEFERRABLE);
3467
3468	timer_setup(&pool->mayday_timer, pool_mayday_timeout, 0);
3469
3470	INIT_LIST_HEAD(&pool->workers);
3471
3472	ida_init(&pool->worker_ida);
3473	INIT_HLIST_NODE(&pool->hash_node);
3474	pool->refcnt = 1;
3475
3476	/* shouldn't fail above this point */
3477	pool->attrs = alloc_workqueue_attrs();
3478	if (!pool->attrs)
3479	return -ENOMEM;
3480	return 0;
3481	}
3482
3483	#ifdef CONFIG_LOCKDEP
3484	static void wq_init_lockdep(struct workqueue_struct *wq)
3485	{
3486	char *lock_name;
3487
3488	lockdep_register_key(&wq->key);
3489	lock_name = kasprintf(GFP_KERNEL, "%s%s", "(wq_completion)", wq->name);
3490	if (!lock_name)
3491	lock_name = wq->name;
3492
3493	wq->lock_name = lock_name;
3494	lockdep_init_map(&wq->lockdep_map, lock_name, &wq->key, 0);
3495	}
3496
3497	static void wq_unregister_lockdep(struct workqueue_struct *wq)
3498	{
3499	lockdep_unregister_key(&wq->key);
3500	}
3501
3502	static void wq_free_lockdep(struct workqueue_struct *wq)
3503	{
3504	if (wq->lock_name != wq->name)
3505	kfree(wq->lock_name);
3506	}
3507	#else
3508	static void wq_init_lockdep(struct workqueue_struct *wq)
3509	{
3510	}
3511
3512	static void wq_unregister_lockdep(struct workqueue_struct *wq)
3513	{
3514	}
3515
3516	static void wq_free_lockdep(struct workqueue_struct *wq)
3517	{
3518	}
3519	#endif
3520
3521	static void rcu_free_wq(struct rcu_head *rcu)
3522	{
3523	struct workqueue_struct *wq =
3524	container_of(rcu, struct workqueue_struct, rcu);
3525
3526	wq_free_lockdep(wq);
3527
3528	if (!(wq->flags & WQ_UNBOUND))
3529	free_percpu(wq->cpu_pwqs);
3530	else
3531	free_workqueue_attrs(wq->unbound_attrs);
3532
3533	kfree(wq);
3534	}
3535
3536	static void rcu_free_pool(struct rcu_head *rcu)
3537	{
3538	struct worker_pool *pool = container_of(rcu, struct worker_pool, rcu);
3539
3540	ida_destroy(&pool->worker_ida);
3541	free_workqueue_attrs(pool->attrs);
3542	kfree(pool);
3543	}
3544
3545	/* This returns with the lock held on success (pool manager is inactive). */
3546	static bool wq_manager_inactive(struct worker_pool *pool)
3547	{
3548	raw_spin_lock_irq(&pool->lock);
3549
3550	if (pool->flags & POOL_MANAGER_ACTIVE) {
3551	raw_spin_unlock_irq(&pool->lock);
3552	return false;
3553	}
3554	return true;
3555	}
3556
3557	/**
3558	* put_unbound_pool - put a worker_pool
3559	* @pool: worker_pool to put
3560	*
3561	* Put @pool. If its refcnt reaches zero, it gets destroyed in RCU
3562	* safe manner. get_unbound_pool() calls this function on its failure path
3563	* and this function should be able to release pools which went through,
3564	* successfully or not, init_worker_pool().
3565	*
3566	* Should be called with wq_pool_mutex held.
3567	*/
3568	static void put_unbound_pool(struct worker_pool *pool)
3569	{
3570	DECLARE_COMPLETION_ONSTACK(detach_completion);
3571	struct worker *worker;
3572
3573	lockdep_assert_held(&wq_pool_mutex);
3574
3575	if (--pool->refcnt)
3576	return;
3577
3578	/* sanity checks */
3579	if (WARN_ON(!(pool->cpu < 0)) \|\|
3580	WARN_ON(!list_empty(&pool->worklist)))
3581	return;
3582
3583	/* release id and unhash */
3584	if (pool->id >= 0)
3585	idr_remove(&worker_pool_idr, pool->id);
3586	hash_del(&pool->hash_node);
3587
3588	/*
3589	* Become the manager and destroy all workers. This prevents
3590	* @pool's workers from blocking on attach_mutex. We're the last
3591	* manager and @pool gets freed with the flag set.
3592	* Because of how wq_manager_inactive() works, we will hold the
3593	* spinlock after a successful wait.
3594	*/
3595	rcuwait_wait_event(&manager_wait, wq_manager_inactive(pool),
3596	TASK_UNINTERRUPTIBLE);
3597	pool->flags \|= POOL_MANAGER_ACTIVE;
3598
3599	while ((worker = first_idle_worker(pool)))
3600	destroy_worker(worker);
3601	WARN_ON(pool->nr_workers \|\| pool->nr_idle);
3602	raw_spin_unlock_irq(&pool->lock);
3603
3604	mutex_lock(&wq_pool_attach_mutex);
3605	if (!list_empty(&pool->workers))
3606	pool->detach_completion = &detach_completion;
3607	mutex_unlock(&wq_pool_attach_mutex);
3608
3609	if (pool->detach_completion)
3610	wait_for_completion(pool->detach_completion);
3611
3612	/* shut down the timers */
3613	del_timer_sync(&pool->idle_timer);
3614	del_timer_sync(&pool->mayday_timer);
3615
3616	/* RCU protected to allow dereferences from get_work_pool() */
3617	call_rcu(&pool->rcu, rcu_free_pool);
3618	}
3619
3620	/**
3621	* get_unbound_pool - get a worker_pool with the specified attributes
3622	* @attrs: the attributes of the worker_pool to get
3623	*
3624	* Obtain a worker_pool which has the same attributes as @attrs, bump the
3625	* reference count and return it. If there already is a matching
3626	* worker_pool, it will be used; otherwise, this function attempts to
3627	* create a new one.
3628	*
3629	* Should be called with wq_pool_mutex held.
3630	*
3631	* Return: On success, a worker_pool with the same attributes as @attrs.
3632	* On failure, %NULL.
3633	*/
3634	static struct worker_pool get_unbound_pool(const struct workqueue_attrs attrs)
3635	{
3636	u32 hash = wqattrs_hash(attrs);
3637	struct worker_pool *pool;
3638	int node;
3639	int target_node = NUMA_NO_NODE;
3640
3641	lockdep_assert_held(&wq_pool_mutex);
3642
3643	/* do we already have a matching pool? */
3644	hash_for_each_possible(unbound_pool_hash, pool, hash_node, hash) {
3645	if (wqattrs_equal(pool->attrs, attrs)) {
3646	pool->refcnt++;
3647	return pool;
3648	}
3649	}
3650
3651	/* if cpumask is contained inside a NUMA node, we belong to that node */
3652	if (wq_numa_enabled) {
3653	for_each_node(node) {
3654	if (cpumask_subset(attrs->cpumask,
3655	wq_numa_possible_cpumask[node])) {
3656	target_node = node;
3657	break;
3658	}
3659	}
3660	}
3661
3662	/* nope, create a new one */
3663	pool = kzalloc_node(sizeof(*pool), GFP_KERNEL, target_node);
3664	if (!pool \|\| init_worker_pool(pool) < 0)
3665	goto fail;
3666
3667	lockdep_set_subclass(&pool->lock, 1); /* see put_pwq() */
3668	copy_workqueue_attrs(pool->attrs, attrs);
3669	pool->node = target_node;
3670
3671	/*
3672	* no_numa isn't a worker_pool attribute, always clear it. See
3673	* 'struct workqueue_attrs' comments for detail.
3674	*/
3675	pool->attrs->no_numa = false;
3676
3677	if (worker_pool_assign_id(pool) < 0)
3678	goto fail;
3679
3680	/* create and start the initial worker */
3681	if (wq_online && !create_worker(pool))
3682	goto fail;
3683
3684	/* install */
3685	hash_add(unbound_pool_hash, &pool->hash_node, hash);
3686
3687	return pool;
3688	fail:
3689	if (pool)
3690	put_unbound_pool(pool);
3691	return NULL;
3692	}
3693
3694	static void rcu_free_pwq(struct rcu_head *rcu)
3695	{
3696	kmem_cache_free(pwq_cache,
3697	container_of(rcu, struct pool_workqueue, rcu));
3698	}
3699
3700	/*
3701	* Scheduled on system_wq by put_pwq() when an unbound pwq hits zero refcnt
3702	* and needs to be destroyed.
3703	*/
3704	static void pwq_unbound_release_workfn(struct work_struct *work)
3705	{
3706	struct pool_workqueue *pwq = container_of(work, struct pool_workqueue,
3707	unbound_release_work);
3708	struct workqueue_struct *wq = pwq->wq;
3709	struct worker_pool *pool = pwq->pool;
3710	bool is_last = false;
3711
3712	/*
3713	* when @pwq is not linked, it doesn't hold any reference to the
3714	* @wq, and @wq is invalid to access.
3715	*/
3716	if (!list_empty(&pwq->pwqs_node)) {
3717	if (WARN_ON_ONCE(!(wq->flags & WQ_UNBOUND)))
3718	return;
3719
3720	mutex_lock(&wq->mutex);
3721	list_del_rcu(&pwq->pwqs_node);
3722	is_last = list_empty(&wq->pwqs);
3723	mutex_unlock(&wq->mutex);
3724	}
3725
3726	mutex_lock(&wq_pool_mutex);
3727	put_unbound_pool(pool);
3728	mutex_unlock(&wq_pool_mutex);
3729
3730	call_rcu(&pwq->rcu, rcu_free_pwq);
3731
3732	/*
3733	* If we're the last pwq going away, @wq is already dead and no one
3734	* is gonna access it anymore. Schedule RCU free.
3735	*/
3736	if (is_last) {
3737	wq_unregister_lockdep(wq);
3738	call_rcu(&wq->rcu, rcu_free_wq);
3739	}
3740	}
3741
3742	/**
3743	* pwq_adjust_max_active - update a pwq's max_active to the current setting
3744	* @pwq: target pool_workqueue
3745	*
3746	* If @pwq isn't freezing, set @pwq->max_active to the associated
3747	* workqueue's saved_max_active and activate inactive work items
3748	* accordingly. If @pwq is freezing, clear @pwq->max_active to zero.
3749	*/
3750	static void pwq_adjust_max_active(struct pool_workqueue *pwq)
3751	{
3752	struct workqueue_struct *wq = pwq->wq;
3753	bool freezable = wq->flags & WQ_FREEZABLE;
3754	unsigned long flags;
3755
3756	/* for @wq->saved_max_active */
3757	lockdep_assert_held(&wq->mutex);
3758
3759	/* fast exit for non-freezable wqs */
3760	if (!freezable && pwq->max_active == wq->saved_max_active)
3761	return;
3762
3763	/* this function can be called during early boot w/ irq disabled */
3764	raw_spin_lock_irqsave(&pwq->pool->lock, flags);
3765
3766	/*
3767	* During [un]freezing, the caller is responsible for ensuring that
3768	* this function is called at least once after @workqueue_freezing
3769	* is updated and visible.
3770	*/
3771	if (!freezable \|\| !workqueue_freezing) {
3772	bool kick = false;
3773
3774	pwq->max_active = wq->saved_max_active;
3775
3776	while (!list_empty(&pwq->inactive_works) &&
3777	pwq->nr_active < pwq->max_active) {
3778	pwq_activate_first_inactive(pwq);
3779	kick = true;
3780	}
3781
3782	/*
3783	* Need to kick a worker after thawed or an unbound wq's
3784	* max_active is bumped. In realtime scenarios, always kicking a
3785	* worker will cause interference on the isolated cpu cores, so
3786	* let's kick iff work items were activated.
3787	*/
3788	if (kick)
3789	wake_up_worker(pwq->pool);
3790	} else {
3791	pwq->max_active = 0;
3792	}
3793
3794	raw_spin_unlock_irqrestore(&pwq->pool->lock, flags);
3795	}
3796
3797	/* initialize newly allocated @pwq which is associated with @wq and @pool */
3798	static void init_pwq(struct pool_workqueue pwq, struct workqueue_struct wq,
3799	struct worker_pool *pool)
3800	{
3801	BUG_ON((unsigned long)pwq & WORK_STRUCT_FLAG_MASK);
3802
3803	memset(pwq, 0, sizeof(*pwq));
3804
3805	pwq->pool = pool;
3806	pwq->wq = wq;
3807	pwq->flush_color = -1;
3808	pwq->refcnt = 1;
3809	INIT_LIST_HEAD(&pwq->inactive_works);
3810	INIT_LIST_HEAD(&pwq->pwqs_node);
3811	INIT_LIST_HEAD(&pwq->mayday_node);
3812	INIT_WORK(&pwq->unbound_release_work, pwq_unbound_release_workfn);
3813	}
3814
3815	/* sync @pwq with the current state of its associated wq and link it */
3816	static void link_pwq(struct pool_workqueue *pwq)
3817	{
3818	struct workqueue_struct *wq = pwq->wq;
3819
3820	lockdep_assert_held(&wq->mutex);
3821
3822	/* may be called multiple times, ignore if already linked */
3823	if (!list_empty(&pwq->pwqs_node))
3824	return;
3825
3826	/* set the matching work_color */
3827	pwq->work_color = wq->work_color;
3828
3829	/* sync max_active to the current setting */
3830	pwq_adjust_max_active(pwq);
3831
3832	/* link in @pwq */
3833	list_add_rcu(&pwq->pwqs_node, &wq->pwqs);
3834	}
3835
3836	/* obtain a pool matching @attr and create a pwq associating the pool and @wq */
3837	static struct pool_workqueue alloc_unbound_pwq(struct workqueue_struct wq,
3838	const struct workqueue_attrs *attrs)
3839	{
3840	struct worker_pool *pool;
3841	struct pool_workqueue *pwq;
3842
3843	lockdep_assert_held(&wq_pool_mutex);
3844
3845	pool = get_unbound_pool(attrs);
3846	if (!pool)
3847	return NULL;
3848
3849	pwq = kmem_cache_alloc_node(pwq_cache, GFP_KERNEL, pool->node);
3850	if (!pwq) {
3851	put_unbound_pool(pool);
3852	return NULL;
3853	}
3854
3855	init_pwq(pwq, wq, pool);
3856	return pwq;
3857	}
3858
3859	/**
3860	* wq_calc_node_cpumask - calculate a wq_attrs' cpumask for the specified node
3861	* @attrs: the wq_attrs of the default pwq of the target workqueue
3862	* @node: the target NUMA node
3863	* @cpu_going_down: if >= 0, the CPU to consider as offline
3864	* @cpumask: outarg, the resulting cpumask
3865	*
3866	* Calculate the cpumask a workqueue with @attrs should use on @node. If
3867	* @cpu_going_down is >= 0, that cpu is considered offline during
3868	* calculation. The result is stored in @cpumask.
3869	*
3870	* If NUMA affinity is not enabled, @attrs->cpumask is always used. If
3871	* enabled and @node has online CPUs requested by @attrs, the returned
3872	* cpumask is the intersection of the possible CPUs of @node and
3873	* @attrs->cpumask.
3874	*
3875	* The caller is responsible for ensuring that the cpumask of @node stays
3876	* stable.
3877	*
3878	* Return: %true if the resulting @cpumask is different from @attrs->cpumask,
3879	* %false if equal.
3880	*/
3881	static bool wq_calc_node_cpumask(const struct workqueue_attrs *attrs, int node,
3882	int cpu_going_down, cpumask_t *cpumask)
3883	{
3884	if (!wq_numa_enabled \|\| attrs->no_numa)
3885	goto use_dfl;
3886
3887	/* does @node have any online CPUs @attrs wants? */
3888	cpumask_and(cpumask, cpumask_of_node(node), attrs->cpumask);
3889	if (cpu_going_down >= 0)
3890	cpumask_clear_cpu(cpu_going_down, cpumask);
3891
3892	if (cpumask_empty(cpumask))
3893	goto use_dfl;
3894
3895	/* yeap, return possible CPUs in @node that @attrs wants */
3896	cpumask_and(cpumask, attrs->cpumask, wq_numa_possible_cpumask[node]);
3897
3898	if (cpumask_empty(cpumask)) {
3899	pr_warn_once("WARNING: workqueue cpumask: online intersect > "
3900	"possible intersect\n");
3901	return false;
3902	}
3903
3904	return !cpumask_equal(cpumask, attrs->cpumask);
3905
3906	use_dfl:
3907	cpumask_copy(cpumask, attrs->cpumask);
3908	return false;
3909	}
3910
3911	/* install @pwq into @wq's numa_pwq_tbl[] for @node and return the old pwq */
3912	static struct pool_workqueue numa_pwq_tbl_install(struct workqueue_struct wq,
3913	int node,
3914	struct pool_workqueue *pwq)
3915	{
3916	struct pool_workqueue *old_pwq;
3917
3918	lockdep_assert_held(&wq_pool_mutex);
3919	lockdep_assert_held(&wq->mutex);
3920
3921	/* link_pwq() can handle duplicate calls */
3922	link_pwq(pwq);
3923
3924	old_pwq = rcu_access_pointer(wq->numa_pwq_tbl[node]);
3925	rcu_assign_pointer(wq->numa_pwq_tbl[node], pwq);
3926	return old_pwq;
3927	}
3928
3929	/* context to store the prepared attrs & pwqs before applying */
3930	struct apply_wqattrs_ctx {
3931	struct workqueue_struct wq; / target workqueue */
3932	struct workqueue_attrs attrs; / attrs to apply */
3933	struct list_head list; /* queued for batching commit */
3934	struct pool_workqueue *dfl_pwq;
3935	struct pool_workqueue *pwq_tbl[];
3936	};
3937
3938	/* free the resources after success or abort */
3939	static void apply_wqattrs_cleanup(struct apply_wqattrs_ctx *ctx)
3940	{
3941	if (ctx) {
3942	int node;
3943
3944	for_each_node(node)
3945	put_pwq_unlocked(ctx->pwq_tbl[node]);
3946	put_pwq_unlocked(ctx->dfl_pwq);
3947
3948	free_workqueue_attrs(ctx->attrs);
3949
3950	kfree(ctx);
3951	}
3952	}
3953
3954	/* allocate the attrs and pwqs for later installation */
3955	static struct apply_wqattrs_ctx *
3956	apply_wqattrs_prepare(struct workqueue_struct *wq,
3957	const struct workqueue_attrs *attrs)
3958	{
3959	struct apply_wqattrs_ctx *ctx;
3960	struct workqueue_attrs new_attrs, tmp_attrs;
3961	int node;
3962
3963	lockdep_assert_held(&wq_pool_mutex);
3964
3965	ctx = kzalloc(struct_size(ctx, pwq_tbl, nr_node_ids), GFP_KERNEL);
3966
3967	new_attrs = alloc_workqueue_attrs();
3968	tmp_attrs = alloc_workqueue_attrs();
3969	if (!ctx \|\| !new_attrs \|\| !tmp_attrs)
3970	goto out_free;
3971
3972	/*
3973	* Calculate the attrs of the default pwq.
3974	* If the user configured cpumask doesn't overlap with the
3975	* wq_unbound_cpumask, we fallback to the wq_unbound_cpumask.
3976	*/
3977	copy_workqueue_attrs(new_attrs, attrs);
3978	cpumask_and(new_attrs->cpumask, new_attrs->cpumask, wq_unbound_cpumask);
3979	if (unlikely(cpumask_empty(new_attrs->cpumask)))
3980	cpumask_copy(new_attrs->cpumask, wq_unbound_cpumask);
3981
3982	/*
3983	* We may create multiple pwqs with differing cpumasks. Make a
3984	* copy of @new_attrs which will be modified and used to obtain
3985	* pools.
3986	*/
3987	copy_workqueue_attrs(tmp_attrs, new_attrs);
3988
3989	/*
3990	* If something goes wrong during CPU up/down, we'll fall back to
3991	* the default pwq covering whole @attrs->cpumask. Always create
3992	* it even if we don't use it immediately.
3993	*/
3994	ctx->dfl_pwq = alloc_unbound_pwq(wq, new_attrs);
3995	if (!ctx->dfl_pwq)
3996	goto out_free;
3997
3998	for_each_node(node) {
3999	if (wq_calc_node_cpumask(new_attrs, node, -1, tmp_attrs->cpumask)) {
4000	ctx->pwq_tbl[node] = alloc_unbound_pwq(wq, tmp_attrs);
4001	if (!ctx->pwq_tbl[node])
4002	goto out_free;
4003	} else {
4004	ctx->dfl_pwq->refcnt++;
4005	ctx->pwq_tbl[node] = ctx->dfl_pwq;
4006	}
4007	}
4008
4009	/* save the user configured attrs and sanitize it. */
4010	copy_workqueue_attrs(new_attrs, attrs);
4011	cpumask_and(new_attrs->cpumask, new_attrs->cpumask, cpu_possible_mask);
4012	ctx->attrs = new_attrs;
4013
4014	ctx->wq = wq;
4015	free_workqueue_attrs(tmp_attrs);
4016	return ctx;
4017
4018	out_free:
4019	free_workqueue_attrs(tmp_attrs);
4020	free_workqueue_attrs(new_attrs);
4021	apply_wqattrs_cleanup(ctx);
4022	return NULL;
4023	}
4024
4025	/* set attrs and install prepared pwqs, @ctx points to old pwqs on return */
4026	static void apply_wqattrs_commit(struct apply_wqattrs_ctx *ctx)
4027	{
4028	int node;
4029
4030	/* all pwqs have been created successfully, let's install'em */
4031	mutex_lock(&ctx->wq->mutex);
4032
4033	copy_workqueue_attrs(ctx->wq->unbound_attrs, ctx->attrs);
4034
4035	/* save the previous pwq and install the new one */
4036	for_each_node(node)
4037	ctx->pwq_tbl[node] = numa_pwq_tbl_install(ctx->wq, node,
4038	ctx->pwq_tbl[node]);
4039
4040	/* @dfl_pwq might not have been used, ensure it's linked */
4041	link_pwq(ctx->dfl_pwq);
4042	swap(ctx->wq->dfl_pwq, ctx->dfl_pwq);
4043
4044	mutex_unlock(&ctx->wq->mutex);
4045	}
4046
4047	static void apply_wqattrs_lock(void)
4048	{
4049	/* CPUs should stay stable across pwq creations and installations */
4050	cpus_read_lock();
4051	mutex_lock(&wq_pool_mutex);
4052	}
4053
4054	static void apply_wqattrs_unlock(void)
4055	{
4056	mutex_unlock(&wq_pool_mutex);
4057	cpus_read_unlock();
4058	}
4059
4060	static int apply_workqueue_attrs_locked(struct workqueue_struct *wq,
4061	const struct workqueue_attrs *attrs)
4062	{
4063	struct apply_wqattrs_ctx *ctx;
4064
4065	/* only unbound workqueues can change attributes */
4066	if (WARN_ON(!(wq->flags & WQ_UNBOUND)))
4067	return -EINVAL;
4068
4069	/* creating multiple pwqs breaks ordering guarantee */
4070	if (!list_empty(&wq->pwqs)) {
4071	if (WARN_ON(wq->flags & __WQ_ORDERED_EXPLICIT))
4072	return -EINVAL;
4073
4074	wq->flags &= ~__WQ_ORDERED;
4075	}
4076
4077	ctx = apply_wqattrs_prepare(wq, attrs);
4078	if (!ctx)
4079	return -ENOMEM;
4080
4081	/* the ctx has been prepared successfully, let's commit it */
4082	apply_wqattrs_commit(ctx);
4083	apply_wqattrs_cleanup(ctx);
4084
4085	return 0;
4086	}
4087
4088	/**
4089	* apply_workqueue_attrs - apply new workqueue_attrs to an unbound workqueue
4090	* @wq: the target workqueue
4091	* @attrs: the workqueue_attrs to apply, allocated with alloc_workqueue_attrs()
4092	*
4093	* Apply @attrs to an unbound workqueue @wq. Unless disabled, on NUMA
4094	* machines, this function maps a separate pwq to each NUMA node with
4095	* possibles CPUs in @attrs->cpumask so that work items are affine to the
4096	* NUMA node it was issued on. Older pwqs are released as in-flight work
4097	* items finish. Note that a work item which repeatedly requeues itself
4098	* back-to-back will stay on its current pwq.
4099	*
4100	* Performs GFP_KERNEL allocations.
4101	*
4102	* Assumes caller has CPU hotplug read exclusion, i.e. cpus_read_lock().
4103	*
4104	* Return: 0 on success and -errno on failure.
4105	*/
4106	int apply_workqueue_attrs(struct workqueue_struct *wq,
4107	const struct workqueue_attrs *attrs)
4108	{
4109	int ret;
4110
4111	lockdep_assert_cpus_held();
4112
4113	mutex_lock(&wq_pool_mutex);
4114	ret = apply_workqueue_attrs_locked(wq, attrs);
4115	mutex_unlock(&wq_pool_mutex);
4116
4117	return ret;
4118	}
4119
4120	/**
4121	* wq_update_unbound_numa - update NUMA affinity of a wq for CPU hot[un]plug
4122	* @wq: the target workqueue
4123	* @cpu: the CPU coming up or going down
4124	* @online: whether @cpu is coming up or going down
4125	*
4126	* This function is to be called from %CPU_DOWN_PREPARE, %CPU_ONLINE and
4127	* %CPU_DOWN_FAILED. @cpu is being hot[un]plugged, update NUMA affinity of
4128	* @wq accordingly.
4129	*
4130	* If NUMA affinity can't be adjusted due to memory allocation failure, it
4131	* falls back to @wq->dfl_pwq which may not be optimal but is always
4132	* correct.
4133	*
4134	* Note that when the last allowed CPU of a NUMA node goes offline for a
4135	* workqueue with a cpumask spanning multiple nodes, the workers which were
4136	* already executing the work items for the workqueue will lose their CPU
4137	* affinity and may execute on any CPU. This is similar to how per-cpu
4138	* workqueues behave on CPU_DOWN. If a workqueue user wants strict
4139	* affinity, it's the user's responsibility to flush the work item from
4140	* CPU_DOWN_PREPARE.
4141	*/
4142	static void wq_update_unbound_numa(struct workqueue_struct *wq, int cpu,
4143	bool online)
4144	{
4145	int node = cpu_to_node(cpu);
4146	int cpu_off = online ? -1 : cpu;
4147	struct pool_workqueue old_pwq = NULL, pwq;
4148	struct workqueue_attrs *target_attrs;
4149	cpumask_t *cpumask;
4150
4151	lockdep_assert_held(&wq_pool_mutex);
4152
4153	if (!wq_numa_enabled \|\| !(wq->flags & WQ_UNBOUND) \|\|
4154	wq->unbound_attrs->no_numa)
4155	return;
4156
4157	/*
4158	* We don't wanna alloc/free wq_attrs for each wq for each CPU.
4159	* Let's use a preallocated one. The following buf is protected by
4160	* CPU hotplug exclusion.
4161	*/
4162	target_attrs = wq_update_unbound_numa_attrs_buf;
4163	cpumask = target_attrs->cpumask;
4164
4165	copy_workqueue_attrs(target_attrs, wq->unbound_attrs);
4166	pwq = unbound_pwq_by_node(wq, node);
4167
4168	/*
4169	* Let's determine what needs to be done. If the target cpumask is
4170	* different from the default pwq's, we need to compare it to @pwq's
4171	* and create a new one if they don't match. If the target cpumask
4172	* equals the default pwq's, the default pwq should be used.
4173	*/
4174	if (wq_calc_node_cpumask(wq->dfl_pwq->pool->attrs, node, cpu_off, cpumask)) {
4175	if (cpumask_equal(cpumask, pwq->pool->attrs->cpumask))
4176	return;
4177	} else {
4178	goto use_dfl_pwq;
4179	}
4180
4181	/* create a new pwq */
4182	pwq = alloc_unbound_pwq(wq, target_attrs);
4183	if (!pwq) {
4184	pr_warn("workqueue: allocation failed while updating NUMA affinity of \"%s\"\n",
4185	wq->name);
4186	goto use_dfl_pwq;
4187	}
4188
4189	/* Install the new pwq. */
4190	mutex_lock(&wq->mutex);
4191	old_pwq = numa_pwq_tbl_install(wq, node, pwq);
4192	goto out_unlock;
4193
4194	use_dfl_pwq:
4195	mutex_lock(&wq->mutex);
4196	raw_spin_lock_irq(&wq->dfl_pwq->pool->lock);
4197	get_pwq(wq->dfl_pwq);
4198	raw_spin_unlock_irq(&wq->dfl_pwq->pool->lock);
4199	old_pwq = numa_pwq_tbl_install(wq, node, wq->dfl_pwq);
4200	out_unlock:
4201	mutex_unlock(&wq->mutex);
4202	put_pwq_unlocked(old_pwq);
4203	}
4204
4205	static int alloc_and_link_pwqs(struct workqueue_struct *wq)
4206	{
4207	bool highpri = wq->flags & WQ_HIGHPRI;
4208	int cpu, ret;
4209
4210	if (!(wq->flags & WQ_UNBOUND)) {
4211	wq->cpu_pwqs = alloc_percpu(struct pool_workqueue);
4212	if (!wq->cpu_pwqs)
4213	return -ENOMEM;
4214
4215	for_each_possible_cpu(cpu) {
4216	struct pool_workqueue *pwq =
4217	per_cpu_ptr(wq->cpu_pwqs, cpu);
4218	struct worker_pool *cpu_pools =
4219	per_cpu(cpu_worker_pools, cpu);
4220
4221	init_pwq(pwq, wq, &cpu_pools[highpri]);
4222
4223	mutex_lock(&wq->mutex);
4224	link_pwq(pwq);
4225	mutex_unlock(&wq->mutex);
4226	}
4227	return 0;
4228	}
4229
4230	cpus_read_lock();
4231	if (wq->flags & __WQ_ORDERED) {
4232	ret = apply_workqueue_attrs(wq, ordered_wq_attrs[highpri]);
4233	/* there should only be single pwq for ordering guarantee */
4234	WARN(!ret && (wq->pwqs.next != &wq->dfl_pwq->pwqs_node \|\|
4235	wq->pwqs.prev != &wq->dfl_pwq->pwqs_node),
4236	"ordering guarantee broken for workqueue %s\n", wq->name);
4237	} else {
4238	ret = apply_workqueue_attrs(wq, unbound_std_wq_attrs[highpri]);
4239	}
4240	cpus_read_unlock();
4241
4242	return ret;
4243	}
4244
4245	static int wq_clamp_max_active(int max_active, unsigned int flags,
4246	const char *name)
4247	{
4248	int lim = flags & WQ_UNBOUND ? WQ_UNBOUND_MAX_ACTIVE : WQ_MAX_ACTIVE;
4249
4250	if (max_active < 1 \|\| max_active > lim)
4251	pr_warn("workqueue: max_active %d requested for %s is out of range, clamping between %d and %d\n",
4252	max_active, name, 1, lim);
4253
4254	return clamp_val(max_active, 1, lim);
4255	}
4256
4257	/*
4258	* Workqueues which may be used during memory reclaim should have a rescuer
4259	* to guarantee forward progress.
4260	*/
4261	static int init_rescuer(struct workqueue_struct *wq)
4262	{
4263	struct worker *rescuer;
4264	int ret;
4265
4266	if (!(wq->flags & WQ_MEM_RECLAIM))
4267	return 0;
4268
4269	rescuer = alloc_worker(NUMA_NO_NODE);
4270	if (!rescuer)
4271	return -ENOMEM;
4272
4273	rescuer->rescue_wq = wq;
4274	rescuer->task = kthread_create(rescuer_thread, rescuer, "%s", wq->name);
4275	if (IS_ERR(rescuer->task)) {
4276	ret = PTR_ERR(rescuer->task);
4277	kfree(rescuer);
4278	return ret;
4279	}
4280
4281	wq->rescuer = rescuer;
4282	kthread_bind_mask(rescuer->task, cpu_possible_mask);
4283	wake_up_process(rescuer->task);
4284
4285	return 0;
4286	}
4287
4288	__printf(1, 4)
4289	struct workqueue_struct alloc_workqueue(const char fmt,
4290	unsigned int flags,
4291	int max_active, ...)
4292	{
4293	size_t tbl_size = 0;
4294	va_list args;
4295	struct workqueue_struct *wq;
4296	struct pool_workqueue *pwq;
4297
4298	/*
4299	* Unbound && max_active == 1 used to imply ordered, which is no
4300	* longer the case on NUMA machines due to per-node pools. While
4301	* alloc_ordered_workqueue() is the right way to create an ordered
4302	* workqueue, keep the previous behavior to avoid subtle breakages
4303	* on NUMA.
4304	*/
4305	if ((flags & WQ_UNBOUND) && max_active == 1)
4306	flags \|= __WQ_ORDERED;
4307
4308	/* see the comment above the definition of WQ_POWER_EFFICIENT */
4309	if ((flags & WQ_POWER_EFFICIENT) && wq_power_efficient)
4310	flags \|= WQ_UNBOUND;
4311
4312	/* allocate wq and format name */
4313	if (flags & WQ_UNBOUND)
4314	tbl_size = nr_node_ids * sizeof(wq->numa_pwq_tbl[0]);
4315
4316	wq = kzalloc(sizeof(*wq) + tbl_size, GFP_KERNEL);
4317	if (!wq)
4318	return NULL;
4319
4320	if (flags & WQ_UNBOUND) {
4321	wq->unbound_attrs = alloc_workqueue_attrs();
4322	if (!wq->unbound_attrs)
4323	goto err_free_wq;
4324	}
4325
4326	va_start(args, max_active);
4327	vsnprintf(wq->name, sizeof(wq->name), fmt, args);
4328	va_end(args);
4329
4330	max_active = max_active ?: WQ_DFL_ACTIVE;
4331	max_active = wq_clamp_max_active(max_active, flags, wq->name);
4332
4333	/* init wq */
4334	wq->flags = flags;
4335	wq->saved_max_active = max_active;
4336	mutex_init(&wq->mutex);
4337	atomic_set(&wq->nr_pwqs_to_flush, 0);
4338	INIT_LIST_HEAD(&wq->pwqs);
4339	INIT_LIST_HEAD(&wq->flusher_queue);
4340	INIT_LIST_HEAD(&wq->flusher_overflow);
4341	INIT_LIST_HEAD(&wq->maydays);
4342
4343	wq_init_lockdep(wq);
4344	INIT_LIST_HEAD(&wq->list);
4345
4346	if (alloc_and_link_pwqs(wq) < 0)
4347	goto err_unreg_lockdep;
4348
4349	if (wq_online && init_rescuer(wq) < 0)
4350	goto err_destroy;
4351
4352	if ((wq->flags & WQ_SYSFS) && workqueue_sysfs_register(wq))
4353	goto err_destroy;
4354
4355	/*
4356	* wq_pool_mutex protects global freeze state and workqueues list.
4357	* Grab it, adjust max_active and add the new @wq to workqueues
4358	* list.
4359	*/
4360	mutex_lock(&wq_pool_mutex);
4361
4362	mutex_lock(&wq->mutex);
4363	for_each_pwq(pwq, wq)
4364	pwq_adjust_max_active(pwq);
4365	mutex_unlock(&wq->mutex);
4366
4367	list_add_tail_rcu(&wq->list, &workqueues);
4368
4369	mutex_unlock(&wq_pool_mutex);
4370
4371	return wq;
4372
4373	err_unreg_lockdep:
4374	wq_unregister_lockdep(wq);
4375	wq_free_lockdep(wq);
4376	err_free_wq:
4377	free_workqueue_attrs(wq->unbound_attrs);
4378	kfree(wq);
4379	return NULL;
4380	err_destroy:
4381	destroy_workqueue(wq);
4382	return NULL;
4383	}
4384	EXPORT_SYMBOL_GPL(alloc_workqueue);
4385
4386	static bool pwq_busy(struct pool_workqueue *pwq)
4387	{
4388	int i;
4389
4390	for (i = 0; i < WORK_NR_COLORS; i++)
4391	if (pwq->nr_in_flight[i])
4392	return true;
4393
4394	if ((pwq != pwq->wq->dfl_pwq) && (pwq->refcnt > 1))
4395	return true;
4396	if (pwq->nr_active \|\| !list_empty(&pwq->inactive_works))
4397	return true;
4398
4399	return false;
4400	}
4401
4402	/**
4403	* destroy_workqueue - safely terminate a workqueue
4404	* @wq: target workqueue
4405	*
4406	* Safely destroy a workqueue. All work currently pending will be done first.
4407	*/
4408	void destroy_workqueue(struct workqueue_struct *wq)
4409	{
4410	struct pool_workqueue *pwq;
4411	int node;
4412
4413	/*
4414	* Remove it from sysfs first so that sanity check failure doesn't
4415	* lead to sysfs name conflicts.
4416	*/
4417	workqueue_sysfs_unregister(wq);
4418
4419	/* drain it before proceeding with destruction */
4420	drain_workqueue(wq);
4421
4422	/* kill rescuer, if sanity checks fail, leave it w/o rescuer */
4423	if (wq->rescuer) {
4424	struct worker *rescuer = wq->rescuer;
4425
4426	/* this prevents new queueing */
4427	raw_spin_lock_irq(&wq_mayday_lock);
4428	wq->rescuer = NULL;
4429	raw_spin_unlock_irq(&wq_mayday_lock);
4430
4431	/* rescuer will empty maydays list before exiting */
4432	kthread_stop(rescuer->task);
4433	kfree(rescuer);
4434	}
4435
4436	/*
4437	* Sanity checks - grab all the locks so that we wait for all
4438	* in-flight operations which may do put_pwq().
4439	*/
4440	mutex_lock(&wq_pool_mutex);
4441	mutex_lock(&wq->mutex);
4442	for_each_pwq(pwq, wq) {
4443	raw_spin_lock_irq(&pwq->pool->lock);
4444	if (WARN_ON(pwq_busy(pwq))) {
4445	pr_warn("%s: %s has the following busy pwq\n",
4446	__func__, wq->name);
4447	show_pwq(pwq);
4448	raw_spin_unlock_irq(&pwq->pool->lock);
4449	mutex_unlock(&wq->mutex);
4450	mutex_unlock(&wq_pool_mutex);
4451	show_one_workqueue(wq);
4452	return;
4453	}
4454	raw_spin_unlock_irq(&pwq->pool->lock);
4455	}
4456	mutex_unlock(&wq->mutex);
4457
4458	/*
4459	* wq list is used to freeze wq, remove from list after
4460	* flushing is complete in case freeze races us.
4461	*/
4462	list_del_rcu(&wq->list);
4463	mutex_unlock(&wq_pool_mutex);
4464
4465	if (!(wq->flags & WQ_UNBOUND)) {
4466	wq_unregister_lockdep(wq);
4467	/*
4468	* The base ref is never dropped on per-cpu pwqs. Directly
4469	* schedule RCU free.
4470	*/
4471	call_rcu(&wq->rcu, rcu_free_wq);
4472	} else {
4473	/*
4474	* We're the sole accessor of @wq at this point. Directly
4475	* access numa_pwq_tbl[] and dfl_pwq to put the base refs.
4476	* @wq will be freed when the last pwq is released.
4477	*/
4478	for_each_node(node) {
4479	pwq = rcu_access_pointer(wq->numa_pwq_tbl[node]);
4480	RCU_INIT_POINTER(wq->numa_pwq_tbl[node], NULL);
4481	put_pwq_unlocked(pwq);
4482	}
4483
4484	/*
4485	* Put dfl_pwq. @wq may be freed any time after dfl_pwq is
4486	* put. Don't access it afterwards.
4487	*/
4488	pwq = wq->dfl_pwq;
4489	wq->dfl_pwq = NULL;
4490	put_pwq_unlocked(pwq);
4491	}
4492	}
4493	EXPORT_SYMBOL_GPL(destroy_workqueue);
4494
4495	/**
4496	* workqueue_set_max_active - adjust max_active of a workqueue
4497	* @wq: target workqueue
4498	* @max_active: new max_active value.
4499	*
4500	* Set max_active of @wq to @max_active.
4501	*
4502	* CONTEXT:
4503	* Don't call from IRQ context.
4504	*/
4505	void workqueue_set_max_active(struct workqueue_struct *wq, int max_active)
4506	{
4507	struct pool_workqueue *pwq;
4508
4509	/* disallow meddling with max_active for ordered workqueues */
4510	if (WARN_ON(wq->flags & __WQ_ORDERED_EXPLICIT))
4511	return;
4512
4513	max_active = wq_clamp_max_active(max_active, wq->flags, wq->name);
4514
4515	mutex_lock(&wq->mutex);
4516
4517	wq->flags &= ~__WQ_ORDERED;
4518	wq->saved_max_active = max_active;
4519
4520	for_each_pwq(pwq, wq)
4521	pwq_adjust_max_active(pwq);
4522
4523	mutex_unlock(&wq->mutex);
4524	}
4525	EXPORT_SYMBOL_GPL(workqueue_set_max_active);
4526
4527	/**
4528	* current_work - retrieve %current task's work struct
4529	*
4530	* Determine if %current task is a workqueue worker and what it's working on.
4531	* Useful to find out the context that the %current task is running in.
4532	*
4533	* Return: work struct if %current task is a workqueue worker, %NULL otherwise.
4534	*/
4535	struct work_struct *current_work(void)
4536	{
4537	struct worker *worker = current_wq_worker();
4538
4539	return worker ? worker->current_work : NULL;
4540	}
4541	EXPORT_SYMBOL(current_work);
4542
4543	/**
4544	* current_is_workqueue_rescuer - is %current workqueue rescuer?
4545	*
4546	* Determine whether %current is a workqueue rescuer. Can be used from
4547	* work functions to determine whether it's being run off the rescuer task.
4548	*
4549	* Return: %true if %current is a workqueue rescuer. %false otherwise.
4550	*/
4551	bool current_is_workqueue_rescuer(void)
4552	{
4553	struct worker *worker = current_wq_worker();
4554
4555	return worker && worker->rescue_wq;
4556	}
4557
4558	/**
4559	* workqueue_congested - test whether a workqueue is congested
4560	* @cpu: CPU in question
4561	* @wq: target workqueue
4562	*
4563	* Test whether @wq's cpu workqueue for @cpu is congested. There is
4564	* no synchronization around this function and the test result is
4565	* unreliable and only useful as advisory hints or for debugging.
4566	*
4567	* If @cpu is WORK_CPU_UNBOUND, the test is performed on the local CPU.
4568	* Note that both per-cpu and unbound workqueues may be associated with
4569	* multiple pool_workqueues which have separate congested states. A
4570	* workqueue being congested on one CPU doesn't mean the workqueue is also
4571	* contested on other CPUs / NUMA nodes.
4572	*
4573	* Return:
4574	* %true if congested, %false otherwise.
4575	*/
4576	bool workqueue_congested(int cpu, struct workqueue_struct *wq)
4577	{
4578	struct pool_workqueue *pwq;
4579	bool ret;
4580
4581	rcu_read_lock();
4582	preempt_disable();
4583
4584	if (cpu == WORK_CPU_UNBOUND)
4585	cpu = smp_processor_id();
4586
4587	if (!(wq->flags & WQ_UNBOUND))
4588	pwq = per_cpu_ptr(wq->cpu_pwqs, cpu);
4589	else
4590	pwq = unbound_pwq_by_node(wq, cpu_to_node(cpu));
4591
4592	ret = !list_empty(&pwq->inactive_works);
4593	preempt_enable();
4594	rcu_read_unlock();
4595
4596	return ret;
4597	}
4598	EXPORT_SYMBOL_GPL(workqueue_congested);
4599
4600	/**
4601	* work_busy - test whether a work is currently pending or running
4602	* @work: the work to be tested
4603	*
4604	* Test whether @work is currently pending or running. There is no
4605	* synchronization around this function and the test result is
4606	* unreliable and only useful as advisory hints or for debugging.
4607	*
4608	* Return:
4609	* OR'd bitmask of WORK_BUSY_* bits.
4610	*/
4611	unsigned int work_busy(struct work_struct *work)
4612	{
4613	struct worker_pool *pool;
4614	unsigned long flags;
4615	unsigned int ret = 0;
4616
4617	if (work_pending(work))
4618	ret \|= WORK_BUSY_PENDING;
4619
4620	rcu_read_lock();
4621	pool = get_work_pool(work);
4622	if (pool) {
4623	raw_spin_lock_irqsave(&pool->lock, flags);
4624	if (find_worker_executing_work(pool, work))
4625	ret \|= WORK_BUSY_RUNNING;
4626	raw_spin_unlock_irqrestore(&pool->lock, flags);
4627	}
4628	rcu_read_unlock();
4629
4630	return ret;
4631	}
4632	EXPORT_SYMBOL_GPL(work_busy);
4633
4634	/**
4635	* set_worker_desc - set description for the current work item
4636	* @fmt: printf-style format string
4637	* @...: arguments for the format string
4638	*
4639	* This function can be called by a running work function to describe what
4640	* the work item is about. If the worker task gets dumped, this
4641	* information will be printed out together to help debugging. The
4642	* description can be at most WORKER_DESC_LEN including the trailing '\0'.
4643	*/
4644	void set_worker_desc(const char *fmt, ...)
4645	{
4646	struct worker *worker = current_wq_worker();
4647	va_list args;
4648
4649	if (worker) {
4650	va_start(args, fmt);
4651	vsnprintf(worker->desc, sizeof(worker->desc), fmt, args);
4652	va_end(args);
4653	}
4654	}
4655	EXPORT_SYMBOL_GPL(set_worker_desc);
4656
4657	/**
4658	* print_worker_info - print out worker information and description
4659	* @log_lvl: the log level to use when printing
4660	* @task: target task
4661	*
4662	* If @task is a worker and currently executing a work item, print out the
4663	* name of the workqueue being serviced and worker description set with
4664	* set_worker_desc() by the currently executing work item.
4665	*
4666	* This function can be safely called on any task as long as the
4667	* task_struct itself is accessible. While safe, this function isn't
4668	* synchronized and may print out mixups or garbages of limited length.
4669	*/
4670	void print_worker_info(const char log_lvl, struct task_struct task)
4671	{
4672	work_func_t *fn = NULL;
4673	char name[WQ_NAME_LEN] = { };
4674	char desc[WORKER_DESC_LEN] = { };
4675	struct pool_workqueue *pwq = NULL;
4676	struct workqueue_struct *wq = NULL;
4677	struct worker *worker;
4678
4679	if (!(task->flags & PF_WQ_WORKER))
4680	return;
4681
4682	/*
4683	* This function is called without any synchronization and @task
4684	* could be in any state. Be careful with dereferences.
4685	*/
4686	worker = kthread_probe_data(task);
4687
4688	/*
4689	* Carefully copy the associated workqueue's workfn, name and desc.
4690	* Keep the original last '\0' in case the original is garbage.
4691	*/
4692	copy_from_kernel_nofault(&fn, &worker->current_func, sizeof(fn));
4693	copy_from_kernel_nofault(&pwq, &worker->current_pwq, sizeof(pwq));
4694	copy_from_kernel_nofault(&wq, &pwq->wq, sizeof(wq));
4695	copy_from_kernel_nofault(name, wq->name, sizeof(name) - 1);
4696	copy_from_kernel_nofault(desc, worker->desc, sizeof(desc) - 1);
4697
4698	if (fn \|\| name[0] \|\| desc[0]) {
4699	printk("%sWorkqueue: %s %ps", log_lvl, name, fn);
4700	if (strcmp(name, desc))
4701	pr_cont(" (%s)", desc);
4702	pr_cont("\n");
4703	}
4704	}
4705
4706	static void pr_cont_pool_info(struct worker_pool *pool)
4707	{
4708	pr_cont(" cpus=%*pbl", nr_cpumask_bits, pool->attrs->cpumask);
4709	if (pool->node != NUMA_NO_NODE)
4710	pr_cont(" node=%d", pool->node);
4711	pr_cont(" flags=0x%x nice=%d", pool->flags, pool->attrs->nice);
4712	}
4713
4714	static void pr_cont_work(bool comma, struct work_struct *work)
4715	{
4716	if (work->func == wq_barrier_func) {
4717	struct wq_barrier *barr;
4718
4719	barr = container_of(work, struct wq_barrier, work);
4720
4721	pr_cont("%s BAR(%d)", comma ? "," : "",
4722	task_pid_nr(barr->task));
4723	} else {
4724	pr_cont("%s %ps", comma ? "," : "", work->func);
4725	}
4726	}
4727
4728	static void show_pwq(struct pool_workqueue *pwq)
4729	{
4730	struct worker_pool *pool = pwq->pool;
4731	struct work_struct *work;
4732	struct worker *worker;
4733	bool has_in_flight = false, has_pending = false;
4734	int bkt;
4735
4736	pr_info(" pwq %d:", pool->id);
4737	pr_cont_pool_info(pool);
4738
4739	pr_cont(" active=%d/%d refcnt=%d%s\n",
4740	pwq->nr_active, pwq->max_active, pwq->refcnt,
4741	!list_empty(&pwq->mayday_node) ? " MAYDAY" : "");
4742
4743	hash_for_each(pool->busy_hash, bkt, worker, hentry) {
4744	if (worker->current_pwq == pwq) {
4745	has_in_flight = true;
4746	break;
4747	}
4748	}
4749	if (has_in_flight) {
4750	bool comma = false;
4751
4752	pr_info(" in-flight:");
4753	hash_for_each(pool->busy_hash, bkt, worker, hentry) {
4754	if (worker->current_pwq != pwq)
4755	continue;
4756
4757	pr_cont("%s %d%s:%ps", comma ? "," : "",
4758	task_pid_nr(worker->task),
4759	worker->rescue_wq ? "(RESCUER)" : "",
4760	worker->current_func);
4761	list_for_each_entry(work, &worker->scheduled, entry)
4762	pr_cont_work(false, work);
4763	comma = true;
4764	}
4765	pr_cont("\n");
4766	}
4767
4768	list_for_each_entry(work, &pool->worklist, entry) {
4769	if (get_work_pwq(work) == pwq) {
4770	has_pending = true;
4771	break;
4772	}
4773	}
4774	if (has_pending) {
4775	bool comma = false;
4776
4777	pr_info(" pending:");
4778	list_for_each_entry(work, &pool->worklist, entry) {
4779	if (get_work_pwq(work) != pwq)
4780	continue;
4781
4782	pr_cont_work(comma, work);
4783	comma = !(*work_data_bits(work) & WORK_STRUCT_LINKED);
4784	}
4785	pr_cont("\n");
4786	}
4787
4788	if (!list_empty(&pwq->inactive_works)) {
4789	bool comma = false;
4790
4791	pr_info(" inactive:");
4792	list_for_each_entry(work, &pwq->inactive_works, entry) {
4793	pr_cont_work(comma, work);
4794	comma = !(*work_data_bits(work) & WORK_STRUCT_LINKED);
4795	}
4796	pr_cont("\n");
4797	}
4798	}
4799
4800	/**
4801	* show_one_workqueue - dump state of specified workqueue
4802	* @wq: workqueue whose state will be printed
4803	*/
4804	void show_one_workqueue(struct workqueue_struct *wq)
4805	{
4806	struct pool_workqueue *pwq;
4807	bool idle = true;
4808	unsigned long flags;
4809
4810	for_each_pwq(pwq, wq) {
4811	if (pwq->nr_active \|\| !list_empty(&pwq->inactive_works)) {
4812	idle = false;
4813	break;
4814	}
4815	}
4816	if (idle) /* Nothing to print for idle workqueue */
4817	return;
4818
4819	pr_info("workqueue %s: flags=0x%x\n", wq->name, wq->flags);
4820
4821	for_each_pwq(pwq, wq) {
4822	raw_spin_lock_irqsave(&pwq->pool->lock, flags);
4823	if (pwq->nr_active \|\| !list_empty(&pwq->inactive_works)) {
4824	/*
4825	* Defer printing to avoid deadlocks in console
4826	* drivers that queue work while holding locks
4827	* also taken in their write paths.
4828	*/
4829	printk_deferred_enter();
4830	show_pwq(pwq);
4831	printk_deferred_exit();
4832	}
4833	raw_spin_unlock_irqrestore(&pwq->pool->lock, flags);
4834	/*
4835	* We could be printing a lot from atomic context, e.g.
4836	* sysrq-t -> show_all_workqueues(). Avoid triggering
4837	* hard lockup.
4838	*/
4839	touch_nmi_watchdog();
4840	}
4841
4842	}
4843
4844	/**
4845	* show_one_worker_pool - dump state of specified worker pool
4846	* @pool: worker pool whose state will be printed
4847	*/
4848	static void show_one_worker_pool(struct worker_pool *pool)
4849	{
4850	struct worker *worker;
4851	bool first = true;
4852	unsigned long flags;
4853
4854	raw_spin_lock_irqsave(&pool->lock, flags);
4855	if (pool->nr_workers == pool->nr_idle)
4856	goto next_pool;
4857	/*
4858	* Defer printing to avoid deadlocks in console drivers that
4859	* queue work while holding locks also taken in their write
4860	* paths.
4861	*/
4862	printk_deferred_enter();
4863	pr_info("pool %d:", pool->id);
4864	pr_cont_pool_info(pool);
4865	pr_cont(" hung=%us workers=%d",
4866	jiffies_to_msecs(jiffies - pool->watchdog_ts) / 1000,
4867	pool->nr_workers);
4868	if (pool->manager)
4869	pr_cont(" manager: %d",
4870	task_pid_nr(pool->manager->task));
4871	list_for_each_entry(worker, &pool->idle_list, entry) {
4872	pr_cont(" %s%d", first ? "idle: " : "",
4873	task_pid_nr(worker->task));
4874	first = false;
4875	}
4876	pr_cont("\n");
4877	printk_deferred_exit();
4878	next_pool:
4879	raw_spin_unlock_irqrestore(&pool->lock, flags);
4880	/*
4881	* We could be printing a lot from atomic context, e.g.
4882	* sysrq-t -> show_all_workqueues(). Avoid triggering
4883	* hard lockup.
4884	*/
4885	touch_nmi_watchdog();
4886
4887	}
4888
4889	/**
4890	* show_all_workqueues - dump workqueue state
4891	*
4892	* Called from a sysrq handler or try_to_freeze_tasks() and prints out
4893	* all busy workqueues and pools.
4894	*/
4895	void show_all_workqueues(void)
4896	{
4897	struct workqueue_struct *wq;
4898	struct worker_pool *pool;
4899	int pi;
4900
4901	rcu_read_lock();
4902
4903	pr_info("Showing busy workqueues and worker pools:\n");
4904
4905	list_for_each_entry_rcu(wq, &workqueues, list)
4906	show_one_workqueue(wq);
4907
4908	for_each_pool(pool, pi)
4909	show_one_worker_pool(pool);
4910
4911	rcu_read_unlock();
4912	}
4913
4914	/* used to show worker information through /proc/PID/{comm,stat,status} */
4915	void wq_worker_comm(char buf, size_t size, struct task_struct task)
4916	{
4917	int off;
4918
4919	/* always show the actual comm */
4920	off = strscpy(buf, task->comm, size);
4921	if (off < 0)
4922	return;
4923
4924	/* stabilize PF_WQ_WORKER and worker pool association */
4925	mutex_lock(&wq_pool_attach_mutex);
4926
4927	if (task->flags & PF_WQ_WORKER) {
4928	struct worker *worker = kthread_data(task);
4929	struct worker_pool *pool = worker->pool;
4930
4931	if (pool) {
4932	raw_spin_lock_irq(&pool->lock);
4933	/*
4934	* ->desc tracks information (wq name or
4935	* set_worker_desc()) for the latest execution. If
4936	* current, prepend '+', otherwise '-'.
4937	*/
4938	if (worker->desc[0] != '\0') {
4939	if (worker->current_work)
4940	scnprintf(buf + off, size - off, "+%s",
4941	worker->desc);
4942	else
4943	scnprintf(buf + off, size - off, "-%s",
4944	worker->desc);
4945	}
4946	raw_spin_unlock_irq(&pool->lock);
4947	}
4948	}
4949
4950	mutex_unlock(&wq_pool_attach_mutex);
4951	}
4952
4953	#ifdef CONFIG_SMP
4954
4955	/*
4956	* CPU hotplug.
4957	*
4958	* There are two challenges in supporting CPU hotplug. Firstly, there
4959	* are a lot of assumptions on strong associations among work, pwq and
4960	* pool which make migrating pending and scheduled works very
4961	* difficult to implement without impacting hot paths. Secondly,
4962	* worker pools serve mix of short, long and very long running works making
4963	* blocked draining impractical.
4964	*
4965	* This is solved by allowing the pools to be disassociated from the CPU
4966	* running as an unbound one and allowing it to be reattached later if the
4967	* cpu comes back online.
4968	*/
4969
4970	static void unbind_workers(int cpu)
4971	{
4972	struct worker_pool *pool;
4973	struct worker *worker;
4974
4975	for_each_cpu_worker_pool(pool, cpu) {
4976	mutex_lock(&wq_pool_attach_mutex);
4977	raw_spin_lock_irq(&pool->lock);
4978
4979	/*
4980	* We've blocked all attach/detach operations. Make all workers
4981	* unbound and set DISASSOCIATED. Before this, all workers
4982	* except for the ones which are still executing works from
4983	* before the last CPU down must be on the cpu. After
4984	* this, they may become diasporas.
4985	*/
4986	for_each_pool_worker(worker, pool)
4987	worker->flags \|= WORKER_UNBOUND;
4988
4989	pool->flags \|= POOL_DISASSOCIATED;
4990
4991	raw_spin_unlock_irq(&pool->lock);
4992
4993	for_each_pool_worker(worker, pool) {
4994	kthread_set_per_cpu(worker->task, -1);
4995	WARN_ON_ONCE(set_cpus_allowed_ptr(worker->task, cpu_possible_mask) < 0);
4996	}
4997
4998	mutex_unlock(&wq_pool_attach_mutex);
4999
5000	/*
5001	* Call schedule() so that we cross rq->lock and thus can
5002	* guarantee sched callbacks see the %WORKER_UNBOUND flag.
5003	* This is necessary as scheduler callbacks may be invoked
5004	* from other cpus.
5005	*/
5006	schedule();
5007
5008	/*
5009	* Sched callbacks are disabled now. Zap nr_running.
5010	* After this, nr_running stays zero and need_more_worker()
5011	* and keep_working() are always true as long as the
5012	* worklist is not empty. This pool now behaves as an
5013	* unbound (in terms of concurrency management) pool which
5014	* are served by workers tied to the pool.
5015	*/
5016	atomic_set(&pool->nr_running, 0);
5017
5018	/*
5019	* With concurrency management just turned off, a busy
5020	* worker blocking could lead to lengthy stalls. Kick off
5021	* unbound chain execution of currently pending work items.
5022	*/
5023	raw_spin_lock_irq(&pool->lock);
5024	wake_up_worker(pool);
5025	raw_spin_unlock_irq(&pool->lock);
5026	}
5027	}
5028
5029	/**
5030	* rebind_workers - rebind all workers of a pool to the associated CPU
5031	* @pool: pool of interest
5032	*
5033	* @pool->cpu is coming online. Rebind all workers to the CPU.
5034	*/
5035	static void rebind_workers(struct worker_pool *pool)
5036	{
5037	struct worker *worker;
5038
5039	lockdep_assert_held(&wq_pool_attach_mutex);
5040
5041	/*
5042	* Restore CPU affinity of all workers. As all idle workers should
5043	* be on the run-queue of the associated CPU before any local
5044	* wake-ups for concurrency management happen, restore CPU affinity
5045	* of all workers first and then clear UNBOUND. As we're called
5046	* from CPU_ONLINE, the following shouldn't fail.
5047	*/
5048	for_each_pool_worker(worker, pool) {
5049	kthread_set_per_cpu(worker->task, pool->cpu);
5050	WARN_ON_ONCE(set_cpus_allowed_ptr(worker->task,
5051	pool->attrs->cpumask) < 0);
5052	}
5053
5054	raw_spin_lock_irq(&pool->lock);
5055
5056	pool->flags &= ~POOL_DISASSOCIATED;
5057
5058	for_each_pool_worker(worker, pool) {
5059	unsigned int worker_flags = worker->flags;
5060
5061	/*
5062	* A bound idle worker should actually be on the runqueue
5063	* of the associated CPU for local wake-ups targeting it to
5064	* work. Kick all idle workers so that they migrate to the
5065	* associated CPU. Doing this in the same loop as
5066	* replacing UNBOUND with REBOUND is safe as no worker will
5067	* be bound before @pool->lock is released.
5068	*/
5069	if (worker_flags & WORKER_IDLE)
5070	wake_up_process(worker->task);
5071
5072	/*
5073	* We want to clear UNBOUND but can't directly call
5074	* worker_clr_flags() or adjust nr_running. Atomically
5075	* replace UNBOUND with another NOT_RUNNING flag REBOUND.
5076	* @worker will clear REBOUND using worker_clr_flags() when
5077	* it initiates the next execution cycle thus restoring
5078	* concurrency management. Note that when or whether
5079	* @worker clears REBOUND doesn't affect correctness.
5080	*
5081	* WRITE_ONCE() is necessary because @worker->flags may be
5082	* tested without holding any lock in
5083	* wq_worker_running(). Without it, NOT_RUNNING test may
5084	* fail incorrectly leading to premature concurrency
5085	* management operations.
5086	*/
5087	WARN_ON_ONCE(!(worker_flags & WORKER_UNBOUND));
5088	worker_flags \|= WORKER_REBOUND;
5089	worker_flags &= ~WORKER_UNBOUND;
5090	WRITE_ONCE(worker->flags, worker_flags);
5091	}
5092
5093	raw_spin_unlock_irq(&pool->lock);
5094	}
5095
5096	/**
5097	* restore_unbound_workers_cpumask - restore cpumask of unbound workers
5098	* @pool: unbound pool of interest
5099	* @cpu: the CPU which is coming up
5100	*
5101	* An unbound pool may end up with a cpumask which doesn't have any online
5102	* CPUs. When a worker of such pool get scheduled, the scheduler resets
5103	* its cpus_allowed. If @cpu is in @pool's cpumask which didn't have any
5104	* online CPU before, cpus_allowed of all its workers should be restored.
5105	*/
5106	static void restore_unbound_workers_cpumask(struct worker_pool *pool, int cpu)
5107	{
5108	static cpumask_t cpumask;
5109	struct worker *worker;
5110
5111	lockdep_assert_held(&wq_pool_attach_mutex);
5112
5113	/* is @cpu allowed for @pool? */
5114	if (!cpumask_test_cpu(cpu, pool->attrs->cpumask))
5115	return;
5116
5117	cpumask_and(&cpumask, pool->attrs->cpumask, cpu_online_mask);
5118
5119	/* as we're called from CPU_ONLINE, the following shouldn't fail */
5120	for_each_pool_worker(worker, pool)
5121	WARN_ON_ONCE(set_cpus_allowed_ptr(worker->task, &cpumask) < 0);
5122	}
5123
5124	int workqueue_prepare_cpu(unsigned int cpu)
5125	{
5126	struct worker_pool *pool;
5127
5128	for_each_cpu_worker_pool(pool, cpu) {
5129	if (pool->nr_workers)
5130	continue;
5131	if (!create_worker(pool))
5132	return -ENOMEM;
5133	}
5134	return 0;
5135	}
5136
5137	int workqueue_online_cpu(unsigned int cpu)
5138	{
5139	struct worker_pool *pool;
5140	struct workqueue_struct *wq;
5141	int pi;
5142
5143	mutex_lock(&wq_pool_mutex);
5144
5145	for_each_pool(pool, pi) {
5146	mutex_lock(&wq_pool_attach_mutex);
5147
5148	if (pool->cpu == cpu)
5149	rebind_workers(pool);
5150	else if (pool->cpu < 0)
5151	restore_unbound_workers_cpumask(pool, cpu);
5152
5153	mutex_unlock(&wq_pool_attach_mutex);
5154	}
5155
5156	/* update NUMA affinity of unbound workqueues */
5157	list_for_each_entry(wq, &workqueues, list)
5158	wq_update_unbound_numa(wq, cpu, true);
5159
5160	mutex_unlock(&wq_pool_mutex);
5161	return 0;
5162	}
5163
5164	int workqueue_offline_cpu(unsigned int cpu)
5165	{
5166	struct workqueue_struct *wq;
5167
5168	/* unbinding per-cpu workers should happen on the local CPU */
5169	if (WARN_ON(cpu != smp_processor_id()))
5170	return -1;
5171
5172	unbind_workers(cpu);
5173
5174	/* update NUMA affinity of unbound workqueues */
5175	mutex_lock(&wq_pool_mutex);
5176	list_for_each_entry(wq, &workqueues, list)
5177	wq_update_unbound_numa(wq, cpu, false);
5178	mutex_unlock(&wq_pool_mutex);
5179
5180	return 0;
5181	}
5182
5183	struct work_for_cpu {
5184	struct work_struct work;
5185	long (fn)(void );
5186	void *arg;
5187	long ret;
5188	};
5189
5190	static void work_for_cpu_fn(struct work_struct *work)
5191	{
5192	struct work_for_cpu *wfc = container_of(work, struct work_for_cpu, work);
5193
5194	wfc->ret = wfc->fn(wfc->arg);
5195	}
5196
5197	/**
5198	* work_on_cpu - run a function in thread context on a particular cpu
5199	* @cpu: the cpu to run on
5200	* @fn: the function to run
5201	* @arg: the function arg
5202	*
5203	* It is up to the caller to ensure that the cpu doesn't go offline.
5204	* The caller must not hold any locks which would prevent @fn from completing.
5205	*
5206	* Return: The value @fn returns.
5207	*/
5208	long work_on_cpu(int cpu, long (fn)(void ), void *arg)
5209	{
5210	struct work_for_cpu wfc = { .fn = fn, .arg = arg };
5211
5212	INIT_WORK_ONSTACK(&wfc.work, work_for_cpu_fn);
5213	schedule_work_on(cpu, &wfc.work);
5214	flush_work(&wfc.work);
5215	destroy_work_on_stack(&wfc.work);
5216	return wfc.ret;
5217	}
5218	EXPORT_SYMBOL_GPL(work_on_cpu);
5219
5220	/**
5221	* work_on_cpu_safe - run a function in thread context on a particular cpu
5222	* @cpu: the cpu to run on
5223	* @fn: the function to run
5224	* @arg: the function argument
5225	*
5226	* Disables CPU hotplug and calls work_on_cpu(). The caller must not hold
5227	* any locks which would prevent @fn from completing.
5228	*
5229	* Return: The value @fn returns.
5230	*/
5231	long work_on_cpu_safe(int cpu, long (fn)(void ), void *arg)
5232	{
5233	long ret = -ENODEV;
5234
5235	cpus_read_lock();
5236	if (cpu_online(cpu))
5237	ret = work_on_cpu(cpu, fn, arg);
5238	cpus_read_unlock();
5239	return ret;
5240	}
5241	EXPORT_SYMBOL_GPL(work_on_cpu_safe);
5242	#endif /* CONFIG_SMP */
5243
5244	#ifdef CONFIG_FREEZER
5245
5246	/**
5247	* freeze_workqueues_begin - begin freezing workqueues
5248	*
5249	* Start freezing workqueues. After this function returns, all freezable
5250	* workqueues will queue new works to their inactive_works list instead of
5251	* pool->worklist.
5252	*
5253	* CONTEXT:
5254	* Grabs and releases wq_pool_mutex, wq->mutex and pool->lock's.
5255	*/
5256	void freeze_workqueues_begin(void)
5257	{
5258	struct workqueue_struct *wq;
5259	struct pool_workqueue *pwq;
5260
5261	mutex_lock(&wq_pool_mutex);
5262
5263	WARN_ON_ONCE(workqueue_freezing);
5264	workqueue_freezing = true;
5265
5266	list_for_each_entry(wq, &workqueues, list) {
5267	mutex_lock(&wq->mutex);
5268	for_each_pwq(pwq, wq)
5269	pwq_adjust_max_active(pwq);
5270	mutex_unlock(&wq->mutex);
5271	}
5272
5273	mutex_unlock(&wq_pool_mutex);
5274	}
5275
5276	/**
5277	* freeze_workqueues_busy - are freezable workqueues still busy?
5278	*
5279	* Check whether freezing is complete. This function must be called
5280	* between freeze_workqueues_begin() and thaw_workqueues().
5281	*
5282	* CONTEXT:
5283	* Grabs and releases wq_pool_mutex.
5284	*
5285	* Return:
5286	* %true if some freezable workqueues are still busy. %false if freezing
5287	* is complete.
5288	*/
5289	bool freeze_workqueues_busy(void)
5290	{
5291	bool busy = false;
5292	struct workqueue_struct *wq;
5293	struct pool_workqueue *pwq;
5294
5295	mutex_lock(&wq_pool_mutex);
5296
5297	WARN_ON_ONCE(!workqueue_freezing);
5298
5299	list_for_each_entry(wq, &workqueues, list) {
5300	if (!(wq->flags & WQ_FREEZABLE))
5301	continue;
5302	/*
5303	* nr_active is monotonically decreasing. It's safe
5304	* to peek without lock.
5305	*/
5306	rcu_read_lock();
5307	for_each_pwq(pwq, wq) {
5308	WARN_ON_ONCE(pwq->nr_active < 0);
5309	if (pwq->nr_active) {
5310	busy = true;
5311	rcu_read_unlock();
5312	goto out_unlock;
5313	}
5314	}
5315	rcu_read_unlock();
5316	}
5317	out_unlock:
5318	mutex_unlock(&wq_pool_mutex);
5319	return busy;
5320	}
5321
5322	/**
5323	* thaw_workqueues - thaw workqueues
5324	*
5325	* Thaw workqueues. Normal queueing is restored and all collected
5326	* frozen works are transferred to their respective pool worklists.
5327	*
5328	* CONTEXT:
5329	* Grabs and releases wq_pool_mutex, wq->mutex and pool->lock's.
5330	*/
5331	void thaw_workqueues(void)
5332	{
5333	struct workqueue_struct *wq;
5334	struct pool_workqueue *pwq;
5335
5336	mutex_lock(&wq_pool_mutex);
5337
5338	if (!workqueue_freezing)
5339	goto out_unlock;
5340
5341	workqueue_freezing = false;
5342
5343	/* restore max_active and repopulate worklist */
5344	list_for_each_entry(wq, &workqueues, list) {
5345	mutex_lock(&wq->mutex);
5346	for_each_pwq(pwq, wq)
5347	pwq_adjust_max_active(pwq);
5348	mutex_unlock(&wq->mutex);
5349	}
5350
5351	out_unlock:
5352	mutex_unlock(&wq_pool_mutex);
5353	}
5354	#endif /* CONFIG_FREEZER */
5355
5356	static int workqueue_apply_unbound_cpumask(void)
5357	{
5358	LIST_HEAD(ctxs);
5359	int ret = 0;
5360	struct workqueue_struct *wq;
5361	struct apply_wqattrs_ctx ctx, n;
5362
5363	lockdep_assert_held(&wq_pool_mutex);
5364
5365	list_for_each_entry(wq, &workqueues, list) {
5366	if (!(wq->flags & WQ_UNBOUND))
5367	continue;
5368	/* creating multiple pwqs breaks ordering guarantee */
5369	if (wq->flags & __WQ_ORDERED)
5370	continue;
5371
5372	ctx = apply_wqattrs_prepare(wq, wq->unbound_attrs);
5373	if (!ctx) {
5374	ret = -ENOMEM;
5375	break;
5376	}
5377
5378	list_add_tail(&ctx->list, &ctxs);
5379	}
5380
5381	list_for_each_entry_safe(ctx, n, &ctxs, list) {
5382	if (!ret)
5383	apply_wqattrs_commit(ctx);
5384	apply_wqattrs_cleanup(ctx);
5385	}
5386
5387	return ret;
5388	}
5389
5390	/**
5391	* workqueue_set_unbound_cpumask - Set the low-level unbound cpumask
5392	* @cpumask: the cpumask to set
5393	*
5394	* The low-level workqueues cpumask is a global cpumask that limits
5395	* the affinity of all unbound workqueues. This function check the @cpumask
5396	* and apply it to all unbound workqueues and updates all pwqs of them.
5397	*
5398	* Return: 0 - Success
5399	* -EINVAL - Invalid @cpumask
5400	* -ENOMEM - Failed to allocate memory for attrs or pwqs.
5401	*/
5402	int workqueue_set_unbound_cpumask(cpumask_var_t cpumask)
5403	{
5404	int ret = -EINVAL;
5405	cpumask_var_t saved_cpumask;
5406
5407	/*
5408	* Not excluding isolated cpus on purpose.
5409	* If the user wishes to include them, we allow that.
5410	*/
5411	cpumask_and(cpumask, cpumask, cpu_possible_mask);
5412	if (!cpumask_empty(cpumask)) {
5413	apply_wqattrs_lock();
5414	if (cpumask_equal(cpumask, wq_unbound_cpumask)) {
5415	ret = 0;
5416	goto out_unlock;
5417	}
5418
5419	if (!zalloc_cpumask_var(&saved_cpumask, GFP_KERNEL)) {
5420	ret = -ENOMEM;
5421	goto out_unlock;
5422	}
5423
5424	/* save the old wq_unbound_cpumask. */
5425	cpumask_copy(saved_cpumask, wq_unbound_cpumask);
5426
5427	/* update wq_unbound_cpumask at first and apply it to wqs. */
5428	cpumask_copy(wq_unbound_cpumask, cpumask);
5429	ret = workqueue_apply_unbound_cpumask();
5430
5431	/* restore the wq_unbound_cpumask when failed. */
5432	if (ret < 0)
5433	cpumask_copy(wq_unbound_cpumask, saved_cpumask);
5434
5435	free_cpumask_var(saved_cpumask);
5436	out_unlock:
5437	apply_wqattrs_unlock();
5438	}
5439
5440	return ret;
5441	}
5442
5443	#ifdef CONFIG_SYSFS
5444	/*
5445	* Workqueues with WQ_SYSFS flag set is visible to userland via
5446	* /sys/bus/workqueue/devices/WQ_NAME. All visible workqueues have the
5447	* following attributes.
5448	*
5449	* per_cpu RO bool : whether the workqueue is per-cpu or unbound
5450	* max_active RW int : maximum number of in-flight work items
5451	*
5452	* Unbound workqueues have the following extra attributes.
5453	*
5454	* pool_ids RO int : the associated pool IDs for each node
5455	* nice RW int : nice value of the workers
5456	* cpumask RW mask : bitmask of allowed CPUs for the workers
5457	* numa RW bool : whether enable NUMA affinity
5458	*/
5459	struct wq_device {
5460	struct workqueue_struct *wq;
5461	struct device dev;
5462	};
5463
5464	static struct workqueue_struct dev_to_wq(struct device dev)
5465	{
5466	struct wq_device *wq_dev = container_of(dev, struct wq_device, dev);
5467
5468	return wq_dev->wq;
5469	}
5470
5471	static ssize_t per_cpu_show(struct device dev, struct device_attribute attr,
5472	char *buf)
5473	{
5474	struct workqueue_struct *wq = dev_to_wq(dev);
5475
5476	return scnprintf(buf, PAGE_SIZE, "%d\n", (bool)!(wq->flags & WQ_UNBOUND));
5477	}
5478	static DEVICE_ATTR_RO(per_cpu);
5479
5480	static ssize_t max_active_show(struct device *dev,
5481	struct device_attribute attr, char buf)
5482	{
5483	struct workqueue_struct *wq = dev_to_wq(dev);
5484
5485	return scnprintf(buf, PAGE_SIZE, "%d\n", wq->saved_max_active);
5486	}
5487
5488	static ssize_t max_active_store(struct device *dev,
5489	struct device_attribute attr, const char buf,
5490	size_t count)
5491	{
5492	struct workqueue_struct *wq = dev_to_wq(dev);
5493	int val;
5494
5495	if (sscanf(buf, "%d", &val) != 1 \|\| val <= 0)
5496	return -EINVAL;
5497
5498	workqueue_set_max_active(wq, val);
5499	return count;
5500	}
5501	static DEVICE_ATTR_RW(max_active);
5502
5503	static struct attribute *wq_sysfs_attrs[] = {
5504	&dev_attr_per_cpu.attr,
5505	&dev_attr_max_active.attr,
5506	NULL,
5507	};
5508	ATTRIBUTE_GROUPS(wq_sysfs);
5509
5510	static ssize_t wq_pool_ids_show(struct device *dev,
5511	struct device_attribute attr, char buf)
5512	{
5513	struct workqueue_struct *wq = dev_to_wq(dev);
5514	const char *delim = "";
5515	int node, written = 0;
5516
5517	cpus_read_lock();
5518	rcu_read_lock();
5519	for_each_node(node) {
5520	written += scnprintf(buf + written, PAGE_SIZE - written,
5521	"%s%d:%d", delim, node,
5522	unbound_pwq_by_node(wq, node)->pool->id);
5523	delim = " ";
5524	}
5525	written += scnprintf(buf + written, PAGE_SIZE - written, "\n");
5526	rcu_read_unlock();
5527	cpus_read_unlock();
5528
5529	return written;
5530	}
5531
5532	static ssize_t wq_nice_show(struct device dev, struct device_attribute attr,
5533	char *buf)
5534	{
5535	struct workqueue_struct *wq = dev_to_wq(dev);
5536	int written;
5537
5538	mutex_lock(&wq->mutex);
5539	written = scnprintf(buf, PAGE_SIZE, "%d\n", wq->unbound_attrs->nice);
5540	mutex_unlock(&wq->mutex);
5541
5542	return written;
5543	}
5544
5545	/* prepare workqueue_attrs for sysfs store operations */
5546	static struct workqueue_attrs wq_sysfs_prep_attrs(struct workqueue_struct wq)
5547	{
5548	struct workqueue_attrs *attrs;
5549
5550	lockdep_assert_held(&wq_pool_mutex);
5551
5552	attrs = alloc_workqueue_attrs();
5553	if (!attrs)
5554	return NULL;
5555
5556	copy_workqueue_attrs(attrs, wq->unbound_attrs);
5557	return attrs;
5558	}
5559
5560	static ssize_t wq_nice_store(struct device dev, struct device_attribute attr,
5561	const char *buf, size_t count)
5562	{
5563	struct workqueue_struct *wq = dev_to_wq(dev);
5564	struct workqueue_attrs *attrs;
5565	int ret = -ENOMEM;
5566
5567	apply_wqattrs_lock();
5568
5569	attrs = wq_sysfs_prep_attrs(wq);
5570	if (!attrs)
5571	goto out_unlock;
5572
5573	if (sscanf(buf, "%d", &attrs->nice) == 1 &&
5574	attrs->nice >= MIN_NICE && attrs->nice <= MAX_NICE)
5575	ret = apply_workqueue_attrs_locked(wq, attrs);
5576	else
5577	ret = -EINVAL;
5578
5579	out_unlock:
5580	apply_wqattrs_unlock();
5581	free_workqueue_attrs(attrs);
5582	return ret ?: count;
5583	}
5584
5585	static ssize_t wq_cpumask_show(struct device *dev,
5586	struct device_attribute attr, char buf)
5587	{
5588	struct workqueue_struct *wq = dev_to_wq(dev);
5589	int written;
5590
5591	mutex_lock(&wq->mutex);
5592	written = scnprintf(buf, PAGE_SIZE, "%*pb\n",
5593	cpumask_pr_args(wq->unbound_attrs->cpumask));
5594	mutex_unlock(&wq->mutex);
5595	return written;
5596	}
5597
5598	static ssize_t wq_cpumask_store(struct device *dev,
5599	struct device_attribute *attr,
5600	const char *buf, size_t count)
5601	{
5602	struct workqueue_struct *wq = dev_to_wq(dev);
5603	struct workqueue_attrs *attrs;
5604	int ret = -ENOMEM;
5605
5606	apply_wqattrs_lock();
5607
5608	attrs = wq_sysfs_prep_attrs(wq);
5609	if (!attrs)
5610	goto out_unlock;
5611
5612	ret = cpumask_parse(buf, attrs->cpumask);
5613	if (!ret)
5614	ret = apply_workqueue_attrs_locked(wq, attrs);
5615
5616	out_unlock:
5617	apply_wqattrs_unlock();
5618	free_workqueue_attrs(attrs);
5619	return ret ?: count;
5620	}
5621
5622	static ssize_t wq_numa_show(struct device dev, struct device_attribute attr,
5623	char *buf)
5624	{
5625	struct workqueue_struct *wq = dev_to_wq(dev);
5626	int written;
5627
5628	mutex_lock(&wq->mutex);
5629	written = scnprintf(buf, PAGE_SIZE, "%d\n",
5630	!wq->unbound_attrs->no_numa);
5631	mutex_unlock(&wq->mutex);
5632
5633	return written;
5634	}
5635
5636	static ssize_t wq_numa_store(struct device dev, struct device_attribute attr,
5637	const char *buf, size_t count)
5638	{
5639	struct workqueue_struct *wq = dev_to_wq(dev);
5640	struct workqueue_attrs *attrs;
5641	int v, ret = -ENOMEM;
5642
5643	apply_wqattrs_lock();
5644
5645	attrs = wq_sysfs_prep_attrs(wq);
5646	if (!attrs)
5647	goto out_unlock;
5648
5649	ret = -EINVAL;
5650	if (sscanf(buf, "%d", &v) == 1) {
5651	attrs->no_numa = !v;
5652	ret = apply_workqueue_attrs_locked(wq, attrs);
5653	}
5654
5655	out_unlock:
5656	apply_wqattrs_unlock();
5657	free_workqueue_attrs(attrs);
5658	return ret ?: count;
5659	}
5660
5661	static struct device_attribute wq_sysfs_unbound_attrs[] = {
5662	__ATTR(pool_ids, 0444, wq_pool_ids_show, NULL),
5663	__ATTR(nice, 0644, wq_nice_show, wq_nice_store),
5664	__ATTR(cpumask, 0644, wq_cpumask_show, wq_cpumask_store),
5665	__ATTR(numa, 0644, wq_numa_show, wq_numa_store),
5666	__ATTR_NULL,
5667	};
5668
5669	static struct bus_type wq_subsys = {
5670	.name = "workqueue",
5671	.dev_groups = wq_sysfs_groups,
5672	};
5673
5674	static ssize_t wq_unbound_cpumask_show(struct device *dev,
5675	struct device_attribute attr, char buf)
5676	{
5677	int written;
5678
5679	mutex_lock(&wq_pool_mutex);
5680	written = scnprintf(buf, PAGE_SIZE, "%*pb\n",
5681	cpumask_pr_args(wq_unbound_cpumask));
5682	mutex_unlock(&wq_pool_mutex);
5683
5684	return written;
5685	}
5686
5687	static ssize_t wq_unbound_cpumask_store(struct device *dev,
5688	struct device_attribute attr, const char buf, size_t count)
5689	{
5690	cpumask_var_t cpumask;
5691	int ret;
5692
5693	if (!zalloc_cpumask_var(&cpumask, GFP_KERNEL))
5694	return -ENOMEM;
5695
5696	ret = cpumask_parse(buf, cpumask);
5697	if (!ret)
5698	ret = workqueue_set_unbound_cpumask(cpumask);
5699
5700	free_cpumask_var(cpumask);
5701	return ret ? ret : count;
5702	}
5703
5704	static struct device_attribute wq_sysfs_cpumask_attr =
5705	__ATTR(cpumask, 0644, wq_unbound_cpumask_show,
5706	wq_unbound_cpumask_store);
5707
5708	static int __init wq_sysfs_init(void)
5709	{
5710	int err;
5711
5712	err = subsys_virtual_register(&wq_subsys, NULL);
5713	if (err)
5714	return err;
5715
5716	return device_create_file(wq_subsys.dev_root, &wq_sysfs_cpumask_attr);
5717	}
5718	core_initcall(wq_sysfs_init);
5719
5720	static void wq_device_release(struct device *dev)
5721	{
5722	struct wq_device *wq_dev = container_of(dev, struct wq_device, dev);
5723
5724	kfree(wq_dev);
5725	}
5726
5727	/**
5728	* workqueue_sysfs_register - make a workqueue visible in sysfs
5729	* @wq: the workqueue to register
5730	*
5731	* Expose @wq in sysfs under /sys/bus/workqueue/devices.
5732	* alloc_workqueue*() automatically calls this function if WQ_SYSFS is set
5733	* which is the preferred method.
5734	*
5735	* Workqueue user should use this function directly iff it wants to apply
5736	* workqueue_attrs before making the workqueue visible in sysfs; otherwise,
5737	* apply_workqueue_attrs() may race against userland updating the
5738	* attributes.
5739	*
5740	* Return: 0 on success, -errno on failure.
5741	*/
5742	int workqueue_sysfs_register(struct workqueue_struct *wq)
5743	{
5744	struct wq_device *wq_dev;
5745	int ret;
5746
5747	/*
5748	* Adjusting max_active or creating new pwqs by applying
5749	* attributes breaks ordering guarantee. Disallow exposing ordered
5750	* workqueues.
5751	*/
5752	if (WARN_ON(wq->flags & __WQ_ORDERED_EXPLICIT))
5753	return -EINVAL;
5754
5755	wq->wq_dev = wq_dev = kzalloc(sizeof(*wq_dev), GFP_KERNEL);
5756	if (!wq_dev)
5757	return -ENOMEM;
5758
5759	wq_dev->wq = wq;
5760	wq_dev->dev.bus = &wq_subsys;
5761	wq_dev->dev.release = wq_device_release;
5762	dev_set_name(&wq_dev->dev, "%s", wq->name);
5763
5764	/*
5765	* unbound_attrs are created separately. Suppress uevent until
5766	* everything is ready.
5767	*/
5768	dev_set_uevent_suppress(&wq_dev->dev, true);
5769
5770	ret = device_register(&wq_dev->dev);
5771	if (ret) {
5772	put_device(&wq_dev->dev);
5773	wq->wq_dev = NULL;
5774	return ret;
5775	}
5776
5777	if (wq->flags & WQ_UNBOUND) {
5778	struct device_attribute *attr;
5779
5780	for (attr = wq_sysfs_unbound_attrs; attr->attr.name; attr++) {
5781	ret = device_create_file(&wq_dev->dev, attr);
5782	if (ret) {
5783	device_unregister(&wq_dev->dev);
5784	wq->wq_dev = NULL;
5785	return ret;
5786	}
5787	}
5788	}
5789
5790	dev_set_uevent_suppress(&wq_dev->dev, false);
5791	kobject_uevent(&wq_dev->dev.kobj, KOBJ_ADD);
5792	return 0;
5793	}
5794
5795	/**
5796	* workqueue_sysfs_unregister - undo workqueue_sysfs_register()
5797	* @wq: the workqueue to unregister
5798	*
5799	* If @wq is registered to sysfs by workqueue_sysfs_register(), unregister.
5800	*/
5801	static void workqueue_sysfs_unregister(struct workqueue_struct *wq)
5802	{
5803	struct wq_device *wq_dev = wq->wq_dev;
5804
5805	if (!wq->wq_dev)
5806	return;
5807
5808	wq->wq_dev = NULL;
5809	device_unregister(&wq_dev->dev);
5810	}
5811	#else /* CONFIG_SYSFS */
5812	static void workqueue_sysfs_unregister(struct workqueue_struct *wq) { }
5813	#endif /* CONFIG_SYSFS */
5814
5815	/*
5816	* Workqueue watchdog.
5817	*
5818	* Stall may be caused by various bugs - missing WQ_MEM_RECLAIM, illegal
5819	* flush dependency, a concurrency managed work item which stays RUNNING
5820	* indefinitely. Workqueue stalls can be very difficult to debug as the
5821	* usual warning mechanisms don't trigger and internal workqueue state is
5822	* largely opaque.
5823	*
5824	* Workqueue watchdog monitors all worker pools periodically and dumps
5825	* state if some pools failed to make forward progress for a while where
5826	* forward progress is defined as the first item on ->worklist changing.
5827	*
5828	* This mechanism is controlled through the kernel parameter
5829	* "workqueue.watchdog_thresh" which can be updated at runtime through the
5830	* corresponding sysfs parameter file.
5831	*/
5832	#ifdef CONFIG_WQ_WATCHDOG
5833
5834	static unsigned long wq_watchdog_thresh = 30;
5835	static struct timer_list wq_watchdog_timer;
5836
5837	static unsigned long wq_watchdog_touched = INITIAL_JIFFIES;
5838	static DEFINE_PER_CPU(unsigned long, wq_watchdog_touched_cpu) = INITIAL_JIFFIES;
5839
5840	static void wq_watchdog_reset_touched(void)
5841	{
5842	int cpu;
5843
5844	wq_watchdog_touched = jiffies;
5845	for_each_possible_cpu(cpu)
5846	per_cpu(wq_watchdog_touched_cpu, cpu) = jiffies;
5847	}
5848
5849	static void wq_watchdog_timer_fn(struct timer_list *unused)
5850	{
5851	unsigned long thresh = READ_ONCE(wq_watchdog_thresh) * HZ;
5852	bool lockup_detected = false;
5853	unsigned long now = jiffies;
5854	struct worker_pool *pool;
5855	int pi;
5856
5857	if (!thresh)
5858	return;
5859
5860	rcu_read_lock();
5861
5862	for_each_pool(pool, pi) {
5863	unsigned long pool_ts, touched, ts;
5864
5865	if (list_empty(&pool->worklist))
5866	continue;
5867
5868	/*
5869	* If a virtual machine is stopped by the host it can look to
5870	* the watchdog like a stall.
5871	*/
5872	kvm_check_and_clear_guest_paused();
5873
5874	/* get the latest of pool and touched timestamps */
5875	if (pool->cpu >= 0)
5876	touched = READ_ONCE(per_cpu(wq_watchdog_touched_cpu, pool->cpu));
5877	else
5878	touched = READ_ONCE(wq_watchdog_touched);
5879	pool_ts = READ_ONCE(pool->watchdog_ts);
5880
5881	if (time_after(pool_ts, touched))
5882	ts = pool_ts;
5883	else
5884	ts = touched;
5885
5886	/* did we stall? */
5887	if (time_after(now, ts + thresh)) {
5888	lockup_detected = true;
5889	pr_emerg("BUG: workqueue lockup - pool");
5890	pr_cont_pool_info(pool);
5891	pr_cont(" stuck for %us!\n",
5892	jiffies_to_msecs(now - pool_ts) / 1000);
5893	}
5894	}
5895
5896	rcu_read_unlock();
5897
5898	if (lockup_detected)
5899	show_all_workqueues();
5900
5901	wq_watchdog_reset_touched();
5902	mod_timer(&wq_watchdog_timer, jiffies + thresh);
5903	}
5904
5905	notrace void wq_watchdog_touch(int cpu)
5906	{
5907	if (cpu >= 0)
5908	per_cpu(wq_watchdog_touched_cpu, cpu) = jiffies;
5909
5910	wq_watchdog_touched = jiffies;
5911	}
5912
5913	static void wq_watchdog_set_thresh(unsigned long thresh)
5914	{
5915	wq_watchdog_thresh = 0;
5916	del_timer_sync(&wq_watchdog_timer);
5917
5918	if (thresh) {
5919	wq_watchdog_thresh = thresh;
5920	wq_watchdog_reset_touched();
5921	mod_timer(&wq_watchdog_timer, jiffies + thresh * HZ);
5922	}
5923	}
5924
5925	static int wq_watchdog_param_set_thresh(const char *val,
5926	const struct kernel_param *kp)
5927	{
5928	unsigned long thresh;
5929	int ret;
5930
5931	ret = kstrtoul(val, 0, &thresh);
5932	if (ret)
5933	return ret;
5934
5935	if (system_wq)
5936	wq_watchdog_set_thresh(thresh);
5937	else
5938	wq_watchdog_thresh = thresh;
5939
5940	return 0;
5941	}
5942
5943	static const struct kernel_param_ops wq_watchdog_thresh_ops = {
5944	.set = wq_watchdog_param_set_thresh,
5945	.get = param_get_ulong,
5946	};
5947
5948	module_param_cb(watchdog_thresh, &wq_watchdog_thresh_ops, &wq_watchdog_thresh,
5949	0644);
5950
5951	static void wq_watchdog_init(void)
5952	{
5953	timer_setup(&wq_watchdog_timer, wq_watchdog_timer_fn, TIMER_DEFERRABLE);
5954	wq_watchdog_set_thresh(wq_watchdog_thresh);
5955	}
5956
5957	#else /* CONFIG_WQ_WATCHDOG */
5958
5959	static inline void wq_watchdog_init(void) { }
5960
5961	#endif /* CONFIG_WQ_WATCHDOG */
5962
5963	static void __init wq_numa_init(void)
5964	{
5965	cpumask_var_t *tbl;
5966	int node, cpu;
5967
5968	if (num_possible_nodes() <= 1)
5969	return;
5970
5971	if (wq_disable_numa) {
5972	pr_info("workqueue: NUMA affinity support disabled\n");
5973	return;
5974	}
5975
5976	for_each_possible_cpu(cpu) {
5977	if (WARN_ON(cpu_to_node(cpu) == NUMA_NO_NODE)) {
5978	pr_warn("workqueue: NUMA node mapping not available for cpu%d, disabling NUMA support\n", cpu);
5979	return;
5980	}
5981	}
5982
5983	wq_update_unbound_numa_attrs_buf = alloc_workqueue_attrs();
5984	BUG_ON(!wq_update_unbound_numa_attrs_buf);
5985
5986	/*
5987	* We want masks of possible CPUs of each node which isn't readily
5988	* available. Build one from cpu_to_node() which should have been
5989	* fully initialized by now.
5990	*/
5991	tbl = kcalloc(nr_node_ids, sizeof(tbl[0]), GFP_KERNEL);
5992	BUG_ON(!tbl);
5993
5994	for_each_node(node)
5995	BUG_ON(!zalloc_cpumask_var_node(&tbl[node], GFP_KERNEL,
5996	node_online(node) ? node : NUMA_NO_NODE));
5997
5998	for_each_possible_cpu(cpu) {
5999	node = cpu_to_node(cpu);
6000	cpumask_set_cpu(cpu, tbl[node]);
6001	}
6002
6003	wq_numa_possible_cpumask = tbl;
6004	wq_numa_enabled = true;
6005	}
6006
6007	/**
6008	* workqueue_init_early - early init for workqueue subsystem
6009	*
6010	* This is the first half of two-staged workqueue subsystem initialization
6011	* and invoked as soon as the bare basics - memory allocation, cpumasks and
6012	* idr are up. It sets up all the data structures and system workqueues
6013	* and allows early boot code to create workqueues and queue/cancel work
6014	* items. Actual work item execution starts only after kthreads can be
6015	* created and scheduled right before early initcalls.
6016	*/
6017	void __init workqueue_init_early(void)
6018	{
6019	int std_nice[NR_STD_WORKER_POOLS] = { 0, HIGHPRI_NICE_LEVEL };
6020	int hk_flags = HK_FLAG_DOMAIN \| HK_FLAG_WQ;
6021	int i, cpu;
6022
6023	BUILD_BUG_ON(__alignof__(struct pool_workqueue) < __alignof__(long long));
6024
6025	BUG_ON(!alloc_cpumask_var(&wq_unbound_cpumask, GFP_KERNEL));
6026	cpumask_copy(wq_unbound_cpumask, housekeeping_cpumask(hk_flags));
6027
6028	pwq_cache = KMEM_CACHE(pool_workqueue, SLAB_PANIC);
6029
6030	/* initialize CPU pools */
6031	for_each_possible_cpu(cpu) {
6032	struct worker_pool *pool;
6033
6034	i = 0;
6035	for_each_cpu_worker_pool(pool, cpu) {
6036	BUG_ON(init_worker_pool(pool));
6037	pool->cpu = cpu;
6038	cpumask_copy(pool->attrs->cpumask, cpumask_of(cpu));
6039	pool->attrs->nice = std_nice[i++];
6040	pool->node = cpu_to_node(cpu);
6041
6042	/* alloc pool ID */
6043	mutex_lock(&wq_pool_mutex);
6044	BUG_ON(worker_pool_assign_id(pool));
6045	mutex_unlock(&wq_pool_mutex);
6046	}
6047	}
6048
6049	/* create default unbound and ordered wq attrs */
6050	for (i = 0; i < NR_STD_WORKER_POOLS; i++) {
6051	struct workqueue_attrs *attrs;
6052
6053	BUG_ON(!(attrs = alloc_workqueue_attrs()));
6054	attrs->nice = std_nice[i];
6055	unbound_std_wq_attrs[i] = attrs;
6056
6057	/*
6058	* An ordered wq should have only one pwq as ordering is
6059	* guaranteed by max_active which is enforced by pwqs.
6060	* Turn off NUMA so that dfl_pwq is used for all nodes.
6061	*/
6062	BUG_ON(!(attrs = alloc_workqueue_attrs()));
6063	attrs->nice = std_nice[i];
6064	attrs->no_numa = true;
6065	ordered_wq_attrs[i] = attrs;
6066	}
6067
6068	system_wq = alloc_workqueue("events", 0, 0);
6069	system_highpri_wq = alloc_workqueue("events_highpri", WQ_HIGHPRI, 0);
6070	system_long_wq = alloc_workqueue("events_long", 0, 0);
6071	system_unbound_wq = alloc_workqueue("events_unbound", WQ_UNBOUND,
6072	WQ_UNBOUND_MAX_ACTIVE);
6073	system_freezable_wq = alloc_workqueue("events_freezable",
6074	WQ_FREEZABLE, 0);
6075	system_power_efficient_wq = alloc_workqueue("events_power_efficient",
6076	WQ_POWER_EFFICIENT, 0);
6077	system_freezable_power_efficient_wq = alloc_workqueue("events_freezable_power_efficient",
6078	WQ_FREEZABLE \| WQ_POWER_EFFICIENT,
6079	0);
6080	BUG_ON(!system_wq \|\| !system_highpri_wq \|\| !system_long_wq \|\|
6081	!system_unbound_wq \|\| !system_freezable_wq \|\|
6082	!system_power_efficient_wq \|\|
6083	!system_freezable_power_efficient_wq);
6084	}
6085
6086	/**
6087	* workqueue_init - bring workqueue subsystem fully online
6088	*
6089	* This is the latter half of two-staged workqueue subsystem initialization
6090	* and invoked as soon as kthreads can be created and scheduled.
6091	* Workqueues have been created and work items queued on them, but there
6092	* are no kworkers executing the work items yet. Populate the worker pools
6093	* with the initial workers and enable future kworker creations.
6094	*/
6095	void __init workqueue_init(void)
6096	{
6097	struct workqueue_struct *wq;
6098	struct worker_pool *pool;
6099	int cpu, bkt;
6100
6101	/*
6102	* It'd be simpler to initialize NUMA in workqueue_init_early() but
6103	* CPU to node mapping may not be available that early on some
6104	* archs such as power and arm64. As per-cpu pools created
6105	* previously could be missing node hint and unbound pools NUMA
6106	* affinity, fix them up.
6107	*
6108	* Also, while iterating workqueues, create rescuers if requested.
6109	*/
6110	wq_numa_init();
6111
6112	mutex_lock(&wq_pool_mutex);
6113
6114	for_each_possible_cpu(cpu) {
6115	for_each_cpu_worker_pool(pool, cpu) {
6116	pool->node = cpu_to_node(cpu);
6117	}
6118	}
6119
6120	list_for_each_entry(wq, &workqueues, list) {
6121	wq_update_unbound_numa(wq, smp_processor_id(), true);
6122	WARN(init_rescuer(wq),
6123	"workqueue: failed to create early rescuer for %s",
6124	wq->name);
6125	}
6126
6127	mutex_unlock(&wq_pool_mutex);
6128
6129	/* create the initial workers */
6130	for_each_online_cpu(cpu) {
6131	for_each_cpu_worker_pool(pool, cpu) {
6132	pool->flags &= ~POOL_DISASSOCIATED;
6133	BUG_ON(!create_worker(pool));
6134	}
6135	}
6136
6137	hash_for_each(unbound_pool_hash, bkt, pool, hash_node)
6138	BUG_ON(!create_worker(pool));
6139
6140	wq_online = true;
6141	wq_watchdog_init();
6142	}