linux - kernel/workqueue.c

6383 lines · 175.7 KiB

1	// SPDX-License-Identifier: GPL-2.0-only
2	/*
3	* kernel/workqueue.c - generic async execution with shared worker pool
4	*
5	* Copyright (C) 2002 Ingo Molnar
6	*
7	* Derived from the taskqueue/keventd code by:
8	* David Woodhouse <[email protected]>
9	* Andrew Morton
10	* Kai Petzke <[email protected]>
11	* Theodore Ts'o <[email protected]>
12	*
13	* Made to use alloc_percpu by Christoph Lameter.
14	*
15	* Copyright (C) 2010 SUSE Linux Products GmbH
16	* Copyright (C) 2010 Tejun Heo <[email protected]>
17	*
18	* This is the generic async execution mechanism. Work items as are
19	* executed in process context. The worker pool is shared and
20	* automatically managed. There are two worker pools for each CPU (one for
21	* normal work items and the other for high priority ones) and some extra
22	* pools for workqueues which are not bound to any specific CPU - the
23	* number of these backing pools is dynamic.
24	*
25	* Please read Documentation/core-api/workqueue.rst for details.
26	*/
27
28	#include <linux/export.h>
29	#include <linux/kernel.h>
30	#include <linux/sched.h>
31	#include <linux/init.h>
32	#include <linux/signal.h>
33	#include <linux/completion.h>
34	#include <linux/workqueue.h>
35	#include <linux/slab.h>
36	#include <linux/cpu.h>
37	#include <linux/notifier.h>
38	#include <linux/kthread.h>
39	#include <linux/hardirq.h>
40	#include <linux/mempolicy.h>
41	#include <linux/freezer.h>
42	#include <linux/debug_locks.h>
43	#include <linux/lockdep.h>
44	#include <linux/idr.h>
45	#include <linux/jhash.h>
46	#include <linux/hashtable.h>
47	#include <linux/rculist.h>
48	#include <linux/nodemask.h>
49	#include <linux/moduleparam.h>
50	#include <linux/uaccess.h>
51	#include <linux/sched/isolation.h>
52	#include <linux/sched/debug.h>
53	#include <linux/nmi.h>
54	#include <linux/kvm_para.h>
55
56	#include "workqueue_internal.h"
57
58	enum {
59	/*
60	* worker_pool flags
61	*
62	* A bound pool is either associated or disassociated with its CPU.
63	* While associated (!DISASSOCIATED), all workers are bound to the
64	* CPU and none has %WORKER_UNBOUND set and concurrency management
65	* is in effect.
66	*
67	* While DISASSOCIATED, the cpu may be offline and all workers have
68	* %WORKER_UNBOUND set and concurrency management disabled, and may
69	* be executing on any CPU. The pool behaves as an unbound one.
70	*
71	* Note that DISASSOCIATED should be flipped only while holding
72	* wq_pool_attach_mutex to avoid changing binding state while
73	* worker_attach_to_pool() is in progress.
74	*/
75	POOL_MANAGER_ACTIVE = 1 << 0, /* being managed */
76	POOL_DISASSOCIATED = 1 << 2, /* cpu can't serve workers */
77
78	/* worker flags */
79	WORKER_DIE = 1 << 1, /* die die die */
80	WORKER_IDLE = 1 << 2, /* is idle */
81	WORKER_PREP = 1 << 3, /* preparing to run works */
82	WORKER_CPU_INTENSIVE = 1 << 6, /* cpu intensive */
83	WORKER_UNBOUND = 1 << 7, /* worker is unbound */
84	WORKER_REBOUND = 1 << 8, /* worker was rebound */
85
86	WORKER_NOT_RUNNING = WORKER_PREP \| WORKER_CPU_INTENSIVE \|
87	WORKER_UNBOUND \| WORKER_REBOUND,
88
89	NR_STD_WORKER_POOLS = 2, /* # standard pools per cpu */
90
91	UNBOUND_POOL_HASH_ORDER = 6, /* hashed by pool->attrs */
92	BUSY_WORKER_HASH_ORDER = 6, /* 64 pointers */
93
94	MAX_IDLE_WORKERS_RATIO = 4, /* 1/4 of busy can be idle */
95	IDLE_WORKER_TIMEOUT = 300 * HZ, /* keep idle ones for 5 mins */
96
97	MAYDAY_INITIAL_TIMEOUT = HZ / 100 >= 2 ? HZ / 100 : 2,
98	/* call for help after 10ms
99	(min two ticks) */
100	MAYDAY_INTERVAL = HZ / 10, /* and then every 100ms */
101	CREATE_COOLDOWN = HZ, /* time to breath after fail */
102
103	/*
104	* Rescue workers are used only on emergencies and shared by
105	* all cpus. Give MIN_NICE.
106	*/
107	RESCUER_NICE_LEVEL = MIN_NICE,
108	HIGHPRI_NICE_LEVEL = MIN_NICE,
109
110	WQ_NAME_LEN = 24,
111	};
112
113	/*
114	* Structure fields follow one of the following exclusion rules.
115	*
116	* I: Modifiable by initialization/destruction paths and read-only for
117	* everyone else.
118	*
119	* P: Preemption protected. Disabling preemption is enough and should
120	* only be modified and accessed from the local cpu.
121	*
122	* L: pool->lock protected. Access with pool->lock held.
123	*
124	* X: During normal operation, modification requires pool->lock and should
125	* be done only from local cpu. Either disabling preemption on local
126	* cpu or grabbing pool->lock is enough for read access. If
127	* POOL_DISASSOCIATED is set, it's identical to L.
128	*
129	* A: wq_pool_attach_mutex protected.
130	*
131	* PL: wq_pool_mutex protected.
132	*
133	* PR: wq_pool_mutex protected for writes. RCU protected for reads.
134	*
135	* PW: wq_pool_mutex and wq->mutex protected for writes. Either for reads.
136	*
137	* PWR: wq_pool_mutex and wq->mutex protected for writes. Either or
138	* RCU for reads.
139	*
140	* WQ: wq->mutex protected.
141	*
142	* WR: wq->mutex protected for writes. RCU protected for reads.
143	*
144	* MD: wq_mayday_lock protected.
145	*
146	* WD: Used internally by the watchdog.
147	*/
148
149	/* struct worker is defined in workqueue_internal.h */
150
151	struct worker_pool {
152	raw_spinlock_t lock; /* the pool lock */
153	int cpu; /* I: the associated cpu */
154	int node; /* I: the associated node ID */
155	int id; /* I: pool ID */
156	unsigned int flags; /* X: flags */
157
158	unsigned long watchdog_ts; /* L: watchdog timestamp */
159	bool cpu_stall; /* WD: stalled cpu bound pool */
160
161	/*
162	* The counter is incremented in a process context on the associated CPU
163	* w/ preemption disabled, and decremented or reset in the same context
164	* but w/ pool->lock held. The readers grab pool->lock and are
165	* guaranteed to see if the counter reached zero.
166	*/
167	int nr_running;
168
169	struct list_head worklist; /* L: list of pending works */
170
171	int nr_workers; /* L: total number of workers */
172	int nr_idle; /* L: currently idle workers */
173
174	struct list_head idle_list; /* L: list of idle workers */
175	struct timer_list idle_timer; /* L: worker idle timeout */
176	struct work_struct idle_cull_work; /* L: worker idle cleanup */
177
178	struct timer_list mayday_timer; /* L: SOS timer for workers */
179
180	/* a workers is either on busy_hash or idle_list, or the manager */
181	DECLARE_HASHTABLE(busy_hash, BUSY_WORKER_HASH_ORDER);
182	/* L: hash of busy workers */
183
184	struct worker manager; / L: purely informational */
185	struct list_head workers; /* A: attached workers */
186	struct list_head dying_workers; /* A: workers about to die */
187	struct completion detach_completion; / all workers detached */
188
189	struct ida worker_ida; /* worker IDs for task name */
190
191	struct workqueue_attrs attrs; / I: worker attributes */
192	struct hlist_node hash_node; /* PL: unbound_pool_hash node */
193	int refcnt; /* PL: refcnt for unbound pools */
194
195	/*
196	* Destruction of pool is RCU protected to allow dereferences
197	* from get_work_pool().
198	*/
199	struct rcu_head rcu;
200	};
201
202	/*
203	* The per-pool workqueue. While queued, the lower WORK_STRUCT_FLAG_BITS
204	* of work_struct->data are used for flags and the remaining high bits
205	* point to the pwq; thus, pwqs need to be aligned at two's power of the
206	* number of flag bits.
207	*/
208	struct pool_workqueue {
209	struct worker_pool pool; / I: the associated pool */
210	struct workqueue_struct wq; / I: the owning workqueue */
211	int work_color; /* L: current color */
212	int flush_color; /* L: flushing color */
213	int refcnt; /* L: reference count */
214	int nr_in_flight[WORK_NR_COLORS];
215	/* L: nr of in_flight works */
216
217	/*
218	* nr_active management and WORK_STRUCT_INACTIVE:
219	*
220	* When pwq->nr_active >= max_active, new work item is queued to
221	* pwq->inactive_works instead of pool->worklist and marked with
222	* WORK_STRUCT_INACTIVE.
223	*
224	* All work items marked with WORK_STRUCT_INACTIVE do not participate
225	* in pwq->nr_active and all work items in pwq->inactive_works are
226	* marked with WORK_STRUCT_INACTIVE. But not all WORK_STRUCT_INACTIVE
227	* work items are in pwq->inactive_works. Some of them are ready to
228	* run in pool->worklist or worker->scheduled. Those work itmes are
229	* only struct wq_barrier which is used for flush_work() and should
230	* not participate in pwq->nr_active. For non-barrier work item, it
231	* is marked with WORK_STRUCT_INACTIVE iff it is in pwq->inactive_works.
232	*/
233	int nr_active; /* L: nr of active works */
234	int max_active; /* L: max active works */
235	struct list_head inactive_works; /* L: inactive works */
236	struct list_head pwqs_node; /* WR: node on wq->pwqs */
237	struct list_head mayday_node; /* MD: node on wq->maydays */
238
239	/*
240	* Release of unbound pwq is punted to system_wq. See put_pwq()
241	* and pwq_unbound_release_workfn() for details. pool_workqueue
242	* itself is also RCU protected so that the first pwq can be
243	* determined without grabbing wq->mutex.
244	*/
245	struct work_struct unbound_release_work;
246	struct rcu_head rcu;
247	} __aligned(1 << WORK_STRUCT_FLAG_BITS);
248
249	/*
250	* Structure used to wait for workqueue flush.
251	*/
252	struct wq_flusher {
253	struct list_head list; /* WQ: list of flushers */
254	int flush_color; /* WQ: flush color waiting for */
255	struct completion done; /* flush completion */
256	};
257
258	struct wq_device;
259
260	/*
261	* The externally visible workqueue. It relays the issued work items to
262	* the appropriate worker_pool through its pool_workqueues.
263	*/
264	struct workqueue_struct {
265	struct list_head pwqs; /* WR: all pwqs of this wq */
266	struct list_head list; /* PR: list of all workqueues */
267
268	struct mutex mutex; /* protects this wq */
269	int work_color; /* WQ: current work color */
270	int flush_color; /* WQ: current flush color */
271	atomic_t nr_pwqs_to_flush; /* flush in progress */
272	struct wq_flusher first_flusher; / WQ: first flusher */
273	struct list_head flusher_queue; /* WQ: flush waiters */
274	struct list_head flusher_overflow; /* WQ: flush overflow list */
275
276	struct list_head maydays; /* MD: pwqs requesting rescue */
277	struct worker rescuer; / MD: rescue worker */
278
279	int nr_drainers; /* WQ: drain in progress */
280	int saved_max_active; /* WQ: saved pwq max_active */
281
282	struct workqueue_attrs unbound_attrs; / PW: only for unbound wqs */
283	struct pool_workqueue dfl_pwq; / PW: only for unbound wqs */
284
285	#ifdef CONFIG_SYSFS
286	struct wq_device wq_dev; / I: for sysfs interface */
287	#endif
288	#ifdef CONFIG_LOCKDEP
289	char *lock_name;
290	struct lock_class_key key;
291	struct lockdep_map lockdep_map;
292	#endif
293	char name[WQ_NAME_LEN]; /* I: workqueue name */
294
295	/*
296	* Destruction of workqueue_struct is RCU protected to allow walking
297	* the workqueues list without grabbing wq_pool_mutex.
298	* This is used to dump all workqueues from sysrq.
299	*/
300	struct rcu_head rcu;
301
302	/* hot fields used during command issue, aligned to cacheline */
303	unsigned int flags ____cacheline_aligned; /* WQ: WQ_* flags */
304	struct pool_workqueue __percpu cpu_pwqs; / I: per-cpu pwqs */
305	struct pool_workqueue __rcu numa_pwq_tbl[]; / PWR: unbound pwqs indexed by node */
306	};
307
308	static struct kmem_cache *pwq_cache;
309
310	static cpumask_var_t *wq_numa_possible_cpumask;
311	/* possible CPUs of each node */
312
313	static bool wq_disable_numa;
314	module_param_named(disable_numa, wq_disable_numa, bool, 0444);
315
316	/* see the comment above the definition of WQ_POWER_EFFICIENT */
317	static bool wq_power_efficient = IS_ENABLED(CONFIG_WQ_POWER_EFFICIENT_DEFAULT);
318	module_param_named(power_efficient, wq_power_efficient, bool, 0444);
319
320	static bool wq_online; /* can kworkers be created yet? */
321
322	static bool wq_numa_enabled; /* unbound NUMA affinity enabled */
323
324	/* buf for wq_update_unbound_numa_attrs(), protected by CPU hotplug exclusion */
325	static struct workqueue_attrs *wq_update_unbound_numa_attrs_buf;
326
327	static DEFINE_MUTEX(wq_pool_mutex); /* protects pools and workqueues list */
328	static DEFINE_MUTEX(wq_pool_attach_mutex); /* protects worker attach/detach */
329	static DEFINE_RAW_SPINLOCK(wq_mayday_lock); /* protects wq->maydays list */
330	/* wait for manager to go away */
331	static struct rcuwait manager_wait = __RCUWAIT_INITIALIZER(manager_wait);
332
333	static LIST_HEAD(workqueues); /* PR: list of all workqueues */
334	static bool workqueue_freezing; /* PL: have wqs started freezing? */
335
336	/* PL&A: allowable cpus for unbound wqs and work items */
337	static cpumask_var_t wq_unbound_cpumask;
338
339	/* CPU where unbound work was last round robin scheduled from this CPU */
340	static DEFINE_PER_CPU(int, wq_rr_cpu_last);
341
342	/*
343	* Local execution of unbound work items is no longer guaranteed. The
344	* following always forces round-robin CPU selection on unbound work items
345	* to uncover usages which depend on it.
346	*/
347	#ifdef CONFIG_DEBUG_WQ_FORCE_RR_CPU
348	static bool wq_debug_force_rr_cpu = true;
349	#else
350	static bool wq_debug_force_rr_cpu = false;
351	#endif
352	module_param_named(debug_force_rr_cpu, wq_debug_force_rr_cpu, bool, 0644);
353
354	/* the per-cpu worker pools */
355	static DEFINE_PER_CPU_SHARED_ALIGNED(struct worker_pool [NR_STD_WORKER_POOLS], cpu_worker_pools);
356
357	static DEFINE_IDR(worker_pool_idr); /* PR: idr of all pools */
358
359	/* PL: hash of all unbound pools keyed by pool->attrs */
360	static DEFINE_HASHTABLE(unbound_pool_hash, UNBOUND_POOL_HASH_ORDER);
361
362	/* I: attributes used when instantiating standard unbound pools on demand */
363	static struct workqueue_attrs *unbound_std_wq_attrs[NR_STD_WORKER_POOLS];
364
365	/* I: attributes used when instantiating ordered pools on demand */
366	static struct workqueue_attrs *ordered_wq_attrs[NR_STD_WORKER_POOLS];
367
368	struct workqueue_struct *system_wq __read_mostly;
369	EXPORT_SYMBOL(system_wq);
370	struct workqueue_struct *system_highpri_wq __read_mostly;
371	EXPORT_SYMBOL_GPL(system_highpri_wq);
372	struct workqueue_struct *system_long_wq __read_mostly;
373	EXPORT_SYMBOL_GPL(system_long_wq);
374	struct workqueue_struct *system_unbound_wq __read_mostly;
375	EXPORT_SYMBOL_GPL(system_unbound_wq);
376	struct workqueue_struct *system_freezable_wq __read_mostly;
377	EXPORT_SYMBOL_GPL(system_freezable_wq);
378	struct workqueue_struct *system_power_efficient_wq __read_mostly;
379	EXPORT_SYMBOL_GPL(system_power_efficient_wq);
380	struct workqueue_struct *system_freezable_power_efficient_wq __read_mostly;
381	EXPORT_SYMBOL_GPL(system_freezable_power_efficient_wq);
382
383	static int worker_thread(void *__worker);
384	static void workqueue_sysfs_unregister(struct workqueue_struct *wq);
385	static void show_pwq(struct pool_workqueue *pwq);
386	static void show_one_worker_pool(struct worker_pool *pool);
387
388	#define CREATE_TRACE_POINTS
389	#include <trace/events/workqueue.h>
390
391	#define assert_rcu_or_pool_mutex() \
392	RCU_LOCKDEP_WARN(!rcu_read_lock_held() && \
393	!lockdep_is_held(&wq_pool_mutex), \
394	"RCU or wq_pool_mutex should be held")
395
396	#define assert_rcu_or_wq_mutex_or_pool_mutex(wq) \
397	RCU_LOCKDEP_WARN(!rcu_read_lock_held() && \
398	!lockdep_is_held(&wq->mutex) && \
399	!lockdep_is_held(&wq_pool_mutex), \
400	"RCU, wq->mutex or wq_pool_mutex should be held")
401
402	#define for_each_cpu_worker_pool(pool, cpu) \
403	for ((pool) = &per_cpu(cpu_worker_pools, cpu)[0]; \
404	(pool) < &per_cpu(cpu_worker_pools, cpu)[NR_STD_WORKER_POOLS]; \
405	(pool)++)
406
407	/**
408	* for_each_pool - iterate through all worker_pools in the system
409	* @pool: iteration cursor
410	* @pi: integer used for iteration
411	*
412	* This must be called either with wq_pool_mutex held or RCU read
413	* locked. If the pool needs to be used beyond the locking in effect, the
414	* caller is responsible for guaranteeing that the pool stays online.
415	*
416	* The if/else clause exists only for the lockdep assertion and can be
417	* ignored.
418	*/
419	#define for_each_pool(pool, pi) \
420	idr_for_each_entry(&worker_pool_idr, pool, pi) \
421	if (({ assert_rcu_or_pool_mutex(); false; })) { } \
422	else
423
424	/**
425	* for_each_pool_worker - iterate through all workers of a worker_pool
426	* @worker: iteration cursor
427	* @pool: worker_pool to iterate workers of
428	*
429	* This must be called with wq_pool_attach_mutex.
430	*
431	* The if/else clause exists only for the lockdep assertion and can be
432	* ignored.
433	*/
434	#define for_each_pool_worker(worker, pool) \
435	list_for_each_entry((worker), &(pool)->workers, node) \
436	if (({ lockdep_assert_held(&wq_pool_attach_mutex); false; })) { } \
437	else
438
439	/**
440	* for_each_pwq - iterate through all pool_workqueues of the specified workqueue
441	* @pwq: iteration cursor
442	* @wq: the target workqueue
443	*
444	* This must be called either with wq->mutex held or RCU read locked.
445	* If the pwq needs to be used beyond the locking in effect, the caller is
446	* responsible for guaranteeing that the pwq stays online.
447	*
448	* The if/else clause exists only for the lockdep assertion and can be
449	* ignored.
450	*/
451	#define for_each_pwq(pwq, wq) \
452	list_for_each_entry_rcu((pwq), &(wq)->pwqs, pwqs_node, \
453	lockdep_is_held(&(wq->mutex)))
454
455	#ifdef CONFIG_DEBUG_OBJECTS_WORK
456
457	static const struct debug_obj_descr work_debug_descr;
458
459	static void work_debug_hint(void addr)
460	{
461	return ((struct work_struct *) addr)->func;
462	}
463
464	static bool work_is_static_object(void *addr)
465	{
466	struct work_struct *work = addr;
467
468	return test_bit(WORK_STRUCT_STATIC_BIT, work_data_bits(work));
469	}
470
471	/*
472	* fixup_init is called when:
473	* - an active object is initialized
474	*/
475	static bool work_fixup_init(void *addr, enum debug_obj_state state)
476	{
477	struct work_struct *work = addr;
478
479	switch (state) {
480	case ODEBUG_STATE_ACTIVE:
481	cancel_work_sync(work);
482	debug_object_init(work, &work_debug_descr);
483	return true;
484	default:
485	return false;
486	}
487	}
488
489	/*
490	* fixup_free is called when:
491	* - an active object is freed
492	*/
493	static bool work_fixup_free(void *addr, enum debug_obj_state state)
494	{
495	struct work_struct *work = addr;
496
497	switch (state) {
498	case ODEBUG_STATE_ACTIVE:
499	cancel_work_sync(work);
500	debug_object_free(work, &work_debug_descr);
501	return true;
502	default:
503	return false;
504	}
505	}
506
507	static const struct debug_obj_descr work_debug_descr = {
508	.name = "work_struct",
509	.debug_hint = work_debug_hint,
510	.is_static_object = work_is_static_object,
511	.fixup_init = work_fixup_init,
512	.fixup_free = work_fixup_free,
513	};
514
515	static inline void debug_work_activate(struct work_struct *work)
516	{
517	debug_object_activate(work, &work_debug_descr);
518	}
519
520	static inline void debug_work_deactivate(struct work_struct *work)
521	{
522	debug_object_deactivate(work, &work_debug_descr);
523	}
524
525	void __init_work(struct work_struct *work, int onstack)
526	{
527	if (onstack)
528	debug_object_init_on_stack(work, &work_debug_descr);
529	else
530	debug_object_init(work, &work_debug_descr);
531	}
532	EXPORT_SYMBOL_GPL(__init_work);
533
534	void destroy_work_on_stack(struct work_struct *work)
535	{
536	debug_object_free(work, &work_debug_descr);
537	}
538	EXPORT_SYMBOL_GPL(destroy_work_on_stack);
539
540	void destroy_delayed_work_on_stack(struct delayed_work *work)
541	{
542	destroy_timer_on_stack(&work->timer);
543	debug_object_free(&work->work, &work_debug_descr);
544	}
545	EXPORT_SYMBOL_GPL(destroy_delayed_work_on_stack);
546
547	#else
548	static inline void debug_work_activate(struct work_struct *work) { }
549	static inline void debug_work_deactivate(struct work_struct *work) { }
550	#endif
551
552	/**
553	* worker_pool_assign_id - allocate ID and assign it to @pool
554	* @pool: the pool pointer of interest
555	*
556	* Returns 0 if ID in [0, WORK_OFFQ_POOL_NONE) is allocated and assigned
557	* successfully, -errno on failure.
558	*/
559	static int worker_pool_assign_id(struct worker_pool *pool)
560	{
561	int ret;
562
563	lockdep_assert_held(&wq_pool_mutex);
564
565	ret = idr_alloc(&worker_pool_idr, pool, 0, WORK_OFFQ_POOL_NONE,
566	GFP_KERNEL);
567	if (ret >= 0) {
568	pool->id = ret;
569	return 0;
570	}
571	return ret;
572	}
573
574	/**
575	* unbound_pwq_by_node - return the unbound pool_workqueue for the given node
576	* @wq: the target workqueue
577	* @node: the node ID
578	*
579	* This must be called with any of wq_pool_mutex, wq->mutex or RCU
580	* read locked.
581	* If the pwq needs to be used beyond the locking in effect, the caller is
582	* responsible for guaranteeing that the pwq stays online.
583	*
584	* Return: The unbound pool_workqueue for @node.
585	*/
586	static struct pool_workqueue unbound_pwq_by_node(struct workqueue_struct wq,
587	int node)
588	{
589	assert_rcu_or_wq_mutex_or_pool_mutex(wq);
590
591	/*
592	* XXX: @node can be NUMA_NO_NODE if CPU goes offline while a
593	* delayed item is pending. The plan is to keep CPU -> NODE
594	* mapping valid and stable across CPU on/offlines. Once that
595	* happens, this workaround can be removed.
596	*/
597	if (unlikely(node == NUMA_NO_NODE))
598	return wq->dfl_pwq;
599
600	return rcu_dereference_raw(wq->numa_pwq_tbl[node]);
601	}
602
603	static unsigned int work_color_to_flags(int color)
604	{
605	return color << WORK_STRUCT_COLOR_SHIFT;
606	}
607
608	static int get_work_color(unsigned long work_data)
609	{
610	return (work_data >> WORK_STRUCT_COLOR_SHIFT) &
611	((1 << WORK_STRUCT_COLOR_BITS) - 1);
612	}
613
614	static int work_next_color(int color)
615	{
616	return (color + 1) % WORK_NR_COLORS;
617	}
618
619	/*
620	* While queued, %WORK_STRUCT_PWQ is set and non flag bits of a work's data
621	* contain the pointer to the queued pwq. Once execution starts, the flag
622	* is cleared and the high bits contain OFFQ flags and pool ID.
623	*
624	* set_work_pwq(), set_work_pool_and_clear_pending(), mark_work_canceling()
625	* and clear_work_data() can be used to set the pwq, pool or clear
626	* work->data. These functions should only be called while the work is
627	* owned - ie. while the PENDING bit is set.
628	*
629	* get_work_pool() and get_work_pwq() can be used to obtain the pool or pwq
630	* corresponding to a work. Pool is available once the work has been
631	* queued anywhere after initialization until it is sync canceled. pwq is
632	* available only while the work item is queued.
633	*
634	* %WORK_OFFQ_CANCELING is used to mark a work item which is being
635	* canceled. While being canceled, a work item may have its PENDING set
636	* but stay off timer and worklist for arbitrarily long and nobody should
637	* try to steal the PENDING bit.
638	*/
639	static inline void set_work_data(struct work_struct *work, unsigned long data,
640	unsigned long flags)
641	{
642	WARN_ON_ONCE(!work_pending(work));
643	atomic_long_set(&work->data, data \| flags \| work_static(work));
644	}
645
646	static void set_work_pwq(struct work_struct work, struct pool_workqueue pwq,
647	unsigned long extra_flags)
648	{
649	set_work_data(work, (unsigned long)pwq,
650	WORK_STRUCT_PENDING \| WORK_STRUCT_PWQ \| extra_flags);
651	}
652
653	static void set_work_pool_and_keep_pending(struct work_struct *work,
654	int pool_id)
655	{
656	set_work_data(work, (unsigned long)pool_id << WORK_OFFQ_POOL_SHIFT,
657	WORK_STRUCT_PENDING);
658	}
659
660	static void set_work_pool_and_clear_pending(struct work_struct *work,
661	int pool_id)
662	{
663	/*
664	* The following wmb is paired with the implied mb in
665	* test_and_set_bit(PENDING) and ensures all updates to @work made
666	* here are visible to and precede any updates by the next PENDING
667	* owner.
668	*/
669	smp_wmb();
670	set_work_data(work, (unsigned long)pool_id << WORK_OFFQ_POOL_SHIFT, 0);
671	/*
672	* The following mb guarantees that previous clear of a PENDING bit
673	* will not be reordered with any speculative LOADS or STORES from
674	* work->current_func, which is executed afterwards. This possible
675	* reordering can lead to a missed execution on attempt to queue
676	* the same @work. E.g. consider this case:
677	*
678	* CPU#0 CPU#1
679	* ---------------------------- --------------------------------
680	*
681	* 1 STORE event_indicated
682	* 2 queue_work_on() {
683	* 3 test_and_set_bit(PENDING)
684	* 4 } set_..._and_clear_pending() {
685	* 5 set_work_data() # clear bit
686	* 6 smp_mb()
687	* 7 work->current_func() {
688	* 8 LOAD event_indicated
689	* }
690	*
691	* Without an explicit full barrier speculative LOAD on line 8 can
692	* be executed before CPU#0 does STORE on line 1. If that happens,
693	* CPU#0 observes the PENDING bit is still set and new execution of
694	* a @work is not queued in a hope, that CPU#1 will eventually
695	* finish the queued @work. Meanwhile CPU#1 does not see
696	* event_indicated is set, because speculative LOAD was executed
697	* before actual STORE.
698	*/
699	smp_mb();
700	}
701
702	static void clear_work_data(struct work_struct *work)
703	{
704	smp_wmb(); /* see set_work_pool_and_clear_pending() */
705	set_work_data(work, WORK_STRUCT_NO_POOL, 0);
706	}
707
708	static struct pool_workqueue get_work_pwq(struct work_struct work)
709	{
710	unsigned long data = atomic_long_read(&work->data);
711
712	if (data & WORK_STRUCT_PWQ)
713	return (void *)(data & WORK_STRUCT_WQ_DATA_MASK);
714	else
715	return NULL;
716	}
717
718	/**
719	* get_work_pool - return the worker_pool a given work was associated with
720	* @work: the work item of interest
721	*
722	* Pools are created and destroyed under wq_pool_mutex, and allows read
723	* access under RCU read lock. As such, this function should be
724	* called under wq_pool_mutex or inside of a rcu_read_lock() region.
725	*
726	* All fields of the returned pool are accessible as long as the above
727	* mentioned locking is in effect. If the returned pool needs to be used
728	* beyond the critical section, the caller is responsible for ensuring the
729	* returned pool is and stays online.
730	*
731	* Return: The worker_pool @work was last associated with. %NULL if none.
732	*/
733	static struct worker_pool get_work_pool(struct work_struct work)
734	{
735	unsigned long data = atomic_long_read(&work->data);
736	int pool_id;
737
738	assert_rcu_or_pool_mutex();
739
740	if (data & WORK_STRUCT_PWQ)
741	return ((struct pool_workqueue *)
742	(data & WORK_STRUCT_WQ_DATA_MASK))->pool;
743
744	pool_id = data >> WORK_OFFQ_POOL_SHIFT;
745	if (pool_id == WORK_OFFQ_POOL_NONE)
746	return NULL;
747
748	return idr_find(&worker_pool_idr, pool_id);
749	}
750
751	/**
752	* get_work_pool_id - return the worker pool ID a given work is associated with
753	* @work: the work item of interest
754	*
755	* Return: The worker_pool ID @work was last associated with.
756	* %WORK_OFFQ_POOL_NONE if none.
757	*/
758	static int get_work_pool_id(struct work_struct *work)
759	{
760	unsigned long data = atomic_long_read(&work->data);
761
762	if (data & WORK_STRUCT_PWQ)
763	return ((struct pool_workqueue *)
764	(data & WORK_STRUCT_WQ_DATA_MASK))->pool->id;
765
766	return data >> WORK_OFFQ_POOL_SHIFT;
767	}
768
769	static void mark_work_canceling(struct work_struct *work)
770	{
771	unsigned long pool_id = get_work_pool_id(work);
772
773	pool_id <<= WORK_OFFQ_POOL_SHIFT;
774	set_work_data(work, pool_id \| WORK_OFFQ_CANCELING, WORK_STRUCT_PENDING);
775	}
776
777	static bool work_is_canceling(struct work_struct *work)
778	{
779	unsigned long data = atomic_long_read(&work->data);
780
781	return !(data & WORK_STRUCT_PWQ) && (data & WORK_OFFQ_CANCELING);
782	}
783
784	/*
785	* Policy functions. These define the policies on how the global worker
786	* pools are managed. Unless noted otherwise, these functions assume that
787	* they're being called with pool->lock held.
788	*/
789
790	static bool __need_more_worker(struct worker_pool *pool)
791	{
792	return !pool->nr_running;
793	}
794
795	/*
796	* Need to wake up a worker? Called from anything but currently
797	* running workers.
798	*
799	* Note that, because unbound workers never contribute to nr_running, this
800	* function will always return %true for unbound pools as long as the
801	* worklist isn't empty.
802	*/
803	static bool need_more_worker(struct worker_pool *pool)
804	{
805	return !list_empty(&pool->worklist) && __need_more_worker(pool);
806	}
807
808	/* Can I start working? Called from busy but !running workers. */
809	static bool may_start_working(struct worker_pool *pool)
810	{
811	return pool->nr_idle;
812	}
813
814	/* Do I need to keep working? Called from currently running workers. */
815	static bool keep_working(struct worker_pool *pool)
816	{
817	return !list_empty(&pool->worklist) && (pool->nr_running <= 1);
818	}
819
820	/* Do we need a new worker? Called from manager. */
821	static bool need_to_create_worker(struct worker_pool *pool)
822	{
823	return need_more_worker(pool) && !may_start_working(pool);
824	}
825
826	/* Do we have too many workers and should some go away? */
827	static bool too_many_workers(struct worker_pool *pool)
828	{
829	bool managing = pool->flags & POOL_MANAGER_ACTIVE;
830	int nr_idle = pool->nr_idle + managing; /* manager is considered idle */
831	int nr_busy = pool->nr_workers - nr_idle;
832
833	return nr_idle > 2 && (nr_idle - 2) * MAX_IDLE_WORKERS_RATIO >= nr_busy;
834	}
835
836	/*
837	* Wake up functions.
838	*/
839
840	/* Return the first idle worker. Called with pool->lock held. */
841	static struct worker first_idle_worker(struct worker_pool pool)
842	{
843	if (unlikely(list_empty(&pool->idle_list)))
844	return NULL;
845
846	return list_first_entry(&pool->idle_list, struct worker, entry);
847	}
848
849	/**
850	* wake_up_worker - wake up an idle worker
851	* @pool: worker pool to wake worker from
852	*
853	* Wake up the first idle worker of @pool.
854	*
855	* CONTEXT:
856	* raw_spin_lock_irq(pool->lock).
857	*/
858	static void wake_up_worker(struct worker_pool *pool)
859	{
860	struct worker *worker = first_idle_worker(pool);
861
862	if (likely(worker))
863	wake_up_process(worker->task);
864	}
865
866	/**
867	* wq_worker_running - a worker is running again
868	* @task: task waking up
869	*
870	* This function is called when a worker returns from schedule()
871	*/
872	void wq_worker_running(struct task_struct *task)
873	{
874	struct worker *worker = kthread_data(task);
875
876	if (!worker->sleeping)
877	return;
878
879	/*
880	* If preempted by unbind_workers() between the WORKER_NOT_RUNNING check
881	* and the nr_running increment below, we may ruin the nr_running reset
882	* and leave with an unexpected pool->nr_running == 1 on the newly unbound
883	* pool. Protect against such race.
884	*/
885	preempt_disable();
886	if (!(worker->flags & WORKER_NOT_RUNNING))
887	worker->pool->nr_running++;
888	preempt_enable();
889	worker->sleeping = 0;
890	}
891
892	/**
893	* wq_worker_sleeping - a worker is going to sleep
894	* @task: task going to sleep
895	*
896	* This function is called from schedule() when a busy worker is
897	* going to sleep.
898	*/
899	void wq_worker_sleeping(struct task_struct *task)
900	{
901	struct worker *worker = kthread_data(task);
902	struct worker_pool *pool;
903
904	/*
905	* Rescuers, which may not have all the fields set up like normal
906	* workers, also reach here, let's not access anything before
907	* checking NOT_RUNNING.
908	*/
909	if (worker->flags & WORKER_NOT_RUNNING)
910	return;
911
912	pool = worker->pool;
913
914	/* Return if preempted before wq_worker_running() was reached */
915	if (worker->sleeping)
916	return;
917
918	worker->sleeping = 1;
919	raw_spin_lock_irq(&pool->lock);
920
921	/*
922	* Recheck in case unbind_workers() preempted us. We don't
923	* want to decrement nr_running after the worker is unbound
924	* and nr_running has been reset.
925	*/
926	if (worker->flags & WORKER_NOT_RUNNING) {
927	raw_spin_unlock_irq(&pool->lock);
928	return;
929	}
930
931	pool->nr_running--;
932	if (need_more_worker(pool))
933	wake_up_worker(pool);
934	raw_spin_unlock_irq(&pool->lock);
935	}
936
937	/**
938	* wq_worker_last_func - retrieve worker's last work function
939	* @task: Task to retrieve last work function of.
940	*
941	* Determine the last function a worker executed. This is called from
942	* the scheduler to get a worker's last known identity.
943	*
944	* CONTEXT:
945	* raw_spin_lock_irq(rq->lock)
946	*
947	* This function is called during schedule() when a kworker is going
948	* to sleep. It's used by psi to identify aggregation workers during
949	* dequeuing, to allow periodic aggregation to shut-off when that
950	* worker is the last task in the system or cgroup to go to sleep.
951	*
952	* As this function doesn't involve any workqueue-related locking, it
953	* only returns stable values when called from inside the scheduler's
954	* queuing and dequeuing paths, when @task, which must be a kworker,
955	* is guaranteed to not be processing any works.
956	*
957	* Return:
958	* The last work function %current executed as a worker, NULL if it
959	* hasn't executed any work yet.
960	*/
961	work_func_t wq_worker_last_func(struct task_struct *task)
962	{
963	struct worker *worker = kthread_data(task);
964
965	return worker->last_func;
966	}
967
968	/**
969	* worker_set_flags - set worker flags and adjust nr_running accordingly
970	* @worker: self
971	* @flags: flags to set
972	*
973	* Set @flags in @worker->flags and adjust nr_running accordingly.
974	*
975	* CONTEXT:
976	* raw_spin_lock_irq(pool->lock)
977	*/
978	static inline void worker_set_flags(struct worker *worker, unsigned int flags)
979	{
980	struct worker_pool *pool = worker->pool;
981
982	WARN_ON_ONCE(worker->task != current);
983
984	/* If transitioning into NOT_RUNNING, adjust nr_running. */
985	if ((flags & WORKER_NOT_RUNNING) &&
986	!(worker->flags & WORKER_NOT_RUNNING)) {
987	pool->nr_running--;
988	}
989
990	worker->flags \|= flags;
991	}
992
993	/**
994	* worker_clr_flags - clear worker flags and adjust nr_running accordingly
995	* @worker: self
996	* @flags: flags to clear
997	*
998	* Clear @flags in @worker->flags and adjust nr_running accordingly.
999	*
1000	* CONTEXT:
1001	* raw_spin_lock_irq(pool->lock)
1002	*/
1003	static inline void worker_clr_flags(struct worker *worker, unsigned int flags)
1004	{
1005	struct worker_pool *pool = worker->pool;
1006	unsigned int oflags = worker->flags;
1007
1008	WARN_ON_ONCE(worker->task != current);
1009
1010	worker->flags &= ~flags;
1011
1012	/*
1013	* If transitioning out of NOT_RUNNING, increment nr_running. Note
1014	* that the nested NOT_RUNNING is not a noop. NOT_RUNNING is mask
1015	* of multiple flags, not a single flag.
1016	*/
1017	if ((flags & WORKER_NOT_RUNNING) && (oflags & WORKER_NOT_RUNNING))
1018	if (!(worker->flags & WORKER_NOT_RUNNING))
1019	pool->nr_running++;
1020	}
1021
1022	/**
1023	* find_worker_executing_work - find worker which is executing a work
1024	* @pool: pool of interest
1025	* @work: work to find worker for
1026	*
1027	* Find a worker which is executing @work on @pool by searching
1028	* @pool->busy_hash which is keyed by the address of @work. For a worker
1029	* to match, its current execution should match the address of @work and
1030	* its work function. This is to avoid unwanted dependency between
1031	* unrelated work executions through a work item being recycled while still
1032	* being executed.
1033	*
1034	* This is a bit tricky. A work item may be freed once its execution
1035	* starts and nothing prevents the freed area from being recycled for
1036	* another work item. If the same work item address ends up being reused
1037	* before the original execution finishes, workqueue will identify the
1038	* recycled work item as currently executing and make it wait until the
1039	* current execution finishes, introducing an unwanted dependency.
1040	*
1041	* This function checks the work item address and work function to avoid
1042	* false positives. Note that this isn't complete as one may construct a
1043	* work function which can introduce dependency onto itself through a
1044	* recycled work item. Well, if somebody wants to shoot oneself in the
1045	* foot that badly, there's only so much we can do, and if such deadlock
1046	* actually occurs, it should be easy to locate the culprit work function.
1047	*
1048	* CONTEXT:
1049	* raw_spin_lock_irq(pool->lock).
1050	*
1051	* Return:
1052	* Pointer to worker which is executing @work if found, %NULL
1053	* otherwise.
1054	*/
1055	static struct worker find_worker_executing_work(struct worker_pool pool,
1056	struct work_struct *work)
1057	{
1058	struct worker *worker;
1059
1060	hash_for_each_possible(pool->busy_hash, worker, hentry,
1061	(unsigned long)work)
1062	if (worker->current_work == work &&
1063	worker->current_func == work->func)
1064	return worker;
1065
1066	return NULL;
1067	}
1068
1069	/**
1070	* move_linked_works - move linked works to a list
1071	* @work: start of series of works to be scheduled
1072	* @head: target list to append @work to
1073	* @nextp: out parameter for nested worklist walking
1074	*
1075	* Schedule linked works starting from @work to @head. Work series to
1076	* be scheduled starts at @work and includes any consecutive work with
1077	* WORK_STRUCT_LINKED set in its predecessor.
1078	*
1079	* If @nextp is not NULL, it's updated to point to the next work of
1080	* the last scheduled work. This allows move_linked_works() to be
1081	* nested inside outer list_for_each_entry_safe().
1082	*
1083	* CONTEXT:
1084	* raw_spin_lock_irq(pool->lock).
1085	*/
1086	static void move_linked_works(struct work_struct work, struct list_head head,
1087	struct work_struct **nextp)
1088	{
1089	struct work_struct *n;
1090
1091	/*
1092	* Linked worklist will always end before the end of the list,
1093	* use NULL for list head.
1094	*/
1095	list_for_each_entry_safe_from(work, n, NULL, entry) {
1096	list_move_tail(&work->entry, head);
1097	if (!(*work_data_bits(work) & WORK_STRUCT_LINKED))
1098	break;
1099	}
1100
1101	/*
1102	* If we're already inside safe list traversal and have moved
1103	* multiple works to the scheduled queue, the next position
1104	* needs to be updated.
1105	*/
1106	if (nextp)
1107	*nextp = n;
1108	}
1109
1110	/**
1111	* get_pwq - get an extra reference on the specified pool_workqueue
1112	* @pwq: pool_workqueue to get
1113	*
1114	* Obtain an extra reference on @pwq. The caller should guarantee that
1115	* @pwq has positive refcnt and be holding the matching pool->lock.
1116	*/
1117	static void get_pwq(struct pool_workqueue *pwq)
1118	{
1119	lockdep_assert_held(&pwq->pool->lock);
1120	WARN_ON_ONCE(pwq->refcnt <= 0);
1121	pwq->refcnt++;
1122	}
1123
1124	/**
1125	* put_pwq - put a pool_workqueue reference
1126	* @pwq: pool_workqueue to put
1127	*
1128	* Drop a reference of @pwq. If its refcnt reaches zero, schedule its
1129	* destruction. The caller should be holding the matching pool->lock.
1130	*/
1131	static void put_pwq(struct pool_workqueue *pwq)
1132	{
1133	lockdep_assert_held(&pwq->pool->lock);
1134	if (likely(--pwq->refcnt))
1135	return;
1136	if (WARN_ON_ONCE(!(pwq->wq->flags & WQ_UNBOUND)))
1137	return;
1138	/*
1139	* @pwq can't be released under pool->lock, bounce to
1140	* pwq_unbound_release_workfn(). This never recurses on the same
1141	* pool->lock as this path is taken only for unbound workqueues and
1142	* the release work item is scheduled on a per-cpu workqueue. To
1143	* avoid lockdep warning, unbound pool->locks are given lockdep
1144	* subclass of 1 in get_unbound_pool().
1145	*/
1146	schedule_work(&pwq->unbound_release_work);
1147	}
1148
1149	/**
1150	* put_pwq_unlocked - put_pwq() with surrounding pool lock/unlock
1151	* @pwq: pool_workqueue to put (can be %NULL)
1152	*
1153	* put_pwq() with locking. This function also allows %NULL @pwq.
1154	*/
1155	static void put_pwq_unlocked(struct pool_workqueue *pwq)
1156	{
1157	if (pwq) {
1158	/*
1159	* As both pwqs and pools are RCU protected, the
1160	* following lock operations are safe.
1161	*/
1162	raw_spin_lock_irq(&pwq->pool->lock);
1163	put_pwq(pwq);
1164	raw_spin_unlock_irq(&pwq->pool->lock);
1165	}
1166	}
1167
1168	static void pwq_activate_inactive_work(struct work_struct *work)
1169	{
1170	struct pool_workqueue *pwq = get_work_pwq(work);
1171
1172	trace_workqueue_activate_work(work);
1173	if (list_empty(&pwq->pool->worklist))
1174	pwq->pool->watchdog_ts = jiffies;
1175	move_linked_works(work, &pwq->pool->worklist, NULL);
1176	__clear_bit(WORK_STRUCT_INACTIVE_BIT, work_data_bits(work));
1177	pwq->nr_active++;
1178	}
1179
1180	static void pwq_activate_first_inactive(struct pool_workqueue *pwq)
1181	{
1182	struct work_struct *work = list_first_entry(&pwq->inactive_works,
1183	struct work_struct, entry);
1184
1185	pwq_activate_inactive_work(work);
1186	}
1187
1188	/**
1189	* pwq_dec_nr_in_flight - decrement pwq's nr_in_flight
1190	* @pwq: pwq of interest
1191	* @work_data: work_data of work which left the queue
1192	*
1193	* A work either has completed or is removed from pending queue,
1194	* decrement nr_in_flight of its pwq and handle workqueue flushing.
1195	*
1196	* CONTEXT:
1197	* raw_spin_lock_irq(pool->lock).
1198	*/
1199	static void pwq_dec_nr_in_flight(struct pool_workqueue *pwq, unsigned long work_data)
1200	{
1201	int color = get_work_color(work_data);
1202
1203	if (!(work_data & WORK_STRUCT_INACTIVE)) {
1204	pwq->nr_active--;
1205	if (!list_empty(&pwq->inactive_works)) {
1206	/* one down, submit an inactive one */
1207	if (pwq->nr_active < pwq->max_active)
1208	pwq_activate_first_inactive(pwq);
1209	}
1210	}
1211
1212	pwq->nr_in_flight[color]--;
1213
1214	/* is flush in progress and are we at the flushing tip? */
1215	if (likely(pwq->flush_color != color))
1216	goto out_put;
1217
1218	/* are there still in-flight works? */
1219	if (pwq->nr_in_flight[color])
1220	goto out_put;
1221
1222	/* this pwq is done, clear flush_color */
1223	pwq->flush_color = -1;
1224
1225	/*
1226	* If this was the last pwq, wake up the first flusher. It
1227	* will handle the rest.
1228	*/
1229	if (atomic_dec_and_test(&pwq->wq->nr_pwqs_to_flush))
1230	complete(&pwq->wq->first_flusher->done);
1231	out_put:
1232	put_pwq(pwq);
1233	}
1234
1235	/**
1236	* try_to_grab_pending - steal work item from worklist and disable irq
1237	* @work: work item to steal
1238	* @is_dwork: @work is a delayed_work
1239	* @flags: place to store irq state
1240	*
1241	* Try to grab PENDING bit of @work. This function can handle @work in any
1242	* stable state - idle, on timer or on worklist.
1243	*
1244	* Return:
1245	*
1246	* ======== ================================================================
1247	* 1 if @work was pending and we successfully stole PENDING
1248	* 0 if @work was idle and we claimed PENDING
1249	* -EAGAIN if PENDING couldn't be grabbed at the moment, safe to busy-retry
1250	* -ENOENT if someone else is canceling @work, this state may persist
1251	* for arbitrarily long
1252	* ======== ================================================================
1253	*
1254	* Note:
1255	* On >= 0 return, the caller owns @work's PENDING bit. To avoid getting
1256	* interrupted while holding PENDING and @work off queue, irq must be
1257	* disabled on entry. This, combined with delayed_work->timer being
1258	* irqsafe, ensures that we return -EAGAIN for finite short period of time.
1259	*
1260	* On successful return, >= 0, irq is disabled and the caller is
1261	* responsible for releasing it using local_irq_restore(*@flags).
1262	*
1263	* This function is safe to call from any context including IRQ handler.
1264	*/
1265	static int try_to_grab_pending(struct work_struct *work, bool is_dwork,
1266	unsigned long *flags)
1267	{
1268	struct worker_pool *pool;
1269	struct pool_workqueue *pwq;
1270
1271	local_irq_save(*flags);
1272
1273	/* try to steal the timer if it exists */
1274	if (is_dwork) {
1275	struct delayed_work *dwork = to_delayed_work(work);
1276
1277	/*
1278	* dwork->timer is irqsafe. If del_timer() fails, it's
1279	* guaranteed that the timer is not queued anywhere and not
1280	* running on the local CPU.
1281	*/
1282	if (likely(del_timer(&dwork->timer)))
1283	return 1;
1284	}
1285
1286	/* try to claim PENDING the normal way */
1287	if (!test_and_set_bit(WORK_STRUCT_PENDING_BIT, work_data_bits(work)))
1288	return 0;
1289
1290	rcu_read_lock();
1291	/*
1292	* The queueing is in progress, or it is already queued. Try to
1293	* steal it from ->worklist without clearing WORK_STRUCT_PENDING.
1294	*/
1295	pool = get_work_pool(work);
1296	if (!pool)
1297	goto fail;
1298
1299	raw_spin_lock(&pool->lock);
1300	/*
1301	* work->data is guaranteed to point to pwq only while the work
1302	* item is queued on pwq->wq, and both updating work->data to point
1303	* to pwq on queueing and to pool on dequeueing are done under
1304	* pwq->pool->lock. This in turn guarantees that, if work->data
1305	* points to pwq which is associated with a locked pool, the work
1306	* item is currently queued on that pool.
1307	*/
1308	pwq = get_work_pwq(work);
1309	if (pwq && pwq->pool == pool) {
1310	debug_work_deactivate(work);
1311
1312	/*
1313	* A cancelable inactive work item must be in the
1314	* pwq->inactive_works since a queued barrier can't be
1315	* canceled (see the comments in insert_wq_barrier()).
1316	*
1317	* An inactive work item cannot be grabbed directly because
1318	* it might have linked barrier work items which, if left
1319	* on the inactive_works list, will confuse pwq->nr_active
1320	* management later on and cause stall. Make sure the work
1321	* item is activated before grabbing.
1322	*/
1323	if (*work_data_bits(work) & WORK_STRUCT_INACTIVE)
1324	pwq_activate_inactive_work(work);
1325
1326	list_del_init(&work->entry);
1327	pwq_dec_nr_in_flight(pwq, *work_data_bits(work));
1328
1329	/* work->data points to pwq iff queued, point to pool */
1330	set_work_pool_and_keep_pending(work, pool->id);
1331
1332	raw_spin_unlock(&pool->lock);
1333	rcu_read_unlock();
1334	return 1;
1335	}
1336	raw_spin_unlock(&pool->lock);
1337	fail:
1338	rcu_read_unlock();
1339	local_irq_restore(*flags);
1340	if (work_is_canceling(work))
1341	return -ENOENT;
1342	cpu_relax();
1343	return -EAGAIN;
1344	}
1345
1346	/**
1347	* insert_work - insert a work into a pool
1348	* @pwq: pwq @work belongs to
1349	* @work: work to insert
1350	* @head: insertion point
1351	* @extra_flags: extra WORK_STRUCT_* flags to set
1352	*
1353	* Insert @work which belongs to @pwq after @head. @extra_flags is or'd to
1354	* work_struct flags.
1355	*
1356	* CONTEXT:
1357	* raw_spin_lock_irq(pool->lock).
1358	*/
1359	static void insert_work(struct pool_workqueue pwq, struct work_struct work,
1360	struct list_head *head, unsigned int extra_flags)
1361	{
1362	struct worker_pool *pool = pwq->pool;
1363
1364	/* record the work call stack in order to print it in KASAN reports */
1365	kasan_record_aux_stack_noalloc(work);
1366
1367	/* we own @work, set data and link */
1368	set_work_pwq(work, pwq, extra_flags);
1369	list_add_tail(&work->entry, head);
1370	get_pwq(pwq);
1371
1372	if (__need_more_worker(pool))
1373	wake_up_worker(pool);
1374	}
1375
1376	/*
1377	* Test whether @work is being queued from another work executing on the
1378	* same workqueue.
1379	*/
1380	static bool is_chained_work(struct workqueue_struct *wq)
1381	{
1382	struct worker *worker;
1383
1384	worker = current_wq_worker();
1385	/*
1386	* Return %true iff I'm a worker executing a work item on @wq. If
1387	* I'm @worker, it's safe to dereference it without locking.
1388	*/
1389	return worker && worker->current_pwq->wq == wq;
1390	}
1391
1392	/*
1393	* When queueing an unbound work item to a wq, prefer local CPU if allowed
1394	* by wq_unbound_cpumask. Otherwise, round robin among the allowed ones to
1395	* avoid perturbing sensitive tasks.
1396	*/
1397	static int wq_select_unbound_cpu(int cpu)
1398	{
1399	int new_cpu;
1400
1401	if (likely(!wq_debug_force_rr_cpu)) {
1402	if (cpumask_test_cpu(cpu, wq_unbound_cpumask))
1403	return cpu;
1404	} else {
1405	pr_warn_once("workqueue: round-robin CPU selection forced, expect performance impact\n");
1406	}
1407
1408	if (cpumask_empty(wq_unbound_cpumask))
1409	return cpu;
1410
1411	new_cpu = __this_cpu_read(wq_rr_cpu_last);
1412	new_cpu = cpumask_next_and(new_cpu, wq_unbound_cpumask, cpu_online_mask);
1413	if (unlikely(new_cpu >= nr_cpu_ids)) {
1414	new_cpu = cpumask_first_and(wq_unbound_cpumask, cpu_online_mask);
1415	if (unlikely(new_cpu >= nr_cpu_ids))
1416	return cpu;
1417	}
1418	__this_cpu_write(wq_rr_cpu_last, new_cpu);
1419
1420	return new_cpu;
1421	}
1422
1423	static void __queue_work(int cpu, struct workqueue_struct *wq,
1424	struct work_struct *work)
1425	{
1426	struct pool_workqueue *pwq;
1427	struct worker_pool *last_pool;
1428	struct list_head *worklist;
1429	unsigned int work_flags;
1430	unsigned int req_cpu = cpu;
1431
1432	/*
1433	* While a work item is PENDING && off queue, a task trying to
1434	* steal the PENDING will busy-loop waiting for it to either get
1435	* queued or lose PENDING. Grabbing PENDING and queueing should
1436	* happen with IRQ disabled.
1437	*/
1438	lockdep_assert_irqs_disabled();
1439
1440
1441	/*
1442	* For a draining wq, only works from the same workqueue are
1443	* allowed. The __WQ_DESTROYING helps to spot the issue that
1444	* queues a new work item to a wq after destroy_workqueue(wq).
1445	*/
1446	if (unlikely(wq->flags & (__WQ_DESTROYING \| __WQ_DRAINING) &&
1447	WARN_ON_ONCE(!is_chained_work(wq))))
1448	return;
1449	rcu_read_lock();
1450	retry:
1451	/* pwq which will be used unless @work is executing elsewhere */
1452	if (wq->flags & WQ_UNBOUND) {
1453	if (req_cpu == WORK_CPU_UNBOUND)
1454	cpu = wq_select_unbound_cpu(raw_smp_processor_id());
1455	pwq = unbound_pwq_by_node(wq, cpu_to_node(cpu));
1456	} else {
1457	if (req_cpu == WORK_CPU_UNBOUND)
1458	cpu = raw_smp_processor_id();
1459	pwq = per_cpu_ptr(wq->cpu_pwqs, cpu);
1460	}
1461
1462	/*
1463	* If @work was previously on a different pool, it might still be
1464	* running there, in which case the work needs to be queued on that
1465	* pool to guarantee non-reentrancy.
1466	*/
1467	last_pool = get_work_pool(work);
1468	if (last_pool && last_pool != pwq->pool) {
1469	struct worker *worker;
1470
1471	raw_spin_lock(&last_pool->lock);
1472
1473	worker = find_worker_executing_work(last_pool, work);
1474
1475	if (worker && worker->current_pwq->wq == wq) {
1476	pwq = worker->current_pwq;
1477	} else {
1478	/* meh... not running there, queue here */
1479	raw_spin_unlock(&last_pool->lock);
1480	raw_spin_lock(&pwq->pool->lock);
1481	}
1482	} else {
1483	raw_spin_lock(&pwq->pool->lock);
1484	}
1485
1486	/*
1487	* pwq is determined and locked. For unbound pools, we could have
1488	* raced with pwq release and it could already be dead. If its
1489	* refcnt is zero, repeat pwq selection. Note that pwqs never die
1490	* without another pwq replacing it in the numa_pwq_tbl or while
1491	* work items are executing on it, so the retrying is guaranteed to
1492	* make forward-progress.
1493	*/
1494	if (unlikely(!pwq->refcnt)) {
1495	if (wq->flags & WQ_UNBOUND) {
1496	raw_spin_unlock(&pwq->pool->lock);
1497	cpu_relax();
1498	goto retry;
1499	}
1500	/* oops */
1501	WARN_ONCE(true, "workqueue: per-cpu pwq for %s on cpu%d has 0 refcnt",
1502	wq->name, cpu);
1503	}
1504
1505	/* pwq determined, queue */
1506	trace_workqueue_queue_work(req_cpu, pwq, work);
1507
1508	if (WARN_ON(!list_empty(&work->entry)))
1509	goto out;
1510
1511	pwq->nr_in_flight[pwq->work_color]++;
1512	work_flags = work_color_to_flags(pwq->work_color);
1513
1514	if (likely(pwq->nr_active < pwq->max_active)) {
1515	trace_workqueue_activate_work(work);
1516	pwq->nr_active++;
1517	worklist = &pwq->pool->worklist;
1518	if (list_empty(worklist))
1519	pwq->pool->watchdog_ts = jiffies;
1520	} else {
1521	work_flags \|= WORK_STRUCT_INACTIVE;
1522	worklist = &pwq->inactive_works;
1523	}
1524
1525	debug_work_activate(work);
1526	insert_work(pwq, work, worklist, work_flags);
1527
1528	out:
1529	raw_spin_unlock(&pwq->pool->lock);
1530	rcu_read_unlock();
1531	}
1532
1533	/**
1534	* queue_work_on - queue work on specific cpu
1535	* @cpu: CPU number to execute work on
1536	* @wq: workqueue to use
1537	* @work: work to queue
1538	*
1539	* We queue the work to a specific CPU, the caller must ensure it
1540	* can't go away. Callers that fail to ensure that the specified
1541	* CPU cannot go away will execute on a randomly chosen CPU.
1542	*
1543	* Return: %false if @work was already on a queue, %true otherwise.
1544	*/
1545	bool queue_work_on(int cpu, struct workqueue_struct *wq,
1546	struct work_struct *work)
1547	{
1548	bool ret = false;
1549	unsigned long flags;
1550
1551	local_irq_save(flags);
1552
1553	if (!test_and_set_bit(WORK_STRUCT_PENDING_BIT, work_data_bits(work))) {
1554	__queue_work(cpu, wq, work);
1555	ret = true;
1556	}
1557
1558	local_irq_restore(flags);
1559	return ret;
1560	}
1561	EXPORT_SYMBOL(queue_work_on);
1562
1563	/**
1564	* workqueue_select_cpu_near - Select a CPU based on NUMA node
1565	* @node: NUMA node ID that we want to select a CPU from
1566	*
1567	* This function will attempt to find a "random" cpu available on a given
1568	* node. If there are no CPUs available on the given node it will return
1569	* WORK_CPU_UNBOUND indicating that we should just schedule to any
1570	* available CPU if we need to schedule this work.
1571	*/
1572	static int workqueue_select_cpu_near(int node)
1573	{
1574	int cpu;
1575
1576	/* No point in doing this if NUMA isn't enabled for workqueues */
1577	if (!wq_numa_enabled)
1578	return WORK_CPU_UNBOUND;
1579
1580	/* Delay binding to CPU if node is not valid or online */
1581	if (node < 0 \|\| node >= MAX_NUMNODES \|\| !node_online(node))
1582	return WORK_CPU_UNBOUND;
1583
1584	/* Use local node/cpu if we are already there */
1585	cpu = raw_smp_processor_id();
1586	if (node == cpu_to_node(cpu))
1587	return cpu;
1588
1589	/* Use "random" otherwise know as "first" online CPU of node */
1590	cpu = cpumask_any_and(cpumask_of_node(node), cpu_online_mask);
1591
1592	/* If CPU is valid return that, otherwise just defer */
1593	return cpu < nr_cpu_ids ? cpu : WORK_CPU_UNBOUND;
1594	}
1595
1596	/**
1597	* queue_work_node - queue work on a "random" cpu for a given NUMA node
1598	* @node: NUMA node that we are targeting the work for
1599	* @wq: workqueue to use
1600	* @work: work to queue
1601	*
1602	* We queue the work to a "random" CPU within a given NUMA node. The basic
1603	* idea here is to provide a way to somehow associate work with a given
1604	* NUMA node.
1605	*
1606	* This function will only make a best effort attempt at getting this onto
1607	* the right NUMA node. If no node is requested or the requested node is
1608	* offline then we just fall back to standard queue_work behavior.
1609	*
1610	* Currently the "random" CPU ends up being the first available CPU in the
1611	* intersection of cpu_online_mask and the cpumask of the node, unless we
1612	* are running on the node. In that case we just use the current CPU.
1613	*
1614	* Return: %false if @work was already on a queue, %true otherwise.
1615	*/
1616	bool queue_work_node(int node, struct workqueue_struct *wq,
1617	struct work_struct *work)
1618	{
1619	unsigned long flags;
1620	bool ret = false;
1621
1622	/*
1623	* This current implementation is specific to unbound workqueues.
1624	* Specifically we only return the first available CPU for a given
1625	* node instead of cycling through individual CPUs within the node.
1626	*
1627	* If this is used with a per-cpu workqueue then the logic in
1628	* workqueue_select_cpu_near would need to be updated to allow for
1629	* some round robin type logic.
1630	*/
1631	WARN_ON_ONCE(!(wq->flags & WQ_UNBOUND));
1632
1633	local_irq_save(flags);
1634
1635	if (!test_and_set_bit(WORK_STRUCT_PENDING_BIT, work_data_bits(work))) {
1636	int cpu = workqueue_select_cpu_near(node);
1637
1638	__queue_work(cpu, wq, work);
1639	ret = true;
1640	}
1641
1642	local_irq_restore(flags);
1643	return ret;
1644	}
1645	EXPORT_SYMBOL_GPL(queue_work_node);
1646
1647	void delayed_work_timer_fn(struct timer_list *t)
1648	{
1649	struct delayed_work *dwork = from_timer(dwork, t, timer);
1650
1651	/* should have been called from irqsafe timer with irq already off */
1652	__queue_work(dwork->cpu, dwork->wq, &dwork->work);
1653	}
1654	EXPORT_SYMBOL(delayed_work_timer_fn);
1655
1656	static void __queue_delayed_work(int cpu, struct workqueue_struct *wq,
1657	struct delayed_work *dwork, unsigned long delay)
1658	{
1659	struct timer_list *timer = &dwork->timer;
1660	struct work_struct *work = &dwork->work;
1661
1662	WARN_ON_ONCE(!wq);
1663	WARN_ON_ONCE(timer->function != delayed_work_timer_fn);
1664	WARN_ON_ONCE(timer_pending(timer));
1665	WARN_ON_ONCE(!list_empty(&work->entry));
1666
1667	/*
1668	* If @delay is 0, queue @dwork->work immediately. This is for
1669	* both optimization and correctness. The earliest @timer can
1670	* expire is on the closest next tick and delayed_work users depend
1671	* on that there's no such delay when @delay is 0.
1672	*/
1673	if (!delay) {
1674	__queue_work(cpu, wq, &dwork->work);
1675	return;
1676	}
1677
1678	dwork->wq = wq;
1679	dwork->cpu = cpu;
1680	timer->expires = jiffies + delay;
1681
1682	if (unlikely(cpu != WORK_CPU_UNBOUND))
1683	add_timer_on(timer, cpu);
1684	else
1685	add_timer(timer);
1686	}
1687
1688	/**
1689	* queue_delayed_work_on - queue work on specific CPU after delay
1690	* @cpu: CPU number to execute work on
1691	* @wq: workqueue to use
1692	* @dwork: work to queue
1693	* @delay: number of jiffies to wait before queueing
1694	*
1695	* Return: %false if @work was already on a queue, %true otherwise. If
1696	* @delay is zero and @dwork is idle, it will be scheduled for immediate
1697	* execution.
1698	*/
1699	bool queue_delayed_work_on(int cpu, struct workqueue_struct *wq,
1700	struct delayed_work *dwork, unsigned long delay)
1701	{
1702	struct work_struct *work = &dwork->work;
1703	bool ret = false;
1704	unsigned long flags;
1705
1706	/* read the comment in __queue_work() */
1707	local_irq_save(flags);
1708
1709	if (!test_and_set_bit(WORK_STRUCT_PENDING_BIT, work_data_bits(work))) {
1710	__queue_delayed_work(cpu, wq, dwork, delay);
1711	ret = true;
1712	}
1713
1714	local_irq_restore(flags);
1715	return ret;
1716	}
1717	EXPORT_SYMBOL(queue_delayed_work_on);
1718
1719	/**
1720	* mod_delayed_work_on - modify delay of or queue a delayed work on specific CPU
1721	* @cpu: CPU number to execute work on
1722	* @wq: workqueue to use
1723	* @dwork: work to queue
1724	* @delay: number of jiffies to wait before queueing
1725	*
1726	* If @dwork is idle, equivalent to queue_delayed_work_on(); otherwise,
1727	* modify @dwork's timer so that it expires after @delay. If @delay is
1728	* zero, @work is guaranteed to be scheduled immediately regardless of its
1729	* current state.
1730	*
1731	* Return: %false if @dwork was idle and queued, %true if @dwork was
1732	* pending and its timer was modified.
1733	*
1734	* This function is safe to call from any context including IRQ handler.
1735	* See try_to_grab_pending() for details.
1736	*/
1737	bool mod_delayed_work_on(int cpu, struct workqueue_struct *wq,
1738	struct delayed_work *dwork, unsigned long delay)
1739	{
1740	unsigned long flags;
1741	int ret;
1742
1743	do {
1744	ret = try_to_grab_pending(&dwork->work, true, &flags);
1745	} while (unlikely(ret == -EAGAIN));
1746
1747	if (likely(ret >= 0)) {
1748	__queue_delayed_work(cpu, wq, dwork, delay);
1749	local_irq_restore(flags);
1750	}
1751
1752	/* -ENOENT from try_to_grab_pending() becomes %true */
1753	return ret;
1754	}
1755	EXPORT_SYMBOL_GPL(mod_delayed_work_on);
1756
1757	static void rcu_work_rcufn(struct rcu_head *rcu)
1758	{
1759	struct rcu_work *rwork = container_of(rcu, struct rcu_work, rcu);
1760
1761	/* read the comment in __queue_work() */
1762	local_irq_disable();
1763	__queue_work(WORK_CPU_UNBOUND, rwork->wq, &rwork->work);
1764	local_irq_enable();
1765	}
1766
1767	/**
1768	* queue_rcu_work - queue work after a RCU grace period
1769	* @wq: workqueue to use
1770	* @rwork: work to queue
1771	*
1772	* Return: %false if @rwork was already pending, %true otherwise. Note
1773	* that a full RCU grace period is guaranteed only after a %true return.
1774	* While @rwork is guaranteed to be executed after a %false return, the
1775	* execution may happen before a full RCU grace period has passed.
1776	*/
1777	bool queue_rcu_work(struct workqueue_struct wq, struct rcu_work rwork)
1778	{
1779	struct work_struct *work = &rwork->work;
1780
1781	if (!test_and_set_bit(WORK_STRUCT_PENDING_BIT, work_data_bits(work))) {
1782	rwork->wq = wq;
1783	call_rcu_hurry(&rwork->rcu, rcu_work_rcufn);
1784	return true;
1785	}
1786
1787	return false;
1788	}
1789	EXPORT_SYMBOL(queue_rcu_work);
1790
1791	/**
1792	* worker_enter_idle - enter idle state
1793	* @worker: worker which is entering idle state
1794	*
1795	* @worker is entering idle state. Update stats and idle timer if
1796	* necessary.
1797	*
1798	* LOCKING:
1799	* raw_spin_lock_irq(pool->lock).
1800	*/
1801	static void worker_enter_idle(struct worker *worker)
1802	{
1803	struct worker_pool *pool = worker->pool;
1804
1805	if (WARN_ON_ONCE(worker->flags & WORKER_IDLE) \|\|
1806	WARN_ON_ONCE(!list_empty(&worker->entry) &&
1807	(worker->hentry.next \|\| worker->hentry.pprev)))
1808	return;
1809
1810	/* can't use worker_set_flags(), also called from create_worker() */
1811	worker->flags \|= WORKER_IDLE;
1812	pool->nr_idle++;
1813	worker->last_active = jiffies;
1814
1815	/* idle_list is LIFO */
1816	list_add(&worker->entry, &pool->idle_list);
1817
1818	if (too_many_workers(pool) && !timer_pending(&pool->idle_timer))
1819	mod_timer(&pool->idle_timer, jiffies + IDLE_WORKER_TIMEOUT);
1820
1821	/* Sanity check nr_running. */
1822	WARN_ON_ONCE(pool->nr_workers == pool->nr_idle && pool->nr_running);
1823	}
1824
1825	/**
1826	* worker_leave_idle - leave idle state
1827	* @worker: worker which is leaving idle state
1828	*
1829	* @worker is leaving idle state. Update stats.
1830	*
1831	* LOCKING:
1832	* raw_spin_lock_irq(pool->lock).
1833	*/
1834	static void worker_leave_idle(struct worker *worker)
1835	{
1836	struct worker_pool *pool = worker->pool;
1837
1838	if (WARN_ON_ONCE(!(worker->flags & WORKER_IDLE)))
1839	return;
1840	worker_clr_flags(worker, WORKER_IDLE);
1841	pool->nr_idle--;
1842	list_del_init(&worker->entry);
1843	}
1844
1845	static struct worker *alloc_worker(int node)
1846	{
1847	struct worker *worker;
1848
1849	worker = kzalloc_node(sizeof(*worker), GFP_KERNEL, node);
1850	if (worker) {
1851	INIT_LIST_HEAD(&worker->entry);
1852	INIT_LIST_HEAD(&worker->scheduled);
1853	INIT_LIST_HEAD(&worker->node);
1854	/* on creation a worker is in !idle && prep state */
1855	worker->flags = WORKER_PREP;
1856	}
1857	return worker;
1858	}
1859
1860	/**
1861	* worker_attach_to_pool() - attach a worker to a pool
1862	* @worker: worker to be attached
1863	* @pool: the target pool
1864	*
1865	* Attach @worker to @pool. Once attached, the %WORKER_UNBOUND flag and
1866	* cpu-binding of @worker are kept coordinated with the pool across
1867	* cpu-[un]hotplugs.
1868	*/
1869	static void worker_attach_to_pool(struct worker *worker,
1870	struct worker_pool *pool)
1871	{
1872	mutex_lock(&wq_pool_attach_mutex);
1873
1874	/*
1875	* The wq_pool_attach_mutex ensures %POOL_DISASSOCIATED remains
1876	* stable across this function. See the comments above the flag
1877	* definition for details.
1878	*/
1879	if (pool->flags & POOL_DISASSOCIATED)
1880	worker->flags \|= WORKER_UNBOUND;
1881	else
1882	kthread_set_per_cpu(worker->task, pool->cpu);
1883
1884	if (worker->rescue_wq)
1885	set_cpus_allowed_ptr(worker->task, pool->attrs->cpumask);
1886
1887	list_add_tail(&worker->node, &pool->workers);
1888	worker->pool = pool;
1889
1890	mutex_unlock(&wq_pool_attach_mutex);
1891	}
1892
1893	/**
1894	* worker_detach_from_pool() - detach a worker from its pool
1895	* @worker: worker which is attached to its pool
1896	*
1897	* Undo the attaching which had been done in worker_attach_to_pool(). The
1898	* caller worker shouldn't access to the pool after detached except it has
1899	* other reference to the pool.
1900	*/
1901	static void worker_detach_from_pool(struct worker *worker)
1902	{
1903	struct worker_pool *pool = worker->pool;
1904	struct completion *detach_completion = NULL;
1905
1906	mutex_lock(&wq_pool_attach_mutex);
1907
1908	kthread_set_per_cpu(worker->task, -1);
1909	list_del(&worker->node);
1910	worker->pool = NULL;
1911
1912	if (list_empty(&pool->workers) && list_empty(&pool->dying_workers))
1913	detach_completion = pool->detach_completion;
1914	mutex_unlock(&wq_pool_attach_mutex);
1915
1916	/* clear leftover flags without pool->lock after it is detached */
1917	worker->flags &= ~(WORKER_UNBOUND \| WORKER_REBOUND);
1918
1919	if (detach_completion)
1920	complete(detach_completion);
1921	}
1922
1923	/**
1924	* create_worker - create a new workqueue worker
1925	* @pool: pool the new worker will belong to
1926	*
1927	* Create and start a new worker which is attached to @pool.
1928	*
1929	* CONTEXT:
1930	* Might sleep. Does GFP_KERNEL allocations.
1931	*
1932	* Return:
1933	* Pointer to the newly created worker.
1934	*/
1935	static struct worker create_worker(struct worker_pool pool)
1936	{
1937	struct worker *worker;
1938	int id;
1939	char id_buf[16];
1940
1941	/* ID is needed to determine kthread name */
1942	id = ida_alloc(&pool->worker_ida, GFP_KERNEL);
1943	if (id < 0) {
1944	pr_err_once("workqueue: Failed to allocate a worker ID: %pe\n",
1945	ERR_PTR(id));
1946	return NULL;
1947	}
1948
1949	worker = alloc_worker(pool->node);
1950	if (!worker) {
1951	pr_err_once("workqueue: Failed to allocate a worker\n");
1952	goto fail;
1953	}
1954
1955	worker->id = id;
1956
1957	if (pool->cpu >= 0)
1958	snprintf(id_buf, sizeof(id_buf), "%d:%d%s", pool->cpu, id,
1959	pool->attrs->nice < 0 ? "H" : "");
1960	else
1961	snprintf(id_buf, sizeof(id_buf), "u%d:%d", pool->id, id);
1962
1963	worker->task = kthread_create_on_node(worker_thread, worker, pool->node,
1964	"kworker/%s", id_buf);
1965	if (IS_ERR(worker->task)) {
1966	if (PTR_ERR(worker->task) == -EINTR) {
1967	pr_err("workqueue: Interrupted when creating a worker thread \"kworker/%s\"\n",
1968	id_buf);
1969	} else {
1970	pr_err_once("workqueue: Failed to create a worker thread: %pe",
1971	worker->task);
1972	}
1973	goto fail;
1974	}
1975
1976	set_user_nice(worker->task, pool->attrs->nice);
1977	kthread_bind_mask(worker->task, pool->attrs->cpumask);
1978
1979	/* successful, attach the worker to the pool */
1980	worker_attach_to_pool(worker, pool);
1981
1982	/* start the newly created worker */
1983	raw_spin_lock_irq(&pool->lock);
1984	worker->pool->nr_workers++;
1985	worker_enter_idle(worker);
1986	wake_up_process(worker->task);
1987	raw_spin_unlock_irq(&pool->lock);
1988
1989	return worker;
1990
1991	fail:
1992	ida_free(&pool->worker_ida, id);
1993	kfree(worker);
1994	return NULL;
1995	}
1996
1997	static void unbind_worker(struct worker *worker)
1998	{
1999	lockdep_assert_held(&wq_pool_attach_mutex);
2000
2001	kthread_set_per_cpu(worker->task, -1);
2002	if (cpumask_intersects(wq_unbound_cpumask, cpu_active_mask))
2003	WARN_ON_ONCE(set_cpus_allowed_ptr(worker->task, wq_unbound_cpumask) < 0);
2004	else
2005	WARN_ON_ONCE(set_cpus_allowed_ptr(worker->task, cpu_possible_mask) < 0);
2006	}
2007
2008	static void wake_dying_workers(struct list_head *cull_list)
2009	{
2010	struct worker worker, tmp;
2011
2012	list_for_each_entry_safe(worker, tmp, cull_list, entry) {
2013	list_del_init(&worker->entry);
2014	unbind_worker(worker);
2015	/*
2016	* If the worker was somehow already running, then it had to be
2017	* in pool->idle_list when set_worker_dying() happened or we
2018	* wouldn't have gotten here.
2019	*
2020	* Thus, the worker must either have observed the WORKER_DIE
2021	* flag, or have set its state to TASK_IDLE. Either way, the
2022	* below will be observed by the worker and is safe to do
2023	* outside of pool->lock.
2024	*/
2025	wake_up_process(worker->task);
2026	}
2027	}
2028
2029	/**
2030	* set_worker_dying - Tag a worker for destruction
2031	* @worker: worker to be destroyed
2032	* @list: transfer worker away from its pool->idle_list and into list
2033	*
2034	* Tag @worker for destruction and adjust @pool stats accordingly. The worker
2035	* should be idle.
2036	*
2037	* CONTEXT:
2038	* raw_spin_lock_irq(pool->lock).
2039	*/
2040	static void set_worker_dying(struct worker worker, struct list_head list)
2041	{
2042	struct worker_pool *pool = worker->pool;
2043
2044	lockdep_assert_held(&pool->lock);
2045	lockdep_assert_held(&wq_pool_attach_mutex);
2046
2047	/* sanity check frenzy */
2048	if (WARN_ON(worker->current_work) \|\|
2049	WARN_ON(!list_empty(&worker->scheduled)) \|\|
2050	WARN_ON(!(worker->flags & WORKER_IDLE)))
2051	return;
2052
2053	pool->nr_workers--;
2054	pool->nr_idle--;
2055
2056	worker->flags \|= WORKER_DIE;
2057
2058	list_move(&worker->entry, list);
2059	list_move(&worker->node, &pool->dying_workers);
2060	}
2061
2062	/**
2063	* idle_worker_timeout - check if some idle workers can now be deleted.
2064	* @t: The pool's idle_timer that just expired
2065	*
2066	* The timer is armed in worker_enter_idle(). Note that it isn't disarmed in
2067	* worker_leave_idle(), as a worker flicking between idle and active while its
2068	* pool is at the too_many_workers() tipping point would cause too much timer
2069	* housekeeping overhead. Since IDLE_WORKER_TIMEOUT is long enough, we just let
2070	* it expire and re-evaluate things from there.
2071	*/
2072	static void idle_worker_timeout(struct timer_list *t)
2073	{
2074	struct worker_pool *pool = from_timer(pool, t, idle_timer);
2075	bool do_cull = false;
2076
2077	if (work_pending(&pool->idle_cull_work))
2078	return;
2079
2080	raw_spin_lock_irq(&pool->lock);
2081
2082	if (too_many_workers(pool)) {
2083	struct worker *worker;
2084	unsigned long expires;
2085
2086	/* idle_list is kept in LIFO order, check the last one */
2087	worker = list_entry(pool->idle_list.prev, struct worker, entry);
2088	expires = worker->last_active + IDLE_WORKER_TIMEOUT;
2089	do_cull = !time_before(jiffies, expires);
2090
2091	if (!do_cull)
2092	mod_timer(&pool->idle_timer, expires);
2093	}
2094	raw_spin_unlock_irq(&pool->lock);
2095
2096	if (do_cull)
2097	queue_work(system_unbound_wq, &pool->idle_cull_work);
2098	}
2099
2100	/**
2101	* idle_cull_fn - cull workers that have been idle for too long.
2102	* @work: the pool's work for handling these idle workers
2103	*
2104	* This goes through a pool's idle workers and gets rid of those that have been
2105	* idle for at least IDLE_WORKER_TIMEOUT seconds.
2106	*
2107	* We don't want to disturb isolated CPUs because of a pcpu kworker being
2108	* culled, so this also resets worker affinity. This requires a sleepable
2109	* context, hence the split between timer callback and work item.
2110	*/
2111	static void idle_cull_fn(struct work_struct *work)
2112	{
2113	struct worker_pool *pool = container_of(work, struct worker_pool, idle_cull_work);
2114	struct list_head cull_list;
2115
2116	INIT_LIST_HEAD(&cull_list);
2117	/*
2118	* Grabbing wq_pool_attach_mutex here ensures an already-running worker
2119	* cannot proceed beyong worker_detach_from_pool() in its self-destruct
2120	* path. This is required as a previously-preempted worker could run after
2121	* set_worker_dying() has happened but before wake_dying_workers() did.
2122	*/
2123	mutex_lock(&wq_pool_attach_mutex);
2124	raw_spin_lock_irq(&pool->lock);
2125
2126	while (too_many_workers(pool)) {
2127	struct worker *worker;
2128	unsigned long expires;
2129
2130	worker = list_entry(pool->idle_list.prev, struct worker, entry);
2131	expires = worker->last_active + IDLE_WORKER_TIMEOUT;
2132
2133	if (time_before(jiffies, expires)) {
2134	mod_timer(&pool->idle_timer, expires);
2135	break;
2136	}
2137
2138	set_worker_dying(worker, &cull_list);
2139	}
2140
2141	raw_spin_unlock_irq(&pool->lock);
2142	wake_dying_workers(&cull_list);
2143	mutex_unlock(&wq_pool_attach_mutex);
2144	}
2145
2146	static void send_mayday(struct work_struct *work)
2147	{
2148	struct pool_workqueue *pwq = get_work_pwq(work);
2149	struct workqueue_struct *wq = pwq->wq;
2150
2151	lockdep_assert_held(&wq_mayday_lock);
2152
2153	if (!wq->rescuer)
2154	return;
2155
2156	/* mayday mayday mayday */
2157	if (list_empty(&pwq->mayday_node)) {
2158	/*
2159	* If @pwq is for an unbound wq, its base ref may be put at
2160	* any time due to an attribute change. Pin @pwq until the
2161	* rescuer is done with it.
2162	*/
2163	get_pwq(pwq);
2164	list_add_tail(&pwq->mayday_node, &wq->maydays);
2165	wake_up_process(wq->rescuer->task);
2166	}
2167	}
2168
2169	static void pool_mayday_timeout(struct timer_list *t)
2170	{
2171	struct worker_pool *pool = from_timer(pool, t, mayday_timer);
2172	struct work_struct *work;
2173
2174	raw_spin_lock_irq(&pool->lock);
2175	raw_spin_lock(&wq_mayday_lock); /* for wq->maydays */
2176
2177	if (need_to_create_worker(pool)) {
2178	/*
2179	* We've been trying to create a new worker but
2180	* haven't been successful. We might be hitting an
2181	* allocation deadlock. Send distress signals to
2182	* rescuers.
2183	*/
2184	list_for_each_entry(work, &pool->worklist, entry)
2185	send_mayday(work);
2186	}
2187
2188	raw_spin_unlock(&wq_mayday_lock);
2189	raw_spin_unlock_irq(&pool->lock);
2190
2191	mod_timer(&pool->mayday_timer, jiffies + MAYDAY_INTERVAL);
2192	}
2193
2194	/**
2195	* maybe_create_worker - create a new worker if necessary
2196	* @pool: pool to create a new worker for
2197	*
2198	* Create a new worker for @pool if necessary. @pool is guaranteed to
2199	* have at least one idle worker on return from this function. If
2200	* creating a new worker takes longer than MAYDAY_INTERVAL, mayday is
2201	* sent to all rescuers with works scheduled on @pool to resolve
2202	* possible allocation deadlock.
2203	*
2204	* On return, need_to_create_worker() is guaranteed to be %false and
2205	* may_start_working() %true.
2206	*
2207	* LOCKING:
2208	* raw_spin_lock_irq(pool->lock) which may be released and regrabbed
2209	* multiple times. Does GFP_KERNEL allocations. Called only from
2210	* manager.
2211	*/
2212	static void maybe_create_worker(struct worker_pool *pool)
2213	__releases(&pool->lock)
2214	__acquires(&pool->lock)
2215	{
2216	restart:
2217	raw_spin_unlock_irq(&pool->lock);
2218
2219	/* if we don't make progress in MAYDAY_INITIAL_TIMEOUT, call for help */
2220	mod_timer(&pool->mayday_timer, jiffies + MAYDAY_INITIAL_TIMEOUT);
2221
2222	while (true) {
2223	if (create_worker(pool) \|\| !need_to_create_worker(pool))
2224	break;
2225
2226	schedule_timeout_interruptible(CREATE_COOLDOWN);
2227
2228	if (!need_to_create_worker(pool))
2229	break;
2230	}
2231
2232	del_timer_sync(&pool->mayday_timer);
2233	raw_spin_lock_irq(&pool->lock);
2234	/*
2235	* This is necessary even after a new worker was just successfully
2236	* created as @pool->lock was dropped and the new worker might have
2237	* already become busy.
2238	*/
2239	if (need_to_create_worker(pool))
2240	goto restart;
2241	}
2242
2243	/**
2244	* manage_workers - manage worker pool
2245	* @worker: self
2246	*
2247	* Assume the manager role and manage the worker pool @worker belongs
2248	* to. At any given time, there can be only zero or one manager per
2249	* pool. The exclusion is handled automatically by this function.
2250	*
2251	* The caller can safely start processing works on false return. On
2252	* true return, it's guaranteed that need_to_create_worker() is false
2253	* and may_start_working() is true.
2254	*
2255	* CONTEXT:
2256	* raw_spin_lock_irq(pool->lock) which may be released and regrabbed
2257	* multiple times. Does GFP_KERNEL allocations.
2258	*
2259	* Return:
2260	* %false if the pool doesn't need management and the caller can safely
2261	* start processing works, %true if management function was performed and
2262	* the conditions that the caller verified before calling the function may
2263	* no longer be true.
2264	*/
2265	static bool manage_workers(struct worker *worker)
2266	{
2267	struct worker_pool *pool = worker->pool;
2268
2269	if (pool->flags & POOL_MANAGER_ACTIVE)
2270	return false;
2271
2272	pool->flags \|= POOL_MANAGER_ACTIVE;
2273	pool->manager = worker;
2274
2275	maybe_create_worker(pool);
2276
2277	pool->manager = NULL;
2278	pool->flags &= ~POOL_MANAGER_ACTIVE;
2279	rcuwait_wake_up(&manager_wait);
2280	return true;
2281	}
2282
2283	/**
2284	* process_one_work - process single work
2285	* @worker: self
2286	* @work: work to process
2287	*
2288	* Process @work. This function contains all the logics necessary to
2289	* process a single work including synchronization against and
2290	* interaction with other workers on the same cpu, queueing and
2291	* flushing. As long as context requirement is met, any worker can
2292	* call this function to process a work.
2293	*
2294	* CONTEXT:
2295	* raw_spin_lock_irq(pool->lock) which is released and regrabbed.
2296	*/
2297	static void process_one_work(struct worker worker, struct work_struct work)
2298	__releases(&pool->lock)
2299	__acquires(&pool->lock)
2300	{
2301	struct pool_workqueue *pwq = get_work_pwq(work);
2302	struct worker_pool *pool = worker->pool;
2303	bool cpu_intensive = pwq->wq->flags & WQ_CPU_INTENSIVE;
2304	unsigned long work_data;
2305	struct worker *collision;
2306	#ifdef CONFIG_LOCKDEP
2307	/*
2308	* It is permissible to free the struct work_struct from
2309	* inside the function that is called from it, this we need to
2310	* take into account for lockdep too. To avoid bogus "held
2311	* lock freed" warnings as well as problems when looking into
2312	* work->lockdep_map, make a copy and use that here.
2313	*/
2314	struct lockdep_map lockdep_map;
2315
2316	lockdep_copy_map(&lockdep_map, &work->lockdep_map);
2317	#endif
2318	/* ensure we're on the correct CPU */
2319	WARN_ON_ONCE(!(pool->flags & POOL_DISASSOCIATED) &&
2320	raw_smp_processor_id() != pool->cpu);
2321
2322	/*
2323	* A single work shouldn't be executed concurrently by
2324	* multiple workers on a single cpu. Check whether anyone is
2325	* already processing the work. If so, defer the work to the
2326	* currently executing one.
2327	*/
2328	collision = find_worker_executing_work(pool, work);
2329	if (unlikely(collision)) {
2330	move_linked_works(work, &collision->scheduled, NULL);
2331	return;
2332	}
2333
2334	/* claim and dequeue */
2335	debug_work_deactivate(work);
2336	hash_add(pool->busy_hash, &worker->hentry, (unsigned long)work);
2337	worker->current_work = work;
2338	worker->current_func = work->func;
2339	worker->current_pwq = pwq;
2340	work_data = *work_data_bits(work);
2341	worker->current_color = get_work_color(work_data);
2342
2343	/*
2344	* Record wq name for cmdline and debug reporting, may get
2345	* overridden through set_worker_desc().
2346	*/
2347	strscpy(worker->desc, pwq->wq->name, WORKER_DESC_LEN);
2348
2349	list_del_init(&work->entry);
2350
2351	/*
2352	* CPU intensive works don't participate in concurrency management.
2353	* They're the scheduler's responsibility. This takes @worker out
2354	* of concurrency management and the next code block will chain
2355	* execution of the pending work items.
2356	*/
2357	if (unlikely(cpu_intensive))
2358	worker_set_flags(worker, WORKER_CPU_INTENSIVE);
2359
2360	/*
2361	* Wake up another worker if necessary. The condition is always
2362	* false for normal per-cpu workers since nr_running would always
2363	* be >= 1 at this point. This is used to chain execution of the
2364	* pending work items for WORKER_NOT_RUNNING workers such as the
2365	* UNBOUND and CPU_INTENSIVE ones.
2366	*/
2367	if (need_more_worker(pool))
2368	wake_up_worker(pool);
2369
2370	/*
2371	* Record the last pool and clear PENDING which should be the last
2372	* update to @work. Also, do this inside @pool->lock so that
2373	* PENDING and queued state changes happen together while IRQ is
2374	* disabled.
2375	*/
2376	set_work_pool_and_clear_pending(work, pool->id);
2377
2378	raw_spin_unlock_irq(&pool->lock);
2379
2380	lock_map_acquire(&pwq->wq->lockdep_map);
2381	lock_map_acquire(&lockdep_map);
2382	/*
2383	* Strictly speaking we should mark the invariant state without holding
2384	* any locks, that is, before these two lock_map_acquire()'s.
2385	*
2386	* However, that would result in:
2387	*
2388	* A(W1)
2389	* WFC(C)
2390	* A(W1)
2391	* C(C)
2392	*
2393	* Which would create W1->C->W1 dependencies, even though there is no
2394	* actual deadlock possible. There are two solutions, using a
2395	* read-recursive acquire on the work(queue) 'locks', but this will then
2396	* hit the lockdep limitation on recursive locks, or simply discard
2397	* these locks.
2398	*
2399	* AFAICT there is no possible deadlock scenario between the
2400	* flush_work() and complete() primitives (except for single-threaded
2401	* workqueues), so hiding them isn't a problem.
2402	*/
2403	lockdep_invariant_state(true);
2404	trace_workqueue_execute_start(work);
2405	worker->current_func(work);
2406	/*
2407	* While we must be careful to not use "work" after this, the trace
2408	* point will only record its address.
2409	*/
2410	trace_workqueue_execute_end(work, worker->current_func);
2411	lock_map_release(&lockdep_map);
2412	lock_map_release(&pwq->wq->lockdep_map);
2413
2414	if (unlikely(in_atomic() \|\| lockdep_depth(current) > 0)) {
2415	pr_err("BUG: workqueue leaked lock or atomic: %s/0x%08x/%d\n"
2416	" last function: %ps\n",
2417	current->comm, preempt_count(), task_pid_nr(current),
2418	worker->current_func);
2419	debug_show_held_locks(current);
2420	dump_stack();
2421	}
2422
2423	/*
2424	* The following prevents a kworker from hogging CPU on !PREEMPTION
2425	* kernels, where a requeueing work item waiting for something to
2426	* happen could deadlock with stop_machine as such work item could
2427	* indefinitely requeue itself while all other CPUs are trapped in
2428	* stop_machine. At the same time, report a quiescent RCU state so
2429	* the same condition doesn't freeze RCU.
2430	*/
2431	cond_resched();
2432
2433	raw_spin_lock_irq(&pool->lock);
2434
2435	/* clear cpu intensive status */
2436	if (unlikely(cpu_intensive))
2437	worker_clr_flags(worker, WORKER_CPU_INTENSIVE);
2438
2439	/* tag the worker for identification in schedule() */
2440	worker->last_func = worker->current_func;
2441
2442	/* we're done with it, release */
2443	hash_del(&worker->hentry);
2444	worker->current_work = NULL;
2445	worker->current_func = NULL;
2446	worker->current_pwq = NULL;
2447	worker->current_color = INT_MAX;
2448	pwq_dec_nr_in_flight(pwq, work_data);
2449	}
2450
2451	/**
2452	* process_scheduled_works - process scheduled works
2453	* @worker: self
2454	*
2455	* Process all scheduled works. Please note that the scheduled list
2456	* may change while processing a work, so this function repeatedly
2457	* fetches a work from the top and executes it.
2458	*
2459	* CONTEXT:
2460	* raw_spin_lock_irq(pool->lock) which may be released and regrabbed
2461	* multiple times.
2462	*/
2463	static void process_scheduled_works(struct worker *worker)
2464	{
2465	while (!list_empty(&worker->scheduled)) {
2466	struct work_struct *work = list_first_entry(&worker->scheduled,
2467	struct work_struct, entry);
2468	process_one_work(worker, work);
2469	}
2470	}
2471
2472	static void set_pf_worker(bool val)
2473	{
2474	mutex_lock(&wq_pool_attach_mutex);
2475	if (val)
2476	current->flags \|= PF_WQ_WORKER;
2477	else
2478	current->flags &= ~PF_WQ_WORKER;
2479	mutex_unlock(&wq_pool_attach_mutex);
2480	}
2481
2482	/**
2483	* worker_thread - the worker thread function
2484	* @__worker: self
2485	*
2486	* The worker thread function. All workers belong to a worker_pool -
2487	* either a per-cpu one or dynamic unbound one. These workers process all
2488	* work items regardless of their specific target workqueue. The only
2489	* exception is work items which belong to workqueues with a rescuer which
2490	* will be explained in rescuer_thread().
2491	*
2492	* Return: 0
2493	*/
2494	static int worker_thread(void *__worker)
2495	{
2496	struct worker *worker = __worker;
2497	struct worker_pool *pool = worker->pool;
2498
2499	/* tell the scheduler that this is a workqueue worker */
2500	set_pf_worker(true);
2501	woke_up:
2502	raw_spin_lock_irq(&pool->lock);
2503
2504	/* am I supposed to die? */
2505	if (unlikely(worker->flags & WORKER_DIE)) {
2506	raw_spin_unlock_irq(&pool->lock);
2507	set_pf_worker(false);
2508
2509	set_task_comm(worker->task, "kworker/dying");
2510	ida_free(&pool->worker_ida, worker->id);
2511	worker_detach_from_pool(worker);
2512	WARN_ON_ONCE(!list_empty(&worker->entry));
2513	kfree(worker);
2514	return 0;
2515	}
2516
2517	worker_leave_idle(worker);
2518	recheck:
2519	/* no more worker necessary? */
2520	if (!need_more_worker(pool))
2521	goto sleep;
2522
2523	/* do we need to manage? */
2524	if (unlikely(!may_start_working(pool)) && manage_workers(worker))
2525	goto recheck;
2526
2527	/*
2528	* ->scheduled list can only be filled while a worker is
2529	* preparing to process a work or actually processing it.
2530	* Make sure nobody diddled with it while I was sleeping.
2531	*/
2532	WARN_ON_ONCE(!list_empty(&worker->scheduled));
2533
2534	/*
2535	* Finish PREP stage. We're guaranteed to have at least one idle
2536	* worker or that someone else has already assumed the manager
2537	* role. This is where @worker starts participating in concurrency
2538	* management if applicable and concurrency management is restored
2539	* after being rebound. See rebind_workers() for details.
2540	*/
2541	worker_clr_flags(worker, WORKER_PREP \| WORKER_REBOUND);
2542
2543	do {
2544	struct work_struct *work =
2545	list_first_entry(&pool->worklist,
2546	struct work_struct, entry);
2547
2548	pool->watchdog_ts = jiffies;
2549
2550	if (likely(!(*work_data_bits(work) & WORK_STRUCT_LINKED))) {
2551	/* optimization path, not strictly necessary */
2552	process_one_work(worker, work);
2553	if (unlikely(!list_empty(&worker->scheduled)))
2554	process_scheduled_works(worker);
2555	} else {
2556	move_linked_works(work, &worker->scheduled, NULL);
2557	process_scheduled_works(worker);
2558	}
2559	} while (keep_working(pool));
2560
2561	worker_set_flags(worker, WORKER_PREP);
2562	sleep:
2563	/*
2564	* pool->lock is held and there's no work to process and no need to
2565	* manage, sleep. Workers are woken up only while holding
2566	* pool->lock or from local cpu, so setting the current state
2567	* before releasing pool->lock is enough to prevent losing any
2568	* event.
2569	*/
2570	worker_enter_idle(worker);
2571	__set_current_state(TASK_IDLE);
2572	raw_spin_unlock_irq(&pool->lock);
2573	schedule();
2574	goto woke_up;
2575	}
2576
2577	/**
2578	* rescuer_thread - the rescuer thread function
2579	* @__rescuer: self
2580	*
2581	* Workqueue rescuer thread function. There's one rescuer for each
2582	* workqueue which has WQ_MEM_RECLAIM set.
2583	*
2584	* Regular work processing on a pool may block trying to create a new
2585	* worker which uses GFP_KERNEL allocation which has slight chance of
2586	* developing into deadlock if some works currently on the same queue
2587	* need to be processed to satisfy the GFP_KERNEL allocation. This is
2588	* the problem rescuer solves.
2589	*
2590	* When such condition is possible, the pool summons rescuers of all
2591	* workqueues which have works queued on the pool and let them process
2592	* those works so that forward progress can be guaranteed.
2593	*
2594	* This should happen rarely.
2595	*
2596	* Return: 0
2597	*/
2598	static int rescuer_thread(void *__rescuer)
2599	{
2600	struct worker *rescuer = __rescuer;
2601	struct workqueue_struct *wq = rescuer->rescue_wq;
2602	struct list_head *scheduled = &rescuer->scheduled;
2603	bool should_stop;
2604
2605	set_user_nice(current, RESCUER_NICE_LEVEL);
2606
2607	/*
2608	* Mark rescuer as worker too. As WORKER_PREP is never cleared, it
2609	* doesn't participate in concurrency management.
2610	*/
2611	set_pf_worker(true);
2612	repeat:
2613	set_current_state(TASK_IDLE);
2614
2615	/*
2616	* By the time the rescuer is requested to stop, the workqueue
2617	* shouldn't have any work pending, but @wq->maydays may still have
2618	* pwq(s) queued. This can happen by non-rescuer workers consuming
2619	* all the work items before the rescuer got to them. Go through
2620	* @wq->maydays processing before acting on should_stop so that the
2621	* list is always empty on exit.
2622	*/
2623	should_stop = kthread_should_stop();
2624
2625	/* see whether any pwq is asking for help */
2626	raw_spin_lock_irq(&wq_mayday_lock);
2627
2628	while (!list_empty(&wq->maydays)) {
2629	struct pool_workqueue *pwq = list_first_entry(&wq->maydays,
2630	struct pool_workqueue, mayday_node);
2631	struct worker_pool *pool = pwq->pool;
2632	struct work_struct work, n;
2633	bool first = true;
2634
2635	__set_current_state(TASK_RUNNING);
2636	list_del_init(&pwq->mayday_node);
2637
2638	raw_spin_unlock_irq(&wq_mayday_lock);
2639
2640	worker_attach_to_pool(rescuer, pool);
2641
2642	raw_spin_lock_irq(&pool->lock);
2643
2644	/*
2645	* Slurp in all works issued via this workqueue and
2646	* process'em.
2647	*/
2648	WARN_ON_ONCE(!list_empty(scheduled));
2649	list_for_each_entry_safe(work, n, &pool->worklist, entry) {
2650	if (get_work_pwq(work) == pwq) {
2651	if (first)
2652	pool->watchdog_ts = jiffies;
2653	move_linked_works(work, scheduled, &n);
2654	}
2655	first = false;
2656	}
2657
2658	if (!list_empty(scheduled)) {
2659	process_scheduled_works(rescuer);
2660
2661	/*
2662	* The above execution of rescued work items could
2663	* have created more to rescue through
2664	* pwq_activate_first_inactive() or chained
2665	* queueing. Let's put @pwq back on mayday list so
2666	* that such back-to-back work items, which may be
2667	* being used to relieve memory pressure, don't
2668	* incur MAYDAY_INTERVAL delay inbetween.
2669	*/
2670	if (pwq->nr_active && need_to_create_worker(pool)) {
2671	raw_spin_lock(&wq_mayday_lock);
2672	/*
2673	* Queue iff we aren't racing destruction
2674	* and somebody else hasn't queued it already.
2675	*/
2676	if (wq->rescuer && list_empty(&pwq->mayday_node)) {
2677	get_pwq(pwq);
2678	list_add_tail(&pwq->mayday_node, &wq->maydays);
2679	}
2680	raw_spin_unlock(&wq_mayday_lock);
2681	}
2682	}
2683
2684	/*
2685	* Put the reference grabbed by send_mayday(). @pool won't
2686	* go away while we're still attached to it.
2687	*/
2688	put_pwq(pwq);
2689
2690	/*
2691	* Leave this pool. If need_more_worker() is %true, notify a
2692	* regular worker; otherwise, we end up with 0 concurrency
2693	* and stalling the execution.
2694	*/
2695	if (need_more_worker(pool))
2696	wake_up_worker(pool);
2697
2698	raw_spin_unlock_irq(&pool->lock);
2699
2700	worker_detach_from_pool(rescuer);
2701
2702	raw_spin_lock_irq(&wq_mayday_lock);
2703	}
2704
2705	raw_spin_unlock_irq(&wq_mayday_lock);
2706
2707	if (should_stop) {
2708	__set_current_state(TASK_RUNNING);
2709	set_pf_worker(false);
2710	return 0;
2711	}
2712
2713	/* rescuers should never participate in concurrency management */
2714	WARN_ON_ONCE(!(rescuer->flags & WORKER_NOT_RUNNING));
2715	schedule();
2716	goto repeat;
2717	}
2718
2719	/**
2720	* check_flush_dependency - check for flush dependency sanity
2721	* @target_wq: workqueue being flushed
2722	* @target_work: work item being flushed (NULL for workqueue flushes)
2723	*
2724	* %current is trying to flush the whole @target_wq or @target_work on it.
2725	* If @target_wq doesn't have %WQ_MEM_RECLAIM, verify that %current is not
2726	* reclaiming memory or running on a workqueue which doesn't have
2727	* %WQ_MEM_RECLAIM as that can break forward-progress guarantee leading to
2728	* a deadlock.
2729	*/
2730	static void check_flush_dependency(struct workqueue_struct *target_wq,
2731	struct work_struct *target_work)
2732	{
2733	work_func_t target_func = target_work ? target_work->func : NULL;
2734	struct worker *worker;
2735
2736	if (target_wq->flags & WQ_MEM_RECLAIM)
2737	return;
2738
2739	worker = current_wq_worker();
2740
2741	WARN_ONCE(current->flags & PF_MEMALLOC,
2742	"workqueue: PF_MEMALLOC task %d(%s) is flushing !WQ_MEM_RECLAIM %s:%ps",
2743	current->pid, current->comm, target_wq->name, target_func);
2744	WARN_ONCE(worker && ((worker->current_pwq->wq->flags &
2745	(WQ_MEM_RECLAIM \| __WQ_LEGACY)) == WQ_MEM_RECLAIM),
2746	"workqueue: WQ_MEM_RECLAIM %s:%ps is flushing !WQ_MEM_RECLAIM %s:%ps",
2747	worker->current_pwq->wq->name, worker->current_func,
2748	target_wq->name, target_func);
2749	}
2750
2751	struct wq_barrier {
2752	struct work_struct work;
2753	struct completion done;
2754	struct task_struct task; / purely informational */
2755	};
2756
2757	static void wq_barrier_func(struct work_struct *work)
2758	{
2759	struct wq_barrier *barr = container_of(work, struct wq_barrier, work);
2760	complete(&barr->done);
2761	}
2762
2763	/**
2764	* insert_wq_barrier - insert a barrier work
2765	* @pwq: pwq to insert barrier into
2766	* @barr: wq_barrier to insert
2767	* @target: target work to attach @barr to
2768	* @worker: worker currently executing @target, NULL if @target is not executing
2769	*
2770	* @barr is linked to @target such that @barr is completed only after
2771	* @target finishes execution. Please note that the ordering
2772	* guarantee is observed only with respect to @target and on the local
2773	* cpu.
2774	*
2775	* Currently, a queued barrier can't be canceled. This is because
2776	* try_to_grab_pending() can't determine whether the work to be
2777	* grabbed is at the head of the queue and thus can't clear LINKED
2778	* flag of the previous work while there must be a valid next work
2779	* after a work with LINKED flag set.
2780	*
2781	* Note that when @worker is non-NULL, @target may be modified
2782	* underneath us, so we can't reliably determine pwq from @target.
2783	*
2784	* CONTEXT:
2785	* raw_spin_lock_irq(pool->lock).
2786	*/
2787	static void insert_wq_barrier(struct pool_workqueue *pwq,
2788	struct wq_barrier *barr,
2789	struct work_struct target, struct worker worker)
2790	{
2791	unsigned int work_flags = 0;
2792	unsigned int work_color;
2793	struct list_head *head;
2794
2795	/*
2796	* debugobject calls are safe here even with pool->lock locked
2797	* as we know for sure that this will not trigger any of the
2798	* checks and call back into the fixup functions where we
2799	* might deadlock.
2800	*/
2801	INIT_WORK_ONSTACK(&barr->work, wq_barrier_func);
2802	__set_bit(WORK_STRUCT_PENDING_BIT, work_data_bits(&barr->work));
2803
2804	init_completion_map(&barr->done, &target->lockdep_map);
2805
2806	barr->task = current;
2807
2808	/* The barrier work item does not participate in pwq->nr_active. */
2809	work_flags \|= WORK_STRUCT_INACTIVE;
2810
2811	/*
2812	* If @target is currently being executed, schedule the
2813	* barrier to the worker; otherwise, put it after @target.
2814	*/
2815	if (worker) {
2816	head = worker->scheduled.next;
2817	work_color = worker->current_color;
2818	} else {
2819	unsigned long *bits = work_data_bits(target);
2820
2821	head = target->entry.next;
2822	/* there can already be other linked works, inherit and set */
2823	work_flags \|= *bits & WORK_STRUCT_LINKED;
2824	work_color = get_work_color(*bits);
2825	__set_bit(WORK_STRUCT_LINKED_BIT, bits);
2826	}
2827
2828	pwq->nr_in_flight[work_color]++;
2829	work_flags \|= work_color_to_flags(work_color);
2830
2831	debug_work_activate(&barr->work);
2832	insert_work(pwq, &barr->work, head, work_flags);
2833	}
2834
2835	/**
2836	* flush_workqueue_prep_pwqs - prepare pwqs for workqueue flushing
2837	* @wq: workqueue being flushed
2838	* @flush_color: new flush color, < 0 for no-op
2839	* @work_color: new work color, < 0 for no-op
2840	*
2841	* Prepare pwqs for workqueue flushing.
2842	*
2843	* If @flush_color is non-negative, flush_color on all pwqs should be
2844	* -1. If no pwq has in-flight commands at the specified color, all
2845	* pwq->flush_color's stay at -1 and %false is returned. If any pwq
2846	* has in flight commands, its pwq->flush_color is set to
2847	* @flush_color, @wq->nr_pwqs_to_flush is updated accordingly, pwq
2848	* wakeup logic is armed and %true is returned.
2849	*
2850	* The caller should have initialized @wq->first_flusher prior to
2851	* calling this function with non-negative @flush_color. If
2852	* @flush_color is negative, no flush color update is done and %false
2853	* is returned.
2854	*
2855	* If @work_color is non-negative, all pwqs should have the same
2856	* work_color which is previous to @work_color and all will be
2857	* advanced to @work_color.
2858	*
2859	* CONTEXT:
2860	* mutex_lock(wq->mutex).
2861	*
2862	* Return:
2863	* %true if @flush_color >= 0 and there's something to flush. %false
2864	* otherwise.
2865	*/
2866	static bool flush_workqueue_prep_pwqs(struct workqueue_struct *wq,
2867	int flush_color, int work_color)
2868	{
2869	bool wait = false;
2870	struct pool_workqueue *pwq;
2871
2872	if (flush_color >= 0) {
2873	WARN_ON_ONCE(atomic_read(&wq->nr_pwqs_to_flush));
2874	atomic_set(&wq->nr_pwqs_to_flush, 1);
2875	}
2876
2877	for_each_pwq(pwq, wq) {
2878	struct worker_pool *pool = pwq->pool;
2879
2880	raw_spin_lock_irq(&pool->lock);
2881
2882	if (flush_color >= 0) {
2883	WARN_ON_ONCE(pwq->flush_color != -1);
2884
2885	if (pwq->nr_in_flight[flush_color]) {
2886	pwq->flush_color = flush_color;
2887	atomic_inc(&wq->nr_pwqs_to_flush);
2888	wait = true;
2889	}
2890	}
2891
2892	if (work_color >= 0) {
2893	WARN_ON_ONCE(work_color != work_next_color(pwq->work_color));
2894	pwq->work_color = work_color;
2895	}
2896
2897	raw_spin_unlock_irq(&pool->lock);
2898	}
2899
2900	if (flush_color >= 0 && atomic_dec_and_test(&wq->nr_pwqs_to_flush))
2901	complete(&wq->first_flusher->done);
2902
2903	return wait;
2904	}
2905
2906	/**
2907	* __flush_workqueue - ensure that any scheduled work has run to completion.
2908	* @wq: workqueue to flush
2909	*
2910	* This function sleeps until all work items which were queued on entry
2911	* have finished execution, but it is not livelocked by new incoming ones.
2912	*/
2913	void __flush_workqueue(struct workqueue_struct *wq)
2914	{
2915	struct wq_flusher this_flusher = {
2916	.list = LIST_HEAD_INIT(this_flusher.list),
2917	.flush_color = -1,
2918	.done = COMPLETION_INITIALIZER_ONSTACK_MAP(this_flusher.done, wq->lockdep_map),
2919	};
2920	int next_color;
2921
2922	if (WARN_ON(!wq_online))
2923	return;
2924
2925	lock_map_acquire(&wq->lockdep_map);
2926	lock_map_release(&wq->lockdep_map);
2927
2928	mutex_lock(&wq->mutex);
2929
2930	/*
2931	* Start-to-wait phase
2932	*/
2933	next_color = work_next_color(wq->work_color);
2934
2935	if (next_color != wq->flush_color) {
2936	/*
2937	* Color space is not full. The current work_color
2938	* becomes our flush_color and work_color is advanced
2939	* by one.
2940	*/
2941	WARN_ON_ONCE(!list_empty(&wq->flusher_overflow));
2942	this_flusher.flush_color = wq->work_color;
2943	wq->work_color = next_color;
2944
2945	if (!wq->first_flusher) {
2946	/* no flush in progress, become the first flusher */
2947	WARN_ON_ONCE(wq->flush_color != this_flusher.flush_color);
2948
2949	wq->first_flusher = &this_flusher;
2950
2951	if (!flush_workqueue_prep_pwqs(wq, wq->flush_color,
2952	wq->work_color)) {
2953	/* nothing to flush, done */
2954	wq->flush_color = next_color;
2955	wq->first_flusher = NULL;
2956	goto out_unlock;
2957	}
2958	} else {
2959	/* wait in queue */
2960	WARN_ON_ONCE(wq->flush_color == this_flusher.flush_color);
2961	list_add_tail(&this_flusher.list, &wq->flusher_queue);
2962	flush_workqueue_prep_pwqs(wq, -1, wq->work_color);
2963	}
2964	} else {
2965	/*
2966	* Oops, color space is full, wait on overflow queue.
2967	* The next flush completion will assign us
2968	* flush_color and transfer to flusher_queue.
2969	*/
2970	list_add_tail(&this_flusher.list, &wq->flusher_overflow);
2971	}
2972
2973	check_flush_dependency(wq, NULL);
2974
2975	mutex_unlock(&wq->mutex);
2976
2977	wait_for_completion(&this_flusher.done);
2978
2979	/*
2980	* Wake-up-and-cascade phase
2981	*
2982	* First flushers are responsible for cascading flushes and
2983	* handling overflow. Non-first flushers can simply return.
2984	*/
2985	if (READ_ONCE(wq->first_flusher) != &this_flusher)
2986	return;
2987
2988	mutex_lock(&wq->mutex);
2989
2990	/* we might have raced, check again with mutex held */
2991	if (wq->first_flusher != &this_flusher)
2992	goto out_unlock;
2993
2994	WRITE_ONCE(wq->first_flusher, NULL);
2995
2996	WARN_ON_ONCE(!list_empty(&this_flusher.list));
2997	WARN_ON_ONCE(wq->flush_color != this_flusher.flush_color);
2998
2999	while (true) {
3000	struct wq_flusher next, tmp;
3001
3002	/* complete all the flushers sharing the current flush color */
3003	list_for_each_entry_safe(next, tmp, &wq->flusher_queue, list) {
3004	if (next->flush_color != wq->flush_color)
3005	break;
3006	list_del_init(&next->list);
3007	complete(&next->done);
3008	}
3009
3010	WARN_ON_ONCE(!list_empty(&wq->flusher_overflow) &&
3011	wq->flush_color != work_next_color(wq->work_color));
3012
3013	/* this flush_color is finished, advance by one */
3014	wq->flush_color = work_next_color(wq->flush_color);
3015
3016	/* one color has been freed, handle overflow queue */
3017	if (!list_empty(&wq->flusher_overflow)) {
3018	/*
3019	* Assign the same color to all overflowed
3020	* flushers, advance work_color and append to
3021	* flusher_queue. This is the start-to-wait
3022	* phase for these overflowed flushers.
3023	*/
3024	list_for_each_entry(tmp, &wq->flusher_overflow, list)
3025	tmp->flush_color = wq->work_color;
3026
3027	wq->work_color = work_next_color(wq->work_color);
3028
3029	list_splice_tail_init(&wq->flusher_overflow,
3030	&wq->flusher_queue);
3031	flush_workqueue_prep_pwqs(wq, -1, wq->work_color);
3032	}
3033
3034	if (list_empty(&wq->flusher_queue)) {
3035	WARN_ON_ONCE(wq->flush_color != wq->work_color);
3036	break;
3037	}
3038
3039	/*
3040	* Need to flush more colors. Make the next flusher
3041	* the new first flusher and arm pwqs.
3042	*/
3043	WARN_ON_ONCE(wq->flush_color == wq->work_color);
3044	WARN_ON_ONCE(wq->flush_color != next->flush_color);
3045
3046	list_del_init(&next->list);
3047	wq->first_flusher = next;
3048
3049	if (flush_workqueue_prep_pwqs(wq, wq->flush_color, -1))
3050	break;
3051
3052	/*
3053	* Meh... this color is already done, clear first
3054	* flusher and repeat cascading.
3055	*/
3056	wq->first_flusher = NULL;
3057	}
3058
3059	out_unlock:
3060	mutex_unlock(&wq->mutex);
3061	}
3062	EXPORT_SYMBOL(__flush_workqueue);
3063
3064	/**
3065	* drain_workqueue - drain a workqueue
3066	* @wq: workqueue to drain
3067	*
3068	* Wait until the workqueue becomes empty. While draining is in progress,
3069	* only chain queueing is allowed. IOW, only currently pending or running
3070	* work items on @wq can queue further work items on it. @wq is flushed
3071	* repeatedly until it becomes empty. The number of flushing is determined
3072	* by the depth of chaining and should be relatively short. Whine if it
3073	* takes too long.
3074	*/
3075	void drain_workqueue(struct workqueue_struct *wq)
3076	{
3077	unsigned int flush_cnt = 0;
3078	struct pool_workqueue *pwq;
3079
3080	/*
3081	* __queue_work() needs to test whether there are drainers, is much
3082	* hotter than drain_workqueue() and already looks at @wq->flags.
3083	* Use __WQ_DRAINING so that queue doesn't have to check nr_drainers.
3084	*/
3085	mutex_lock(&wq->mutex);
3086	if (!wq->nr_drainers++)
3087	wq->flags \|= __WQ_DRAINING;
3088	mutex_unlock(&wq->mutex);
3089	reflush:
3090	__flush_workqueue(wq);
3091
3092	mutex_lock(&wq->mutex);
3093
3094	for_each_pwq(pwq, wq) {
3095	bool drained;
3096
3097	raw_spin_lock_irq(&pwq->pool->lock);
3098	drained = !pwq->nr_active && list_empty(&pwq->inactive_works);
3099	raw_spin_unlock_irq(&pwq->pool->lock);
3100
3101	if (drained)
3102	continue;
3103
3104	if (++flush_cnt == 10 \|\|
3105	(flush_cnt % 100 == 0 && flush_cnt <= 1000))
3106	pr_warn("workqueue %s: %s() isn't complete after %u tries\n",
3107	wq->name, __func__, flush_cnt);
3108
3109	mutex_unlock(&wq->mutex);
3110	goto reflush;
3111	}
3112
3113	if (!--wq->nr_drainers)
3114	wq->flags &= ~__WQ_DRAINING;
3115	mutex_unlock(&wq->mutex);
3116	}
3117	EXPORT_SYMBOL_GPL(drain_workqueue);
3118
3119	static bool start_flush_work(struct work_struct work, struct wq_barrier barr,
3120	bool from_cancel)
3121	{
3122	struct worker *worker = NULL;
3123	struct worker_pool *pool;
3124	struct pool_workqueue *pwq;
3125
3126	might_sleep();
3127
3128	rcu_read_lock();
3129	pool = get_work_pool(work);
3130	if (!pool) {
3131	rcu_read_unlock();
3132	return false;
3133	}
3134
3135	raw_spin_lock_irq(&pool->lock);
3136	/* see the comment in try_to_grab_pending() with the same code */
3137	pwq = get_work_pwq(work);
3138	if (pwq) {
3139	if (unlikely(pwq->pool != pool))
3140	goto already_gone;
3141	} else {
3142	worker = find_worker_executing_work(pool, work);
3143	if (!worker)
3144	goto already_gone;
3145	pwq = worker->current_pwq;
3146	}
3147
3148	check_flush_dependency(pwq->wq, work);
3149
3150	insert_wq_barrier(pwq, barr, work, worker);
3151	raw_spin_unlock_irq(&pool->lock);
3152
3153	/*
3154	* Force a lock recursion deadlock when using flush_work() inside a
3155	* single-threaded or rescuer equipped workqueue.
3156	*
3157	* For single threaded workqueues the deadlock happens when the work
3158	* is after the work issuing the flush_work(). For rescuer equipped
3159	* workqueues the deadlock happens when the rescuer stalls, blocking
3160	* forward progress.
3161	*/
3162	if (!from_cancel &&
3163	(pwq->wq->saved_max_active == 1 \|\| pwq->wq->rescuer)) {
3164	lock_map_acquire(&pwq->wq->lockdep_map);
3165	lock_map_release(&pwq->wq->lockdep_map);
3166	}
3167	rcu_read_unlock();
3168	return true;
3169	already_gone:
3170	raw_spin_unlock_irq(&pool->lock);
3171	rcu_read_unlock();
3172	return false;
3173	}
3174
3175	static bool __flush_work(struct work_struct *work, bool from_cancel)
3176	{
3177	struct wq_barrier barr;
3178
3179	if (WARN_ON(!wq_online))
3180	return false;
3181
3182	if (WARN_ON(!work->func))
3183	return false;
3184
3185	lock_map_acquire(&work->lockdep_map);
3186	lock_map_release(&work->lockdep_map);
3187
3188	if (start_flush_work(work, &barr, from_cancel)) {
3189	wait_for_completion(&barr.done);
3190	destroy_work_on_stack(&barr.work);
3191	return true;
3192	} else {
3193	return false;
3194	}
3195	}
3196
3197	/**
3198	* flush_work - wait for a work to finish executing the last queueing instance
3199	* @work: the work to flush
3200	*
3201	* Wait until @work has finished execution. @work is guaranteed to be idle
3202	* on return if it hasn't been requeued since flush started.
3203	*
3204	* Return:
3205	* %true if flush_work() waited for the work to finish execution,
3206	* %false if it was already idle.
3207	*/
3208	bool flush_work(struct work_struct *work)
3209	{
3210	return __flush_work(work, false);
3211	}
3212	EXPORT_SYMBOL_GPL(flush_work);
3213
3214	struct cwt_wait {
3215	wait_queue_entry_t wait;
3216	struct work_struct *work;
3217	};
3218
3219	static int cwt_wakefn(wait_queue_entry_t wait, unsigned mode, int sync, void key)
3220	{
3221	struct cwt_wait *cwait = container_of(wait, struct cwt_wait, wait);
3222
3223	if (cwait->work != key)
3224	return 0;
3225	return autoremove_wake_function(wait, mode, sync, key);
3226	}
3227
3228	static bool __cancel_work_timer(struct work_struct *work, bool is_dwork)
3229	{
3230	static DECLARE_WAIT_QUEUE_HEAD(cancel_waitq);
3231	unsigned long flags;
3232	int ret;
3233
3234	do {
3235	ret = try_to_grab_pending(work, is_dwork, &flags);
3236	/*
3237	* If someone else is already canceling, wait for it to
3238	* finish. flush_work() doesn't work for PREEMPT_NONE
3239	* because we may get scheduled between @work's completion
3240	* and the other canceling task resuming and clearing
3241	* CANCELING - flush_work() will return false immediately
3242	* as @work is no longer busy, try_to_grab_pending() will
3243	* return -ENOENT as @work is still being canceled and the
3244	* other canceling task won't be able to clear CANCELING as
3245	* we're hogging the CPU.
3246	*
3247	* Let's wait for completion using a waitqueue. As this
3248	* may lead to the thundering herd problem, use a custom
3249	* wake function which matches @work along with exclusive
3250	* wait and wakeup.
3251	*/
3252	if (unlikely(ret == -ENOENT)) {
3253	struct cwt_wait cwait;
3254
3255	init_wait(&cwait.wait);
3256	cwait.wait.func = cwt_wakefn;
3257	cwait.work = work;
3258
3259	prepare_to_wait_exclusive(&cancel_waitq, &cwait.wait,
3260	TASK_UNINTERRUPTIBLE);
3261	if (work_is_canceling(work))
3262	schedule();
3263	finish_wait(&cancel_waitq, &cwait.wait);
3264	}
3265	} while (unlikely(ret < 0));
3266
3267	/* tell other tasks trying to grab @work to back off */
3268	mark_work_canceling(work);
3269	local_irq_restore(flags);
3270
3271	/*
3272	* This allows canceling during early boot. We know that @work
3273	* isn't executing.
3274	*/
3275	if (wq_online)
3276	__flush_work(work, true);
3277
3278	clear_work_data(work);
3279
3280	/*
3281	* Paired with prepare_to_wait() above so that either
3282	* waitqueue_active() is visible here or !work_is_canceling() is
3283	* visible there.
3284	*/
3285	smp_mb();
3286	if (waitqueue_active(&cancel_waitq))
3287	__wake_up(&cancel_waitq, TASK_NORMAL, 1, work);
3288
3289	return ret;
3290	}
3291
3292	/**
3293	* cancel_work_sync - cancel a work and wait for it to finish
3294	* @work: the work to cancel
3295	*
3296	* Cancel @work and wait for its execution to finish. This function
3297	* can be used even if the work re-queues itself or migrates to
3298	* another workqueue. On return from this function, @work is
3299	* guaranteed to be not pending or executing on any CPU.
3300	*
3301	* cancel_work_sync(&delayed_work->work) must not be used for
3302	* delayed_work's. Use cancel_delayed_work_sync() instead.
3303	*
3304	* The caller must ensure that the workqueue on which @work was last
3305	* queued can't be destroyed before this function returns.
3306	*
3307	* Return:
3308	* %true if @work was pending, %false otherwise.
3309	*/
3310	bool cancel_work_sync(struct work_struct *work)
3311	{
3312	return __cancel_work_timer(work, false);
3313	}
3314	EXPORT_SYMBOL_GPL(cancel_work_sync);
3315
3316	/**
3317	* flush_delayed_work - wait for a dwork to finish executing the last queueing
3318	* @dwork: the delayed work to flush
3319	*
3320	* Delayed timer is cancelled and the pending work is queued for
3321	* immediate execution. Like flush_work(), this function only
3322	* considers the last queueing instance of @dwork.
3323	*
3324	* Return:
3325	* %true if flush_work() waited for the work to finish execution,
3326	* %false if it was already idle.
3327	*/
3328	bool flush_delayed_work(struct delayed_work *dwork)
3329	{
3330	local_irq_disable();
3331	if (del_timer_sync(&dwork->timer))
3332	__queue_work(dwork->cpu, dwork->wq, &dwork->work);
3333	local_irq_enable();
3334	return flush_work(&dwork->work);
3335	}
3336	EXPORT_SYMBOL(flush_delayed_work);
3337
3338	/**
3339	* flush_rcu_work - wait for a rwork to finish executing the last queueing
3340	* @rwork: the rcu work to flush
3341	*
3342	* Return:
3343	* %true if flush_rcu_work() waited for the work to finish execution,
3344	* %false if it was already idle.
3345	*/
3346	bool flush_rcu_work(struct rcu_work *rwork)
3347	{
3348	if (test_bit(WORK_STRUCT_PENDING_BIT, work_data_bits(&rwork->work))) {
3349	rcu_barrier();
3350	flush_work(&rwork->work);
3351	return true;
3352	} else {
3353	return flush_work(&rwork->work);
3354	}
3355	}
3356	EXPORT_SYMBOL(flush_rcu_work);
3357
3358	static bool __cancel_work(struct work_struct *work, bool is_dwork)
3359	{
3360	unsigned long flags;
3361	int ret;
3362
3363	do {
3364	ret = try_to_grab_pending(work, is_dwork, &flags);
3365	} while (unlikely(ret == -EAGAIN));
3366
3367	if (unlikely(ret < 0))
3368	return false;
3369
3370	set_work_pool_and_clear_pending(work, get_work_pool_id(work));
3371	local_irq_restore(flags);
3372	return ret;
3373	}
3374
3375	/*
3376	* See cancel_delayed_work()
3377	*/
3378	bool cancel_work(struct work_struct *work)
3379	{
3380	return __cancel_work(work, false);
3381	}
3382	EXPORT_SYMBOL(cancel_work);
3383
3384	/**
3385	* cancel_delayed_work - cancel a delayed work
3386	* @dwork: delayed_work to cancel
3387	*
3388	* Kill off a pending delayed_work.
3389	*
3390	* Return: %true if @dwork was pending and canceled; %false if it wasn't
3391	* pending.
3392	*
3393	* Note:
3394	* The work callback function may still be running on return, unless
3395	* it returns %true and the work doesn't re-arm itself. Explicitly flush or
3396	* use cancel_delayed_work_sync() to wait on it.
3397	*
3398	* This function is safe to call from any context including IRQ handler.
3399	*/
3400	bool cancel_delayed_work(struct delayed_work *dwork)
3401	{
3402	return __cancel_work(&dwork->work, true);
3403	}
3404	EXPORT_SYMBOL(cancel_delayed_work);
3405
3406	/**
3407	* cancel_delayed_work_sync - cancel a delayed work and wait for it to finish
3408	* @dwork: the delayed work cancel
3409	*
3410	* This is cancel_work_sync() for delayed works.
3411	*
3412	* Return:
3413	* %true if @dwork was pending, %false otherwise.
3414	*/
3415	bool cancel_delayed_work_sync(struct delayed_work *dwork)
3416	{
3417	return __cancel_work_timer(&dwork->work, true);
3418	}
3419	EXPORT_SYMBOL(cancel_delayed_work_sync);
3420
3421	/**
3422	* schedule_on_each_cpu - execute a function synchronously on each online CPU
3423	* @func: the function to call
3424	*
3425	* schedule_on_each_cpu() executes @func on each online CPU using the
3426	* system workqueue and blocks until all CPUs have completed.
3427	* schedule_on_each_cpu() is very slow.
3428	*
3429	* Return:
3430	* 0 on success, -errno on failure.
3431	*/
3432	int schedule_on_each_cpu(work_func_t func)
3433	{
3434	int cpu;
3435	struct work_struct __percpu *works;
3436
3437	works = alloc_percpu(struct work_struct);
3438	if (!works)
3439	return -ENOMEM;
3440
3441	cpus_read_lock();
3442
3443	for_each_online_cpu(cpu) {
3444	struct work_struct *work = per_cpu_ptr(works, cpu);
3445
3446	INIT_WORK(work, func);
3447	schedule_work_on(cpu, work);
3448	}
3449
3450	for_each_online_cpu(cpu)
3451	flush_work(per_cpu_ptr(works, cpu));
3452
3453	cpus_read_unlock();
3454	free_percpu(works);
3455	return 0;
3456	}
3457
3458	/**
3459	* execute_in_process_context - reliably execute the routine with user context
3460	* @fn: the function to execute
3461	* @ew: guaranteed storage for the execute work structure (must
3462	* be available when the work executes)
3463	*
3464	* Executes the function immediately if process context is available,
3465	* otherwise schedules the function for delayed execution.
3466	*
3467	* Return: 0 - function was executed
3468	* 1 - function was scheduled for execution
3469	*/
3470	int execute_in_process_context(work_func_t fn, struct execute_work *ew)
3471	{
3472	if (!in_interrupt()) {
3473	fn(&ew->work);
3474	return 0;
3475	}
3476
3477	INIT_WORK(&ew->work, fn);
3478	schedule_work(&ew->work);
3479
3480	return 1;
3481	}
3482	EXPORT_SYMBOL_GPL(execute_in_process_context);
3483
3484	/**
3485	* free_workqueue_attrs - free a workqueue_attrs
3486	* @attrs: workqueue_attrs to free
3487	*
3488	* Undo alloc_workqueue_attrs().
3489	*/
3490	void free_workqueue_attrs(struct workqueue_attrs *attrs)
3491	{
3492	if (attrs) {
3493	free_cpumask_var(attrs->cpumask);
3494	kfree(attrs);
3495	}
3496	}
3497
3498	/**
3499	* alloc_workqueue_attrs - allocate a workqueue_attrs
3500	*
3501	* Allocate a new workqueue_attrs, initialize with default settings and
3502	* return it.
3503	*
3504	* Return: The allocated new workqueue_attr on success. %NULL on failure.
3505	*/
3506	struct workqueue_attrs *alloc_workqueue_attrs(void)
3507	{
3508	struct workqueue_attrs *attrs;
3509
3510	attrs = kzalloc(sizeof(*attrs), GFP_KERNEL);
3511	if (!attrs)
3512	goto fail;
3513	if (!alloc_cpumask_var(&attrs->cpumask, GFP_KERNEL))
3514	goto fail;
3515
3516	cpumask_copy(attrs->cpumask, cpu_possible_mask);
3517	return attrs;
3518	fail:
3519	free_workqueue_attrs(attrs);
3520	return NULL;
3521	}
3522
3523	static void copy_workqueue_attrs(struct workqueue_attrs *to,
3524	const struct workqueue_attrs *from)
3525	{
3526	to->nice = from->nice;
3527	cpumask_copy(to->cpumask, from->cpumask);
3528	/*
3529	* Unlike hash and equality test, this function doesn't ignore
3530	* ->no_numa as it is used for both pool and wq attrs. Instead,
3531	* get_unbound_pool() explicitly clears ->no_numa after copying.
3532	*/
3533	to->no_numa = from->no_numa;
3534	}
3535
3536	/* hash value of the content of @attr */
3537	static u32 wqattrs_hash(const struct workqueue_attrs *attrs)
3538	{
3539	u32 hash = 0;
3540
3541	hash = jhash_1word(attrs->nice, hash);
3542	hash = jhash(cpumask_bits(attrs->cpumask),
3543	BITS_TO_LONGS(nr_cpumask_bits) * sizeof(long), hash);
3544	return hash;
3545	}
3546
3547	/* content equality test */
3548	static bool wqattrs_equal(const struct workqueue_attrs *a,
3549	const struct workqueue_attrs *b)
3550	{
3551	if (a->nice != b->nice)
3552	return false;
3553	if (!cpumask_equal(a->cpumask, b->cpumask))
3554	return false;
3555	return true;
3556	}
3557
3558	/**
3559	* init_worker_pool - initialize a newly zalloc'd worker_pool
3560	* @pool: worker_pool to initialize
3561	*
3562	* Initialize a newly zalloc'd @pool. It also allocates @pool->attrs.
3563	*
3564	* Return: 0 on success, -errno on failure. Even on failure, all fields
3565	* inside @pool proper are initialized and put_unbound_pool() can be called
3566	* on @pool safely to release it.
3567	*/
3568	static int init_worker_pool(struct worker_pool *pool)
3569	{
3570	raw_spin_lock_init(&pool->lock);
3571	pool->id = -1;
3572	pool->cpu = -1;
3573	pool->node = NUMA_NO_NODE;
3574	pool->flags \|= POOL_DISASSOCIATED;
3575	pool->watchdog_ts = jiffies;
3576	INIT_LIST_HEAD(&pool->worklist);
3577	INIT_LIST_HEAD(&pool->idle_list);
3578	hash_init(pool->busy_hash);
3579
3580	timer_setup(&pool->idle_timer, idle_worker_timeout, TIMER_DEFERRABLE);
3581	INIT_WORK(&pool->idle_cull_work, idle_cull_fn);
3582
3583	timer_setup(&pool->mayday_timer, pool_mayday_timeout, 0);
3584
3585	INIT_LIST_HEAD(&pool->workers);
3586	INIT_LIST_HEAD(&pool->dying_workers);
3587
3588	ida_init(&pool->worker_ida);
3589	INIT_HLIST_NODE(&pool->hash_node);
3590	pool->refcnt = 1;
3591
3592	/* shouldn't fail above this point */
3593	pool->attrs = alloc_workqueue_attrs();
3594	if (!pool->attrs)
3595	return -ENOMEM;
3596	return 0;
3597	}
3598
3599	#ifdef CONFIG_LOCKDEP
3600	static void wq_init_lockdep(struct workqueue_struct *wq)
3601	{
3602	char *lock_name;
3603
3604	lockdep_register_key(&wq->key);
3605	lock_name = kasprintf(GFP_KERNEL, "%s%s", "(wq_completion)", wq->name);
3606	if (!lock_name)
3607	lock_name = wq->name;
3608
3609	wq->lock_name = lock_name;
3610	lockdep_init_map(&wq->lockdep_map, lock_name, &wq->key, 0);
3611	}
3612
3613	static void wq_unregister_lockdep(struct workqueue_struct *wq)
3614	{
3615	lockdep_unregister_key(&wq->key);
3616	}
3617
3618	static void wq_free_lockdep(struct workqueue_struct *wq)
3619	{
3620	if (wq->lock_name != wq->name)
3621	kfree(wq->lock_name);
3622	}
3623	#else
3624	static void wq_init_lockdep(struct workqueue_struct *wq)
3625	{
3626	}
3627
3628	static void wq_unregister_lockdep(struct workqueue_struct *wq)
3629	{
3630	}
3631
3632	static void wq_free_lockdep(struct workqueue_struct *wq)
3633	{
3634	}
3635	#endif
3636
3637	static void rcu_free_wq(struct rcu_head *rcu)
3638	{
3639	struct workqueue_struct *wq =
3640	container_of(rcu, struct workqueue_struct, rcu);
3641
3642	wq_free_lockdep(wq);
3643
3644	if (!(wq->flags & WQ_UNBOUND))
3645	free_percpu(wq->cpu_pwqs);
3646	else
3647	free_workqueue_attrs(wq->unbound_attrs);
3648
3649	kfree(wq);
3650	}
3651
3652	static void rcu_free_pool(struct rcu_head *rcu)
3653	{
3654	struct worker_pool *pool = container_of(rcu, struct worker_pool, rcu);
3655
3656	ida_destroy(&pool->worker_ida);
3657	free_workqueue_attrs(pool->attrs);
3658	kfree(pool);
3659	}
3660
3661	/**
3662	* put_unbound_pool - put a worker_pool
3663	* @pool: worker_pool to put
3664	*
3665	* Put @pool. If its refcnt reaches zero, it gets destroyed in RCU
3666	* safe manner. get_unbound_pool() calls this function on its failure path
3667	* and this function should be able to release pools which went through,
3668	* successfully or not, init_worker_pool().
3669	*
3670	* Should be called with wq_pool_mutex held.
3671	*/
3672	static void put_unbound_pool(struct worker_pool *pool)
3673	{
3674	DECLARE_COMPLETION_ONSTACK(detach_completion);
3675	struct list_head cull_list;
3676	struct worker *worker;
3677
3678	INIT_LIST_HEAD(&cull_list);
3679
3680	lockdep_assert_held(&wq_pool_mutex);
3681
3682	if (--pool->refcnt)
3683	return;
3684
3685	/* sanity checks */
3686	if (WARN_ON(!(pool->cpu < 0)) \|\|
3687	WARN_ON(!list_empty(&pool->worklist)))
3688	return;
3689
3690	/* release id and unhash */
3691	if (pool->id >= 0)
3692	idr_remove(&worker_pool_idr, pool->id);
3693	hash_del(&pool->hash_node);
3694
3695	/*
3696	* Become the manager and destroy all workers. This prevents
3697	* @pool's workers from blocking on attach_mutex. We're the last
3698	* manager and @pool gets freed with the flag set.
3699	*
3700	* Having a concurrent manager is quite unlikely to happen as we can
3701	* only get here with
3702	* pwq->refcnt == pool->refcnt == 0
3703	* which implies no work queued to the pool, which implies no worker can
3704	* become the manager. However a worker could have taken the role of
3705	* manager before the refcnts dropped to 0, since maybe_create_worker()
3706	* drops pool->lock
3707	*/
3708	while (true) {
3709	rcuwait_wait_event(&manager_wait,
3710	!(pool->flags & POOL_MANAGER_ACTIVE),
3711	TASK_UNINTERRUPTIBLE);
3712
3713	mutex_lock(&wq_pool_attach_mutex);
3714	raw_spin_lock_irq(&pool->lock);
3715	if (!(pool->flags & POOL_MANAGER_ACTIVE)) {
3716	pool->flags \|= POOL_MANAGER_ACTIVE;
3717	break;
3718	}
3719	raw_spin_unlock_irq(&pool->lock);
3720	mutex_unlock(&wq_pool_attach_mutex);
3721	}
3722
3723	while ((worker = first_idle_worker(pool)))
3724	set_worker_dying(worker, &cull_list);
3725	WARN_ON(pool->nr_workers \|\| pool->nr_idle);
3726	raw_spin_unlock_irq(&pool->lock);
3727
3728	wake_dying_workers(&cull_list);
3729
3730	if (!list_empty(&pool->workers) \|\| !list_empty(&pool->dying_workers))
3731	pool->detach_completion = &detach_completion;
3732	mutex_unlock(&wq_pool_attach_mutex);
3733
3734	if (pool->detach_completion)
3735	wait_for_completion(pool->detach_completion);
3736
3737	/* shut down the timers */
3738	del_timer_sync(&pool->idle_timer);
3739	cancel_work_sync(&pool->idle_cull_work);
3740	del_timer_sync(&pool->mayday_timer);
3741
3742	/* RCU protected to allow dereferences from get_work_pool() */
3743	call_rcu(&pool->rcu, rcu_free_pool);
3744	}
3745
3746	/**
3747	* get_unbound_pool - get a worker_pool with the specified attributes
3748	* @attrs: the attributes of the worker_pool to get
3749	*
3750	* Obtain a worker_pool which has the same attributes as @attrs, bump the
3751	* reference count and return it. If there already is a matching
3752	* worker_pool, it will be used; otherwise, this function attempts to
3753	* create a new one.
3754	*
3755	* Should be called with wq_pool_mutex held.
3756	*
3757	* Return: On success, a worker_pool with the same attributes as @attrs.
3758	* On failure, %NULL.
3759	*/
3760	static struct worker_pool get_unbound_pool(const struct workqueue_attrs attrs)
3761	{
3762	u32 hash = wqattrs_hash(attrs);
3763	struct worker_pool *pool;
3764	int node;
3765	int target_node = NUMA_NO_NODE;
3766
3767	lockdep_assert_held(&wq_pool_mutex);
3768
3769	/* do we already have a matching pool? */
3770	hash_for_each_possible(unbound_pool_hash, pool, hash_node, hash) {
3771	if (wqattrs_equal(pool->attrs, attrs)) {
3772	pool->refcnt++;
3773	return pool;
3774	}
3775	}
3776
3777	/* if cpumask is contained inside a NUMA node, we belong to that node */
3778	if (wq_numa_enabled) {
3779	for_each_node(node) {
3780	if (cpumask_subset(attrs->cpumask,
3781	wq_numa_possible_cpumask[node])) {
3782	target_node = node;
3783	break;
3784	}
3785	}
3786	}
3787
3788	/* nope, create a new one */
3789	pool = kzalloc_node(sizeof(*pool), GFP_KERNEL, target_node);
3790	if (!pool \|\| init_worker_pool(pool) < 0)
3791	goto fail;
3792
3793	lockdep_set_subclass(&pool->lock, 1); /* see put_pwq() */
3794	copy_workqueue_attrs(pool->attrs, attrs);
3795	pool->node = target_node;
3796
3797	/*
3798	* no_numa isn't a worker_pool attribute, always clear it. See
3799	* 'struct workqueue_attrs' comments for detail.
3800	*/
3801	pool->attrs->no_numa = false;
3802
3803	if (worker_pool_assign_id(pool) < 0)
3804	goto fail;
3805
3806	/* create and start the initial worker */
3807	if (wq_online && !create_worker(pool))
3808	goto fail;
3809
3810	/* install */
3811	hash_add(unbound_pool_hash, &pool->hash_node, hash);
3812
3813	return pool;
3814	fail:
3815	if (pool)
3816	put_unbound_pool(pool);
3817	return NULL;
3818	}
3819
3820	static void rcu_free_pwq(struct rcu_head *rcu)
3821	{
3822	kmem_cache_free(pwq_cache,
3823	container_of(rcu, struct pool_workqueue, rcu));
3824	}
3825
3826	/*
3827	* Scheduled on system_wq by put_pwq() when an unbound pwq hits zero refcnt
3828	* and needs to be destroyed.
3829	*/
3830	static void pwq_unbound_release_workfn(struct work_struct *work)
3831	{
3832	struct pool_workqueue *pwq = container_of(work, struct pool_workqueue,
3833	unbound_release_work);
3834	struct workqueue_struct *wq = pwq->wq;
3835	struct worker_pool *pool = pwq->pool;
3836	bool is_last = false;
3837
3838	/*
3839	* when @pwq is not linked, it doesn't hold any reference to the
3840	* @wq, and @wq is invalid to access.
3841	*/
3842	if (!list_empty(&pwq->pwqs_node)) {
3843	if (WARN_ON_ONCE(!(wq->flags & WQ_UNBOUND)))
3844	return;
3845
3846	mutex_lock(&wq->mutex);
3847	list_del_rcu(&pwq->pwqs_node);
3848	is_last = list_empty(&wq->pwqs);
3849	mutex_unlock(&wq->mutex);
3850	}
3851
3852	mutex_lock(&wq_pool_mutex);
3853	put_unbound_pool(pool);
3854	mutex_unlock(&wq_pool_mutex);
3855
3856	call_rcu(&pwq->rcu, rcu_free_pwq);
3857
3858	/*
3859	* If we're the last pwq going away, @wq is already dead and no one
3860	* is gonna access it anymore. Schedule RCU free.
3861	*/
3862	if (is_last) {
3863	wq_unregister_lockdep(wq);
3864	call_rcu(&wq->rcu, rcu_free_wq);
3865	}
3866	}
3867
3868	/**
3869	* pwq_adjust_max_active - update a pwq's max_active to the current setting
3870	* @pwq: target pool_workqueue
3871	*
3872	* If @pwq isn't freezing, set @pwq->max_active to the associated
3873	* workqueue's saved_max_active and activate inactive work items
3874	* accordingly. If @pwq is freezing, clear @pwq->max_active to zero.
3875	*/
3876	static void pwq_adjust_max_active(struct pool_workqueue *pwq)
3877	{
3878	struct workqueue_struct *wq = pwq->wq;
3879	bool freezable = wq->flags & WQ_FREEZABLE;
3880	unsigned long flags;
3881
3882	/* for @wq->saved_max_active */
3883	lockdep_assert_held(&wq->mutex);
3884
3885	/* fast exit for non-freezable wqs */
3886	if (!freezable && pwq->max_active == wq->saved_max_active)
3887	return;
3888
3889	/* this function can be called during early boot w/ irq disabled */
3890	raw_spin_lock_irqsave(&pwq->pool->lock, flags);
3891
3892	/*
3893	* During [un]freezing, the caller is responsible for ensuring that
3894	* this function is called at least once after @workqueue_freezing
3895	* is updated and visible.
3896	*/
3897	if (!freezable \|\| !workqueue_freezing) {
3898	bool kick = false;
3899
3900	pwq->max_active = wq->saved_max_active;
3901
3902	while (!list_empty(&pwq->inactive_works) &&
3903	pwq->nr_active < pwq->max_active) {
3904	pwq_activate_first_inactive(pwq);
3905	kick = true;
3906	}
3907
3908	/*
3909	* Need to kick a worker after thawed or an unbound wq's
3910	* max_active is bumped. In realtime scenarios, always kicking a
3911	* worker will cause interference on the isolated cpu cores, so
3912	* let's kick iff work items were activated.
3913	*/
3914	if (kick)
3915	wake_up_worker(pwq->pool);
3916	} else {
3917	pwq->max_active = 0;
3918	}
3919
3920	raw_spin_unlock_irqrestore(&pwq->pool->lock, flags);
3921	}
3922
3923	/* initialize newly allocated @pwq which is associated with @wq and @pool */
3924	static void init_pwq(struct pool_workqueue pwq, struct workqueue_struct wq,
3925	struct worker_pool *pool)
3926	{
3927	BUG_ON((unsigned long)pwq & WORK_STRUCT_FLAG_MASK);
3928
3929	memset(pwq, 0, sizeof(*pwq));
3930
3931	pwq->pool = pool;
3932	pwq->wq = wq;
3933	pwq->flush_color = -1;
3934	pwq->refcnt = 1;
3935	INIT_LIST_HEAD(&pwq->inactive_works);
3936	INIT_LIST_HEAD(&pwq->pwqs_node);
3937	INIT_LIST_HEAD(&pwq->mayday_node);
3938	INIT_WORK(&pwq->unbound_release_work, pwq_unbound_release_workfn);
3939	}
3940
3941	/* sync @pwq with the current state of its associated wq and link it */
3942	static void link_pwq(struct pool_workqueue *pwq)
3943	{
3944	struct workqueue_struct *wq = pwq->wq;
3945
3946	lockdep_assert_held(&wq->mutex);
3947
3948	/* may be called multiple times, ignore if already linked */
3949	if (!list_empty(&pwq->pwqs_node))
3950	return;
3951
3952	/* set the matching work_color */
3953	pwq->work_color = wq->work_color;
3954
3955	/* sync max_active to the current setting */
3956	pwq_adjust_max_active(pwq);
3957
3958	/* link in @pwq */
3959	list_add_rcu(&pwq->pwqs_node, &wq->pwqs);
3960	}
3961
3962	/* obtain a pool matching @attr and create a pwq associating the pool and @wq */
3963	static struct pool_workqueue alloc_unbound_pwq(struct workqueue_struct wq,
3964	const struct workqueue_attrs *attrs)
3965	{
3966	struct worker_pool *pool;
3967	struct pool_workqueue *pwq;
3968
3969	lockdep_assert_held(&wq_pool_mutex);
3970
3971	pool = get_unbound_pool(attrs);
3972	if (!pool)
3973	return NULL;
3974
3975	pwq = kmem_cache_alloc_node(pwq_cache, GFP_KERNEL, pool->node);
3976	if (!pwq) {
3977	put_unbound_pool(pool);
3978	return NULL;
3979	}
3980
3981	init_pwq(pwq, wq, pool);
3982	return pwq;
3983	}
3984
3985	/**
3986	* wq_calc_node_cpumask - calculate a wq_attrs' cpumask for the specified node
3987	* @attrs: the wq_attrs of the default pwq of the target workqueue
3988	* @node: the target NUMA node
3989	* @cpu_going_down: if >= 0, the CPU to consider as offline
3990	* @cpumask: outarg, the resulting cpumask
3991	*
3992	* Calculate the cpumask a workqueue with @attrs should use on @node. If
3993	* @cpu_going_down is >= 0, that cpu is considered offline during
3994	* calculation. The result is stored in @cpumask.
3995	*
3996	* If NUMA affinity is not enabled, @attrs->cpumask is always used. If
3997	* enabled and @node has online CPUs requested by @attrs, the returned
3998	* cpumask is the intersection of the possible CPUs of @node and
3999	* @attrs->cpumask.
4000	*
4001	* The caller is responsible for ensuring that the cpumask of @node stays
4002	* stable.
4003	*
4004	* Return: %true if the resulting @cpumask is different from @attrs->cpumask,
4005	* %false if equal.
4006	*/
4007	static bool wq_calc_node_cpumask(const struct workqueue_attrs *attrs, int node,
4008	int cpu_going_down, cpumask_t *cpumask)
4009	{
4010	if (!wq_numa_enabled \|\| attrs->no_numa)
4011	goto use_dfl;
4012
4013	/* does @node have any online CPUs @attrs wants? */
4014	cpumask_and(cpumask, cpumask_of_node(node), attrs->cpumask);
4015	if (cpu_going_down >= 0)
4016	cpumask_clear_cpu(cpu_going_down, cpumask);
4017
4018	if (cpumask_empty(cpumask))
4019	goto use_dfl;
4020
4021	/* yeap, return possible CPUs in @node that @attrs wants */
4022	cpumask_and(cpumask, attrs->cpumask, wq_numa_possible_cpumask[node]);
4023
4024	if (cpumask_empty(cpumask)) {
4025	pr_warn_once("WARNING: workqueue cpumask: online intersect > "
4026	"possible intersect\n");
4027	return false;
4028	}
4029
4030	return !cpumask_equal(cpumask, attrs->cpumask);
4031
4032	use_dfl:
4033	cpumask_copy(cpumask, attrs->cpumask);
4034	return false;
4035	}
4036
4037	/* install @pwq into @wq's numa_pwq_tbl[] for @node and return the old pwq */
4038	static struct pool_workqueue numa_pwq_tbl_install(struct workqueue_struct wq,
4039	int node,
4040	struct pool_workqueue *pwq)
4041	{
4042	struct pool_workqueue *old_pwq;
4043
4044	lockdep_assert_held(&wq_pool_mutex);
4045	lockdep_assert_held(&wq->mutex);
4046
4047	/* link_pwq() can handle duplicate calls */
4048	link_pwq(pwq);
4049
4050	old_pwq = rcu_access_pointer(wq->numa_pwq_tbl[node]);
4051	rcu_assign_pointer(wq->numa_pwq_tbl[node], pwq);
4052	return old_pwq;
4053	}
4054
4055	/* context to store the prepared attrs & pwqs before applying */
4056	struct apply_wqattrs_ctx {
4057	struct workqueue_struct wq; / target workqueue */
4058	struct workqueue_attrs attrs; / attrs to apply */
4059	struct list_head list; /* queued for batching commit */
4060	struct pool_workqueue *dfl_pwq;
4061	struct pool_workqueue *pwq_tbl[];
4062	};
4063
4064	/* free the resources after success or abort */
4065	static void apply_wqattrs_cleanup(struct apply_wqattrs_ctx *ctx)
4066	{
4067	if (ctx) {
4068	int node;
4069
4070	for_each_node(node)
4071	put_pwq_unlocked(ctx->pwq_tbl[node]);
4072	put_pwq_unlocked(ctx->dfl_pwq);
4073
4074	free_workqueue_attrs(ctx->attrs);
4075
4076	kfree(ctx);
4077	}
4078	}
4079
4080	/* allocate the attrs and pwqs for later installation */
4081	static struct apply_wqattrs_ctx *
4082	apply_wqattrs_prepare(struct workqueue_struct *wq,
4083	const struct workqueue_attrs *attrs,
4084	const cpumask_var_t unbound_cpumask)
4085	{
4086	struct apply_wqattrs_ctx *ctx;
4087	struct workqueue_attrs new_attrs, tmp_attrs;
4088	int node;
4089
4090	lockdep_assert_held(&wq_pool_mutex);
4091
4092	ctx = kzalloc(struct_size(ctx, pwq_tbl, nr_node_ids), GFP_KERNEL);
4093
4094	new_attrs = alloc_workqueue_attrs();
4095	tmp_attrs = alloc_workqueue_attrs();
4096	if (!ctx \|\| !new_attrs \|\| !tmp_attrs)
4097	goto out_free;
4098
4099	/*
4100	* Calculate the attrs of the default pwq with unbound_cpumask
4101	* which is wq_unbound_cpumask or to set to wq_unbound_cpumask.
4102	* If the user configured cpumask doesn't overlap with the
4103	* wq_unbound_cpumask, we fallback to the wq_unbound_cpumask.
4104	*/
4105	copy_workqueue_attrs(new_attrs, attrs);
4106	cpumask_and(new_attrs->cpumask, new_attrs->cpumask, unbound_cpumask);
4107	if (unlikely(cpumask_empty(new_attrs->cpumask)))
4108	cpumask_copy(new_attrs->cpumask, unbound_cpumask);
4109
4110	/*
4111	* We may create multiple pwqs with differing cpumasks. Make a
4112	* copy of @new_attrs which will be modified and used to obtain
4113	* pools.
4114	*/
4115	copy_workqueue_attrs(tmp_attrs, new_attrs);
4116
4117	/*
4118	* If something goes wrong during CPU up/down, we'll fall back to
4119	* the default pwq covering whole @attrs->cpumask. Always create
4120	* it even if we don't use it immediately.
4121	*/
4122	ctx->dfl_pwq = alloc_unbound_pwq(wq, new_attrs);
4123	if (!ctx->dfl_pwq)
4124	goto out_free;
4125
4126	for_each_node(node) {
4127	if (wq_calc_node_cpumask(new_attrs, node, -1, tmp_attrs->cpumask)) {
4128	ctx->pwq_tbl[node] = alloc_unbound_pwq(wq, tmp_attrs);
4129	if (!ctx->pwq_tbl[node])
4130	goto out_free;
4131	} else {
4132	ctx->dfl_pwq->refcnt++;
4133	ctx->pwq_tbl[node] = ctx->dfl_pwq;
4134	}
4135	}
4136
4137	/* save the user configured attrs and sanitize it. */
4138	copy_workqueue_attrs(new_attrs, attrs);
4139	cpumask_and(new_attrs->cpumask, new_attrs->cpumask, cpu_possible_mask);
4140	ctx->attrs = new_attrs;
4141
4142	ctx->wq = wq;
4143	free_workqueue_attrs(tmp_attrs);
4144	return ctx;
4145
4146	out_free:
4147	free_workqueue_attrs(tmp_attrs);
4148	free_workqueue_attrs(new_attrs);
4149	apply_wqattrs_cleanup(ctx);
4150	return NULL;
4151	}
4152
4153	/* set attrs and install prepared pwqs, @ctx points to old pwqs on return */
4154	static void apply_wqattrs_commit(struct apply_wqattrs_ctx *ctx)
4155	{
4156	int node;
4157
4158	/* all pwqs have been created successfully, let's install'em */
4159	mutex_lock(&ctx->wq->mutex);
4160
4161	copy_workqueue_attrs(ctx->wq->unbound_attrs, ctx->attrs);
4162
4163	/* save the previous pwq and install the new one */
4164	for_each_node(node)
4165	ctx->pwq_tbl[node] = numa_pwq_tbl_install(ctx->wq, node,
4166	ctx->pwq_tbl[node]);
4167
4168	/* @dfl_pwq might not have been used, ensure it's linked */
4169	link_pwq(ctx->dfl_pwq);
4170	swap(ctx->wq->dfl_pwq, ctx->dfl_pwq);
4171
4172	mutex_unlock(&ctx->wq->mutex);
4173	}
4174
4175	static void apply_wqattrs_lock(void)
4176	{
4177	/* CPUs should stay stable across pwq creations and installations */
4178	cpus_read_lock();
4179	mutex_lock(&wq_pool_mutex);
4180	}
4181
4182	static void apply_wqattrs_unlock(void)
4183	{
4184	mutex_unlock(&wq_pool_mutex);
4185	cpus_read_unlock();
4186	}
4187
4188	static int apply_workqueue_attrs_locked(struct workqueue_struct *wq,
4189	const struct workqueue_attrs *attrs)
4190	{
4191	struct apply_wqattrs_ctx *ctx;
4192
4193	/* only unbound workqueues can change attributes */
4194	if (WARN_ON(!(wq->flags & WQ_UNBOUND)))
4195	return -EINVAL;
4196
4197	/* creating multiple pwqs breaks ordering guarantee */
4198	if (!list_empty(&wq->pwqs)) {
4199	if (WARN_ON(wq->flags & __WQ_ORDERED_EXPLICIT))
4200	return -EINVAL;
4201
4202	wq->flags &= ~__WQ_ORDERED;
4203	}
4204
4205	ctx = apply_wqattrs_prepare(wq, attrs, wq_unbound_cpumask);
4206	if (!ctx)
4207	return -ENOMEM;
4208
4209	/* the ctx has been prepared successfully, let's commit it */
4210	apply_wqattrs_commit(ctx);
4211	apply_wqattrs_cleanup(ctx);
4212
4213	return 0;
4214	}
4215
4216	/**
4217	* apply_workqueue_attrs - apply new workqueue_attrs to an unbound workqueue
4218	* @wq: the target workqueue
4219	* @attrs: the workqueue_attrs to apply, allocated with alloc_workqueue_attrs()
4220	*
4221	* Apply @attrs to an unbound workqueue @wq. Unless disabled, on NUMA
4222	* machines, this function maps a separate pwq to each NUMA node with
4223	* possibles CPUs in @attrs->cpumask so that work items are affine to the
4224	* NUMA node it was issued on. Older pwqs are released as in-flight work
4225	* items finish. Note that a work item which repeatedly requeues itself
4226	* back-to-back will stay on its current pwq.
4227	*
4228	* Performs GFP_KERNEL allocations.
4229	*
4230	* Assumes caller has CPU hotplug read exclusion, i.e. cpus_read_lock().
4231	*
4232	* Return: 0 on success and -errno on failure.
4233	*/
4234	int apply_workqueue_attrs(struct workqueue_struct *wq,
4235	const struct workqueue_attrs *attrs)
4236	{
4237	int ret;
4238
4239	lockdep_assert_cpus_held();
4240
4241	mutex_lock(&wq_pool_mutex);
4242	ret = apply_workqueue_attrs_locked(wq, attrs);
4243	mutex_unlock(&wq_pool_mutex);
4244
4245	return ret;
4246	}
4247
4248	/**
4249	* wq_update_unbound_numa - update NUMA affinity of a wq for CPU hot[un]plug
4250	* @wq: the target workqueue
4251	* @cpu: the CPU coming up or going down
4252	* @online: whether @cpu is coming up or going down
4253	*
4254	* This function is to be called from %CPU_DOWN_PREPARE, %CPU_ONLINE and
4255	* %CPU_DOWN_FAILED. @cpu is being hot[un]plugged, update NUMA affinity of
4256	* @wq accordingly.
4257	*
4258	* If NUMA affinity can't be adjusted due to memory allocation failure, it
4259	* falls back to @wq->dfl_pwq which may not be optimal but is always
4260	* correct.
4261	*
4262	* Note that when the last allowed CPU of a NUMA node goes offline for a
4263	* workqueue with a cpumask spanning multiple nodes, the workers which were
4264	* already executing the work items for the workqueue will lose their CPU
4265	* affinity and may execute on any CPU. This is similar to how per-cpu
4266	* workqueues behave on CPU_DOWN. If a workqueue user wants strict
4267	* affinity, it's the user's responsibility to flush the work item from
4268	* CPU_DOWN_PREPARE.
4269	*/
4270	static void wq_update_unbound_numa(struct workqueue_struct *wq, int cpu,
4271	bool online)
4272	{
4273	int node = cpu_to_node(cpu);
4274	int cpu_off = online ? -1 : cpu;
4275	struct pool_workqueue old_pwq = NULL, pwq;
4276	struct workqueue_attrs *target_attrs;
4277	cpumask_t *cpumask;
4278
4279	lockdep_assert_held(&wq_pool_mutex);
4280
4281	if (!wq_numa_enabled \|\| !(wq->flags & WQ_UNBOUND) \|\|
4282	wq->unbound_attrs->no_numa)
4283	return;
4284
4285	/*
4286	* We don't wanna alloc/free wq_attrs for each wq for each CPU.
4287	* Let's use a preallocated one. The following buf is protected by
4288	* CPU hotplug exclusion.
4289	*/
4290	target_attrs = wq_update_unbound_numa_attrs_buf;
4291	cpumask = target_attrs->cpumask;
4292
4293	copy_workqueue_attrs(target_attrs, wq->unbound_attrs);
4294	pwq = unbound_pwq_by_node(wq, node);
4295
4296	/*
4297	* Let's determine what needs to be done. If the target cpumask is
4298	* different from the default pwq's, we need to compare it to @pwq's
4299	* and create a new one if they don't match. If the target cpumask
4300	* equals the default pwq's, the default pwq should be used.
4301	*/
4302	if (wq_calc_node_cpumask(wq->dfl_pwq->pool->attrs, node, cpu_off, cpumask)) {
4303	if (cpumask_equal(cpumask, pwq->pool->attrs->cpumask))
4304	return;
4305	} else {
4306	goto use_dfl_pwq;
4307	}
4308
4309	/* create a new pwq */
4310	pwq = alloc_unbound_pwq(wq, target_attrs);
4311	if (!pwq) {
4312	pr_warn("workqueue: allocation failed while updating NUMA affinity of \"%s\"\n",
4313	wq->name);
4314	goto use_dfl_pwq;
4315	}
4316
4317	/* Install the new pwq. */
4318	mutex_lock(&wq->mutex);
4319	old_pwq = numa_pwq_tbl_install(wq, node, pwq);
4320	goto out_unlock;
4321
4322	use_dfl_pwq:
4323	mutex_lock(&wq->mutex);
4324	raw_spin_lock_irq(&wq->dfl_pwq->pool->lock);
4325	get_pwq(wq->dfl_pwq);
4326	raw_spin_unlock_irq(&wq->dfl_pwq->pool->lock);
4327	old_pwq = numa_pwq_tbl_install(wq, node, wq->dfl_pwq);
4328	out_unlock:
4329	mutex_unlock(&wq->mutex);
4330	put_pwq_unlocked(old_pwq);
4331	}
4332
4333	static int alloc_and_link_pwqs(struct workqueue_struct *wq)
4334	{
4335	bool highpri = wq->flags & WQ_HIGHPRI;
4336	int cpu, ret;
4337
4338	if (!(wq->flags & WQ_UNBOUND)) {
4339	wq->cpu_pwqs = alloc_percpu(struct pool_workqueue);
4340	if (!wq->cpu_pwqs)
4341	return -ENOMEM;
4342
4343	for_each_possible_cpu(cpu) {
4344	struct pool_workqueue *pwq =
4345	per_cpu_ptr(wq->cpu_pwqs, cpu);
4346	struct worker_pool *cpu_pools =
4347	per_cpu(cpu_worker_pools, cpu);
4348
4349	init_pwq(pwq, wq, &cpu_pools[highpri]);
4350
4351	mutex_lock(&wq->mutex);
4352	link_pwq(pwq);
4353	mutex_unlock(&wq->mutex);
4354	}
4355	return 0;
4356	}
4357
4358	cpus_read_lock();
4359	if (wq->flags & __WQ_ORDERED) {
4360	ret = apply_workqueue_attrs(wq, ordered_wq_attrs[highpri]);
4361	/* there should only be single pwq for ordering guarantee */
4362	WARN(!ret && (wq->pwqs.next != &wq->dfl_pwq->pwqs_node \|\|
4363	wq->pwqs.prev != &wq->dfl_pwq->pwqs_node),
4364	"ordering guarantee broken for workqueue %s\n", wq->name);
4365	} else {
4366	ret = apply_workqueue_attrs(wq, unbound_std_wq_attrs[highpri]);
4367	}
4368	cpus_read_unlock();
4369
4370	return ret;
4371	}
4372
4373	static int wq_clamp_max_active(int max_active, unsigned int flags,
4374	const char *name)
4375	{
4376	int lim = flags & WQ_UNBOUND ? WQ_UNBOUND_MAX_ACTIVE : WQ_MAX_ACTIVE;
4377
4378	if (max_active < 1 \|\| max_active > lim)
4379	pr_warn("workqueue: max_active %d requested for %s is out of range, clamping between %d and %d\n",
4380	max_active, name, 1, lim);
4381
4382	return clamp_val(max_active, 1, lim);
4383	}
4384
4385	/*
4386	* Workqueues which may be used during memory reclaim should have a rescuer
4387	* to guarantee forward progress.
4388	*/
4389	static int init_rescuer(struct workqueue_struct *wq)
4390	{
4391	struct worker *rescuer;
4392	int ret;
4393
4394	if (!(wq->flags & WQ_MEM_RECLAIM))
4395	return 0;
4396
4397	rescuer = alloc_worker(NUMA_NO_NODE);
4398	if (!rescuer) {
4399	pr_err("workqueue: Failed to allocate a rescuer for wq \"%s\"\n",
4400	wq->name);
4401	return -ENOMEM;
4402	}
4403
4404	rescuer->rescue_wq = wq;
4405	rescuer->task = kthread_create(rescuer_thread, rescuer, "%s", wq->name);
4406	if (IS_ERR(rescuer->task)) {
4407	ret = PTR_ERR(rescuer->task);
4408	pr_err("workqueue: Failed to create a rescuer kthread for wq \"%s\": %pe",
4409	wq->name, ERR_PTR(ret));
4410	kfree(rescuer);
4411	return ret;
4412	}
4413
4414	wq->rescuer = rescuer;
4415	kthread_bind_mask(rescuer->task, cpu_possible_mask);
4416	wake_up_process(rescuer->task);
4417
4418	return 0;
4419	}
4420
4421	__printf(1, 4)
4422	struct workqueue_struct alloc_workqueue(const char fmt,
4423	unsigned int flags,
4424	int max_active, ...)
4425	{
4426	size_t tbl_size = 0;
4427	va_list args;
4428	struct workqueue_struct *wq;
4429	struct pool_workqueue *pwq;
4430
4431	/*
4432	* Unbound && max_active == 1 used to imply ordered, which is no
4433	* longer the case on NUMA machines due to per-node pools. While
4434	* alloc_ordered_workqueue() is the right way to create an ordered
4435	* workqueue, keep the previous behavior to avoid subtle breakages
4436	* on NUMA.
4437	*/
4438	if ((flags & WQ_UNBOUND) && max_active == 1)
4439	flags \|= __WQ_ORDERED;
4440
4441	/* see the comment above the definition of WQ_POWER_EFFICIENT */
4442	if ((flags & WQ_POWER_EFFICIENT) && wq_power_efficient)
4443	flags \|= WQ_UNBOUND;
4444
4445	/* allocate wq and format name */
4446	if (flags & WQ_UNBOUND)
4447	tbl_size = nr_node_ids * sizeof(wq->numa_pwq_tbl[0]);
4448
4449	wq = kzalloc(sizeof(*wq) + tbl_size, GFP_KERNEL);
4450	if (!wq)
4451	return NULL;
4452
4453	if (flags & WQ_UNBOUND) {
4454	wq->unbound_attrs = alloc_workqueue_attrs();
4455	if (!wq->unbound_attrs)
4456	goto err_free_wq;
4457	}
4458
4459	va_start(args, max_active);
4460	vsnprintf(wq->name, sizeof(wq->name), fmt, args);
4461	va_end(args);
4462
4463	max_active = max_active ?: WQ_DFL_ACTIVE;
4464	max_active = wq_clamp_max_active(max_active, flags, wq->name);
4465
4466	/* init wq */
4467	wq->flags = flags;
4468	wq->saved_max_active = max_active;
4469	mutex_init(&wq->mutex);
4470	atomic_set(&wq->nr_pwqs_to_flush, 0);
4471	INIT_LIST_HEAD(&wq->pwqs);
4472	INIT_LIST_HEAD(&wq->flusher_queue);
4473	INIT_LIST_HEAD(&wq->flusher_overflow);
4474	INIT_LIST_HEAD(&wq->maydays);
4475
4476	wq_init_lockdep(wq);
4477	INIT_LIST_HEAD(&wq->list);
4478
4479	if (alloc_and_link_pwqs(wq) < 0)
4480	goto err_unreg_lockdep;
4481
4482	if (wq_online && init_rescuer(wq) < 0)
4483	goto err_destroy;
4484
4485	if ((wq->flags & WQ_SYSFS) && workqueue_sysfs_register(wq))
4486	goto err_destroy;
4487
4488	/*
4489	* wq_pool_mutex protects global freeze state and workqueues list.
4490	* Grab it, adjust max_active and add the new @wq to workqueues
4491	* list.
4492	*/
4493	mutex_lock(&wq_pool_mutex);
4494
4495	mutex_lock(&wq->mutex);
4496	for_each_pwq(pwq, wq)
4497	pwq_adjust_max_active(pwq);
4498	mutex_unlock(&wq->mutex);
4499
4500	list_add_tail_rcu(&wq->list, &workqueues);
4501
4502	mutex_unlock(&wq_pool_mutex);
4503
4504	return wq;
4505
4506	err_unreg_lockdep:
4507	wq_unregister_lockdep(wq);
4508	wq_free_lockdep(wq);
4509	err_free_wq:
4510	free_workqueue_attrs(wq->unbound_attrs);
4511	kfree(wq);
4512	return NULL;
4513	err_destroy:
4514	destroy_workqueue(wq);
4515	return NULL;
4516	}
4517	EXPORT_SYMBOL_GPL(alloc_workqueue);
4518
4519	static bool pwq_busy(struct pool_workqueue *pwq)
4520	{
4521	int i;
4522
4523	for (i = 0; i < WORK_NR_COLORS; i++)
4524	if (pwq->nr_in_flight[i])
4525	return true;
4526
4527	if ((pwq != pwq->wq->dfl_pwq) && (pwq->refcnt > 1))
4528	return true;
4529	if (pwq->nr_active \|\| !list_empty(&pwq->inactive_works))
4530	return true;
4531
4532	return false;
4533	}
4534
4535	/**
4536	* destroy_workqueue - safely terminate a workqueue
4537	* @wq: target workqueue
4538	*
4539	* Safely destroy a workqueue. All work currently pending will be done first.
4540	*/
4541	void destroy_workqueue(struct workqueue_struct *wq)
4542	{
4543	struct pool_workqueue *pwq;
4544	int node;
4545
4546	/*
4547	* Remove it from sysfs first so that sanity check failure doesn't
4548	* lead to sysfs name conflicts.
4549	*/
4550	workqueue_sysfs_unregister(wq);
4551
4552	/* mark the workqueue destruction is in progress */
4553	mutex_lock(&wq->mutex);
4554	wq->flags \|= __WQ_DESTROYING;
4555	mutex_unlock(&wq->mutex);
4556
4557	/* drain it before proceeding with destruction */
4558	drain_workqueue(wq);
4559
4560	/* kill rescuer, if sanity checks fail, leave it w/o rescuer */
4561	if (wq->rescuer) {
4562	struct worker *rescuer = wq->rescuer;
4563
4564	/* this prevents new queueing */
4565	raw_spin_lock_irq(&wq_mayday_lock);
4566	wq->rescuer = NULL;
4567	raw_spin_unlock_irq(&wq_mayday_lock);
4568
4569	/* rescuer will empty maydays list before exiting */
4570	kthread_stop(rescuer->task);
4571	kfree(rescuer);
4572	}
4573
4574	/*
4575	* Sanity checks - grab all the locks so that we wait for all
4576	* in-flight operations which may do put_pwq().
4577	*/
4578	mutex_lock(&wq_pool_mutex);
4579	mutex_lock(&wq->mutex);
4580	for_each_pwq(pwq, wq) {
4581	raw_spin_lock_irq(&pwq->pool->lock);
4582	if (WARN_ON(pwq_busy(pwq))) {
4583	pr_warn("%s: %s has the following busy pwq\n",
4584	__func__, wq->name);
4585	show_pwq(pwq);
4586	raw_spin_unlock_irq(&pwq->pool->lock);
4587	mutex_unlock(&wq->mutex);
4588	mutex_unlock(&wq_pool_mutex);
4589	show_one_workqueue(wq);
4590	return;
4591	}
4592	raw_spin_unlock_irq(&pwq->pool->lock);
4593	}
4594	mutex_unlock(&wq->mutex);
4595
4596	/*
4597	* wq list is used to freeze wq, remove from list after
4598	* flushing is complete in case freeze races us.
4599	*/
4600	list_del_rcu(&wq->list);
4601	mutex_unlock(&wq_pool_mutex);
4602
4603	if (!(wq->flags & WQ_UNBOUND)) {
4604	wq_unregister_lockdep(wq);
4605	/*
4606	* The base ref is never dropped on per-cpu pwqs. Directly
4607	* schedule RCU free.
4608	*/
4609	call_rcu(&wq->rcu, rcu_free_wq);
4610	} else {
4611	/*
4612	* We're the sole accessor of @wq at this point. Directly
4613	* access numa_pwq_tbl[] and dfl_pwq to put the base refs.
4614	* @wq will be freed when the last pwq is released.
4615	*/
4616	for_each_node(node) {
4617	pwq = rcu_access_pointer(wq->numa_pwq_tbl[node]);
4618	RCU_INIT_POINTER(wq->numa_pwq_tbl[node], NULL);
4619	put_pwq_unlocked(pwq);
4620	}
4621
4622	/*
4623	* Put dfl_pwq. @wq may be freed any time after dfl_pwq is
4624	* put. Don't access it afterwards.
4625	*/
4626	pwq = wq->dfl_pwq;
4627	wq->dfl_pwq = NULL;
4628	put_pwq_unlocked(pwq);
4629	}
4630	}
4631	EXPORT_SYMBOL_GPL(destroy_workqueue);
4632
4633	/**
4634	* workqueue_set_max_active - adjust max_active of a workqueue
4635	* @wq: target workqueue
4636	* @max_active: new max_active value.
4637	*
4638	* Set max_active of @wq to @max_active.
4639	*
4640	* CONTEXT:
4641	* Don't call from IRQ context.
4642	*/
4643	void workqueue_set_max_active(struct workqueue_struct *wq, int max_active)
4644	{
4645	struct pool_workqueue *pwq;
4646
4647	/* disallow meddling with max_active for ordered workqueues */
4648	if (WARN_ON(wq->flags & __WQ_ORDERED_EXPLICIT))
4649	return;
4650
4651	max_active = wq_clamp_max_active(max_active, wq->flags, wq->name);
4652
4653	mutex_lock(&wq->mutex);
4654
4655	wq->flags &= ~__WQ_ORDERED;
4656	wq->saved_max_active = max_active;
4657
4658	for_each_pwq(pwq, wq)
4659	pwq_adjust_max_active(pwq);
4660
4661	mutex_unlock(&wq->mutex);
4662	}
4663	EXPORT_SYMBOL_GPL(workqueue_set_max_active);
4664
4665	/**
4666	* current_work - retrieve %current task's work struct
4667	*
4668	* Determine if %current task is a workqueue worker and what it's working on.
4669	* Useful to find out the context that the %current task is running in.
4670	*
4671	* Return: work struct if %current task is a workqueue worker, %NULL otherwise.
4672	*/
4673	struct work_struct *current_work(void)
4674	{
4675	struct worker *worker = current_wq_worker();
4676
4677	return worker ? worker->current_work : NULL;
4678	}
4679	EXPORT_SYMBOL(current_work);
4680
4681	/**
4682	* current_is_workqueue_rescuer - is %current workqueue rescuer?
4683	*
4684	* Determine whether %current is a workqueue rescuer. Can be used from
4685	* work functions to determine whether it's being run off the rescuer task.
4686	*
4687	* Return: %true if %current is a workqueue rescuer. %false otherwise.
4688	*/
4689	bool current_is_workqueue_rescuer(void)
4690	{
4691	struct worker *worker = current_wq_worker();
4692
4693	return worker && worker->rescue_wq;
4694	}
4695
4696	/**
4697	* workqueue_congested - test whether a workqueue is congested
4698	* @cpu: CPU in question
4699	* @wq: target workqueue
4700	*
4701	* Test whether @wq's cpu workqueue for @cpu is congested. There is
4702	* no synchronization around this function and the test result is
4703	* unreliable and only useful as advisory hints or for debugging.
4704	*
4705	* If @cpu is WORK_CPU_UNBOUND, the test is performed on the local CPU.
4706	* Note that both per-cpu and unbound workqueues may be associated with
4707	* multiple pool_workqueues which have separate congested states. A
4708	* workqueue being congested on one CPU doesn't mean the workqueue is also
4709	* contested on other CPUs / NUMA nodes.
4710	*
4711	* Return:
4712	* %true if congested, %false otherwise.
4713	*/
4714	bool workqueue_congested(int cpu, struct workqueue_struct *wq)
4715	{
4716	struct pool_workqueue *pwq;
4717	bool ret;
4718
4719	rcu_read_lock();
4720	preempt_disable();
4721
4722	if (cpu == WORK_CPU_UNBOUND)
4723	cpu = smp_processor_id();
4724
4725	if (!(wq->flags & WQ_UNBOUND))
4726	pwq = per_cpu_ptr(wq->cpu_pwqs, cpu);
4727	else
4728	pwq = unbound_pwq_by_node(wq, cpu_to_node(cpu));
4729
4730	ret = !list_empty(&pwq->inactive_works);
4731	preempt_enable();
4732	rcu_read_unlock();
4733
4734	return ret;
4735	}
4736	EXPORT_SYMBOL_GPL(workqueue_congested);
4737
4738	/**
4739	* work_busy - test whether a work is currently pending or running
4740	* @work: the work to be tested
4741	*
4742	* Test whether @work is currently pending or running. There is no
4743	* synchronization around this function and the test result is
4744	* unreliable and only useful as advisory hints or for debugging.
4745	*
4746	* Return:
4747	* OR'd bitmask of WORK_BUSY_* bits.
4748	*/
4749	unsigned int work_busy(struct work_struct *work)
4750	{
4751	struct worker_pool *pool;
4752	unsigned long flags;
4753	unsigned int ret = 0;
4754
4755	if (work_pending(work))
4756	ret \|= WORK_BUSY_PENDING;
4757
4758	rcu_read_lock();
4759	pool = get_work_pool(work);
4760	if (pool) {
4761	raw_spin_lock_irqsave(&pool->lock, flags);
4762	if (find_worker_executing_work(pool, work))
4763	ret \|= WORK_BUSY_RUNNING;
4764	raw_spin_unlock_irqrestore(&pool->lock, flags);
4765	}
4766	rcu_read_unlock();
4767
4768	return ret;
4769	}
4770	EXPORT_SYMBOL_GPL(work_busy);
4771
4772	/**
4773	* set_worker_desc - set description for the current work item
4774	* @fmt: printf-style format string
4775	* @...: arguments for the format string
4776	*
4777	* This function can be called by a running work function to describe what
4778	* the work item is about. If the worker task gets dumped, this
4779	* information will be printed out together to help debugging. The
4780	* description can be at most WORKER_DESC_LEN including the trailing '\0'.
4781	*/
4782	void set_worker_desc(const char *fmt, ...)
4783	{
4784	struct worker *worker = current_wq_worker();
4785	va_list args;
4786
4787	if (worker) {
4788	va_start(args, fmt);
4789	vsnprintf(worker->desc, sizeof(worker->desc), fmt, args);
4790	va_end(args);
4791	}
4792	}
4793	EXPORT_SYMBOL_GPL(set_worker_desc);
4794
4795	/**
4796	* print_worker_info - print out worker information and description
4797	* @log_lvl: the log level to use when printing
4798	* @task: target task
4799	*
4800	* If @task is a worker and currently executing a work item, print out the
4801	* name of the workqueue being serviced and worker description set with
4802	* set_worker_desc() by the currently executing work item.
4803	*
4804	* This function can be safely called on any task as long as the
4805	* task_struct itself is accessible. While safe, this function isn't
4806	* synchronized and may print out mixups or garbages of limited length.
4807	*/
4808	void print_worker_info(const char log_lvl, struct task_struct task)
4809	{
4810	work_func_t *fn = NULL;
4811	char name[WQ_NAME_LEN] = { };
4812	char desc[WORKER_DESC_LEN] = { };
4813	struct pool_workqueue *pwq = NULL;
4814	struct workqueue_struct *wq = NULL;
4815	struct worker *worker;
4816
4817	if (!(task->flags & PF_WQ_WORKER))
4818	return;
4819
4820	/*
4821	* This function is called without any synchronization and @task
4822	* could be in any state. Be careful with dereferences.
4823	*/
4824	worker = kthread_probe_data(task);
4825
4826	/*
4827	* Carefully copy the associated workqueue's workfn, name and desc.
4828	* Keep the original last '\0' in case the original is garbage.
4829	*/
4830	copy_from_kernel_nofault(&fn, &worker->current_func, sizeof(fn));
4831	copy_from_kernel_nofault(&pwq, &worker->current_pwq, sizeof(pwq));
4832	copy_from_kernel_nofault(&wq, &pwq->wq, sizeof(wq));
4833	copy_from_kernel_nofault(name, wq->name, sizeof(name) - 1);
4834	copy_from_kernel_nofault(desc, worker->desc, sizeof(desc) - 1);
4835
4836	if (fn \|\| name[0] \|\| desc[0]) {
4837	printk("%sWorkqueue: %s %ps", log_lvl, name, fn);
4838	if (strcmp(name, desc))
4839	pr_cont(" (%s)", desc);
4840	pr_cont("\n");
4841	}
4842	}
4843
4844	static void pr_cont_pool_info(struct worker_pool *pool)
4845	{
4846	pr_cont(" cpus=%*pbl", nr_cpumask_bits, pool->attrs->cpumask);
4847	if (pool->node != NUMA_NO_NODE)
4848	pr_cont(" node=%d", pool->node);
4849	pr_cont(" flags=0x%x nice=%d", pool->flags, pool->attrs->nice);
4850	}
4851
4852	struct pr_cont_work_struct {
4853	bool comma;
4854	work_func_t func;
4855	long ctr;
4856	};
4857
4858	static void pr_cont_work_flush(bool comma, work_func_t func, struct pr_cont_work_struct *pcwsp)
4859	{
4860	if (!pcwsp->ctr)
4861	goto out_record;
4862	if (func == pcwsp->func) {
4863	pcwsp->ctr++;
4864	return;
4865	}
4866	if (pcwsp->ctr == 1)
4867	pr_cont("%s %ps", pcwsp->comma ? "," : "", pcwsp->func);
4868	else
4869	pr_cont("%s %ld*%ps", pcwsp->comma ? "," : "", pcwsp->ctr, pcwsp->func);
4870	pcwsp->ctr = 0;
4871	out_record:
4872	if ((long)func == -1L)
4873	return;
4874	pcwsp->comma = comma;
4875	pcwsp->func = func;
4876	pcwsp->ctr = 1;
4877	}
4878
4879	static void pr_cont_work(bool comma, struct work_struct work, struct pr_cont_work_struct pcwsp)
4880	{
4881	if (work->func == wq_barrier_func) {
4882	struct wq_barrier *barr;
4883
4884	barr = container_of(work, struct wq_barrier, work);
4885
4886	pr_cont_work_flush(comma, (work_func_t)-1, pcwsp);
4887	pr_cont("%s BAR(%d)", comma ? "," : "",
4888	task_pid_nr(barr->task));
4889	} else {
4890	if (!comma)
4891	pr_cont_work_flush(comma, (work_func_t)-1, pcwsp);
4892	pr_cont_work_flush(comma, work->func, pcwsp);
4893	}
4894	}
4895
4896	static void show_pwq(struct pool_workqueue *pwq)
4897	{
4898	struct pr_cont_work_struct pcws = { .ctr = 0, };
4899	struct worker_pool *pool = pwq->pool;
4900	struct work_struct *work;
4901	struct worker *worker;
4902	bool has_in_flight = false, has_pending = false;
4903	int bkt;
4904
4905	pr_info(" pwq %d:", pool->id);
4906	pr_cont_pool_info(pool);
4907
4908	pr_cont(" active=%d/%d refcnt=%d%s\n",
4909	pwq->nr_active, pwq->max_active, pwq->refcnt,
4910	!list_empty(&pwq->mayday_node) ? " MAYDAY" : "");
4911
4912	hash_for_each(pool->busy_hash, bkt, worker, hentry) {
4913	if (worker->current_pwq == pwq) {
4914	has_in_flight = true;
4915	break;
4916	}
4917	}
4918	if (has_in_flight) {
4919	bool comma = false;
4920
4921	pr_info(" in-flight:");
4922	hash_for_each(pool->busy_hash, bkt, worker, hentry) {
4923	if (worker->current_pwq != pwq)
4924	continue;
4925
4926	pr_cont("%s %d%s:%ps", comma ? "," : "",
4927	task_pid_nr(worker->task),
4928	worker->rescue_wq ? "(RESCUER)" : "",
4929	worker->current_func);
4930	list_for_each_entry(work, &worker->scheduled, entry)
4931	pr_cont_work(false, work, &pcws);
4932	pr_cont_work_flush(comma, (work_func_t)-1L, &pcws);
4933	comma = true;
4934	}
4935	pr_cont("\n");
4936	}
4937
4938	list_for_each_entry(work, &pool->worklist, entry) {
4939	if (get_work_pwq(work) == pwq) {
4940	has_pending = true;
4941	break;
4942	}
4943	}
4944	if (has_pending) {
4945	bool comma = false;
4946
4947	pr_info(" pending:");
4948	list_for_each_entry(work, &pool->worklist, entry) {
4949	if (get_work_pwq(work) != pwq)
4950	continue;
4951
4952	pr_cont_work(comma, work, &pcws);
4953	comma = !(*work_data_bits(work) & WORK_STRUCT_LINKED);
4954	}
4955	pr_cont_work_flush(comma, (work_func_t)-1L, &pcws);
4956	pr_cont("\n");
4957	}
4958
4959	if (!list_empty(&pwq->inactive_works)) {
4960	bool comma = false;
4961
4962	pr_info(" inactive:");
4963	list_for_each_entry(work, &pwq->inactive_works, entry) {
4964	pr_cont_work(comma, work, &pcws);
4965	comma = !(*work_data_bits(work) & WORK_STRUCT_LINKED);
4966	}
4967	pr_cont_work_flush(comma, (work_func_t)-1L, &pcws);
4968	pr_cont("\n");
4969	}
4970	}
4971
4972	/**
4973	* show_one_workqueue - dump state of specified workqueue
4974	* @wq: workqueue whose state will be printed
4975	*/
4976	void show_one_workqueue(struct workqueue_struct *wq)
4977	{
4978	struct pool_workqueue *pwq;
4979	bool idle = true;
4980	unsigned long flags;
4981
4982	for_each_pwq(pwq, wq) {
4983	if (pwq->nr_active \|\| !list_empty(&pwq->inactive_works)) {
4984	idle = false;
4985	break;
4986	}
4987	}
4988	if (idle) /* Nothing to print for idle workqueue */
4989	return;
4990
4991	pr_info("workqueue %s: flags=0x%x\n", wq->name, wq->flags);
4992
4993	for_each_pwq(pwq, wq) {
4994	raw_spin_lock_irqsave(&pwq->pool->lock, flags);
4995	if (pwq->nr_active \|\| !list_empty(&pwq->inactive_works)) {
4996	/*
4997	* Defer printing to avoid deadlocks in console
4998	* drivers that queue work while holding locks
4999	* also taken in their write paths.
5000	*/
5001	printk_deferred_enter();
5002	show_pwq(pwq);
5003	printk_deferred_exit();
5004	}
5005	raw_spin_unlock_irqrestore(&pwq->pool->lock, flags);
5006	/*
5007	* We could be printing a lot from atomic context, e.g.
5008	* sysrq-t -> show_all_workqueues(). Avoid triggering
5009	* hard lockup.
5010	*/
5011	touch_nmi_watchdog();
5012	}
5013
5014	}
5015
5016	/**
5017	* show_one_worker_pool - dump state of specified worker pool
5018	* @pool: worker pool whose state will be printed
5019	*/
5020	static void show_one_worker_pool(struct worker_pool *pool)
5021	{
5022	struct worker *worker;
5023	bool first = true;
5024	unsigned long flags;
5025	unsigned long hung = 0;
5026
5027	raw_spin_lock_irqsave(&pool->lock, flags);
5028	if (pool->nr_workers == pool->nr_idle)
5029	goto next_pool;
5030
5031	/* How long the first pending work is waiting for a worker. */
5032	if (!list_empty(&pool->worklist))
5033	hung = jiffies_to_msecs(jiffies - pool->watchdog_ts) / 1000;
5034
5035	/*
5036	* Defer printing to avoid deadlocks in console drivers that
5037	* queue work while holding locks also taken in their write
5038	* paths.
5039	*/
5040	printk_deferred_enter();
5041	pr_info("pool %d:", pool->id);
5042	pr_cont_pool_info(pool);
5043	pr_cont(" hung=%lus workers=%d", hung, pool->nr_workers);
5044	if (pool->manager)
5045	pr_cont(" manager: %d",
5046	task_pid_nr(pool->manager->task));
5047	list_for_each_entry(worker, &pool->idle_list, entry) {
5048	pr_cont(" %s%d", first ? "idle: " : "",
5049	task_pid_nr(worker->task));
5050	first = false;
5051	}
5052	pr_cont("\n");
5053	printk_deferred_exit();
5054	next_pool:
5055	raw_spin_unlock_irqrestore(&pool->lock, flags);
5056	/*
5057	* We could be printing a lot from atomic context, e.g.
5058	* sysrq-t -> show_all_workqueues(). Avoid triggering
5059	* hard lockup.
5060	*/
5061	touch_nmi_watchdog();
5062
5063	}
5064
5065	/**
5066	* show_all_workqueues - dump workqueue state
5067	*
5068	* Called from a sysrq handler and prints out all busy workqueues and pools.
5069	*/
5070	void show_all_workqueues(void)
5071	{
5072	struct workqueue_struct *wq;
5073	struct worker_pool *pool;
5074	int pi;
5075
5076	rcu_read_lock();
5077
5078	pr_info("Showing busy workqueues and worker pools:\n");
5079
5080	list_for_each_entry_rcu(wq, &workqueues, list)
5081	show_one_workqueue(wq);
5082
5083	for_each_pool(pool, pi)
5084	show_one_worker_pool(pool);
5085
5086	rcu_read_unlock();
5087	}
5088
5089	/**
5090	* show_freezable_workqueues - dump freezable workqueue state
5091	*
5092	* Called from try_to_freeze_tasks() and prints out all freezable workqueues
5093	* still busy.
5094	*/
5095	void show_freezable_workqueues(void)
5096	{
5097	struct workqueue_struct *wq;
5098
5099	rcu_read_lock();
5100
5101	pr_info("Showing freezable workqueues that are still busy:\n");
5102
5103	list_for_each_entry_rcu(wq, &workqueues, list) {
5104	if (!(wq->flags & WQ_FREEZABLE))
5105	continue;
5106	show_one_workqueue(wq);
5107	}
5108
5109	rcu_read_unlock();
5110	}
5111
5112	/* used to show worker information through /proc/PID/{comm,stat,status} */
5113	void wq_worker_comm(char buf, size_t size, struct task_struct task)
5114	{
5115	int off;
5116
5117	/* always show the actual comm */
5118	off = strscpy(buf, task->comm, size);
5119	if (off < 0)
5120	return;
5121
5122	/* stabilize PF_WQ_WORKER and worker pool association */
5123	mutex_lock(&wq_pool_attach_mutex);
5124
5125	if (task->flags & PF_WQ_WORKER) {
5126	struct worker *worker = kthread_data(task);
5127	struct worker_pool *pool = worker->pool;
5128
5129	if (pool) {
5130	raw_spin_lock_irq(&pool->lock);
5131	/*
5132	* ->desc tracks information (wq name or
5133	* set_worker_desc()) for the latest execution. If
5134	* current, prepend '+', otherwise '-'.
5135	*/
5136	if (worker->desc[0] != '\0') {
5137	if (worker->current_work)
5138	scnprintf(buf + off, size - off, "+%s",
5139	worker->desc);
5140	else
5141	scnprintf(buf + off, size - off, "-%s",
5142	worker->desc);
5143	}
5144	raw_spin_unlock_irq(&pool->lock);
5145	}
5146	}
5147
5148	mutex_unlock(&wq_pool_attach_mutex);
5149	}
5150
5151	#ifdef CONFIG_SMP
5152
5153	/*
5154	* CPU hotplug.
5155	*
5156	* There are two challenges in supporting CPU hotplug. Firstly, there
5157	* are a lot of assumptions on strong associations among work, pwq and
5158	* pool which make migrating pending and scheduled works very
5159	* difficult to implement without impacting hot paths. Secondly,
5160	* worker pools serve mix of short, long and very long running works making
5161	* blocked draining impractical.
5162	*
5163	* This is solved by allowing the pools to be disassociated from the CPU
5164	* running as an unbound one and allowing it to be reattached later if the
5165	* cpu comes back online.
5166	*/
5167
5168	static void unbind_workers(int cpu)
5169	{
5170	struct worker_pool *pool;
5171	struct worker *worker;
5172
5173	for_each_cpu_worker_pool(pool, cpu) {
5174	mutex_lock(&wq_pool_attach_mutex);
5175	raw_spin_lock_irq(&pool->lock);
5176
5177	/*
5178	* We've blocked all attach/detach operations. Make all workers
5179	* unbound and set DISASSOCIATED. Before this, all workers
5180	* must be on the cpu. After this, they may become diasporas.
5181	* And the preemption disabled section in their sched callbacks
5182	* are guaranteed to see WORKER_UNBOUND since the code here
5183	* is on the same cpu.
5184	*/
5185	for_each_pool_worker(worker, pool)
5186	worker->flags \|= WORKER_UNBOUND;
5187
5188	pool->flags \|= POOL_DISASSOCIATED;
5189
5190	/*
5191	* The handling of nr_running in sched callbacks are disabled
5192	* now. Zap nr_running. After this, nr_running stays zero and
5193	* need_more_worker() and keep_working() are always true as
5194	* long as the worklist is not empty. This pool now behaves as
5195	* an unbound (in terms of concurrency management) pool which
5196	* are served by workers tied to the pool.
5197	*/
5198	pool->nr_running = 0;
5199
5200	/*
5201	* With concurrency management just turned off, a busy
5202	* worker blocking could lead to lengthy stalls. Kick off
5203	* unbound chain execution of currently pending work items.
5204	*/
5205	wake_up_worker(pool);
5206
5207	raw_spin_unlock_irq(&pool->lock);
5208
5209	for_each_pool_worker(worker, pool)
5210	unbind_worker(worker);
5211
5212	mutex_unlock(&wq_pool_attach_mutex);
5213	}
5214	}
5215
5216	/**
5217	* rebind_workers - rebind all workers of a pool to the associated CPU
5218	* @pool: pool of interest
5219	*
5220	* @pool->cpu is coming online. Rebind all workers to the CPU.
5221	*/
5222	static void rebind_workers(struct worker_pool *pool)
5223	{
5224	struct worker *worker;
5225
5226	lockdep_assert_held(&wq_pool_attach_mutex);
5227
5228	/*
5229	* Restore CPU affinity of all workers. As all idle workers should
5230	* be on the run-queue of the associated CPU before any local
5231	* wake-ups for concurrency management happen, restore CPU affinity
5232	* of all workers first and then clear UNBOUND. As we're called
5233	* from CPU_ONLINE, the following shouldn't fail.
5234	*/
5235	for_each_pool_worker(worker, pool) {
5236	kthread_set_per_cpu(worker->task, pool->cpu);
5237	WARN_ON_ONCE(set_cpus_allowed_ptr(worker->task,
5238	pool->attrs->cpumask) < 0);
5239	}
5240
5241	raw_spin_lock_irq(&pool->lock);
5242
5243	pool->flags &= ~POOL_DISASSOCIATED;
5244
5245	for_each_pool_worker(worker, pool) {
5246	unsigned int worker_flags = worker->flags;
5247
5248	/*
5249	* We want to clear UNBOUND but can't directly call
5250	* worker_clr_flags() or adjust nr_running. Atomically
5251	* replace UNBOUND with another NOT_RUNNING flag REBOUND.
5252	* @worker will clear REBOUND using worker_clr_flags() when
5253	* it initiates the next execution cycle thus restoring
5254	* concurrency management. Note that when or whether
5255	* @worker clears REBOUND doesn't affect correctness.
5256	*
5257	* WRITE_ONCE() is necessary because @worker->flags may be
5258	* tested without holding any lock in
5259	* wq_worker_running(). Without it, NOT_RUNNING test may
5260	* fail incorrectly leading to premature concurrency
5261	* management operations.
5262	*/
5263	WARN_ON_ONCE(!(worker_flags & WORKER_UNBOUND));
5264	worker_flags \|= WORKER_REBOUND;
5265	worker_flags &= ~WORKER_UNBOUND;
5266	WRITE_ONCE(worker->flags, worker_flags);
5267	}
5268
5269	raw_spin_unlock_irq(&pool->lock);
5270	}
5271
5272	/**
5273	* restore_unbound_workers_cpumask - restore cpumask of unbound workers
5274	* @pool: unbound pool of interest
5275	* @cpu: the CPU which is coming up
5276	*
5277	* An unbound pool may end up with a cpumask which doesn't have any online
5278	* CPUs. When a worker of such pool get scheduled, the scheduler resets
5279	* its cpus_allowed. If @cpu is in @pool's cpumask which didn't have any
5280	* online CPU before, cpus_allowed of all its workers should be restored.
5281	*/
5282	static void restore_unbound_workers_cpumask(struct worker_pool *pool, int cpu)
5283	{
5284	static cpumask_t cpumask;
5285	struct worker *worker;
5286
5287	lockdep_assert_held(&wq_pool_attach_mutex);
5288
5289	/* is @cpu allowed for @pool? */
5290	if (!cpumask_test_cpu(cpu, pool->attrs->cpumask))
5291	return;
5292
5293	cpumask_and(&cpumask, pool->attrs->cpumask, cpu_online_mask);
5294
5295	/* as we're called from CPU_ONLINE, the following shouldn't fail */
5296	for_each_pool_worker(worker, pool)
5297	WARN_ON_ONCE(set_cpus_allowed_ptr(worker->task, &cpumask) < 0);
5298	}
5299
5300	int workqueue_prepare_cpu(unsigned int cpu)
5301	{
5302	struct worker_pool *pool;
5303
5304	for_each_cpu_worker_pool(pool, cpu) {
5305	if (pool->nr_workers)
5306	continue;
5307	if (!create_worker(pool))
5308	return -ENOMEM;
5309	}
5310	return 0;
5311	}
5312
5313	int workqueue_online_cpu(unsigned int cpu)
5314	{
5315	struct worker_pool *pool;
5316	struct workqueue_struct *wq;
5317	int pi;
5318
5319	mutex_lock(&wq_pool_mutex);
5320
5321	for_each_pool(pool, pi) {
5322	mutex_lock(&wq_pool_attach_mutex);
5323
5324	if (pool->cpu == cpu)
5325	rebind_workers(pool);
5326	else if (pool->cpu < 0)
5327	restore_unbound_workers_cpumask(pool, cpu);
5328
5329	mutex_unlock(&wq_pool_attach_mutex);
5330	}
5331
5332	/* update NUMA affinity of unbound workqueues */
5333	list_for_each_entry(wq, &workqueues, list)
5334	wq_update_unbound_numa(wq, cpu, true);
5335
5336	mutex_unlock(&wq_pool_mutex);
5337	return 0;
5338	}
5339
5340	int workqueue_offline_cpu(unsigned int cpu)
5341	{
5342	struct workqueue_struct *wq;
5343
5344	/* unbinding per-cpu workers should happen on the local CPU */
5345	if (WARN_ON(cpu != smp_processor_id()))
5346	return -1;
5347
5348	unbind_workers(cpu);
5349
5350	/* update NUMA affinity of unbound workqueues */
5351	mutex_lock(&wq_pool_mutex);
5352	list_for_each_entry(wq, &workqueues, list)
5353	wq_update_unbound_numa(wq, cpu, false);
5354	mutex_unlock(&wq_pool_mutex);
5355
5356	return 0;
5357	}
5358
5359	struct work_for_cpu {
5360	struct work_struct work;
5361	long (fn)(void );
5362	void *arg;
5363	long ret;
5364	};
5365
5366	static void work_for_cpu_fn(struct work_struct *work)
5367	{
5368	struct work_for_cpu *wfc = container_of(work, struct work_for_cpu, work);
5369
5370	wfc->ret = wfc->fn(wfc->arg);
5371	}
5372
5373	/**
5374	* work_on_cpu - run a function in thread context on a particular cpu
5375	* @cpu: the cpu to run on
5376	* @fn: the function to run
5377	* @arg: the function arg
5378	*
5379	* It is up to the caller to ensure that the cpu doesn't go offline.
5380	* The caller must not hold any locks which would prevent @fn from completing.
5381	*
5382	* Return: The value @fn returns.
5383	*/
5384	long work_on_cpu(int cpu, long (fn)(void ), void *arg)
5385	{
5386	struct work_for_cpu wfc = { .fn = fn, .arg = arg };
5387
5388	INIT_WORK_ONSTACK(&wfc.work, work_for_cpu_fn);
5389	schedule_work_on(cpu, &wfc.work);
5390	flush_work(&wfc.work);
5391	destroy_work_on_stack(&wfc.work);
5392	return wfc.ret;
5393	}
5394	EXPORT_SYMBOL_GPL(work_on_cpu);
5395
5396	/**
5397	* work_on_cpu_safe - run a function in thread context on a particular cpu
5398	* @cpu: the cpu to run on
5399	* @fn: the function to run
5400	* @arg: the function argument
5401	*
5402	* Disables CPU hotplug and calls work_on_cpu(). The caller must not hold
5403	* any locks which would prevent @fn from completing.
5404	*
5405	* Return: The value @fn returns.
5406	*/
5407	long work_on_cpu_safe(int cpu, long (fn)(void ), void *arg)
5408	{
5409	long ret = -ENODEV;
5410
5411	cpus_read_lock();
5412	if (cpu_online(cpu))
5413	ret = work_on_cpu(cpu, fn, arg);
5414	cpus_read_unlock();
5415	return ret;
5416	}
5417	EXPORT_SYMBOL_GPL(work_on_cpu_safe);
5418	#endif /* CONFIG_SMP */
5419
5420	#ifdef CONFIG_FREEZER
5421
5422	/**
5423	* freeze_workqueues_begin - begin freezing workqueues
5424	*
5425	* Start freezing workqueues. After this function returns, all freezable
5426	* workqueues will queue new works to their inactive_works list instead of
5427	* pool->worklist.
5428	*
5429	* CONTEXT:
5430	* Grabs and releases wq_pool_mutex, wq->mutex and pool->lock's.
5431	*/
5432	void freeze_workqueues_begin(void)
5433	{
5434	struct workqueue_struct *wq;
5435	struct pool_workqueue *pwq;
5436
5437	mutex_lock(&wq_pool_mutex);
5438
5439	WARN_ON_ONCE(workqueue_freezing);
5440	workqueue_freezing = true;
5441
5442	list_for_each_entry(wq, &workqueues, list) {
5443	mutex_lock(&wq->mutex);
5444	for_each_pwq(pwq, wq)
5445	pwq_adjust_max_active(pwq);
5446	mutex_unlock(&wq->mutex);
5447	}
5448
5449	mutex_unlock(&wq_pool_mutex);
5450	}
5451
5452	/**
5453	* freeze_workqueues_busy - are freezable workqueues still busy?
5454	*
5455	* Check whether freezing is complete. This function must be called
5456	* between freeze_workqueues_begin() and thaw_workqueues().
5457	*
5458	* CONTEXT:
5459	* Grabs and releases wq_pool_mutex.
5460	*
5461	* Return:
5462	* %true if some freezable workqueues are still busy. %false if freezing
5463	* is complete.
5464	*/
5465	bool freeze_workqueues_busy(void)
5466	{
5467	bool busy = false;
5468	struct workqueue_struct *wq;
5469	struct pool_workqueue *pwq;
5470
5471	mutex_lock(&wq_pool_mutex);
5472
5473	WARN_ON_ONCE(!workqueue_freezing);
5474
5475	list_for_each_entry(wq, &workqueues, list) {
5476	if (!(wq->flags & WQ_FREEZABLE))
5477	continue;
5478	/*
5479	* nr_active is monotonically decreasing. It's safe
5480	* to peek without lock.
5481	*/
5482	rcu_read_lock();
5483	for_each_pwq(pwq, wq) {
5484	WARN_ON_ONCE(pwq->nr_active < 0);
5485	if (pwq->nr_active) {
5486	busy = true;
5487	rcu_read_unlock();
5488	goto out_unlock;
5489	}
5490	}
5491	rcu_read_unlock();
5492	}
5493	out_unlock:
5494	mutex_unlock(&wq_pool_mutex);
5495	return busy;
5496	}
5497
5498	/**
5499	* thaw_workqueues - thaw workqueues
5500	*
5501	* Thaw workqueues. Normal queueing is restored and all collected
5502	* frozen works are transferred to their respective pool worklists.
5503	*
5504	* CONTEXT:
5505	* Grabs and releases wq_pool_mutex, wq->mutex and pool->lock's.
5506	*/
5507	void thaw_workqueues(void)
5508	{
5509	struct workqueue_struct *wq;
5510	struct pool_workqueue *pwq;
5511
5512	mutex_lock(&wq_pool_mutex);
5513
5514	if (!workqueue_freezing)
5515	goto out_unlock;
5516
5517	workqueue_freezing = false;
5518
5519	/* restore max_active and repopulate worklist */
5520	list_for_each_entry(wq, &workqueues, list) {
5521	mutex_lock(&wq->mutex);
5522	for_each_pwq(pwq, wq)
5523	pwq_adjust_max_active(pwq);
5524	mutex_unlock(&wq->mutex);
5525	}
5526
5527	out_unlock:
5528	mutex_unlock(&wq_pool_mutex);
5529	}
5530	#endif /* CONFIG_FREEZER */
5531
5532	static int workqueue_apply_unbound_cpumask(const cpumask_var_t unbound_cpumask)
5533	{
5534	LIST_HEAD(ctxs);
5535	int ret = 0;
5536	struct workqueue_struct *wq;
5537	struct apply_wqattrs_ctx ctx, n;
5538
5539	lockdep_assert_held(&wq_pool_mutex);
5540
5541	list_for_each_entry(wq, &workqueues, list) {
5542	if (!(wq->flags & WQ_UNBOUND))
5543	continue;
5544	/* creating multiple pwqs breaks ordering guarantee */
5545	if (wq->flags & __WQ_ORDERED)
5546	continue;
5547
5548	ctx = apply_wqattrs_prepare(wq, wq->unbound_attrs, unbound_cpumask);
5549	if (!ctx) {
5550	ret = -ENOMEM;
5551	break;
5552	}
5553
5554	list_add_tail(&ctx->list, &ctxs);
5555	}
5556
5557	list_for_each_entry_safe(ctx, n, &ctxs, list) {
5558	if (!ret)
5559	apply_wqattrs_commit(ctx);
5560	apply_wqattrs_cleanup(ctx);
5561	}
5562
5563	if (!ret) {
5564	mutex_lock(&wq_pool_attach_mutex);
5565	cpumask_copy(wq_unbound_cpumask, unbound_cpumask);
5566	mutex_unlock(&wq_pool_attach_mutex);
5567	}
5568	return ret;
5569	}
5570
5571	/**
5572	* workqueue_set_unbound_cpumask - Set the low-level unbound cpumask
5573	* @cpumask: the cpumask to set
5574	*
5575	* The low-level workqueues cpumask is a global cpumask that limits
5576	* the affinity of all unbound workqueues. This function check the @cpumask
5577	* and apply it to all unbound workqueues and updates all pwqs of them.
5578	*
5579	* Return: 0 - Success
5580	* -EINVAL - Invalid @cpumask
5581	* -ENOMEM - Failed to allocate memory for attrs or pwqs.
5582	*/
5583	int workqueue_set_unbound_cpumask(cpumask_var_t cpumask)
5584	{
5585	int ret = -EINVAL;
5586
5587	/*
5588	* Not excluding isolated cpus on purpose.
5589	* If the user wishes to include them, we allow that.
5590	*/
5591	cpumask_and(cpumask, cpumask, cpu_possible_mask);
5592	if (!cpumask_empty(cpumask)) {
5593	apply_wqattrs_lock();
5594	if (cpumask_equal(cpumask, wq_unbound_cpumask)) {
5595	ret = 0;
5596	goto out_unlock;
5597	}
5598
5599	ret = workqueue_apply_unbound_cpumask(cpumask);
5600
5601	out_unlock:
5602	apply_wqattrs_unlock();
5603	}
5604
5605	return ret;
5606	}
5607
5608	#ifdef CONFIG_SYSFS
5609	/*
5610	* Workqueues with WQ_SYSFS flag set is visible to userland via
5611	* /sys/bus/workqueue/devices/WQ_NAME. All visible workqueues have the
5612	* following attributes.
5613	*
5614	* per_cpu RO bool : whether the workqueue is per-cpu or unbound
5615	* max_active RW int : maximum number of in-flight work items
5616	*
5617	* Unbound workqueues have the following extra attributes.
5618	*
5619	* pool_ids RO int : the associated pool IDs for each node
5620	* nice RW int : nice value of the workers
5621	* cpumask RW mask : bitmask of allowed CPUs for the workers
5622	* numa RW bool : whether enable NUMA affinity
5623	*/
5624	struct wq_device {
5625	struct workqueue_struct *wq;
5626	struct device dev;
5627	};
5628
5629	static struct workqueue_struct dev_to_wq(struct device dev)
5630	{
5631	struct wq_device *wq_dev = container_of(dev, struct wq_device, dev);
5632
5633	return wq_dev->wq;
5634	}
5635
5636	static ssize_t per_cpu_show(struct device dev, struct device_attribute attr,
5637	char *buf)
5638	{
5639	struct workqueue_struct *wq = dev_to_wq(dev);
5640
5641	return scnprintf(buf, PAGE_SIZE, "%d\n", (bool)!(wq->flags & WQ_UNBOUND));
5642	}
5643	static DEVICE_ATTR_RO(per_cpu);
5644
5645	static ssize_t max_active_show(struct device *dev,
5646	struct device_attribute attr, char buf)
5647	{
5648	struct workqueue_struct *wq = dev_to_wq(dev);
5649
5650	return scnprintf(buf, PAGE_SIZE, "%d\n", wq->saved_max_active);
5651	}
5652
5653	static ssize_t max_active_store(struct device *dev,
5654	struct device_attribute attr, const char buf,
5655	size_t count)
5656	{
5657	struct workqueue_struct *wq = dev_to_wq(dev);
5658	int val;
5659
5660	if (sscanf(buf, "%d", &val) != 1 \|\| val <= 0)
5661	return -EINVAL;
5662
5663	workqueue_set_max_active(wq, val);
5664	return count;
5665	}
5666	static DEVICE_ATTR_RW(max_active);
5667
5668	static struct attribute *wq_sysfs_attrs[] = {
5669	&dev_attr_per_cpu.attr,
5670	&dev_attr_max_active.attr,
5671	NULL,
5672	};
5673	ATTRIBUTE_GROUPS(wq_sysfs);
5674
5675	static ssize_t wq_pool_ids_show(struct device *dev,
5676	struct device_attribute attr, char buf)
5677	{
5678	struct workqueue_struct *wq = dev_to_wq(dev);
5679	const char *delim = "";
5680	int node, written = 0;
5681
5682	cpus_read_lock();
5683	rcu_read_lock();
5684	for_each_node(node) {
5685	written += scnprintf(buf + written, PAGE_SIZE - written,
5686	"%s%d:%d", delim, node,
5687	unbound_pwq_by_node(wq, node)->pool->id);
5688	delim = " ";
5689	}
5690	written += scnprintf(buf + written, PAGE_SIZE - written, "\n");
5691	rcu_read_unlock();
5692	cpus_read_unlock();
5693
5694	return written;
5695	}
5696
5697	static ssize_t wq_nice_show(struct device dev, struct device_attribute attr,
5698	char *buf)
5699	{
5700	struct workqueue_struct *wq = dev_to_wq(dev);
5701	int written;
5702
5703	mutex_lock(&wq->mutex);
5704	written = scnprintf(buf, PAGE_SIZE, "%d\n", wq->unbound_attrs->nice);
5705	mutex_unlock(&wq->mutex);
5706
5707	return written;
5708	}
5709
5710	/* prepare workqueue_attrs for sysfs store operations */
5711	static struct workqueue_attrs wq_sysfs_prep_attrs(struct workqueue_struct wq)
5712	{
5713	struct workqueue_attrs *attrs;
5714
5715	lockdep_assert_held(&wq_pool_mutex);
5716
5717	attrs = alloc_workqueue_attrs();
5718	if (!attrs)
5719	return NULL;
5720
5721	copy_workqueue_attrs(attrs, wq->unbound_attrs);
5722	return attrs;
5723	}
5724
5725	static ssize_t wq_nice_store(struct device dev, struct device_attribute attr,
5726	const char *buf, size_t count)
5727	{
5728	struct workqueue_struct *wq = dev_to_wq(dev);
5729	struct workqueue_attrs *attrs;
5730	int ret = -ENOMEM;
5731
5732	apply_wqattrs_lock();
5733
5734	attrs = wq_sysfs_prep_attrs(wq);
5735	if (!attrs)
5736	goto out_unlock;
5737
5738	if (sscanf(buf, "%d", &attrs->nice) == 1 &&
5739	attrs->nice >= MIN_NICE && attrs->nice <= MAX_NICE)
5740	ret = apply_workqueue_attrs_locked(wq, attrs);
5741	else
5742	ret = -EINVAL;
5743
5744	out_unlock:
5745	apply_wqattrs_unlock();
5746	free_workqueue_attrs(attrs);
5747	return ret ?: count;
5748	}
5749
5750	static ssize_t wq_cpumask_show(struct device *dev,
5751	struct device_attribute attr, char buf)
5752	{
5753	struct workqueue_struct *wq = dev_to_wq(dev);
5754	int written;
5755
5756	mutex_lock(&wq->mutex);
5757	written = scnprintf(buf, PAGE_SIZE, "%*pb\n",
5758	cpumask_pr_args(wq->unbound_attrs->cpumask));
5759	mutex_unlock(&wq->mutex);
5760	return written;
5761	}
5762
5763	static ssize_t wq_cpumask_store(struct device *dev,
5764	struct device_attribute *attr,
5765	const char *buf, size_t count)
5766	{
5767	struct workqueue_struct *wq = dev_to_wq(dev);
5768	struct workqueue_attrs *attrs;
5769	int ret = -ENOMEM;
5770
5771	apply_wqattrs_lock();
5772
5773	attrs = wq_sysfs_prep_attrs(wq);
5774	if (!attrs)
5775	goto out_unlock;
5776
5777	ret = cpumask_parse(buf, attrs->cpumask);
5778	if (!ret)
5779	ret = apply_workqueue_attrs_locked(wq, attrs);
5780
5781	out_unlock:
5782	apply_wqattrs_unlock();
5783	free_workqueue_attrs(attrs);
5784	return ret ?: count;
5785	}
5786
5787	static ssize_t wq_numa_show(struct device dev, struct device_attribute attr,
5788	char *buf)
5789	{
5790	struct workqueue_struct *wq = dev_to_wq(dev);
5791	int written;
5792
5793	mutex_lock(&wq->mutex);
5794	written = scnprintf(buf, PAGE_SIZE, "%d\n",
5795	!wq->unbound_attrs->no_numa);
5796	mutex_unlock(&wq->mutex);
5797
5798	return written;
5799	}
5800
5801	static ssize_t wq_numa_store(struct device dev, struct device_attribute attr,
5802	const char *buf, size_t count)
5803	{
5804	struct workqueue_struct *wq = dev_to_wq(dev);
5805	struct workqueue_attrs *attrs;
5806	int v, ret = -ENOMEM;
5807
5808	apply_wqattrs_lock();
5809
5810	attrs = wq_sysfs_prep_attrs(wq);
5811	if (!attrs)
5812	goto out_unlock;
5813
5814	ret = -EINVAL;
5815	if (sscanf(buf, "%d", &v) == 1) {
5816	attrs->no_numa = !v;
5817	ret = apply_workqueue_attrs_locked(wq, attrs);
5818	}
5819
5820	out_unlock:
5821	apply_wqattrs_unlock();
5822	free_workqueue_attrs(attrs);
5823	return ret ?: count;
5824	}
5825
5826	static struct device_attribute wq_sysfs_unbound_attrs[] = {
5827	__ATTR(pool_ids, 0444, wq_pool_ids_show, NULL),
5828	__ATTR(nice, 0644, wq_nice_show, wq_nice_store),
5829	__ATTR(cpumask, 0644, wq_cpumask_show, wq_cpumask_store),
5830	__ATTR(numa, 0644, wq_numa_show, wq_numa_store),
5831	__ATTR_NULL,
5832	};
5833
5834	static struct bus_type wq_subsys = {
5835	.name = "workqueue",
5836	.dev_groups = wq_sysfs_groups,
5837	};
5838
5839	static ssize_t wq_unbound_cpumask_show(struct device *dev,
5840	struct device_attribute attr, char buf)
5841	{
5842	int written;
5843
5844	mutex_lock(&wq_pool_mutex);
5845	written = scnprintf(buf, PAGE_SIZE, "%*pb\n",
5846	cpumask_pr_args(wq_unbound_cpumask));
5847	mutex_unlock(&wq_pool_mutex);
5848
5849	return written;
5850	}
5851
5852	static ssize_t wq_unbound_cpumask_store(struct device *dev,
5853	struct device_attribute attr, const char buf, size_t count)
5854	{
5855	cpumask_var_t cpumask;
5856	int ret;
5857
5858	if (!zalloc_cpumask_var(&cpumask, GFP_KERNEL))
5859	return -ENOMEM;
5860
5861	ret = cpumask_parse(buf, cpumask);
5862	if (!ret)
5863	ret = workqueue_set_unbound_cpumask(cpumask);
5864
5865	free_cpumask_var(cpumask);
5866	return ret ? ret : count;
5867	}
5868
5869	static struct device_attribute wq_sysfs_cpumask_attr =
5870	__ATTR(cpumask, 0644, wq_unbound_cpumask_show,
5871	wq_unbound_cpumask_store);
5872
5873	static int __init wq_sysfs_init(void)
5874	{
5875	struct device *dev_root;
5876	int err;
5877
5878	err = subsys_virtual_register(&wq_subsys, NULL);
5879	if (err)
5880	return err;
5881
5882	dev_root = bus_get_dev_root(&wq_subsys);
5883	if (dev_root) {
5884	err = device_create_file(dev_root, &wq_sysfs_cpumask_attr);
5885	put_device(dev_root);
5886	}
5887	return err;
5888	}
5889	core_initcall(wq_sysfs_init);
5890
5891	static void wq_device_release(struct device *dev)
5892	{
5893	struct wq_device *wq_dev = container_of(dev, struct wq_device, dev);
5894
5895	kfree(wq_dev);
5896	}
5897
5898	/**
5899	* workqueue_sysfs_register - make a workqueue visible in sysfs
5900	* @wq: the workqueue to register
5901	*
5902	* Expose @wq in sysfs under /sys/bus/workqueue/devices.
5903	* alloc_workqueue*() automatically calls this function if WQ_SYSFS is set
5904	* which is the preferred method.
5905	*
5906	* Workqueue user should use this function directly iff it wants to apply
5907	* workqueue_attrs before making the workqueue visible in sysfs; otherwise,
5908	* apply_workqueue_attrs() may race against userland updating the
5909	* attributes.
5910	*
5911	* Return: 0 on success, -errno on failure.
5912	*/
5913	int workqueue_sysfs_register(struct workqueue_struct *wq)
5914	{
5915	struct wq_device *wq_dev;
5916	int ret;
5917
5918	/*
5919	* Adjusting max_active or creating new pwqs by applying
5920	* attributes breaks ordering guarantee. Disallow exposing ordered
5921	* workqueues.
5922	*/
5923	if (WARN_ON(wq->flags & __WQ_ORDERED_EXPLICIT))
5924	return -EINVAL;
5925
5926	wq->wq_dev = wq_dev = kzalloc(sizeof(*wq_dev), GFP_KERNEL);
5927	if (!wq_dev)
5928	return -ENOMEM;
5929
5930	wq_dev->wq = wq;
5931	wq_dev->dev.bus = &wq_subsys;
5932	wq_dev->dev.release = wq_device_release;
5933	dev_set_name(&wq_dev->dev, "%s", wq->name);
5934
5935	/*
5936	* unbound_attrs are created separately. Suppress uevent until
5937	* everything is ready.
5938	*/
5939	dev_set_uevent_suppress(&wq_dev->dev, true);
5940
5941	ret = device_register(&wq_dev->dev);
5942	if (ret) {
5943	put_device(&wq_dev->dev);
5944	wq->wq_dev = NULL;
5945	return ret;
5946	}
5947
5948	if (wq->flags & WQ_UNBOUND) {
5949	struct device_attribute *attr;
5950
5951	for (attr = wq_sysfs_unbound_attrs; attr->attr.name; attr++) {
5952	ret = device_create_file(&wq_dev->dev, attr);
5953	if (ret) {
5954	device_unregister(&wq_dev->dev);
5955	wq->wq_dev = NULL;
5956	return ret;
5957	}
5958	}
5959	}
5960
5961	dev_set_uevent_suppress(&wq_dev->dev, false);
5962	kobject_uevent(&wq_dev->dev.kobj, KOBJ_ADD);
5963	return 0;
5964	}
5965
5966	/**
5967	* workqueue_sysfs_unregister - undo workqueue_sysfs_register()
5968	* @wq: the workqueue to unregister
5969	*
5970	* If @wq is registered to sysfs by workqueue_sysfs_register(), unregister.
5971	*/
5972	static void workqueue_sysfs_unregister(struct workqueue_struct *wq)
5973	{
5974	struct wq_device *wq_dev = wq->wq_dev;
5975
5976	if (!wq->wq_dev)
5977	return;
5978
5979	wq->wq_dev = NULL;
5980	device_unregister(&wq_dev->dev);
5981	}
5982	#else /* CONFIG_SYSFS */
5983	static void workqueue_sysfs_unregister(struct workqueue_struct *wq) { }
5984	#endif /* CONFIG_SYSFS */
5985
5986	/*
5987	* Workqueue watchdog.
5988	*
5989	* Stall may be caused by various bugs - missing WQ_MEM_RECLAIM, illegal
5990	* flush dependency, a concurrency managed work item which stays RUNNING
5991	* indefinitely. Workqueue stalls can be very difficult to debug as the
5992	* usual warning mechanisms don't trigger and internal workqueue state is
5993	* largely opaque.
5994	*
5995	* Workqueue watchdog monitors all worker pools periodically and dumps
5996	* state if some pools failed to make forward progress for a while where
5997	* forward progress is defined as the first item on ->worklist changing.
5998	*
5999	* This mechanism is controlled through the kernel parameter
6000	* "workqueue.watchdog_thresh" which can be updated at runtime through the
6001	* corresponding sysfs parameter file.
6002	*/
6003	#ifdef CONFIG_WQ_WATCHDOG
6004
6005	static unsigned long wq_watchdog_thresh = 30;
6006	static struct timer_list wq_watchdog_timer;
6007
6008	static unsigned long wq_watchdog_touched = INITIAL_JIFFIES;
6009	static DEFINE_PER_CPU(unsigned long, wq_watchdog_touched_cpu) = INITIAL_JIFFIES;
6010
6011	/*
6012	* Show workers that might prevent the processing of pending work items.
6013	* The only candidates are CPU-bound workers in the running state.
6014	* Pending work items should be handled by another idle worker
6015	* in all other situations.
6016	*/
6017	static void show_cpu_pool_hog(struct worker_pool *pool)
6018	{
6019	struct worker *worker;
6020	unsigned long flags;
6021	int bkt;
6022
6023	raw_spin_lock_irqsave(&pool->lock, flags);
6024
6025	hash_for_each(pool->busy_hash, bkt, worker, hentry) {
6026	if (task_is_running(worker->task)) {
6027	/*
6028	* Defer printing to avoid deadlocks in console
6029	* drivers that queue work while holding locks
6030	* also taken in their write paths.
6031	*/
6032	printk_deferred_enter();
6033
6034	pr_info("pool %d:\n", pool->id);
6035	sched_show_task(worker->task);
6036
6037	printk_deferred_exit();
6038	}
6039	}
6040
6041	raw_spin_unlock_irqrestore(&pool->lock, flags);
6042	}
6043
6044	static void show_cpu_pools_hogs(void)
6045	{
6046	struct worker_pool *pool;
6047	int pi;
6048
6049	pr_info("Showing backtraces of running workers in stalled CPU-bound worker pools:\n");
6050
6051	rcu_read_lock();
6052
6053	for_each_pool(pool, pi) {
6054	if (pool->cpu_stall)
6055	show_cpu_pool_hog(pool);
6056
6057	}
6058
6059	rcu_read_unlock();
6060	}
6061
6062	static void wq_watchdog_reset_touched(void)
6063	{
6064	int cpu;
6065
6066	wq_watchdog_touched = jiffies;
6067	for_each_possible_cpu(cpu)
6068	per_cpu(wq_watchdog_touched_cpu, cpu) = jiffies;
6069	}
6070
6071	static void wq_watchdog_timer_fn(struct timer_list *unused)
6072	{
6073	unsigned long thresh = READ_ONCE(wq_watchdog_thresh) * HZ;
6074	bool lockup_detected = false;
6075	bool cpu_pool_stall = false;
6076	unsigned long now = jiffies;
6077	struct worker_pool *pool;
6078	int pi;
6079
6080	if (!thresh)
6081	return;
6082
6083	rcu_read_lock();
6084
6085	for_each_pool(pool, pi) {
6086	unsigned long pool_ts, touched, ts;
6087
6088	pool->cpu_stall = false;
6089	if (list_empty(&pool->worklist))
6090	continue;
6091
6092	/*
6093	* If a virtual machine is stopped by the host it can look to
6094	* the watchdog like a stall.
6095	*/
6096	kvm_check_and_clear_guest_paused();
6097
6098	/* get the latest of pool and touched timestamps */
6099	if (pool->cpu >= 0)
6100	touched = READ_ONCE(per_cpu(wq_watchdog_touched_cpu, pool->cpu));
6101	else
6102	touched = READ_ONCE(wq_watchdog_touched);
6103	pool_ts = READ_ONCE(pool->watchdog_ts);
6104
6105	if (time_after(pool_ts, touched))
6106	ts = pool_ts;
6107	else
6108	ts = touched;
6109
6110	/* did we stall? */
6111	if (time_after(now, ts + thresh)) {
6112	lockup_detected = true;
6113	if (pool->cpu >= 0) {
6114	pool->cpu_stall = true;
6115	cpu_pool_stall = true;
6116	}
6117	pr_emerg("BUG: workqueue lockup - pool");
6118	pr_cont_pool_info(pool);
6119	pr_cont(" stuck for %us!\n",
6120	jiffies_to_msecs(now - pool_ts) / 1000);
6121	}
6122
6123
6124	}
6125
6126	rcu_read_unlock();
6127
6128	if (lockup_detected)
6129	show_all_workqueues();
6130
6131	if (cpu_pool_stall)
6132	show_cpu_pools_hogs();
6133
6134	wq_watchdog_reset_touched();
6135	mod_timer(&wq_watchdog_timer, jiffies + thresh);
6136	}
6137
6138	notrace void wq_watchdog_touch(int cpu)
6139	{
6140	if (cpu >= 0)
6141	per_cpu(wq_watchdog_touched_cpu, cpu) = jiffies;
6142
6143	wq_watchdog_touched = jiffies;
6144	}
6145
6146	static void wq_watchdog_set_thresh(unsigned long thresh)
6147	{
6148	wq_watchdog_thresh = 0;
6149	del_timer_sync(&wq_watchdog_timer);
6150
6151	if (thresh) {
6152	wq_watchdog_thresh = thresh;
6153	wq_watchdog_reset_touched();
6154	mod_timer(&wq_watchdog_timer, jiffies + thresh * HZ);
6155	}
6156	}
6157
6158	static int wq_watchdog_param_set_thresh(const char *val,
6159	const struct kernel_param *kp)
6160	{
6161	unsigned long thresh;
6162	int ret;
6163
6164	ret = kstrtoul(val, 0, &thresh);
6165	if (ret)
6166	return ret;
6167
6168	if (system_wq)
6169	wq_watchdog_set_thresh(thresh);
6170	else
6171	wq_watchdog_thresh = thresh;
6172
6173	return 0;
6174	}
6175
6176	static const struct kernel_param_ops wq_watchdog_thresh_ops = {
6177	.set = wq_watchdog_param_set_thresh,
6178	.get = param_get_ulong,
6179	};
6180
6181	module_param_cb(watchdog_thresh, &wq_watchdog_thresh_ops, &wq_watchdog_thresh,
6182	0644);
6183
6184	static void wq_watchdog_init(void)
6185	{
6186	timer_setup(&wq_watchdog_timer, wq_watchdog_timer_fn, TIMER_DEFERRABLE);
6187	wq_watchdog_set_thresh(wq_watchdog_thresh);
6188	}
6189
6190	#else /* CONFIG_WQ_WATCHDOG */
6191
6192	static inline void wq_watchdog_init(void) { }
6193
6194	#endif /* CONFIG_WQ_WATCHDOG */
6195
6196	static void __init wq_numa_init(void)
6197	{
6198	cpumask_var_t *tbl;
6199	int node, cpu;
6200
6201	if (num_possible_nodes() <= 1)
6202	return;
6203
6204	if (wq_disable_numa) {
6205	pr_info("workqueue: NUMA affinity support disabled\n");
6206	return;
6207	}
6208
6209	for_each_possible_cpu(cpu) {
6210	if (WARN_ON(cpu_to_node(cpu) == NUMA_NO_NODE)) {
6211	pr_warn("workqueue: NUMA node mapping not available for cpu%d, disabling NUMA support\n", cpu);
6212	return;
6213	}
6214	}
6215
6216	wq_update_unbound_numa_attrs_buf = alloc_workqueue_attrs();
6217	BUG_ON(!wq_update_unbound_numa_attrs_buf);
6218
6219	/*
6220	* We want masks of possible CPUs of each node which isn't readily
6221	* available. Build one from cpu_to_node() which should have been
6222	* fully initialized by now.
6223	*/
6224	tbl = kcalloc(nr_node_ids, sizeof(tbl[0]), GFP_KERNEL);
6225	BUG_ON(!tbl);
6226
6227	for_each_node(node)
6228	BUG_ON(!zalloc_cpumask_var_node(&tbl[node], GFP_KERNEL,
6229	node_online(node) ? node : NUMA_NO_NODE));
6230
6231	for_each_possible_cpu(cpu) {
6232	node = cpu_to_node(cpu);
6233	cpumask_set_cpu(cpu, tbl[node]);
6234	}
6235
6236	wq_numa_possible_cpumask = tbl;
6237	wq_numa_enabled = true;
6238	}
6239
6240	/**
6241	* workqueue_init_early - early init for workqueue subsystem
6242	*
6243	* This is the first half of two-staged workqueue subsystem initialization
6244	* and invoked as soon as the bare basics - memory allocation, cpumasks and
6245	* idr are up. It sets up all the data structures and system workqueues
6246	* and allows early boot code to create workqueues and queue/cancel work
6247	* items. Actual work item execution starts only after kthreads can be
6248	* created and scheduled right before early initcalls.
6249	*/
6250	void __init workqueue_init_early(void)
6251	{
6252	int std_nice[NR_STD_WORKER_POOLS] = { 0, HIGHPRI_NICE_LEVEL };
6253	int i, cpu;
6254
6255	BUILD_BUG_ON(__alignof__(struct pool_workqueue) < __alignof__(long long));
6256
6257	BUG_ON(!alloc_cpumask_var(&wq_unbound_cpumask, GFP_KERNEL));
6258	cpumask_copy(wq_unbound_cpumask, housekeeping_cpumask(HK_TYPE_WQ));
6259	cpumask_and(wq_unbound_cpumask, wq_unbound_cpumask, housekeeping_cpumask(HK_TYPE_DOMAIN));
6260
6261	pwq_cache = KMEM_CACHE(pool_workqueue, SLAB_PANIC);
6262
6263	/* initialize CPU pools */
6264	for_each_possible_cpu(cpu) {
6265	struct worker_pool *pool;
6266
6267	i = 0;
6268	for_each_cpu_worker_pool(pool, cpu) {
6269	BUG_ON(init_worker_pool(pool));
6270	pool->cpu = cpu;
6271	cpumask_copy(pool->attrs->cpumask, cpumask_of(cpu));
6272	pool->attrs->nice = std_nice[i++];
6273	pool->node = cpu_to_node(cpu);
6274
6275	/* alloc pool ID */
6276	mutex_lock(&wq_pool_mutex);
6277	BUG_ON(worker_pool_assign_id(pool));
6278	mutex_unlock(&wq_pool_mutex);
6279	}
6280	}
6281
6282	/* create default unbound and ordered wq attrs */
6283	for (i = 0; i < NR_STD_WORKER_POOLS; i++) {
6284	struct workqueue_attrs *attrs;
6285
6286	BUG_ON(!(attrs = alloc_workqueue_attrs()));
6287	attrs->nice = std_nice[i];
6288	unbound_std_wq_attrs[i] = attrs;
6289
6290	/*
6291	* An ordered wq should have only one pwq as ordering is
6292	* guaranteed by max_active which is enforced by pwqs.
6293	* Turn off NUMA so that dfl_pwq is used for all nodes.
6294	*/
6295	BUG_ON(!(attrs = alloc_workqueue_attrs()));
6296	attrs->nice = std_nice[i];
6297	attrs->no_numa = true;
6298	ordered_wq_attrs[i] = attrs;
6299	}
6300
6301	system_wq = alloc_workqueue("events", 0, 0);
6302	system_highpri_wq = alloc_workqueue("events_highpri", WQ_HIGHPRI, 0);
6303	system_long_wq = alloc_workqueue("events_long", 0, 0);
6304	system_unbound_wq = alloc_workqueue("events_unbound", WQ_UNBOUND,
6305	WQ_UNBOUND_MAX_ACTIVE);
6306	system_freezable_wq = alloc_workqueue("events_freezable",
6307	WQ_FREEZABLE, 0);
6308	system_power_efficient_wq = alloc_workqueue("events_power_efficient",
6309	WQ_POWER_EFFICIENT, 0);
6310	system_freezable_power_efficient_wq = alloc_workqueue("events_freezable_power_efficient",
6311	WQ_FREEZABLE \| WQ_POWER_EFFICIENT,
6312	0);
6313	BUG_ON(!system_wq \|\| !system_highpri_wq \|\| !system_long_wq \|\|
6314	!system_unbound_wq \|\| !system_freezable_wq \|\|
6315	!system_power_efficient_wq \|\|
6316	!system_freezable_power_efficient_wq);
6317	}
6318
6319	/**
6320	* workqueue_init - bring workqueue subsystem fully online
6321	*
6322	* This is the latter half of two-staged workqueue subsystem initialization
6323	* and invoked as soon as kthreads can be created and scheduled.
6324	* Workqueues have been created and work items queued on them, but there
6325	* are no kworkers executing the work items yet. Populate the worker pools
6326	* with the initial workers and enable future kworker creations.
6327	*/
6328	void __init workqueue_init(void)
6329	{
6330	struct workqueue_struct *wq;
6331	struct worker_pool *pool;
6332	int cpu, bkt;
6333
6334	/*
6335	* It'd be simpler to initialize NUMA in workqueue_init_early() but
6336	* CPU to node mapping may not be available that early on some
6337	* archs such as power and arm64. As per-cpu pools created
6338	* previously could be missing node hint and unbound pools NUMA
6339	* affinity, fix them up.
6340	*
6341	* Also, while iterating workqueues, create rescuers if requested.
6342	*/
6343	wq_numa_init();
6344
6345	mutex_lock(&wq_pool_mutex);
6346
6347	for_each_possible_cpu(cpu) {
6348	for_each_cpu_worker_pool(pool, cpu) {
6349	pool->node = cpu_to_node(cpu);
6350	}
6351	}
6352
6353	list_for_each_entry(wq, &workqueues, list) {
6354	wq_update_unbound_numa(wq, smp_processor_id(), true);
6355	WARN(init_rescuer(wq),
6356	"workqueue: failed to create early rescuer for %s",
6357	wq->name);
6358	}
6359
6360	mutex_unlock(&wq_pool_mutex);
6361
6362	/* create the initial workers */
6363	for_each_online_cpu(cpu) {
6364	for_each_cpu_worker_pool(pool, cpu) {
6365	pool->flags &= ~POOL_DISASSOCIATED;
6366	BUG_ON(!create_worker(pool));
6367	}
6368	}
6369
6370	hash_for_each(unbound_pool_hash, bkt, pool, hash_node)
6371	BUG_ON(!create_worker(pool));
6372
6373	wq_online = true;
6374	wq_watchdog_init();
6375	}
6376
6377	/*
6378	* Despite the naming, this is a no-op function which is here only for avoiding
6379	* link error. Since compile-time warning may fail to catch, we will need to
6380	* emit run-time warning from __flush_workqueue().
6381	*/
6382	void __warn_flushing_systemwide_wq(void) { }
6383	EXPORT_SYMBOL(__warn_flushing_systemwide_wq);