linux - kernel/workqueue.c

5709 lines · 156.8 KiB

1	/*
2	* kernel/workqueue.c - generic async execution with shared worker pool
3	*
4	* Copyright (C) 2002 Ingo Molnar
5	*
6	* Derived from the taskqueue/keventd code by:
7	* David Woodhouse <[email protected]>
8	* Andrew Morton
9	* Kai Petzke <[email protected]>
10	* Theodore Ts'o <[email protected]>
11	*
12	* Made to use alloc_percpu by Christoph Lameter.
13	*
14	* Copyright (C) 2010 SUSE Linux Products GmbH
15	* Copyright (C) 2010 Tejun Heo <[email protected]>
16	*
17	* This is the generic async execution mechanism. Work items as are
18	* executed in process context. The worker pool is shared and
19	* automatically managed. There are two worker pools for each CPU (one for
20	* normal work items and the other for high priority ones) and some extra
21	* pools for workqueues which are not bound to any specific CPU - the
22	* number of these backing pools is dynamic.
23	*
24	* Please read Documentation/core-api/workqueue.rst for details.
25	*/
26
27	#include <linux/export.h>
28	#include <linux/kernel.h>
29	#include <linux/sched.h>
30	#include <linux/init.h>
31	#include <linux/signal.h>
32	#include <linux/completion.h>
33	#include <linux/workqueue.h>
34	#include <linux/slab.h>
35	#include <linux/cpu.h>
36	#include <linux/notifier.h>
37	#include <linux/kthread.h>
38	#include <linux/hardirq.h>
39	#include <linux/mempolicy.h>
40	#include <linux/freezer.h>
41	#include <linux/debug_locks.h>
42	#include <linux/lockdep.h>
43	#include <linux/idr.h>
44	#include <linux/jhash.h>
45	#include <linux/hashtable.h>
46	#include <linux/rculist.h>
47	#include <linux/nodemask.h>
48	#include <linux/moduleparam.h>
49	#include <linux/uaccess.h>
50	#include <linux/sched/isolation.h>
51	#include <linux/nmi.h>
52
53	#include "workqueue_internal.h"
54
55	enum {
56	/*
57	* worker_pool flags
58	*
59	* A bound pool is either associated or disassociated with its CPU.
60	* While associated (!DISASSOCIATED), all workers are bound to the
61	* CPU and none has %WORKER_UNBOUND set and concurrency management
62	* is in effect.
63	*
64	* While DISASSOCIATED, the cpu may be offline and all workers have
65	* %WORKER_UNBOUND set and concurrency management disabled, and may
66	* be executing on any CPU. The pool behaves as an unbound one.
67	*
68	* Note that DISASSOCIATED should be flipped only while holding
69	* attach_mutex to avoid changing binding state while
70	* worker_attach_to_pool() is in progress.
71	*/
72	POOL_MANAGER_ACTIVE = 1 << 0, /* being managed */
73	POOL_DISASSOCIATED = 1 << 2, /* cpu can't serve workers */
74
75	/* worker flags */
76	WORKER_DIE = 1 << 1, /* die die die */
77	WORKER_IDLE = 1 << 2, /* is idle */
78	WORKER_PREP = 1 << 3, /* preparing to run works */
79	WORKER_CPU_INTENSIVE = 1 << 6, /* cpu intensive */
80	WORKER_UNBOUND = 1 << 7, /* worker is unbound */
81	WORKER_REBOUND = 1 << 8, /* worker was rebound */
82
83	WORKER_NOT_RUNNING = WORKER_PREP \| WORKER_CPU_INTENSIVE \|
84	WORKER_UNBOUND \| WORKER_REBOUND,
85
86	NR_STD_WORKER_POOLS = 2, /* # standard pools per cpu */
87
88	UNBOUND_POOL_HASH_ORDER = 6, /* hashed by pool->attrs */
89	BUSY_WORKER_HASH_ORDER = 6, /* 64 pointers */
90
91	MAX_IDLE_WORKERS_RATIO = 4, /* 1/4 of busy can be idle */
92	IDLE_WORKER_TIMEOUT = 300 * HZ, /* keep idle ones for 5 mins */
93
94	MAYDAY_INITIAL_TIMEOUT = HZ / 100 >= 2 ? HZ / 100 : 2,
95	/* call for help after 10ms
96	(min two ticks) */
97	MAYDAY_INTERVAL = HZ / 10, /* and then every 100ms */
98	CREATE_COOLDOWN = HZ, /* time to breath after fail */
99
100	/*
101	* Rescue workers are used only on emergencies and shared by
102	* all cpus. Give MIN_NICE.
103	*/
104	RESCUER_NICE_LEVEL = MIN_NICE,
105	HIGHPRI_NICE_LEVEL = MIN_NICE,
106
107	WQ_NAME_LEN = 24,
108	};
109
110	/*
111	* Structure fields follow one of the following exclusion rules.
112	*
113	* I: Modifiable by initialization/destruction paths and read-only for
114	* everyone else.
115	*
116	* P: Preemption protected. Disabling preemption is enough and should
117	* only be modified and accessed from the local cpu.
118	*
119	* L: pool->lock protected. Access with pool->lock held.
120	*
121	* X: During normal operation, modification requires pool->lock and should
122	* be done only from local cpu. Either disabling preemption on local
123	* cpu or grabbing pool->lock is enough for read access. If
124	* POOL_DISASSOCIATED is set, it's identical to L.
125	*
126	* A: pool->attach_mutex protected.
127	*
128	* PL: wq_pool_mutex protected.
129	*
130	* PR: wq_pool_mutex protected for writes. Sched-RCU protected for reads.
131	*
132	* PW: wq_pool_mutex and wq->mutex protected for writes. Either for reads.
133	*
134	* PWR: wq_pool_mutex and wq->mutex protected for writes. Either or
135	* sched-RCU for reads.
136	*
137	* WQ: wq->mutex protected.
138	*
139	* WR: wq->mutex protected for writes. Sched-RCU protected for reads.
140	*
141	* MD: wq_mayday_lock protected.
142	*/
143
144	/* struct worker is defined in workqueue_internal.h */
145
146	struct worker_pool {
147	spinlock_t lock; /* the pool lock */
148	int cpu; /* I: the associated cpu */
149	int node; /* I: the associated node ID */
150	int id; /* I: pool ID */
151	unsigned int flags; /* X: flags */
152
153	unsigned long watchdog_ts; /* L: watchdog timestamp */
154
155	struct list_head worklist; /* L: list of pending works */
156	int nr_workers; /* L: total number of workers */
157
158	/* nr_idle includes the ones off idle_list for rebinding */
159	int nr_idle; /* L: currently idle ones */
160
161	struct list_head idle_list; /* X: list of idle workers */
162	struct timer_list idle_timer; /* L: worker idle timeout */
163	struct timer_list mayday_timer; /* L: SOS timer for workers */
164
165	/* a workers is either on busy_hash or idle_list, or the manager */
166	DECLARE_HASHTABLE(busy_hash, BUSY_WORKER_HASH_ORDER);
167	/* L: hash of busy workers */
168
169	/* see manage_workers() for details on the two manager mutexes */
170	struct worker manager; / L: purely informational */
171	struct mutex attach_mutex; /* attach/detach exclusion */
172	struct list_head workers; /* A: attached workers */
173	struct completion detach_completion; / all workers detached */
174
175	struct ida worker_ida; /* worker IDs for task name */
176
177	struct workqueue_attrs attrs; / I: worker attributes */
178	struct hlist_node hash_node; /* PL: unbound_pool_hash node */
179	int refcnt; /* PL: refcnt for unbound pools */
180
181	/*
182	* The current concurrency level. As it's likely to be accessed
183	* from other CPUs during try_to_wake_up(), put it in a separate
184	* cacheline.
185	*/
186	atomic_t nr_running ____cacheline_aligned_in_smp;
187
188	/*
189	* Destruction of pool is sched-RCU protected to allow dereferences
190	* from get_work_pool().
191	*/
192	struct rcu_head rcu;
193	} ____cacheline_aligned_in_smp;
194
195	/*
196	* The per-pool workqueue. While queued, the lower WORK_STRUCT_FLAG_BITS
197	* of work_struct->data are used for flags and the remaining high bits
198	* point to the pwq; thus, pwqs need to be aligned at two's power of the
199	* number of flag bits.
200	*/
201	struct pool_workqueue {
202	struct worker_pool pool; / I: the associated pool */
203	struct workqueue_struct wq; / I: the owning workqueue */
204	int work_color; /* L: current color */
205	int flush_color; /* L: flushing color */
206	int refcnt; /* L: reference count */
207	int nr_in_flight[WORK_NR_COLORS];
208	/* L: nr of in_flight works */
209	int nr_active; /* L: nr of active works */
210	int max_active; /* L: max active works */
211	struct list_head delayed_works; /* L: delayed works */
212	struct list_head pwqs_node; /* WR: node on wq->pwqs */
213	struct list_head mayday_node; /* MD: node on wq->maydays */
214
215	/*
216	* Release of unbound pwq is punted to system_wq. See put_pwq()
217	* and pwq_unbound_release_workfn() for details. pool_workqueue
218	* itself is also sched-RCU protected so that the first pwq can be
219	* determined without grabbing wq->mutex.
220	*/
221	struct work_struct unbound_release_work;
222	struct rcu_head rcu;
223	} __aligned(1 << WORK_STRUCT_FLAG_BITS);
224
225	/*
226	* Structure used to wait for workqueue flush.
227	*/
228	struct wq_flusher {
229	struct list_head list; /* WQ: list of flushers */
230	int flush_color; /* WQ: flush color waiting for */
231	struct completion done; /* flush completion */
232	};
233
234	struct wq_device;
235
236	/*
237	* The externally visible workqueue. It relays the issued work items to
238	* the appropriate worker_pool through its pool_workqueues.
239	*/
240	struct workqueue_struct {
241	struct list_head pwqs; /* WR: all pwqs of this wq */
242	struct list_head list; /* PR: list of all workqueues */
243
244	struct mutex mutex; /* protects this wq */
245	int work_color; /* WQ: current work color */
246	int flush_color; /* WQ: current flush color */
247	atomic_t nr_pwqs_to_flush; /* flush in progress */
248	struct wq_flusher first_flusher; / WQ: first flusher */
249	struct list_head flusher_queue; /* WQ: flush waiters */
250	struct list_head flusher_overflow; /* WQ: flush overflow list */
251
252	struct list_head maydays; /* MD: pwqs requesting rescue */
253	struct worker rescuer; / I: rescue worker */
254
255	int nr_drainers; /* WQ: drain in progress */
256	int saved_max_active; /* WQ: saved pwq max_active */
257
258	struct workqueue_attrs unbound_attrs; / PW: only for unbound wqs */
259	struct pool_workqueue dfl_pwq; / PW: only for unbound wqs */
260
261	#ifdef CONFIG_SYSFS
262	struct wq_device wq_dev; / I: for sysfs interface */
263	#endif
264	#ifdef CONFIG_LOCKDEP
265	struct lockdep_map lockdep_map;
266	#endif
267	char name[WQ_NAME_LEN]; /* I: workqueue name */
268
269	/*
270	* Destruction of workqueue_struct is sched-RCU protected to allow
271	* walking the workqueues list without grabbing wq_pool_mutex.
272	* This is used to dump all workqueues from sysrq.
273	*/
274	struct rcu_head rcu;
275
276	/* hot fields used during command issue, aligned to cacheline */
277	unsigned int flags ____cacheline_aligned; /* WQ: WQ_* flags */
278	struct pool_workqueue __percpu cpu_pwqs; / I: per-cpu pwqs */
279	struct pool_workqueue __rcu numa_pwq_tbl[]; / PWR: unbound pwqs indexed by node */
280	};
281
282	static struct kmem_cache *pwq_cache;
283
284	static cpumask_var_t *wq_numa_possible_cpumask;
285	/* possible CPUs of each node */
286
287	static bool wq_disable_numa;
288	module_param_named(disable_numa, wq_disable_numa, bool, 0444);
289
290	/* see the comment above the definition of WQ_POWER_EFFICIENT */
291	static bool wq_power_efficient = IS_ENABLED(CONFIG_WQ_POWER_EFFICIENT_DEFAULT);
292	module_param_named(power_efficient, wq_power_efficient, bool, 0444);
293
294	static bool wq_online; /* can kworkers be created yet? */
295
296	static bool wq_numa_enabled; /* unbound NUMA affinity enabled */
297
298	/* buf for wq_update_unbound_numa_attrs(), protected by CPU hotplug exclusion */
299	static struct workqueue_attrs *wq_update_unbound_numa_attrs_buf;
300
301	static DEFINE_MUTEX(wq_pool_mutex); /* protects pools and workqueues list */
302	static DEFINE_SPINLOCK(wq_mayday_lock); /* protects wq->maydays list */
303	static DECLARE_WAIT_QUEUE_HEAD(wq_manager_wait); /* wait for manager to go away */
304
305	static LIST_HEAD(workqueues); /* PR: list of all workqueues */
306	static bool workqueue_freezing; /* PL: have wqs started freezing? */
307
308	/* PL: allowable cpus for unbound wqs and work items */
309	static cpumask_var_t wq_unbound_cpumask;
310
311	/* CPU where unbound work was last round robin scheduled from this CPU */
312	static DEFINE_PER_CPU(int, wq_rr_cpu_last);
313
314	/*
315	* Local execution of unbound work items is no longer guaranteed. The
316	* following always forces round-robin CPU selection on unbound work items
317	* to uncover usages which depend on it.
318	*/
319	#ifdef CONFIG_DEBUG_WQ_FORCE_RR_CPU
320	static bool wq_debug_force_rr_cpu = true;
321	#else
322	static bool wq_debug_force_rr_cpu = false;
323	#endif
324	module_param_named(debug_force_rr_cpu, wq_debug_force_rr_cpu, bool, 0644);
325
326	/* the per-cpu worker pools */
327	static DEFINE_PER_CPU_SHARED_ALIGNED(struct worker_pool [NR_STD_WORKER_POOLS], cpu_worker_pools);
328
329	static DEFINE_IDR(worker_pool_idr); /* PR: idr of all pools */
330
331	/* PL: hash of all unbound pools keyed by pool->attrs */
332	static DEFINE_HASHTABLE(unbound_pool_hash, UNBOUND_POOL_HASH_ORDER);
333
334	/* I: attributes used when instantiating standard unbound pools on demand */
335	static struct workqueue_attrs *unbound_std_wq_attrs[NR_STD_WORKER_POOLS];
336
337	/* I: attributes used when instantiating ordered pools on demand */
338	static struct workqueue_attrs *ordered_wq_attrs[NR_STD_WORKER_POOLS];
339
340	struct workqueue_struct *system_wq __read_mostly;
341	EXPORT_SYMBOL(system_wq);
342	struct workqueue_struct *system_highpri_wq __read_mostly;
343	EXPORT_SYMBOL_GPL(system_highpri_wq);
344	struct workqueue_struct *system_long_wq __read_mostly;
345	EXPORT_SYMBOL_GPL(system_long_wq);
346	struct workqueue_struct *system_unbound_wq __read_mostly;
347	EXPORT_SYMBOL_GPL(system_unbound_wq);
348	struct workqueue_struct *system_freezable_wq __read_mostly;
349	EXPORT_SYMBOL_GPL(system_freezable_wq);
350	struct workqueue_struct *system_power_efficient_wq __read_mostly;
351	EXPORT_SYMBOL_GPL(system_power_efficient_wq);
352	struct workqueue_struct *system_freezable_power_efficient_wq __read_mostly;
353	EXPORT_SYMBOL_GPL(system_freezable_power_efficient_wq);
354
355	static int worker_thread(void *__worker);
356	static void workqueue_sysfs_unregister(struct workqueue_struct *wq);
357
358	#define CREATE_TRACE_POINTS
359	#include <trace/events/workqueue.h>
360
361	#define assert_rcu_or_pool_mutex() \
362	RCU_LOCKDEP_WARN(!rcu_read_lock_sched_held() && \
363	!lockdep_is_held(&wq_pool_mutex), \
364	"sched RCU or wq_pool_mutex should be held")
365
366	#define assert_rcu_or_wq_mutex(wq) \
367	RCU_LOCKDEP_WARN(!rcu_read_lock_sched_held() && \
368	!lockdep_is_held(&wq->mutex), \
369	"sched RCU or wq->mutex should be held")
370
371	#define assert_rcu_or_wq_mutex_or_pool_mutex(wq) \
372	RCU_LOCKDEP_WARN(!rcu_read_lock_sched_held() && \
373	!lockdep_is_held(&wq->mutex) && \
374	!lockdep_is_held(&wq_pool_mutex), \
375	"sched RCU, wq->mutex or wq_pool_mutex should be held")
376
377	#define for_each_cpu_worker_pool(pool, cpu) \
378	for ((pool) = &per_cpu(cpu_worker_pools, cpu)[0]; \
379	(pool) < &per_cpu(cpu_worker_pools, cpu)[NR_STD_WORKER_POOLS]; \
380	(pool)++)
381
382	/**
383	* for_each_pool - iterate through all worker_pools in the system
384	* @pool: iteration cursor
385	* @pi: integer used for iteration
386	*
387	* This must be called either with wq_pool_mutex held or sched RCU read
388	* locked. If the pool needs to be used beyond the locking in effect, the
389	* caller is responsible for guaranteeing that the pool stays online.
390	*
391	* The if/else clause exists only for the lockdep assertion and can be
392	* ignored.
393	*/
394	#define for_each_pool(pool, pi) \
395	idr_for_each_entry(&worker_pool_idr, pool, pi) \
396	if (({ assert_rcu_or_pool_mutex(); false; })) { } \
397	else
398
399	/**
400	* for_each_pool_worker - iterate through all workers of a worker_pool
401	* @worker: iteration cursor
402	* @pool: worker_pool to iterate workers of
403	*
404	* This must be called with @pool->attach_mutex.
405	*
406	* The if/else clause exists only for the lockdep assertion and can be
407	* ignored.
408	*/
409	#define for_each_pool_worker(worker, pool) \
410	list_for_each_entry((worker), &(pool)->workers, node) \
411	if (({ lockdep_assert_held(&pool->attach_mutex); false; })) { } \
412	else
413
414	/**
415	* for_each_pwq - iterate through all pool_workqueues of the specified workqueue
416	* @pwq: iteration cursor
417	* @wq: the target workqueue
418	*
419	* This must be called either with wq->mutex held or sched RCU read locked.
420	* If the pwq needs to be used beyond the locking in effect, the caller is
421	* responsible for guaranteeing that the pwq stays online.
422	*
423	* The if/else clause exists only for the lockdep assertion and can be
424	* ignored.
425	*/
426	#define for_each_pwq(pwq, wq) \
427	list_for_each_entry_rcu((pwq), &(wq)->pwqs, pwqs_node) \
428	if (({ assert_rcu_or_wq_mutex(wq); false; })) { } \
429	else
430
431	#ifdef CONFIG_DEBUG_OBJECTS_WORK
432
433	static struct debug_obj_descr work_debug_descr;
434
435	static void work_debug_hint(void addr)
436	{
437	return ((struct work_struct *) addr)->func;
438	}
439
440	static bool work_is_static_object(void *addr)
441	{
442	struct work_struct *work = addr;
443
444	return test_bit(WORK_STRUCT_STATIC_BIT, work_data_bits(work));
445	}
446
447	/*
448	* fixup_init is called when:
449	* - an active object is initialized
450	*/
451	static bool work_fixup_init(void *addr, enum debug_obj_state state)
452	{
453	struct work_struct *work = addr;
454
455	switch (state) {
456	case ODEBUG_STATE_ACTIVE:
457	cancel_work_sync(work);
458	debug_object_init(work, &work_debug_descr);
459	return true;
460	default:
461	return false;
462	}
463	}
464
465	/*
466	* fixup_free is called when:
467	* - an active object is freed
468	*/
469	static bool work_fixup_free(void *addr, enum debug_obj_state state)
470	{
471	struct work_struct *work = addr;
472
473	switch (state) {
474	case ODEBUG_STATE_ACTIVE:
475	cancel_work_sync(work);
476	debug_object_free(work, &work_debug_descr);
477	return true;
478	default:
479	return false;
480	}
481	}
482
483	static struct debug_obj_descr work_debug_descr = {
484	.name = "work_struct",
485	.debug_hint = work_debug_hint,
486	.is_static_object = work_is_static_object,
487	.fixup_init = work_fixup_init,
488	.fixup_free = work_fixup_free,
489	};
490
491	static inline void debug_work_activate(struct work_struct *work)
492	{
493	debug_object_activate(work, &work_debug_descr);
494	}
495
496	static inline void debug_work_deactivate(struct work_struct *work)
497	{
498	debug_object_deactivate(work, &work_debug_descr);
499	}
500
501	void __init_work(struct work_struct *work, int onstack)
502	{
503	if (onstack)
504	debug_object_init_on_stack(work, &work_debug_descr);
505	else
506	debug_object_init(work, &work_debug_descr);
507	}
508	EXPORT_SYMBOL_GPL(__init_work);
509
510	void destroy_work_on_stack(struct work_struct *work)
511	{
512	debug_object_free(work, &work_debug_descr);
513	}
514	EXPORT_SYMBOL_GPL(destroy_work_on_stack);
515
516	void destroy_delayed_work_on_stack(struct delayed_work *work)
517	{
518	destroy_timer_on_stack(&work->timer);
519	debug_object_free(&work->work, &work_debug_descr);
520	}
521	EXPORT_SYMBOL_GPL(destroy_delayed_work_on_stack);
522
523	#else
524	static inline void debug_work_activate(struct work_struct *work) { }
525	static inline void debug_work_deactivate(struct work_struct *work) { }
526	#endif
527
528	/**
529	* worker_pool_assign_id - allocate ID and assing it to @pool
530	* @pool: the pool pointer of interest
531	*
532	* Returns 0 if ID in [0, WORK_OFFQ_POOL_NONE) is allocated and assigned
533	* successfully, -errno on failure.
534	*/
535	static int worker_pool_assign_id(struct worker_pool *pool)
536	{
537	int ret;
538
539	lockdep_assert_held(&wq_pool_mutex);
540
541	ret = idr_alloc(&worker_pool_idr, pool, 0, WORK_OFFQ_POOL_NONE,
542	GFP_KERNEL);
543	if (ret >= 0) {
544	pool->id = ret;
545	return 0;
546	}
547	return ret;
548	}
549
550	/**
551	* unbound_pwq_by_node - return the unbound pool_workqueue for the given node
552	* @wq: the target workqueue
553	* @node: the node ID
554	*
555	* This must be called with any of wq_pool_mutex, wq->mutex or sched RCU
556	* read locked.
557	* If the pwq needs to be used beyond the locking in effect, the caller is
558	* responsible for guaranteeing that the pwq stays online.
559	*
560	* Return: The unbound pool_workqueue for @node.
561	*/
562	static struct pool_workqueue unbound_pwq_by_node(struct workqueue_struct wq,
563	int node)
564	{
565	assert_rcu_or_wq_mutex_or_pool_mutex(wq);
566
567	/*
568	* XXX: @node can be NUMA_NO_NODE if CPU goes offline while a
569	* delayed item is pending. The plan is to keep CPU -> NODE
570	* mapping valid and stable across CPU on/offlines. Once that
571	* happens, this workaround can be removed.
572	*/
573	if (unlikely(node == NUMA_NO_NODE))
574	return wq->dfl_pwq;
575
576	return rcu_dereference_raw(wq->numa_pwq_tbl[node]);
577	}
578
579	static unsigned int work_color_to_flags(int color)
580	{
581	return color << WORK_STRUCT_COLOR_SHIFT;
582	}
583
584	static int get_work_color(struct work_struct *work)
585	{
586	return (*work_data_bits(work) >> WORK_STRUCT_COLOR_SHIFT) &
587	((1 << WORK_STRUCT_COLOR_BITS) - 1);
588	}
589
590	static int work_next_color(int color)
591	{
592	return (color + 1) % WORK_NR_COLORS;
593	}
594
595	/*
596	* While queued, %WORK_STRUCT_PWQ is set and non flag bits of a work's data
597	* contain the pointer to the queued pwq. Once execution starts, the flag
598	* is cleared and the high bits contain OFFQ flags and pool ID.
599	*
600	* set_work_pwq(), set_work_pool_and_clear_pending(), mark_work_canceling()
601	* and clear_work_data() can be used to set the pwq, pool or clear
602	* work->data. These functions should only be called while the work is
603	* owned - ie. while the PENDING bit is set.
604	*
605	* get_work_pool() and get_work_pwq() can be used to obtain the pool or pwq
606	* corresponding to a work. Pool is available once the work has been
607	* queued anywhere after initialization until it is sync canceled. pwq is
608	* available only while the work item is queued.
609	*
610	* %WORK_OFFQ_CANCELING is used to mark a work item which is being
611	* canceled. While being canceled, a work item may have its PENDING set
612	* but stay off timer and worklist for arbitrarily long and nobody should
613	* try to steal the PENDING bit.
614	*/
615	static inline void set_work_data(struct work_struct *work, unsigned long data,
616	unsigned long flags)
617	{
618	WARN_ON_ONCE(!work_pending(work));
619	atomic_long_set(&work->data, data \| flags \| work_static(work));
620	}
621
622	static void set_work_pwq(struct work_struct work, struct pool_workqueue pwq,
623	unsigned long extra_flags)
624	{
625	set_work_data(work, (unsigned long)pwq,
626	WORK_STRUCT_PENDING \| WORK_STRUCT_PWQ \| extra_flags);
627	}
628
629	static void set_work_pool_and_keep_pending(struct work_struct *work,
630	int pool_id)
631	{
632	set_work_data(work, (unsigned long)pool_id << WORK_OFFQ_POOL_SHIFT,
633	WORK_STRUCT_PENDING);
634	}
635
636	static void set_work_pool_and_clear_pending(struct work_struct *work,
637	int pool_id)
638	{
639	/*
640	* The following wmb is paired with the implied mb in
641	* test_and_set_bit(PENDING) and ensures all updates to @work made
642	* here are visible to and precede any updates by the next PENDING
643	* owner.
644	*/
645	smp_wmb();
646	set_work_data(work, (unsigned long)pool_id << WORK_OFFQ_POOL_SHIFT, 0);
647	/*
648	* The following mb guarantees that previous clear of a PENDING bit
649	* will not be reordered with any speculative LOADS or STORES from
650	* work->current_func, which is executed afterwards. This possible
651	* reordering can lead to a missed execution on attempt to qeueue
652	* the same @work. E.g. consider this case:
653	*
654	* CPU#0 CPU#1
655	* ---------------------------- --------------------------------
656	*
657	* 1 STORE event_indicated
658	* 2 queue_work_on() {
659	* 3 test_and_set_bit(PENDING)
660	* 4 } set_..._and_clear_pending() {
661	* 5 set_work_data() # clear bit
662	* 6 smp_mb()
663	* 7 work->current_func() {
664	* 8 LOAD event_indicated
665	* }
666	*
667	* Without an explicit full barrier speculative LOAD on line 8 can
668	* be executed before CPU#0 does STORE on line 1. If that happens,
669	* CPU#0 observes the PENDING bit is still set and new execution of
670	* a @work is not queued in a hope, that CPU#1 will eventually
671	* finish the queued @work. Meanwhile CPU#1 does not see
672	* event_indicated is set, because speculative LOAD was executed
673	* before actual STORE.
674	*/
675	smp_mb();
676	}
677
678	static void clear_work_data(struct work_struct *work)
679	{
680	smp_wmb(); /* see set_work_pool_and_clear_pending() */
681	set_work_data(work, WORK_STRUCT_NO_POOL, 0);
682	}
683
684	static struct pool_workqueue get_work_pwq(struct work_struct work)
685	{
686	unsigned long data = atomic_long_read(&work->data);
687
688	if (data & WORK_STRUCT_PWQ)
689	return (void *)(data & WORK_STRUCT_WQ_DATA_MASK);
690	else
691	return NULL;
692	}
693
694	/**
695	* get_work_pool - return the worker_pool a given work was associated with
696	* @work: the work item of interest
697	*
698	* Pools are created and destroyed under wq_pool_mutex, and allows read
699	* access under sched-RCU read lock. As such, this function should be
700	* called under wq_pool_mutex or with preemption disabled.
701	*
702	* All fields of the returned pool are accessible as long as the above
703	* mentioned locking is in effect. If the returned pool needs to be used
704	* beyond the critical section, the caller is responsible for ensuring the
705	* returned pool is and stays online.
706	*
707	* Return: The worker_pool @work was last associated with. %NULL if none.
708	*/
709	static struct worker_pool get_work_pool(struct work_struct work)
710	{
711	unsigned long data = atomic_long_read(&work->data);
712	int pool_id;
713
714	assert_rcu_or_pool_mutex();
715
716	if (data & WORK_STRUCT_PWQ)
717	return ((struct pool_workqueue *)
718	(data & WORK_STRUCT_WQ_DATA_MASK))->pool;
719
720	pool_id = data >> WORK_OFFQ_POOL_SHIFT;
721	if (pool_id == WORK_OFFQ_POOL_NONE)
722	return NULL;
723
724	return idr_find(&worker_pool_idr, pool_id);
725	}
726
727	/**
728	* get_work_pool_id - return the worker pool ID a given work is associated with
729	* @work: the work item of interest
730	*
731	* Return: The worker_pool ID @work was last associated with.
732	* %WORK_OFFQ_POOL_NONE if none.
733	*/
734	static int get_work_pool_id(struct work_struct *work)
735	{
736	unsigned long data = atomic_long_read(&work->data);
737
738	if (data & WORK_STRUCT_PWQ)
739	return ((struct pool_workqueue *)
740	(data & WORK_STRUCT_WQ_DATA_MASK))->pool->id;
741
742	return data >> WORK_OFFQ_POOL_SHIFT;
743	}
744
745	static void mark_work_canceling(struct work_struct *work)
746	{
747	unsigned long pool_id = get_work_pool_id(work);
748
749	pool_id <<= WORK_OFFQ_POOL_SHIFT;
750	set_work_data(work, pool_id \| WORK_OFFQ_CANCELING, WORK_STRUCT_PENDING);
751	}
752
753	static bool work_is_canceling(struct work_struct *work)
754	{
755	unsigned long data = atomic_long_read(&work->data);
756
757	return !(data & WORK_STRUCT_PWQ) && (data & WORK_OFFQ_CANCELING);
758	}
759
760	/*
761	* Policy functions. These define the policies on how the global worker
762	* pools are managed. Unless noted otherwise, these functions assume that
763	* they're being called with pool->lock held.
764	*/
765
766	static bool __need_more_worker(struct worker_pool *pool)
767	{
768	return !atomic_read(&pool->nr_running);
769	}
770
771	/*
772	* Need to wake up a worker? Called from anything but currently
773	* running workers.
774	*
775	* Note that, because unbound workers never contribute to nr_running, this
776	* function will always return %true for unbound pools as long as the
777	* worklist isn't empty.
778	*/
779	static bool need_more_worker(struct worker_pool *pool)
780	{
781	return !list_empty(&pool->worklist) && __need_more_worker(pool);
782	}
783
784	/* Can I start working? Called from busy but !running workers. */
785	static bool may_start_working(struct worker_pool *pool)
786	{
787	return pool->nr_idle;
788	}
789
790	/* Do I need to keep working? Called from currently running workers. */
791	static bool keep_working(struct worker_pool *pool)
792	{
793	return !list_empty(&pool->worklist) &&
794	atomic_read(&pool->nr_running) <= 1;
795	}
796
797	/* Do we need a new worker? Called from manager. */
798	static bool need_to_create_worker(struct worker_pool *pool)
799	{
800	return need_more_worker(pool) && !may_start_working(pool);
801	}
802
803	/* Do we have too many workers and should some go away? */
804	static bool too_many_workers(struct worker_pool *pool)
805	{
806	bool managing = pool->flags & POOL_MANAGER_ACTIVE;
807	int nr_idle = pool->nr_idle + managing; /* manager is considered idle */
808	int nr_busy = pool->nr_workers - nr_idle;
809
810	return nr_idle > 2 && (nr_idle - 2) * MAX_IDLE_WORKERS_RATIO >= nr_busy;
811	}
812
813	/*
814	* Wake up functions.
815	*/
816
817	/* Return the first idle worker. Safe with preemption disabled */
818	static struct worker first_idle_worker(struct worker_pool pool)
819	{
820	if (unlikely(list_empty(&pool->idle_list)))
821	return NULL;
822
823	return list_first_entry(&pool->idle_list, struct worker, entry);
824	}
825
826	/**
827	* wake_up_worker - wake up an idle worker
828	* @pool: worker pool to wake worker from
829	*
830	* Wake up the first idle worker of @pool.
831	*
832	* CONTEXT:
833	* spin_lock_irq(pool->lock).
834	*/
835	static void wake_up_worker(struct worker_pool *pool)
836	{
837	struct worker *worker = first_idle_worker(pool);
838
839	if (likely(worker))
840	wake_up_process(worker->task);
841	}
842
843	/**
844	* wq_worker_waking_up - a worker is waking up
845	* @task: task waking up
846	* @cpu: CPU @task is waking up to
847	*
848	* This function is called during try_to_wake_up() when a worker is
849	* being awoken.
850	*
851	* CONTEXT:
852	* spin_lock_irq(rq->lock)
853	*/
854	void wq_worker_waking_up(struct task_struct *task, int cpu)
855	{
856	struct worker *worker = kthread_data(task);
857
858	if (!(worker->flags & WORKER_NOT_RUNNING)) {
859	WARN_ON_ONCE(worker->pool->cpu != cpu);
860	atomic_inc(&worker->pool->nr_running);
861	}
862	}
863
864	/**
865	* wq_worker_sleeping - a worker is going to sleep
866	* @task: task going to sleep
867	*
868	* This function is called during schedule() when a busy worker is
869	* going to sleep. Worker on the same cpu can be woken up by
870	* returning pointer to its task.
871	*
872	* CONTEXT:
873	* spin_lock_irq(rq->lock)
874	*
875	* Return:
876	* Worker task on @cpu to wake up, %NULL if none.
877	*/
878	struct task_struct wq_worker_sleeping(struct task_struct task)
879	{
880	struct worker worker = kthread_data(task), to_wakeup = NULL;
881	struct worker_pool *pool;
882
883	/*
884	* Rescuers, which may not have all the fields set up like normal
885	* workers, also reach here, let's not access anything before
886	* checking NOT_RUNNING.
887	*/
888	if (worker->flags & WORKER_NOT_RUNNING)
889	return NULL;
890
891	pool = worker->pool;
892
893	/* this can only happen on the local cpu */
894	if (WARN_ON_ONCE(pool->cpu != raw_smp_processor_id()))
895	return NULL;
896
897	/*
898	* The counterpart of the following dec_and_test, implied mb,
899	* worklist not empty test sequence is in insert_work().
900	* Please read comment there.
901	*
902	* NOT_RUNNING is clear. This means that we're bound to and
903	* running on the local cpu w/ rq lock held and preemption
904	* disabled, which in turn means that none else could be
905	* manipulating idle_list, so dereferencing idle_list without pool
906	* lock is safe.
907	*/
908	if (atomic_dec_and_test(&pool->nr_running) &&
909	!list_empty(&pool->worklist))
910	to_wakeup = first_idle_worker(pool);
911	return to_wakeup ? to_wakeup->task : NULL;
912	}
913
914	/**
915	* worker_set_flags - set worker flags and adjust nr_running accordingly
916	* @worker: self
917	* @flags: flags to set
918	*
919	* Set @flags in @worker->flags and adjust nr_running accordingly.
920	*
921	* CONTEXT:
922	* spin_lock_irq(pool->lock)
923	*/
924	static inline void worker_set_flags(struct worker *worker, unsigned int flags)
925	{
926	struct worker_pool *pool = worker->pool;
927
928	WARN_ON_ONCE(worker->task != current);
929
930	/* If transitioning into NOT_RUNNING, adjust nr_running. */
931	if ((flags & WORKER_NOT_RUNNING) &&
932	!(worker->flags & WORKER_NOT_RUNNING)) {
933	atomic_dec(&pool->nr_running);
934	}
935
936	worker->flags \|= flags;
937	}
938
939	/**
940	* worker_clr_flags - clear worker flags and adjust nr_running accordingly
941	* @worker: self
942	* @flags: flags to clear
943	*
944	* Clear @flags in @worker->flags and adjust nr_running accordingly.
945	*
946	* CONTEXT:
947	* spin_lock_irq(pool->lock)
948	*/
949	static inline void worker_clr_flags(struct worker *worker, unsigned int flags)
950	{
951	struct worker_pool *pool = worker->pool;
952	unsigned int oflags = worker->flags;
953
954	WARN_ON_ONCE(worker->task != current);
955
956	worker->flags &= ~flags;
957
958	/*
959	* If transitioning out of NOT_RUNNING, increment nr_running. Note
960	* that the nested NOT_RUNNING is not a noop. NOT_RUNNING is mask
961	* of multiple flags, not a single flag.
962	*/
963	if ((flags & WORKER_NOT_RUNNING) && (oflags & WORKER_NOT_RUNNING))
964	if (!(worker->flags & WORKER_NOT_RUNNING))
965	atomic_inc(&pool->nr_running);
966	}
967
968	/**
969	* find_worker_executing_work - find worker which is executing a work
970	* @pool: pool of interest
971	* @work: work to find worker for
972	*
973	* Find a worker which is executing @work on @pool by searching
974	* @pool->busy_hash which is keyed by the address of @work. For a worker
975	* to match, its current execution should match the address of @work and
976	* its work function. This is to avoid unwanted dependency between
977	* unrelated work executions through a work item being recycled while still
978	* being executed.
979	*
980	* This is a bit tricky. A work item may be freed once its execution
981	* starts and nothing prevents the freed area from being recycled for
982	* another work item. If the same work item address ends up being reused
983	* before the original execution finishes, workqueue will identify the
984	* recycled work item as currently executing and make it wait until the
985	* current execution finishes, introducing an unwanted dependency.
986	*
987	* This function checks the work item address and work function to avoid
988	* false positives. Note that this isn't complete as one may construct a
989	* work function which can introduce dependency onto itself through a
990	* recycled work item. Well, if somebody wants to shoot oneself in the
991	* foot that badly, there's only so much we can do, and if such deadlock
992	* actually occurs, it should be easy to locate the culprit work function.
993	*
994	* CONTEXT:
995	* spin_lock_irq(pool->lock).
996	*
997	* Return:
998	* Pointer to worker which is executing @work if found, %NULL
999	* otherwise.
1000	*/
1001	static struct worker find_worker_executing_work(struct worker_pool pool,
1002	struct work_struct *work)
1003	{
1004	struct worker *worker;
1005
1006	hash_for_each_possible(pool->busy_hash, worker, hentry,
1007	(unsigned long)work)
1008	if (worker->current_work == work &&
1009	worker->current_func == work->func)
1010	return worker;
1011
1012	return NULL;
1013	}
1014
1015	/**
1016	* move_linked_works - move linked works to a list
1017	* @work: start of series of works to be scheduled
1018	* @head: target list to append @work to
1019	* @nextp: out parameter for nested worklist walking
1020	*
1021	* Schedule linked works starting from @work to @head. Work series to
1022	* be scheduled starts at @work and includes any consecutive work with
1023	* WORK_STRUCT_LINKED set in its predecessor.
1024	*
1025	* If @nextp is not NULL, it's updated to point to the next work of
1026	* the last scheduled work. This allows move_linked_works() to be
1027	* nested inside outer list_for_each_entry_safe().
1028	*
1029	* CONTEXT:
1030	* spin_lock_irq(pool->lock).
1031	*/
1032	static void move_linked_works(struct work_struct work, struct list_head head,
1033	struct work_struct **nextp)
1034	{
1035	struct work_struct *n;
1036
1037	/*
1038	* Linked worklist will always end before the end of the list,
1039	* use NULL for list head.
1040	*/
1041	list_for_each_entry_safe_from(work, n, NULL, entry) {
1042	list_move_tail(&work->entry, head);
1043	if (!(*work_data_bits(work) & WORK_STRUCT_LINKED))
1044	break;
1045	}
1046
1047	/*
1048	* If we're already inside safe list traversal and have moved
1049	* multiple works to the scheduled queue, the next position
1050	* needs to be updated.
1051	*/
1052	if (nextp)
1053	*nextp = n;
1054	}
1055
1056	/**
1057	* get_pwq - get an extra reference on the specified pool_workqueue
1058	* @pwq: pool_workqueue to get
1059	*
1060	* Obtain an extra reference on @pwq. The caller should guarantee that
1061	* @pwq has positive refcnt and be holding the matching pool->lock.
1062	*/
1063	static void get_pwq(struct pool_workqueue *pwq)
1064	{
1065	lockdep_assert_held(&pwq->pool->lock);
1066	WARN_ON_ONCE(pwq->refcnt <= 0);
1067	pwq->refcnt++;
1068	}
1069
1070	/**
1071	* put_pwq - put a pool_workqueue reference
1072	* @pwq: pool_workqueue to put
1073	*
1074	* Drop a reference of @pwq. If its refcnt reaches zero, schedule its
1075	* destruction. The caller should be holding the matching pool->lock.
1076	*/
1077	static void put_pwq(struct pool_workqueue *pwq)
1078	{
1079	lockdep_assert_held(&pwq->pool->lock);
1080	if (likely(--pwq->refcnt))
1081	return;
1082	if (WARN_ON_ONCE(!(pwq->wq->flags & WQ_UNBOUND)))
1083	return;
1084	/*
1085	* @pwq can't be released under pool->lock, bounce to
1086	* pwq_unbound_release_workfn(). This never recurses on the same
1087	* pool->lock as this path is taken only for unbound workqueues and
1088	* the release work item is scheduled on a per-cpu workqueue. To
1089	* avoid lockdep warning, unbound pool->locks are given lockdep
1090	* subclass of 1 in get_unbound_pool().
1091	*/
1092	schedule_work(&pwq->unbound_release_work);
1093	}
1094
1095	/**
1096	* put_pwq_unlocked - put_pwq() with surrounding pool lock/unlock
1097	* @pwq: pool_workqueue to put (can be %NULL)
1098	*
1099	* put_pwq() with locking. This function also allows %NULL @pwq.
1100	*/
1101	static void put_pwq_unlocked(struct pool_workqueue *pwq)
1102	{
1103	if (pwq) {
1104	/*
1105	* As both pwqs and pools are sched-RCU protected, the
1106	* following lock operations are safe.
1107	*/
1108	spin_lock_irq(&pwq->pool->lock);
1109	put_pwq(pwq);
1110	spin_unlock_irq(&pwq->pool->lock);
1111	}
1112	}
1113
1114	static void pwq_activate_delayed_work(struct work_struct *work)
1115	{
1116	struct pool_workqueue *pwq = get_work_pwq(work);
1117
1118	trace_workqueue_activate_work(work);
1119	if (list_empty(&pwq->pool->worklist))
1120	pwq->pool->watchdog_ts = jiffies;
1121	move_linked_works(work, &pwq->pool->worklist, NULL);
1122	__clear_bit(WORK_STRUCT_DELAYED_BIT, work_data_bits(work));
1123	pwq->nr_active++;
1124	}
1125
1126	static void pwq_activate_first_delayed(struct pool_workqueue *pwq)
1127	{
1128	struct work_struct *work = list_first_entry(&pwq->delayed_works,
1129	struct work_struct, entry);
1130
1131	pwq_activate_delayed_work(work);
1132	}
1133
1134	/**
1135	* pwq_dec_nr_in_flight - decrement pwq's nr_in_flight
1136	* @pwq: pwq of interest
1137	* @color: color of work which left the queue
1138	*
1139	* A work either has completed or is removed from pending queue,
1140	* decrement nr_in_flight of its pwq and handle workqueue flushing.
1141	*
1142	* CONTEXT:
1143	* spin_lock_irq(pool->lock).
1144	*/
1145	static void pwq_dec_nr_in_flight(struct pool_workqueue *pwq, int color)
1146	{
1147	/* uncolored work items don't participate in flushing or nr_active */
1148	if (color == WORK_NO_COLOR)
1149	goto out_put;
1150
1151	pwq->nr_in_flight[color]--;
1152
1153	pwq->nr_active--;
1154	if (!list_empty(&pwq->delayed_works)) {
1155	/* one down, submit a delayed one */
1156	if (pwq->nr_active < pwq->max_active)
1157	pwq_activate_first_delayed(pwq);
1158	}
1159
1160	/* is flush in progress and are we at the flushing tip? */
1161	if (likely(pwq->flush_color != color))
1162	goto out_put;
1163
1164	/* are there still in-flight works? */
1165	if (pwq->nr_in_flight[color])
1166	goto out_put;
1167
1168	/* this pwq is done, clear flush_color */
1169	pwq->flush_color = -1;
1170
1171	/*
1172	* If this was the last pwq, wake up the first flusher. It
1173	* will handle the rest.
1174	*/
1175	if (atomic_dec_and_test(&pwq->wq->nr_pwqs_to_flush))
1176	complete(&pwq->wq->first_flusher->done);
1177	out_put:
1178	put_pwq(pwq);
1179	}
1180
1181	/**
1182	* try_to_grab_pending - steal work item from worklist and disable irq
1183	* @work: work item to steal
1184	* @is_dwork: @work is a delayed_work
1185	* @flags: place to store irq state
1186	*
1187	* Try to grab PENDING bit of @work. This function can handle @work in any
1188	* stable state - idle, on timer or on worklist.
1189	*
1190	* Return:
1191	* 1 if @work was pending and we successfully stole PENDING
1192	* 0 if @work was idle and we claimed PENDING
1193	* -EAGAIN if PENDING couldn't be grabbed at the moment, safe to busy-retry
1194	* -ENOENT if someone else is canceling @work, this state may persist
1195	* for arbitrarily long
1196	*
1197	* Note:
1198	* On >= 0 return, the caller owns @work's PENDING bit. To avoid getting
1199	* interrupted while holding PENDING and @work off queue, irq must be
1200	* disabled on entry. This, combined with delayed_work->timer being
1201	* irqsafe, ensures that we return -EAGAIN for finite short period of time.
1202	*
1203	* On successful return, >= 0, irq is disabled and the caller is
1204	* responsible for releasing it using local_irq_restore(*@flags).
1205	*
1206	* This function is safe to call from any context including IRQ handler.
1207	*/
1208	static int try_to_grab_pending(struct work_struct *work, bool is_dwork,
1209	unsigned long *flags)
1210	{
1211	struct worker_pool *pool;
1212	struct pool_workqueue *pwq;
1213
1214	local_irq_save(*flags);
1215
1216	/* try to steal the timer if it exists */
1217	if (is_dwork) {
1218	struct delayed_work *dwork = to_delayed_work(work);
1219
1220	/*
1221	* dwork->timer is irqsafe. If del_timer() fails, it's
1222	* guaranteed that the timer is not queued anywhere and not
1223	* running on the local CPU.
1224	*/
1225	if (likely(del_timer(&dwork->timer)))
1226	return 1;
1227	}
1228
1229	/* try to claim PENDING the normal way */
1230	if (!test_and_set_bit(WORK_STRUCT_PENDING_BIT, work_data_bits(work)))
1231	return 0;
1232
1233	/*
1234	* The queueing is in progress, or it is already queued. Try to
1235	* steal it from ->worklist without clearing WORK_STRUCT_PENDING.
1236	*/
1237	pool = get_work_pool(work);
1238	if (!pool)
1239	goto fail;
1240
1241	spin_lock(&pool->lock);
1242	/*
1243	* work->data is guaranteed to point to pwq only while the work
1244	* item is queued on pwq->wq, and both updating work->data to point
1245	* to pwq on queueing and to pool on dequeueing are done under
1246	* pwq->pool->lock. This in turn guarantees that, if work->data
1247	* points to pwq which is associated with a locked pool, the work
1248	* item is currently queued on that pool.
1249	*/
1250	pwq = get_work_pwq(work);
1251	if (pwq && pwq->pool == pool) {
1252	debug_work_deactivate(work);
1253
1254	/*
1255	* A delayed work item cannot be grabbed directly because
1256	* it might have linked NO_COLOR work items which, if left
1257	* on the delayed_list, will confuse pwq->nr_active
1258	* management later on and cause stall. Make sure the work
1259	* item is activated before grabbing.
1260	*/
1261	if (*work_data_bits(work) & WORK_STRUCT_DELAYED)
1262	pwq_activate_delayed_work(work);
1263
1264	list_del_init(&work->entry);
1265	pwq_dec_nr_in_flight(pwq, get_work_color(work));
1266
1267	/* work->data points to pwq iff queued, point to pool */
1268	set_work_pool_and_keep_pending(work, pool->id);
1269
1270	spin_unlock(&pool->lock);
1271	return 1;
1272	}
1273	spin_unlock(&pool->lock);
1274	fail:
1275	local_irq_restore(*flags);
1276	if (work_is_canceling(work))
1277	return -ENOENT;
1278	cpu_relax();
1279	return -EAGAIN;
1280	}
1281
1282	/**
1283	* insert_work - insert a work into a pool
1284	* @pwq: pwq @work belongs to
1285	* @work: work to insert
1286	* @head: insertion point
1287	* @extra_flags: extra WORK_STRUCT_* flags to set
1288	*
1289	* Insert @work which belongs to @pwq after @head. @extra_flags is or'd to
1290	* work_struct flags.
1291	*
1292	* CONTEXT:
1293	* spin_lock_irq(pool->lock).
1294	*/
1295	static void insert_work(struct pool_workqueue pwq, struct work_struct work,
1296	struct list_head *head, unsigned int extra_flags)
1297	{
1298	struct worker_pool *pool = pwq->pool;
1299
1300	/* we own @work, set data and link */
1301	set_work_pwq(work, pwq, extra_flags);
1302	list_add_tail(&work->entry, head);
1303	get_pwq(pwq);
1304
1305	/*
1306	* Ensure either wq_worker_sleeping() sees the above
1307	* list_add_tail() or we see zero nr_running to avoid workers lying
1308	* around lazily while there are works to be processed.
1309	*/
1310	smp_mb();
1311
1312	if (__need_more_worker(pool))
1313	wake_up_worker(pool);
1314	}
1315
1316	/*
1317	* Test whether @work is being queued from another work executing on the
1318	* same workqueue.
1319	*/
1320	static bool is_chained_work(struct workqueue_struct *wq)
1321	{
1322	struct worker *worker;
1323
1324	worker = current_wq_worker();
1325	/*
1326	* Return %true iff I'm a worker execuing a work item on @wq. If
1327	* I'm @worker, it's safe to dereference it without locking.
1328	*/
1329	return worker && worker->current_pwq->wq == wq;
1330	}
1331
1332	/*
1333	* When queueing an unbound work item to a wq, prefer local CPU if allowed
1334	* by wq_unbound_cpumask. Otherwise, round robin among the allowed ones to
1335	* avoid perturbing sensitive tasks.
1336	*/
1337	static int wq_select_unbound_cpu(int cpu)
1338	{
1339	static bool printed_dbg_warning;
1340	int new_cpu;
1341
1342	if (likely(!wq_debug_force_rr_cpu)) {
1343	if (cpumask_test_cpu(cpu, wq_unbound_cpumask))
1344	return cpu;
1345	} else if (!printed_dbg_warning) {
1346	pr_warn("workqueue: round-robin CPU selection forced, expect performance impact\n");
1347	printed_dbg_warning = true;
1348	}
1349
1350	if (cpumask_empty(wq_unbound_cpumask))
1351	return cpu;
1352
1353	new_cpu = __this_cpu_read(wq_rr_cpu_last);
1354	new_cpu = cpumask_next_and(new_cpu, wq_unbound_cpumask, cpu_online_mask);
1355	if (unlikely(new_cpu >= nr_cpu_ids)) {
1356	new_cpu = cpumask_first_and(wq_unbound_cpumask, cpu_online_mask);
1357	if (unlikely(new_cpu >= nr_cpu_ids))
1358	return cpu;
1359	}
1360	__this_cpu_write(wq_rr_cpu_last, new_cpu);
1361
1362	return new_cpu;
1363	}
1364
1365	static void __queue_work(int cpu, struct workqueue_struct *wq,
1366	struct work_struct *work)
1367	{
1368	struct pool_workqueue *pwq;
1369	struct worker_pool *last_pool;
1370	struct list_head *worklist;
1371	unsigned int work_flags;
1372	unsigned int req_cpu = cpu;
1373
1374	/*
1375	* While a work item is PENDING && off queue, a task trying to
1376	* steal the PENDING will busy-loop waiting for it to either get
1377	* queued or lose PENDING. Grabbing PENDING and queueing should
1378	* happen with IRQ disabled.
1379	*/
1380	lockdep_assert_irqs_disabled();
1381
1382	debug_work_activate(work);
1383
1384	/* if draining, only works from the same workqueue are allowed */
1385	if (unlikely(wq->flags & __WQ_DRAINING) &&
1386	WARN_ON_ONCE(!is_chained_work(wq)))
1387	return;
1388	retry:
1389	if (req_cpu == WORK_CPU_UNBOUND)
1390	cpu = wq_select_unbound_cpu(raw_smp_processor_id());
1391
1392	/* pwq which will be used unless @work is executing elsewhere */
1393	if (!(wq->flags & WQ_UNBOUND))
1394	pwq = per_cpu_ptr(wq->cpu_pwqs, cpu);
1395	else
1396	pwq = unbound_pwq_by_node(wq, cpu_to_node(cpu));
1397
1398	/*
1399	* If @work was previously on a different pool, it might still be
1400	* running there, in which case the work needs to be queued on that
1401	* pool to guarantee non-reentrancy.
1402	*/
1403	last_pool = get_work_pool(work);
1404	if (last_pool && last_pool != pwq->pool) {
1405	struct worker *worker;
1406
1407	spin_lock(&last_pool->lock);
1408
1409	worker = find_worker_executing_work(last_pool, work);
1410
1411	if (worker && worker->current_pwq->wq == wq) {
1412	pwq = worker->current_pwq;
1413	} else {
1414	/* meh... not running there, queue here */
1415	spin_unlock(&last_pool->lock);
1416	spin_lock(&pwq->pool->lock);
1417	}
1418	} else {
1419	spin_lock(&pwq->pool->lock);
1420	}
1421
1422	/*
1423	* pwq is determined and locked. For unbound pools, we could have
1424	* raced with pwq release and it could already be dead. If its
1425	* refcnt is zero, repeat pwq selection. Note that pwqs never die
1426	* without another pwq replacing it in the numa_pwq_tbl or while
1427	* work items are executing on it, so the retrying is guaranteed to
1428	* make forward-progress.
1429	*/
1430	if (unlikely(!pwq->refcnt)) {
1431	if (wq->flags & WQ_UNBOUND) {
1432	spin_unlock(&pwq->pool->lock);
1433	cpu_relax();
1434	goto retry;
1435	}
1436	/* oops */
1437	WARN_ONCE(true, "workqueue: per-cpu pwq for %s on cpu%d has 0 refcnt",
1438	wq->name, cpu);
1439	}
1440
1441	/* pwq determined, queue */
1442	trace_workqueue_queue_work(req_cpu, pwq, work);
1443
1444	if (WARN_ON(!list_empty(&work->entry))) {
1445	spin_unlock(&pwq->pool->lock);
1446	return;
1447	}
1448
1449	pwq->nr_in_flight[pwq->work_color]++;
1450	work_flags = work_color_to_flags(pwq->work_color);
1451
1452	if (likely(pwq->nr_active < pwq->max_active)) {
1453	trace_workqueue_activate_work(work);
1454	pwq->nr_active++;
1455	worklist = &pwq->pool->worklist;
1456	if (list_empty(worklist))
1457	pwq->pool->watchdog_ts = jiffies;
1458	} else {
1459	work_flags \|= WORK_STRUCT_DELAYED;
1460	worklist = &pwq->delayed_works;
1461	}
1462
1463	insert_work(pwq, work, worklist, work_flags);
1464
1465	spin_unlock(&pwq->pool->lock);
1466	}
1467
1468	/**
1469	* queue_work_on - queue work on specific cpu
1470	* @cpu: CPU number to execute work on
1471	* @wq: workqueue to use
1472	* @work: work to queue
1473	*
1474	* We queue the work to a specific CPU, the caller must ensure it
1475	* can't go away.
1476	*
1477	* Return: %false if @work was already on a queue, %true otherwise.
1478	*/
1479	bool queue_work_on(int cpu, struct workqueue_struct *wq,
1480	struct work_struct *work)
1481	{
1482	bool ret = false;
1483	unsigned long flags;
1484
1485	local_irq_save(flags);
1486
1487	if (!test_and_set_bit(WORK_STRUCT_PENDING_BIT, work_data_bits(work))) {
1488	__queue_work(cpu, wq, work);
1489	ret = true;
1490	}
1491
1492	local_irq_restore(flags);
1493	return ret;
1494	}
1495	EXPORT_SYMBOL(queue_work_on);
1496
1497	void delayed_work_timer_fn(struct timer_list *t)
1498	{
1499	struct delayed_work *dwork = from_timer(dwork, t, timer);
1500
1501	/* should have been called from irqsafe timer with irq already off */
1502	__queue_work(dwork->cpu, dwork->wq, &dwork->work);
1503	}
1504	EXPORT_SYMBOL(delayed_work_timer_fn);
1505
1506	static void __queue_delayed_work(int cpu, struct workqueue_struct *wq,
1507	struct delayed_work *dwork, unsigned long delay)
1508	{
1509	struct timer_list *timer = &dwork->timer;
1510	struct work_struct *work = &dwork->work;
1511
1512	WARN_ON_ONCE(!wq);
1513	WARN_ON_ONCE(timer->function != delayed_work_timer_fn);
1514	WARN_ON_ONCE(timer_pending(timer));
1515	WARN_ON_ONCE(!list_empty(&work->entry));
1516
1517	/*
1518	* If @delay is 0, queue @dwork->work immediately. This is for
1519	* both optimization and correctness. The earliest @timer can
1520	* expire is on the closest next tick and delayed_work users depend
1521	* on that there's no such delay when @delay is 0.
1522	*/
1523	if (!delay) {
1524	__queue_work(cpu, wq, &dwork->work);
1525	return;
1526	}
1527
1528	dwork->wq = wq;
1529	dwork->cpu = cpu;
1530	timer->expires = jiffies + delay;
1531
1532	if (unlikely(cpu != WORK_CPU_UNBOUND))
1533	add_timer_on(timer, cpu);
1534	else
1535	add_timer(timer);
1536	}
1537
1538	/**
1539	* queue_delayed_work_on - queue work on specific CPU after delay
1540	* @cpu: CPU number to execute work on
1541	* @wq: workqueue to use
1542	* @dwork: work to queue
1543	* @delay: number of jiffies to wait before queueing
1544	*
1545	* Return: %false if @work was already on a queue, %true otherwise. If
1546	* @delay is zero and @dwork is idle, it will be scheduled for immediate
1547	* execution.
1548	*/
1549	bool queue_delayed_work_on(int cpu, struct workqueue_struct *wq,
1550	struct delayed_work *dwork, unsigned long delay)
1551	{
1552	struct work_struct *work = &dwork->work;
1553	bool ret = false;
1554	unsigned long flags;
1555
1556	/* read the comment in __queue_work() */
1557	local_irq_save(flags);
1558
1559	if (!test_and_set_bit(WORK_STRUCT_PENDING_BIT, work_data_bits(work))) {
1560	__queue_delayed_work(cpu, wq, dwork, delay);
1561	ret = true;
1562	}
1563
1564	local_irq_restore(flags);
1565	return ret;
1566	}
1567	EXPORT_SYMBOL(queue_delayed_work_on);
1568
1569	/**
1570	* mod_delayed_work_on - modify delay of or queue a delayed work on specific CPU
1571	* @cpu: CPU number to execute work on
1572	* @wq: workqueue to use
1573	* @dwork: work to queue
1574	* @delay: number of jiffies to wait before queueing
1575	*
1576	* If @dwork is idle, equivalent to queue_delayed_work_on(); otherwise,
1577	* modify @dwork's timer so that it expires after @delay. If @delay is
1578	* zero, @work is guaranteed to be scheduled immediately regardless of its
1579	* current state.
1580	*
1581	* Return: %false if @dwork was idle and queued, %true if @dwork was
1582	* pending and its timer was modified.
1583	*
1584	* This function is safe to call from any context including IRQ handler.
1585	* See try_to_grab_pending() for details.
1586	*/
1587	bool mod_delayed_work_on(int cpu, struct workqueue_struct *wq,
1588	struct delayed_work *dwork, unsigned long delay)
1589	{
1590	unsigned long flags;
1591	int ret;
1592
1593	do {
1594	ret = try_to_grab_pending(&dwork->work, true, &flags);
1595	} while (unlikely(ret == -EAGAIN));
1596
1597	if (likely(ret >= 0)) {
1598	__queue_delayed_work(cpu, wq, dwork, delay);
1599	local_irq_restore(flags);
1600	}
1601
1602	/* -ENOENT from try_to_grab_pending() becomes %true */
1603	return ret;
1604	}
1605	EXPORT_SYMBOL_GPL(mod_delayed_work_on);
1606
1607	/**
1608	* worker_enter_idle - enter idle state
1609	* @worker: worker which is entering idle state
1610	*
1611	* @worker is entering idle state. Update stats and idle timer if
1612	* necessary.
1613	*
1614	* LOCKING:
1615	* spin_lock_irq(pool->lock).
1616	*/
1617	static void worker_enter_idle(struct worker *worker)
1618	{
1619	struct worker_pool *pool = worker->pool;
1620
1621	if (WARN_ON_ONCE(worker->flags & WORKER_IDLE) \|\|
1622	WARN_ON_ONCE(!list_empty(&worker->entry) &&
1623	(worker->hentry.next \|\| worker->hentry.pprev)))
1624	return;
1625
1626	/* can't use worker_set_flags(), also called from create_worker() */
1627	worker->flags \|= WORKER_IDLE;
1628	pool->nr_idle++;
1629	worker->last_active = jiffies;
1630
1631	/* idle_list is LIFO */
1632	list_add(&worker->entry, &pool->idle_list);
1633
1634	if (too_many_workers(pool) && !timer_pending(&pool->idle_timer))
1635	mod_timer(&pool->idle_timer, jiffies + IDLE_WORKER_TIMEOUT);
1636
1637	/*
1638	* Sanity check nr_running. Because unbind_workers() releases
1639	* pool->lock between setting %WORKER_UNBOUND and zapping
1640	* nr_running, the warning may trigger spuriously. Check iff
1641	* unbind is not in progress.
1642	*/
1643	WARN_ON_ONCE(!(pool->flags & POOL_DISASSOCIATED) &&
1644	pool->nr_workers == pool->nr_idle &&
1645	atomic_read(&pool->nr_running));
1646	}
1647
1648	/**
1649	* worker_leave_idle - leave idle state
1650	* @worker: worker which is leaving idle state
1651	*
1652	* @worker is leaving idle state. Update stats.
1653	*
1654	* LOCKING:
1655	* spin_lock_irq(pool->lock).
1656	*/
1657	static void worker_leave_idle(struct worker *worker)
1658	{
1659	struct worker_pool *pool = worker->pool;
1660
1661	if (WARN_ON_ONCE(!(worker->flags & WORKER_IDLE)))
1662	return;
1663	worker_clr_flags(worker, WORKER_IDLE);
1664	pool->nr_idle--;
1665	list_del_init(&worker->entry);
1666	}
1667
1668	static struct worker *alloc_worker(int node)
1669	{
1670	struct worker *worker;
1671
1672	worker = kzalloc_node(sizeof(*worker), GFP_KERNEL, node);
1673	if (worker) {
1674	INIT_LIST_HEAD(&worker->entry);
1675	INIT_LIST_HEAD(&worker->scheduled);
1676	INIT_LIST_HEAD(&worker->node);
1677	/* on creation a worker is in !idle && prep state */
1678	worker->flags = WORKER_PREP;
1679	}
1680	return worker;
1681	}
1682
1683	/**
1684	* worker_attach_to_pool() - attach a worker to a pool
1685	* @worker: worker to be attached
1686	* @pool: the target pool
1687	*
1688	* Attach @worker to @pool. Once attached, the %WORKER_UNBOUND flag and
1689	* cpu-binding of @worker are kept coordinated with the pool across
1690	* cpu-[un]hotplugs.
1691	*/
1692	static void worker_attach_to_pool(struct worker *worker,
1693	struct worker_pool *pool)
1694	{
1695	mutex_lock(&pool->attach_mutex);
1696
1697	/*
1698	* set_cpus_allowed_ptr() will fail if the cpumask doesn't have any
1699	* online CPUs. It'll be re-applied when any of the CPUs come up.
1700	*/
1701	set_cpus_allowed_ptr(worker->task, pool->attrs->cpumask);
1702
1703	/*
1704	* The pool->attach_mutex ensures %POOL_DISASSOCIATED remains
1705	* stable across this function. See the comments above the
1706	* flag definition for details.
1707	*/
1708	if (pool->flags & POOL_DISASSOCIATED)
1709	worker->flags \|= WORKER_UNBOUND;
1710
1711	list_add_tail(&worker->node, &pool->workers);
1712
1713	mutex_unlock(&pool->attach_mutex);
1714	}
1715
1716	/**
1717	* worker_detach_from_pool() - detach a worker from its pool
1718	* @worker: worker which is attached to its pool
1719	* @pool: the pool @worker is attached to
1720	*
1721	* Undo the attaching which had been done in worker_attach_to_pool(). The
1722	* caller worker shouldn't access to the pool after detached except it has
1723	* other reference to the pool.
1724	*/
1725	static void worker_detach_from_pool(struct worker *worker,
1726	struct worker_pool *pool)
1727	{
1728	struct completion *detach_completion = NULL;
1729
1730	mutex_lock(&pool->attach_mutex);
1731	list_del(&worker->node);
1732	if (list_empty(&pool->workers))
1733	detach_completion = pool->detach_completion;
1734	mutex_unlock(&pool->attach_mutex);
1735
1736	/* clear leftover flags without pool->lock after it is detached */
1737	worker->flags &= ~(WORKER_UNBOUND \| WORKER_REBOUND);
1738
1739	if (detach_completion)
1740	complete(detach_completion);
1741	}
1742
1743	/**
1744	* create_worker - create a new workqueue worker
1745	* @pool: pool the new worker will belong to
1746	*
1747	* Create and start a new worker which is attached to @pool.
1748	*
1749	* CONTEXT:
1750	* Might sleep. Does GFP_KERNEL allocations.
1751	*
1752	* Return:
1753	* Pointer to the newly created worker.
1754	*/
1755	static struct worker create_worker(struct worker_pool pool)
1756	{
1757	struct worker *worker = NULL;
1758	int id = -1;
1759	char id_buf[16];
1760
1761	/* ID is needed to determine kthread name */
1762	id = ida_simple_get(&pool->worker_ida, 0, 0, GFP_KERNEL);
1763	if (id < 0)
1764	goto fail;
1765
1766	worker = alloc_worker(pool->node);
1767	if (!worker)
1768	goto fail;
1769
1770	worker->pool = pool;
1771	worker->id = id;
1772
1773	if (pool->cpu >= 0)
1774	snprintf(id_buf, sizeof(id_buf), "%d:%d%s", pool->cpu, id,
1775	pool->attrs->nice < 0 ? "H" : "");
1776	else
1777	snprintf(id_buf, sizeof(id_buf), "u%d:%d", pool->id, id);
1778
1779	worker->task = kthread_create_on_node(worker_thread, worker, pool->node,
1780	"kworker/%s", id_buf);
1781	if (IS_ERR(worker->task))
1782	goto fail;
1783
1784	set_user_nice(worker->task, pool->attrs->nice);
1785	kthread_bind_mask(worker->task, pool->attrs->cpumask);
1786
1787	/* successful, attach the worker to the pool */
1788	worker_attach_to_pool(worker, pool);
1789
1790	/* start the newly created worker */
1791	spin_lock_irq(&pool->lock);
1792	worker->pool->nr_workers++;
1793	worker_enter_idle(worker);
1794	wake_up_process(worker->task);
1795	spin_unlock_irq(&pool->lock);
1796
1797	return worker;
1798
1799	fail:
1800	if (id >= 0)
1801	ida_simple_remove(&pool->worker_ida, id);
1802	kfree(worker);
1803	return NULL;
1804	}
1805
1806	/**
1807	* destroy_worker - destroy a workqueue worker
1808	* @worker: worker to be destroyed
1809	*
1810	* Destroy @worker and adjust @pool stats accordingly. The worker should
1811	* be idle.
1812	*
1813	* CONTEXT:
1814	* spin_lock_irq(pool->lock).
1815	*/
1816	static void destroy_worker(struct worker *worker)
1817	{
1818	struct worker_pool *pool = worker->pool;
1819
1820	lockdep_assert_held(&pool->lock);
1821
1822	/* sanity check frenzy */
1823	if (WARN_ON(worker->current_work) \|\|
1824	WARN_ON(!list_empty(&worker->scheduled)) \|\|
1825	WARN_ON(!(worker->flags & WORKER_IDLE)))
1826	return;
1827
1828	pool->nr_workers--;
1829	pool->nr_idle--;
1830
1831	list_del_init(&worker->entry);
1832	worker->flags \|= WORKER_DIE;
1833	wake_up_process(worker->task);
1834	}
1835
1836	static void idle_worker_timeout(struct timer_list *t)
1837	{
1838	struct worker_pool *pool = from_timer(pool, t, idle_timer);
1839
1840	spin_lock_irq(&pool->lock);
1841
1842	while (too_many_workers(pool)) {
1843	struct worker *worker;
1844	unsigned long expires;
1845
1846	/* idle_list is kept in LIFO order, check the last one */
1847	worker = list_entry(pool->idle_list.prev, struct worker, entry);
1848	expires = worker->last_active + IDLE_WORKER_TIMEOUT;
1849
1850	if (time_before(jiffies, expires)) {
1851	mod_timer(&pool->idle_timer, expires);
1852	break;
1853	}
1854
1855	destroy_worker(worker);
1856	}
1857
1858	spin_unlock_irq(&pool->lock);
1859	}
1860
1861	static void send_mayday(struct work_struct *work)
1862	{
1863	struct pool_workqueue *pwq = get_work_pwq(work);
1864	struct workqueue_struct *wq = pwq->wq;
1865
1866	lockdep_assert_held(&wq_mayday_lock);
1867
1868	if (!wq->rescuer)
1869	return;
1870
1871	/* mayday mayday mayday */
1872	if (list_empty(&pwq->mayday_node)) {
1873	/*
1874	* If @pwq is for an unbound wq, its base ref may be put at
1875	* any time due to an attribute change. Pin @pwq until the
1876	* rescuer is done with it.
1877	*/
1878	get_pwq(pwq);
1879	list_add_tail(&pwq->mayday_node, &wq->maydays);
1880	wake_up_process(wq->rescuer->task);
1881	}
1882	}
1883
1884	static void pool_mayday_timeout(struct timer_list *t)
1885	{
1886	struct worker_pool *pool = from_timer(pool, t, mayday_timer);
1887	struct work_struct *work;
1888
1889	spin_lock_irq(&pool->lock);
1890	spin_lock(&wq_mayday_lock); /* for wq->maydays */
1891
1892	if (need_to_create_worker(pool)) {
1893	/*
1894	* We've been trying to create a new worker but
1895	* haven't been successful. We might be hitting an
1896	* allocation deadlock. Send distress signals to
1897	* rescuers.
1898	*/
1899	list_for_each_entry(work, &pool->worklist, entry)
1900	send_mayday(work);
1901	}
1902
1903	spin_unlock(&wq_mayday_lock);
1904	spin_unlock_irq(&pool->lock);
1905
1906	mod_timer(&pool->mayday_timer, jiffies + MAYDAY_INTERVAL);
1907	}
1908
1909	/**
1910	* maybe_create_worker - create a new worker if necessary
1911	* @pool: pool to create a new worker for
1912	*
1913	* Create a new worker for @pool if necessary. @pool is guaranteed to
1914	* have at least one idle worker on return from this function. If
1915	* creating a new worker takes longer than MAYDAY_INTERVAL, mayday is
1916	* sent to all rescuers with works scheduled on @pool to resolve
1917	* possible allocation deadlock.
1918	*
1919	* On return, need_to_create_worker() is guaranteed to be %false and
1920	* may_start_working() %true.
1921	*
1922	* LOCKING:
1923	* spin_lock_irq(pool->lock) which may be released and regrabbed
1924	* multiple times. Does GFP_KERNEL allocations. Called only from
1925	* manager.
1926	*/
1927	static void maybe_create_worker(struct worker_pool *pool)
1928	__releases(&pool->lock)
1929	__acquires(&pool->lock)
1930	{
1931	restart:
1932	spin_unlock_irq(&pool->lock);
1933
1934	/* if we don't make progress in MAYDAY_INITIAL_TIMEOUT, call for help */
1935	mod_timer(&pool->mayday_timer, jiffies + MAYDAY_INITIAL_TIMEOUT);
1936
1937	while (true) {
1938	if (create_worker(pool) \|\| !need_to_create_worker(pool))
1939	break;
1940
1941	schedule_timeout_interruptible(CREATE_COOLDOWN);
1942
1943	if (!need_to_create_worker(pool))
1944	break;
1945	}
1946
1947	del_timer_sync(&pool->mayday_timer);
1948	spin_lock_irq(&pool->lock);
1949	/*
1950	* This is necessary even after a new worker was just successfully
1951	* created as @pool->lock was dropped and the new worker might have
1952	* already become busy.
1953	*/
1954	if (need_to_create_worker(pool))
1955	goto restart;
1956	}
1957
1958	/**
1959	* manage_workers - manage worker pool
1960	* @worker: self
1961	*
1962	* Assume the manager role and manage the worker pool @worker belongs
1963	* to. At any given time, there can be only zero or one manager per
1964	* pool. The exclusion is handled automatically by this function.
1965	*
1966	* The caller can safely start processing works on false return. On
1967	* true return, it's guaranteed that need_to_create_worker() is false
1968	* and may_start_working() is true.
1969	*
1970	* CONTEXT:
1971	* spin_lock_irq(pool->lock) which may be released and regrabbed
1972	* multiple times. Does GFP_KERNEL allocations.
1973	*
1974	* Return:
1975	* %false if the pool doesn't need management and the caller can safely
1976	* start processing works, %true if management function was performed and
1977	* the conditions that the caller verified before calling the function may
1978	* no longer be true.
1979	*/
1980	static bool manage_workers(struct worker *worker)
1981	{
1982	struct worker_pool *pool = worker->pool;
1983
1984	if (pool->flags & POOL_MANAGER_ACTIVE)
1985	return false;
1986
1987	pool->flags \|= POOL_MANAGER_ACTIVE;
1988	pool->manager = worker;
1989
1990	maybe_create_worker(pool);
1991
1992	pool->manager = NULL;
1993	pool->flags &= ~POOL_MANAGER_ACTIVE;
1994	wake_up(&wq_manager_wait);
1995	return true;
1996	}
1997
1998	/**
1999	* process_one_work - process single work
2000	* @worker: self
2001	* @work: work to process
2002	*
2003	* Process @work. This function contains all the logics necessary to
2004	* process a single work including synchronization against and
2005	* interaction with other workers on the same cpu, queueing and
2006	* flushing. As long as context requirement is met, any worker can
2007	* call this function to process a work.
2008	*
2009	* CONTEXT:
2010	* spin_lock_irq(pool->lock) which is released and regrabbed.
2011	*/
2012	static void process_one_work(struct worker worker, struct work_struct work)
2013	__releases(&pool->lock)
2014	__acquires(&pool->lock)
2015	{
2016	struct pool_workqueue *pwq = get_work_pwq(work);
2017	struct worker_pool *pool = worker->pool;
2018	bool cpu_intensive = pwq->wq->flags & WQ_CPU_INTENSIVE;
2019	int work_color;
2020	struct worker *collision;
2021	#ifdef CONFIG_LOCKDEP
2022	/*
2023	* It is permissible to free the struct work_struct from
2024	* inside the function that is called from it, this we need to
2025	* take into account for lockdep too. To avoid bogus "held
2026	* lock freed" warnings as well as problems when looking into
2027	* work->lockdep_map, make a copy and use that here.
2028	*/
2029	struct lockdep_map lockdep_map;
2030
2031	lockdep_copy_map(&lockdep_map, &work->lockdep_map);
2032	#endif
2033	/* ensure we're on the correct CPU */
2034	WARN_ON_ONCE(!(pool->flags & POOL_DISASSOCIATED) &&
2035	raw_smp_processor_id() != pool->cpu);
2036
2037	/*
2038	* A single work shouldn't be executed concurrently by
2039	* multiple workers on a single cpu. Check whether anyone is
2040	* already processing the work. If so, defer the work to the
2041	* currently executing one.
2042	*/
2043	collision = find_worker_executing_work(pool, work);
2044	if (unlikely(collision)) {
2045	move_linked_works(work, &collision->scheduled, NULL);
2046	return;
2047	}
2048
2049	/* claim and dequeue */
2050	debug_work_deactivate(work);
2051	hash_add(pool->busy_hash, &worker->hentry, (unsigned long)work);
2052	worker->current_work = work;
2053	worker->current_func = work->func;
2054	worker->current_pwq = pwq;
2055	work_color = get_work_color(work);
2056
2057	list_del_init(&work->entry);
2058
2059	/*
2060	* CPU intensive works don't participate in concurrency management.
2061	* They're the scheduler's responsibility. This takes @worker out
2062	* of concurrency management and the next code block will chain
2063	* execution of the pending work items.
2064	*/
2065	if (unlikely(cpu_intensive))
2066	worker_set_flags(worker, WORKER_CPU_INTENSIVE);
2067
2068	/*
2069	* Wake up another worker if necessary. The condition is always
2070	* false for normal per-cpu workers since nr_running would always
2071	* be >= 1 at this point. This is used to chain execution of the
2072	* pending work items for WORKER_NOT_RUNNING workers such as the
2073	* UNBOUND and CPU_INTENSIVE ones.
2074	*/
2075	if (need_more_worker(pool))
2076	wake_up_worker(pool);
2077
2078	/*
2079	* Record the last pool and clear PENDING which should be the last
2080	* update to @work. Also, do this inside @pool->lock so that
2081	* PENDING and queued state changes happen together while IRQ is
2082	* disabled.
2083	*/
2084	set_work_pool_and_clear_pending(work, pool->id);
2085
2086	spin_unlock_irq(&pool->lock);
2087
2088	lock_map_acquire(&pwq->wq->lockdep_map);
2089	lock_map_acquire(&lockdep_map);
2090	/*
2091	* Strictly speaking we should mark the invariant state without holding
2092	* any locks, that is, before these two lock_map_acquire()'s.
2093	*
2094	* However, that would result in:
2095	*
2096	* A(W1)
2097	* WFC(C)
2098	* A(W1)
2099	* C(C)
2100	*
2101	* Which would create W1->C->W1 dependencies, even though there is no
2102	* actual deadlock possible. There are two solutions, using a
2103	* read-recursive acquire on the work(queue) 'locks', but this will then
2104	* hit the lockdep limitation on recursive locks, or simply discard
2105	* these locks.
2106	*
2107	* AFAICT there is no possible deadlock scenario between the
2108	* flush_work() and complete() primitives (except for single-threaded
2109	* workqueues), so hiding them isn't a problem.
2110	*/
2111	lockdep_invariant_state(true);
2112	trace_workqueue_execute_start(work);
2113	worker->current_func(work);
2114	/*
2115	* While we must be careful to not use "work" after this, the trace
2116	* point will only record its address.
2117	*/
2118	trace_workqueue_execute_end(work);
2119	lock_map_release(&lockdep_map);
2120	lock_map_release(&pwq->wq->lockdep_map);
2121
2122	if (unlikely(in_atomic() \|\| lockdep_depth(current) > 0)) {
2123	pr_err("BUG: workqueue leaked lock or atomic: %s/0x%08x/%d\n"
2124	" last function: %pf\n",
2125	current->comm, preempt_count(), task_pid_nr(current),
2126	worker->current_func);
2127	debug_show_held_locks(current);
2128	dump_stack();
2129	}
2130
2131	/*
2132	* The following prevents a kworker from hogging CPU on !PREEMPT
2133	* kernels, where a requeueing work item waiting for something to
2134	* happen could deadlock with stop_machine as such work item could
2135	* indefinitely requeue itself while all other CPUs are trapped in
2136	* stop_machine. At the same time, report a quiescent RCU state so
2137	* the same condition doesn't freeze RCU.
2138	*/
2139	cond_resched();
2140
2141	spin_lock_irq(&pool->lock);
2142
2143	/* clear cpu intensive status */
2144	if (unlikely(cpu_intensive))
2145	worker_clr_flags(worker, WORKER_CPU_INTENSIVE);
2146
2147	/* we're done with it, release */
2148	hash_del(&worker->hentry);
2149	worker->current_work = NULL;
2150	worker->current_func = NULL;
2151	worker->current_pwq = NULL;
2152	worker->desc_valid = false;
2153	pwq_dec_nr_in_flight(pwq, work_color);
2154	}
2155
2156	/**
2157	* process_scheduled_works - process scheduled works
2158	* @worker: self
2159	*
2160	* Process all scheduled works. Please note that the scheduled list
2161	* may change while processing a work, so this function repeatedly
2162	* fetches a work from the top and executes it.
2163	*
2164	* CONTEXT:
2165	* spin_lock_irq(pool->lock) which may be released and regrabbed
2166	* multiple times.
2167	*/
2168	static void process_scheduled_works(struct worker *worker)
2169	{
2170	while (!list_empty(&worker->scheduled)) {
2171	struct work_struct *work = list_first_entry(&worker->scheduled,
2172	struct work_struct, entry);
2173	process_one_work(worker, work);
2174	}
2175	}
2176
2177	/**
2178	* worker_thread - the worker thread function
2179	* @__worker: self
2180	*
2181	* The worker thread function. All workers belong to a worker_pool -
2182	* either a per-cpu one or dynamic unbound one. These workers process all
2183	* work items regardless of their specific target workqueue. The only
2184	* exception is work items which belong to workqueues with a rescuer which
2185	* will be explained in rescuer_thread().
2186	*
2187	* Return: 0
2188	*/
2189	static int worker_thread(void *__worker)
2190	{
2191	struct worker *worker = __worker;
2192	struct worker_pool *pool = worker->pool;
2193
2194	/* tell the scheduler that this is a workqueue worker */
2195	worker->task->flags \|= PF_WQ_WORKER;
2196	woke_up:
2197	spin_lock_irq(&pool->lock);
2198
2199	/* am I supposed to die? */
2200	if (unlikely(worker->flags & WORKER_DIE)) {
2201	spin_unlock_irq(&pool->lock);
2202	WARN_ON_ONCE(!list_empty(&worker->entry));
2203	worker->task->flags &= ~PF_WQ_WORKER;
2204
2205	set_task_comm(worker->task, "kworker/dying");
2206	ida_simple_remove(&pool->worker_ida, worker->id);
2207	worker_detach_from_pool(worker, pool);
2208	kfree(worker);
2209	return 0;
2210	}
2211
2212	worker_leave_idle(worker);
2213	recheck:
2214	/* no more worker necessary? */
2215	if (!need_more_worker(pool))
2216	goto sleep;
2217
2218	/* do we need to manage? */
2219	if (unlikely(!may_start_working(pool)) && manage_workers(worker))
2220	goto recheck;
2221
2222	/*
2223	* ->scheduled list can only be filled while a worker is
2224	* preparing to process a work or actually processing it.
2225	* Make sure nobody diddled with it while I was sleeping.
2226	*/
2227	WARN_ON_ONCE(!list_empty(&worker->scheduled));
2228
2229	/*
2230	* Finish PREP stage. We're guaranteed to have at least one idle
2231	* worker or that someone else has already assumed the manager
2232	* role. This is where @worker starts participating in concurrency
2233	* management if applicable and concurrency management is restored
2234	* after being rebound. See rebind_workers() for details.
2235	*/
2236	worker_clr_flags(worker, WORKER_PREP \| WORKER_REBOUND);
2237
2238	do {
2239	struct work_struct *work =
2240	list_first_entry(&pool->worklist,
2241	struct work_struct, entry);
2242
2243	pool->watchdog_ts = jiffies;
2244
2245	if (likely(!(*work_data_bits(work) & WORK_STRUCT_LINKED))) {
2246	/* optimization path, not strictly necessary */
2247	process_one_work(worker, work);
2248	if (unlikely(!list_empty(&worker->scheduled)))
2249	process_scheduled_works(worker);
2250	} else {
2251	move_linked_works(work, &worker->scheduled, NULL);
2252	process_scheduled_works(worker);
2253	}
2254	} while (keep_working(pool));
2255
2256	worker_set_flags(worker, WORKER_PREP);
2257	sleep:
2258	/*
2259	* pool->lock is held and there's no work to process and no need to
2260	* manage, sleep. Workers are woken up only while holding
2261	* pool->lock or from local cpu, so setting the current state
2262	* before releasing pool->lock is enough to prevent losing any
2263	* event.
2264	*/
2265	worker_enter_idle(worker);
2266	__set_current_state(TASK_IDLE);
2267	spin_unlock_irq(&pool->lock);
2268	schedule();
2269	goto woke_up;
2270	}
2271
2272	/**
2273	* rescuer_thread - the rescuer thread function
2274	* @__rescuer: self
2275	*
2276	* Workqueue rescuer thread function. There's one rescuer for each
2277	* workqueue which has WQ_MEM_RECLAIM set.
2278	*
2279	* Regular work processing on a pool may block trying to create a new
2280	* worker which uses GFP_KERNEL allocation which has slight chance of
2281	* developing into deadlock if some works currently on the same queue
2282	* need to be processed to satisfy the GFP_KERNEL allocation. This is
2283	* the problem rescuer solves.
2284	*
2285	* When such condition is possible, the pool summons rescuers of all
2286	* workqueues which have works queued on the pool and let them process
2287	* those works so that forward progress can be guaranteed.
2288	*
2289	* This should happen rarely.
2290	*
2291	* Return: 0
2292	*/
2293	static int rescuer_thread(void *__rescuer)
2294	{
2295	struct worker *rescuer = __rescuer;
2296	struct workqueue_struct *wq = rescuer->rescue_wq;
2297	struct list_head *scheduled = &rescuer->scheduled;
2298	bool should_stop;
2299
2300	set_user_nice(current, RESCUER_NICE_LEVEL);
2301
2302	/*
2303	* Mark rescuer as worker too. As WORKER_PREP is never cleared, it
2304	* doesn't participate in concurrency management.
2305	*/
2306	rescuer->task->flags \|= PF_WQ_WORKER;
2307	repeat:
2308	set_current_state(TASK_IDLE);
2309
2310	/*
2311	* By the time the rescuer is requested to stop, the workqueue
2312	* shouldn't have any work pending, but @wq->maydays may still have
2313	* pwq(s) queued. This can happen by non-rescuer workers consuming
2314	* all the work items before the rescuer got to them. Go through
2315	* @wq->maydays processing before acting on should_stop so that the
2316	* list is always empty on exit.
2317	*/
2318	should_stop = kthread_should_stop();
2319
2320	/* see whether any pwq is asking for help */
2321	spin_lock_irq(&wq_mayday_lock);
2322
2323	while (!list_empty(&wq->maydays)) {
2324	struct pool_workqueue *pwq = list_first_entry(&wq->maydays,
2325	struct pool_workqueue, mayday_node);
2326	struct worker_pool *pool = pwq->pool;
2327	struct work_struct work, n;
2328	bool first = true;
2329
2330	__set_current_state(TASK_RUNNING);
2331	list_del_init(&pwq->mayday_node);
2332
2333	spin_unlock_irq(&wq_mayday_lock);
2334
2335	worker_attach_to_pool(rescuer, pool);
2336
2337	spin_lock_irq(&pool->lock);
2338	rescuer->pool = pool;
2339
2340	/*
2341	* Slurp in all works issued via this workqueue and
2342	* process'em.
2343	*/
2344	WARN_ON_ONCE(!list_empty(scheduled));
2345	list_for_each_entry_safe(work, n, &pool->worklist, entry) {
2346	if (get_work_pwq(work) == pwq) {
2347	if (first)
2348	pool->watchdog_ts = jiffies;
2349	move_linked_works(work, scheduled, &n);
2350	}
2351	first = false;
2352	}
2353
2354	if (!list_empty(scheduled)) {
2355	process_scheduled_works(rescuer);
2356
2357	/*
2358	* The above execution of rescued work items could
2359	* have created more to rescue through
2360	* pwq_activate_first_delayed() or chained
2361	* queueing. Let's put @pwq back on mayday list so
2362	* that such back-to-back work items, which may be
2363	* being used to relieve memory pressure, don't
2364	* incur MAYDAY_INTERVAL delay inbetween.
2365	*/
2366	if (need_to_create_worker(pool)) {
2367	spin_lock(&wq_mayday_lock);
2368	get_pwq(pwq);
2369	list_move_tail(&pwq->mayday_node, &wq->maydays);
2370	spin_unlock(&wq_mayday_lock);
2371	}
2372	}
2373
2374	/*
2375	* Put the reference grabbed by send_mayday(). @pool won't
2376	* go away while we're still attached to it.
2377	*/
2378	put_pwq(pwq);
2379
2380	/*
2381	* Leave this pool. If need_more_worker() is %true, notify a
2382	* regular worker; otherwise, we end up with 0 concurrency
2383	* and stalling the execution.
2384	*/
2385	if (need_more_worker(pool))
2386	wake_up_worker(pool);
2387
2388	rescuer->pool = NULL;
2389	spin_unlock_irq(&pool->lock);
2390
2391	worker_detach_from_pool(rescuer, pool);
2392
2393	spin_lock_irq(&wq_mayday_lock);
2394	}
2395
2396	spin_unlock_irq(&wq_mayday_lock);
2397
2398	if (should_stop) {
2399	__set_current_state(TASK_RUNNING);
2400	rescuer->task->flags &= ~PF_WQ_WORKER;
2401	return 0;
2402	}
2403
2404	/* rescuers should never participate in concurrency management */
2405	WARN_ON_ONCE(!(rescuer->flags & WORKER_NOT_RUNNING));
2406	schedule();
2407	goto repeat;
2408	}
2409
2410	/**
2411	* check_flush_dependency - check for flush dependency sanity
2412	* @target_wq: workqueue being flushed
2413	* @target_work: work item being flushed (NULL for workqueue flushes)
2414	*
2415	* %current is trying to flush the whole @target_wq or @target_work on it.
2416	* If @target_wq doesn't have %WQ_MEM_RECLAIM, verify that %current is not
2417	* reclaiming memory or running on a workqueue which doesn't have
2418	* %WQ_MEM_RECLAIM as that can break forward-progress guarantee leading to
2419	* a deadlock.
2420	*/
2421	static void check_flush_dependency(struct workqueue_struct *target_wq,
2422	struct work_struct *target_work)
2423	{
2424	work_func_t target_func = target_work ? target_work->func : NULL;
2425	struct worker *worker;
2426
2427	if (target_wq->flags & WQ_MEM_RECLAIM)
2428	return;
2429
2430	worker = current_wq_worker();
2431
2432	WARN_ONCE(current->flags & PF_MEMALLOC,
2433	"workqueue: PF_MEMALLOC task %d(%s) is flushing !WQ_MEM_RECLAIM %s:%pf",
2434	current->pid, current->comm, target_wq->name, target_func);
2435	WARN_ONCE(worker && ((worker->current_pwq->wq->flags &
2436	(WQ_MEM_RECLAIM \| __WQ_LEGACY)) == WQ_MEM_RECLAIM),
2437	"workqueue: WQ_MEM_RECLAIM %s:%pf is flushing !WQ_MEM_RECLAIM %s:%pf",
2438	worker->current_pwq->wq->name, worker->current_func,
2439	target_wq->name, target_func);
2440	}
2441
2442	struct wq_barrier {
2443	struct work_struct work;
2444	struct completion done;
2445	struct task_struct task; / purely informational */
2446	};
2447
2448	static void wq_barrier_func(struct work_struct *work)
2449	{
2450	struct wq_barrier *barr = container_of(work, struct wq_barrier, work);
2451	complete(&barr->done);
2452	}
2453
2454	/**
2455	* insert_wq_barrier - insert a barrier work
2456	* @pwq: pwq to insert barrier into
2457	* @barr: wq_barrier to insert
2458	* @target: target work to attach @barr to
2459	* @worker: worker currently executing @target, NULL if @target is not executing
2460	*
2461	* @barr is linked to @target such that @barr is completed only after
2462	* @target finishes execution. Please note that the ordering
2463	* guarantee is observed only with respect to @target and on the local
2464	* cpu.
2465	*
2466	* Currently, a queued barrier can't be canceled. This is because
2467	* try_to_grab_pending() can't determine whether the work to be
2468	* grabbed is at the head of the queue and thus can't clear LINKED
2469	* flag of the previous work while there must be a valid next work
2470	* after a work with LINKED flag set.
2471	*
2472	* Note that when @worker is non-NULL, @target may be modified
2473	* underneath us, so we can't reliably determine pwq from @target.
2474	*
2475	* CONTEXT:
2476	* spin_lock_irq(pool->lock).
2477	*/
2478	static void insert_wq_barrier(struct pool_workqueue *pwq,
2479	struct wq_barrier *barr,
2480	struct work_struct target, struct worker worker)
2481	{
2482	struct list_head *head;
2483	unsigned int linked = 0;
2484
2485	/*
2486	* debugobject calls are safe here even with pool->lock locked
2487	* as we know for sure that this will not trigger any of the
2488	* checks and call back into the fixup functions where we
2489	* might deadlock.
2490	*/
2491	INIT_WORK_ONSTACK(&barr->work, wq_barrier_func);
2492	__set_bit(WORK_STRUCT_PENDING_BIT, work_data_bits(&barr->work));
2493
2494	init_completion_map(&barr->done, &target->lockdep_map);
2495
2496	barr->task = current;
2497
2498	/*
2499	* If @target is currently being executed, schedule the
2500	* barrier to the worker; otherwise, put it after @target.
2501	*/
2502	if (worker)
2503	head = worker->scheduled.next;
2504	else {
2505	unsigned long *bits = work_data_bits(target);
2506
2507	head = target->entry.next;
2508	/* there can already be other linked works, inherit and set */
2509	linked = *bits & WORK_STRUCT_LINKED;
2510	__set_bit(WORK_STRUCT_LINKED_BIT, bits);
2511	}
2512
2513	debug_work_activate(&barr->work);
2514	insert_work(pwq, &barr->work, head,
2515	work_color_to_flags(WORK_NO_COLOR) \| linked);
2516	}
2517
2518	/**
2519	* flush_workqueue_prep_pwqs - prepare pwqs for workqueue flushing
2520	* @wq: workqueue being flushed
2521	* @flush_color: new flush color, < 0 for no-op
2522	* @work_color: new work color, < 0 for no-op
2523	*
2524	* Prepare pwqs for workqueue flushing.
2525	*
2526	* If @flush_color is non-negative, flush_color on all pwqs should be
2527	* -1. If no pwq has in-flight commands at the specified color, all
2528	* pwq->flush_color's stay at -1 and %false is returned. If any pwq
2529	* has in flight commands, its pwq->flush_color is set to
2530	* @flush_color, @wq->nr_pwqs_to_flush is updated accordingly, pwq
2531	* wakeup logic is armed and %true is returned.
2532	*
2533	* The caller should have initialized @wq->first_flusher prior to
2534	* calling this function with non-negative @flush_color. If
2535	* @flush_color is negative, no flush color update is done and %false
2536	* is returned.
2537	*
2538	* If @work_color is non-negative, all pwqs should have the same
2539	* work_color which is previous to @work_color and all will be
2540	* advanced to @work_color.
2541	*
2542	* CONTEXT:
2543	* mutex_lock(wq->mutex).
2544	*
2545	* Return:
2546	* %true if @flush_color >= 0 and there's something to flush. %false
2547	* otherwise.
2548	*/
2549	static bool flush_workqueue_prep_pwqs(struct workqueue_struct *wq,
2550	int flush_color, int work_color)
2551	{
2552	bool wait = false;
2553	struct pool_workqueue *pwq;
2554
2555	if (flush_color >= 0) {
2556	WARN_ON_ONCE(atomic_read(&wq->nr_pwqs_to_flush));
2557	atomic_set(&wq->nr_pwqs_to_flush, 1);
2558	}
2559
2560	for_each_pwq(pwq, wq) {
2561	struct worker_pool *pool = pwq->pool;
2562
2563	spin_lock_irq(&pool->lock);
2564
2565	if (flush_color >= 0) {
2566	WARN_ON_ONCE(pwq->flush_color != -1);
2567
2568	if (pwq->nr_in_flight[flush_color]) {
2569	pwq->flush_color = flush_color;
2570	atomic_inc(&wq->nr_pwqs_to_flush);
2571	wait = true;
2572	}
2573	}
2574
2575	if (work_color >= 0) {
2576	WARN_ON_ONCE(work_color != work_next_color(pwq->work_color));
2577	pwq->work_color = work_color;
2578	}
2579
2580	spin_unlock_irq(&pool->lock);
2581	}
2582
2583	if (flush_color >= 0 && atomic_dec_and_test(&wq->nr_pwqs_to_flush))
2584	complete(&wq->first_flusher->done);
2585
2586	return wait;
2587	}
2588
2589	/**
2590	* flush_workqueue - ensure that any scheduled work has run to completion.
2591	* @wq: workqueue to flush
2592	*
2593	* This function sleeps until all work items which were queued on entry
2594	* have finished execution, but it is not livelocked by new incoming ones.
2595	*/
2596	void flush_workqueue(struct workqueue_struct *wq)
2597	{
2598	struct wq_flusher this_flusher = {
2599	.list = LIST_HEAD_INIT(this_flusher.list),
2600	.flush_color = -1,
2601	.done = COMPLETION_INITIALIZER_ONSTACK_MAP(this_flusher.done, wq->lockdep_map),
2602	};
2603	int next_color;
2604
2605	if (WARN_ON(!wq_online))
2606	return;
2607
2608	mutex_lock(&wq->mutex);
2609
2610	/*
2611	* Start-to-wait phase
2612	*/
2613	next_color = work_next_color(wq->work_color);
2614
2615	if (next_color != wq->flush_color) {
2616	/*
2617	* Color space is not full. The current work_color
2618	* becomes our flush_color and work_color is advanced
2619	* by one.
2620	*/
2621	WARN_ON_ONCE(!list_empty(&wq->flusher_overflow));
2622	this_flusher.flush_color = wq->work_color;
2623	wq->work_color = next_color;
2624
2625	if (!wq->first_flusher) {
2626	/* no flush in progress, become the first flusher */
2627	WARN_ON_ONCE(wq->flush_color != this_flusher.flush_color);
2628
2629	wq->first_flusher = &this_flusher;
2630
2631	if (!flush_workqueue_prep_pwqs(wq, wq->flush_color,
2632	wq->work_color)) {
2633	/* nothing to flush, done */
2634	wq->flush_color = next_color;
2635	wq->first_flusher = NULL;
2636	goto out_unlock;
2637	}
2638	} else {
2639	/* wait in queue */
2640	WARN_ON_ONCE(wq->flush_color == this_flusher.flush_color);
2641	list_add_tail(&this_flusher.list, &wq->flusher_queue);
2642	flush_workqueue_prep_pwqs(wq, -1, wq->work_color);
2643	}
2644	} else {
2645	/*
2646	* Oops, color space is full, wait on overflow queue.
2647	* The next flush completion will assign us
2648	* flush_color and transfer to flusher_queue.
2649	*/
2650	list_add_tail(&this_flusher.list, &wq->flusher_overflow);
2651	}
2652
2653	check_flush_dependency(wq, NULL);
2654
2655	mutex_unlock(&wq->mutex);
2656
2657	wait_for_completion(&this_flusher.done);
2658
2659	/*
2660	* Wake-up-and-cascade phase
2661	*
2662	* First flushers are responsible for cascading flushes and
2663	* handling overflow. Non-first flushers can simply return.
2664	*/
2665	if (wq->first_flusher != &this_flusher)
2666	return;
2667
2668	mutex_lock(&wq->mutex);
2669
2670	/* we might have raced, check again with mutex held */
2671	if (wq->first_flusher != &this_flusher)
2672	goto out_unlock;
2673
2674	wq->first_flusher = NULL;
2675
2676	WARN_ON_ONCE(!list_empty(&this_flusher.list));
2677	WARN_ON_ONCE(wq->flush_color != this_flusher.flush_color);
2678
2679	while (true) {
2680	struct wq_flusher next, tmp;
2681
2682	/* complete all the flushers sharing the current flush color */
2683	list_for_each_entry_safe(next, tmp, &wq->flusher_queue, list) {
2684	if (next->flush_color != wq->flush_color)
2685	break;
2686	list_del_init(&next->list);
2687	complete(&next->done);
2688	}
2689
2690	WARN_ON_ONCE(!list_empty(&wq->flusher_overflow) &&
2691	wq->flush_color != work_next_color(wq->work_color));
2692
2693	/* this flush_color is finished, advance by one */
2694	wq->flush_color = work_next_color(wq->flush_color);
2695
2696	/* one color has been freed, handle overflow queue */
2697	if (!list_empty(&wq->flusher_overflow)) {
2698	/*
2699	* Assign the same color to all overflowed
2700	* flushers, advance work_color and append to
2701	* flusher_queue. This is the start-to-wait
2702	* phase for these overflowed flushers.
2703	*/
2704	list_for_each_entry(tmp, &wq->flusher_overflow, list)
2705	tmp->flush_color = wq->work_color;
2706
2707	wq->work_color = work_next_color(wq->work_color);
2708
2709	list_splice_tail_init(&wq->flusher_overflow,
2710	&wq->flusher_queue);
2711	flush_workqueue_prep_pwqs(wq, -1, wq->work_color);
2712	}
2713
2714	if (list_empty(&wq->flusher_queue)) {
2715	WARN_ON_ONCE(wq->flush_color != wq->work_color);
2716	break;
2717	}
2718
2719	/*
2720	* Need to flush more colors. Make the next flusher
2721	* the new first flusher and arm pwqs.
2722	*/
2723	WARN_ON_ONCE(wq->flush_color == wq->work_color);
2724	WARN_ON_ONCE(wq->flush_color != next->flush_color);
2725
2726	list_del_init(&next->list);
2727	wq->first_flusher = next;
2728
2729	if (flush_workqueue_prep_pwqs(wq, wq->flush_color, -1))
2730	break;
2731
2732	/*
2733	* Meh... this color is already done, clear first
2734	* flusher and repeat cascading.
2735	*/
2736	wq->first_flusher = NULL;
2737	}
2738
2739	out_unlock:
2740	mutex_unlock(&wq->mutex);
2741	}
2742	EXPORT_SYMBOL(flush_workqueue);
2743
2744	/**
2745	* drain_workqueue - drain a workqueue
2746	* @wq: workqueue to drain
2747	*
2748	* Wait until the workqueue becomes empty. While draining is in progress,
2749	* only chain queueing is allowed. IOW, only currently pending or running
2750	* work items on @wq can queue further work items on it. @wq is flushed
2751	* repeatedly until it becomes empty. The number of flushing is determined
2752	* by the depth of chaining and should be relatively short. Whine if it
2753	* takes too long.
2754	*/
2755	void drain_workqueue(struct workqueue_struct *wq)
2756	{
2757	unsigned int flush_cnt = 0;
2758	struct pool_workqueue *pwq;
2759
2760	/*
2761	* __queue_work() needs to test whether there are drainers, is much
2762	* hotter than drain_workqueue() and already looks at @wq->flags.
2763	* Use __WQ_DRAINING so that queue doesn't have to check nr_drainers.
2764	*/
2765	mutex_lock(&wq->mutex);
2766	if (!wq->nr_drainers++)
2767	wq->flags \|= __WQ_DRAINING;
2768	mutex_unlock(&wq->mutex);
2769	reflush:
2770	flush_workqueue(wq);
2771
2772	mutex_lock(&wq->mutex);
2773
2774	for_each_pwq(pwq, wq) {
2775	bool drained;
2776
2777	spin_lock_irq(&pwq->pool->lock);
2778	drained = !pwq->nr_active && list_empty(&pwq->delayed_works);
2779	spin_unlock_irq(&pwq->pool->lock);
2780
2781	if (drained)
2782	continue;
2783
2784	if (++flush_cnt == 10 \|\|
2785	(flush_cnt % 100 == 0 && flush_cnt <= 1000))
2786	pr_warn("workqueue %s: drain_workqueue() isn't complete after %u tries\n",
2787	wq->name, flush_cnt);
2788
2789	mutex_unlock(&wq->mutex);
2790	goto reflush;
2791	}
2792
2793	if (!--wq->nr_drainers)
2794	wq->flags &= ~__WQ_DRAINING;
2795	mutex_unlock(&wq->mutex);
2796	}
2797	EXPORT_SYMBOL_GPL(drain_workqueue);
2798
2799	static bool start_flush_work(struct work_struct work, struct wq_barrier barr)
2800	{
2801	struct worker *worker = NULL;
2802	struct worker_pool *pool;
2803	struct pool_workqueue *pwq;
2804
2805	might_sleep();
2806
2807	local_irq_disable();
2808	pool = get_work_pool(work);
2809	if (!pool) {
2810	local_irq_enable();
2811	return false;
2812	}
2813
2814	spin_lock(&pool->lock);
2815	/* see the comment in try_to_grab_pending() with the same code */
2816	pwq = get_work_pwq(work);
2817	if (pwq) {
2818	if (unlikely(pwq->pool != pool))
2819	goto already_gone;
2820	} else {
2821	worker = find_worker_executing_work(pool, work);
2822	if (!worker)
2823	goto already_gone;
2824	pwq = worker->current_pwq;
2825	}
2826
2827	check_flush_dependency(pwq->wq, work);
2828
2829	insert_wq_barrier(pwq, barr, work, worker);
2830	spin_unlock_irq(&pool->lock);
2831
2832	/*
2833	* Force a lock recursion deadlock when using flush_work() inside a
2834	* single-threaded or rescuer equipped workqueue.
2835	*
2836	* For single threaded workqueues the deadlock happens when the work
2837	* is after the work issuing the flush_work(). For rescuer equipped
2838	* workqueues the deadlock happens when the rescuer stalls, blocking
2839	* forward progress.
2840	*/
2841	if (pwq->wq->saved_max_active == 1 \|\| pwq->wq->rescuer) {
2842	lock_map_acquire(&pwq->wq->lockdep_map);
2843	lock_map_release(&pwq->wq->lockdep_map);
2844	}
2845
2846	return true;
2847	already_gone:
2848	spin_unlock_irq(&pool->lock);
2849	return false;
2850	}
2851
2852	/**
2853	* flush_work - wait for a work to finish executing the last queueing instance
2854	* @work: the work to flush
2855	*
2856	* Wait until @work has finished execution. @work is guaranteed to be idle
2857	* on return if it hasn't been requeued since flush started.
2858	*
2859	* Return:
2860	* %true if flush_work() waited for the work to finish execution,
2861	* %false if it was already idle.
2862	*/
2863	bool flush_work(struct work_struct *work)
2864	{
2865	struct wq_barrier barr;
2866
2867	if (WARN_ON(!wq_online))
2868	return false;
2869
2870	if (start_flush_work(work, &barr)) {
2871	wait_for_completion(&barr.done);
2872	destroy_work_on_stack(&barr.work);
2873	return true;
2874	} else {
2875	return false;
2876	}
2877	}
2878	EXPORT_SYMBOL_GPL(flush_work);
2879
2880	struct cwt_wait {
2881	wait_queue_entry_t wait;
2882	struct work_struct *work;
2883	};
2884
2885	static int cwt_wakefn(wait_queue_entry_t wait, unsigned mode, int sync, void key)
2886	{
2887	struct cwt_wait *cwait = container_of(wait, struct cwt_wait, wait);
2888
2889	if (cwait->work != key)
2890	return 0;
2891	return autoremove_wake_function(wait, mode, sync, key);
2892	}
2893
2894	static bool __cancel_work_timer(struct work_struct *work, bool is_dwork)
2895	{
2896	static DECLARE_WAIT_QUEUE_HEAD(cancel_waitq);
2897	unsigned long flags;
2898	int ret;
2899
2900	do {
2901	ret = try_to_grab_pending(work, is_dwork, &flags);
2902	/*
2903	* If someone else is already canceling, wait for it to
2904	* finish. flush_work() doesn't work for PREEMPT_NONE
2905	* because we may get scheduled between @work's completion
2906	* and the other canceling task resuming and clearing
2907	* CANCELING - flush_work() will return false immediately
2908	* as @work is no longer busy, try_to_grab_pending() will
2909	* return -ENOENT as @work is still being canceled and the
2910	* other canceling task won't be able to clear CANCELING as
2911	* we're hogging the CPU.
2912	*
2913	* Let's wait for completion using a waitqueue. As this
2914	* may lead to the thundering herd problem, use a custom
2915	* wake function which matches @work along with exclusive
2916	* wait and wakeup.
2917	*/
2918	if (unlikely(ret == -ENOENT)) {
2919	struct cwt_wait cwait;
2920
2921	init_wait(&cwait.wait);
2922	cwait.wait.func = cwt_wakefn;
2923	cwait.work = work;
2924
2925	prepare_to_wait_exclusive(&cancel_waitq, &cwait.wait,
2926	TASK_UNINTERRUPTIBLE);
2927	if (work_is_canceling(work))
2928	schedule();
2929	finish_wait(&cancel_waitq, &cwait.wait);
2930	}
2931	} while (unlikely(ret < 0));
2932
2933	/* tell other tasks trying to grab @work to back off */
2934	mark_work_canceling(work);
2935	local_irq_restore(flags);
2936
2937	/*
2938	* This allows canceling during early boot. We know that @work
2939	* isn't executing.
2940	*/
2941	if (wq_online)
2942	flush_work(work);
2943
2944	clear_work_data(work);
2945
2946	/*
2947	* Paired with prepare_to_wait() above so that either
2948	* waitqueue_active() is visible here or !work_is_canceling() is
2949	* visible there.
2950	*/
2951	smp_mb();
2952	if (waitqueue_active(&cancel_waitq))
2953	__wake_up(&cancel_waitq, TASK_NORMAL, 1, work);
2954
2955	return ret;
2956	}
2957
2958	/**
2959	* cancel_work_sync - cancel a work and wait for it to finish
2960	* @work: the work to cancel
2961	*
2962	* Cancel @work and wait for its execution to finish. This function
2963	* can be used even if the work re-queues itself or migrates to
2964	* another workqueue. On return from this function, @work is
2965	* guaranteed to be not pending or executing on any CPU.
2966	*
2967	* cancel_work_sync(&delayed_work->work) must not be used for
2968	* delayed_work's. Use cancel_delayed_work_sync() instead.
2969	*
2970	* The caller must ensure that the workqueue on which @work was last
2971	* queued can't be destroyed before this function returns.
2972	*
2973	* Return:
2974	* %true if @work was pending, %false otherwise.
2975	*/
2976	bool cancel_work_sync(struct work_struct *work)
2977	{
2978	return __cancel_work_timer(work, false);
2979	}
2980	EXPORT_SYMBOL_GPL(cancel_work_sync);
2981
2982	/**
2983	* flush_delayed_work - wait for a dwork to finish executing the last queueing
2984	* @dwork: the delayed work to flush
2985	*
2986	* Delayed timer is cancelled and the pending work is queued for
2987	* immediate execution. Like flush_work(), this function only
2988	* considers the last queueing instance of @dwork.
2989	*
2990	* Return:
2991	* %true if flush_work() waited for the work to finish execution,
2992	* %false if it was already idle.
2993	*/
2994	bool flush_delayed_work(struct delayed_work *dwork)
2995	{
2996	local_irq_disable();
2997	if (del_timer_sync(&dwork->timer))
2998	__queue_work(dwork->cpu, dwork->wq, &dwork->work);
2999	local_irq_enable();
3000	return flush_work(&dwork->work);
3001	}
3002	EXPORT_SYMBOL(flush_delayed_work);
3003
3004	static bool __cancel_work(struct work_struct *work, bool is_dwork)
3005	{
3006	unsigned long flags;
3007	int ret;
3008
3009	do {
3010	ret = try_to_grab_pending(work, is_dwork, &flags);
3011	} while (unlikely(ret == -EAGAIN));
3012
3013	if (unlikely(ret < 0))
3014	return false;
3015
3016	set_work_pool_and_clear_pending(work, get_work_pool_id(work));
3017	local_irq_restore(flags);
3018	return ret;
3019	}
3020
3021	/*
3022	* See cancel_delayed_work()
3023	*/
3024	bool cancel_work(struct work_struct *work)
3025	{
3026	return __cancel_work(work, false);
3027	}
3028
3029	/**
3030	* cancel_delayed_work - cancel a delayed work
3031	* @dwork: delayed_work to cancel
3032	*
3033	* Kill off a pending delayed_work.
3034	*
3035	* Return: %true if @dwork was pending and canceled; %false if it wasn't
3036	* pending.
3037	*
3038	* Note:
3039	* The work callback function may still be running on return, unless
3040	* it returns %true and the work doesn't re-arm itself. Explicitly flush or
3041	* use cancel_delayed_work_sync() to wait on it.
3042	*
3043	* This function is safe to call from any context including IRQ handler.
3044	*/
3045	bool cancel_delayed_work(struct delayed_work *dwork)
3046	{
3047	return __cancel_work(&dwork->work, true);
3048	}
3049	EXPORT_SYMBOL(cancel_delayed_work);
3050
3051	/**
3052	* cancel_delayed_work_sync - cancel a delayed work and wait for it to finish
3053	* @dwork: the delayed work cancel
3054	*
3055	* This is cancel_work_sync() for delayed works.
3056	*
3057	* Return:
3058	* %true if @dwork was pending, %false otherwise.
3059	*/
3060	bool cancel_delayed_work_sync(struct delayed_work *dwork)
3061	{
3062	return __cancel_work_timer(&dwork->work, true);
3063	}
3064	EXPORT_SYMBOL(cancel_delayed_work_sync);
3065
3066	/**
3067	* schedule_on_each_cpu - execute a function synchronously on each online CPU
3068	* @func: the function to call
3069	*
3070	* schedule_on_each_cpu() executes @func on each online CPU using the
3071	* system workqueue and blocks until all CPUs have completed.
3072	* schedule_on_each_cpu() is very slow.
3073	*
3074	* Return:
3075	* 0 on success, -errno on failure.
3076	*/
3077	int schedule_on_each_cpu(work_func_t func)
3078	{
3079	int cpu;
3080	struct work_struct __percpu *works;
3081
3082	works = alloc_percpu(struct work_struct);
3083	if (!works)
3084	return -ENOMEM;
3085
3086	get_online_cpus();
3087
3088	for_each_online_cpu(cpu) {
3089	struct work_struct *work = per_cpu_ptr(works, cpu);
3090
3091	INIT_WORK(work, func);
3092	schedule_work_on(cpu, work);
3093	}
3094
3095	for_each_online_cpu(cpu)
3096	flush_work(per_cpu_ptr(works, cpu));
3097
3098	put_online_cpus();
3099	free_percpu(works);
3100	return 0;
3101	}
3102
3103	/**
3104	* execute_in_process_context - reliably execute the routine with user context
3105	* @fn: the function to execute
3106	* @ew: guaranteed storage for the execute work structure (must
3107	* be available when the work executes)
3108	*
3109	* Executes the function immediately if process context is available,
3110	* otherwise schedules the function for delayed execution.
3111	*
3112	* Return: 0 - function was executed
3113	* 1 - function was scheduled for execution
3114	*/
3115	int execute_in_process_context(work_func_t fn, struct execute_work *ew)
3116	{
3117	if (!in_interrupt()) {
3118	fn(&ew->work);
3119	return 0;
3120	}
3121
3122	INIT_WORK(&ew->work, fn);
3123	schedule_work(&ew->work);
3124
3125	return 1;
3126	}
3127	EXPORT_SYMBOL_GPL(execute_in_process_context);
3128
3129	/**
3130	* free_workqueue_attrs - free a workqueue_attrs
3131	* @attrs: workqueue_attrs to free
3132	*
3133	* Undo alloc_workqueue_attrs().
3134	*/
3135	void free_workqueue_attrs(struct workqueue_attrs *attrs)
3136	{
3137	if (attrs) {
3138	free_cpumask_var(attrs->cpumask);
3139	kfree(attrs);
3140	}
3141	}
3142
3143	/**
3144	* alloc_workqueue_attrs - allocate a workqueue_attrs
3145	* @gfp_mask: allocation mask to use
3146	*
3147	* Allocate a new workqueue_attrs, initialize with default settings and
3148	* return it.
3149	*
3150	* Return: The allocated new workqueue_attr on success. %NULL on failure.
3151	*/
3152	struct workqueue_attrs *alloc_workqueue_attrs(gfp_t gfp_mask)
3153	{
3154	struct workqueue_attrs *attrs;
3155
3156	attrs = kzalloc(sizeof(*attrs), gfp_mask);
3157	if (!attrs)
3158	goto fail;
3159	if (!alloc_cpumask_var(&attrs->cpumask, gfp_mask))
3160	goto fail;
3161
3162	cpumask_copy(attrs->cpumask, cpu_possible_mask);
3163	return attrs;
3164	fail:
3165	free_workqueue_attrs(attrs);
3166	return NULL;
3167	}
3168
3169	static void copy_workqueue_attrs(struct workqueue_attrs *to,
3170	const struct workqueue_attrs *from)
3171	{
3172	to->nice = from->nice;
3173	cpumask_copy(to->cpumask, from->cpumask);
3174	/*
3175	* Unlike hash and equality test, this function doesn't ignore
3176	* ->no_numa as it is used for both pool and wq attrs. Instead,
3177	* get_unbound_pool() explicitly clears ->no_numa after copying.
3178	*/
3179	to->no_numa = from->no_numa;
3180	}
3181
3182	/* hash value of the content of @attr */
3183	static u32 wqattrs_hash(const struct workqueue_attrs *attrs)
3184	{
3185	u32 hash = 0;
3186
3187	hash = jhash_1word(attrs->nice, hash);
3188	hash = jhash(cpumask_bits(attrs->cpumask),
3189	BITS_TO_LONGS(nr_cpumask_bits) * sizeof(long), hash);
3190	return hash;
3191	}
3192
3193	/* content equality test */
3194	static bool wqattrs_equal(const struct workqueue_attrs *a,
3195	const struct workqueue_attrs *b)
3196	{
3197	if (a->nice != b->nice)
3198	return false;
3199	if (!cpumask_equal(a->cpumask, b->cpumask))
3200	return false;
3201	return true;
3202	}
3203
3204	/**
3205	* init_worker_pool - initialize a newly zalloc'd worker_pool
3206	* @pool: worker_pool to initialize
3207	*
3208	* Initialize a newly zalloc'd @pool. It also allocates @pool->attrs.
3209	*
3210	* Return: 0 on success, -errno on failure. Even on failure, all fields
3211	* inside @pool proper are initialized and put_unbound_pool() can be called
3212	* on @pool safely to release it.
3213	*/
3214	static int init_worker_pool(struct worker_pool *pool)
3215	{
3216	spin_lock_init(&pool->lock);
3217	pool->id = -1;
3218	pool->cpu = -1;
3219	pool->node = NUMA_NO_NODE;
3220	pool->flags \|= POOL_DISASSOCIATED;
3221	pool->watchdog_ts = jiffies;
3222	INIT_LIST_HEAD(&pool->worklist);
3223	INIT_LIST_HEAD(&pool->idle_list);
3224	hash_init(pool->busy_hash);
3225
3226	timer_setup(&pool->idle_timer, idle_worker_timeout, TIMER_DEFERRABLE);
3227
3228	timer_setup(&pool->mayday_timer, pool_mayday_timeout, 0);
3229
3230	mutex_init(&pool->attach_mutex);
3231	INIT_LIST_HEAD(&pool->workers);
3232
3233	ida_init(&pool->worker_ida);
3234	INIT_HLIST_NODE(&pool->hash_node);
3235	pool->refcnt = 1;
3236
3237	/* shouldn't fail above this point */
3238	pool->attrs = alloc_workqueue_attrs(GFP_KERNEL);
3239	if (!pool->attrs)
3240	return -ENOMEM;
3241	return 0;
3242	}
3243
3244	static void rcu_free_wq(struct rcu_head *rcu)
3245	{
3246	struct workqueue_struct *wq =
3247	container_of(rcu, struct workqueue_struct, rcu);
3248
3249	if (!(wq->flags & WQ_UNBOUND))
3250	free_percpu(wq->cpu_pwqs);
3251	else
3252	free_workqueue_attrs(wq->unbound_attrs);
3253
3254	kfree(wq->rescuer);
3255	kfree(wq);
3256	}
3257
3258	static void rcu_free_pool(struct rcu_head *rcu)
3259	{
3260	struct worker_pool *pool = container_of(rcu, struct worker_pool, rcu);
3261
3262	ida_destroy(&pool->worker_ida);
3263	free_workqueue_attrs(pool->attrs);
3264	kfree(pool);
3265	}
3266
3267	/**
3268	* put_unbound_pool - put a worker_pool
3269	* @pool: worker_pool to put
3270	*
3271	* Put @pool. If its refcnt reaches zero, it gets destroyed in sched-RCU
3272	* safe manner. get_unbound_pool() calls this function on its failure path
3273	* and this function should be able to release pools which went through,
3274	* successfully or not, init_worker_pool().
3275	*
3276	* Should be called with wq_pool_mutex held.
3277	*/
3278	static void put_unbound_pool(struct worker_pool *pool)
3279	{
3280	DECLARE_COMPLETION_ONSTACK(detach_completion);
3281	struct worker *worker;
3282
3283	lockdep_assert_held(&wq_pool_mutex);
3284
3285	if (--pool->refcnt)
3286	return;
3287
3288	/* sanity checks */
3289	if (WARN_ON(!(pool->cpu < 0)) \|\|
3290	WARN_ON(!list_empty(&pool->worklist)))
3291	return;
3292
3293	/* release id and unhash */
3294	if (pool->id >= 0)
3295	idr_remove(&worker_pool_idr, pool->id);
3296	hash_del(&pool->hash_node);
3297
3298	/*
3299	* Become the manager and destroy all workers. This prevents
3300	* @pool's workers from blocking on attach_mutex. We're the last
3301	* manager and @pool gets freed with the flag set.
3302	*/
3303	spin_lock_irq(&pool->lock);
3304	wait_event_lock_irq(wq_manager_wait,
3305	!(pool->flags & POOL_MANAGER_ACTIVE), pool->lock);
3306	pool->flags \|= POOL_MANAGER_ACTIVE;
3307
3308	while ((worker = first_idle_worker(pool)))
3309	destroy_worker(worker);
3310	WARN_ON(pool->nr_workers \|\| pool->nr_idle);
3311	spin_unlock_irq(&pool->lock);
3312
3313	mutex_lock(&pool->attach_mutex);
3314	if (!list_empty(&pool->workers))
3315	pool->detach_completion = &detach_completion;
3316	mutex_unlock(&pool->attach_mutex);
3317
3318	if (pool->detach_completion)
3319	wait_for_completion(pool->detach_completion);
3320
3321	/* shut down the timers */
3322	del_timer_sync(&pool->idle_timer);
3323	del_timer_sync(&pool->mayday_timer);
3324
3325	/* sched-RCU protected to allow dereferences from get_work_pool() */
3326	call_rcu_sched(&pool->rcu, rcu_free_pool);
3327	}
3328
3329	/**
3330	* get_unbound_pool - get a worker_pool with the specified attributes
3331	* @attrs: the attributes of the worker_pool to get
3332	*
3333	* Obtain a worker_pool which has the same attributes as @attrs, bump the
3334	* reference count and return it. If there already is a matching
3335	* worker_pool, it will be used; otherwise, this function attempts to
3336	* create a new one.
3337	*
3338	* Should be called with wq_pool_mutex held.
3339	*
3340	* Return: On success, a worker_pool with the same attributes as @attrs.
3341	* On failure, %NULL.
3342	*/
3343	static struct worker_pool get_unbound_pool(const struct workqueue_attrs attrs)
3344	{
3345	u32 hash = wqattrs_hash(attrs);
3346	struct worker_pool *pool;
3347	int node;
3348	int target_node = NUMA_NO_NODE;
3349
3350	lockdep_assert_held(&wq_pool_mutex);
3351
3352	/* do we already have a matching pool? */
3353	hash_for_each_possible(unbound_pool_hash, pool, hash_node, hash) {
3354	if (wqattrs_equal(pool->attrs, attrs)) {
3355	pool->refcnt++;
3356	return pool;
3357	}
3358	}
3359
3360	/* if cpumask is contained inside a NUMA node, we belong to that node */
3361	if (wq_numa_enabled) {
3362	for_each_node(node) {
3363	if (cpumask_subset(attrs->cpumask,
3364	wq_numa_possible_cpumask[node])) {
3365	target_node = node;
3366	break;
3367	}
3368	}
3369	}
3370
3371	/* nope, create a new one */
3372	pool = kzalloc_node(sizeof(*pool), GFP_KERNEL, target_node);
3373	if (!pool \|\| init_worker_pool(pool) < 0)
3374	goto fail;
3375
3376	lockdep_set_subclass(&pool->lock, 1); /* see put_pwq() */
3377	copy_workqueue_attrs(pool->attrs, attrs);
3378	pool->node = target_node;
3379
3380	/*
3381	* no_numa isn't a worker_pool attribute, always clear it. See
3382	* 'struct workqueue_attrs' comments for detail.
3383	*/
3384	pool->attrs->no_numa = false;
3385
3386	if (worker_pool_assign_id(pool) < 0)
3387	goto fail;
3388
3389	/* create and start the initial worker */
3390	if (wq_online && !create_worker(pool))
3391	goto fail;
3392
3393	/* install */
3394	hash_add(unbound_pool_hash, &pool->hash_node, hash);
3395
3396	return pool;
3397	fail:
3398	if (pool)
3399	put_unbound_pool(pool);
3400	return NULL;
3401	}
3402
3403	static void rcu_free_pwq(struct rcu_head *rcu)
3404	{
3405	kmem_cache_free(pwq_cache,
3406	container_of(rcu, struct pool_workqueue, rcu));
3407	}
3408
3409	/*
3410	* Scheduled on system_wq by put_pwq() when an unbound pwq hits zero refcnt
3411	* and needs to be destroyed.
3412	*/
3413	static void pwq_unbound_release_workfn(struct work_struct *work)
3414	{
3415	struct pool_workqueue *pwq = container_of(work, struct pool_workqueue,
3416	unbound_release_work);
3417	struct workqueue_struct *wq = pwq->wq;
3418	struct worker_pool *pool = pwq->pool;
3419	bool is_last;
3420
3421	if (WARN_ON_ONCE(!(wq->flags & WQ_UNBOUND)))
3422	return;
3423
3424	mutex_lock(&wq->mutex);
3425	list_del_rcu(&pwq->pwqs_node);
3426	is_last = list_empty(&wq->pwqs);
3427	mutex_unlock(&wq->mutex);
3428
3429	mutex_lock(&wq_pool_mutex);
3430	put_unbound_pool(pool);
3431	mutex_unlock(&wq_pool_mutex);
3432
3433	call_rcu_sched(&pwq->rcu, rcu_free_pwq);
3434
3435	/*
3436	* If we're the last pwq going away, @wq is already dead and no one
3437	* is gonna access it anymore. Schedule RCU free.
3438	*/
3439	if (is_last)
3440	call_rcu_sched(&wq->rcu, rcu_free_wq);
3441	}
3442
3443	/**
3444	* pwq_adjust_max_active - update a pwq's max_active to the current setting
3445	* @pwq: target pool_workqueue
3446	*
3447	* If @pwq isn't freezing, set @pwq->max_active to the associated
3448	* workqueue's saved_max_active and activate delayed work items
3449	* accordingly. If @pwq is freezing, clear @pwq->max_active to zero.
3450	*/
3451	static void pwq_adjust_max_active(struct pool_workqueue *pwq)
3452	{
3453	struct workqueue_struct *wq = pwq->wq;
3454	bool freezable = wq->flags & WQ_FREEZABLE;
3455	unsigned long flags;
3456
3457	/* for @wq->saved_max_active */
3458	lockdep_assert_held(&wq->mutex);
3459
3460	/* fast exit for non-freezable wqs */
3461	if (!freezable && pwq->max_active == wq->saved_max_active)
3462	return;
3463
3464	/* this function can be called during early boot w/ irq disabled */
3465	spin_lock_irqsave(&pwq->pool->lock, flags);
3466
3467	/*
3468	* During [un]freezing, the caller is responsible for ensuring that
3469	* this function is called at least once after @workqueue_freezing
3470	* is updated and visible.
3471	*/
3472	if (!freezable \|\| !workqueue_freezing) {
3473	pwq->max_active = wq->saved_max_active;
3474
3475	while (!list_empty(&pwq->delayed_works) &&
3476	pwq->nr_active < pwq->max_active)
3477	pwq_activate_first_delayed(pwq);
3478
3479	/*
3480	* Need to kick a worker after thawed or an unbound wq's
3481	* max_active is bumped. It's a slow path. Do it always.
3482	*/
3483	wake_up_worker(pwq->pool);
3484	} else {
3485	pwq->max_active = 0;
3486	}
3487
3488	spin_unlock_irqrestore(&pwq->pool->lock, flags);
3489	}
3490
3491	/* initialize newly alloced @pwq which is associated with @wq and @pool */
3492	static void init_pwq(struct pool_workqueue pwq, struct workqueue_struct wq,
3493	struct worker_pool *pool)
3494	{
3495	BUG_ON((unsigned long)pwq & WORK_STRUCT_FLAG_MASK);
3496
3497	memset(pwq, 0, sizeof(*pwq));
3498
3499	pwq->pool = pool;
3500	pwq->wq = wq;
3501	pwq->flush_color = -1;
3502	pwq->refcnt = 1;
3503	INIT_LIST_HEAD(&pwq->delayed_works);
3504	INIT_LIST_HEAD(&pwq->pwqs_node);
3505	INIT_LIST_HEAD(&pwq->mayday_node);
3506	INIT_WORK(&pwq->unbound_release_work, pwq_unbound_release_workfn);
3507	}
3508
3509	/* sync @pwq with the current state of its associated wq and link it */
3510	static void link_pwq(struct pool_workqueue *pwq)
3511	{
3512	struct workqueue_struct *wq = pwq->wq;
3513
3514	lockdep_assert_held(&wq->mutex);
3515
3516	/* may be called multiple times, ignore if already linked */
3517	if (!list_empty(&pwq->pwqs_node))
3518	return;
3519
3520	/* set the matching work_color */
3521	pwq->work_color = wq->work_color;
3522
3523	/* sync max_active to the current setting */
3524	pwq_adjust_max_active(pwq);
3525
3526	/* link in @pwq */
3527	list_add_rcu(&pwq->pwqs_node, &wq->pwqs);
3528	}
3529
3530	/* obtain a pool matching @attr and create a pwq associating the pool and @wq */
3531	static struct pool_workqueue alloc_unbound_pwq(struct workqueue_struct wq,
3532	const struct workqueue_attrs *attrs)
3533	{
3534	struct worker_pool *pool;
3535	struct pool_workqueue *pwq;
3536
3537	lockdep_assert_held(&wq_pool_mutex);
3538
3539	pool = get_unbound_pool(attrs);
3540	if (!pool)
3541	return NULL;
3542
3543	pwq = kmem_cache_alloc_node(pwq_cache, GFP_KERNEL, pool->node);
3544	if (!pwq) {
3545	put_unbound_pool(pool);
3546	return NULL;
3547	}
3548
3549	init_pwq(pwq, wq, pool);
3550	return pwq;
3551	}
3552
3553	/**
3554	* wq_calc_node_cpumask - calculate a wq_attrs' cpumask for the specified node
3555	* @attrs: the wq_attrs of the default pwq of the target workqueue
3556	* @node: the target NUMA node
3557	* @cpu_going_down: if >= 0, the CPU to consider as offline
3558	* @cpumask: outarg, the resulting cpumask
3559	*
3560	* Calculate the cpumask a workqueue with @attrs should use on @node. If
3561	* @cpu_going_down is >= 0, that cpu is considered offline during
3562	* calculation. The result is stored in @cpumask.
3563	*
3564	* If NUMA affinity is not enabled, @attrs->cpumask is always used. If
3565	* enabled and @node has online CPUs requested by @attrs, the returned
3566	* cpumask is the intersection of the possible CPUs of @node and
3567	* @attrs->cpumask.
3568	*
3569	* The caller is responsible for ensuring that the cpumask of @node stays
3570	* stable.
3571	*
3572	* Return: %true if the resulting @cpumask is different from @attrs->cpumask,
3573	* %false if equal.
3574	*/
3575	static bool wq_calc_node_cpumask(const struct workqueue_attrs *attrs, int node,
3576	int cpu_going_down, cpumask_t *cpumask)
3577	{
3578	if (!wq_numa_enabled \|\| attrs->no_numa)
3579	goto use_dfl;
3580
3581	/* does @node have any online CPUs @attrs wants? */
3582	cpumask_and(cpumask, cpumask_of_node(node), attrs->cpumask);
3583	if (cpu_going_down >= 0)
3584	cpumask_clear_cpu(cpu_going_down, cpumask);
3585
3586	if (cpumask_empty(cpumask))
3587	goto use_dfl;
3588
3589	/* yeap, return possible CPUs in @node that @attrs wants */
3590	cpumask_and(cpumask, attrs->cpumask, wq_numa_possible_cpumask[node]);
3591
3592	if (cpumask_empty(cpumask)) {
3593	pr_warn_once("WARNING: workqueue cpumask: online intersect > "
3594	"possible intersect\n");
3595	return false;
3596	}
3597
3598	return !cpumask_equal(cpumask, attrs->cpumask);
3599
3600	use_dfl:
3601	cpumask_copy(cpumask, attrs->cpumask);
3602	return false;
3603	}
3604
3605	/* install @pwq into @wq's numa_pwq_tbl[] for @node and return the old pwq */
3606	static struct pool_workqueue numa_pwq_tbl_install(struct workqueue_struct wq,
3607	int node,
3608	struct pool_workqueue *pwq)
3609	{
3610	struct pool_workqueue *old_pwq;
3611
3612	lockdep_assert_held(&wq_pool_mutex);
3613	lockdep_assert_held(&wq->mutex);
3614
3615	/* link_pwq() can handle duplicate calls */
3616	link_pwq(pwq);
3617
3618	old_pwq = rcu_access_pointer(wq->numa_pwq_tbl[node]);
3619	rcu_assign_pointer(wq->numa_pwq_tbl[node], pwq);
3620	return old_pwq;
3621	}
3622
3623	/* context to store the prepared attrs & pwqs before applying */
3624	struct apply_wqattrs_ctx {
3625	struct workqueue_struct wq; / target workqueue */
3626	struct workqueue_attrs attrs; / attrs to apply */
3627	struct list_head list; /* queued for batching commit */
3628	struct pool_workqueue *dfl_pwq;
3629	struct pool_workqueue *pwq_tbl[];
3630	};
3631
3632	/* free the resources after success or abort */
3633	static void apply_wqattrs_cleanup(struct apply_wqattrs_ctx *ctx)
3634	{
3635	if (ctx) {
3636	int node;
3637
3638	for_each_node(node)
3639	put_pwq_unlocked(ctx->pwq_tbl[node]);
3640	put_pwq_unlocked(ctx->dfl_pwq);
3641
3642	free_workqueue_attrs(ctx->attrs);
3643
3644	kfree(ctx);
3645	}
3646	}
3647
3648	/* allocate the attrs and pwqs for later installation */
3649	static struct apply_wqattrs_ctx *
3650	apply_wqattrs_prepare(struct workqueue_struct *wq,
3651	const struct workqueue_attrs *attrs)
3652	{
3653	struct apply_wqattrs_ctx *ctx;
3654	struct workqueue_attrs new_attrs, tmp_attrs;
3655	int node;
3656
3657	lockdep_assert_held(&wq_pool_mutex);
3658
3659	ctx = kzalloc(sizeof(ctx) + nr_node_ids sizeof(ctx->pwq_tbl[0]),
3660	GFP_KERNEL);
3661
3662	new_attrs = alloc_workqueue_attrs(GFP_KERNEL);
3663	tmp_attrs = alloc_workqueue_attrs(GFP_KERNEL);
3664	if (!ctx \|\| !new_attrs \|\| !tmp_attrs)
3665	goto out_free;
3666
3667	/*
3668	* Calculate the attrs of the default pwq.
3669	* If the user configured cpumask doesn't overlap with the
3670	* wq_unbound_cpumask, we fallback to the wq_unbound_cpumask.
3671	*/
3672	copy_workqueue_attrs(new_attrs, attrs);
3673	cpumask_and(new_attrs->cpumask, new_attrs->cpumask, wq_unbound_cpumask);
3674	if (unlikely(cpumask_empty(new_attrs->cpumask)))
3675	cpumask_copy(new_attrs->cpumask, wq_unbound_cpumask);
3676
3677	/*
3678	* We may create multiple pwqs with differing cpumasks. Make a
3679	* copy of @new_attrs which will be modified and used to obtain
3680	* pools.
3681	*/
3682	copy_workqueue_attrs(tmp_attrs, new_attrs);
3683
3684	/*
3685	* If something goes wrong during CPU up/down, we'll fall back to
3686	* the default pwq covering whole @attrs->cpumask. Always create
3687	* it even if we don't use it immediately.
3688	*/
3689	ctx->dfl_pwq = alloc_unbound_pwq(wq, new_attrs);
3690	if (!ctx->dfl_pwq)
3691	goto out_free;
3692
3693	for_each_node(node) {
3694	if (wq_calc_node_cpumask(new_attrs, node, -1, tmp_attrs->cpumask)) {
3695	ctx->pwq_tbl[node] = alloc_unbound_pwq(wq, tmp_attrs);
3696	if (!ctx->pwq_tbl[node])
3697	goto out_free;
3698	} else {
3699	ctx->dfl_pwq->refcnt++;
3700	ctx->pwq_tbl[node] = ctx->dfl_pwq;
3701	}
3702	}
3703
3704	/* save the user configured attrs and sanitize it. */
3705	copy_workqueue_attrs(new_attrs, attrs);
3706	cpumask_and(new_attrs->cpumask, new_attrs->cpumask, cpu_possible_mask);
3707	ctx->attrs = new_attrs;
3708
3709	ctx->wq = wq;
3710	free_workqueue_attrs(tmp_attrs);
3711	return ctx;
3712
3713	out_free:
3714	free_workqueue_attrs(tmp_attrs);
3715	free_workqueue_attrs(new_attrs);
3716	apply_wqattrs_cleanup(ctx);
3717	return NULL;
3718	}
3719
3720	/* set attrs and install prepared pwqs, @ctx points to old pwqs on return */
3721	static void apply_wqattrs_commit(struct apply_wqattrs_ctx *ctx)
3722	{
3723	int node;
3724
3725	/* all pwqs have been created successfully, let's install'em */
3726	mutex_lock(&ctx->wq->mutex);
3727
3728	copy_workqueue_attrs(ctx->wq->unbound_attrs, ctx->attrs);
3729
3730	/* save the previous pwq and install the new one */
3731	for_each_node(node)
3732	ctx->pwq_tbl[node] = numa_pwq_tbl_install(ctx->wq, node,
3733	ctx->pwq_tbl[node]);
3734
3735	/* @dfl_pwq might not have been used, ensure it's linked */
3736	link_pwq(ctx->dfl_pwq);
3737	swap(ctx->wq->dfl_pwq, ctx->dfl_pwq);
3738
3739	mutex_unlock(&ctx->wq->mutex);
3740	}
3741
3742	static void apply_wqattrs_lock(void)
3743	{
3744	/* CPUs should stay stable across pwq creations and installations */
3745	get_online_cpus();
3746	mutex_lock(&wq_pool_mutex);
3747	}
3748
3749	static void apply_wqattrs_unlock(void)
3750	{
3751	mutex_unlock(&wq_pool_mutex);
3752	put_online_cpus();
3753	}
3754
3755	static int apply_workqueue_attrs_locked(struct workqueue_struct *wq,
3756	const struct workqueue_attrs *attrs)
3757	{
3758	struct apply_wqattrs_ctx *ctx;
3759
3760	/* only unbound workqueues can change attributes */
3761	if (WARN_ON(!(wq->flags & WQ_UNBOUND)))
3762	return -EINVAL;
3763
3764	/* creating multiple pwqs breaks ordering guarantee */
3765	if (!list_empty(&wq->pwqs)) {
3766	if (WARN_ON(wq->flags & __WQ_ORDERED_EXPLICIT))
3767	return -EINVAL;
3768
3769	wq->flags &= ~__WQ_ORDERED;
3770	}
3771
3772	ctx = apply_wqattrs_prepare(wq, attrs);
3773	if (!ctx)
3774	return -ENOMEM;
3775
3776	/* the ctx has been prepared successfully, let's commit it */
3777	apply_wqattrs_commit(ctx);
3778	apply_wqattrs_cleanup(ctx);
3779
3780	return 0;
3781	}
3782
3783	/**
3784	* apply_workqueue_attrs - apply new workqueue_attrs to an unbound workqueue
3785	* @wq: the target workqueue
3786	* @attrs: the workqueue_attrs to apply, allocated with alloc_workqueue_attrs()
3787	*
3788	* Apply @attrs to an unbound workqueue @wq. Unless disabled, on NUMA
3789	* machines, this function maps a separate pwq to each NUMA node with
3790	* possibles CPUs in @attrs->cpumask so that work items are affine to the
3791	* NUMA node it was issued on. Older pwqs are released as in-flight work
3792	* items finish. Note that a work item which repeatedly requeues itself
3793	* back-to-back will stay on its current pwq.
3794	*
3795	* Performs GFP_KERNEL allocations.
3796	*
3797	* Return: 0 on success and -errno on failure.
3798	*/
3799	int apply_workqueue_attrs(struct workqueue_struct *wq,
3800	const struct workqueue_attrs *attrs)
3801	{
3802	int ret;
3803
3804	apply_wqattrs_lock();
3805	ret = apply_workqueue_attrs_locked(wq, attrs);
3806	apply_wqattrs_unlock();
3807
3808	return ret;
3809	}
3810	EXPORT_SYMBOL_GPL(apply_workqueue_attrs);
3811
3812	/**
3813	* wq_update_unbound_numa - update NUMA affinity of a wq for CPU hot[un]plug
3814	* @wq: the target workqueue
3815	* @cpu: the CPU coming up or going down
3816	* @online: whether @cpu is coming up or going down
3817	*
3818	* This function is to be called from %CPU_DOWN_PREPARE, %CPU_ONLINE and
3819	* %CPU_DOWN_FAILED. @cpu is being hot[un]plugged, update NUMA affinity of
3820	* @wq accordingly.
3821	*
3822	* If NUMA affinity can't be adjusted due to memory allocation failure, it
3823	* falls back to @wq->dfl_pwq which may not be optimal but is always
3824	* correct.
3825	*
3826	* Note that when the last allowed CPU of a NUMA node goes offline for a
3827	* workqueue with a cpumask spanning multiple nodes, the workers which were
3828	* already executing the work items for the workqueue will lose their CPU
3829	* affinity and may execute on any CPU. This is similar to how per-cpu
3830	* workqueues behave on CPU_DOWN. If a workqueue user wants strict
3831	* affinity, it's the user's responsibility to flush the work item from
3832	* CPU_DOWN_PREPARE.
3833	*/
3834	static void wq_update_unbound_numa(struct workqueue_struct *wq, int cpu,
3835	bool online)
3836	{
3837	int node = cpu_to_node(cpu);
3838	int cpu_off = online ? -1 : cpu;
3839	struct pool_workqueue old_pwq = NULL, pwq;
3840	struct workqueue_attrs *target_attrs;
3841	cpumask_t *cpumask;
3842
3843	lockdep_assert_held(&wq_pool_mutex);
3844
3845	if (!wq_numa_enabled \|\| !(wq->flags & WQ_UNBOUND) \|\|
3846	wq->unbound_attrs->no_numa)
3847	return;
3848
3849	/*
3850	* We don't wanna alloc/free wq_attrs for each wq for each CPU.
3851	* Let's use a preallocated one. The following buf is protected by
3852	* CPU hotplug exclusion.
3853	*/
3854	target_attrs = wq_update_unbound_numa_attrs_buf;
3855	cpumask = target_attrs->cpumask;
3856
3857	copy_workqueue_attrs(target_attrs, wq->unbound_attrs);
3858	pwq = unbound_pwq_by_node(wq, node);
3859
3860	/*
3861	* Let's determine what needs to be done. If the target cpumask is
3862	* different from the default pwq's, we need to compare it to @pwq's
3863	* and create a new one if they don't match. If the target cpumask
3864	* equals the default pwq's, the default pwq should be used.
3865	*/
3866	if (wq_calc_node_cpumask(wq->dfl_pwq->pool->attrs, node, cpu_off, cpumask)) {
3867	if (cpumask_equal(cpumask, pwq->pool->attrs->cpumask))
3868	return;
3869	} else {
3870	goto use_dfl_pwq;
3871	}
3872
3873	/* create a new pwq */
3874	pwq = alloc_unbound_pwq(wq, target_attrs);
3875	if (!pwq) {
3876	pr_warn("workqueue: allocation failed while updating NUMA affinity of \"%s\"\n",
3877	wq->name);
3878	goto use_dfl_pwq;
3879	}
3880
3881	/* Install the new pwq. */
3882	mutex_lock(&wq->mutex);
3883	old_pwq = numa_pwq_tbl_install(wq, node, pwq);
3884	goto out_unlock;
3885
3886	use_dfl_pwq:
3887	mutex_lock(&wq->mutex);
3888	spin_lock_irq(&wq->dfl_pwq->pool->lock);
3889	get_pwq(wq->dfl_pwq);
3890	spin_unlock_irq(&wq->dfl_pwq->pool->lock);
3891	old_pwq = numa_pwq_tbl_install(wq, node, wq->dfl_pwq);
3892	out_unlock:
3893	mutex_unlock(&wq->mutex);
3894	put_pwq_unlocked(old_pwq);
3895	}
3896
3897	static int alloc_and_link_pwqs(struct workqueue_struct *wq)
3898	{
3899	bool highpri = wq->flags & WQ_HIGHPRI;
3900	int cpu, ret;
3901
3902	if (!(wq->flags & WQ_UNBOUND)) {
3903	wq->cpu_pwqs = alloc_percpu(struct pool_workqueue);
3904	if (!wq->cpu_pwqs)
3905	return -ENOMEM;
3906
3907	for_each_possible_cpu(cpu) {
3908	struct pool_workqueue *pwq =
3909	per_cpu_ptr(wq->cpu_pwqs, cpu);
3910	struct worker_pool *cpu_pools =
3911	per_cpu(cpu_worker_pools, cpu);
3912
3913	init_pwq(pwq, wq, &cpu_pools[highpri]);
3914
3915	mutex_lock(&wq->mutex);
3916	link_pwq(pwq);
3917	mutex_unlock(&wq->mutex);
3918	}
3919	return 0;
3920	} else if (wq->flags & __WQ_ORDERED) {
3921	ret = apply_workqueue_attrs(wq, ordered_wq_attrs[highpri]);
3922	/* there should only be single pwq for ordering guarantee */
3923	WARN(!ret && (wq->pwqs.next != &wq->dfl_pwq->pwqs_node \|\|
3924	wq->pwqs.prev != &wq->dfl_pwq->pwqs_node),
3925	"ordering guarantee broken for workqueue %s\n", wq->name);
3926	return ret;
3927	} else {
3928	return apply_workqueue_attrs(wq, unbound_std_wq_attrs[highpri]);
3929	}
3930	}
3931
3932	static int wq_clamp_max_active(int max_active, unsigned int flags,
3933	const char *name)
3934	{
3935	int lim = flags & WQ_UNBOUND ? WQ_UNBOUND_MAX_ACTIVE : WQ_MAX_ACTIVE;
3936
3937	if (max_active < 1 \|\| max_active > lim)
3938	pr_warn("workqueue: max_active %d requested for %s is out of range, clamping between %d and %d\n",
3939	max_active, name, 1, lim);
3940
3941	return clamp_val(max_active, 1, lim);
3942	}
3943
3944	/*
3945	* Workqueues which may be used during memory reclaim should have a rescuer
3946	* to guarantee forward progress.
3947	*/
3948	static int init_rescuer(struct workqueue_struct *wq)
3949	{
3950	struct worker *rescuer;
3951	int ret;
3952
3953	if (!(wq->flags & WQ_MEM_RECLAIM))
3954	return 0;
3955
3956	rescuer = alloc_worker(NUMA_NO_NODE);
3957	if (!rescuer)
3958	return -ENOMEM;
3959
3960	rescuer->rescue_wq = wq;
3961	rescuer->task = kthread_create(rescuer_thread, rescuer, "%s", wq->name);
3962	ret = PTR_ERR_OR_ZERO(rescuer->task);
3963	if (ret) {
3964	kfree(rescuer);
3965	return ret;
3966	}
3967
3968	wq->rescuer = rescuer;
3969	kthread_bind_mask(rescuer->task, cpu_possible_mask);
3970	wake_up_process(rescuer->task);
3971
3972	return 0;
3973	}
3974
3975	struct workqueue_struct __alloc_workqueue_key(const char fmt,
3976	unsigned int flags,
3977	int max_active,
3978	struct lock_class_key *key,
3979	const char *lock_name, ...)
3980	{
3981	size_t tbl_size = 0;
3982	va_list args;
3983	struct workqueue_struct *wq;
3984	struct pool_workqueue *pwq;
3985
3986	/*
3987	* Unbound && max_active == 1 used to imply ordered, which is no
3988	* longer the case on NUMA machines due to per-node pools. While
3989	* alloc_ordered_workqueue() is the right way to create an ordered
3990	* workqueue, keep the previous behavior to avoid subtle breakages
3991	* on NUMA.
3992	*/
3993	if ((flags & WQ_UNBOUND) && max_active == 1)
3994	flags \|= __WQ_ORDERED;
3995
3996	/* see the comment above the definition of WQ_POWER_EFFICIENT */
3997	if ((flags & WQ_POWER_EFFICIENT) && wq_power_efficient)
3998	flags \|= WQ_UNBOUND;
3999
4000	/* allocate wq and format name */
4001	if (flags & WQ_UNBOUND)
4002	tbl_size = nr_node_ids * sizeof(wq->numa_pwq_tbl[0]);
4003
4004	wq = kzalloc(sizeof(*wq) + tbl_size, GFP_KERNEL);
4005	if (!wq)
4006	return NULL;
4007
4008	if (flags & WQ_UNBOUND) {
4009	wq->unbound_attrs = alloc_workqueue_attrs(GFP_KERNEL);
4010	if (!wq->unbound_attrs)
4011	goto err_free_wq;
4012	}
4013
4014	va_start(args, lock_name);
4015	vsnprintf(wq->name, sizeof(wq->name), fmt, args);
4016	va_end(args);
4017
4018	max_active = max_active ?: WQ_DFL_ACTIVE;
4019	max_active = wq_clamp_max_active(max_active, flags, wq->name);
4020
4021	/* init wq */
4022	wq->flags = flags;
4023	wq->saved_max_active = max_active;
4024	mutex_init(&wq->mutex);
4025	atomic_set(&wq->nr_pwqs_to_flush, 0);
4026	INIT_LIST_HEAD(&wq->pwqs);
4027	INIT_LIST_HEAD(&wq->flusher_queue);
4028	INIT_LIST_HEAD(&wq->flusher_overflow);
4029	INIT_LIST_HEAD(&wq->maydays);
4030
4031	lockdep_init_map(&wq->lockdep_map, lock_name, key, 0);
4032	INIT_LIST_HEAD(&wq->list);
4033
4034	if (alloc_and_link_pwqs(wq) < 0)
4035	goto err_free_wq;
4036
4037	if (wq_online && init_rescuer(wq) < 0)
4038	goto err_destroy;
4039
4040	if ((wq->flags & WQ_SYSFS) && workqueue_sysfs_register(wq))
4041	goto err_destroy;
4042
4043	/*
4044	* wq_pool_mutex protects global freeze state and workqueues list.
4045	* Grab it, adjust max_active and add the new @wq to workqueues
4046	* list.
4047	*/
4048	mutex_lock(&wq_pool_mutex);
4049
4050	mutex_lock(&wq->mutex);
4051	for_each_pwq(pwq, wq)
4052	pwq_adjust_max_active(pwq);
4053	mutex_unlock(&wq->mutex);
4054
4055	list_add_tail_rcu(&wq->list, &workqueues);
4056
4057	mutex_unlock(&wq_pool_mutex);
4058
4059	return wq;
4060
4061	err_free_wq:
4062	free_workqueue_attrs(wq->unbound_attrs);
4063	kfree(wq);
4064	return NULL;
4065	err_destroy:
4066	destroy_workqueue(wq);
4067	return NULL;
4068	}
4069	EXPORT_SYMBOL_GPL(__alloc_workqueue_key);
4070
4071	/**
4072	* destroy_workqueue - safely terminate a workqueue
4073	* @wq: target workqueue
4074	*
4075	* Safely destroy a workqueue. All work currently pending will be done first.
4076	*/
4077	void destroy_workqueue(struct workqueue_struct *wq)
4078	{
4079	struct pool_workqueue *pwq;
4080	int node;
4081
4082	/* drain it before proceeding with destruction */
4083	drain_workqueue(wq);
4084
4085	/* sanity checks */
4086	mutex_lock(&wq->mutex);
4087	for_each_pwq(pwq, wq) {
4088	int i;
4089
4090	for (i = 0; i < WORK_NR_COLORS; i++) {
4091	if (WARN_ON(pwq->nr_in_flight[i])) {
4092	mutex_unlock(&wq->mutex);
4093	show_workqueue_state();
4094	return;
4095	}
4096	}
4097
4098	if (WARN_ON((pwq != wq->dfl_pwq) && (pwq->refcnt > 1)) \|\|
4099	WARN_ON(pwq->nr_active) \|\|
4100	WARN_ON(!list_empty(&pwq->delayed_works))) {
4101	mutex_unlock(&wq->mutex);
4102	show_workqueue_state();
4103	return;
4104	}
4105	}
4106	mutex_unlock(&wq->mutex);
4107
4108	/*
4109	* wq list is used to freeze wq, remove from list after
4110	* flushing is complete in case freeze races us.
4111	*/
4112	mutex_lock(&wq_pool_mutex);
4113	list_del_rcu(&wq->list);
4114	mutex_unlock(&wq_pool_mutex);
4115
4116	workqueue_sysfs_unregister(wq);
4117
4118	if (wq->rescuer)
4119	kthread_stop(wq->rescuer->task);
4120
4121	if (!(wq->flags & WQ_UNBOUND)) {
4122	/*
4123	* The base ref is never dropped on per-cpu pwqs. Directly
4124	* schedule RCU free.
4125	*/
4126	call_rcu_sched(&wq->rcu, rcu_free_wq);
4127	} else {
4128	/*
4129	* We're the sole accessor of @wq at this point. Directly
4130	* access numa_pwq_tbl[] and dfl_pwq to put the base refs.
4131	* @wq will be freed when the last pwq is released.
4132	*/
4133	for_each_node(node) {
4134	pwq = rcu_access_pointer(wq->numa_pwq_tbl[node]);
4135	RCU_INIT_POINTER(wq->numa_pwq_tbl[node], NULL);
4136	put_pwq_unlocked(pwq);
4137	}
4138
4139	/*
4140	* Put dfl_pwq. @wq may be freed any time after dfl_pwq is
4141	* put. Don't access it afterwards.
4142	*/
4143	pwq = wq->dfl_pwq;
4144	wq->dfl_pwq = NULL;
4145	put_pwq_unlocked(pwq);
4146	}
4147	}
4148	EXPORT_SYMBOL_GPL(destroy_workqueue);
4149
4150	/**
4151	* workqueue_set_max_active - adjust max_active of a workqueue
4152	* @wq: target workqueue
4153	* @max_active: new max_active value.
4154	*
4155	* Set max_active of @wq to @max_active.
4156	*
4157	* CONTEXT:
4158	* Don't call from IRQ context.
4159	*/
4160	void workqueue_set_max_active(struct workqueue_struct *wq, int max_active)
4161	{
4162	struct pool_workqueue *pwq;
4163
4164	/* disallow meddling with max_active for ordered workqueues */
4165	if (WARN_ON(wq->flags & __WQ_ORDERED_EXPLICIT))
4166	return;
4167
4168	max_active = wq_clamp_max_active(max_active, wq->flags, wq->name);
4169
4170	mutex_lock(&wq->mutex);
4171
4172	wq->flags &= ~__WQ_ORDERED;
4173	wq->saved_max_active = max_active;
4174
4175	for_each_pwq(pwq, wq)
4176	pwq_adjust_max_active(pwq);
4177
4178	mutex_unlock(&wq->mutex);
4179	}
4180	EXPORT_SYMBOL_GPL(workqueue_set_max_active);
4181
4182	/**
4183	* current_work - retrieve %current task's work struct
4184	*
4185	* Determine if %current task is a workqueue worker and what it's working on.
4186	* Useful to find out the context that the %current task is running in.
4187	*
4188	* Return: work struct if %current task is a workqueue worker, %NULL otherwise.
4189	*/
4190	struct work_struct *current_work(void)
4191	{
4192	struct worker *worker = current_wq_worker();
4193
4194	return worker ? worker->current_work : NULL;
4195	}
4196	EXPORT_SYMBOL(current_work);
4197
4198	/**
4199	* current_is_workqueue_rescuer - is %current workqueue rescuer?
4200	*
4201	* Determine whether %current is a workqueue rescuer. Can be used from
4202	* work functions to determine whether it's being run off the rescuer task.
4203	*
4204	* Return: %true if %current is a workqueue rescuer. %false otherwise.
4205	*/
4206	bool current_is_workqueue_rescuer(void)
4207	{
4208	struct worker *worker = current_wq_worker();
4209
4210	return worker && worker->rescue_wq;
4211	}
4212
4213	/**
4214	* workqueue_congested - test whether a workqueue is congested
4215	* @cpu: CPU in question
4216	* @wq: target workqueue
4217	*
4218	* Test whether @wq's cpu workqueue for @cpu is congested. There is
4219	* no synchronization around this function and the test result is
4220	* unreliable and only useful as advisory hints or for debugging.
4221	*
4222	* If @cpu is WORK_CPU_UNBOUND, the test is performed on the local CPU.
4223	* Note that both per-cpu and unbound workqueues may be associated with
4224	* multiple pool_workqueues which have separate congested states. A
4225	* workqueue being congested on one CPU doesn't mean the workqueue is also
4226	* contested on other CPUs / NUMA nodes.
4227	*
4228	* Return:
4229	* %true if congested, %false otherwise.
4230	*/
4231	bool workqueue_congested(int cpu, struct workqueue_struct *wq)
4232	{
4233	struct pool_workqueue *pwq;
4234	bool ret;
4235
4236	rcu_read_lock_sched();
4237
4238	if (cpu == WORK_CPU_UNBOUND)
4239	cpu = smp_processor_id();
4240
4241	if (!(wq->flags & WQ_UNBOUND))
4242	pwq = per_cpu_ptr(wq->cpu_pwqs, cpu);
4243	else
4244	pwq = unbound_pwq_by_node(wq, cpu_to_node(cpu));
4245
4246	ret = !list_empty(&pwq->delayed_works);
4247	rcu_read_unlock_sched();
4248
4249	return ret;
4250	}
4251	EXPORT_SYMBOL_GPL(workqueue_congested);
4252
4253	/**
4254	* work_busy - test whether a work is currently pending or running
4255	* @work: the work to be tested
4256	*
4257	* Test whether @work is currently pending or running. There is no
4258	* synchronization around this function and the test result is
4259	* unreliable and only useful as advisory hints or for debugging.
4260	*
4261	* Return:
4262	* OR'd bitmask of WORK_BUSY_* bits.
4263	*/
4264	unsigned int work_busy(struct work_struct *work)
4265	{
4266	struct worker_pool *pool;
4267	unsigned long flags;
4268	unsigned int ret = 0;
4269
4270	if (work_pending(work))
4271	ret \|= WORK_BUSY_PENDING;
4272
4273	local_irq_save(flags);
4274	pool = get_work_pool(work);
4275	if (pool) {
4276	spin_lock(&pool->lock);
4277	if (find_worker_executing_work(pool, work))
4278	ret \|= WORK_BUSY_RUNNING;
4279	spin_unlock(&pool->lock);
4280	}
4281	local_irq_restore(flags);
4282
4283	return ret;
4284	}
4285	EXPORT_SYMBOL_GPL(work_busy);
4286
4287	/**
4288	* set_worker_desc - set description for the current work item
4289	* @fmt: printf-style format string
4290	* @...: arguments for the format string
4291	*
4292	* This function can be called by a running work function to describe what
4293	* the work item is about. If the worker task gets dumped, this
4294	* information will be printed out together to help debugging. The
4295	* description can be at most WORKER_DESC_LEN including the trailing '\0'.
4296	*/
4297	void set_worker_desc(const char *fmt, ...)
4298	{
4299	struct worker *worker = current_wq_worker();
4300	va_list args;
4301
4302	if (worker) {
4303	va_start(args, fmt);
4304	vsnprintf(worker->desc, sizeof(worker->desc), fmt, args);
4305	va_end(args);
4306	worker->desc_valid = true;
4307	}
4308	}
4309
4310	/**
4311	* print_worker_info - print out worker information and description
4312	* @log_lvl: the log level to use when printing
4313	* @task: target task
4314	*
4315	* If @task is a worker and currently executing a work item, print out the
4316	* name of the workqueue being serviced and worker description set with
4317	* set_worker_desc() by the currently executing work item.
4318	*
4319	* This function can be safely called on any task as long as the
4320	* task_struct itself is accessible. While safe, this function isn't
4321	* synchronized and may print out mixups or garbages of limited length.
4322	*/
4323	void print_worker_info(const char log_lvl, struct task_struct task)
4324	{
4325	work_func_t *fn = NULL;
4326	char name[WQ_NAME_LEN] = { };
4327	char desc[WORKER_DESC_LEN] = { };
4328	struct pool_workqueue *pwq = NULL;
4329	struct workqueue_struct *wq = NULL;
4330	bool desc_valid = false;
4331	struct worker *worker;
4332
4333	if (!(task->flags & PF_WQ_WORKER))
4334	return;
4335
4336	/*
4337	* This function is called without any synchronization and @task
4338	* could be in any state. Be careful with dereferences.
4339	*/
4340	worker = kthread_probe_data(task);
4341
4342	/*
4343	* Carefully copy the associated workqueue's workfn and name. Keep
4344	* the original last '\0' in case the original contains garbage.
4345	*/
4346	probe_kernel_read(&fn, &worker->current_func, sizeof(fn));
4347	probe_kernel_read(&pwq, &worker->current_pwq, sizeof(pwq));
4348	probe_kernel_read(&wq, &pwq->wq, sizeof(wq));
4349	probe_kernel_read(name, wq->name, sizeof(name) - 1);
4350
4351	/* copy worker description */
4352	probe_kernel_read(&desc_valid, &worker->desc_valid, sizeof(desc_valid));
4353	if (desc_valid)
4354	probe_kernel_read(desc, worker->desc, sizeof(desc) - 1);
4355
4356	if (fn \|\| name[0] \|\| desc[0]) {
4357	printk("%sWorkqueue: %s %pf", log_lvl, name, fn);
4358	if (desc[0])
4359	pr_cont(" (%s)", desc);
4360	pr_cont("\n");
4361	}
4362	}
4363
4364	static void pr_cont_pool_info(struct worker_pool *pool)
4365	{
4366	pr_cont(" cpus=%*pbl", nr_cpumask_bits, pool->attrs->cpumask);
4367	if (pool->node != NUMA_NO_NODE)
4368	pr_cont(" node=%d", pool->node);
4369	pr_cont(" flags=0x%x nice=%d", pool->flags, pool->attrs->nice);
4370	}
4371
4372	static void pr_cont_work(bool comma, struct work_struct *work)
4373	{
4374	if (work->func == wq_barrier_func) {
4375	struct wq_barrier *barr;
4376
4377	barr = container_of(work, struct wq_barrier, work);
4378
4379	pr_cont("%s BAR(%d)", comma ? "," : "",
4380	task_pid_nr(barr->task));
4381	} else {
4382	pr_cont("%s %pf", comma ? "," : "", work->func);
4383	}
4384	}
4385
4386	static void show_pwq(struct pool_workqueue *pwq)
4387	{
4388	struct worker_pool *pool = pwq->pool;
4389	struct work_struct *work;
4390	struct worker *worker;
4391	bool has_in_flight = false, has_pending = false;
4392	int bkt;
4393
4394	pr_info(" pwq %d:", pool->id);
4395	pr_cont_pool_info(pool);
4396
4397	pr_cont(" active=%d/%d%s\n", pwq->nr_active, pwq->max_active,
4398	!list_empty(&pwq->mayday_node) ? " MAYDAY" : "");
4399
4400	hash_for_each(pool->busy_hash, bkt, worker, hentry) {
4401	if (worker->current_pwq == pwq) {
4402	has_in_flight = true;
4403	break;
4404	}
4405	}
4406	if (has_in_flight) {
4407	bool comma = false;
4408
4409	pr_info(" in-flight:");
4410	hash_for_each(pool->busy_hash, bkt, worker, hentry) {
4411	if (worker->current_pwq != pwq)
4412	continue;
4413
4414	pr_cont("%s %d%s:%pf", comma ? "," : "",
4415	task_pid_nr(worker->task),
4416	worker == pwq->wq->rescuer ? "(RESCUER)" : "",
4417	worker->current_func);
4418	list_for_each_entry(work, &worker->scheduled, entry)
4419	pr_cont_work(false, work);
4420	comma = true;
4421	}
4422	pr_cont("\n");
4423	}
4424
4425	list_for_each_entry(work, &pool->worklist, entry) {
4426	if (get_work_pwq(work) == pwq) {
4427	has_pending = true;
4428	break;
4429	}
4430	}
4431	if (has_pending) {
4432	bool comma = false;
4433
4434	pr_info(" pending:");
4435	list_for_each_entry(work, &pool->worklist, entry) {
4436	if (get_work_pwq(work) != pwq)
4437	continue;
4438
4439	pr_cont_work(comma, work);
4440	comma = !(*work_data_bits(work) & WORK_STRUCT_LINKED);
4441	}
4442	pr_cont("\n");
4443	}
4444
4445	if (!list_empty(&pwq->delayed_works)) {
4446	bool comma = false;
4447
4448	pr_info(" delayed:");
4449	list_for_each_entry(work, &pwq->delayed_works, entry) {
4450	pr_cont_work(comma, work);
4451	comma = !(*work_data_bits(work) & WORK_STRUCT_LINKED);
4452	}
4453	pr_cont("\n");
4454	}
4455	}
4456
4457	/**
4458	* show_workqueue_state - dump workqueue state
4459	*
4460	* Called from a sysrq handler or try_to_freeze_tasks() and prints out
4461	* all busy workqueues and pools.
4462	*/
4463	void show_workqueue_state(void)
4464	{
4465	struct workqueue_struct *wq;
4466	struct worker_pool *pool;
4467	unsigned long flags;
4468	int pi;
4469
4470	rcu_read_lock_sched();
4471
4472	pr_info("Showing busy workqueues and worker pools:\n");
4473
4474	list_for_each_entry_rcu(wq, &workqueues, list) {
4475	struct pool_workqueue *pwq;
4476	bool idle = true;
4477
4478	for_each_pwq(pwq, wq) {
4479	if (pwq->nr_active \|\| !list_empty(&pwq->delayed_works)) {
4480	idle = false;
4481	break;
4482	}
4483	}
4484	if (idle)
4485	continue;
4486
4487	pr_info("workqueue %s: flags=0x%x\n", wq->name, wq->flags);
4488
4489	for_each_pwq(pwq, wq) {
4490	spin_lock_irqsave(&pwq->pool->lock, flags);
4491	if (pwq->nr_active \|\| !list_empty(&pwq->delayed_works))
4492	show_pwq(pwq);
4493	spin_unlock_irqrestore(&pwq->pool->lock, flags);
4494	/*
4495	* We could be printing a lot from atomic context, e.g.
4496	* sysrq-t -> show_workqueue_state(). Avoid triggering
4497	* hard lockup.
4498	*/
4499	touch_nmi_watchdog();
4500	}
4501	}
4502
4503	for_each_pool(pool, pi) {
4504	struct worker *worker;
4505	bool first = true;
4506
4507	spin_lock_irqsave(&pool->lock, flags);
4508	if (pool->nr_workers == pool->nr_idle)
4509	goto next_pool;
4510
4511	pr_info("pool %d:", pool->id);
4512	pr_cont_pool_info(pool);
4513	pr_cont(" hung=%us workers=%d",
4514	jiffies_to_msecs(jiffies - pool->watchdog_ts) / 1000,
4515	pool->nr_workers);
4516	if (pool->manager)
4517	pr_cont(" manager: %d",
4518	task_pid_nr(pool->manager->task));
4519	list_for_each_entry(worker, &pool->idle_list, entry) {
4520	pr_cont(" %s%d", first ? "idle: " : "",
4521	task_pid_nr(worker->task));
4522	first = false;
4523	}
4524	pr_cont("\n");
4525	next_pool:
4526	spin_unlock_irqrestore(&pool->lock, flags);
4527	/*
4528	* We could be printing a lot from atomic context, e.g.
4529	* sysrq-t -> show_workqueue_state(). Avoid triggering
4530	* hard lockup.
4531	*/
4532	touch_nmi_watchdog();
4533	}
4534
4535	rcu_read_unlock_sched();
4536	}
4537
4538	/*
4539	* CPU hotplug.
4540	*
4541	* There are two challenges in supporting CPU hotplug. Firstly, there
4542	* are a lot of assumptions on strong associations among work, pwq and
4543	* pool which make migrating pending and scheduled works very
4544	* difficult to implement without impacting hot paths. Secondly,
4545	* worker pools serve mix of short, long and very long running works making
4546	* blocked draining impractical.
4547	*
4548	* This is solved by allowing the pools to be disassociated from the CPU
4549	* running as an unbound one and allowing it to be reattached later if the
4550	* cpu comes back online.
4551	*/
4552
4553	static void unbind_workers(int cpu)
4554	{
4555	struct worker_pool *pool;
4556	struct worker *worker;
4557
4558	for_each_cpu_worker_pool(pool, cpu) {
4559	mutex_lock(&pool->attach_mutex);
4560	spin_lock_irq(&pool->lock);
4561
4562	/*
4563	* We've blocked all attach/detach operations. Make all workers
4564	* unbound and set DISASSOCIATED. Before this, all workers
4565	* except for the ones which are still executing works from
4566	* before the last CPU down must be on the cpu. After
4567	* this, they may become diasporas.
4568	*/
4569	for_each_pool_worker(worker, pool)
4570	worker->flags \|= WORKER_UNBOUND;
4571
4572	pool->flags \|= POOL_DISASSOCIATED;
4573
4574	spin_unlock_irq(&pool->lock);
4575	mutex_unlock(&pool->attach_mutex);
4576
4577	/*
4578	* Call schedule() so that we cross rq->lock and thus can
4579	* guarantee sched callbacks see the %WORKER_UNBOUND flag.
4580	* This is necessary as scheduler callbacks may be invoked
4581	* from other cpus.
4582	*/
4583	schedule();
4584
4585	/*
4586	* Sched callbacks are disabled now. Zap nr_running.
4587	* After this, nr_running stays zero and need_more_worker()
4588	* and keep_working() are always true as long as the
4589	* worklist is not empty. This pool now behaves as an
4590	* unbound (in terms of concurrency management) pool which
4591	* are served by workers tied to the pool.
4592	*/
4593	atomic_set(&pool->nr_running, 0);
4594
4595	/*
4596	* With concurrency management just turned off, a busy
4597	* worker blocking could lead to lengthy stalls. Kick off
4598	* unbound chain execution of currently pending work items.
4599	*/
4600	spin_lock_irq(&pool->lock);
4601	wake_up_worker(pool);
4602	spin_unlock_irq(&pool->lock);
4603	}
4604	}
4605
4606	/**
4607	* rebind_workers - rebind all workers of a pool to the associated CPU
4608	* @pool: pool of interest
4609	*
4610	* @pool->cpu is coming online. Rebind all workers to the CPU.
4611	*/
4612	static void rebind_workers(struct worker_pool *pool)
4613	{
4614	struct worker *worker;
4615
4616	lockdep_assert_held(&pool->attach_mutex);
4617
4618	/*
4619	* Restore CPU affinity of all workers. As all idle workers should
4620	* be on the run-queue of the associated CPU before any local
4621	* wake-ups for concurrency management happen, restore CPU affinity
4622	* of all workers first and then clear UNBOUND. As we're called
4623	* from CPU_ONLINE, the following shouldn't fail.
4624	*/
4625	for_each_pool_worker(worker, pool)
4626	WARN_ON_ONCE(set_cpus_allowed_ptr(worker->task,
4627	pool->attrs->cpumask) < 0);
4628
4629	spin_lock_irq(&pool->lock);
4630
4631	pool->flags &= ~POOL_DISASSOCIATED;
4632
4633	for_each_pool_worker(worker, pool) {
4634	unsigned int worker_flags = worker->flags;
4635
4636	/*
4637	* A bound idle worker should actually be on the runqueue
4638	* of the associated CPU for local wake-ups targeting it to
4639	* work. Kick all idle workers so that they migrate to the
4640	* associated CPU. Doing this in the same loop as
4641	* replacing UNBOUND with REBOUND is safe as no worker will
4642	* be bound before @pool->lock is released.
4643	*/
4644	if (worker_flags & WORKER_IDLE)
4645	wake_up_process(worker->task);
4646
4647	/*
4648	* We want to clear UNBOUND but can't directly call
4649	* worker_clr_flags() or adjust nr_running. Atomically
4650	* replace UNBOUND with another NOT_RUNNING flag REBOUND.
4651	* @worker will clear REBOUND using worker_clr_flags() when
4652	* it initiates the next execution cycle thus restoring
4653	* concurrency management. Note that when or whether
4654	* @worker clears REBOUND doesn't affect correctness.
4655	*
4656	* WRITE_ONCE() is necessary because @worker->flags may be
4657	* tested without holding any lock in
4658	* wq_worker_waking_up(). Without it, NOT_RUNNING test may
4659	* fail incorrectly leading to premature concurrency
4660	* management operations.
4661	*/
4662	WARN_ON_ONCE(!(worker_flags & WORKER_UNBOUND));
4663	worker_flags \|= WORKER_REBOUND;
4664	worker_flags &= ~WORKER_UNBOUND;
4665	WRITE_ONCE(worker->flags, worker_flags);
4666	}
4667
4668	spin_unlock_irq(&pool->lock);
4669	}
4670
4671	/**
4672	* restore_unbound_workers_cpumask - restore cpumask of unbound workers
4673	* @pool: unbound pool of interest
4674	* @cpu: the CPU which is coming up
4675	*
4676	* An unbound pool may end up with a cpumask which doesn't have any online
4677	* CPUs. When a worker of such pool get scheduled, the scheduler resets
4678	* its cpus_allowed. If @cpu is in @pool's cpumask which didn't have any
4679	* online CPU before, cpus_allowed of all its workers should be restored.
4680	*/
4681	static void restore_unbound_workers_cpumask(struct worker_pool *pool, int cpu)
4682	{
4683	static cpumask_t cpumask;
4684	struct worker *worker;
4685
4686	lockdep_assert_held(&pool->attach_mutex);
4687
4688	/* is @cpu allowed for @pool? */
4689	if (!cpumask_test_cpu(cpu, pool->attrs->cpumask))
4690	return;
4691
4692	cpumask_and(&cpumask, pool->attrs->cpumask, cpu_online_mask);
4693
4694	/* as we're called from CPU_ONLINE, the following shouldn't fail */
4695	for_each_pool_worker(worker, pool)
4696	WARN_ON_ONCE(set_cpus_allowed_ptr(worker->task, &cpumask) < 0);
4697	}
4698
4699	int workqueue_prepare_cpu(unsigned int cpu)
4700	{
4701	struct worker_pool *pool;
4702
4703	for_each_cpu_worker_pool(pool, cpu) {
4704	if (pool->nr_workers)
4705	continue;
4706	if (!create_worker(pool))
4707	return -ENOMEM;
4708	}
4709	return 0;
4710	}
4711
4712	int workqueue_online_cpu(unsigned int cpu)
4713	{
4714	struct worker_pool *pool;
4715	struct workqueue_struct *wq;
4716	int pi;
4717
4718	mutex_lock(&wq_pool_mutex);
4719
4720	for_each_pool(pool, pi) {
4721	mutex_lock(&pool->attach_mutex);
4722
4723	if (pool->cpu == cpu)
4724	rebind_workers(pool);
4725	else if (pool->cpu < 0)
4726	restore_unbound_workers_cpumask(pool, cpu);
4727
4728	mutex_unlock(&pool->attach_mutex);
4729	}
4730
4731	/* update NUMA affinity of unbound workqueues */
4732	list_for_each_entry(wq, &workqueues, list)
4733	wq_update_unbound_numa(wq, cpu, true);
4734
4735	mutex_unlock(&wq_pool_mutex);
4736	return 0;
4737	}
4738
4739	int workqueue_offline_cpu(unsigned int cpu)
4740	{
4741	struct workqueue_struct *wq;
4742
4743	/* unbinding per-cpu workers should happen on the local CPU */
4744	if (WARN_ON(cpu != smp_processor_id()))
4745	return -1;
4746
4747	unbind_workers(cpu);
4748
4749	/* update NUMA affinity of unbound workqueues */
4750	mutex_lock(&wq_pool_mutex);
4751	list_for_each_entry(wq, &workqueues, list)
4752	wq_update_unbound_numa(wq, cpu, false);
4753	mutex_unlock(&wq_pool_mutex);
4754
4755	return 0;
4756	}
4757
4758	#ifdef CONFIG_SMP
4759
4760	struct work_for_cpu {
4761	struct work_struct work;
4762	long (fn)(void );
4763	void *arg;
4764	long ret;
4765	};
4766
4767	static void work_for_cpu_fn(struct work_struct *work)
4768	{
4769	struct work_for_cpu *wfc = container_of(work, struct work_for_cpu, work);
4770
4771	wfc->ret = wfc->fn(wfc->arg);
4772	}
4773
4774	/**
4775	* work_on_cpu - run a function in thread context on a particular cpu
4776	* @cpu: the cpu to run on
4777	* @fn: the function to run
4778	* @arg: the function arg
4779	*
4780	* It is up to the caller to ensure that the cpu doesn't go offline.
4781	* The caller must not hold any locks which would prevent @fn from completing.
4782	*
4783	* Return: The value @fn returns.
4784	*/
4785	long work_on_cpu(int cpu, long (fn)(void ), void *arg)
4786	{
4787	struct work_for_cpu wfc = { .fn = fn, .arg = arg };
4788
4789	INIT_WORK_ONSTACK(&wfc.work, work_for_cpu_fn);
4790	schedule_work_on(cpu, &wfc.work);
4791	flush_work(&wfc.work);
4792	destroy_work_on_stack(&wfc.work);
4793	return wfc.ret;
4794	}
4795	EXPORT_SYMBOL_GPL(work_on_cpu);
4796
4797	/**
4798	* work_on_cpu_safe - run a function in thread context on a particular cpu
4799	* @cpu: the cpu to run on
4800	* @fn: the function to run
4801	* @arg: the function argument
4802	*
4803	* Disables CPU hotplug and calls work_on_cpu(). The caller must not hold
4804	* any locks which would prevent @fn from completing.
4805	*
4806	* Return: The value @fn returns.
4807	*/
4808	long work_on_cpu_safe(int cpu, long (fn)(void ), void *arg)
4809	{
4810	long ret = -ENODEV;
4811
4812	get_online_cpus();
4813	if (cpu_online(cpu))
4814	ret = work_on_cpu(cpu, fn, arg);
4815	put_online_cpus();
4816	return ret;
4817	}
4818	EXPORT_SYMBOL_GPL(work_on_cpu_safe);
4819	#endif /* CONFIG_SMP */
4820
4821	#ifdef CONFIG_FREEZER
4822
4823	/**
4824	* freeze_workqueues_begin - begin freezing workqueues
4825	*
4826	* Start freezing workqueues. After this function returns, all freezable
4827	* workqueues will queue new works to their delayed_works list instead of
4828	* pool->worklist.
4829	*
4830	* CONTEXT:
4831	* Grabs and releases wq_pool_mutex, wq->mutex and pool->lock's.
4832	*/
4833	void freeze_workqueues_begin(void)
4834	{
4835	struct workqueue_struct *wq;
4836	struct pool_workqueue *pwq;
4837
4838	mutex_lock(&wq_pool_mutex);
4839
4840	WARN_ON_ONCE(workqueue_freezing);
4841	workqueue_freezing = true;
4842
4843	list_for_each_entry(wq, &workqueues, list) {
4844	mutex_lock(&wq->mutex);
4845	for_each_pwq(pwq, wq)
4846	pwq_adjust_max_active(pwq);
4847	mutex_unlock(&wq->mutex);
4848	}
4849
4850	mutex_unlock(&wq_pool_mutex);
4851	}
4852
4853	/**
4854	* freeze_workqueues_busy - are freezable workqueues still busy?
4855	*
4856	* Check whether freezing is complete. This function must be called
4857	* between freeze_workqueues_begin() and thaw_workqueues().
4858	*
4859	* CONTEXT:
4860	* Grabs and releases wq_pool_mutex.
4861	*
4862	* Return:
4863	* %true if some freezable workqueues are still busy. %false if freezing
4864	* is complete.
4865	*/
4866	bool freeze_workqueues_busy(void)
4867	{
4868	bool busy = false;
4869	struct workqueue_struct *wq;
4870	struct pool_workqueue *pwq;
4871
4872	mutex_lock(&wq_pool_mutex);
4873
4874	WARN_ON_ONCE(!workqueue_freezing);
4875
4876	list_for_each_entry(wq, &workqueues, list) {
4877	if (!(wq->flags & WQ_FREEZABLE))
4878	continue;
4879	/*
4880	* nr_active is monotonically decreasing. It's safe
4881	* to peek without lock.
4882	*/
4883	rcu_read_lock_sched();
4884	for_each_pwq(pwq, wq) {
4885	WARN_ON_ONCE(pwq->nr_active < 0);
4886	if (pwq->nr_active) {
4887	busy = true;
4888	rcu_read_unlock_sched();
4889	goto out_unlock;
4890	}
4891	}
4892	rcu_read_unlock_sched();
4893	}
4894	out_unlock:
4895	mutex_unlock(&wq_pool_mutex);
4896	return busy;
4897	}
4898
4899	/**
4900	* thaw_workqueues - thaw workqueues
4901	*
4902	* Thaw workqueues. Normal queueing is restored and all collected
4903	* frozen works are transferred to their respective pool worklists.
4904	*
4905	* CONTEXT:
4906	* Grabs and releases wq_pool_mutex, wq->mutex and pool->lock's.
4907	*/
4908	void thaw_workqueues(void)
4909	{
4910	struct workqueue_struct *wq;
4911	struct pool_workqueue *pwq;
4912
4913	mutex_lock(&wq_pool_mutex);
4914
4915	if (!workqueue_freezing)
4916	goto out_unlock;
4917
4918	workqueue_freezing = false;
4919
4920	/* restore max_active and repopulate worklist */
4921	list_for_each_entry(wq, &workqueues, list) {
4922	mutex_lock(&wq->mutex);
4923	for_each_pwq(pwq, wq)
4924	pwq_adjust_max_active(pwq);
4925	mutex_unlock(&wq->mutex);
4926	}
4927
4928	out_unlock:
4929	mutex_unlock(&wq_pool_mutex);
4930	}
4931	#endif /* CONFIG_FREEZER */
4932
4933	static int workqueue_apply_unbound_cpumask(void)
4934	{
4935	LIST_HEAD(ctxs);
4936	int ret = 0;
4937	struct workqueue_struct *wq;
4938	struct apply_wqattrs_ctx ctx, n;
4939
4940	lockdep_assert_held(&wq_pool_mutex);
4941
4942	list_for_each_entry(wq, &workqueues, list) {
4943	if (!(wq->flags & WQ_UNBOUND))
4944	continue;
4945	/* creating multiple pwqs breaks ordering guarantee */
4946	if (wq->flags & __WQ_ORDERED)
4947	continue;
4948
4949	ctx = apply_wqattrs_prepare(wq, wq->unbound_attrs);
4950	if (!ctx) {
4951	ret = -ENOMEM;
4952	break;
4953	}
4954
4955	list_add_tail(&ctx->list, &ctxs);
4956	}
4957
4958	list_for_each_entry_safe(ctx, n, &ctxs, list) {
4959	if (!ret)
4960	apply_wqattrs_commit(ctx);
4961	apply_wqattrs_cleanup(ctx);
4962	}
4963
4964	return ret;
4965	}
4966
4967	/**
4968	* workqueue_set_unbound_cpumask - Set the low-level unbound cpumask
4969	* @cpumask: the cpumask to set
4970	*
4971	* The low-level workqueues cpumask is a global cpumask that limits
4972	* the affinity of all unbound workqueues. This function check the @cpumask
4973	* and apply it to all unbound workqueues and updates all pwqs of them.
4974	*
4975	* Retun: 0 - Success
4976	* -EINVAL - Invalid @cpumask
4977	* -ENOMEM - Failed to allocate memory for attrs or pwqs.
4978	*/
4979	int workqueue_set_unbound_cpumask(cpumask_var_t cpumask)
4980	{
4981	int ret = -EINVAL;
4982	cpumask_var_t saved_cpumask;
4983
4984	if (!zalloc_cpumask_var(&saved_cpumask, GFP_KERNEL))
4985	return -ENOMEM;
4986
4987	/*
4988	* Not excluding isolated cpus on purpose.
4989	* If the user wishes to include them, we allow that.
4990	*/
4991	cpumask_and(cpumask, cpumask, cpu_possible_mask);
4992	if (!cpumask_empty(cpumask)) {
4993	apply_wqattrs_lock();
4994
4995	/* save the old wq_unbound_cpumask. */
4996	cpumask_copy(saved_cpumask, wq_unbound_cpumask);
4997
4998	/* update wq_unbound_cpumask at first and apply it to wqs. */
4999	cpumask_copy(wq_unbound_cpumask, cpumask);
5000	ret = workqueue_apply_unbound_cpumask();
5001
5002	/* restore the wq_unbound_cpumask when failed. */
5003	if (ret < 0)
5004	cpumask_copy(wq_unbound_cpumask, saved_cpumask);
5005
5006	apply_wqattrs_unlock();
5007	}
5008
5009	free_cpumask_var(saved_cpumask);
5010	return ret;
5011	}
5012
5013	#ifdef CONFIG_SYSFS
5014	/*
5015	* Workqueues with WQ_SYSFS flag set is visible to userland via
5016	* /sys/bus/workqueue/devices/WQ_NAME. All visible workqueues have the
5017	* following attributes.
5018	*
5019	* per_cpu RO bool : whether the workqueue is per-cpu or unbound
5020	* max_active RW int : maximum number of in-flight work items
5021	*
5022	* Unbound workqueues have the following extra attributes.
5023	*
5024	* pool_ids RO int : the associated pool IDs for each node
5025	* nice RW int : nice value of the workers
5026	* cpumask RW mask : bitmask of allowed CPUs for the workers
5027	* numa RW bool : whether enable NUMA affinity
5028	*/
5029	struct wq_device {
5030	struct workqueue_struct *wq;
5031	struct device dev;
5032	};
5033
5034	static struct workqueue_struct dev_to_wq(struct device dev)
5035	{
5036	struct wq_device *wq_dev = container_of(dev, struct wq_device, dev);
5037
5038	return wq_dev->wq;
5039	}
5040
5041	static ssize_t per_cpu_show(struct device dev, struct device_attribute attr,
5042	char *buf)
5043	{
5044	struct workqueue_struct *wq = dev_to_wq(dev);
5045
5046	return scnprintf(buf, PAGE_SIZE, "%d\n", (bool)!(wq->flags & WQ_UNBOUND));
5047	}
5048	static DEVICE_ATTR_RO(per_cpu);
5049
5050	static ssize_t max_active_show(struct device *dev,
5051	struct device_attribute attr, char buf)
5052	{
5053	struct workqueue_struct *wq = dev_to_wq(dev);
5054
5055	return scnprintf(buf, PAGE_SIZE, "%d\n", wq->saved_max_active);
5056	}
5057
5058	static ssize_t max_active_store(struct device *dev,
5059	struct device_attribute attr, const char buf,
5060	size_t count)
5061	{
5062	struct workqueue_struct *wq = dev_to_wq(dev);
5063	int val;
5064
5065	if (sscanf(buf, "%d", &val) != 1 \|\| val <= 0)
5066	return -EINVAL;
5067
5068	workqueue_set_max_active(wq, val);
5069	return count;
5070	}
5071	static DEVICE_ATTR_RW(max_active);
5072
5073	static struct attribute *wq_sysfs_attrs[] = {
5074	&dev_attr_per_cpu.attr,
5075	&dev_attr_max_active.attr,
5076	NULL,
5077	};
5078	ATTRIBUTE_GROUPS(wq_sysfs);
5079
5080	static ssize_t wq_pool_ids_show(struct device *dev,
5081	struct device_attribute attr, char buf)
5082	{
5083	struct workqueue_struct *wq = dev_to_wq(dev);
5084	const char *delim = "";
5085	int node, written = 0;
5086
5087	rcu_read_lock_sched();
5088	for_each_node(node) {
5089	written += scnprintf(buf + written, PAGE_SIZE - written,
5090	"%s%d:%d", delim, node,
5091	unbound_pwq_by_node(wq, node)->pool->id);
5092	delim = " ";
5093	}
5094	written += scnprintf(buf + written, PAGE_SIZE - written, "\n");
5095	rcu_read_unlock_sched();
5096
5097	return written;
5098	}
5099
5100	static ssize_t wq_nice_show(struct device dev, struct device_attribute attr,
5101	char *buf)
5102	{
5103	struct workqueue_struct *wq = dev_to_wq(dev);
5104	int written;
5105
5106	mutex_lock(&wq->mutex);
5107	written = scnprintf(buf, PAGE_SIZE, "%d\n", wq->unbound_attrs->nice);
5108	mutex_unlock(&wq->mutex);
5109
5110	return written;
5111	}
5112
5113	/* prepare workqueue_attrs for sysfs store operations */
5114	static struct workqueue_attrs wq_sysfs_prep_attrs(struct workqueue_struct wq)
5115	{
5116	struct workqueue_attrs *attrs;
5117
5118	lockdep_assert_held(&wq_pool_mutex);
5119
5120	attrs = alloc_workqueue_attrs(GFP_KERNEL);
5121	if (!attrs)
5122	return NULL;
5123
5124	copy_workqueue_attrs(attrs, wq->unbound_attrs);
5125	return attrs;
5126	}
5127
5128	static ssize_t wq_nice_store(struct device dev, struct device_attribute attr,
5129	const char *buf, size_t count)
5130	{
5131	struct workqueue_struct *wq = dev_to_wq(dev);
5132	struct workqueue_attrs *attrs;
5133	int ret = -ENOMEM;
5134
5135	apply_wqattrs_lock();
5136
5137	attrs = wq_sysfs_prep_attrs(wq);
5138	if (!attrs)
5139	goto out_unlock;
5140
5141	if (sscanf(buf, "%d", &attrs->nice) == 1 &&
5142	attrs->nice >= MIN_NICE && attrs->nice <= MAX_NICE)
5143	ret = apply_workqueue_attrs_locked(wq, attrs);
5144	else
5145	ret = -EINVAL;
5146
5147	out_unlock:
5148	apply_wqattrs_unlock();
5149	free_workqueue_attrs(attrs);
5150	return ret ?: count;
5151	}
5152
5153	static ssize_t wq_cpumask_show(struct device *dev,
5154	struct device_attribute attr, char buf)
5155	{
5156	struct workqueue_struct *wq = dev_to_wq(dev);
5157	int written;
5158
5159	mutex_lock(&wq->mutex);
5160	written = scnprintf(buf, PAGE_SIZE, "%*pb\n",
5161	cpumask_pr_args(wq->unbound_attrs->cpumask));
5162	mutex_unlock(&wq->mutex);
5163	return written;
5164	}
5165
5166	static ssize_t wq_cpumask_store(struct device *dev,
5167	struct device_attribute *attr,
5168	const char *buf, size_t count)
5169	{
5170	struct workqueue_struct *wq = dev_to_wq(dev);
5171	struct workqueue_attrs *attrs;
5172	int ret = -ENOMEM;
5173
5174	apply_wqattrs_lock();
5175
5176	attrs = wq_sysfs_prep_attrs(wq);
5177	if (!attrs)
5178	goto out_unlock;
5179
5180	ret = cpumask_parse(buf, attrs->cpumask);
5181	if (!ret)
5182	ret = apply_workqueue_attrs_locked(wq, attrs);
5183
5184	out_unlock:
5185	apply_wqattrs_unlock();
5186	free_workqueue_attrs(attrs);
5187	return ret ?: count;
5188	}
5189
5190	static ssize_t wq_numa_show(struct device dev, struct device_attribute attr,
5191	char *buf)
5192	{
5193	struct workqueue_struct *wq = dev_to_wq(dev);
5194	int written;
5195
5196	mutex_lock(&wq->mutex);
5197	written = scnprintf(buf, PAGE_SIZE, "%d\n",
5198	!wq->unbound_attrs->no_numa);
5199	mutex_unlock(&wq->mutex);
5200
5201	return written;
5202	}
5203
5204	static ssize_t wq_numa_store(struct device dev, struct device_attribute attr,
5205	const char *buf, size_t count)
5206	{
5207	struct workqueue_struct *wq = dev_to_wq(dev);
5208	struct workqueue_attrs *attrs;
5209	int v, ret = -ENOMEM;
5210
5211	apply_wqattrs_lock();
5212
5213	attrs = wq_sysfs_prep_attrs(wq);
5214	if (!attrs)
5215	goto out_unlock;
5216
5217	ret = -EINVAL;
5218	if (sscanf(buf, "%d", &v) == 1) {
5219	attrs->no_numa = !v;
5220	ret = apply_workqueue_attrs_locked(wq, attrs);
5221	}
5222
5223	out_unlock:
5224	apply_wqattrs_unlock();
5225	free_workqueue_attrs(attrs);
5226	return ret ?: count;
5227	}
5228
5229	static struct device_attribute wq_sysfs_unbound_attrs[] = {
5230	__ATTR(pool_ids, 0444, wq_pool_ids_show, NULL),
5231	__ATTR(nice, 0644, wq_nice_show, wq_nice_store),
5232	__ATTR(cpumask, 0644, wq_cpumask_show, wq_cpumask_store),
5233	__ATTR(numa, 0644, wq_numa_show, wq_numa_store),
5234	__ATTR_NULL,
5235	};
5236
5237	static struct bus_type wq_subsys = {
5238	.name = "workqueue",
5239	.dev_groups = wq_sysfs_groups,
5240	};
5241
5242	static ssize_t wq_unbound_cpumask_show(struct device *dev,
5243	struct device_attribute attr, char buf)
5244	{
5245	int written;
5246
5247	mutex_lock(&wq_pool_mutex);
5248	written = scnprintf(buf, PAGE_SIZE, "%*pb\n",
5249	cpumask_pr_args(wq_unbound_cpumask));
5250	mutex_unlock(&wq_pool_mutex);
5251
5252	return written;
5253	}
5254
5255	static ssize_t wq_unbound_cpumask_store(struct device *dev,
5256	struct device_attribute attr, const char buf, size_t count)
5257	{
5258	cpumask_var_t cpumask;
5259	int ret;
5260
5261	if (!zalloc_cpumask_var(&cpumask, GFP_KERNEL))
5262	return -ENOMEM;
5263
5264	ret = cpumask_parse(buf, cpumask);
5265	if (!ret)
5266	ret = workqueue_set_unbound_cpumask(cpumask);
5267
5268	free_cpumask_var(cpumask);
5269	return ret ? ret : count;
5270	}
5271
5272	static struct device_attribute wq_sysfs_cpumask_attr =
5273	__ATTR(cpumask, 0644, wq_unbound_cpumask_show,
5274	wq_unbound_cpumask_store);
5275
5276	static int __init wq_sysfs_init(void)
5277	{
5278	int err;
5279
5280	err = subsys_virtual_register(&wq_subsys, NULL);
5281	if (err)
5282	return err;
5283
5284	return device_create_file(wq_subsys.dev_root, &wq_sysfs_cpumask_attr);
5285	}
5286	core_initcall(wq_sysfs_init);
5287
5288	static void wq_device_release(struct device *dev)
5289	{
5290	struct wq_device *wq_dev = container_of(dev, struct wq_device, dev);
5291
5292	kfree(wq_dev);
5293	}
5294
5295	/**
5296	* workqueue_sysfs_register - make a workqueue visible in sysfs
5297	* @wq: the workqueue to register
5298	*
5299	* Expose @wq in sysfs under /sys/bus/workqueue/devices.
5300	* alloc_workqueue*() automatically calls this function if WQ_SYSFS is set
5301	* which is the preferred method.
5302	*
5303	* Workqueue user should use this function directly iff it wants to apply
5304	* workqueue_attrs before making the workqueue visible in sysfs; otherwise,
5305	* apply_workqueue_attrs() may race against userland updating the
5306	* attributes.
5307	*
5308	* Return: 0 on success, -errno on failure.
5309	*/
5310	int workqueue_sysfs_register(struct workqueue_struct *wq)
5311	{
5312	struct wq_device *wq_dev;
5313	int ret;
5314
5315	/*
5316	* Adjusting max_active or creating new pwqs by applying
5317	* attributes breaks ordering guarantee. Disallow exposing ordered
5318	* workqueues.
5319	*/
5320	if (WARN_ON(wq->flags & __WQ_ORDERED_EXPLICIT))
5321	return -EINVAL;
5322
5323	wq->wq_dev = wq_dev = kzalloc(sizeof(*wq_dev), GFP_KERNEL);
5324	if (!wq_dev)
5325	return -ENOMEM;
5326
5327	wq_dev->wq = wq;
5328	wq_dev->dev.bus = &wq_subsys;
5329	wq_dev->dev.release = wq_device_release;
5330	dev_set_name(&wq_dev->dev, "%s", wq->name);
5331
5332	/*
5333	* unbound_attrs are created separately. Suppress uevent until
5334	* everything is ready.
5335	*/
5336	dev_set_uevent_suppress(&wq_dev->dev, true);
5337
5338	ret = device_register(&wq_dev->dev);
5339	if (ret) {
5340	kfree(wq_dev);
5341	wq->wq_dev = NULL;
5342	return ret;
5343	}
5344
5345	if (wq->flags & WQ_UNBOUND) {
5346	struct device_attribute *attr;
5347
5348	for (attr = wq_sysfs_unbound_attrs; attr->attr.name; attr++) {
5349	ret = device_create_file(&wq_dev->dev, attr);
5350	if (ret) {
5351	device_unregister(&wq_dev->dev);
5352	wq->wq_dev = NULL;
5353	return ret;
5354	}
5355	}
5356	}
5357
5358	dev_set_uevent_suppress(&wq_dev->dev, false);
5359	kobject_uevent(&wq_dev->dev.kobj, KOBJ_ADD);
5360	return 0;
5361	}
5362
5363	/**
5364	* workqueue_sysfs_unregister - undo workqueue_sysfs_register()
5365	* @wq: the workqueue to unregister
5366	*
5367	* If @wq is registered to sysfs by workqueue_sysfs_register(), unregister.
5368	*/
5369	static void workqueue_sysfs_unregister(struct workqueue_struct *wq)
5370	{
5371	struct wq_device *wq_dev = wq->wq_dev;
5372
5373	if (!wq->wq_dev)
5374	return;
5375
5376	wq->wq_dev = NULL;
5377	device_unregister(&wq_dev->dev);
5378	}
5379	#else /* CONFIG_SYSFS */
5380	static void workqueue_sysfs_unregister(struct workqueue_struct *wq) { }
5381	#endif /* CONFIG_SYSFS */
5382
5383	/*
5384	* Workqueue watchdog.
5385	*
5386	* Stall may be caused by various bugs - missing WQ_MEM_RECLAIM, illegal
5387	* flush dependency, a concurrency managed work item which stays RUNNING
5388	* indefinitely. Workqueue stalls can be very difficult to debug as the
5389	* usual warning mechanisms don't trigger and internal workqueue state is
5390	* largely opaque.
5391	*
5392	* Workqueue watchdog monitors all worker pools periodically and dumps
5393	* state if some pools failed to make forward progress for a while where
5394	* forward progress is defined as the first item on ->worklist changing.
5395	*
5396	* This mechanism is controlled through the kernel parameter
5397	* "workqueue.watchdog_thresh" which can be updated at runtime through the
5398	* corresponding sysfs parameter file.
5399	*/
5400	#ifdef CONFIG_WQ_WATCHDOG
5401
5402	static unsigned long wq_watchdog_thresh = 30;
5403	static struct timer_list wq_watchdog_timer;
5404
5405	static unsigned long wq_watchdog_touched = INITIAL_JIFFIES;
5406	static DEFINE_PER_CPU(unsigned long, wq_watchdog_touched_cpu) = INITIAL_JIFFIES;
5407
5408	static void wq_watchdog_reset_touched(void)
5409	{
5410	int cpu;
5411
5412	wq_watchdog_touched = jiffies;
5413	for_each_possible_cpu(cpu)
5414	per_cpu(wq_watchdog_touched_cpu, cpu) = jiffies;
5415	}
5416
5417	static void wq_watchdog_timer_fn(struct timer_list *unused)
5418	{
5419	unsigned long thresh = READ_ONCE(wq_watchdog_thresh) * HZ;
5420	bool lockup_detected = false;
5421	struct worker_pool *pool;
5422	int pi;
5423
5424	if (!thresh)
5425	return;
5426
5427	rcu_read_lock();
5428
5429	for_each_pool(pool, pi) {
5430	unsigned long pool_ts, touched, ts;
5431
5432	if (list_empty(&pool->worklist))
5433	continue;
5434
5435	/* get the latest of pool and touched timestamps */
5436	pool_ts = READ_ONCE(pool->watchdog_ts);
5437	touched = READ_ONCE(wq_watchdog_touched);
5438
5439	if (time_after(pool_ts, touched))
5440	ts = pool_ts;
5441	else
5442	ts = touched;
5443
5444	if (pool->cpu >= 0) {
5445	unsigned long cpu_touched =
5446	READ_ONCE(per_cpu(wq_watchdog_touched_cpu,
5447	pool->cpu));
5448	if (time_after(cpu_touched, ts))
5449	ts = cpu_touched;
5450	}
5451
5452	/* did we stall? */
5453	if (time_after(jiffies, ts + thresh)) {
5454	lockup_detected = true;
5455	pr_emerg("BUG: workqueue lockup - pool");
5456	pr_cont_pool_info(pool);
5457	pr_cont(" stuck for %us!\n",
5458	jiffies_to_msecs(jiffies - pool_ts) / 1000);
5459	}
5460	}
5461
5462	rcu_read_unlock();
5463
5464	if (lockup_detected)
5465	show_workqueue_state();
5466
5467	wq_watchdog_reset_touched();
5468	mod_timer(&wq_watchdog_timer, jiffies + thresh);
5469	}
5470
5471	void wq_watchdog_touch(int cpu)
5472	{
5473	if (cpu >= 0)
5474	per_cpu(wq_watchdog_touched_cpu, cpu) = jiffies;
5475	else
5476	wq_watchdog_touched = jiffies;
5477	}
5478
5479	static void wq_watchdog_set_thresh(unsigned long thresh)
5480	{
5481	wq_watchdog_thresh = 0;
5482	del_timer_sync(&wq_watchdog_timer);
5483
5484	if (thresh) {
5485	wq_watchdog_thresh = thresh;
5486	wq_watchdog_reset_touched();
5487	mod_timer(&wq_watchdog_timer, jiffies + thresh * HZ);
5488	}
5489	}
5490
5491	static int wq_watchdog_param_set_thresh(const char *val,
5492	const struct kernel_param *kp)
5493	{
5494	unsigned long thresh;
5495	int ret;
5496
5497	ret = kstrtoul(val, 0, &thresh);
5498	if (ret)
5499	return ret;
5500
5501	if (system_wq)
5502	wq_watchdog_set_thresh(thresh);
5503	else
5504	wq_watchdog_thresh = thresh;
5505
5506	return 0;
5507	}
5508
5509	static const struct kernel_param_ops wq_watchdog_thresh_ops = {
5510	.set = wq_watchdog_param_set_thresh,
5511	.get = param_get_ulong,
5512	};
5513
5514	module_param_cb(watchdog_thresh, &wq_watchdog_thresh_ops, &wq_watchdog_thresh,
5515	0644);
5516
5517	static void wq_watchdog_init(void)
5518	{
5519	timer_setup(&wq_watchdog_timer, wq_watchdog_timer_fn, TIMER_DEFERRABLE);
5520	wq_watchdog_set_thresh(wq_watchdog_thresh);
5521	}
5522
5523	#else /* CONFIG_WQ_WATCHDOG */
5524
5525	static inline void wq_watchdog_init(void) { }
5526
5527	#endif /* CONFIG_WQ_WATCHDOG */
5528
5529	static void __init wq_numa_init(void)
5530	{
5531	cpumask_var_t *tbl;
5532	int node, cpu;
5533
5534	if (num_possible_nodes() <= 1)
5535	return;
5536
5537	if (wq_disable_numa) {
5538	pr_info("workqueue: NUMA affinity support disabled\n");
5539	return;
5540	}
5541
5542	wq_update_unbound_numa_attrs_buf = alloc_workqueue_attrs(GFP_KERNEL);
5543	BUG_ON(!wq_update_unbound_numa_attrs_buf);
5544
5545	/*
5546	* We want masks of possible CPUs of each node which isn't readily
5547	* available. Build one from cpu_to_node() which should have been
5548	* fully initialized by now.
5549	*/
5550	tbl = kzalloc(nr_node_ids * sizeof(tbl[0]), GFP_KERNEL);
5551	BUG_ON(!tbl);
5552
5553	for_each_node(node)
5554	BUG_ON(!zalloc_cpumask_var_node(&tbl[node], GFP_KERNEL,
5555	node_online(node) ? node : NUMA_NO_NODE));
5556
5557	for_each_possible_cpu(cpu) {
5558	node = cpu_to_node(cpu);
5559	if (WARN_ON(node == NUMA_NO_NODE)) {
5560	pr_warn("workqueue: NUMA node mapping not available for cpu%d, disabling NUMA support\n", cpu);
5561	/* happens iff arch is bonkers, let's just proceed */
5562	return;
5563	}
5564	cpumask_set_cpu(cpu, tbl[node]);
5565	}
5566
5567	wq_numa_possible_cpumask = tbl;
5568	wq_numa_enabled = true;
5569	}
5570
5571	/**
5572	* workqueue_init_early - early init for workqueue subsystem
5573	*
5574	* This is the first half of two-staged workqueue subsystem initialization
5575	* and invoked as soon as the bare basics - memory allocation, cpumasks and
5576	* idr are up. It sets up all the data structures and system workqueues
5577	* and allows early boot code to create workqueues and queue/cancel work
5578	* items. Actual work item execution starts only after kthreads can be
5579	* created and scheduled right before early initcalls.
5580	*/
5581	int __init workqueue_init_early(void)
5582	{
5583	int std_nice[NR_STD_WORKER_POOLS] = { 0, HIGHPRI_NICE_LEVEL };
5584	int i, cpu;
5585
5586	WARN_ON(__alignof__(struct pool_workqueue) < __alignof__(long long));
5587
5588	BUG_ON(!alloc_cpumask_var(&wq_unbound_cpumask, GFP_KERNEL));
5589	cpumask_copy(wq_unbound_cpumask, housekeeping_cpumask(HK_FLAG_DOMAIN));
5590
5591	pwq_cache = KMEM_CACHE(pool_workqueue, SLAB_PANIC);
5592
5593	/* initialize CPU pools */
5594	for_each_possible_cpu(cpu) {
5595	struct worker_pool *pool;
5596
5597	i = 0;
5598	for_each_cpu_worker_pool(pool, cpu) {
5599	BUG_ON(init_worker_pool(pool));
5600	pool->cpu = cpu;
5601	cpumask_copy(pool->attrs->cpumask, cpumask_of(cpu));
5602	pool->attrs->nice = std_nice[i++];
5603	pool->node = cpu_to_node(cpu);
5604
5605	/* alloc pool ID */
5606	mutex_lock(&wq_pool_mutex);
5607	BUG_ON(worker_pool_assign_id(pool));
5608	mutex_unlock(&wq_pool_mutex);
5609	}
5610	}
5611
5612	/* create default unbound and ordered wq attrs */
5613	for (i = 0; i < NR_STD_WORKER_POOLS; i++) {
5614	struct workqueue_attrs *attrs;
5615
5616	BUG_ON(!(attrs = alloc_workqueue_attrs(GFP_KERNEL)));
5617	attrs->nice = std_nice[i];
5618	unbound_std_wq_attrs[i] = attrs;
5619
5620	/*
5621	* An ordered wq should have only one pwq as ordering is
5622	* guaranteed by max_active which is enforced by pwqs.
5623	* Turn off NUMA so that dfl_pwq is used for all nodes.
5624	*/
5625	BUG_ON(!(attrs = alloc_workqueue_attrs(GFP_KERNEL)));
5626	attrs->nice = std_nice[i];
5627	attrs->no_numa = true;
5628	ordered_wq_attrs[i] = attrs;
5629	}
5630
5631	system_wq = alloc_workqueue("events", 0, 0);
5632	system_highpri_wq = alloc_workqueue("events_highpri", WQ_HIGHPRI, 0);
5633	system_long_wq = alloc_workqueue("events_long", 0, 0);
5634	system_unbound_wq = alloc_workqueue("events_unbound", WQ_UNBOUND,
5635	WQ_UNBOUND_MAX_ACTIVE);
5636	system_freezable_wq = alloc_workqueue("events_freezable",
5637	WQ_FREEZABLE, 0);
5638	system_power_efficient_wq = alloc_workqueue("events_power_efficient",
5639	WQ_POWER_EFFICIENT, 0);
5640	system_freezable_power_efficient_wq = alloc_workqueue("events_freezable_power_efficient",
5641	WQ_FREEZABLE \| WQ_POWER_EFFICIENT,
5642	0);
5643	BUG_ON(!system_wq \|\| !system_highpri_wq \|\| !system_long_wq \|\|
5644	!system_unbound_wq \|\| !system_freezable_wq \|\|
5645	!system_power_efficient_wq \|\|
5646	!system_freezable_power_efficient_wq);
5647
5648	return 0;
5649	}
5650
5651	/**
5652	* workqueue_init - bring workqueue subsystem fully online
5653	*
5654	* This is the latter half of two-staged workqueue subsystem initialization
5655	* and invoked as soon as kthreads can be created and scheduled.
5656	* Workqueues have been created and work items queued on them, but there
5657	* are no kworkers executing the work items yet. Populate the worker pools
5658	* with the initial workers and enable future kworker creations.
5659	*/
5660	int __init workqueue_init(void)
5661	{
5662	struct workqueue_struct *wq;
5663	struct worker_pool *pool;
5664	int cpu, bkt;
5665
5666	/*
5667	* It'd be simpler to initialize NUMA in workqueue_init_early() but
5668	* CPU to node mapping may not be available that early on some
5669	* archs such as power and arm64. As per-cpu pools created
5670	* previously could be missing node hint and unbound pools NUMA
5671	* affinity, fix them up.
5672	*
5673	* Also, while iterating workqueues, create rescuers if requested.
5674	*/
5675	wq_numa_init();
5676
5677	mutex_lock(&wq_pool_mutex);
5678
5679	for_each_possible_cpu(cpu) {
5680	for_each_cpu_worker_pool(pool, cpu) {
5681	pool->node = cpu_to_node(cpu);
5682	}
5683	}
5684
5685	list_for_each_entry(wq, &workqueues, list) {
5686	wq_update_unbound_numa(wq, smp_processor_id(), true);
5687	WARN(init_rescuer(wq),
5688	"workqueue: failed to create early rescuer for %s",
5689	wq->name);
5690	}
5691
5692	mutex_unlock(&wq_pool_mutex);
5693
5694	/* create the initial workers */
5695	for_each_online_cpu(cpu) {
5696	for_each_cpu_worker_pool(pool, cpu) {
5697	pool->flags &= ~POOL_DISASSOCIATED;
5698	BUG_ON(!create_worker(pool));
5699	}
5700	}
5701
5702	hash_for_each(unbound_pool_hash, bkt, pool, hash_node)
5703	BUG_ON(!create_worker(pool));
5704
5705	wq_online = true;
5706	wq_watchdog_init();
5707
5708	return 0;
5709	}