linux - kernel/workqueue.c

5081 lines · 140.1 KiB

1	/*
2	* kernel/workqueue.c - generic async execution with shared worker pool
3	*
4	* Copyright (C) 2002 Ingo Molnar
5	*
6	* Derived from the taskqueue/keventd code by:
7	* David Woodhouse <[email protected]>
8	* Andrew Morton
9	* Kai Petzke <[email protected]>
10	* Theodore Ts'o <[email protected]>
11	*
12	* Made to use alloc_percpu by Christoph Lameter.
13	*
14	* Copyright (C) 2010 SUSE Linux Products GmbH
15	* Copyright (C) 2010 Tejun Heo <[email protected]>
16	*
17	* This is the generic async execution mechanism. Work items as are
18	* executed in process context. The worker pool is shared and
19	* automatically managed. There are two worker pools for each CPU (one for
20	* normal work items and the other for high priority ones) and some extra
21	* pools for workqueues which are not bound to any specific CPU - the
22	* number of these backing pools is dynamic.
23	*
24	* Please read Documentation/workqueue.txt for details.
25	*/
26
27	#include <linux/export.h>
28	#include <linux/kernel.h>
29	#include <linux/sched.h>
30	#include <linux/init.h>
31	#include <linux/signal.h>
32	#include <linux/completion.h>
33	#include <linux/workqueue.h>
34	#include <linux/slab.h>
35	#include <linux/cpu.h>
36	#include <linux/notifier.h>
37	#include <linux/kthread.h>
38	#include <linux/hardirq.h>
39	#include <linux/mempolicy.h>
40	#include <linux/freezer.h>
41	#include <linux/kallsyms.h>
42	#include <linux/debug_locks.h>
43	#include <linux/lockdep.h>
44	#include <linux/idr.h>
45	#include <linux/jhash.h>
46	#include <linux/hashtable.h>
47	#include <linux/rculist.h>
48	#include <linux/nodemask.h>
49	#include <linux/moduleparam.h>
50	#include <linux/uaccess.h>
51
52	#include "workqueue_internal.h"
53
54	enum {
55	/*
56	* worker_pool flags
57	*
58	* A bound pool is either associated or disassociated with its CPU.
59	* While associated (!DISASSOCIATED), all workers are bound to the
60	* CPU and none has %WORKER_UNBOUND set and concurrency management
61	* is in effect.
62	*
63	* While DISASSOCIATED, the cpu may be offline and all workers have
64	* %WORKER_UNBOUND set and concurrency management disabled, and may
65	* be executing on any CPU. The pool behaves as an unbound one.
66	*
67	* Note that DISASSOCIATED should be flipped only while holding
68	* manager_mutex to avoid changing binding state while
69	* create_worker() is in progress.
70	*/
71	POOL_MANAGE_WORKERS = 1 << 0, /* need to manage workers */
72	POOL_DISASSOCIATED = 1 << 2, /* cpu can't serve workers */
73	POOL_FREEZING = 1 << 3, /* freeze in progress */
74
75	/* worker flags */
76	WORKER_STARTED = 1 << 0, /* started */
77	WORKER_DIE = 1 << 1, /* die die die */
78	WORKER_IDLE = 1 << 2, /* is idle */
79	WORKER_PREP = 1 << 3, /* preparing to run works */
80	WORKER_CPU_INTENSIVE = 1 << 6, /* cpu intensive */
81	WORKER_UNBOUND = 1 << 7, /* worker is unbound */
82	WORKER_REBOUND = 1 << 8, /* worker was rebound */
83
84	WORKER_NOT_RUNNING = WORKER_PREP \| WORKER_CPU_INTENSIVE \|
85	WORKER_UNBOUND \| WORKER_REBOUND,
86
87	NR_STD_WORKER_POOLS = 2, /* # standard pools per cpu */
88
89	UNBOUND_POOL_HASH_ORDER = 6, /* hashed by pool->attrs */
90	BUSY_WORKER_HASH_ORDER = 6, /* 64 pointers */
91
92	MAX_IDLE_WORKERS_RATIO = 4, /* 1/4 of busy can be idle */
93	IDLE_WORKER_TIMEOUT = 300 * HZ, /* keep idle ones for 5 mins */
94
95	MAYDAY_INITIAL_TIMEOUT = HZ / 100 >= 2 ? HZ / 100 : 2,
96	/* call for help after 10ms
97	(min two ticks) */
98	MAYDAY_INTERVAL = HZ / 10, /* and then every 100ms */
99	CREATE_COOLDOWN = HZ, /* time to breath after fail */
100
101	/*
102	* Rescue workers are used only on emergencies and shared by
103	* all cpus. Give -20.
104	*/
105	RESCUER_NICE_LEVEL = -20,
106	HIGHPRI_NICE_LEVEL = -20,
107
108	WQ_NAME_LEN = 24,
109	};
110
111	/*
112	* Structure fields follow one of the following exclusion rules.
113	*
114	* I: Modifiable by initialization/destruction paths and read-only for
115	* everyone else.
116	*
117	* P: Preemption protected. Disabling preemption is enough and should
118	* only be modified and accessed from the local cpu.
119	*
120	* L: pool->lock protected. Access with pool->lock held.
121	*
122	* X: During normal operation, modification requires pool->lock and should
123	* be done only from local cpu. Either disabling preemption on local
124	* cpu or grabbing pool->lock is enough for read access. If
125	* POOL_DISASSOCIATED is set, it's identical to L.
126	*
127	* MG: pool->manager_mutex and pool->lock protected. Writes require both
128	* locks. Reads can happen under either lock.
129	*
130	* PL: wq_pool_mutex protected.
131	*
132	* PR: wq_pool_mutex protected for writes. Sched-RCU protected for reads.
133	*
134	* WQ: wq->mutex protected.
135	*
136	* WR: wq->mutex protected for writes. Sched-RCU protected for reads.
137	*
138	* MD: wq_mayday_lock protected.
139	*/
140
141	/* struct worker is defined in workqueue_internal.h */
142
143	struct worker_pool {
144	spinlock_t lock; /* the pool lock */
145	int cpu; /* I: the associated cpu */
146	int node; /* I: the associated node ID */
147	int id; /* I: pool ID */
148	unsigned int flags; /* X: flags */
149
150	struct list_head worklist; /* L: list of pending works */
151	int nr_workers; /* L: total number of workers */
152
153	/* nr_idle includes the ones off idle_list for rebinding */
154	int nr_idle; /* L: currently idle ones */
155
156	struct list_head idle_list; /* X: list of idle workers */
157	struct timer_list idle_timer; /* L: worker idle timeout */
158	struct timer_list mayday_timer; /* L: SOS timer for workers */
159
160	/* a workers is either on busy_hash or idle_list, or the manager */
161	DECLARE_HASHTABLE(busy_hash, BUSY_WORKER_HASH_ORDER);
162	/* L: hash of busy workers */
163
164	/* see manage_workers() for details on the two manager mutexes */
165	struct mutex manager_arb; /* manager arbitration */
166	struct mutex manager_mutex; /* manager exclusion */
167	struct idr worker_idr; /* MG: worker IDs and iteration */
168
169	struct workqueue_attrs attrs; / I: worker attributes */
170	struct hlist_node hash_node; /* PL: unbound_pool_hash node */
171	int refcnt; /* PL: refcnt for unbound pools */
172
173	/*
174	* The current concurrency level. As it's likely to be accessed
175	* from other CPUs during try_to_wake_up(), put it in a separate
176	* cacheline.
177	*/
178	atomic_t nr_running ____cacheline_aligned_in_smp;
179
180	/*
181	* Destruction of pool is sched-RCU protected to allow dereferences
182	* from get_work_pool().
183	*/
184	struct rcu_head rcu;
185	} ____cacheline_aligned_in_smp;
186
187	/*
188	* The per-pool workqueue. While queued, the lower WORK_STRUCT_FLAG_BITS
189	* of work_struct->data are used for flags and the remaining high bits
190	* point to the pwq; thus, pwqs need to be aligned at two's power of the
191	* number of flag bits.
192	*/
193	struct pool_workqueue {
194	struct worker_pool pool; / I: the associated pool */
195	struct workqueue_struct wq; / I: the owning workqueue */
196	int work_color; /* L: current color */
197	int flush_color; /* L: flushing color */
198	int refcnt; /* L: reference count */
199	int nr_in_flight[WORK_NR_COLORS];
200	/* L: nr of in_flight works */
201	int nr_active; /* L: nr of active works */
202	int max_active; /* L: max active works */
203	struct list_head delayed_works; /* L: delayed works */
204	struct list_head pwqs_node; /* WR: node on wq->pwqs */
205	struct list_head mayday_node; /* MD: node on wq->maydays */
206
207	/*
208	* Release of unbound pwq is punted to system_wq. See put_pwq()
209	* and pwq_unbound_release_workfn() for details. pool_workqueue
210	* itself is also sched-RCU protected so that the first pwq can be
211	* determined without grabbing wq->mutex.
212	*/
213	struct work_struct unbound_release_work;
214	struct rcu_head rcu;
215	} __aligned(1 << WORK_STRUCT_FLAG_BITS);
216
217	/*
218	* Structure used to wait for workqueue flush.
219	*/
220	struct wq_flusher {
221	struct list_head list; /* WQ: list of flushers */
222	int flush_color; /* WQ: flush color waiting for */
223	struct completion done; /* flush completion */
224	};
225
226	struct wq_device;
227
228	/*
229	* The externally visible workqueue. It relays the issued work items to
230	* the appropriate worker_pool through its pool_workqueues.
231	*/
232	struct workqueue_struct {
233	struct list_head pwqs; /* WR: all pwqs of this wq */
234	struct list_head list; /* PL: list of all workqueues */
235
236	struct mutex mutex; /* protects this wq */
237	int work_color; /* WQ: current work color */
238	int flush_color; /* WQ: current flush color */
239	atomic_t nr_pwqs_to_flush; /* flush in progress */
240	struct wq_flusher first_flusher; / WQ: first flusher */
241	struct list_head flusher_queue; /* WQ: flush waiters */
242	struct list_head flusher_overflow; /* WQ: flush overflow list */
243
244	struct list_head maydays; /* MD: pwqs requesting rescue */
245	struct worker rescuer; / I: rescue worker */
246
247	int nr_drainers; /* WQ: drain in progress */
248	int saved_max_active; /* WQ: saved pwq max_active */
249
250	struct workqueue_attrs unbound_attrs; / WQ: only for unbound wqs */
251	struct pool_workqueue dfl_pwq; / WQ: only for unbound wqs */
252
253	#ifdef CONFIG_SYSFS
254	struct wq_device wq_dev; / I: for sysfs interface */
255	#endif
256	#ifdef CONFIG_LOCKDEP
257	struct lockdep_map lockdep_map;
258	#endif
259	char name[WQ_NAME_LEN]; /* I: workqueue name */
260
261	/* hot fields used during command issue, aligned to cacheline */
262	unsigned int flags ____cacheline_aligned; /* WQ: WQ_* flags */
263	struct pool_workqueue __percpu cpu_pwqs; / I: per-cpu pwqs */
264	struct pool_workqueue __rcu numa_pwq_tbl[]; / FR: unbound pwqs indexed by node */
265	};
266
267	static struct kmem_cache *pwq_cache;
268
269	static int wq_numa_tbl_len; /* highest possible NUMA node id + 1 */
270	static cpumask_var_t *wq_numa_possible_cpumask;
271	/* possible CPUs of each node */
272
273	static bool wq_disable_numa;
274	module_param_named(disable_numa, wq_disable_numa, bool, 0444);
275
276	/* see the comment above the definition of WQ_POWER_EFFICIENT */
277	#ifdef CONFIG_WQ_POWER_EFFICIENT_DEFAULT
278	static bool wq_power_efficient = true;
279	#else
280	static bool wq_power_efficient;
281	#endif
282
283	module_param_named(power_efficient, wq_power_efficient, bool, 0444);
284
285	static bool wq_numa_enabled; /* unbound NUMA affinity enabled */
286
287	/* buf for wq_update_unbound_numa_attrs(), protected by CPU hotplug exclusion */
288	static struct workqueue_attrs *wq_update_unbound_numa_attrs_buf;
289
290	static DEFINE_MUTEX(wq_pool_mutex); /* protects pools and workqueues list */
291	static DEFINE_SPINLOCK(wq_mayday_lock); /* protects wq->maydays list */
292
293	static LIST_HEAD(workqueues); /* PL: list of all workqueues */
294	static bool workqueue_freezing; /* PL: have wqs started freezing? */
295
296	/* the per-cpu worker pools */
297	static DEFINE_PER_CPU_SHARED_ALIGNED(struct worker_pool [NR_STD_WORKER_POOLS],
298	cpu_worker_pools);
299
300	static DEFINE_IDR(worker_pool_idr); /* PR: idr of all pools */
301
302	/* PL: hash of all unbound pools keyed by pool->attrs */
303	static DEFINE_HASHTABLE(unbound_pool_hash, UNBOUND_POOL_HASH_ORDER);
304
305	/* I: attributes used when instantiating standard unbound pools on demand */
306	static struct workqueue_attrs *unbound_std_wq_attrs[NR_STD_WORKER_POOLS];
307
308	struct workqueue_struct *system_wq __read_mostly;
309	EXPORT_SYMBOL(system_wq);
310	struct workqueue_struct *system_highpri_wq __read_mostly;
311	EXPORT_SYMBOL_GPL(system_highpri_wq);
312	struct workqueue_struct *system_long_wq __read_mostly;
313	EXPORT_SYMBOL_GPL(system_long_wq);
314	struct workqueue_struct *system_unbound_wq __read_mostly;
315	EXPORT_SYMBOL_GPL(system_unbound_wq);
316	struct workqueue_struct *system_freezable_wq __read_mostly;
317	EXPORT_SYMBOL_GPL(system_freezable_wq);
318	struct workqueue_struct *system_power_efficient_wq __read_mostly;
319	EXPORT_SYMBOL_GPL(system_power_efficient_wq);
320	struct workqueue_struct *system_freezable_power_efficient_wq __read_mostly;
321	EXPORT_SYMBOL_GPL(system_freezable_power_efficient_wq);
322
323	static int worker_thread(void *__worker);
324	static void copy_workqueue_attrs(struct workqueue_attrs *to,
325	const struct workqueue_attrs *from);
326
327	#define CREATE_TRACE_POINTS
328	#include <trace/events/workqueue.h>
329
330	#define assert_rcu_or_pool_mutex() \
331	rcu_lockdep_assert(rcu_read_lock_sched_held() \|\| \
332	lockdep_is_held(&wq_pool_mutex), \
333	"sched RCU or wq_pool_mutex should be held")
334
335	#define assert_rcu_or_wq_mutex(wq) \
336	rcu_lockdep_assert(rcu_read_lock_sched_held() \|\| \
337	lockdep_is_held(&wq->mutex), \
338	"sched RCU or wq->mutex should be held")
339
340	#ifdef CONFIG_LOCKDEP
341	#define assert_manager_or_pool_lock(pool) \
342	WARN_ONCE(debug_locks && \
343	!lockdep_is_held(&(pool)->manager_mutex) && \
344	!lockdep_is_held(&(pool)->lock), \
345	"pool->manager_mutex or ->lock should be held")
346	#else
347	#define assert_manager_or_pool_lock(pool) do { } while (0)
348	#endif
349
350	#define for_each_cpu_worker_pool(pool, cpu) \
351	for ((pool) = &per_cpu(cpu_worker_pools, cpu)[0]; \
352	(pool) < &per_cpu(cpu_worker_pools, cpu)[NR_STD_WORKER_POOLS]; \
353	(pool)++)
354
355	/**
356	* for_each_pool - iterate through all worker_pools in the system
357	* @pool: iteration cursor
358	* @pi: integer used for iteration
359	*
360	* This must be called either with wq_pool_mutex held or sched RCU read
361	* locked. If the pool needs to be used beyond the locking in effect, the
362	* caller is responsible for guaranteeing that the pool stays online.
363	*
364	* The if/else clause exists only for the lockdep assertion and can be
365	* ignored.
366	*/
367	#define for_each_pool(pool, pi) \
368	idr_for_each_entry(&worker_pool_idr, pool, pi) \
369	if (({ assert_rcu_or_pool_mutex(); false; })) { } \
370	else
371
372	/**
373	* for_each_pool_worker - iterate through all workers of a worker_pool
374	* @worker: iteration cursor
375	* @wi: integer used for iteration
376	* @pool: worker_pool to iterate workers of
377	*
378	* This must be called with either @pool->manager_mutex or ->lock held.
379	*
380	* The if/else clause exists only for the lockdep assertion and can be
381	* ignored.
382	*/
383	#define for_each_pool_worker(worker, wi, pool) \
384	idr_for_each_entry(&(pool)->worker_idr, (worker), (wi)) \
385	if (({ assert_manager_or_pool_lock((pool)); false; })) { } \
386	else
387
388	/**
389	* for_each_pwq - iterate through all pool_workqueues of the specified workqueue
390	* @pwq: iteration cursor
391	* @wq: the target workqueue
392	*
393	* This must be called either with wq->mutex held or sched RCU read locked.
394	* If the pwq needs to be used beyond the locking in effect, the caller is
395	* responsible for guaranteeing that the pwq stays online.
396	*
397	* The if/else clause exists only for the lockdep assertion and can be
398	* ignored.
399	*/
400	#define for_each_pwq(pwq, wq) \
401	list_for_each_entry_rcu((pwq), &(wq)->pwqs, pwqs_node) \
402	if (({ assert_rcu_or_wq_mutex(wq); false; })) { } \
403	else
404
405	#ifdef CONFIG_DEBUG_OBJECTS_WORK
406
407	static struct debug_obj_descr work_debug_descr;
408
409	static void work_debug_hint(void addr)
410	{
411	return ((struct work_struct *) addr)->func;
412	}
413
414	/*
415	* fixup_init is called when:
416	* - an active object is initialized
417	*/
418	static int work_fixup_init(void *addr, enum debug_obj_state state)
419	{
420	struct work_struct *work = addr;
421
422	switch (state) {
423	case ODEBUG_STATE_ACTIVE:
424	cancel_work_sync(work);
425	debug_object_init(work, &work_debug_descr);
426	return 1;
427	default:
428	return 0;
429	}
430	}
431
432	/*
433	* fixup_activate is called when:
434	* - an active object is activated
435	* - an unknown object is activated (might be a statically initialized object)
436	*/
437	static int work_fixup_activate(void *addr, enum debug_obj_state state)
438	{
439	struct work_struct *work = addr;
440
441	switch (state) {
442
443	case ODEBUG_STATE_NOTAVAILABLE:
444	/*
445	* This is not really a fixup. The work struct was
446	* statically initialized. We just make sure that it
447	* is tracked in the object tracker.
448	*/
449	if (test_bit(WORK_STRUCT_STATIC_BIT, work_data_bits(work))) {
450	debug_object_init(work, &work_debug_descr);
451	debug_object_activate(work, &work_debug_descr);
452	return 0;
453	}
454	WARN_ON_ONCE(1);
455	return 0;
456
457	case ODEBUG_STATE_ACTIVE:
458	WARN_ON(1);
459
460	default:
461	return 0;
462	}
463	}
464
465	/*
466	* fixup_free is called when:
467	* - an active object is freed
468	*/
469	static int work_fixup_free(void *addr, enum debug_obj_state state)
470	{
471	struct work_struct *work = addr;
472
473	switch (state) {
474	case ODEBUG_STATE_ACTIVE:
475	cancel_work_sync(work);
476	debug_object_free(work, &work_debug_descr);
477	return 1;
478	default:
479	return 0;
480	}
481	}
482
483	static struct debug_obj_descr work_debug_descr = {
484	.name = "work_struct",
485	.debug_hint = work_debug_hint,
486	.fixup_init = work_fixup_init,
487	.fixup_activate = work_fixup_activate,
488	.fixup_free = work_fixup_free,
489	};
490
491	static inline void debug_work_activate(struct work_struct *work)
492	{
493	debug_object_activate(work, &work_debug_descr);
494	}
495
496	static inline void debug_work_deactivate(struct work_struct *work)
497	{
498	debug_object_deactivate(work, &work_debug_descr);
499	}
500
501	void __init_work(struct work_struct *work, int onstack)
502	{
503	if (onstack)
504	debug_object_init_on_stack(work, &work_debug_descr);
505	else
506	debug_object_init(work, &work_debug_descr);
507	}
508	EXPORT_SYMBOL_GPL(__init_work);
509
510	void destroy_work_on_stack(struct work_struct *work)
511	{
512	debug_object_free(work, &work_debug_descr);
513	}
514	EXPORT_SYMBOL_GPL(destroy_work_on_stack);
515
516	#else
517	static inline void debug_work_activate(struct work_struct *work) { }
518	static inline void debug_work_deactivate(struct work_struct *work) { }
519	#endif
520
521	/* allocate ID and assign it to @pool */
522	static int worker_pool_assign_id(struct worker_pool *pool)
523	{
524	int ret;
525
526	lockdep_assert_held(&wq_pool_mutex);
527
528	ret = idr_alloc(&worker_pool_idr, pool, 0, 0, GFP_KERNEL);
529	if (ret >= 0) {
530	pool->id = ret;
531	return 0;
532	}
533	return ret;
534	}
535
536	/**
537	* unbound_pwq_by_node - return the unbound pool_workqueue for the given node
538	* @wq: the target workqueue
539	* @node: the node ID
540	*
541	* This must be called either with pwq_lock held or sched RCU read locked.
542	* If the pwq needs to be used beyond the locking in effect, the caller is
543	* responsible for guaranteeing that the pwq stays online.
544	*
545	* Return: The unbound pool_workqueue for @node.
546	*/
547	static struct pool_workqueue unbound_pwq_by_node(struct workqueue_struct wq,
548	int node)
549	{
550	assert_rcu_or_wq_mutex(wq);
551	return rcu_dereference_raw(wq->numa_pwq_tbl[node]);
552	}
553
554	static unsigned int work_color_to_flags(int color)
555	{
556	return color << WORK_STRUCT_COLOR_SHIFT;
557	}
558
559	static int get_work_color(struct work_struct *work)
560	{
561	return (*work_data_bits(work) >> WORK_STRUCT_COLOR_SHIFT) &
562	((1 << WORK_STRUCT_COLOR_BITS) - 1);
563	}
564
565	static int work_next_color(int color)
566	{
567	return (color + 1) % WORK_NR_COLORS;
568	}
569
570	/*
571	* While queued, %WORK_STRUCT_PWQ is set and non flag bits of a work's data
572	* contain the pointer to the queued pwq. Once execution starts, the flag
573	* is cleared and the high bits contain OFFQ flags and pool ID.
574	*
575	* set_work_pwq(), set_work_pool_and_clear_pending(), mark_work_canceling()
576	* and clear_work_data() can be used to set the pwq, pool or clear
577	* work->data. These functions should only be called while the work is
578	* owned - ie. while the PENDING bit is set.
579	*
580	* get_work_pool() and get_work_pwq() can be used to obtain the pool or pwq
581	* corresponding to a work. Pool is available once the work has been
582	* queued anywhere after initialization until it is sync canceled. pwq is
583	* available only while the work item is queued.
584	*
585	* %WORK_OFFQ_CANCELING is used to mark a work item which is being
586	* canceled. While being canceled, a work item may have its PENDING set
587	* but stay off timer and worklist for arbitrarily long and nobody should
588	* try to steal the PENDING bit.
589	*/
590	static inline void set_work_data(struct work_struct *work, unsigned long data,
591	unsigned long flags)
592	{
593	WARN_ON_ONCE(!work_pending(work));
594	atomic_long_set(&work->data, data \| flags \| work_static(work));
595	}
596
597	static void set_work_pwq(struct work_struct work, struct pool_workqueue pwq,
598	unsigned long extra_flags)
599	{
600	set_work_data(work, (unsigned long)pwq,
601	WORK_STRUCT_PENDING \| WORK_STRUCT_PWQ \| extra_flags);
602	}
603
604	static void set_work_pool_and_keep_pending(struct work_struct *work,
605	int pool_id)
606	{
607	set_work_data(work, (unsigned long)pool_id << WORK_OFFQ_POOL_SHIFT,
608	WORK_STRUCT_PENDING);
609	}
610
611	static void set_work_pool_and_clear_pending(struct work_struct *work,
612	int pool_id)
613	{
614	/*
615	* The following wmb is paired with the implied mb in
616	* test_and_set_bit(PENDING) and ensures all updates to @work made
617	* here are visible to and precede any updates by the next PENDING
618	* owner.
619	*/
620	smp_wmb();
621	set_work_data(work, (unsigned long)pool_id << WORK_OFFQ_POOL_SHIFT, 0);
622	}
623
624	static void clear_work_data(struct work_struct *work)
625	{
626	smp_wmb(); /* see set_work_pool_and_clear_pending() */
627	set_work_data(work, WORK_STRUCT_NO_POOL, 0);
628	}
629
630	static struct pool_workqueue get_work_pwq(struct work_struct work)
631	{
632	unsigned long data = atomic_long_read(&work->data);
633
634	if (data & WORK_STRUCT_PWQ)
635	return (void *)(data & WORK_STRUCT_WQ_DATA_MASK);
636	else
637	return NULL;
638	}
639
640	/**
641	* get_work_pool - return the worker_pool a given work was associated with
642	* @work: the work item of interest
643	*
644	* Pools are created and destroyed under wq_pool_mutex, and allows read
645	* access under sched-RCU read lock. As such, this function should be
646	* called under wq_pool_mutex or with preemption disabled.
647	*
648	* All fields of the returned pool are accessible as long as the above
649	* mentioned locking is in effect. If the returned pool needs to be used
650	* beyond the critical section, the caller is responsible for ensuring the
651	* returned pool is and stays online.
652	*
653	* Return: The worker_pool @work was last associated with. %NULL if none.
654	*/
655	static struct worker_pool get_work_pool(struct work_struct work)
656	{
657	unsigned long data = atomic_long_read(&work->data);
658	int pool_id;
659
660	assert_rcu_or_pool_mutex();
661
662	if (data & WORK_STRUCT_PWQ)
663	return ((struct pool_workqueue *)
664	(data & WORK_STRUCT_WQ_DATA_MASK))->pool;
665
666	pool_id = data >> WORK_OFFQ_POOL_SHIFT;
667	if (pool_id == WORK_OFFQ_POOL_NONE)
668	return NULL;
669
670	return idr_find(&worker_pool_idr, pool_id);
671	}
672
673	/**
674	* get_work_pool_id - return the worker pool ID a given work is associated with
675	* @work: the work item of interest
676	*
677	* Return: The worker_pool ID @work was last associated with.
678	* %WORK_OFFQ_POOL_NONE if none.
679	*/
680	static int get_work_pool_id(struct work_struct *work)
681	{
682	unsigned long data = atomic_long_read(&work->data);
683
684	if (data & WORK_STRUCT_PWQ)
685	return ((struct pool_workqueue *)
686	(data & WORK_STRUCT_WQ_DATA_MASK))->pool->id;
687
688	return data >> WORK_OFFQ_POOL_SHIFT;
689	}
690
691	static void mark_work_canceling(struct work_struct *work)
692	{
693	unsigned long pool_id = get_work_pool_id(work);
694
695	pool_id <<= WORK_OFFQ_POOL_SHIFT;
696	set_work_data(work, pool_id \| WORK_OFFQ_CANCELING, WORK_STRUCT_PENDING);
697	}
698
699	static bool work_is_canceling(struct work_struct *work)
700	{
701	unsigned long data = atomic_long_read(&work->data);
702
703	return !(data & WORK_STRUCT_PWQ) && (data & WORK_OFFQ_CANCELING);
704	}
705
706	/*
707	* Policy functions. These define the policies on how the global worker
708	* pools are managed. Unless noted otherwise, these functions assume that
709	* they're being called with pool->lock held.
710	*/
711
712	static bool __need_more_worker(struct worker_pool *pool)
713	{
714	return !atomic_read(&pool->nr_running);
715	}
716
717	/*
718	* Need to wake up a worker? Called from anything but currently
719	* running workers.
720	*
721	* Note that, because unbound workers never contribute to nr_running, this
722	* function will always return %true for unbound pools as long as the
723	* worklist isn't empty.
724	*/
725	static bool need_more_worker(struct worker_pool *pool)
726	{
727	return !list_empty(&pool->worklist) && __need_more_worker(pool);
728	}
729
730	/* Can I start working? Called from busy but !running workers. */
731	static bool may_start_working(struct worker_pool *pool)
732	{
733	return pool->nr_idle;
734	}
735
736	/* Do I need to keep working? Called from currently running workers. */
737	static bool keep_working(struct worker_pool *pool)
738	{
739	return !list_empty(&pool->worklist) &&
740	atomic_read(&pool->nr_running) <= 1;
741	}
742
743	/* Do we need a new worker? Called from manager. */
744	static bool need_to_create_worker(struct worker_pool *pool)
745	{
746	return need_more_worker(pool) && !may_start_working(pool);
747	}
748
749	/* Do I need to be the manager? */
750	static bool need_to_manage_workers(struct worker_pool *pool)
751	{
752	return need_to_create_worker(pool) \|\|
753	(pool->flags & POOL_MANAGE_WORKERS);
754	}
755
756	/* Do we have too many workers and should some go away? */
757	static bool too_many_workers(struct worker_pool *pool)
758	{
759	bool managing = mutex_is_locked(&pool->manager_arb);
760	int nr_idle = pool->nr_idle + managing; /* manager is considered idle */
761	int nr_busy = pool->nr_workers - nr_idle;
762
763	/*
764	* nr_idle and idle_list may disagree if idle rebinding is in
765	* progress. Never return %true if idle_list is empty.
766	*/
767	if (list_empty(&pool->idle_list))
768	return false;
769
770	return nr_idle > 2 && (nr_idle - 2) * MAX_IDLE_WORKERS_RATIO >= nr_busy;
771	}
772
773	/*
774	* Wake up functions.
775	*/
776
777	/* Return the first worker. Safe with preemption disabled */
778	static struct worker first_worker(struct worker_pool pool)
779	{
780	if (unlikely(list_empty(&pool->idle_list)))
781	return NULL;
782
783	return list_first_entry(&pool->idle_list, struct worker, entry);
784	}
785
786	/**
787	* wake_up_worker - wake up an idle worker
788	* @pool: worker pool to wake worker from
789	*
790	* Wake up the first idle worker of @pool.
791	*
792	* CONTEXT:
793	* spin_lock_irq(pool->lock).
794	*/
795	static void wake_up_worker(struct worker_pool *pool)
796	{
797	struct worker *worker = first_worker(pool);
798
799	if (likely(worker))
800	wake_up_process(worker->task);
801	}
802
803	/**
804	* wq_worker_waking_up - a worker is waking up
805	* @task: task waking up
806	* @cpu: CPU @task is waking up to
807	*
808	* This function is called during try_to_wake_up() when a worker is
809	* being awoken.
810	*
811	* CONTEXT:
812	* spin_lock_irq(rq->lock)
813	*/
814	void wq_worker_waking_up(struct task_struct *task, int cpu)
815	{
816	struct worker *worker = kthread_data(task);
817
818	if (!(worker->flags & WORKER_NOT_RUNNING)) {
819	WARN_ON_ONCE(worker->pool->cpu != cpu);
820	atomic_inc(&worker->pool->nr_running);
821	}
822	}
823
824	/**
825	* wq_worker_sleeping - a worker is going to sleep
826	* @task: task going to sleep
827	* @cpu: CPU in question, must be the current CPU number
828	*
829	* This function is called during schedule() when a busy worker is
830	* going to sleep. Worker on the same cpu can be woken up by
831	* returning pointer to its task.
832	*
833	* CONTEXT:
834	* spin_lock_irq(rq->lock)
835	*
836	* Return:
837	* Worker task on @cpu to wake up, %NULL if none.
838	*/
839	struct task_struct wq_worker_sleeping(struct task_struct task, int cpu)
840	{
841	struct worker worker = kthread_data(task), to_wakeup = NULL;
842	struct worker_pool *pool;
843
844	/*
845	* Rescuers, which may not have all the fields set up like normal
846	* workers, also reach here, let's not access anything before
847	* checking NOT_RUNNING.
848	*/
849	if (worker->flags & WORKER_NOT_RUNNING)
850	return NULL;
851
852	pool = worker->pool;
853
854	/* this can only happen on the local cpu */
855	if (WARN_ON_ONCE(cpu != raw_smp_processor_id()))
856	return NULL;
857
858	/*
859	* The counterpart of the following dec_and_test, implied mb,
860	* worklist not empty test sequence is in insert_work().
861	* Please read comment there.
862	*
863	* NOT_RUNNING is clear. This means that we're bound to and
864	* running on the local cpu w/ rq lock held and preemption
865	* disabled, which in turn means that none else could be
866	* manipulating idle_list, so dereferencing idle_list without pool
867	* lock is safe.
868	*/
869	if (atomic_dec_and_test(&pool->nr_running) &&
870	!list_empty(&pool->worklist))
871	to_wakeup = first_worker(pool);
872	return to_wakeup ? to_wakeup->task : NULL;
873	}
874
875	/**
876	* worker_set_flags - set worker flags and adjust nr_running accordingly
877	* @worker: self
878	* @flags: flags to set
879	* @wakeup: wakeup an idle worker if necessary
880	*
881	* Set @flags in @worker->flags and adjust nr_running accordingly. If
882	* nr_running becomes zero and @wakeup is %true, an idle worker is
883	* woken up.
884	*
885	* CONTEXT:
886	* spin_lock_irq(pool->lock)
887	*/
888	static inline void worker_set_flags(struct worker *worker, unsigned int flags,
889	bool wakeup)
890	{
891	struct worker_pool *pool = worker->pool;
892
893	WARN_ON_ONCE(worker->task != current);
894
895	/*
896	* If transitioning into NOT_RUNNING, adjust nr_running and
897	* wake up an idle worker as necessary if requested by
898	* @wakeup.
899	*/
900	if ((flags & WORKER_NOT_RUNNING) &&
901	!(worker->flags & WORKER_NOT_RUNNING)) {
902	if (wakeup) {
903	if (atomic_dec_and_test(&pool->nr_running) &&
904	!list_empty(&pool->worklist))
905	wake_up_worker(pool);
906	} else
907	atomic_dec(&pool->nr_running);
908	}
909
910	worker->flags \|= flags;
911	}
912
913	/**
914	* worker_clr_flags - clear worker flags and adjust nr_running accordingly
915	* @worker: self
916	* @flags: flags to clear
917	*
918	* Clear @flags in @worker->flags and adjust nr_running accordingly.
919	*
920	* CONTEXT:
921	* spin_lock_irq(pool->lock)
922	*/
923	static inline void worker_clr_flags(struct worker *worker, unsigned int flags)
924	{
925	struct worker_pool *pool = worker->pool;
926	unsigned int oflags = worker->flags;
927
928	WARN_ON_ONCE(worker->task != current);
929
930	worker->flags &= ~flags;
931
932	/*
933	* If transitioning out of NOT_RUNNING, increment nr_running. Note
934	* that the nested NOT_RUNNING is not a noop. NOT_RUNNING is mask
935	* of multiple flags, not a single flag.
936	*/
937	if ((flags & WORKER_NOT_RUNNING) && (oflags & WORKER_NOT_RUNNING))
938	if (!(worker->flags & WORKER_NOT_RUNNING))
939	atomic_inc(&pool->nr_running);
940	}
941
942	/**
943	* find_worker_executing_work - find worker which is executing a work
944	* @pool: pool of interest
945	* @work: work to find worker for
946	*
947	* Find a worker which is executing @work on @pool by searching
948	* @pool->busy_hash which is keyed by the address of @work. For a worker
949	* to match, its current execution should match the address of @work and
950	* its work function. This is to avoid unwanted dependency between
951	* unrelated work executions through a work item being recycled while still
952	* being executed.
953	*
954	* This is a bit tricky. A work item may be freed once its execution
955	* starts and nothing prevents the freed area from being recycled for
956	* another work item. If the same work item address ends up being reused
957	* before the original execution finishes, workqueue will identify the
958	* recycled work item as currently executing and make it wait until the
959	* current execution finishes, introducing an unwanted dependency.
960	*
961	* This function checks the work item address and work function to avoid
962	* false positives. Note that this isn't complete as one may construct a
963	* work function which can introduce dependency onto itself through a
964	* recycled work item. Well, if somebody wants to shoot oneself in the
965	* foot that badly, there's only so much we can do, and if such deadlock
966	* actually occurs, it should be easy to locate the culprit work function.
967	*
968	* CONTEXT:
969	* spin_lock_irq(pool->lock).
970	*
971	* Return:
972	* Pointer to worker which is executing @work if found, %NULL
973	* otherwise.
974	*/
975	static struct worker find_worker_executing_work(struct worker_pool pool,
976	struct work_struct *work)
977	{
978	struct worker *worker;
979
980	hash_for_each_possible(pool->busy_hash, worker, hentry,
981	(unsigned long)work)
982	if (worker->current_work == work &&
983	worker->current_func == work->func)
984	return worker;
985
986	return NULL;
987	}
988
989	/**
990	* move_linked_works - move linked works to a list
991	* @work: start of series of works to be scheduled
992	* @head: target list to append @work to
993	* @nextp: out paramter for nested worklist walking
994	*
995	* Schedule linked works starting from @work to @head. Work series to
996	* be scheduled starts at @work and includes any consecutive work with
997	* WORK_STRUCT_LINKED set in its predecessor.
998	*
999	* If @nextp is not NULL, it's updated to point to the next work of
1000	* the last scheduled work. This allows move_linked_works() to be
1001	* nested inside outer list_for_each_entry_safe().
1002	*
1003	* CONTEXT:
1004	* spin_lock_irq(pool->lock).
1005	*/
1006	static void move_linked_works(struct work_struct work, struct list_head head,
1007	struct work_struct **nextp)
1008	{
1009	struct work_struct *n;
1010
1011	/*
1012	* Linked worklist will always end before the end of the list,
1013	* use NULL for list head.
1014	*/
1015	list_for_each_entry_safe_from(work, n, NULL, entry) {
1016	list_move_tail(&work->entry, head);
1017	if (!(*work_data_bits(work) & WORK_STRUCT_LINKED))
1018	break;
1019	}
1020
1021	/*
1022	* If we're already inside safe list traversal and have moved
1023	* multiple works to the scheduled queue, the next position
1024	* needs to be updated.
1025	*/
1026	if (nextp)
1027	*nextp = n;
1028	}
1029
1030	/**
1031	* get_pwq - get an extra reference on the specified pool_workqueue
1032	* @pwq: pool_workqueue to get
1033	*
1034	* Obtain an extra reference on @pwq. The caller should guarantee that
1035	* @pwq has positive refcnt and be holding the matching pool->lock.
1036	*/
1037	static void get_pwq(struct pool_workqueue *pwq)
1038	{
1039	lockdep_assert_held(&pwq->pool->lock);
1040	WARN_ON_ONCE(pwq->refcnt <= 0);
1041	pwq->refcnt++;
1042	}
1043
1044	/**
1045	* put_pwq - put a pool_workqueue reference
1046	* @pwq: pool_workqueue to put
1047	*
1048	* Drop a reference of @pwq. If its refcnt reaches zero, schedule its
1049	* destruction. The caller should be holding the matching pool->lock.
1050	*/
1051	static void put_pwq(struct pool_workqueue *pwq)
1052	{
1053	lockdep_assert_held(&pwq->pool->lock);
1054	if (likely(--pwq->refcnt))
1055	return;
1056	if (WARN_ON_ONCE(!(pwq->wq->flags & WQ_UNBOUND)))
1057	return;
1058	/*
1059	* @pwq can't be released under pool->lock, bounce to
1060	* pwq_unbound_release_workfn(). This never recurses on the same
1061	* pool->lock as this path is taken only for unbound workqueues and
1062	* the release work item is scheduled on a per-cpu workqueue. To
1063	* avoid lockdep warning, unbound pool->locks are given lockdep
1064	* subclass of 1 in get_unbound_pool().
1065	*/
1066	schedule_work(&pwq->unbound_release_work);
1067	}
1068
1069	/**
1070	* put_pwq_unlocked - put_pwq() with surrounding pool lock/unlock
1071	* @pwq: pool_workqueue to put (can be %NULL)
1072	*
1073	* put_pwq() with locking. This function also allows %NULL @pwq.
1074	*/
1075	static void put_pwq_unlocked(struct pool_workqueue *pwq)
1076	{
1077	if (pwq) {
1078	/*
1079	* As both pwqs and pools are sched-RCU protected, the
1080	* following lock operations are safe.
1081	*/
1082	spin_lock_irq(&pwq->pool->lock);
1083	put_pwq(pwq);
1084	spin_unlock_irq(&pwq->pool->lock);
1085	}
1086	}
1087
1088	static void pwq_activate_delayed_work(struct work_struct *work)
1089	{
1090	struct pool_workqueue *pwq = get_work_pwq(work);
1091
1092	trace_workqueue_activate_work(work);
1093	move_linked_works(work, &pwq->pool->worklist, NULL);
1094	__clear_bit(WORK_STRUCT_DELAYED_BIT, work_data_bits(work));
1095	pwq->nr_active++;
1096	}
1097
1098	static void pwq_activate_first_delayed(struct pool_workqueue *pwq)
1099	{
1100	struct work_struct *work = list_first_entry(&pwq->delayed_works,
1101	struct work_struct, entry);
1102
1103	pwq_activate_delayed_work(work);
1104	}
1105
1106	/**
1107	* pwq_dec_nr_in_flight - decrement pwq's nr_in_flight
1108	* @pwq: pwq of interest
1109	* @color: color of work which left the queue
1110	*
1111	* A work either has completed or is removed from pending queue,
1112	* decrement nr_in_flight of its pwq and handle workqueue flushing.
1113	*
1114	* CONTEXT:
1115	* spin_lock_irq(pool->lock).
1116	*/
1117	static void pwq_dec_nr_in_flight(struct pool_workqueue *pwq, int color)
1118	{
1119	/* uncolored work items don't participate in flushing or nr_active */
1120	if (color == WORK_NO_COLOR)
1121	goto out_put;
1122
1123	pwq->nr_in_flight[color]--;
1124
1125	pwq->nr_active--;
1126	if (!list_empty(&pwq->delayed_works)) {
1127	/* one down, submit a delayed one */
1128	if (pwq->nr_active < pwq->max_active)
1129	pwq_activate_first_delayed(pwq);
1130	}
1131
1132	/* is flush in progress and are we at the flushing tip? */
1133	if (likely(pwq->flush_color != color))
1134	goto out_put;
1135
1136	/* are there still in-flight works? */
1137	if (pwq->nr_in_flight[color])
1138	goto out_put;
1139
1140	/* this pwq is done, clear flush_color */
1141	pwq->flush_color = -1;
1142
1143	/*
1144	* If this was the last pwq, wake up the first flusher. It
1145	* will handle the rest.
1146	*/
1147	if (atomic_dec_and_test(&pwq->wq->nr_pwqs_to_flush))
1148	complete(&pwq->wq->first_flusher->done);
1149	out_put:
1150	put_pwq(pwq);
1151	}
1152
1153	/**
1154	* try_to_grab_pending - steal work item from worklist and disable irq
1155	* @work: work item to steal
1156	* @is_dwork: @work is a delayed_work
1157	* @flags: place to store irq state
1158	*
1159	* Try to grab PENDING bit of @work. This function can handle @work in any
1160	* stable state - idle, on timer or on worklist.
1161	*
1162	* Return:
1163	* 1 if @work was pending and we successfully stole PENDING
1164	* 0 if @work was idle and we claimed PENDING
1165	* -EAGAIN if PENDING couldn't be grabbed at the moment, safe to busy-retry
1166	* -ENOENT if someone else is canceling @work, this state may persist
1167	* for arbitrarily long
1168	*
1169	* Note:
1170	* On >= 0 return, the caller owns @work's PENDING bit. To avoid getting
1171	* interrupted while holding PENDING and @work off queue, irq must be
1172	* disabled on entry. This, combined with delayed_work->timer being
1173	* irqsafe, ensures that we return -EAGAIN for finite short period of time.
1174	*
1175	* On successful return, >= 0, irq is disabled and the caller is
1176	* responsible for releasing it using local_irq_restore(*@flags).
1177	*
1178	* This function is safe to call from any context including IRQ handler.
1179	*/
1180	static int try_to_grab_pending(struct work_struct *work, bool is_dwork,
1181	unsigned long *flags)
1182	{
1183	struct worker_pool *pool;
1184	struct pool_workqueue *pwq;
1185
1186	local_irq_save(*flags);
1187
1188	/* try to steal the timer if it exists */
1189	if (is_dwork) {
1190	struct delayed_work *dwork = to_delayed_work(work);
1191
1192	/*
1193	* dwork->timer is irqsafe. If del_timer() fails, it's
1194	* guaranteed that the timer is not queued anywhere and not
1195	* running on the local CPU.
1196	*/
1197	if (likely(del_timer(&dwork->timer)))
1198	return 1;
1199	}
1200
1201	/* try to claim PENDING the normal way */
1202	if (!test_and_set_bit(WORK_STRUCT_PENDING_BIT, work_data_bits(work)))
1203	return 0;
1204
1205	/*
1206	* The queueing is in progress, or it is already queued. Try to
1207	* steal it from ->worklist without clearing WORK_STRUCT_PENDING.
1208	*/
1209	pool = get_work_pool(work);
1210	if (!pool)
1211	goto fail;
1212
1213	spin_lock(&pool->lock);
1214	/*
1215	* work->data is guaranteed to point to pwq only while the work
1216	* item is queued on pwq->wq, and both updating work->data to point
1217	* to pwq on queueing and to pool on dequeueing are done under
1218	* pwq->pool->lock. This in turn guarantees that, if work->data
1219	* points to pwq which is associated with a locked pool, the work
1220	* item is currently queued on that pool.
1221	*/
1222	pwq = get_work_pwq(work);
1223	if (pwq && pwq->pool == pool) {
1224	debug_work_deactivate(work);
1225
1226	/*
1227	* A delayed work item cannot be grabbed directly because
1228	* it might have linked NO_COLOR work items which, if left
1229	* on the delayed_list, will confuse pwq->nr_active
1230	* management later on and cause stall. Make sure the work
1231	* item is activated before grabbing.
1232	*/
1233	if (*work_data_bits(work) & WORK_STRUCT_DELAYED)
1234	pwq_activate_delayed_work(work);
1235
1236	list_del_init(&work->entry);
1237	pwq_dec_nr_in_flight(get_work_pwq(work), get_work_color(work));
1238
1239	/* work->data points to pwq iff queued, point to pool */
1240	set_work_pool_and_keep_pending(work, pool->id);
1241
1242	spin_unlock(&pool->lock);
1243	return 1;
1244	}
1245	spin_unlock(&pool->lock);
1246	fail:
1247	local_irq_restore(*flags);
1248	if (work_is_canceling(work))
1249	return -ENOENT;
1250	cpu_relax();
1251	return -EAGAIN;
1252	}
1253
1254	/**
1255	* insert_work - insert a work into a pool
1256	* @pwq: pwq @work belongs to
1257	* @work: work to insert
1258	* @head: insertion point
1259	* @extra_flags: extra WORK_STRUCT_* flags to set
1260	*
1261	* Insert @work which belongs to @pwq after @head. @extra_flags is or'd to
1262	* work_struct flags.
1263	*
1264	* CONTEXT:
1265	* spin_lock_irq(pool->lock).
1266	*/
1267	static void insert_work(struct pool_workqueue pwq, struct work_struct work,
1268	struct list_head *head, unsigned int extra_flags)
1269	{
1270	struct worker_pool *pool = pwq->pool;
1271
1272	/* we own @work, set data and link */
1273	set_work_pwq(work, pwq, extra_flags);
1274	list_add_tail(&work->entry, head);
1275	get_pwq(pwq);
1276
1277	/*
1278	* Ensure either wq_worker_sleeping() sees the above
1279	* list_add_tail() or we see zero nr_running to avoid workers lying
1280	* around lazily while there are works to be processed.
1281	*/
1282	smp_mb();
1283
1284	if (__need_more_worker(pool))
1285	wake_up_worker(pool);
1286	}
1287
1288	/*
1289	* Test whether @work is being queued from another work executing on the
1290	* same workqueue.
1291	*/
1292	static bool is_chained_work(struct workqueue_struct *wq)
1293	{
1294	struct worker *worker;
1295
1296	worker = current_wq_worker();
1297	/*
1298	* Return %true iff I'm a worker execuing a work item on @wq. If
1299	* I'm @worker, it's safe to dereference it without locking.
1300	*/
1301	return worker && worker->current_pwq->wq == wq;
1302	}
1303
1304	static void __queue_work(int cpu, struct workqueue_struct *wq,
1305	struct work_struct *work)
1306	{
1307	struct pool_workqueue *pwq;
1308	struct worker_pool *last_pool;
1309	struct list_head *worklist;
1310	unsigned int work_flags;
1311	unsigned int req_cpu = cpu;
1312
1313	/*
1314	* While a work item is PENDING && off queue, a task trying to
1315	* steal the PENDING will busy-loop waiting for it to either get
1316	* queued or lose PENDING. Grabbing PENDING and queueing should
1317	* happen with IRQ disabled.
1318	*/
1319	WARN_ON_ONCE(!irqs_disabled());
1320
1321	debug_work_activate(work);
1322
1323	/* if dying, only works from the same workqueue are allowed */
1324	if (unlikely(wq->flags & __WQ_DRAINING) &&
1325	WARN_ON_ONCE(!is_chained_work(wq)))
1326	return;
1327	retry:
1328	if (req_cpu == WORK_CPU_UNBOUND)
1329	cpu = raw_smp_processor_id();
1330
1331	/* pwq which will be used unless @work is executing elsewhere */
1332	if (!(wq->flags & WQ_UNBOUND))
1333	pwq = per_cpu_ptr(wq->cpu_pwqs, cpu);
1334	else
1335	pwq = unbound_pwq_by_node(wq, cpu_to_node(cpu));
1336
1337	/*
1338	* If @work was previously on a different pool, it might still be
1339	* running there, in which case the work needs to be queued on that
1340	* pool to guarantee non-reentrancy.
1341	*/
1342	last_pool = get_work_pool(work);
1343	if (last_pool && last_pool != pwq->pool) {
1344	struct worker *worker;
1345
1346	spin_lock(&last_pool->lock);
1347
1348	worker = find_worker_executing_work(last_pool, work);
1349
1350	if (worker && worker->current_pwq->wq == wq) {
1351	pwq = worker->current_pwq;
1352	} else {
1353	/* meh... not running there, queue here */
1354	spin_unlock(&last_pool->lock);
1355	spin_lock(&pwq->pool->lock);
1356	}
1357	} else {
1358	spin_lock(&pwq->pool->lock);
1359	}
1360
1361	/*
1362	* pwq is determined and locked. For unbound pools, we could have
1363	* raced with pwq release and it could already be dead. If its
1364	* refcnt is zero, repeat pwq selection. Note that pwqs never die
1365	* without another pwq replacing it in the numa_pwq_tbl or while
1366	* work items are executing on it, so the retrying is guaranteed to
1367	* make forward-progress.
1368	*/
1369	if (unlikely(!pwq->refcnt)) {
1370	if (wq->flags & WQ_UNBOUND) {
1371	spin_unlock(&pwq->pool->lock);
1372	cpu_relax();
1373	goto retry;
1374	}
1375	/* oops */
1376	WARN_ONCE(true, "workqueue: per-cpu pwq for %s on cpu%d has 0 refcnt",
1377	wq->name, cpu);
1378	}
1379
1380	/* pwq determined, queue */
1381	trace_workqueue_queue_work(req_cpu, pwq, work);
1382
1383	if (WARN_ON(!list_empty(&work->entry))) {
1384	spin_unlock(&pwq->pool->lock);
1385	return;
1386	}
1387
1388	pwq->nr_in_flight[pwq->work_color]++;
1389	work_flags = work_color_to_flags(pwq->work_color);
1390
1391	if (likely(pwq->nr_active < pwq->max_active)) {
1392	trace_workqueue_activate_work(work);
1393	pwq->nr_active++;
1394	worklist = &pwq->pool->worklist;
1395	} else {
1396	work_flags \|= WORK_STRUCT_DELAYED;
1397	worklist = &pwq->delayed_works;
1398	}
1399
1400	insert_work(pwq, work, worklist, work_flags);
1401
1402	spin_unlock(&pwq->pool->lock);
1403	}
1404
1405	/**
1406	* queue_work_on - queue work on specific cpu
1407	* @cpu: CPU number to execute work on
1408	* @wq: workqueue to use
1409	* @work: work to queue
1410	*
1411	* We queue the work to a specific CPU, the caller must ensure it
1412	* can't go away.
1413	*
1414	* Return: %false if @work was already on a queue, %true otherwise.
1415	*/
1416	bool queue_work_on(int cpu, struct workqueue_struct *wq,
1417	struct work_struct *work)
1418	{
1419	bool ret = false;
1420	unsigned long flags;
1421
1422	local_irq_save(flags);
1423
1424	if (!test_and_set_bit(WORK_STRUCT_PENDING_BIT, work_data_bits(work))) {
1425	__queue_work(cpu, wq, work);
1426	ret = true;
1427	}
1428
1429	local_irq_restore(flags);
1430	return ret;
1431	}
1432	EXPORT_SYMBOL(queue_work_on);
1433
1434	void delayed_work_timer_fn(unsigned long __data)
1435	{
1436	struct delayed_work dwork = (struct delayed_work )__data;
1437
1438	/* should have been called from irqsafe timer with irq already off */
1439	__queue_work(dwork->cpu, dwork->wq, &dwork->work);
1440	}
1441	EXPORT_SYMBOL(delayed_work_timer_fn);
1442
1443	static void __queue_delayed_work(int cpu, struct workqueue_struct *wq,
1444	struct delayed_work *dwork, unsigned long delay)
1445	{
1446	struct timer_list *timer = &dwork->timer;
1447	struct work_struct *work = &dwork->work;
1448
1449	WARN_ON_ONCE(timer->function != delayed_work_timer_fn \|\|
1450	timer->data != (unsigned long)dwork);
1451	WARN_ON_ONCE(timer_pending(timer));
1452	WARN_ON_ONCE(!list_empty(&work->entry));
1453
1454	/*
1455	* If @delay is 0, queue @dwork->work immediately. This is for
1456	* both optimization and correctness. The earliest @timer can
1457	* expire is on the closest next tick and delayed_work users depend
1458	* on that there's no such delay when @delay is 0.
1459	*/
1460	if (!delay) {
1461	__queue_work(cpu, wq, &dwork->work);
1462	return;
1463	}
1464
1465	timer_stats_timer_set_start_info(&dwork->timer);
1466
1467	dwork->wq = wq;
1468	dwork->cpu = cpu;
1469	timer->expires = jiffies + delay;
1470
1471	if (unlikely(cpu != WORK_CPU_UNBOUND))
1472	add_timer_on(timer, cpu);
1473	else
1474	add_timer(timer);
1475	}
1476
1477	/**
1478	* queue_delayed_work_on - queue work on specific CPU after delay
1479	* @cpu: CPU number to execute work on
1480	* @wq: workqueue to use
1481	* @dwork: work to queue
1482	* @delay: number of jiffies to wait before queueing
1483	*
1484	* Return: %false if @work was already on a queue, %true otherwise. If
1485	* @delay is zero and @dwork is idle, it will be scheduled for immediate
1486	* execution.
1487	*/
1488	bool queue_delayed_work_on(int cpu, struct workqueue_struct *wq,
1489	struct delayed_work *dwork, unsigned long delay)
1490	{
1491	struct work_struct *work = &dwork->work;
1492	bool ret = false;
1493	unsigned long flags;
1494
1495	/* read the comment in __queue_work() */
1496	local_irq_save(flags);
1497
1498	if (!test_and_set_bit(WORK_STRUCT_PENDING_BIT, work_data_bits(work))) {
1499	__queue_delayed_work(cpu, wq, dwork, delay);
1500	ret = true;
1501	}
1502
1503	local_irq_restore(flags);
1504	return ret;
1505	}
1506	EXPORT_SYMBOL(queue_delayed_work_on);
1507
1508	/**
1509	* mod_delayed_work_on - modify delay of or queue a delayed work on specific CPU
1510	* @cpu: CPU number to execute work on
1511	* @wq: workqueue to use
1512	* @dwork: work to queue
1513	* @delay: number of jiffies to wait before queueing
1514	*
1515	* If @dwork is idle, equivalent to queue_delayed_work_on(); otherwise,
1516	* modify @dwork's timer so that it expires after @delay. If @delay is
1517	* zero, @work is guaranteed to be scheduled immediately regardless of its
1518	* current state.
1519	*
1520	* Return: %false if @dwork was idle and queued, %true if @dwork was
1521	* pending and its timer was modified.
1522	*
1523	* This function is safe to call from any context including IRQ handler.
1524	* See try_to_grab_pending() for details.
1525	*/
1526	bool mod_delayed_work_on(int cpu, struct workqueue_struct *wq,
1527	struct delayed_work *dwork, unsigned long delay)
1528	{
1529	unsigned long flags;
1530	int ret;
1531
1532	do {
1533	ret = try_to_grab_pending(&dwork->work, true, &flags);
1534	} while (unlikely(ret == -EAGAIN));
1535
1536	if (likely(ret >= 0)) {
1537	__queue_delayed_work(cpu, wq, dwork, delay);
1538	local_irq_restore(flags);
1539	}
1540
1541	/* -ENOENT from try_to_grab_pending() becomes %true */
1542	return ret;
1543	}
1544	EXPORT_SYMBOL_GPL(mod_delayed_work_on);
1545
1546	/**
1547	* worker_enter_idle - enter idle state
1548	* @worker: worker which is entering idle state
1549	*
1550	* @worker is entering idle state. Update stats and idle timer if
1551	* necessary.
1552	*
1553	* LOCKING:
1554	* spin_lock_irq(pool->lock).
1555	*/
1556	static void worker_enter_idle(struct worker *worker)
1557	{
1558	struct worker_pool *pool = worker->pool;
1559
1560	if (WARN_ON_ONCE(worker->flags & WORKER_IDLE) \|\|
1561	WARN_ON_ONCE(!list_empty(&worker->entry) &&
1562	(worker->hentry.next \|\| worker->hentry.pprev)))
1563	return;
1564
1565	/* can't use worker_set_flags(), also called from start_worker() */
1566	worker->flags \|= WORKER_IDLE;
1567	pool->nr_idle++;
1568	worker->last_active = jiffies;
1569
1570	/* idle_list is LIFO */
1571	list_add(&worker->entry, &pool->idle_list);
1572
1573	if (too_many_workers(pool) && !timer_pending(&pool->idle_timer))
1574	mod_timer(&pool->idle_timer, jiffies + IDLE_WORKER_TIMEOUT);
1575
1576	/*
1577	* Sanity check nr_running. Because wq_unbind_fn() releases
1578	* pool->lock between setting %WORKER_UNBOUND and zapping
1579	* nr_running, the warning may trigger spuriously. Check iff
1580	* unbind is not in progress.
1581	*/
1582	WARN_ON_ONCE(!(pool->flags & POOL_DISASSOCIATED) &&
1583	pool->nr_workers == pool->nr_idle &&
1584	atomic_read(&pool->nr_running));
1585	}
1586
1587	/**
1588	* worker_leave_idle - leave idle state
1589	* @worker: worker which is leaving idle state
1590	*
1591	* @worker is leaving idle state. Update stats.
1592	*
1593	* LOCKING:
1594	* spin_lock_irq(pool->lock).
1595	*/
1596	static void worker_leave_idle(struct worker *worker)
1597	{
1598	struct worker_pool *pool = worker->pool;
1599
1600	if (WARN_ON_ONCE(!(worker->flags & WORKER_IDLE)))
1601	return;
1602	worker_clr_flags(worker, WORKER_IDLE);
1603	pool->nr_idle--;
1604	list_del_init(&worker->entry);
1605	}
1606
1607	/**
1608	* worker_maybe_bind_and_lock - try to bind %current to worker_pool and lock it
1609	* @pool: target worker_pool
1610	*
1611	* Bind %current to the cpu of @pool if it is associated and lock @pool.
1612	*
1613	* Works which are scheduled while the cpu is online must at least be
1614	* scheduled to a worker which is bound to the cpu so that if they are
1615	* flushed from cpu callbacks while cpu is going down, they are
1616	* guaranteed to execute on the cpu.
1617	*
1618	* This function is to be used by unbound workers and rescuers to bind
1619	* themselves to the target cpu and may race with cpu going down or
1620	* coming online. kthread_bind() can't be used because it may put the
1621	* worker to already dead cpu and set_cpus_allowed_ptr() can't be used
1622	* verbatim as it's best effort and blocking and pool may be
1623	* [dis]associated in the meantime.
1624	*
1625	* This function tries set_cpus_allowed() and locks pool and verifies the
1626	* binding against %POOL_DISASSOCIATED which is set during
1627	* %CPU_DOWN_PREPARE and cleared during %CPU_ONLINE, so if the worker
1628	* enters idle state or fetches works without dropping lock, it can
1629	* guarantee the scheduling requirement described in the first paragraph.
1630	*
1631	* CONTEXT:
1632	* Might sleep. Called without any lock but returns with pool->lock
1633	* held.
1634	*
1635	* Return:
1636	* %true if the associated pool is online (@worker is successfully
1637	* bound), %false if offline.
1638	*/
1639	static bool worker_maybe_bind_and_lock(struct worker_pool *pool)
1640	__acquires(&pool->lock)
1641	{
1642	while (true) {
1643	/*
1644	* The following call may fail, succeed or succeed
1645	* without actually migrating the task to the cpu if
1646	* it races with cpu hotunplug operation. Verify
1647	* against POOL_DISASSOCIATED.
1648	*/
1649	if (!(pool->flags & POOL_DISASSOCIATED))
1650	set_cpus_allowed_ptr(current, pool->attrs->cpumask);
1651
1652	spin_lock_irq(&pool->lock);
1653	if (pool->flags & POOL_DISASSOCIATED)
1654	return false;
1655	if (task_cpu(current) == pool->cpu &&
1656	cpumask_equal(&current->cpus_allowed, pool->attrs->cpumask))
1657	return true;
1658	spin_unlock_irq(&pool->lock);
1659
1660	/*
1661	* We've raced with CPU hot[un]plug. Give it a breather
1662	* and retry migration. cond_resched() is required here;
1663	* otherwise, we might deadlock against cpu_stop trying to
1664	* bring down the CPU on non-preemptive kernel.
1665	*/
1666	cpu_relax();
1667	cond_resched();
1668	}
1669	}
1670
1671	static struct worker *alloc_worker(void)
1672	{
1673	struct worker *worker;
1674
1675	worker = kzalloc(sizeof(*worker), GFP_KERNEL);
1676	if (worker) {
1677	INIT_LIST_HEAD(&worker->entry);
1678	INIT_LIST_HEAD(&worker->scheduled);
1679	/* on creation a worker is in !idle && prep state */
1680	worker->flags = WORKER_PREP;
1681	}
1682	return worker;
1683	}
1684
1685	/**
1686	* create_worker - create a new workqueue worker
1687	* @pool: pool the new worker will belong to
1688	*
1689	* Create a new worker which is bound to @pool. The returned worker
1690	* can be started by calling start_worker() or destroyed using
1691	* destroy_worker().
1692	*
1693	* CONTEXT:
1694	* Might sleep. Does GFP_KERNEL allocations.
1695	*
1696	* Return:
1697	* Pointer to the newly created worker.
1698	*/
1699	static struct worker create_worker(struct worker_pool pool)
1700	{
1701	struct worker *worker = NULL;
1702	int id = -1;
1703	char id_buf[16];
1704
1705	lockdep_assert_held(&pool->manager_mutex);
1706
1707	/*
1708	* ID is needed to determine kthread name. Allocate ID first
1709	* without installing the pointer.
1710	*/
1711	idr_preload(GFP_KERNEL);
1712	spin_lock_irq(&pool->lock);
1713
1714	id = idr_alloc(&pool->worker_idr, NULL, 0, 0, GFP_NOWAIT);
1715
1716	spin_unlock_irq(&pool->lock);
1717	idr_preload_end();
1718	if (id < 0)
1719	goto fail;
1720
1721	worker = alloc_worker();
1722	if (!worker)
1723	goto fail;
1724
1725	worker->pool = pool;
1726	worker->id = id;
1727
1728	if (pool->cpu >= 0)
1729	snprintf(id_buf, sizeof(id_buf), "%d:%d%s", pool->cpu, id,
1730	pool->attrs->nice < 0 ? "H" : "");
1731	else
1732	snprintf(id_buf, sizeof(id_buf), "u%d:%d", pool->id, id);
1733
1734	worker->task = kthread_create_on_node(worker_thread, worker, pool->node,
1735	"kworker/%s", id_buf);
1736	if (IS_ERR(worker->task))
1737	goto fail;
1738
1739	/*
1740	* set_cpus_allowed_ptr() will fail if the cpumask doesn't have any
1741	* online CPUs. It'll be re-applied when any of the CPUs come up.
1742	*/
1743	set_user_nice(worker->task, pool->attrs->nice);
1744	set_cpus_allowed_ptr(worker->task, pool->attrs->cpumask);
1745
1746	/* prevent userland from meddling with cpumask of workqueue workers */
1747	worker->task->flags \|= PF_NO_SETAFFINITY;
1748
1749	/*
1750	* The caller is responsible for ensuring %POOL_DISASSOCIATED
1751	* remains stable across this function. See the comments above the
1752	* flag definition for details.
1753	*/
1754	if (pool->flags & POOL_DISASSOCIATED)
1755	worker->flags \|= WORKER_UNBOUND;
1756
1757	/* successful, commit the pointer to idr */
1758	spin_lock_irq(&pool->lock);
1759	idr_replace(&pool->worker_idr, worker, worker->id);
1760	spin_unlock_irq(&pool->lock);
1761
1762	return worker;
1763
1764	fail:
1765	if (id >= 0) {
1766	spin_lock_irq(&pool->lock);
1767	idr_remove(&pool->worker_idr, id);
1768	spin_unlock_irq(&pool->lock);
1769	}
1770	kfree(worker);
1771	return NULL;
1772	}
1773
1774	/**
1775	* start_worker - start a newly created worker
1776	* @worker: worker to start
1777	*
1778	* Make the pool aware of @worker and start it.
1779	*
1780	* CONTEXT:
1781	* spin_lock_irq(pool->lock).
1782	*/
1783	static void start_worker(struct worker *worker)
1784	{
1785	worker->flags \|= WORKER_STARTED;
1786	worker->pool->nr_workers++;
1787	worker_enter_idle(worker);
1788	wake_up_process(worker->task);
1789	}
1790
1791	/**
1792	* create_and_start_worker - create and start a worker for a pool
1793	* @pool: the target pool
1794	*
1795	* Grab the managership of @pool and create and start a new worker for it.
1796	*
1797	* Return: 0 on success. A negative error code otherwise.
1798	*/
1799	static int create_and_start_worker(struct worker_pool *pool)
1800	{
1801	struct worker *worker;
1802
1803	mutex_lock(&pool->manager_mutex);
1804
1805	worker = create_worker(pool);
1806	if (worker) {
1807	spin_lock_irq(&pool->lock);
1808	start_worker(worker);
1809	spin_unlock_irq(&pool->lock);
1810	}
1811
1812	mutex_unlock(&pool->manager_mutex);
1813
1814	return worker ? 0 : -ENOMEM;
1815	}
1816
1817	/**
1818	* destroy_worker - destroy a workqueue worker
1819	* @worker: worker to be destroyed
1820	*
1821	* Destroy @worker and adjust @pool stats accordingly.
1822	*
1823	* CONTEXT:
1824	* spin_lock_irq(pool->lock) which is released and regrabbed.
1825	*/
1826	static void destroy_worker(struct worker *worker)
1827	{
1828	struct worker_pool *pool = worker->pool;
1829
1830	lockdep_assert_held(&pool->manager_mutex);
1831	lockdep_assert_held(&pool->lock);
1832
1833	/* sanity check frenzy */
1834	if (WARN_ON(worker->current_work) \|\|
1835	WARN_ON(!list_empty(&worker->scheduled)))
1836	return;
1837
1838	if (worker->flags & WORKER_STARTED)
1839	pool->nr_workers--;
1840	if (worker->flags & WORKER_IDLE)
1841	pool->nr_idle--;
1842
1843	list_del_init(&worker->entry);
1844	worker->flags \|= WORKER_DIE;
1845
1846	idr_remove(&pool->worker_idr, worker->id);
1847
1848	spin_unlock_irq(&pool->lock);
1849
1850	kthread_stop(worker->task);
1851	kfree(worker);
1852
1853	spin_lock_irq(&pool->lock);
1854	}
1855
1856	static void idle_worker_timeout(unsigned long __pool)
1857	{
1858	struct worker_pool pool = (void )__pool;
1859
1860	spin_lock_irq(&pool->lock);
1861
1862	if (too_many_workers(pool)) {
1863	struct worker *worker;
1864	unsigned long expires;
1865
1866	/* idle_list is kept in LIFO order, check the last one */
1867	worker = list_entry(pool->idle_list.prev, struct worker, entry);
1868	expires = worker->last_active + IDLE_WORKER_TIMEOUT;
1869
1870	if (time_before(jiffies, expires))
1871	mod_timer(&pool->idle_timer, expires);
1872	else {
1873	/* it's been idle for too long, wake up manager */
1874	pool->flags \|= POOL_MANAGE_WORKERS;
1875	wake_up_worker(pool);
1876	}
1877	}
1878
1879	spin_unlock_irq(&pool->lock);
1880	}
1881
1882	static void send_mayday(struct work_struct *work)
1883	{
1884	struct pool_workqueue *pwq = get_work_pwq(work);
1885	struct workqueue_struct *wq = pwq->wq;
1886
1887	lockdep_assert_held(&wq_mayday_lock);
1888
1889	if (!wq->rescuer)
1890	return;
1891
1892	/* mayday mayday mayday */
1893	if (list_empty(&pwq->mayday_node)) {
1894	list_add_tail(&pwq->mayday_node, &wq->maydays);
1895	wake_up_process(wq->rescuer->task);
1896	}
1897	}
1898
1899	static void pool_mayday_timeout(unsigned long __pool)
1900	{
1901	struct worker_pool pool = (void )__pool;
1902	struct work_struct *work;
1903
1904	spin_lock_irq(&wq_mayday_lock); /* for wq->maydays */
1905	spin_lock(&pool->lock);
1906
1907	if (need_to_create_worker(pool)) {
1908	/*
1909	* We've been trying to create a new worker but
1910	* haven't been successful. We might be hitting an
1911	* allocation deadlock. Send distress signals to
1912	* rescuers.
1913	*/
1914	list_for_each_entry(work, &pool->worklist, entry)
1915	send_mayday(work);
1916	}
1917
1918	spin_unlock(&pool->lock);
1919	spin_unlock_irq(&wq_mayday_lock);
1920
1921	mod_timer(&pool->mayday_timer, jiffies + MAYDAY_INTERVAL);
1922	}
1923
1924	/**
1925	* maybe_create_worker - create a new worker if necessary
1926	* @pool: pool to create a new worker for
1927	*
1928	* Create a new worker for @pool if necessary. @pool is guaranteed to
1929	* have at least one idle worker on return from this function. If
1930	* creating a new worker takes longer than MAYDAY_INTERVAL, mayday is
1931	* sent to all rescuers with works scheduled on @pool to resolve
1932	* possible allocation deadlock.
1933	*
1934	* On return, need_to_create_worker() is guaranteed to be %false and
1935	* may_start_working() %true.
1936	*
1937	* LOCKING:
1938	* spin_lock_irq(pool->lock) which may be released and regrabbed
1939	* multiple times. Does GFP_KERNEL allocations. Called only from
1940	* manager.
1941	*
1942	* Return:
1943	* %false if no action was taken and pool->lock stayed locked, %true
1944	* otherwise.
1945	*/
1946	static bool maybe_create_worker(struct worker_pool *pool)
1947	__releases(&pool->lock)
1948	__acquires(&pool->lock)
1949	{
1950	if (!need_to_create_worker(pool))
1951	return false;
1952	restart:
1953	spin_unlock_irq(&pool->lock);
1954
1955	/* if we don't make progress in MAYDAY_INITIAL_TIMEOUT, call for help */
1956	mod_timer(&pool->mayday_timer, jiffies + MAYDAY_INITIAL_TIMEOUT);
1957
1958	while (true) {
1959	struct worker *worker;
1960
1961	worker = create_worker(pool);
1962	if (worker) {
1963	del_timer_sync(&pool->mayday_timer);
1964	spin_lock_irq(&pool->lock);
1965	start_worker(worker);
1966	if (WARN_ON_ONCE(need_to_create_worker(pool)))
1967	goto restart;
1968	return true;
1969	}
1970
1971	if (!need_to_create_worker(pool))
1972	break;
1973
1974	__set_current_state(TASK_INTERRUPTIBLE);
1975	schedule_timeout(CREATE_COOLDOWN);
1976
1977	if (!need_to_create_worker(pool))
1978	break;
1979	}
1980
1981	del_timer_sync(&pool->mayday_timer);
1982	spin_lock_irq(&pool->lock);
1983	if (need_to_create_worker(pool))
1984	goto restart;
1985	return true;
1986	}
1987
1988	/**
1989	* maybe_destroy_worker - destroy workers which have been idle for a while
1990	* @pool: pool to destroy workers for
1991	*
1992	* Destroy @pool workers which have been idle for longer than
1993	* IDLE_WORKER_TIMEOUT.
1994	*
1995	* LOCKING:
1996	* spin_lock_irq(pool->lock) which may be released and regrabbed
1997	* multiple times. Called only from manager.
1998	*
1999	* Return:
2000	* %false if no action was taken and pool->lock stayed locked, %true
2001	* otherwise.
2002	*/
2003	static bool maybe_destroy_workers(struct worker_pool *pool)
2004	{
2005	bool ret = false;
2006
2007	while (too_many_workers(pool)) {
2008	struct worker *worker;
2009	unsigned long expires;
2010
2011	worker = list_entry(pool->idle_list.prev, struct worker, entry);
2012	expires = worker->last_active + IDLE_WORKER_TIMEOUT;
2013
2014	if (time_before(jiffies, expires)) {
2015	mod_timer(&pool->idle_timer, expires);
2016	break;
2017	}
2018
2019	destroy_worker(worker);
2020	ret = true;
2021	}
2022
2023	return ret;
2024	}
2025
2026	/**
2027	* manage_workers - manage worker pool
2028	* @worker: self
2029	*
2030	* Assume the manager role and manage the worker pool @worker belongs
2031	* to. At any given time, there can be only zero or one manager per
2032	* pool. The exclusion is handled automatically by this function.
2033	*
2034	* The caller can safely start processing works on false return. On
2035	* true return, it's guaranteed that need_to_create_worker() is false
2036	* and may_start_working() is true.
2037	*
2038	* CONTEXT:
2039	* spin_lock_irq(pool->lock) which may be released and regrabbed
2040	* multiple times. Does GFP_KERNEL allocations.
2041	*
2042	* Return:
2043	* %false if the pool don't need management and the caller can safely start
2044	* processing works, %true indicates that the function released pool->lock
2045	* and reacquired it to perform some management function and that the
2046	* conditions that the caller verified while holding the lock before
2047	* calling the function might no longer be true.
2048	*/
2049	static bool manage_workers(struct worker *worker)
2050	{
2051	struct worker_pool *pool = worker->pool;
2052	bool ret = false;
2053
2054	/*
2055	* Managership is governed by two mutexes - manager_arb and
2056	* manager_mutex. manager_arb handles arbitration of manager role.
2057	* Anyone who successfully grabs manager_arb wins the arbitration
2058	* and becomes the manager. mutex_trylock() on pool->manager_arb
2059	* failure while holding pool->lock reliably indicates that someone
2060	* else is managing the pool and the worker which failed trylock
2061	* can proceed to executing work items. This means that anyone
2062	* grabbing manager_arb is responsible for actually performing
2063	* manager duties. If manager_arb is grabbed and released without
2064	* actual management, the pool may stall indefinitely.
2065	*
2066	* manager_mutex is used for exclusion of actual management
2067	* operations. The holder of manager_mutex can be sure that none
2068	* of management operations, including creation and destruction of
2069	* workers, won't take place until the mutex is released. Because
2070	* manager_mutex doesn't interfere with manager role arbitration,
2071	* it is guaranteed that the pool's management, while may be
2072	* delayed, won't be disturbed by someone else grabbing
2073	* manager_mutex.
2074	*/
2075	if (!mutex_trylock(&pool->manager_arb))
2076	return ret;
2077
2078	/*
2079	* With manager arbitration won, manager_mutex would be free in
2080	* most cases. trylock first without dropping @pool->lock.
2081	*/
2082	if (unlikely(!mutex_trylock(&pool->manager_mutex))) {
2083	spin_unlock_irq(&pool->lock);
2084	mutex_lock(&pool->manager_mutex);
2085	spin_lock_irq(&pool->lock);
2086	ret = true;
2087	}
2088
2089	pool->flags &= ~POOL_MANAGE_WORKERS;
2090
2091	/*
2092	* Destroy and then create so that may_start_working() is true
2093	* on return.
2094	*/
2095	ret \|= maybe_destroy_workers(pool);
2096	ret \|= maybe_create_worker(pool);
2097
2098	mutex_unlock(&pool->manager_mutex);
2099	mutex_unlock(&pool->manager_arb);
2100	return ret;
2101	}
2102
2103	/**
2104	* process_one_work - process single work
2105	* @worker: self
2106	* @work: work to process
2107	*
2108	* Process @work. This function contains all the logics necessary to
2109	* process a single work including synchronization against and
2110	* interaction with other workers on the same cpu, queueing and
2111	* flushing. As long as context requirement is met, any worker can
2112	* call this function to process a work.
2113	*
2114	* CONTEXT:
2115	* spin_lock_irq(pool->lock) which is released and regrabbed.
2116	*/
2117	static void process_one_work(struct worker worker, struct work_struct work)
2118	__releases(&pool->lock)
2119	__acquires(&pool->lock)
2120	{
2121	struct pool_workqueue *pwq = get_work_pwq(work);
2122	struct worker_pool *pool = worker->pool;
2123	bool cpu_intensive = pwq->wq->flags & WQ_CPU_INTENSIVE;
2124	int work_color;
2125	struct worker *collision;
2126	#ifdef CONFIG_LOCKDEP
2127	/*
2128	* It is permissible to free the struct work_struct from
2129	* inside the function that is called from it, this we need to
2130	* take into account for lockdep too. To avoid bogus "held
2131	* lock freed" warnings as well as problems when looking into
2132	* work->lockdep_map, make a copy and use that here.
2133	*/
2134	struct lockdep_map lockdep_map;
2135
2136	lockdep_copy_map(&lockdep_map, &work->lockdep_map);
2137	#endif
2138	/*
2139	* Ensure we're on the correct CPU. DISASSOCIATED test is
2140	* necessary to avoid spurious warnings from rescuers servicing the
2141	* unbound or a disassociated pool.
2142	*/
2143	WARN_ON_ONCE(!(worker->flags & WORKER_UNBOUND) &&
2144	!(pool->flags & POOL_DISASSOCIATED) &&
2145	raw_smp_processor_id() != pool->cpu);
2146
2147	/*
2148	* A single work shouldn't be executed concurrently by
2149	* multiple workers on a single cpu. Check whether anyone is
2150	* already processing the work. If so, defer the work to the
2151	* currently executing one.
2152	*/
2153	collision = find_worker_executing_work(pool, work);
2154	if (unlikely(collision)) {
2155	move_linked_works(work, &collision->scheduled, NULL);
2156	return;
2157	}
2158
2159	/* claim and dequeue */
2160	debug_work_deactivate(work);
2161	hash_add(pool->busy_hash, &worker->hentry, (unsigned long)work);
2162	worker->current_work = work;
2163	worker->current_func = work->func;
2164	worker->current_pwq = pwq;
2165	work_color = get_work_color(work);
2166
2167	list_del_init(&work->entry);
2168
2169	/*
2170	* CPU intensive works don't participate in concurrency
2171	* management. They're the scheduler's responsibility.
2172	*/
2173	if (unlikely(cpu_intensive))
2174	worker_set_flags(worker, WORKER_CPU_INTENSIVE, true);
2175
2176	/*
2177	* Unbound pool isn't concurrency managed and work items should be
2178	* executed ASAP. Wake up another worker if necessary.
2179	*/
2180	if ((worker->flags & WORKER_UNBOUND) && need_more_worker(pool))
2181	wake_up_worker(pool);
2182
2183	/*
2184	* Record the last pool and clear PENDING which should be the last
2185	* update to @work. Also, do this inside @pool->lock so that
2186	* PENDING and queued state changes happen together while IRQ is
2187	* disabled.
2188	*/
2189	set_work_pool_and_clear_pending(work, pool->id);
2190
2191	spin_unlock_irq(&pool->lock);
2192
2193	lock_map_acquire_read(&pwq->wq->lockdep_map);
2194	lock_map_acquire(&lockdep_map);
2195	trace_workqueue_execute_start(work);
2196	worker->current_func(work);
2197	/*
2198	* While we must be careful to not use "work" after this, the trace
2199	* point will only record its address.
2200	*/
2201	trace_workqueue_execute_end(work);
2202	lock_map_release(&lockdep_map);
2203	lock_map_release(&pwq->wq->lockdep_map);
2204
2205	if (unlikely(in_atomic() \|\| lockdep_depth(current) > 0)) {
2206	pr_err("BUG: workqueue leaked lock or atomic: %s/0x%08x/%d\n"
2207	" last function: %pf\n",
2208	current->comm, preempt_count(), task_pid_nr(current),
2209	worker->current_func);
2210	debug_show_held_locks(current);
2211	dump_stack();
2212	}
2213
2214	/*
2215	* The following prevents a kworker from hogging CPU on !PREEMPT
2216	* kernels, where a requeueing work item waiting for something to
2217	* happen could deadlock with stop_machine as such work item could
2218	* indefinitely requeue itself while all other CPUs are trapped in
2219	* stop_machine.
2220	*/
2221	cond_resched();
2222
2223	spin_lock_irq(&pool->lock);
2224
2225	/* clear cpu intensive status */
2226	if (unlikely(cpu_intensive))
2227	worker_clr_flags(worker, WORKER_CPU_INTENSIVE);
2228
2229	/* we're done with it, release */
2230	hash_del(&worker->hentry);
2231	worker->current_work = NULL;
2232	worker->current_func = NULL;
2233	worker->current_pwq = NULL;
2234	worker->desc_valid = false;
2235	pwq_dec_nr_in_flight(pwq, work_color);
2236	}
2237
2238	/**
2239	* process_scheduled_works - process scheduled works
2240	* @worker: self
2241	*
2242	* Process all scheduled works. Please note that the scheduled list
2243	* may change while processing a work, so this function repeatedly
2244	* fetches a work from the top and executes it.
2245	*
2246	* CONTEXT:
2247	* spin_lock_irq(pool->lock) which may be released and regrabbed
2248	* multiple times.
2249	*/
2250	static void process_scheduled_works(struct worker *worker)
2251	{
2252	while (!list_empty(&worker->scheduled)) {
2253	struct work_struct *work = list_first_entry(&worker->scheduled,
2254	struct work_struct, entry);
2255	process_one_work(worker, work);
2256	}
2257	}
2258
2259	/**
2260	* worker_thread - the worker thread function
2261	* @__worker: self
2262	*
2263	* The worker thread function. All workers belong to a worker_pool -
2264	* either a per-cpu one or dynamic unbound one. These workers process all
2265	* work items regardless of their specific target workqueue. The only
2266	* exception is work items which belong to workqueues with a rescuer which
2267	* will be explained in rescuer_thread().
2268	*
2269	* Return: 0
2270	*/
2271	static int worker_thread(void *__worker)
2272	{
2273	struct worker *worker = __worker;
2274	struct worker_pool *pool = worker->pool;
2275
2276	/* tell the scheduler that this is a workqueue worker */
2277	worker->task->flags \|= PF_WQ_WORKER;
2278	woke_up:
2279	spin_lock_irq(&pool->lock);
2280
2281	/* am I supposed to die? */
2282	if (unlikely(worker->flags & WORKER_DIE)) {
2283	spin_unlock_irq(&pool->lock);
2284	WARN_ON_ONCE(!list_empty(&worker->entry));
2285	worker->task->flags &= ~PF_WQ_WORKER;
2286	return 0;
2287	}
2288
2289	worker_leave_idle(worker);
2290	recheck:
2291	/* no more worker necessary? */
2292	if (!need_more_worker(pool))
2293	goto sleep;
2294
2295	/* do we need to manage? */
2296	if (unlikely(!may_start_working(pool)) && manage_workers(worker))
2297	goto recheck;
2298
2299	/*
2300	* ->scheduled list can only be filled while a worker is
2301	* preparing to process a work or actually processing it.
2302	* Make sure nobody diddled with it while I was sleeping.
2303	*/
2304	WARN_ON_ONCE(!list_empty(&worker->scheduled));
2305
2306	/*
2307	* Finish PREP stage. We're guaranteed to have at least one idle
2308	* worker or that someone else has already assumed the manager
2309	* role. This is where @worker starts participating in concurrency
2310	* management if applicable and concurrency management is restored
2311	* after being rebound. See rebind_workers() for details.
2312	*/
2313	worker_clr_flags(worker, WORKER_PREP \| WORKER_REBOUND);
2314
2315	do {
2316	struct work_struct *work =
2317	list_first_entry(&pool->worklist,
2318	struct work_struct, entry);
2319
2320	if (likely(!(*work_data_bits(work) & WORK_STRUCT_LINKED))) {
2321	/* optimization path, not strictly necessary */
2322	process_one_work(worker, work);
2323	if (unlikely(!list_empty(&worker->scheduled)))
2324	process_scheduled_works(worker);
2325	} else {
2326	move_linked_works(work, &worker->scheduled, NULL);
2327	process_scheduled_works(worker);
2328	}
2329	} while (keep_working(pool));
2330
2331	worker_set_flags(worker, WORKER_PREP, false);
2332	sleep:
2333	if (unlikely(need_to_manage_workers(pool)) && manage_workers(worker))
2334	goto recheck;
2335
2336	/*
2337	* pool->lock is held and there's no work to process and no need to
2338	* manage, sleep. Workers are woken up only while holding
2339	* pool->lock or from local cpu, so setting the current state
2340	* before releasing pool->lock is enough to prevent losing any
2341	* event.
2342	*/
2343	worker_enter_idle(worker);
2344	__set_current_state(TASK_INTERRUPTIBLE);
2345	spin_unlock_irq(&pool->lock);
2346	schedule();
2347	goto woke_up;
2348	}
2349
2350	/**
2351	* rescuer_thread - the rescuer thread function
2352	* @__rescuer: self
2353	*
2354	* Workqueue rescuer thread function. There's one rescuer for each
2355	* workqueue which has WQ_MEM_RECLAIM set.
2356	*
2357	* Regular work processing on a pool may block trying to create a new
2358	* worker which uses GFP_KERNEL allocation which has slight chance of
2359	* developing into deadlock if some works currently on the same queue
2360	* need to be processed to satisfy the GFP_KERNEL allocation. This is
2361	* the problem rescuer solves.
2362	*
2363	* When such condition is possible, the pool summons rescuers of all
2364	* workqueues which have works queued on the pool and let them process
2365	* those works so that forward progress can be guaranteed.
2366	*
2367	* This should happen rarely.
2368	*
2369	* Return: 0
2370	*/
2371	static int rescuer_thread(void *__rescuer)
2372	{
2373	struct worker *rescuer = __rescuer;
2374	struct workqueue_struct *wq = rescuer->rescue_wq;
2375	struct list_head *scheduled = &rescuer->scheduled;
2376
2377	set_user_nice(current, RESCUER_NICE_LEVEL);
2378
2379	/*
2380	* Mark rescuer as worker too. As WORKER_PREP is never cleared, it
2381	* doesn't participate in concurrency management.
2382	*/
2383	rescuer->task->flags \|= PF_WQ_WORKER;
2384	repeat:
2385	set_current_state(TASK_INTERRUPTIBLE);
2386
2387	if (kthread_should_stop()) {
2388	__set_current_state(TASK_RUNNING);
2389	rescuer->task->flags &= ~PF_WQ_WORKER;
2390	return 0;
2391	}
2392
2393	/* see whether any pwq is asking for help */
2394	spin_lock_irq(&wq_mayday_lock);
2395
2396	while (!list_empty(&wq->maydays)) {
2397	struct pool_workqueue *pwq = list_first_entry(&wq->maydays,
2398	struct pool_workqueue, mayday_node);
2399	struct worker_pool *pool = pwq->pool;
2400	struct work_struct work, n;
2401
2402	__set_current_state(TASK_RUNNING);
2403	list_del_init(&pwq->mayday_node);
2404
2405	spin_unlock_irq(&wq_mayday_lock);
2406
2407	/* migrate to the target cpu if possible */
2408	worker_maybe_bind_and_lock(pool);
2409	rescuer->pool = pool;
2410
2411	/*
2412	* Slurp in all works issued via this workqueue and
2413	* process'em.
2414	*/
2415	WARN_ON_ONCE(!list_empty(&rescuer->scheduled));
2416	list_for_each_entry_safe(work, n, &pool->worklist, entry)
2417	if (get_work_pwq(work) == pwq)
2418	move_linked_works(work, scheduled, &n);
2419
2420	process_scheduled_works(rescuer);
2421
2422	/*
2423	* Leave this pool. If keep_working() is %true, notify a
2424	* regular worker; otherwise, we end up with 0 concurrency
2425	* and stalling the execution.
2426	*/
2427	if (keep_working(pool))
2428	wake_up_worker(pool);
2429
2430	rescuer->pool = NULL;
2431	spin_unlock(&pool->lock);
2432	spin_lock(&wq_mayday_lock);
2433	}
2434
2435	spin_unlock_irq(&wq_mayday_lock);
2436
2437	/* rescuers should never participate in concurrency management */
2438	WARN_ON_ONCE(!(rescuer->flags & WORKER_NOT_RUNNING));
2439	schedule();
2440	goto repeat;
2441	}
2442
2443	struct wq_barrier {
2444	struct work_struct work;
2445	struct completion done;
2446	};
2447
2448	static void wq_barrier_func(struct work_struct *work)
2449	{
2450	struct wq_barrier *barr = container_of(work, struct wq_barrier, work);
2451	complete(&barr->done);
2452	}
2453
2454	/**
2455	* insert_wq_barrier - insert a barrier work
2456	* @pwq: pwq to insert barrier into
2457	* @barr: wq_barrier to insert
2458	* @target: target work to attach @barr to
2459	* @worker: worker currently executing @target, NULL if @target is not executing
2460	*
2461	* @barr is linked to @target such that @barr is completed only after
2462	* @target finishes execution. Please note that the ordering
2463	* guarantee is observed only with respect to @target and on the local
2464	* cpu.
2465	*
2466	* Currently, a queued barrier can't be canceled. This is because
2467	* try_to_grab_pending() can't determine whether the work to be
2468	* grabbed is at the head of the queue and thus can't clear LINKED
2469	* flag of the previous work while there must be a valid next work
2470	* after a work with LINKED flag set.
2471	*
2472	* Note that when @worker is non-NULL, @target may be modified
2473	* underneath us, so we can't reliably determine pwq from @target.
2474	*
2475	* CONTEXT:
2476	* spin_lock_irq(pool->lock).
2477	*/
2478	static void insert_wq_barrier(struct pool_workqueue *pwq,
2479	struct wq_barrier *barr,
2480	struct work_struct target, struct worker worker)
2481	{
2482	struct list_head *head;
2483	unsigned int linked = 0;
2484
2485	/*
2486	* debugobject calls are safe here even with pool->lock locked
2487	* as we know for sure that this will not trigger any of the
2488	* checks and call back into the fixup functions where we
2489	* might deadlock.
2490	*/
2491	INIT_WORK_ONSTACK(&barr->work, wq_barrier_func);
2492	__set_bit(WORK_STRUCT_PENDING_BIT, work_data_bits(&barr->work));
2493	init_completion(&barr->done);
2494
2495	/*
2496	* If @target is currently being executed, schedule the
2497	* barrier to the worker; otherwise, put it after @target.
2498	*/
2499	if (worker)
2500	head = worker->scheduled.next;
2501	else {
2502	unsigned long *bits = work_data_bits(target);
2503
2504	head = target->entry.next;
2505	/* there can already be other linked works, inherit and set */
2506	linked = *bits & WORK_STRUCT_LINKED;
2507	__set_bit(WORK_STRUCT_LINKED_BIT, bits);
2508	}
2509
2510	debug_work_activate(&barr->work);
2511	insert_work(pwq, &barr->work, head,
2512	work_color_to_flags(WORK_NO_COLOR) \| linked);
2513	}
2514
2515	/**
2516	* flush_workqueue_prep_pwqs - prepare pwqs for workqueue flushing
2517	* @wq: workqueue being flushed
2518	* @flush_color: new flush color, < 0 for no-op
2519	* @work_color: new work color, < 0 for no-op
2520	*
2521	* Prepare pwqs for workqueue flushing.
2522	*
2523	* If @flush_color is non-negative, flush_color on all pwqs should be
2524	* -1. If no pwq has in-flight commands at the specified color, all
2525	* pwq->flush_color's stay at -1 and %false is returned. If any pwq
2526	* has in flight commands, its pwq->flush_color is set to
2527	* @flush_color, @wq->nr_pwqs_to_flush is updated accordingly, pwq
2528	* wakeup logic is armed and %true is returned.
2529	*
2530	* The caller should have initialized @wq->first_flusher prior to
2531	* calling this function with non-negative @flush_color. If
2532	* @flush_color is negative, no flush color update is done and %false
2533	* is returned.
2534	*
2535	* If @work_color is non-negative, all pwqs should have the same
2536	* work_color which is previous to @work_color and all will be
2537	* advanced to @work_color.
2538	*
2539	* CONTEXT:
2540	* mutex_lock(wq->mutex).
2541	*
2542	* Return:
2543	* %true if @flush_color >= 0 and there's something to flush. %false
2544	* otherwise.
2545	*/
2546	static bool flush_workqueue_prep_pwqs(struct workqueue_struct *wq,
2547	int flush_color, int work_color)
2548	{
2549	bool wait = false;
2550	struct pool_workqueue *pwq;
2551
2552	if (flush_color >= 0) {
2553	WARN_ON_ONCE(atomic_read(&wq->nr_pwqs_to_flush));
2554	atomic_set(&wq->nr_pwqs_to_flush, 1);
2555	}
2556
2557	for_each_pwq(pwq, wq) {
2558	struct worker_pool *pool = pwq->pool;
2559
2560	spin_lock_irq(&pool->lock);
2561
2562	if (flush_color >= 0) {
2563	WARN_ON_ONCE(pwq->flush_color != -1);
2564
2565	if (pwq->nr_in_flight[flush_color]) {
2566	pwq->flush_color = flush_color;
2567	atomic_inc(&wq->nr_pwqs_to_flush);
2568	wait = true;
2569	}
2570	}
2571
2572	if (work_color >= 0) {
2573	WARN_ON_ONCE(work_color != work_next_color(pwq->work_color));
2574	pwq->work_color = work_color;
2575	}
2576
2577	spin_unlock_irq(&pool->lock);
2578	}
2579
2580	if (flush_color >= 0 && atomic_dec_and_test(&wq->nr_pwqs_to_flush))
2581	complete(&wq->first_flusher->done);
2582
2583	return wait;
2584	}
2585
2586	/**
2587	* flush_workqueue - ensure that any scheduled work has run to completion.
2588	* @wq: workqueue to flush
2589	*
2590	* This function sleeps until all work items which were queued on entry
2591	* have finished execution, but it is not livelocked by new incoming ones.
2592	*/
2593	void flush_workqueue(struct workqueue_struct *wq)
2594	{
2595	struct wq_flusher this_flusher = {
2596	.list = LIST_HEAD_INIT(this_flusher.list),
2597	.flush_color = -1,
2598	.done = COMPLETION_INITIALIZER_ONSTACK(this_flusher.done),
2599	};
2600	int next_color;
2601
2602	lock_map_acquire(&wq->lockdep_map);
2603	lock_map_release(&wq->lockdep_map);
2604
2605	mutex_lock(&wq->mutex);
2606
2607	/*
2608	* Start-to-wait phase
2609	*/
2610	next_color = work_next_color(wq->work_color);
2611
2612	if (next_color != wq->flush_color) {
2613	/*
2614	* Color space is not full. The current work_color
2615	* becomes our flush_color and work_color is advanced
2616	* by one.
2617	*/
2618	WARN_ON_ONCE(!list_empty(&wq->flusher_overflow));
2619	this_flusher.flush_color = wq->work_color;
2620	wq->work_color = next_color;
2621
2622	if (!wq->first_flusher) {
2623	/* no flush in progress, become the first flusher */
2624	WARN_ON_ONCE(wq->flush_color != this_flusher.flush_color);
2625
2626	wq->first_flusher = &this_flusher;
2627
2628	if (!flush_workqueue_prep_pwqs(wq, wq->flush_color,
2629	wq->work_color)) {
2630	/* nothing to flush, done */
2631	wq->flush_color = next_color;
2632	wq->first_flusher = NULL;
2633	goto out_unlock;
2634	}
2635	} else {
2636	/* wait in queue */
2637	WARN_ON_ONCE(wq->flush_color == this_flusher.flush_color);
2638	list_add_tail(&this_flusher.list, &wq->flusher_queue);
2639	flush_workqueue_prep_pwqs(wq, -1, wq->work_color);
2640	}
2641	} else {
2642	/*
2643	* Oops, color space is full, wait on overflow queue.
2644	* The next flush completion will assign us
2645	* flush_color and transfer to flusher_queue.
2646	*/
2647	list_add_tail(&this_flusher.list, &wq->flusher_overflow);
2648	}
2649
2650	mutex_unlock(&wq->mutex);
2651
2652	wait_for_completion(&this_flusher.done);
2653
2654	/*
2655	* Wake-up-and-cascade phase
2656	*
2657	* First flushers are responsible for cascading flushes and
2658	* handling overflow. Non-first flushers can simply return.
2659	*/
2660	if (wq->first_flusher != &this_flusher)
2661	return;
2662
2663	mutex_lock(&wq->mutex);
2664
2665	/* we might have raced, check again with mutex held */
2666	if (wq->first_flusher != &this_flusher)
2667	goto out_unlock;
2668
2669	wq->first_flusher = NULL;
2670
2671	WARN_ON_ONCE(!list_empty(&this_flusher.list));
2672	WARN_ON_ONCE(wq->flush_color != this_flusher.flush_color);
2673
2674	while (true) {
2675	struct wq_flusher next, tmp;
2676
2677	/* complete all the flushers sharing the current flush color */
2678	list_for_each_entry_safe(next, tmp, &wq->flusher_queue, list) {
2679	if (next->flush_color != wq->flush_color)
2680	break;
2681	list_del_init(&next->list);
2682	complete(&next->done);
2683	}
2684
2685	WARN_ON_ONCE(!list_empty(&wq->flusher_overflow) &&
2686	wq->flush_color != work_next_color(wq->work_color));
2687
2688	/* this flush_color is finished, advance by one */
2689	wq->flush_color = work_next_color(wq->flush_color);
2690
2691	/* one color has been freed, handle overflow queue */
2692	if (!list_empty(&wq->flusher_overflow)) {
2693	/*
2694	* Assign the same color to all overflowed
2695	* flushers, advance work_color and append to
2696	* flusher_queue. This is the start-to-wait
2697	* phase for these overflowed flushers.
2698	*/
2699	list_for_each_entry(tmp, &wq->flusher_overflow, list)
2700	tmp->flush_color = wq->work_color;
2701
2702	wq->work_color = work_next_color(wq->work_color);
2703
2704	list_splice_tail_init(&wq->flusher_overflow,
2705	&wq->flusher_queue);
2706	flush_workqueue_prep_pwqs(wq, -1, wq->work_color);
2707	}
2708
2709	if (list_empty(&wq->flusher_queue)) {
2710	WARN_ON_ONCE(wq->flush_color != wq->work_color);
2711	break;
2712	}
2713
2714	/*
2715	* Need to flush more colors. Make the next flusher
2716	* the new first flusher and arm pwqs.
2717	*/
2718	WARN_ON_ONCE(wq->flush_color == wq->work_color);
2719	WARN_ON_ONCE(wq->flush_color != next->flush_color);
2720
2721	list_del_init(&next->list);
2722	wq->first_flusher = next;
2723
2724	if (flush_workqueue_prep_pwqs(wq, wq->flush_color, -1))
2725	break;
2726
2727	/*
2728	* Meh... this color is already done, clear first
2729	* flusher and repeat cascading.
2730	*/
2731	wq->first_flusher = NULL;
2732	}
2733
2734	out_unlock:
2735	mutex_unlock(&wq->mutex);
2736	}
2737	EXPORT_SYMBOL_GPL(flush_workqueue);
2738
2739	/**
2740	* drain_workqueue - drain a workqueue
2741	* @wq: workqueue to drain
2742	*
2743	* Wait until the workqueue becomes empty. While draining is in progress,
2744	* only chain queueing is allowed. IOW, only currently pending or running
2745	* work items on @wq can queue further work items on it. @wq is flushed
2746	* repeatedly until it becomes empty. The number of flushing is detemined
2747	* by the depth of chaining and should be relatively short. Whine if it
2748	* takes too long.
2749	*/
2750	void drain_workqueue(struct workqueue_struct *wq)
2751	{
2752	unsigned int flush_cnt = 0;
2753	struct pool_workqueue *pwq;
2754
2755	/*
2756	* __queue_work() needs to test whether there are drainers, is much
2757	* hotter than drain_workqueue() and already looks at @wq->flags.
2758	* Use __WQ_DRAINING so that queue doesn't have to check nr_drainers.
2759	*/
2760	mutex_lock(&wq->mutex);
2761	if (!wq->nr_drainers++)
2762	wq->flags \|= __WQ_DRAINING;
2763	mutex_unlock(&wq->mutex);
2764	reflush:
2765	flush_workqueue(wq);
2766
2767	mutex_lock(&wq->mutex);
2768
2769	for_each_pwq(pwq, wq) {
2770	bool drained;
2771
2772	spin_lock_irq(&pwq->pool->lock);
2773	drained = !pwq->nr_active && list_empty(&pwq->delayed_works);
2774	spin_unlock_irq(&pwq->pool->lock);
2775
2776	if (drained)
2777	continue;
2778
2779	if (++flush_cnt == 10 \|\|
2780	(flush_cnt % 100 == 0 && flush_cnt <= 1000))
2781	pr_warn("workqueue %s: drain_workqueue() isn't complete after %u tries\n",
2782	wq->name, flush_cnt);
2783
2784	mutex_unlock(&wq->mutex);
2785	goto reflush;
2786	}
2787
2788	if (!--wq->nr_drainers)
2789	wq->flags &= ~__WQ_DRAINING;
2790	mutex_unlock(&wq->mutex);
2791	}
2792	EXPORT_SYMBOL_GPL(drain_workqueue);
2793
2794	static bool start_flush_work(struct work_struct work, struct wq_barrier barr)
2795	{
2796	struct worker *worker = NULL;
2797	struct worker_pool *pool;
2798	struct pool_workqueue *pwq;
2799
2800	might_sleep();
2801
2802	local_irq_disable();
2803	pool = get_work_pool(work);
2804	if (!pool) {
2805	local_irq_enable();
2806	return false;
2807	}
2808
2809	spin_lock(&pool->lock);
2810	/* see the comment in try_to_grab_pending() with the same code */
2811	pwq = get_work_pwq(work);
2812	if (pwq) {
2813	if (unlikely(pwq->pool != pool))
2814	goto already_gone;
2815	} else {
2816	worker = find_worker_executing_work(pool, work);
2817	if (!worker)
2818	goto already_gone;
2819	pwq = worker->current_pwq;
2820	}
2821
2822	insert_wq_barrier(pwq, barr, work, worker);
2823	spin_unlock_irq(&pool->lock);
2824
2825	/*
2826	* If @max_active is 1 or rescuer is in use, flushing another work
2827	* item on the same workqueue may lead to deadlock. Make sure the
2828	* flusher is not running on the same workqueue by verifying write
2829	* access.
2830	*/
2831	if (pwq->wq->saved_max_active == 1 \|\| pwq->wq->rescuer)
2832	lock_map_acquire(&pwq->wq->lockdep_map);
2833	else
2834	lock_map_acquire_read(&pwq->wq->lockdep_map);
2835	lock_map_release(&pwq->wq->lockdep_map);
2836
2837	return true;
2838	already_gone:
2839	spin_unlock_irq(&pool->lock);
2840	return false;
2841	}
2842
2843	static bool __flush_work(struct work_struct *work)
2844	{
2845	struct wq_barrier barr;
2846
2847	if (start_flush_work(work, &barr)) {
2848	wait_for_completion(&barr.done);
2849	destroy_work_on_stack(&barr.work);
2850	return true;
2851	} else {
2852	return false;
2853	}
2854	}
2855
2856	/**
2857	* flush_work - wait for a work to finish executing the last queueing instance
2858	* @work: the work to flush
2859	*
2860	* Wait until @work has finished execution. @work is guaranteed to be idle
2861	* on return if it hasn't been requeued since flush started.
2862	*
2863	* Return:
2864	* %true if flush_work() waited for the work to finish execution,
2865	* %false if it was already idle.
2866	*/
2867	bool flush_work(struct work_struct *work)
2868	{
2869	lock_map_acquire(&work->lockdep_map);
2870	lock_map_release(&work->lockdep_map);
2871
2872	return __flush_work(work);
2873	}
2874	EXPORT_SYMBOL_GPL(flush_work);
2875
2876	static bool __cancel_work_timer(struct work_struct *work, bool is_dwork)
2877	{
2878	unsigned long flags;
2879	int ret;
2880
2881	do {
2882	ret = try_to_grab_pending(work, is_dwork, &flags);
2883	/*
2884	* If someone else is canceling, wait for the same event it
2885	* would be waiting for before retrying.
2886	*/
2887	if (unlikely(ret == -ENOENT))
2888	flush_work(work);
2889	} while (unlikely(ret < 0));
2890
2891	/* tell other tasks trying to grab @work to back off */
2892	mark_work_canceling(work);
2893	local_irq_restore(flags);
2894
2895	flush_work(work);
2896	clear_work_data(work);
2897	return ret;
2898	}
2899
2900	/**
2901	* cancel_work_sync - cancel a work and wait for it to finish
2902	* @work: the work to cancel
2903	*
2904	* Cancel @work and wait for its execution to finish. This function
2905	* can be used even if the work re-queues itself or migrates to
2906	* another workqueue. On return from this function, @work is
2907	* guaranteed to be not pending or executing on any CPU.
2908	*
2909	* cancel_work_sync(&delayed_work->work) must not be used for
2910	* delayed_work's. Use cancel_delayed_work_sync() instead.
2911	*
2912	* The caller must ensure that the workqueue on which @work was last
2913	* queued can't be destroyed before this function returns.
2914	*
2915	* Return:
2916	* %true if @work was pending, %false otherwise.
2917	*/
2918	bool cancel_work_sync(struct work_struct *work)
2919	{
2920	return __cancel_work_timer(work, false);
2921	}
2922	EXPORT_SYMBOL_GPL(cancel_work_sync);
2923
2924	/**
2925	* flush_delayed_work - wait for a dwork to finish executing the last queueing
2926	* @dwork: the delayed work to flush
2927	*
2928	* Delayed timer is cancelled and the pending work is queued for
2929	* immediate execution. Like flush_work(), this function only
2930	* considers the last queueing instance of @dwork.
2931	*
2932	* Return:
2933	* %true if flush_work() waited for the work to finish execution,
2934	* %false if it was already idle.
2935	*/
2936	bool flush_delayed_work(struct delayed_work *dwork)
2937	{
2938	local_irq_disable();
2939	if (del_timer_sync(&dwork->timer))
2940	__queue_work(dwork->cpu, dwork->wq, &dwork->work);
2941	local_irq_enable();
2942	return flush_work(&dwork->work);
2943	}
2944	EXPORT_SYMBOL(flush_delayed_work);
2945
2946	/**
2947	* cancel_delayed_work - cancel a delayed work
2948	* @dwork: delayed_work to cancel
2949	*
2950	* Kill off a pending delayed_work.
2951	*
2952	* Return: %true if @dwork was pending and canceled; %false if it wasn't
2953	* pending.
2954	*
2955	* Note:
2956	* The work callback function may still be running on return, unless
2957	* it returns %true and the work doesn't re-arm itself. Explicitly flush or
2958	* use cancel_delayed_work_sync() to wait on it.
2959	*
2960	* This function is safe to call from any context including IRQ handler.
2961	*/
2962	bool cancel_delayed_work(struct delayed_work *dwork)
2963	{
2964	unsigned long flags;
2965	int ret;
2966
2967	do {
2968	ret = try_to_grab_pending(&dwork->work, true, &flags);
2969	} while (unlikely(ret == -EAGAIN));
2970
2971	if (unlikely(ret < 0))
2972	return false;
2973
2974	set_work_pool_and_clear_pending(&dwork->work,
2975	get_work_pool_id(&dwork->work));
2976	local_irq_restore(flags);
2977	return ret;
2978	}
2979	EXPORT_SYMBOL(cancel_delayed_work);
2980
2981	/**
2982	* cancel_delayed_work_sync - cancel a delayed work and wait for it to finish
2983	* @dwork: the delayed work cancel
2984	*
2985	* This is cancel_work_sync() for delayed works.
2986	*
2987	* Return:
2988	* %true if @dwork was pending, %false otherwise.
2989	*/
2990	bool cancel_delayed_work_sync(struct delayed_work *dwork)
2991	{
2992	return __cancel_work_timer(&dwork->work, true);
2993	}
2994	EXPORT_SYMBOL(cancel_delayed_work_sync);
2995
2996	/**
2997	* schedule_on_each_cpu - execute a function synchronously on each online CPU
2998	* @func: the function to call
2999	*
3000	* schedule_on_each_cpu() executes @func on each online CPU using the
3001	* system workqueue and blocks until all CPUs have completed.
3002	* schedule_on_each_cpu() is very slow.
3003	*
3004	* Return:
3005	* 0 on success, -errno on failure.
3006	*/
3007	int schedule_on_each_cpu(work_func_t func)
3008	{
3009	int cpu;
3010	struct work_struct __percpu *works;
3011
3012	works = alloc_percpu(struct work_struct);
3013	if (!works)
3014	return -ENOMEM;
3015
3016	get_online_cpus();
3017
3018	for_each_online_cpu(cpu) {
3019	struct work_struct *work = per_cpu_ptr(works, cpu);
3020
3021	INIT_WORK(work, func);
3022	schedule_work_on(cpu, work);
3023	}
3024
3025	for_each_online_cpu(cpu)
3026	flush_work(per_cpu_ptr(works, cpu));
3027
3028	put_online_cpus();
3029	free_percpu(works);
3030	return 0;
3031	}
3032
3033	/**
3034	* flush_scheduled_work - ensure that any scheduled work has run to completion.
3035	*
3036	* Forces execution of the kernel-global workqueue and blocks until its
3037	* completion.
3038	*
3039	* Think twice before calling this function! It's very easy to get into
3040	* trouble if you don't take great care. Either of the following situations
3041	* will lead to deadlock:
3042	*
3043	* One of the work items currently on the workqueue needs to acquire
3044	* a lock held by your code or its caller.
3045	*
3046	* Your code is running in the context of a work routine.
3047	*
3048	* They will be detected by lockdep when they occur, but the first might not
3049	* occur very often. It depends on what work items are on the workqueue and
3050	* what locks they need, which you have no control over.
3051	*
3052	* In most situations flushing the entire workqueue is overkill; you merely
3053	* need to know that a particular work item isn't queued and isn't running.
3054	* In such cases you should use cancel_delayed_work_sync() or
3055	* cancel_work_sync() instead.
3056	*/
3057	void flush_scheduled_work(void)
3058	{
3059	flush_workqueue(system_wq);
3060	}
3061	EXPORT_SYMBOL(flush_scheduled_work);
3062
3063	/**
3064	* execute_in_process_context - reliably execute the routine with user context
3065	* @fn: the function to execute
3066	* @ew: guaranteed storage for the execute work structure (must
3067	* be available when the work executes)
3068	*
3069	* Executes the function immediately if process context is available,
3070	* otherwise schedules the function for delayed execution.
3071	*
3072	* Return: 0 - function was executed
3073	* 1 - function was scheduled for execution
3074	*/
3075	int execute_in_process_context(work_func_t fn, struct execute_work *ew)
3076	{
3077	if (!in_interrupt()) {
3078	fn(&ew->work);
3079	return 0;
3080	}
3081
3082	INIT_WORK(&ew->work, fn);
3083	schedule_work(&ew->work);
3084
3085	return 1;
3086	}
3087	EXPORT_SYMBOL_GPL(execute_in_process_context);
3088
3089	#ifdef CONFIG_SYSFS
3090	/*
3091	* Workqueues with WQ_SYSFS flag set is visible to userland via
3092	* /sys/bus/workqueue/devices/WQ_NAME. All visible workqueues have the
3093	* following attributes.
3094	*
3095	* per_cpu RO bool : whether the workqueue is per-cpu or unbound
3096	* max_active RW int : maximum number of in-flight work items
3097	*
3098	* Unbound workqueues have the following extra attributes.
3099	*
3100	* id RO int : the associated pool ID
3101	* nice RW int : nice value of the workers
3102	* cpumask RW mask : bitmask of allowed CPUs for the workers
3103	*/
3104	struct wq_device {
3105	struct workqueue_struct *wq;
3106	struct device dev;
3107	};
3108
3109	static struct workqueue_struct dev_to_wq(struct device dev)
3110	{
3111	struct wq_device *wq_dev = container_of(dev, struct wq_device, dev);
3112
3113	return wq_dev->wq;
3114	}
3115
3116	static ssize_t per_cpu_show(struct device dev, struct device_attribute attr,
3117	char *buf)
3118	{
3119	struct workqueue_struct *wq = dev_to_wq(dev);
3120
3121	return scnprintf(buf, PAGE_SIZE, "%d\n", (bool)!(wq->flags & WQ_UNBOUND));
3122	}
3123	static DEVICE_ATTR_RO(per_cpu);
3124
3125	static ssize_t max_active_show(struct device *dev,
3126	struct device_attribute attr, char buf)
3127	{
3128	struct workqueue_struct *wq = dev_to_wq(dev);
3129
3130	return scnprintf(buf, PAGE_SIZE, "%d\n", wq->saved_max_active);
3131	}
3132
3133	static ssize_t max_active_store(struct device *dev,
3134	struct device_attribute attr, const char buf,
3135	size_t count)
3136	{
3137	struct workqueue_struct *wq = dev_to_wq(dev);
3138	int val;
3139
3140	if (sscanf(buf, "%d", &val) != 1 \|\| val <= 0)
3141	return -EINVAL;
3142
3143	workqueue_set_max_active(wq, val);
3144	return count;
3145	}
3146	static DEVICE_ATTR_RW(max_active);
3147
3148	static struct attribute *wq_sysfs_attrs[] = {
3149	&dev_attr_per_cpu.attr,
3150	&dev_attr_max_active.attr,
3151	NULL,
3152	};
3153	ATTRIBUTE_GROUPS(wq_sysfs);
3154
3155	static ssize_t wq_pool_ids_show(struct device *dev,
3156	struct device_attribute attr, char buf)
3157	{
3158	struct workqueue_struct *wq = dev_to_wq(dev);
3159	const char *delim = "";
3160	int node, written = 0;
3161
3162	rcu_read_lock_sched();
3163	for_each_node(node) {
3164	written += scnprintf(buf + written, PAGE_SIZE - written,
3165	"%s%d:%d", delim, node,
3166	unbound_pwq_by_node(wq, node)->pool->id);
3167	delim = " ";
3168	}
3169	written += scnprintf(buf + written, PAGE_SIZE - written, "\n");
3170	rcu_read_unlock_sched();
3171
3172	return written;
3173	}
3174
3175	static ssize_t wq_nice_show(struct device dev, struct device_attribute attr,
3176	char *buf)
3177	{
3178	struct workqueue_struct *wq = dev_to_wq(dev);
3179	int written;
3180
3181	mutex_lock(&wq->mutex);
3182	written = scnprintf(buf, PAGE_SIZE, "%d\n", wq->unbound_attrs->nice);
3183	mutex_unlock(&wq->mutex);
3184
3185	return written;
3186	}
3187
3188	/* prepare workqueue_attrs for sysfs store operations */
3189	static struct workqueue_attrs wq_sysfs_prep_attrs(struct workqueue_struct wq)
3190	{
3191	struct workqueue_attrs *attrs;
3192
3193	attrs = alloc_workqueue_attrs(GFP_KERNEL);
3194	if (!attrs)
3195	return NULL;
3196
3197	mutex_lock(&wq->mutex);
3198	copy_workqueue_attrs(attrs, wq->unbound_attrs);
3199	mutex_unlock(&wq->mutex);
3200	return attrs;
3201	}
3202
3203	static ssize_t wq_nice_store(struct device dev, struct device_attribute attr,
3204	const char *buf, size_t count)
3205	{
3206	struct workqueue_struct *wq = dev_to_wq(dev);
3207	struct workqueue_attrs *attrs;
3208	int ret;
3209
3210	attrs = wq_sysfs_prep_attrs(wq);
3211	if (!attrs)
3212	return -ENOMEM;
3213
3214	if (sscanf(buf, "%d", &attrs->nice) == 1 &&
3215	attrs->nice >= -20 && attrs->nice <= 19)
3216	ret = apply_workqueue_attrs(wq, attrs);
3217	else
3218	ret = -EINVAL;
3219
3220	free_workqueue_attrs(attrs);
3221	return ret ?: count;
3222	}
3223
3224	static ssize_t wq_cpumask_show(struct device *dev,
3225	struct device_attribute attr, char buf)
3226	{
3227	struct workqueue_struct *wq = dev_to_wq(dev);
3228	int written;
3229
3230	mutex_lock(&wq->mutex);
3231	written = cpumask_scnprintf(buf, PAGE_SIZE, wq->unbound_attrs->cpumask);
3232	mutex_unlock(&wq->mutex);
3233
3234	written += scnprintf(buf + written, PAGE_SIZE - written, "\n");
3235	return written;
3236	}
3237
3238	static ssize_t wq_cpumask_store(struct device *dev,
3239	struct device_attribute *attr,
3240	const char *buf, size_t count)
3241	{
3242	struct workqueue_struct *wq = dev_to_wq(dev);
3243	struct workqueue_attrs *attrs;
3244	int ret;
3245
3246	attrs = wq_sysfs_prep_attrs(wq);
3247	if (!attrs)
3248	return -ENOMEM;
3249
3250	ret = cpumask_parse(buf, attrs->cpumask);
3251	if (!ret)
3252	ret = apply_workqueue_attrs(wq, attrs);
3253
3254	free_workqueue_attrs(attrs);
3255	return ret ?: count;
3256	}
3257
3258	static ssize_t wq_numa_show(struct device dev, struct device_attribute attr,
3259	char *buf)
3260	{
3261	struct workqueue_struct *wq = dev_to_wq(dev);
3262	int written;
3263
3264	mutex_lock(&wq->mutex);
3265	written = scnprintf(buf, PAGE_SIZE, "%d\n",
3266	!wq->unbound_attrs->no_numa);
3267	mutex_unlock(&wq->mutex);
3268
3269	return written;
3270	}
3271
3272	static ssize_t wq_numa_store(struct device dev, struct device_attribute attr,
3273	const char *buf, size_t count)
3274	{
3275	struct workqueue_struct *wq = dev_to_wq(dev);
3276	struct workqueue_attrs *attrs;
3277	int v, ret;
3278
3279	attrs = wq_sysfs_prep_attrs(wq);
3280	if (!attrs)
3281	return -ENOMEM;
3282
3283	ret = -EINVAL;
3284	if (sscanf(buf, "%d", &v) == 1) {
3285	attrs->no_numa = !v;
3286	ret = apply_workqueue_attrs(wq, attrs);
3287	}
3288
3289	free_workqueue_attrs(attrs);
3290	return ret ?: count;
3291	}
3292
3293	static struct device_attribute wq_sysfs_unbound_attrs[] = {
3294	__ATTR(pool_ids, 0444, wq_pool_ids_show, NULL),
3295	__ATTR(nice, 0644, wq_nice_show, wq_nice_store),
3296	__ATTR(cpumask, 0644, wq_cpumask_show, wq_cpumask_store),
3297	__ATTR(numa, 0644, wq_numa_show, wq_numa_store),
3298	__ATTR_NULL,
3299	};
3300
3301	static struct bus_type wq_subsys = {
3302	.name = "workqueue",
3303	.dev_groups = wq_sysfs_groups,
3304	};
3305
3306	static int __init wq_sysfs_init(void)
3307	{
3308	return subsys_virtual_register(&wq_subsys, NULL);
3309	}
3310	core_initcall(wq_sysfs_init);
3311
3312	static void wq_device_release(struct device *dev)
3313	{
3314	struct wq_device *wq_dev = container_of(dev, struct wq_device, dev);
3315
3316	kfree(wq_dev);
3317	}
3318
3319	/**
3320	* workqueue_sysfs_register - make a workqueue visible in sysfs
3321	* @wq: the workqueue to register
3322	*
3323	* Expose @wq in sysfs under /sys/bus/workqueue/devices.
3324	* alloc_workqueue*() automatically calls this function if WQ_SYSFS is set
3325	* which is the preferred method.
3326	*
3327	* Workqueue user should use this function directly iff it wants to apply
3328	* workqueue_attrs before making the workqueue visible in sysfs; otherwise,
3329	* apply_workqueue_attrs() may race against userland updating the
3330	* attributes.
3331	*
3332	* Return: 0 on success, -errno on failure.
3333	*/
3334	int workqueue_sysfs_register(struct workqueue_struct *wq)
3335	{
3336	struct wq_device *wq_dev;
3337	int ret;
3338
3339	/*
3340	* Adjusting max_active or creating new pwqs by applyting
3341	* attributes breaks ordering guarantee. Disallow exposing ordered
3342	* workqueues.
3343	*/
3344	if (WARN_ON(wq->flags & __WQ_ORDERED))
3345	return -EINVAL;
3346
3347	wq->wq_dev = wq_dev = kzalloc(sizeof(*wq_dev), GFP_KERNEL);
3348	if (!wq_dev)
3349	return -ENOMEM;
3350
3351	wq_dev->wq = wq;
3352	wq_dev->dev.bus = &wq_subsys;
3353	wq_dev->dev.init_name = wq->name;
3354	wq_dev->dev.release = wq_device_release;
3355
3356	/*
3357	* unbound_attrs are created separately. Suppress uevent until
3358	* everything is ready.
3359	*/
3360	dev_set_uevent_suppress(&wq_dev->dev, true);
3361
3362	ret = device_register(&wq_dev->dev);
3363	if (ret) {
3364	kfree(wq_dev);
3365	wq->wq_dev = NULL;
3366	return ret;
3367	}
3368
3369	if (wq->flags & WQ_UNBOUND) {
3370	struct device_attribute *attr;
3371
3372	for (attr = wq_sysfs_unbound_attrs; attr->attr.name; attr++) {
3373	ret = device_create_file(&wq_dev->dev, attr);
3374	if (ret) {
3375	device_unregister(&wq_dev->dev);
3376	wq->wq_dev = NULL;
3377	return ret;
3378	}
3379	}
3380	}
3381
3382	kobject_uevent(&wq_dev->dev.kobj, KOBJ_ADD);
3383	return 0;
3384	}
3385
3386	/**
3387	* workqueue_sysfs_unregister - undo workqueue_sysfs_register()
3388	* @wq: the workqueue to unregister
3389	*
3390	* If @wq is registered to sysfs by workqueue_sysfs_register(), unregister.
3391	*/
3392	static void workqueue_sysfs_unregister(struct workqueue_struct *wq)
3393	{
3394	struct wq_device *wq_dev = wq->wq_dev;
3395
3396	if (!wq->wq_dev)
3397	return;
3398
3399	wq->wq_dev = NULL;
3400	device_unregister(&wq_dev->dev);
3401	}
3402	#else /* CONFIG_SYSFS */
3403	static void workqueue_sysfs_unregister(struct workqueue_struct *wq) { }
3404	#endif /* CONFIG_SYSFS */
3405
3406	/**
3407	* free_workqueue_attrs - free a workqueue_attrs
3408	* @attrs: workqueue_attrs to free
3409	*
3410	* Undo alloc_workqueue_attrs().
3411	*/
3412	void free_workqueue_attrs(struct workqueue_attrs *attrs)
3413	{
3414	if (attrs) {
3415	free_cpumask_var(attrs->cpumask);
3416	kfree(attrs);
3417	}
3418	}
3419
3420	/**
3421	* alloc_workqueue_attrs - allocate a workqueue_attrs
3422	* @gfp_mask: allocation mask to use
3423	*
3424	* Allocate a new workqueue_attrs, initialize with default settings and
3425	* return it.
3426	*
3427	* Return: The allocated new workqueue_attr on success. %NULL on failure.
3428	*/
3429	struct workqueue_attrs *alloc_workqueue_attrs(gfp_t gfp_mask)
3430	{
3431	struct workqueue_attrs *attrs;
3432
3433	attrs = kzalloc(sizeof(*attrs), gfp_mask);
3434	if (!attrs)
3435	goto fail;
3436	if (!alloc_cpumask_var(&attrs->cpumask, gfp_mask))
3437	goto fail;
3438
3439	cpumask_copy(attrs->cpumask, cpu_possible_mask);
3440	return attrs;
3441	fail:
3442	free_workqueue_attrs(attrs);
3443	return NULL;
3444	}
3445
3446	static void copy_workqueue_attrs(struct workqueue_attrs *to,
3447	const struct workqueue_attrs *from)
3448	{
3449	to->nice = from->nice;
3450	cpumask_copy(to->cpumask, from->cpumask);
3451	/*
3452	* Unlike hash and equality test, this function doesn't ignore
3453	* ->no_numa as it is used for both pool and wq attrs. Instead,
3454	* get_unbound_pool() explicitly clears ->no_numa after copying.
3455	*/
3456	to->no_numa = from->no_numa;
3457	}
3458
3459	/* hash value of the content of @attr */
3460	static u32 wqattrs_hash(const struct workqueue_attrs *attrs)
3461	{
3462	u32 hash = 0;
3463
3464	hash = jhash_1word(attrs->nice, hash);
3465	hash = jhash(cpumask_bits(attrs->cpumask),
3466	BITS_TO_LONGS(nr_cpumask_bits) * sizeof(long), hash);
3467	return hash;
3468	}
3469
3470	/* content equality test */
3471	static bool wqattrs_equal(const struct workqueue_attrs *a,
3472	const struct workqueue_attrs *b)
3473	{
3474	if (a->nice != b->nice)
3475	return false;
3476	if (!cpumask_equal(a->cpumask, b->cpumask))
3477	return false;
3478	return true;
3479	}
3480
3481	/**
3482	* init_worker_pool - initialize a newly zalloc'd worker_pool
3483	* @pool: worker_pool to initialize
3484	*
3485	* Initiailize a newly zalloc'd @pool. It also allocates @pool->attrs.
3486	*
3487	* Return: 0 on success, -errno on failure. Even on failure, all fields
3488	* inside @pool proper are initialized and put_unbound_pool() can be called
3489	* on @pool safely to release it.
3490	*/
3491	static int init_worker_pool(struct worker_pool *pool)
3492	{
3493	spin_lock_init(&pool->lock);
3494	pool->id = -1;
3495	pool->cpu = -1;
3496	pool->node = NUMA_NO_NODE;
3497	pool->flags \|= POOL_DISASSOCIATED;
3498	INIT_LIST_HEAD(&pool->worklist);
3499	INIT_LIST_HEAD(&pool->idle_list);
3500	hash_init(pool->busy_hash);
3501
3502	init_timer_deferrable(&pool->idle_timer);
3503	pool->idle_timer.function = idle_worker_timeout;
3504	pool->idle_timer.data = (unsigned long)pool;
3505
3506	setup_timer(&pool->mayday_timer, pool_mayday_timeout,
3507	(unsigned long)pool);
3508
3509	mutex_init(&pool->manager_arb);
3510	mutex_init(&pool->manager_mutex);
3511	idr_init(&pool->worker_idr);
3512
3513	INIT_HLIST_NODE(&pool->hash_node);
3514	pool->refcnt = 1;
3515
3516	/* shouldn't fail above this point */
3517	pool->attrs = alloc_workqueue_attrs(GFP_KERNEL);
3518	if (!pool->attrs)
3519	return -ENOMEM;
3520	return 0;
3521	}
3522
3523	static void rcu_free_pool(struct rcu_head *rcu)
3524	{
3525	struct worker_pool *pool = container_of(rcu, struct worker_pool, rcu);
3526
3527	idr_destroy(&pool->worker_idr);
3528	free_workqueue_attrs(pool->attrs);
3529	kfree(pool);
3530	}
3531
3532	/**
3533	* put_unbound_pool - put a worker_pool
3534	* @pool: worker_pool to put
3535	*
3536	* Put @pool. If its refcnt reaches zero, it gets destroyed in sched-RCU
3537	* safe manner. get_unbound_pool() calls this function on its failure path
3538	* and this function should be able to release pools which went through,
3539	* successfully or not, init_worker_pool().
3540	*
3541	* Should be called with wq_pool_mutex held.
3542	*/
3543	static void put_unbound_pool(struct worker_pool *pool)
3544	{
3545	struct worker *worker;
3546
3547	lockdep_assert_held(&wq_pool_mutex);
3548
3549	if (--pool->refcnt)
3550	return;
3551
3552	/* sanity checks */
3553	if (WARN_ON(!(pool->flags & POOL_DISASSOCIATED)) \|\|
3554	WARN_ON(!list_empty(&pool->worklist)))
3555	return;
3556
3557	/* release id and unhash */
3558	if (pool->id >= 0)
3559	idr_remove(&worker_pool_idr, pool->id);
3560	hash_del(&pool->hash_node);
3561
3562	/*
3563	* Become the manager and destroy all workers. Grabbing
3564	* manager_arb prevents @pool's workers from blocking on
3565	* manager_mutex.
3566	*/
3567	mutex_lock(&pool->manager_arb);
3568	mutex_lock(&pool->manager_mutex);
3569	spin_lock_irq(&pool->lock);
3570
3571	while ((worker = first_worker(pool)))
3572	destroy_worker(worker);
3573	WARN_ON(pool->nr_workers \|\| pool->nr_idle);
3574
3575	spin_unlock_irq(&pool->lock);
3576	mutex_unlock(&pool->manager_mutex);
3577	mutex_unlock(&pool->manager_arb);
3578
3579	/* shut down the timers */
3580	del_timer_sync(&pool->idle_timer);
3581	del_timer_sync(&pool->mayday_timer);
3582
3583	/* sched-RCU protected to allow dereferences from get_work_pool() */
3584	call_rcu_sched(&pool->rcu, rcu_free_pool);
3585	}
3586
3587	/**
3588	* get_unbound_pool - get a worker_pool with the specified attributes
3589	* @attrs: the attributes of the worker_pool to get
3590	*
3591	* Obtain a worker_pool which has the same attributes as @attrs, bump the
3592	* reference count and return it. If there already is a matching
3593	* worker_pool, it will be used; otherwise, this function attempts to
3594	* create a new one.
3595	*
3596	* Should be called with wq_pool_mutex held.
3597	*
3598	* Return: On success, a worker_pool with the same attributes as @attrs.
3599	* On failure, %NULL.
3600	*/
3601	static struct worker_pool get_unbound_pool(const struct workqueue_attrs attrs)
3602	{
3603	u32 hash = wqattrs_hash(attrs);
3604	struct worker_pool *pool;
3605	int node;
3606
3607	lockdep_assert_held(&wq_pool_mutex);
3608
3609	/* do we already have a matching pool? */
3610	hash_for_each_possible(unbound_pool_hash, pool, hash_node, hash) {
3611	if (wqattrs_equal(pool->attrs, attrs)) {
3612	pool->refcnt++;
3613	goto out_unlock;
3614	}
3615	}
3616
3617	/* nope, create a new one */
3618	pool = kzalloc(sizeof(*pool), GFP_KERNEL);
3619	if (!pool \|\| init_worker_pool(pool) < 0)
3620	goto fail;
3621
3622	if (workqueue_freezing)
3623	pool->flags \|= POOL_FREEZING;
3624
3625	lockdep_set_subclass(&pool->lock, 1); /* see put_pwq() */
3626	copy_workqueue_attrs(pool->attrs, attrs);
3627
3628	/*
3629	* no_numa isn't a worker_pool attribute, always clear it. See
3630	* 'struct workqueue_attrs' comments for detail.
3631	*/
3632	pool->attrs->no_numa = false;
3633
3634	/* if cpumask is contained inside a NUMA node, we belong to that node */
3635	if (wq_numa_enabled) {
3636	for_each_node(node) {
3637	if (cpumask_subset(pool->attrs->cpumask,
3638	wq_numa_possible_cpumask[node])) {
3639	pool->node = node;
3640	break;
3641	}
3642	}
3643	}
3644
3645	if (worker_pool_assign_id(pool) < 0)
3646	goto fail;
3647
3648	/* create and start the initial worker */
3649	if (create_and_start_worker(pool) < 0)
3650	goto fail;
3651
3652	/* install */
3653	hash_add(unbound_pool_hash, &pool->hash_node, hash);
3654	out_unlock:
3655	return pool;
3656	fail:
3657	if (pool)
3658	put_unbound_pool(pool);
3659	return NULL;
3660	}
3661
3662	static void rcu_free_pwq(struct rcu_head *rcu)
3663	{
3664	kmem_cache_free(pwq_cache,
3665	container_of(rcu, struct pool_workqueue, rcu));
3666	}
3667
3668	/*
3669	* Scheduled on system_wq by put_pwq() when an unbound pwq hits zero refcnt
3670	* and needs to be destroyed.
3671	*/
3672	static void pwq_unbound_release_workfn(struct work_struct *work)
3673	{
3674	struct pool_workqueue *pwq = container_of(work, struct pool_workqueue,
3675	unbound_release_work);
3676	struct workqueue_struct *wq = pwq->wq;
3677	struct worker_pool *pool = pwq->pool;
3678	bool is_last;
3679
3680	if (WARN_ON_ONCE(!(wq->flags & WQ_UNBOUND)))
3681	return;
3682
3683	/*
3684	* Unlink @pwq. Synchronization against wq->mutex isn't strictly
3685	* necessary on release but do it anyway. It's easier to verify
3686	* and consistent with the linking path.
3687	*/
3688	mutex_lock(&wq->mutex);
3689	list_del_rcu(&pwq->pwqs_node);
3690	is_last = list_empty(&wq->pwqs);
3691	mutex_unlock(&wq->mutex);
3692
3693	mutex_lock(&wq_pool_mutex);
3694	put_unbound_pool(pool);
3695	mutex_unlock(&wq_pool_mutex);
3696
3697	call_rcu_sched(&pwq->rcu, rcu_free_pwq);
3698
3699	/*
3700	* If we're the last pwq going away, @wq is already dead and no one
3701	* is gonna access it anymore. Free it.
3702	*/
3703	if (is_last) {
3704	free_workqueue_attrs(wq->unbound_attrs);
3705	kfree(wq);
3706	}
3707	}
3708
3709	/**
3710	* pwq_adjust_max_active - update a pwq's max_active to the current setting
3711	* @pwq: target pool_workqueue
3712	*
3713	* If @pwq isn't freezing, set @pwq->max_active to the associated
3714	* workqueue's saved_max_active and activate delayed work items
3715	* accordingly. If @pwq is freezing, clear @pwq->max_active to zero.
3716	*/
3717	static void pwq_adjust_max_active(struct pool_workqueue *pwq)
3718	{
3719	struct workqueue_struct *wq = pwq->wq;
3720	bool freezable = wq->flags & WQ_FREEZABLE;
3721
3722	/* for @wq->saved_max_active */
3723	lockdep_assert_held(&wq->mutex);
3724
3725	/* fast exit for non-freezable wqs */
3726	if (!freezable && pwq->max_active == wq->saved_max_active)
3727	return;
3728
3729	spin_lock_irq(&pwq->pool->lock);
3730
3731	if (!freezable \|\| !(pwq->pool->flags & POOL_FREEZING)) {
3732	pwq->max_active = wq->saved_max_active;
3733
3734	while (!list_empty(&pwq->delayed_works) &&
3735	pwq->nr_active < pwq->max_active)
3736	pwq_activate_first_delayed(pwq);
3737
3738	/*
3739	* Need to kick a worker after thawed or an unbound wq's
3740	* max_active is bumped. It's a slow path. Do it always.
3741	*/
3742	wake_up_worker(pwq->pool);
3743	} else {
3744	pwq->max_active = 0;
3745	}
3746
3747	spin_unlock_irq(&pwq->pool->lock);
3748	}
3749
3750	/* initialize newly alloced @pwq which is associated with @wq and @pool */
3751	static void init_pwq(struct pool_workqueue pwq, struct workqueue_struct wq,
3752	struct worker_pool *pool)
3753	{
3754	BUG_ON((unsigned long)pwq & WORK_STRUCT_FLAG_MASK);
3755
3756	memset(pwq, 0, sizeof(*pwq));
3757
3758	pwq->pool = pool;
3759	pwq->wq = wq;
3760	pwq->flush_color = -1;
3761	pwq->refcnt = 1;
3762	INIT_LIST_HEAD(&pwq->delayed_works);
3763	INIT_LIST_HEAD(&pwq->pwqs_node);
3764	INIT_LIST_HEAD(&pwq->mayday_node);
3765	INIT_WORK(&pwq->unbound_release_work, pwq_unbound_release_workfn);
3766	}
3767
3768	/* sync @pwq with the current state of its associated wq and link it */
3769	static void link_pwq(struct pool_workqueue *pwq)
3770	{
3771	struct workqueue_struct *wq = pwq->wq;
3772
3773	lockdep_assert_held(&wq->mutex);
3774
3775	/* may be called multiple times, ignore if already linked */
3776	if (!list_empty(&pwq->pwqs_node))
3777	return;
3778
3779	/*
3780	* Set the matching work_color. This is synchronized with
3781	* wq->mutex to avoid confusing flush_workqueue().
3782	*/
3783	pwq->work_color = wq->work_color;
3784
3785	/* sync max_active to the current setting */
3786	pwq_adjust_max_active(pwq);
3787
3788	/* link in @pwq */
3789	list_add_rcu(&pwq->pwqs_node, &wq->pwqs);
3790	}
3791
3792	/* obtain a pool matching @attr and create a pwq associating the pool and @wq */
3793	static struct pool_workqueue alloc_unbound_pwq(struct workqueue_struct wq,
3794	const struct workqueue_attrs *attrs)
3795	{
3796	struct worker_pool *pool;
3797	struct pool_workqueue *pwq;
3798
3799	lockdep_assert_held(&wq_pool_mutex);
3800
3801	pool = get_unbound_pool(attrs);
3802	if (!pool)
3803	return NULL;
3804
3805	pwq = kmem_cache_alloc_node(pwq_cache, GFP_KERNEL, pool->node);
3806	if (!pwq) {
3807	put_unbound_pool(pool);
3808	return NULL;
3809	}
3810
3811	init_pwq(pwq, wq, pool);
3812	return pwq;
3813	}
3814
3815	/* undo alloc_unbound_pwq(), used only in the error path */
3816	static void free_unbound_pwq(struct pool_workqueue *pwq)
3817	{
3818	lockdep_assert_held(&wq_pool_mutex);
3819
3820	if (pwq) {
3821	put_unbound_pool(pwq->pool);
3822	kmem_cache_free(pwq_cache, pwq);
3823	}
3824	}
3825
3826	/**
3827	* wq_calc_node_mask - calculate a wq_attrs' cpumask for the specified node
3828	* @attrs: the wq_attrs of interest
3829	* @node: the target NUMA node
3830	* @cpu_going_down: if >= 0, the CPU to consider as offline
3831	* @cpumask: outarg, the resulting cpumask
3832	*
3833	* Calculate the cpumask a workqueue with @attrs should use on @node. If
3834	* @cpu_going_down is >= 0, that cpu is considered offline during
3835	* calculation. The result is stored in @cpumask.
3836	*
3837	* If NUMA affinity is not enabled, @attrs->cpumask is always used. If
3838	* enabled and @node has online CPUs requested by @attrs, the returned
3839	* cpumask is the intersection of the possible CPUs of @node and
3840	* @attrs->cpumask.
3841	*
3842	* The caller is responsible for ensuring that the cpumask of @node stays
3843	* stable.
3844	*
3845	* Return: %true if the resulting @cpumask is different from @attrs->cpumask,
3846	* %false if equal.
3847	*/
3848	static bool wq_calc_node_cpumask(const struct workqueue_attrs *attrs, int node,
3849	int cpu_going_down, cpumask_t *cpumask)
3850	{
3851	if (!wq_numa_enabled \|\| attrs->no_numa)
3852	goto use_dfl;
3853
3854	/* does @node have any online CPUs @attrs wants? */
3855	cpumask_and(cpumask, cpumask_of_node(node), attrs->cpumask);
3856	if (cpu_going_down >= 0)
3857	cpumask_clear_cpu(cpu_going_down, cpumask);
3858
3859	if (cpumask_empty(cpumask))
3860	goto use_dfl;
3861
3862	/* yeap, return possible CPUs in @node that @attrs wants */
3863	cpumask_and(cpumask, attrs->cpumask, wq_numa_possible_cpumask[node]);
3864	return !cpumask_equal(cpumask, attrs->cpumask);
3865
3866	use_dfl:
3867	cpumask_copy(cpumask, attrs->cpumask);
3868	return false;
3869	}
3870
3871	/* install @pwq into @wq's numa_pwq_tbl[] for @node and return the old pwq */
3872	static struct pool_workqueue numa_pwq_tbl_install(struct workqueue_struct wq,
3873	int node,
3874	struct pool_workqueue *pwq)
3875	{
3876	struct pool_workqueue *old_pwq;
3877
3878	lockdep_assert_held(&wq->mutex);
3879
3880	/* link_pwq() can handle duplicate calls */
3881	link_pwq(pwq);
3882
3883	old_pwq = rcu_access_pointer(wq->numa_pwq_tbl[node]);
3884	rcu_assign_pointer(wq->numa_pwq_tbl[node], pwq);
3885	return old_pwq;
3886	}
3887
3888	/**
3889	* apply_workqueue_attrs - apply new workqueue_attrs to an unbound workqueue
3890	* @wq: the target workqueue
3891	* @attrs: the workqueue_attrs to apply, allocated with alloc_workqueue_attrs()
3892	*
3893	* Apply @attrs to an unbound workqueue @wq. Unless disabled, on NUMA
3894	* machines, this function maps a separate pwq to each NUMA node with
3895	* possibles CPUs in @attrs->cpumask so that work items are affine to the
3896	* NUMA node it was issued on. Older pwqs are released as in-flight work
3897	* items finish. Note that a work item which repeatedly requeues itself
3898	* back-to-back will stay on its current pwq.
3899	*
3900	* Performs GFP_KERNEL allocations.
3901	*
3902	* Return: 0 on success and -errno on failure.
3903	*/
3904	int apply_workqueue_attrs(struct workqueue_struct *wq,
3905	const struct workqueue_attrs *attrs)
3906	{
3907	struct workqueue_attrs new_attrs, tmp_attrs;
3908	struct pool_workqueue *pwq_tbl, dfl_pwq;
3909	int node, ret;
3910
3911	/* only unbound workqueues can change attributes */
3912	if (WARN_ON(!(wq->flags & WQ_UNBOUND)))
3913	return -EINVAL;
3914
3915	/* creating multiple pwqs breaks ordering guarantee */
3916	if (WARN_ON((wq->flags & __WQ_ORDERED) && !list_empty(&wq->pwqs)))
3917	return -EINVAL;
3918
3919	pwq_tbl = kzalloc(wq_numa_tbl_len * sizeof(pwq_tbl[0]), GFP_KERNEL);
3920	new_attrs = alloc_workqueue_attrs(GFP_KERNEL);
3921	tmp_attrs = alloc_workqueue_attrs(GFP_KERNEL);
3922	if (!pwq_tbl \|\| !new_attrs \|\| !tmp_attrs)
3923	goto enomem;
3924
3925	/* make a copy of @attrs and sanitize it */
3926	copy_workqueue_attrs(new_attrs, attrs);
3927	cpumask_and(new_attrs->cpumask, new_attrs->cpumask, cpu_possible_mask);
3928
3929	/*
3930	* We may create multiple pwqs with differing cpumasks. Make a
3931	* copy of @new_attrs which will be modified and used to obtain
3932	* pools.
3933	*/
3934	copy_workqueue_attrs(tmp_attrs, new_attrs);
3935
3936	/*
3937	* CPUs should stay stable across pwq creations and installations.
3938	* Pin CPUs, determine the target cpumask for each node and create
3939	* pwqs accordingly.
3940	*/
3941	get_online_cpus();
3942
3943	mutex_lock(&wq_pool_mutex);
3944
3945	/*
3946	* If something goes wrong during CPU up/down, we'll fall back to
3947	* the default pwq covering whole @attrs->cpumask. Always create
3948	* it even if we don't use it immediately.
3949	*/
3950	dfl_pwq = alloc_unbound_pwq(wq, new_attrs);
3951	if (!dfl_pwq)
3952	goto enomem_pwq;
3953
3954	for_each_node(node) {
3955	if (wq_calc_node_cpumask(attrs, node, -1, tmp_attrs->cpumask)) {
3956	pwq_tbl[node] = alloc_unbound_pwq(wq, tmp_attrs);
3957	if (!pwq_tbl[node])
3958	goto enomem_pwq;
3959	} else {
3960	dfl_pwq->refcnt++;
3961	pwq_tbl[node] = dfl_pwq;
3962	}
3963	}
3964
3965	mutex_unlock(&wq_pool_mutex);
3966
3967	/* all pwqs have been created successfully, let's install'em */
3968	mutex_lock(&wq->mutex);
3969
3970	copy_workqueue_attrs(wq->unbound_attrs, new_attrs);
3971
3972	/* save the previous pwq and install the new one */
3973	for_each_node(node)
3974	pwq_tbl[node] = numa_pwq_tbl_install(wq, node, pwq_tbl[node]);
3975
3976	/* @dfl_pwq might not have been used, ensure it's linked */
3977	link_pwq(dfl_pwq);
3978	swap(wq->dfl_pwq, dfl_pwq);
3979
3980	mutex_unlock(&wq->mutex);
3981
3982	/* put the old pwqs */
3983	for_each_node(node)
3984	put_pwq_unlocked(pwq_tbl[node]);
3985	put_pwq_unlocked(dfl_pwq);
3986
3987	put_online_cpus();
3988	ret = 0;
3989	/* fall through */
3990	out_free:
3991	free_workqueue_attrs(tmp_attrs);
3992	free_workqueue_attrs(new_attrs);
3993	kfree(pwq_tbl);
3994	return ret;
3995
3996	enomem_pwq:
3997	free_unbound_pwq(dfl_pwq);
3998	for_each_node(node)
3999	if (pwq_tbl && pwq_tbl[node] != dfl_pwq)
4000	free_unbound_pwq(pwq_tbl[node]);
4001	mutex_unlock(&wq_pool_mutex);
4002	put_online_cpus();
4003	enomem:
4004	ret = -ENOMEM;
4005	goto out_free;
4006	}
4007
4008	/**
4009	* wq_update_unbound_numa - update NUMA affinity of a wq for CPU hot[un]plug
4010	* @wq: the target workqueue
4011	* @cpu: the CPU coming up or going down
4012	* @online: whether @cpu is coming up or going down
4013	*
4014	* This function is to be called from %CPU_DOWN_PREPARE, %CPU_ONLINE and
4015	* %CPU_DOWN_FAILED. @cpu is being hot[un]plugged, update NUMA affinity of
4016	* @wq accordingly.
4017	*
4018	* If NUMA affinity can't be adjusted due to memory allocation failure, it
4019	* falls back to @wq->dfl_pwq which may not be optimal but is always
4020	* correct.
4021	*
4022	* Note that when the last allowed CPU of a NUMA node goes offline for a
4023	* workqueue with a cpumask spanning multiple nodes, the workers which were
4024	* already executing the work items for the workqueue will lose their CPU
4025	* affinity and may execute on any CPU. This is similar to how per-cpu
4026	* workqueues behave on CPU_DOWN. If a workqueue user wants strict
4027	* affinity, it's the user's responsibility to flush the work item from
4028	* CPU_DOWN_PREPARE.
4029	*/
4030	static void wq_update_unbound_numa(struct workqueue_struct *wq, int cpu,
4031	bool online)
4032	{
4033	int node = cpu_to_node(cpu);
4034	int cpu_off = online ? -1 : cpu;
4035	struct pool_workqueue old_pwq = NULL, pwq;
4036	struct workqueue_attrs *target_attrs;
4037	cpumask_t *cpumask;
4038
4039	lockdep_assert_held(&wq_pool_mutex);
4040
4041	if (!wq_numa_enabled \|\| !(wq->flags & WQ_UNBOUND))
4042	return;
4043
4044	/*
4045	* We don't wanna alloc/free wq_attrs for each wq for each CPU.
4046	* Let's use a preallocated one. The following buf is protected by
4047	* CPU hotplug exclusion.
4048	*/
4049	target_attrs = wq_update_unbound_numa_attrs_buf;
4050	cpumask = target_attrs->cpumask;
4051
4052	mutex_lock(&wq->mutex);
4053	if (wq->unbound_attrs->no_numa)
4054	goto out_unlock;
4055
4056	copy_workqueue_attrs(target_attrs, wq->unbound_attrs);
4057	pwq = unbound_pwq_by_node(wq, node);
4058
4059	/*
4060	* Let's determine what needs to be done. If the target cpumask is
4061	* different from wq's, we need to compare it to @pwq's and create
4062	* a new one if they don't match. If the target cpumask equals
4063	* wq's, the default pwq should be used. If @pwq is already the
4064	* default one, nothing to do; otherwise, install the default one.
4065	*/
4066	if (wq_calc_node_cpumask(wq->unbound_attrs, node, cpu_off, cpumask)) {
4067	if (cpumask_equal(cpumask, pwq->pool->attrs->cpumask))
4068	goto out_unlock;
4069	} else {
4070	if (pwq == wq->dfl_pwq)
4071	goto out_unlock;
4072	else
4073	goto use_dfl_pwq;
4074	}
4075
4076	mutex_unlock(&wq->mutex);
4077
4078	/* create a new pwq */
4079	pwq = alloc_unbound_pwq(wq, target_attrs);
4080	if (!pwq) {
4081	pr_warning("workqueue: allocation failed while updating NUMA affinity of \"%s\"\n",
4082	wq->name);
4083	goto out_unlock;
4084	}
4085
4086	/*
4087	* Install the new pwq. As this function is called only from CPU
4088	* hotplug callbacks and applying a new attrs is wrapped with
4089	* get/put_online_cpus(), @wq->unbound_attrs couldn't have changed
4090	* inbetween.
4091	*/
4092	mutex_lock(&wq->mutex);
4093	old_pwq = numa_pwq_tbl_install(wq, node, pwq);
4094	goto out_unlock;
4095
4096	use_dfl_pwq:
4097	spin_lock_irq(&wq->dfl_pwq->pool->lock);
4098	get_pwq(wq->dfl_pwq);
4099	spin_unlock_irq(&wq->dfl_pwq->pool->lock);
4100	old_pwq = numa_pwq_tbl_install(wq, node, wq->dfl_pwq);
4101	out_unlock:
4102	mutex_unlock(&wq->mutex);
4103	put_pwq_unlocked(old_pwq);
4104	}
4105
4106	static int alloc_and_link_pwqs(struct workqueue_struct *wq)
4107	{
4108	bool highpri = wq->flags & WQ_HIGHPRI;
4109	int cpu;
4110
4111	if (!(wq->flags & WQ_UNBOUND)) {
4112	wq->cpu_pwqs = alloc_percpu(struct pool_workqueue);
4113	if (!wq->cpu_pwqs)
4114	return -ENOMEM;
4115
4116	for_each_possible_cpu(cpu) {
4117	struct pool_workqueue *pwq =
4118	per_cpu_ptr(wq->cpu_pwqs, cpu);
4119	struct worker_pool *cpu_pools =
4120	per_cpu(cpu_worker_pools, cpu);
4121
4122	init_pwq(pwq, wq, &cpu_pools[highpri]);
4123
4124	mutex_lock(&wq->mutex);
4125	link_pwq(pwq);
4126	mutex_unlock(&wq->mutex);
4127	}
4128	return 0;
4129	} else {
4130	return apply_workqueue_attrs(wq, unbound_std_wq_attrs[highpri]);
4131	}
4132	}
4133
4134	static int wq_clamp_max_active(int max_active, unsigned int flags,
4135	const char *name)
4136	{
4137	int lim = flags & WQ_UNBOUND ? WQ_UNBOUND_MAX_ACTIVE : WQ_MAX_ACTIVE;
4138
4139	if (max_active < 1 \|\| max_active > lim)
4140	pr_warn("workqueue: max_active %d requested for %s is out of range, clamping between %d and %d\n",
4141	max_active, name, 1, lim);
4142
4143	return clamp_val(max_active, 1, lim);
4144	}
4145
4146	struct workqueue_struct __alloc_workqueue_key(const char fmt,
4147	unsigned int flags,
4148	int max_active,
4149	struct lock_class_key *key,
4150	const char *lock_name, ...)
4151	{
4152	size_t tbl_size = 0;
4153	va_list args;
4154	struct workqueue_struct *wq;
4155	struct pool_workqueue *pwq;
4156
4157	/* see the comment above the definition of WQ_POWER_EFFICIENT */
4158	if ((flags & WQ_POWER_EFFICIENT) && wq_power_efficient)
4159	flags \|= WQ_UNBOUND;
4160
4161	/* allocate wq and format name */
4162	if (flags & WQ_UNBOUND)
4163	tbl_size = wq_numa_tbl_len * sizeof(wq->numa_pwq_tbl[0]);
4164
4165	wq = kzalloc(sizeof(*wq) + tbl_size, GFP_KERNEL);
4166	if (!wq)
4167	return NULL;
4168
4169	if (flags & WQ_UNBOUND) {
4170	wq->unbound_attrs = alloc_workqueue_attrs(GFP_KERNEL);
4171	if (!wq->unbound_attrs)
4172	goto err_free_wq;
4173	}
4174
4175	va_start(args, lock_name);
4176	vsnprintf(wq->name, sizeof(wq->name), fmt, args);
4177	va_end(args);
4178
4179	max_active = max_active ?: WQ_DFL_ACTIVE;
4180	max_active = wq_clamp_max_active(max_active, flags, wq->name);
4181
4182	/* init wq */
4183	wq->flags = flags;
4184	wq->saved_max_active = max_active;
4185	mutex_init(&wq->mutex);
4186	atomic_set(&wq->nr_pwqs_to_flush, 0);
4187	INIT_LIST_HEAD(&wq->pwqs);
4188	INIT_LIST_HEAD(&wq->flusher_queue);
4189	INIT_LIST_HEAD(&wq->flusher_overflow);
4190	INIT_LIST_HEAD(&wq->maydays);
4191
4192	lockdep_init_map(&wq->lockdep_map, lock_name, key, 0);
4193	INIT_LIST_HEAD(&wq->list);
4194
4195	if (alloc_and_link_pwqs(wq) < 0)
4196	goto err_free_wq;
4197
4198	/*
4199	* Workqueues which may be used during memory reclaim should
4200	* have a rescuer to guarantee forward progress.
4201	*/
4202	if (flags & WQ_MEM_RECLAIM) {
4203	struct worker *rescuer;
4204
4205	rescuer = alloc_worker();
4206	if (!rescuer)
4207	goto err_destroy;
4208
4209	rescuer->rescue_wq = wq;
4210	rescuer->task = kthread_create(rescuer_thread, rescuer, "%s",
4211	wq->name);
4212	if (IS_ERR(rescuer->task)) {
4213	kfree(rescuer);
4214	goto err_destroy;
4215	}
4216
4217	wq->rescuer = rescuer;
4218	rescuer->task->flags \|= PF_NO_SETAFFINITY;
4219	wake_up_process(rescuer->task);
4220	}
4221
4222	if ((wq->flags & WQ_SYSFS) && workqueue_sysfs_register(wq))
4223	goto err_destroy;
4224
4225	/*
4226	* wq_pool_mutex protects global freeze state and workqueues list.
4227	* Grab it, adjust max_active and add the new @wq to workqueues
4228	* list.
4229	*/
4230	mutex_lock(&wq_pool_mutex);
4231
4232	mutex_lock(&wq->mutex);
4233	for_each_pwq(pwq, wq)
4234	pwq_adjust_max_active(pwq);
4235	mutex_unlock(&wq->mutex);
4236
4237	list_add(&wq->list, &workqueues);
4238
4239	mutex_unlock(&wq_pool_mutex);
4240
4241	return wq;
4242
4243	err_free_wq:
4244	free_workqueue_attrs(wq->unbound_attrs);
4245	kfree(wq);
4246	return NULL;
4247	err_destroy:
4248	destroy_workqueue(wq);
4249	return NULL;
4250	}
4251	EXPORT_SYMBOL_GPL(__alloc_workqueue_key);
4252
4253	/**
4254	* destroy_workqueue - safely terminate a workqueue
4255	* @wq: target workqueue
4256	*
4257	* Safely destroy a workqueue. All work currently pending will be done first.
4258	*/
4259	void destroy_workqueue(struct workqueue_struct *wq)
4260	{
4261	struct pool_workqueue *pwq;
4262	int node;
4263
4264	/* drain it before proceeding with destruction */
4265	drain_workqueue(wq);
4266
4267	/* sanity checks */
4268	mutex_lock(&wq->mutex);
4269	for_each_pwq(pwq, wq) {
4270	int i;
4271
4272	for (i = 0; i < WORK_NR_COLORS; i++) {
4273	if (WARN_ON(pwq->nr_in_flight[i])) {
4274	mutex_unlock(&wq->mutex);
4275	return;
4276	}
4277	}
4278
4279	if (WARN_ON((pwq != wq->dfl_pwq) && (pwq->refcnt > 1)) \|\|
4280	WARN_ON(pwq->nr_active) \|\|
4281	WARN_ON(!list_empty(&pwq->delayed_works))) {
4282	mutex_unlock(&wq->mutex);
4283	return;
4284	}
4285	}
4286	mutex_unlock(&wq->mutex);
4287
4288	/*
4289	* wq list is used to freeze wq, remove from list after
4290	* flushing is complete in case freeze races us.
4291	*/
4292	mutex_lock(&wq_pool_mutex);
4293	list_del_init(&wq->list);
4294	mutex_unlock(&wq_pool_mutex);
4295
4296	workqueue_sysfs_unregister(wq);
4297
4298	if (wq->rescuer) {
4299	kthread_stop(wq->rescuer->task);
4300	kfree(wq->rescuer);
4301	wq->rescuer = NULL;
4302	}
4303
4304	if (!(wq->flags & WQ_UNBOUND)) {
4305	/*
4306	* The base ref is never dropped on per-cpu pwqs. Directly
4307	* free the pwqs and wq.
4308	*/
4309	free_percpu(wq->cpu_pwqs);
4310	kfree(wq);
4311	} else {
4312	/*
4313	* We're the sole accessor of @wq at this point. Directly
4314	* access numa_pwq_tbl[] and dfl_pwq to put the base refs.
4315	* @wq will be freed when the last pwq is released.
4316	*/
4317	for_each_node(node) {
4318	pwq = rcu_access_pointer(wq->numa_pwq_tbl[node]);
4319	RCU_INIT_POINTER(wq->numa_pwq_tbl[node], NULL);
4320	put_pwq_unlocked(pwq);
4321	}
4322
4323	/*
4324	* Put dfl_pwq. @wq may be freed any time after dfl_pwq is
4325	* put. Don't access it afterwards.
4326	*/
4327	pwq = wq->dfl_pwq;
4328	wq->dfl_pwq = NULL;
4329	put_pwq_unlocked(pwq);
4330	}
4331	}
4332	EXPORT_SYMBOL_GPL(destroy_workqueue);
4333
4334	/**
4335	* workqueue_set_max_active - adjust max_active of a workqueue
4336	* @wq: target workqueue
4337	* @max_active: new max_active value.
4338	*
4339	* Set max_active of @wq to @max_active.
4340	*
4341	* CONTEXT:
4342	* Don't call from IRQ context.
4343	*/
4344	void workqueue_set_max_active(struct workqueue_struct *wq, int max_active)
4345	{
4346	struct pool_workqueue *pwq;
4347
4348	/* disallow meddling with max_active for ordered workqueues */
4349	if (WARN_ON(wq->flags & __WQ_ORDERED))
4350	return;
4351
4352	max_active = wq_clamp_max_active(max_active, wq->flags, wq->name);
4353
4354	mutex_lock(&wq->mutex);
4355
4356	wq->saved_max_active = max_active;
4357
4358	for_each_pwq(pwq, wq)
4359	pwq_adjust_max_active(pwq);
4360
4361	mutex_unlock(&wq->mutex);
4362	}
4363	EXPORT_SYMBOL_GPL(workqueue_set_max_active);
4364
4365	/**
4366	* current_is_workqueue_rescuer - is %current workqueue rescuer?
4367	*
4368	* Determine whether %current is a workqueue rescuer. Can be used from
4369	* work functions to determine whether it's being run off the rescuer task.
4370	*
4371	* Return: %true if %current is a workqueue rescuer. %false otherwise.
4372	*/
4373	bool current_is_workqueue_rescuer(void)
4374	{
4375	struct worker *worker = current_wq_worker();
4376
4377	return worker && worker->rescue_wq;
4378	}
4379
4380	/**
4381	* workqueue_congested - test whether a workqueue is congested
4382	* @cpu: CPU in question
4383	* @wq: target workqueue
4384	*
4385	* Test whether @wq's cpu workqueue for @cpu is congested. There is
4386	* no synchronization around this function and the test result is
4387	* unreliable and only useful as advisory hints or for debugging.
4388	*
4389	* If @cpu is WORK_CPU_UNBOUND, the test is performed on the local CPU.
4390	* Note that both per-cpu and unbound workqueues may be associated with
4391	* multiple pool_workqueues which have separate congested states. A
4392	* workqueue being congested on one CPU doesn't mean the workqueue is also
4393	* contested on other CPUs / NUMA nodes.
4394	*
4395	* Return:
4396	* %true if congested, %false otherwise.
4397	*/
4398	bool workqueue_congested(int cpu, struct workqueue_struct *wq)
4399	{
4400	struct pool_workqueue *pwq;
4401	bool ret;
4402
4403	rcu_read_lock_sched();
4404
4405	if (cpu == WORK_CPU_UNBOUND)
4406	cpu = smp_processor_id();
4407
4408	if (!(wq->flags & WQ_UNBOUND))
4409	pwq = per_cpu_ptr(wq->cpu_pwqs, cpu);
4410	else
4411	pwq = unbound_pwq_by_node(wq, cpu_to_node(cpu));
4412
4413	ret = !list_empty(&pwq->delayed_works);
4414	rcu_read_unlock_sched();
4415
4416	return ret;
4417	}
4418	EXPORT_SYMBOL_GPL(workqueue_congested);
4419
4420	/**
4421	* work_busy - test whether a work is currently pending or running
4422	* @work: the work to be tested
4423	*
4424	* Test whether @work is currently pending or running. There is no
4425	* synchronization around this function and the test result is
4426	* unreliable and only useful as advisory hints or for debugging.
4427	*
4428	* Return:
4429	* OR'd bitmask of WORK_BUSY_* bits.
4430	*/
4431	unsigned int work_busy(struct work_struct *work)
4432	{
4433	struct worker_pool *pool;
4434	unsigned long flags;
4435	unsigned int ret = 0;
4436
4437	if (work_pending(work))
4438	ret \|= WORK_BUSY_PENDING;
4439
4440	local_irq_save(flags);
4441	pool = get_work_pool(work);
4442	if (pool) {
4443	spin_lock(&pool->lock);
4444	if (find_worker_executing_work(pool, work))
4445	ret \|= WORK_BUSY_RUNNING;
4446	spin_unlock(&pool->lock);
4447	}
4448	local_irq_restore(flags);
4449
4450	return ret;
4451	}
4452	EXPORT_SYMBOL_GPL(work_busy);
4453
4454	/**
4455	* set_worker_desc - set description for the current work item
4456	* @fmt: printf-style format string
4457	* @...: arguments for the format string
4458	*
4459	* This function can be called by a running work function to describe what
4460	* the work item is about. If the worker task gets dumped, this
4461	* information will be printed out together to help debugging. The
4462	* description can be at most WORKER_DESC_LEN including the trailing '\0'.
4463	*/
4464	void set_worker_desc(const char *fmt, ...)
4465	{
4466	struct worker *worker = current_wq_worker();
4467	va_list args;
4468
4469	if (worker) {
4470	va_start(args, fmt);
4471	vsnprintf(worker->desc, sizeof(worker->desc), fmt, args);
4472	va_end(args);
4473	worker->desc_valid = true;
4474	}
4475	}
4476
4477	/**
4478	* print_worker_info - print out worker information and description
4479	* @log_lvl: the log level to use when printing
4480	* @task: target task
4481	*
4482	* If @task is a worker and currently executing a work item, print out the
4483	* name of the workqueue being serviced and worker description set with
4484	* set_worker_desc() by the currently executing work item.
4485	*
4486	* This function can be safely called on any task as long as the
4487	* task_struct itself is accessible. While safe, this function isn't
4488	* synchronized and may print out mixups or garbages of limited length.
4489	*/
4490	void print_worker_info(const char log_lvl, struct task_struct task)
4491	{
4492	work_func_t *fn = NULL;
4493	char name[WQ_NAME_LEN] = { };
4494	char desc[WORKER_DESC_LEN] = { };
4495	struct pool_workqueue *pwq = NULL;
4496	struct workqueue_struct *wq = NULL;
4497	bool desc_valid = false;
4498	struct worker *worker;
4499
4500	if (!(task->flags & PF_WQ_WORKER))
4501	return;
4502
4503	/*
4504	* This function is called without any synchronization and @task
4505	* could be in any state. Be careful with dereferences.
4506	*/
4507	worker = probe_kthread_data(task);
4508
4509	/*
4510	* Carefully copy the associated workqueue's workfn and name. Keep
4511	* the original last '\0' in case the original contains garbage.
4512	*/
4513	probe_kernel_read(&fn, &worker->current_func, sizeof(fn));
4514	probe_kernel_read(&pwq, &worker->current_pwq, sizeof(pwq));
4515	probe_kernel_read(&wq, &pwq->wq, sizeof(wq));
4516	probe_kernel_read(name, wq->name, sizeof(name) - 1);
4517
4518	/* copy worker description */
4519	probe_kernel_read(&desc_valid, &worker->desc_valid, sizeof(desc_valid));
4520	if (desc_valid)
4521	probe_kernel_read(desc, worker->desc, sizeof(desc) - 1);
4522
4523	if (fn \|\| name[0] \|\| desc[0]) {
4524	printk("%sWorkqueue: %s %pf", log_lvl, name, fn);
4525	if (desc[0])
4526	pr_cont(" (%s)", desc);
4527	pr_cont("\n");
4528	}
4529	}
4530
4531	/*
4532	* CPU hotplug.
4533	*
4534	* There are two challenges in supporting CPU hotplug. Firstly, there
4535	* are a lot of assumptions on strong associations among work, pwq and
4536	* pool which make migrating pending and scheduled works very
4537	* difficult to implement without impacting hot paths. Secondly,
4538	* worker pools serve mix of short, long and very long running works making
4539	* blocked draining impractical.
4540	*
4541	* This is solved by allowing the pools to be disassociated from the CPU
4542	* running as an unbound one and allowing it to be reattached later if the
4543	* cpu comes back online.
4544	*/
4545
4546	static void wq_unbind_fn(struct work_struct *work)
4547	{
4548	int cpu = smp_processor_id();
4549	struct worker_pool *pool;
4550	struct worker *worker;
4551	int wi;
4552
4553	for_each_cpu_worker_pool(pool, cpu) {
4554	WARN_ON_ONCE(cpu != smp_processor_id());
4555
4556	mutex_lock(&pool->manager_mutex);
4557	spin_lock_irq(&pool->lock);
4558
4559	/*
4560	* We've blocked all manager operations. Make all workers
4561	* unbound and set DISASSOCIATED. Before this, all workers
4562	* except for the ones which are still executing works from
4563	* before the last CPU down must be on the cpu. After
4564	* this, they may become diasporas.
4565	*/
4566	for_each_pool_worker(worker, wi, pool)
4567	worker->flags \|= WORKER_UNBOUND;
4568
4569	pool->flags \|= POOL_DISASSOCIATED;
4570
4571	spin_unlock_irq(&pool->lock);
4572	mutex_unlock(&pool->manager_mutex);
4573
4574	/*
4575	* Call schedule() so that we cross rq->lock and thus can
4576	* guarantee sched callbacks see the %WORKER_UNBOUND flag.
4577	* This is necessary as scheduler callbacks may be invoked
4578	* from other cpus.
4579	*/
4580	schedule();
4581
4582	/*
4583	* Sched callbacks are disabled now. Zap nr_running.
4584	* After this, nr_running stays zero and need_more_worker()
4585	* and keep_working() are always true as long as the
4586	* worklist is not empty. This pool now behaves as an
4587	* unbound (in terms of concurrency management) pool which
4588	* are served by workers tied to the pool.
4589	*/
4590	atomic_set(&pool->nr_running, 0);
4591
4592	/*
4593	* With concurrency management just turned off, a busy
4594	* worker blocking could lead to lengthy stalls. Kick off
4595	* unbound chain execution of currently pending work items.
4596	*/
4597	spin_lock_irq(&pool->lock);
4598	wake_up_worker(pool);
4599	spin_unlock_irq(&pool->lock);
4600	}
4601	}
4602
4603	/**
4604	* rebind_workers - rebind all workers of a pool to the associated CPU
4605	* @pool: pool of interest
4606	*
4607	* @pool->cpu is coming online. Rebind all workers to the CPU.
4608	*/
4609	static void rebind_workers(struct worker_pool *pool)
4610	{
4611	struct worker *worker;
4612	int wi;
4613
4614	lockdep_assert_held(&pool->manager_mutex);
4615
4616	/*
4617	* Restore CPU affinity of all workers. As all idle workers should
4618	* be on the run-queue of the associated CPU before any local
4619	* wake-ups for concurrency management happen, restore CPU affinty
4620	* of all workers first and then clear UNBOUND. As we're called
4621	* from CPU_ONLINE, the following shouldn't fail.
4622	*/
4623	for_each_pool_worker(worker, wi, pool)
4624	WARN_ON_ONCE(set_cpus_allowed_ptr(worker->task,
4625	pool->attrs->cpumask) < 0);
4626
4627	spin_lock_irq(&pool->lock);
4628
4629	for_each_pool_worker(worker, wi, pool) {
4630	unsigned int worker_flags = worker->flags;
4631
4632	/*
4633	* A bound idle worker should actually be on the runqueue
4634	* of the associated CPU for local wake-ups targeting it to
4635	* work. Kick all idle workers so that they migrate to the
4636	* associated CPU. Doing this in the same loop as
4637	* replacing UNBOUND with REBOUND is safe as no worker will
4638	* be bound before @pool->lock is released.
4639	*/
4640	if (worker_flags & WORKER_IDLE)
4641	wake_up_process(worker->task);
4642
4643	/*
4644	* We want to clear UNBOUND but can't directly call
4645	* worker_clr_flags() or adjust nr_running. Atomically
4646	* replace UNBOUND with another NOT_RUNNING flag REBOUND.
4647	* @worker will clear REBOUND using worker_clr_flags() when
4648	* it initiates the next execution cycle thus restoring
4649	* concurrency management. Note that when or whether
4650	* @worker clears REBOUND doesn't affect correctness.
4651	*
4652	* ACCESS_ONCE() is necessary because @worker->flags may be
4653	* tested without holding any lock in
4654	* wq_worker_waking_up(). Without it, NOT_RUNNING test may
4655	* fail incorrectly leading to premature concurrency
4656	* management operations.
4657	*/
4658	WARN_ON_ONCE(!(worker_flags & WORKER_UNBOUND));
4659	worker_flags \|= WORKER_REBOUND;
4660	worker_flags &= ~WORKER_UNBOUND;
4661	ACCESS_ONCE(worker->flags) = worker_flags;
4662	}
4663
4664	spin_unlock_irq(&pool->lock);
4665	}
4666
4667	/**
4668	* restore_unbound_workers_cpumask - restore cpumask of unbound workers
4669	* @pool: unbound pool of interest
4670	* @cpu: the CPU which is coming up
4671	*
4672	* An unbound pool may end up with a cpumask which doesn't have any online
4673	* CPUs. When a worker of such pool get scheduled, the scheduler resets
4674	* its cpus_allowed. If @cpu is in @pool's cpumask which didn't have any
4675	* online CPU before, cpus_allowed of all its workers should be restored.
4676	*/
4677	static void restore_unbound_workers_cpumask(struct worker_pool *pool, int cpu)
4678	{
4679	static cpumask_t cpumask;
4680	struct worker *worker;
4681	int wi;
4682
4683	lockdep_assert_held(&pool->manager_mutex);
4684
4685	/* is @cpu allowed for @pool? */
4686	if (!cpumask_test_cpu(cpu, pool->attrs->cpumask))
4687	return;
4688
4689	/* is @cpu the only online CPU? */
4690	cpumask_and(&cpumask, pool->attrs->cpumask, cpu_online_mask);
4691	if (cpumask_weight(&cpumask) != 1)
4692	return;
4693
4694	/* as we're called from CPU_ONLINE, the following shouldn't fail */
4695	for_each_pool_worker(worker, wi, pool)
4696	WARN_ON_ONCE(set_cpus_allowed_ptr(worker->task,
4697	pool->attrs->cpumask) < 0);
4698	}
4699
4700	/*
4701	* Workqueues should be brought up before normal priority CPU notifiers.
4702	* This will be registered high priority CPU notifier.
4703	*/
4704	static int workqueue_cpu_up_callback(struct notifier_block *nfb,
4705	unsigned long action,
4706	void *hcpu)
4707	{
4708	int cpu = (unsigned long)hcpu;
4709	struct worker_pool *pool;
4710	struct workqueue_struct *wq;
4711	int pi;
4712
4713	switch (action & ~CPU_TASKS_FROZEN) {
4714	case CPU_UP_PREPARE:
4715	for_each_cpu_worker_pool(pool, cpu) {
4716	if (pool->nr_workers)
4717	continue;
4718	if (create_and_start_worker(pool) < 0)
4719	return NOTIFY_BAD;
4720	}
4721	break;
4722
4723	case CPU_DOWN_FAILED:
4724	case CPU_ONLINE:
4725	mutex_lock(&wq_pool_mutex);
4726
4727	for_each_pool(pool, pi) {
4728	mutex_lock(&pool->manager_mutex);
4729
4730	if (pool->cpu == cpu) {
4731	spin_lock_irq(&pool->lock);
4732	pool->flags &= ~POOL_DISASSOCIATED;
4733	spin_unlock_irq(&pool->lock);
4734
4735	rebind_workers(pool);
4736	} else if (pool->cpu < 0) {
4737	restore_unbound_workers_cpumask(pool, cpu);
4738	}
4739
4740	mutex_unlock(&pool->manager_mutex);
4741	}
4742
4743	/* update NUMA affinity of unbound workqueues */
4744	list_for_each_entry(wq, &workqueues, list)
4745	wq_update_unbound_numa(wq, cpu, true);
4746
4747	mutex_unlock(&wq_pool_mutex);
4748	break;
4749	}
4750	return NOTIFY_OK;
4751	}
4752
4753	/*
4754	* Workqueues should be brought down after normal priority CPU notifiers.
4755	* This will be registered as low priority CPU notifier.
4756	*/
4757	static int workqueue_cpu_down_callback(struct notifier_block *nfb,
4758	unsigned long action,
4759	void *hcpu)
4760	{
4761	int cpu = (unsigned long)hcpu;
4762	struct work_struct unbind_work;
4763	struct workqueue_struct *wq;
4764
4765	switch (action & ~CPU_TASKS_FROZEN) {
4766	case CPU_DOWN_PREPARE:
4767	/* unbinding per-cpu workers should happen on the local CPU */
4768	INIT_WORK_ONSTACK(&unbind_work, wq_unbind_fn);
4769	queue_work_on(cpu, system_highpri_wq, &unbind_work);
4770
4771	/* update NUMA affinity of unbound workqueues */
4772	mutex_lock(&wq_pool_mutex);
4773	list_for_each_entry(wq, &workqueues, list)
4774	wq_update_unbound_numa(wq, cpu, false);
4775	mutex_unlock(&wq_pool_mutex);
4776
4777	/* wait for per-cpu unbinding to finish */
4778	flush_work(&unbind_work);
4779	break;
4780	}
4781	return NOTIFY_OK;
4782	}
4783
4784	#ifdef CONFIG_SMP
4785
4786	struct work_for_cpu {
4787	struct work_struct work;
4788	long (fn)(void );
4789	void *arg;
4790	long ret;
4791	};
4792
4793	static void work_for_cpu_fn(struct work_struct *work)
4794	{
4795	struct work_for_cpu *wfc = container_of(work, struct work_for_cpu, work);
4796
4797	wfc->ret = wfc->fn(wfc->arg);
4798	}
4799
4800	/**
4801	* work_on_cpu - run a function in user context on a particular cpu
4802	* @cpu: the cpu to run on
4803	* @fn: the function to run
4804	* @arg: the function arg
4805	*
4806	* It is up to the caller to ensure that the cpu doesn't go offline.
4807	* The caller must not hold any locks which would prevent @fn from completing.
4808	*
4809	* Return: The value @fn returns.
4810	*/
4811	long work_on_cpu(int cpu, long (fn)(void ), void *arg)
4812	{
4813	struct work_for_cpu wfc = { .fn = fn, .arg = arg };
4814
4815	INIT_WORK_ONSTACK(&wfc.work, work_for_cpu_fn);
4816	schedule_work_on(cpu, &wfc.work);
4817
4818	/*
4819	* The work item is on-stack and can't lead to deadlock through
4820	* flushing. Use __flush_work() to avoid spurious lockdep warnings
4821	* when work_on_cpu()s are nested.
4822	*/
4823	__flush_work(&wfc.work);
4824
4825	return wfc.ret;
4826	}
4827	EXPORT_SYMBOL_GPL(work_on_cpu);
4828	#endif /* CONFIG_SMP */
4829
4830	#ifdef CONFIG_FREEZER
4831
4832	/**
4833	* freeze_workqueues_begin - begin freezing workqueues
4834	*
4835	* Start freezing workqueues. After this function returns, all freezable
4836	* workqueues will queue new works to their delayed_works list instead of
4837	* pool->worklist.
4838	*
4839	* CONTEXT:
4840	* Grabs and releases wq_pool_mutex, wq->mutex and pool->lock's.
4841	*/
4842	void freeze_workqueues_begin(void)
4843	{
4844	struct worker_pool *pool;
4845	struct workqueue_struct *wq;
4846	struct pool_workqueue *pwq;
4847	int pi;
4848
4849	mutex_lock(&wq_pool_mutex);
4850
4851	WARN_ON_ONCE(workqueue_freezing);
4852	workqueue_freezing = true;
4853
4854	/* set FREEZING */
4855	for_each_pool(pool, pi) {
4856	spin_lock_irq(&pool->lock);
4857	WARN_ON_ONCE(pool->flags & POOL_FREEZING);
4858	pool->flags \|= POOL_FREEZING;
4859	spin_unlock_irq(&pool->lock);
4860	}
4861
4862	list_for_each_entry(wq, &workqueues, list) {
4863	mutex_lock(&wq->mutex);
4864	for_each_pwq(pwq, wq)
4865	pwq_adjust_max_active(pwq);
4866	mutex_unlock(&wq->mutex);
4867	}
4868
4869	mutex_unlock(&wq_pool_mutex);
4870	}
4871
4872	/**
4873	* freeze_workqueues_busy - are freezable workqueues still busy?
4874	*
4875	* Check whether freezing is complete. This function must be called
4876	* between freeze_workqueues_begin() and thaw_workqueues().
4877	*
4878	* CONTEXT:
4879	* Grabs and releases wq_pool_mutex.
4880	*
4881	* Return:
4882	* %true if some freezable workqueues are still busy. %false if freezing
4883	* is complete.
4884	*/
4885	bool freeze_workqueues_busy(void)
4886	{
4887	bool busy = false;
4888	struct workqueue_struct *wq;
4889	struct pool_workqueue *pwq;
4890
4891	mutex_lock(&wq_pool_mutex);
4892
4893	WARN_ON_ONCE(!workqueue_freezing);
4894
4895	list_for_each_entry(wq, &workqueues, list) {
4896	if (!(wq->flags & WQ_FREEZABLE))
4897	continue;
4898	/*
4899	* nr_active is monotonically decreasing. It's safe
4900	* to peek without lock.
4901	*/
4902	rcu_read_lock_sched();
4903	for_each_pwq(pwq, wq) {
4904	WARN_ON_ONCE(pwq->nr_active < 0);
4905	if (pwq->nr_active) {
4906	busy = true;
4907	rcu_read_unlock_sched();
4908	goto out_unlock;
4909	}
4910	}
4911	rcu_read_unlock_sched();
4912	}
4913	out_unlock:
4914	mutex_unlock(&wq_pool_mutex);
4915	return busy;
4916	}
4917
4918	/**
4919	* thaw_workqueues - thaw workqueues
4920	*
4921	* Thaw workqueues. Normal queueing is restored and all collected
4922	* frozen works are transferred to their respective pool worklists.
4923	*
4924	* CONTEXT:
4925	* Grabs and releases wq_pool_mutex, wq->mutex and pool->lock's.
4926	*/
4927	void thaw_workqueues(void)
4928	{
4929	struct workqueue_struct *wq;
4930	struct pool_workqueue *pwq;
4931	struct worker_pool *pool;
4932	int pi;
4933
4934	mutex_lock(&wq_pool_mutex);
4935
4936	if (!workqueue_freezing)
4937	goto out_unlock;
4938
4939	/* clear FREEZING */
4940	for_each_pool(pool, pi) {
4941	spin_lock_irq(&pool->lock);
4942	WARN_ON_ONCE(!(pool->flags & POOL_FREEZING));
4943	pool->flags &= ~POOL_FREEZING;
4944	spin_unlock_irq(&pool->lock);
4945	}
4946
4947	/* restore max_active and repopulate worklist */
4948	list_for_each_entry(wq, &workqueues, list) {
4949	mutex_lock(&wq->mutex);
4950	for_each_pwq(pwq, wq)
4951	pwq_adjust_max_active(pwq);
4952	mutex_unlock(&wq->mutex);
4953	}
4954
4955	workqueue_freezing = false;
4956	out_unlock:
4957	mutex_unlock(&wq_pool_mutex);
4958	}
4959	#endif /* CONFIG_FREEZER */
4960
4961	static void __init wq_numa_init(void)
4962	{
4963	cpumask_var_t *tbl;
4964	int node, cpu;
4965
4966	/* determine NUMA pwq table len - highest node id + 1 */
4967	for_each_node(node)
4968	wq_numa_tbl_len = max(wq_numa_tbl_len, node + 1);
4969
4970	if (num_possible_nodes() <= 1)
4971	return;
4972
4973	if (wq_disable_numa) {
4974	pr_info("workqueue: NUMA affinity support disabled\n");
4975	return;
4976	}
4977
4978	wq_update_unbound_numa_attrs_buf = alloc_workqueue_attrs(GFP_KERNEL);
4979	BUG_ON(!wq_update_unbound_numa_attrs_buf);
4980
4981	/*
4982	* We want masks of possible CPUs of each node which isn't readily
4983	* available. Build one from cpu_to_node() which should have been
4984	* fully initialized by now.
4985	*/
4986	tbl = kzalloc(wq_numa_tbl_len * sizeof(tbl[0]), GFP_KERNEL);
4987	BUG_ON(!tbl);
4988
4989	for_each_node(node)
4990	BUG_ON(!alloc_cpumask_var_node(&tbl[node], GFP_KERNEL,
4991	node_online(node) ? node : NUMA_NO_NODE));
4992
4993	for_each_possible_cpu(cpu) {
4994	node = cpu_to_node(cpu);
4995	if (WARN_ON(node == NUMA_NO_NODE)) {
4996	pr_warn("workqueue: NUMA node mapping not available for cpu%d, disabling NUMA support\n", cpu);
4997	/* happens iff arch is bonkers, let's just proceed */
4998	return;
4999	}
5000	cpumask_set_cpu(cpu, tbl[node]);
5001	}
5002
5003	wq_numa_possible_cpumask = tbl;
5004	wq_numa_enabled = true;
5005	}
5006
5007	static int __init init_workqueues(void)
5008	{
5009	int std_nice[NR_STD_WORKER_POOLS] = { 0, HIGHPRI_NICE_LEVEL };
5010	int i, cpu;
5011
5012	/* make sure we have enough bits for OFFQ pool ID */
5013	BUILD_BUG_ON((1LU << (BITS_PER_LONG - WORK_OFFQ_POOL_SHIFT)) <
5014	WORK_CPU_END * NR_STD_WORKER_POOLS);
5015
5016	WARN_ON(__alignof__(struct pool_workqueue) < __alignof__(long long));
5017
5018	pwq_cache = KMEM_CACHE(pool_workqueue, SLAB_PANIC);
5019
5020	cpu_notifier(workqueue_cpu_up_callback, CPU_PRI_WORKQUEUE_UP);
5021	hotcpu_notifier(workqueue_cpu_down_callback, CPU_PRI_WORKQUEUE_DOWN);
5022
5023	wq_numa_init();
5024
5025	/* initialize CPU pools */
5026	for_each_possible_cpu(cpu) {
5027	struct worker_pool *pool;
5028
5029	i = 0;
5030	for_each_cpu_worker_pool(pool, cpu) {
5031	BUG_ON(init_worker_pool(pool));
5032	pool->cpu = cpu;
5033	cpumask_copy(pool->attrs->cpumask, cpumask_of(cpu));
5034	pool->attrs->nice = std_nice[i++];
5035	pool->node = cpu_to_node(cpu);
5036
5037	/* alloc pool ID */
5038	mutex_lock(&wq_pool_mutex);
5039	BUG_ON(worker_pool_assign_id(pool));
5040	mutex_unlock(&wq_pool_mutex);
5041	}
5042	}
5043
5044	/* create the initial worker */
5045	for_each_online_cpu(cpu) {
5046	struct worker_pool *pool;
5047
5048	for_each_cpu_worker_pool(pool, cpu) {
5049	pool->flags &= ~POOL_DISASSOCIATED;
5050	BUG_ON(create_and_start_worker(pool) < 0);
5051	}
5052	}
5053
5054	/* create default unbound wq attrs */
5055	for (i = 0; i < NR_STD_WORKER_POOLS; i++) {
5056	struct workqueue_attrs *attrs;
5057
5058	BUG_ON(!(attrs = alloc_workqueue_attrs(GFP_KERNEL)));
5059	attrs->nice = std_nice[i];
5060	unbound_std_wq_attrs[i] = attrs;
5061	}
5062
5063	system_wq = alloc_workqueue("events", 0, 0);
5064	system_highpri_wq = alloc_workqueue("events_highpri", WQ_HIGHPRI, 0);
5065	system_long_wq = alloc_workqueue("events_long", 0, 0);
5066	system_unbound_wq = alloc_workqueue("events_unbound", WQ_UNBOUND,
5067	WQ_UNBOUND_MAX_ACTIVE);
5068	system_freezable_wq = alloc_workqueue("events_freezable",
5069	WQ_FREEZABLE, 0);
5070	system_power_efficient_wq = alloc_workqueue("events_power_efficient",
5071	WQ_POWER_EFFICIENT, 0);
5072	system_freezable_power_efficient_wq = alloc_workqueue("events_freezable_power_efficient",
5073	WQ_FREEZABLE \| WQ_POWER_EFFICIENT,
5074	0);
5075	BUG_ON(!system_wq \|\| !system_highpri_wq \|\| !system_long_wq \|\|
5076	!system_unbound_wq \|\| !system_freezable_wq \|\|
5077	!system_power_efficient_wq \|\|
5078	!system_freezable_power_efficient_wq);
5079	return 0;
5080	}
5081	early_initcall(init_workqueues);