linux - kernel/workqueue.c

5016 lines · 138.5 KiB

1	/*
2	* kernel/workqueue.c - generic async execution with shared worker pool
3	*
4	* Copyright (C) 2002 Ingo Molnar
5	*
6	* Derived from the taskqueue/keventd code by:
7	* David Woodhouse <[email protected]>
8	* Andrew Morton
9	* Kai Petzke <[email protected]>
10	* Theodore Ts'o <[email protected]>
11	*
12	* Made to use alloc_percpu by Christoph Lameter.
13	*
14	* Copyright (C) 2010 SUSE Linux Products GmbH
15	* Copyright (C) 2010 Tejun Heo <[email protected]>
16	*
17	* This is the generic async execution mechanism. Work items as are
18	* executed in process context. The worker pool is shared and
19	* automatically managed. There is one worker pool for each CPU and
20	* one extra for works which are better served by workers which are
21	* not bound to any specific CPU.
22	*
23	* Please read Documentation/workqueue.txt for details.
24	*/
25
26	#include <linux/export.h>
27	#include <linux/kernel.h>
28	#include <linux/sched.h>
29	#include <linux/init.h>
30	#include <linux/signal.h>
31	#include <linux/completion.h>
32	#include <linux/workqueue.h>
33	#include <linux/slab.h>
34	#include <linux/cpu.h>
35	#include <linux/notifier.h>
36	#include <linux/kthread.h>
37	#include <linux/hardirq.h>
38	#include <linux/mempolicy.h>
39	#include <linux/freezer.h>
40	#include <linux/kallsyms.h>
41	#include <linux/debug_locks.h>
42	#include <linux/lockdep.h>
43	#include <linux/idr.h>
44	#include <linux/jhash.h>
45	#include <linux/hashtable.h>
46	#include <linux/rculist.h>
47	#include <linux/nodemask.h>
48	#include <linux/moduleparam.h>
49	#include <linux/uaccess.h>
50
51	#include "workqueue_internal.h"
52
53	enum {
54	/*
55	* worker_pool flags
56	*
57	* A bound pool is either associated or disassociated with its CPU.
58	* While associated (!DISASSOCIATED), all workers are bound to the
59	* CPU and none has %WORKER_UNBOUND set and concurrency management
60	* is in effect.
61	*
62	* While DISASSOCIATED, the cpu may be offline and all workers have
63	* %WORKER_UNBOUND set and concurrency management disabled, and may
64	* be executing on any CPU. The pool behaves as an unbound one.
65	*
66	* Note that DISASSOCIATED should be flipped only while holding
67	* manager_mutex to avoid changing binding state while
68	* create_worker() is in progress.
69	*/
70	POOL_MANAGE_WORKERS = 1 << 0, /* need to manage workers */
71	POOL_DISASSOCIATED = 1 << 2, /* cpu can't serve workers */
72	POOL_FREEZING = 1 << 3, /* freeze in progress */
73
74	/* worker flags */
75	WORKER_STARTED = 1 << 0, /* started */
76	WORKER_DIE = 1 << 1, /* die die die */
77	WORKER_IDLE = 1 << 2, /* is idle */
78	WORKER_PREP = 1 << 3, /* preparing to run works */
79	WORKER_CPU_INTENSIVE = 1 << 6, /* cpu intensive */
80	WORKER_UNBOUND = 1 << 7, /* worker is unbound */
81	WORKER_REBOUND = 1 << 8, /* worker was rebound */
82
83	WORKER_NOT_RUNNING = WORKER_PREP \| WORKER_CPU_INTENSIVE \|
84	WORKER_UNBOUND \| WORKER_REBOUND,
85
86	NR_STD_WORKER_POOLS = 2, /* # standard pools per cpu */
87
88	UNBOUND_POOL_HASH_ORDER = 6, /* hashed by pool->attrs */
89	BUSY_WORKER_HASH_ORDER = 6, /* 64 pointers */
90
91	MAX_IDLE_WORKERS_RATIO = 4, /* 1/4 of busy can be idle */
92	IDLE_WORKER_TIMEOUT = 300 * HZ, /* keep idle ones for 5 mins */
93
94	MAYDAY_INITIAL_TIMEOUT = HZ / 100 >= 2 ? HZ / 100 : 2,
95	/* call for help after 10ms
96	(min two ticks) */
97	MAYDAY_INTERVAL = HZ / 10, /* and then every 100ms */
98	CREATE_COOLDOWN = HZ, /* time to breath after fail */
99
100	/*
101	* Rescue workers are used only on emergencies and shared by
102	* all cpus. Give -20.
103	*/
104	RESCUER_NICE_LEVEL = -20,
105	HIGHPRI_NICE_LEVEL = -20,
106
107	WQ_NAME_LEN = 24,
108	};
109
110	/*
111	* Structure fields follow one of the following exclusion rules.
112	*
113	* I: Modifiable by initialization/destruction paths and read-only for
114	* everyone else.
115	*
116	* P: Preemption protected. Disabling preemption is enough and should
117	* only be modified and accessed from the local cpu.
118	*
119	* L: pool->lock protected. Access with pool->lock held.
120	*
121	* X: During normal operation, modification requires pool->lock and should
122	* be done only from local cpu. Either disabling preemption on local
123	* cpu or grabbing pool->lock is enough for read access. If
124	* POOL_DISASSOCIATED is set, it's identical to L.
125	*
126	* MG: pool->manager_mutex and pool->lock protected. Writes require both
127	* locks. Reads can happen under either lock.
128	*
129	* PL: wq_pool_mutex protected.
130	*
131	* PR: wq_pool_mutex protected for writes. Sched-RCU protected for reads.
132	*
133	* WQ: wq->mutex protected.
134	*
135	* WR: wq->mutex protected for writes. Sched-RCU protected for reads.
136	*
137	* MD: wq_mayday_lock protected.
138	*/
139
140	/* struct worker is defined in workqueue_internal.h */
141
142	struct worker_pool {
143	spinlock_t lock; /* the pool lock */
144	int cpu; /* I: the associated cpu */
145	int node; /* I: the associated node ID */
146	int id; /* I: pool ID */
147	unsigned int flags; /* X: flags */
148
149	struct list_head worklist; /* L: list of pending works */
150	int nr_workers; /* L: total number of workers */
151
152	/* nr_idle includes the ones off idle_list for rebinding */
153	int nr_idle; /* L: currently idle ones */
154
155	struct list_head idle_list; /* X: list of idle workers */
156	struct timer_list idle_timer; /* L: worker idle timeout */
157	struct timer_list mayday_timer; /* L: SOS timer for workers */
158
159	/* a workers is either on busy_hash or idle_list, or the manager */
160	DECLARE_HASHTABLE(busy_hash, BUSY_WORKER_HASH_ORDER);
161	/* L: hash of busy workers */
162
163	/* see manage_workers() for details on the two manager mutexes */
164	struct mutex manager_arb; /* manager arbitration */
165	struct mutex manager_mutex; /* manager exclusion */
166	struct idr worker_idr; /* MG: worker IDs and iteration */
167
168	struct workqueue_attrs attrs; / I: worker attributes */
169	struct hlist_node hash_node; /* PL: unbound_pool_hash node */
170	int refcnt; /* PL: refcnt for unbound pools */
171
172	/*
173	* The current concurrency level. As it's likely to be accessed
174	* from other CPUs during try_to_wake_up(), put it in a separate
175	* cacheline.
176	*/
177	atomic_t nr_running ____cacheline_aligned_in_smp;
178
179	/*
180	* Destruction of pool is sched-RCU protected to allow dereferences
181	* from get_work_pool().
182	*/
183	struct rcu_head rcu;
184	} ____cacheline_aligned_in_smp;
185
186	/*
187	* The per-pool workqueue. While queued, the lower WORK_STRUCT_FLAG_BITS
188	* of work_struct->data are used for flags and the remaining high bits
189	* point to the pwq; thus, pwqs need to be aligned at two's power of the
190	* number of flag bits.
191	*/
192	struct pool_workqueue {
193	struct worker_pool pool; / I: the associated pool */
194	struct workqueue_struct wq; / I: the owning workqueue */
195	int work_color; /* L: current color */
196	int flush_color; /* L: flushing color */
197	int refcnt; /* L: reference count */
198	int nr_in_flight[WORK_NR_COLORS];
199	/* L: nr of in_flight works */
200	int nr_active; /* L: nr of active works */
201	int max_active; /* L: max active works */
202	struct list_head delayed_works; /* L: delayed works */
203	struct list_head pwqs_node; /* WR: node on wq->pwqs */
204	struct list_head mayday_node; /* MD: node on wq->maydays */
205
206	/*
207	* Release of unbound pwq is punted to system_wq. See put_pwq()
208	* and pwq_unbound_release_workfn() for details. pool_workqueue
209	* itself is also sched-RCU protected so that the first pwq can be
210	* determined without grabbing wq->mutex.
211	*/
212	struct work_struct unbound_release_work;
213	struct rcu_head rcu;
214	} __aligned(1 << WORK_STRUCT_FLAG_BITS);
215
216	/*
217	* Structure used to wait for workqueue flush.
218	*/
219	struct wq_flusher {
220	struct list_head list; /* WQ: list of flushers */
221	int flush_color; /* WQ: flush color waiting for */
222	struct completion done; /* flush completion */
223	};
224
225	struct wq_device;
226
227	/*
228	* The externally visible workqueue. It relays the issued work items to
229	* the appropriate worker_pool through its pool_workqueues.
230	*/
231	struct workqueue_struct {
232	struct list_head pwqs; /* WR: all pwqs of this wq */
233	struct list_head list; /* PL: list of all workqueues */
234
235	struct mutex mutex; /* protects this wq */
236	int work_color; /* WQ: current work color */
237	int flush_color; /* WQ: current flush color */
238	atomic_t nr_pwqs_to_flush; /* flush in progress */
239	struct wq_flusher first_flusher; / WQ: first flusher */
240	struct list_head flusher_queue; /* WQ: flush waiters */
241	struct list_head flusher_overflow; /* WQ: flush overflow list */
242
243	struct list_head maydays; /* MD: pwqs requesting rescue */
244	struct worker rescuer; / I: rescue worker */
245
246	int nr_drainers; /* WQ: drain in progress */
247	int saved_max_active; /* WQ: saved pwq max_active */
248
249	struct workqueue_attrs unbound_attrs; / WQ: only for unbound wqs */
250	struct pool_workqueue dfl_pwq; / WQ: only for unbound wqs */
251
252	#ifdef CONFIG_SYSFS
253	struct wq_device wq_dev; / I: for sysfs interface */
254	#endif
255	#ifdef CONFIG_LOCKDEP
256	struct lockdep_map lockdep_map;
257	#endif
258	char name[WQ_NAME_LEN]; /* I: workqueue name */
259
260	/* hot fields used during command issue, aligned to cacheline */
261	unsigned int flags ____cacheline_aligned; /* WQ: WQ_* flags */
262	struct pool_workqueue __percpu cpu_pwqs; / I: per-cpu pwqs */
263	struct pool_workqueue __rcu numa_pwq_tbl[]; / FR: unbound pwqs indexed by node */
264	};
265
266	static struct kmem_cache *pwq_cache;
267
268	static int wq_numa_tbl_len; /* highest possible NUMA node id + 1 */
269	static cpumask_var_t *wq_numa_possible_cpumask;
270	/* possible CPUs of each node */
271
272	static bool wq_disable_numa;
273	module_param_named(disable_numa, wq_disable_numa, bool, 0444);
274
275	/* see the comment above the definition of WQ_POWER_EFFICIENT */
276	#ifdef CONFIG_WQ_POWER_EFFICIENT_DEFAULT
277	static bool wq_power_efficient = true;
278	#else
279	static bool wq_power_efficient;
280	#endif
281
282	module_param_named(power_efficient, wq_power_efficient, bool, 0444);
283
284	static bool wq_numa_enabled; /* unbound NUMA affinity enabled */
285
286	/* buf for wq_update_unbound_numa_attrs(), protected by CPU hotplug exclusion */
287	static struct workqueue_attrs *wq_update_unbound_numa_attrs_buf;
288
289	static DEFINE_MUTEX(wq_pool_mutex); /* protects pools and workqueues list */
290	static DEFINE_SPINLOCK(wq_mayday_lock); /* protects wq->maydays list */
291
292	static LIST_HEAD(workqueues); /* PL: list of all workqueues */
293	static bool workqueue_freezing; /* PL: have wqs started freezing? */
294
295	/* the per-cpu worker pools */
296	static DEFINE_PER_CPU_SHARED_ALIGNED(struct worker_pool [NR_STD_WORKER_POOLS],
297	cpu_worker_pools);
298
299	static DEFINE_IDR(worker_pool_idr); /* PR: idr of all pools */
300
301	/* PL: hash of all unbound pools keyed by pool->attrs */
302	static DEFINE_HASHTABLE(unbound_pool_hash, UNBOUND_POOL_HASH_ORDER);
303
304	/* I: attributes used when instantiating standard unbound pools on demand */
305	static struct workqueue_attrs *unbound_std_wq_attrs[NR_STD_WORKER_POOLS];
306
307	struct workqueue_struct *system_wq __read_mostly;
308	EXPORT_SYMBOL(system_wq);
309	struct workqueue_struct *system_highpri_wq __read_mostly;
310	EXPORT_SYMBOL_GPL(system_highpri_wq);
311	struct workqueue_struct *system_long_wq __read_mostly;
312	EXPORT_SYMBOL_GPL(system_long_wq);
313	struct workqueue_struct *system_unbound_wq __read_mostly;
314	EXPORT_SYMBOL_GPL(system_unbound_wq);
315	struct workqueue_struct *system_freezable_wq __read_mostly;
316	EXPORT_SYMBOL_GPL(system_freezable_wq);
317	struct workqueue_struct *system_power_efficient_wq __read_mostly;
318	EXPORT_SYMBOL_GPL(system_power_efficient_wq);
319	struct workqueue_struct *system_freezable_power_efficient_wq __read_mostly;
320	EXPORT_SYMBOL_GPL(system_freezable_power_efficient_wq);
321
322	static int worker_thread(void *__worker);
323	static void copy_workqueue_attrs(struct workqueue_attrs *to,
324	const struct workqueue_attrs *from);
325
326	#define CREATE_TRACE_POINTS
327	#include <trace/events/workqueue.h>
328
329	#define assert_rcu_or_pool_mutex() \
330	rcu_lockdep_assert(rcu_read_lock_sched_held() \|\| \
331	lockdep_is_held(&wq_pool_mutex), \
332	"sched RCU or wq_pool_mutex should be held")
333
334	#define assert_rcu_or_wq_mutex(wq) \
335	rcu_lockdep_assert(rcu_read_lock_sched_held() \|\| \
336	lockdep_is_held(&wq->mutex), \
337	"sched RCU or wq->mutex should be held")
338
339	#ifdef CONFIG_LOCKDEP
340	#define assert_manager_or_pool_lock(pool) \
341	WARN_ONCE(debug_locks && \
342	!lockdep_is_held(&(pool)->manager_mutex) && \
343	!lockdep_is_held(&(pool)->lock), \
344	"pool->manager_mutex or ->lock should be held")
345	#else
346	#define assert_manager_or_pool_lock(pool) do { } while (0)
347	#endif
348
349	#define for_each_cpu_worker_pool(pool, cpu) \
350	for ((pool) = &per_cpu(cpu_worker_pools, cpu)[0]; \
351	(pool) < &per_cpu(cpu_worker_pools, cpu)[NR_STD_WORKER_POOLS]; \
352	(pool)++)
353
354	/**
355	* for_each_pool - iterate through all worker_pools in the system
356	* @pool: iteration cursor
357	* @pi: integer used for iteration
358	*
359	* This must be called either with wq_pool_mutex held or sched RCU read
360	* locked. If the pool needs to be used beyond the locking in effect, the
361	* caller is responsible for guaranteeing that the pool stays online.
362	*
363	* The if/else clause exists only for the lockdep assertion and can be
364	* ignored.
365	*/
366	#define for_each_pool(pool, pi) \
367	idr_for_each_entry(&worker_pool_idr, pool, pi) \
368	if (({ assert_rcu_or_pool_mutex(); false; })) { } \
369	else
370
371	/**
372	* for_each_pool_worker - iterate through all workers of a worker_pool
373	* @worker: iteration cursor
374	* @wi: integer used for iteration
375	* @pool: worker_pool to iterate workers of
376	*
377	* This must be called with either @pool->manager_mutex or ->lock held.
378	*
379	* The if/else clause exists only for the lockdep assertion and can be
380	* ignored.
381	*/
382	#define for_each_pool_worker(worker, wi, pool) \
383	idr_for_each_entry(&(pool)->worker_idr, (worker), (wi)) \
384	if (({ assert_manager_or_pool_lock((pool)); false; })) { } \
385	else
386
387	/**
388	* for_each_pwq - iterate through all pool_workqueues of the specified workqueue
389	* @pwq: iteration cursor
390	* @wq: the target workqueue
391	*
392	* This must be called either with wq->mutex held or sched RCU read locked.
393	* If the pwq needs to be used beyond the locking in effect, the caller is
394	* responsible for guaranteeing that the pwq stays online.
395	*
396	* The if/else clause exists only for the lockdep assertion and can be
397	* ignored.
398	*/
399	#define for_each_pwq(pwq, wq) \
400	list_for_each_entry_rcu((pwq), &(wq)->pwqs, pwqs_node) \
401	if (({ assert_rcu_or_wq_mutex(wq); false; })) { } \
402	else
403
404	#ifdef CONFIG_DEBUG_OBJECTS_WORK
405
406	static struct debug_obj_descr work_debug_descr;
407
408	static void work_debug_hint(void addr)
409	{
410	return ((struct work_struct *) addr)->func;
411	}
412
413	/*
414	* fixup_init is called when:
415	* - an active object is initialized
416	*/
417	static int work_fixup_init(void *addr, enum debug_obj_state state)
418	{
419	struct work_struct *work = addr;
420
421	switch (state) {
422	case ODEBUG_STATE_ACTIVE:
423	cancel_work_sync(work);
424	debug_object_init(work, &work_debug_descr);
425	return 1;
426	default:
427	return 0;
428	}
429	}
430
431	/*
432	* fixup_activate is called when:
433	* - an active object is activated
434	* - an unknown object is activated (might be a statically initialized object)
435	*/
436	static int work_fixup_activate(void *addr, enum debug_obj_state state)
437	{
438	struct work_struct *work = addr;
439
440	switch (state) {
441
442	case ODEBUG_STATE_NOTAVAILABLE:
443	/*
444	* This is not really a fixup. The work struct was
445	* statically initialized. We just make sure that it
446	* is tracked in the object tracker.
447	*/
448	if (test_bit(WORK_STRUCT_STATIC_BIT, work_data_bits(work))) {
449	debug_object_init(work, &work_debug_descr);
450	debug_object_activate(work, &work_debug_descr);
451	return 0;
452	}
453	WARN_ON_ONCE(1);
454	return 0;
455
456	case ODEBUG_STATE_ACTIVE:
457	WARN_ON(1);
458
459	default:
460	return 0;
461	}
462	}
463
464	/*
465	* fixup_free is called when:
466	* - an active object is freed
467	*/
468	static int work_fixup_free(void *addr, enum debug_obj_state state)
469	{
470	struct work_struct *work = addr;
471
472	switch (state) {
473	case ODEBUG_STATE_ACTIVE:
474	cancel_work_sync(work);
475	debug_object_free(work, &work_debug_descr);
476	return 1;
477	default:
478	return 0;
479	}
480	}
481
482	static struct debug_obj_descr work_debug_descr = {
483	.name = "work_struct",
484	.debug_hint = work_debug_hint,
485	.fixup_init = work_fixup_init,
486	.fixup_activate = work_fixup_activate,
487	.fixup_free = work_fixup_free,
488	};
489
490	static inline void debug_work_activate(struct work_struct *work)
491	{
492	debug_object_activate(work, &work_debug_descr);
493	}
494
495	static inline void debug_work_deactivate(struct work_struct *work)
496	{
497	debug_object_deactivate(work, &work_debug_descr);
498	}
499
500	void __init_work(struct work_struct *work, int onstack)
501	{
502	if (onstack)
503	debug_object_init_on_stack(work, &work_debug_descr);
504	else
505	debug_object_init(work, &work_debug_descr);
506	}
507	EXPORT_SYMBOL_GPL(__init_work);
508
509	void destroy_work_on_stack(struct work_struct *work)
510	{
511	debug_object_free(work, &work_debug_descr);
512	}
513	EXPORT_SYMBOL_GPL(destroy_work_on_stack);
514
515	#else
516	static inline void debug_work_activate(struct work_struct *work) { }
517	static inline void debug_work_deactivate(struct work_struct *work) { }
518	#endif
519
520	/* allocate ID and assign it to @pool */
521	static int worker_pool_assign_id(struct worker_pool *pool)
522	{
523	int ret;
524
525	lockdep_assert_held(&wq_pool_mutex);
526
527	ret = idr_alloc(&worker_pool_idr, pool, 0, 0, GFP_KERNEL);
528	if (ret >= 0) {
529	pool->id = ret;
530	return 0;
531	}
532	return ret;
533	}
534
535	/**
536	* unbound_pwq_by_node - return the unbound pool_workqueue for the given node
537	* @wq: the target workqueue
538	* @node: the node ID
539	*
540	* This must be called either with pwq_lock held or sched RCU read locked.
541	* If the pwq needs to be used beyond the locking in effect, the caller is
542	* responsible for guaranteeing that the pwq stays online.
543	*/
544	static struct pool_workqueue unbound_pwq_by_node(struct workqueue_struct wq,
545	int node)
546	{
547	assert_rcu_or_wq_mutex(wq);
548	return rcu_dereference_raw(wq->numa_pwq_tbl[node]);
549	}
550
551	static unsigned int work_color_to_flags(int color)
552	{
553	return color << WORK_STRUCT_COLOR_SHIFT;
554	}
555
556	static int get_work_color(struct work_struct *work)
557	{
558	return (*work_data_bits(work) >> WORK_STRUCT_COLOR_SHIFT) &
559	((1 << WORK_STRUCT_COLOR_BITS) - 1);
560	}
561
562	static int work_next_color(int color)
563	{
564	return (color + 1) % WORK_NR_COLORS;
565	}
566
567	/*
568	* While queued, %WORK_STRUCT_PWQ is set and non flag bits of a work's data
569	* contain the pointer to the queued pwq. Once execution starts, the flag
570	* is cleared and the high bits contain OFFQ flags and pool ID.
571	*
572	* set_work_pwq(), set_work_pool_and_clear_pending(), mark_work_canceling()
573	* and clear_work_data() can be used to set the pwq, pool or clear
574	* work->data. These functions should only be called while the work is
575	* owned - ie. while the PENDING bit is set.
576	*
577	* get_work_pool() and get_work_pwq() can be used to obtain the pool or pwq
578	* corresponding to a work. Pool is available once the work has been
579	* queued anywhere after initialization until it is sync canceled. pwq is
580	* available only while the work item is queued.
581	*
582	* %WORK_OFFQ_CANCELING is used to mark a work item which is being
583	* canceled. While being canceled, a work item may have its PENDING set
584	* but stay off timer and worklist for arbitrarily long and nobody should
585	* try to steal the PENDING bit.
586	*/
587	static inline void set_work_data(struct work_struct *work, unsigned long data,
588	unsigned long flags)
589	{
590	WARN_ON_ONCE(!work_pending(work));
591	atomic_long_set(&work->data, data \| flags \| work_static(work));
592	}
593
594	static void set_work_pwq(struct work_struct work, struct pool_workqueue pwq,
595	unsigned long extra_flags)
596	{
597	set_work_data(work, (unsigned long)pwq,
598	WORK_STRUCT_PENDING \| WORK_STRUCT_PWQ \| extra_flags);
599	}
600
601	static void set_work_pool_and_keep_pending(struct work_struct *work,
602	int pool_id)
603	{
604	set_work_data(work, (unsigned long)pool_id << WORK_OFFQ_POOL_SHIFT,
605	WORK_STRUCT_PENDING);
606	}
607
608	static void set_work_pool_and_clear_pending(struct work_struct *work,
609	int pool_id)
610	{
611	/*
612	* The following wmb is paired with the implied mb in
613	* test_and_set_bit(PENDING) and ensures all updates to @work made
614	* here are visible to and precede any updates by the next PENDING
615	* owner.
616	*/
617	smp_wmb();
618	set_work_data(work, (unsigned long)pool_id << WORK_OFFQ_POOL_SHIFT, 0);
619	}
620
621	static void clear_work_data(struct work_struct *work)
622	{
623	smp_wmb(); /* see set_work_pool_and_clear_pending() */
624	set_work_data(work, WORK_STRUCT_NO_POOL, 0);
625	}
626
627	static struct pool_workqueue get_work_pwq(struct work_struct work)
628	{
629	unsigned long data = atomic_long_read(&work->data);
630
631	if (data & WORK_STRUCT_PWQ)
632	return (void *)(data & WORK_STRUCT_WQ_DATA_MASK);
633	else
634	return NULL;
635	}
636
637	/**
638	* get_work_pool - return the worker_pool a given work was associated with
639	* @work: the work item of interest
640	*
641	* Return the worker_pool @work was last associated with. %NULL if none.
642	*
643	* Pools are created and destroyed under wq_pool_mutex, and allows read
644	* access under sched-RCU read lock. As such, this function should be
645	* called under wq_pool_mutex or with preemption disabled.
646	*
647	* All fields of the returned pool are accessible as long as the above
648	* mentioned locking is in effect. If the returned pool needs to be used
649	* beyond the critical section, the caller is responsible for ensuring the
650	* returned pool is and stays online.
651	*/
652	static struct worker_pool get_work_pool(struct work_struct work)
653	{
654	unsigned long data = atomic_long_read(&work->data);
655	int pool_id;
656
657	assert_rcu_or_pool_mutex();
658
659	if (data & WORK_STRUCT_PWQ)
660	return ((struct pool_workqueue *)
661	(data & WORK_STRUCT_WQ_DATA_MASK))->pool;
662
663	pool_id = data >> WORK_OFFQ_POOL_SHIFT;
664	if (pool_id == WORK_OFFQ_POOL_NONE)
665	return NULL;
666
667	return idr_find(&worker_pool_idr, pool_id);
668	}
669
670	/**
671	* get_work_pool_id - return the worker pool ID a given work is associated with
672	* @work: the work item of interest
673	*
674	* Return the worker_pool ID @work was last associated with.
675	* %WORK_OFFQ_POOL_NONE if none.
676	*/
677	static int get_work_pool_id(struct work_struct *work)
678	{
679	unsigned long data = atomic_long_read(&work->data);
680
681	if (data & WORK_STRUCT_PWQ)
682	return ((struct pool_workqueue *)
683	(data & WORK_STRUCT_WQ_DATA_MASK))->pool->id;
684
685	return data >> WORK_OFFQ_POOL_SHIFT;
686	}
687
688	static void mark_work_canceling(struct work_struct *work)
689	{
690	unsigned long pool_id = get_work_pool_id(work);
691
692	pool_id <<= WORK_OFFQ_POOL_SHIFT;
693	set_work_data(work, pool_id \| WORK_OFFQ_CANCELING, WORK_STRUCT_PENDING);
694	}
695
696	static bool work_is_canceling(struct work_struct *work)
697	{
698	unsigned long data = atomic_long_read(&work->data);
699
700	return !(data & WORK_STRUCT_PWQ) && (data & WORK_OFFQ_CANCELING);
701	}
702
703	/*
704	* Policy functions. These define the policies on how the global worker
705	* pools are managed. Unless noted otherwise, these functions assume that
706	* they're being called with pool->lock held.
707	*/
708
709	static bool __need_more_worker(struct worker_pool *pool)
710	{
711	return !atomic_read(&pool->nr_running);
712	}
713
714	/*
715	* Need to wake up a worker? Called from anything but currently
716	* running workers.
717	*
718	* Note that, because unbound workers never contribute to nr_running, this
719	* function will always return %true for unbound pools as long as the
720	* worklist isn't empty.
721	*/
722	static bool need_more_worker(struct worker_pool *pool)
723	{
724	return !list_empty(&pool->worklist) && __need_more_worker(pool);
725	}
726
727	/* Can I start working? Called from busy but !running workers. */
728	static bool may_start_working(struct worker_pool *pool)
729	{
730	return pool->nr_idle;
731	}
732
733	/* Do I need to keep working? Called from currently running workers. */
734	static bool keep_working(struct worker_pool *pool)
735	{
736	return !list_empty(&pool->worklist) &&
737	atomic_read(&pool->nr_running) <= 1;
738	}
739
740	/* Do we need a new worker? Called from manager. */
741	static bool need_to_create_worker(struct worker_pool *pool)
742	{
743	return need_more_worker(pool) && !may_start_working(pool);
744	}
745
746	/* Do I need to be the manager? */
747	static bool need_to_manage_workers(struct worker_pool *pool)
748	{
749	return need_to_create_worker(pool) \|\|
750	(pool->flags & POOL_MANAGE_WORKERS);
751	}
752
753	/* Do we have too many workers and should some go away? */
754	static bool too_many_workers(struct worker_pool *pool)
755	{
756	bool managing = mutex_is_locked(&pool->manager_arb);
757	int nr_idle = pool->nr_idle + managing; /* manager is considered idle */
758	int nr_busy = pool->nr_workers - nr_idle;
759
760	/*
761	* nr_idle and idle_list may disagree if idle rebinding is in
762	* progress. Never return %true if idle_list is empty.
763	*/
764	if (list_empty(&pool->idle_list))
765	return false;
766
767	return nr_idle > 2 && (nr_idle - 2) * MAX_IDLE_WORKERS_RATIO >= nr_busy;
768	}
769
770	/*
771	* Wake up functions.
772	*/
773
774	/* Return the first worker. Safe with preemption disabled */
775	static struct worker first_worker(struct worker_pool pool)
776	{
777	if (unlikely(list_empty(&pool->idle_list)))
778	return NULL;
779
780	return list_first_entry(&pool->idle_list, struct worker, entry);
781	}
782
783	/**
784	* wake_up_worker - wake up an idle worker
785	* @pool: worker pool to wake worker from
786	*
787	* Wake up the first idle worker of @pool.
788	*
789	* CONTEXT:
790	* spin_lock_irq(pool->lock).
791	*/
792	static void wake_up_worker(struct worker_pool *pool)
793	{
794	struct worker *worker = first_worker(pool);
795
796	if (likely(worker))
797	wake_up_process(worker->task);
798	}
799
800	/**
801	* wq_worker_waking_up - a worker is waking up
802	* @task: task waking up
803	* @cpu: CPU @task is waking up to
804	*
805	* This function is called during try_to_wake_up() when a worker is
806	* being awoken.
807	*
808	* CONTEXT:
809	* spin_lock_irq(rq->lock)
810	*/
811	void wq_worker_waking_up(struct task_struct *task, int cpu)
812	{
813	struct worker *worker = kthread_data(task);
814
815	if (!(worker->flags & WORKER_NOT_RUNNING)) {
816	WARN_ON_ONCE(worker->pool->cpu != cpu);
817	atomic_inc(&worker->pool->nr_running);
818	}
819	}
820
821	/**
822	* wq_worker_sleeping - a worker is going to sleep
823	* @task: task going to sleep
824	* @cpu: CPU in question, must be the current CPU number
825	*
826	* This function is called during schedule() when a busy worker is
827	* going to sleep. Worker on the same cpu can be woken up by
828	* returning pointer to its task.
829	*
830	* CONTEXT:
831	* spin_lock_irq(rq->lock)
832	*
833	* RETURNS:
834	* Worker task on @cpu to wake up, %NULL if none.
835	*/
836	struct task_struct wq_worker_sleeping(struct task_struct task, int cpu)
837	{
838	struct worker worker = kthread_data(task), to_wakeup = NULL;
839	struct worker_pool *pool;
840
841	/*
842	* Rescuers, which may not have all the fields set up like normal
843	* workers, also reach here, let's not access anything before
844	* checking NOT_RUNNING.
845	*/
846	if (worker->flags & WORKER_NOT_RUNNING)
847	return NULL;
848
849	pool = worker->pool;
850
851	/* this can only happen on the local cpu */
852	if (WARN_ON_ONCE(cpu != raw_smp_processor_id()))
853	return NULL;
854
855	/*
856	* The counterpart of the following dec_and_test, implied mb,
857	* worklist not empty test sequence is in insert_work().
858	* Please read comment there.
859	*
860	* NOT_RUNNING is clear. This means that we're bound to and
861	* running on the local cpu w/ rq lock held and preemption
862	* disabled, which in turn means that none else could be
863	* manipulating idle_list, so dereferencing idle_list without pool
864	* lock is safe.
865	*/
866	if (atomic_dec_and_test(&pool->nr_running) &&
867	!list_empty(&pool->worklist))
868	to_wakeup = first_worker(pool);
869	return to_wakeup ? to_wakeup->task : NULL;
870	}
871
872	/**
873	* worker_set_flags - set worker flags and adjust nr_running accordingly
874	* @worker: self
875	* @flags: flags to set
876	* @wakeup: wakeup an idle worker if necessary
877	*
878	* Set @flags in @worker->flags and adjust nr_running accordingly. If
879	* nr_running becomes zero and @wakeup is %true, an idle worker is
880	* woken up.
881	*
882	* CONTEXT:
883	* spin_lock_irq(pool->lock)
884	*/
885	static inline void worker_set_flags(struct worker *worker, unsigned int flags,
886	bool wakeup)
887	{
888	struct worker_pool *pool = worker->pool;
889
890	WARN_ON_ONCE(worker->task != current);
891
892	/*
893	* If transitioning into NOT_RUNNING, adjust nr_running and
894	* wake up an idle worker as necessary if requested by
895	* @wakeup.
896	*/
897	if ((flags & WORKER_NOT_RUNNING) &&
898	!(worker->flags & WORKER_NOT_RUNNING)) {
899	if (wakeup) {
900	if (atomic_dec_and_test(&pool->nr_running) &&
901	!list_empty(&pool->worklist))
902	wake_up_worker(pool);
903	} else
904	atomic_dec(&pool->nr_running);
905	}
906
907	worker->flags \|= flags;
908	}
909
910	/**
911	* worker_clr_flags - clear worker flags and adjust nr_running accordingly
912	* @worker: self
913	* @flags: flags to clear
914	*
915	* Clear @flags in @worker->flags and adjust nr_running accordingly.
916	*
917	* CONTEXT:
918	* spin_lock_irq(pool->lock)
919	*/
920	static inline void worker_clr_flags(struct worker *worker, unsigned int flags)
921	{
922	struct worker_pool *pool = worker->pool;
923	unsigned int oflags = worker->flags;
924
925	WARN_ON_ONCE(worker->task != current);
926
927	worker->flags &= ~flags;
928
929	/*
930	* If transitioning out of NOT_RUNNING, increment nr_running. Note
931	* that the nested NOT_RUNNING is not a noop. NOT_RUNNING is mask
932	* of multiple flags, not a single flag.
933	*/
934	if ((flags & WORKER_NOT_RUNNING) && (oflags & WORKER_NOT_RUNNING))
935	if (!(worker->flags & WORKER_NOT_RUNNING))
936	atomic_inc(&pool->nr_running);
937	}
938
939	/**
940	* find_worker_executing_work - find worker which is executing a work
941	* @pool: pool of interest
942	* @work: work to find worker for
943	*
944	* Find a worker which is executing @work on @pool by searching
945	* @pool->busy_hash which is keyed by the address of @work. For a worker
946	* to match, its current execution should match the address of @work and
947	* its work function. This is to avoid unwanted dependency between
948	* unrelated work executions through a work item being recycled while still
949	* being executed.
950	*
951	* This is a bit tricky. A work item may be freed once its execution
952	* starts and nothing prevents the freed area from being recycled for
953	* another work item. If the same work item address ends up being reused
954	* before the original execution finishes, workqueue will identify the
955	* recycled work item as currently executing and make it wait until the
956	* current execution finishes, introducing an unwanted dependency.
957	*
958	* This function checks the work item address and work function to avoid
959	* false positives. Note that this isn't complete as one may construct a
960	* work function which can introduce dependency onto itself through a
961	* recycled work item. Well, if somebody wants to shoot oneself in the
962	* foot that badly, there's only so much we can do, and if such deadlock
963	* actually occurs, it should be easy to locate the culprit work function.
964	*
965	* CONTEXT:
966	* spin_lock_irq(pool->lock).
967	*
968	* RETURNS:
969	* Pointer to worker which is executing @work if found, NULL
970	* otherwise.
971	*/
972	static struct worker find_worker_executing_work(struct worker_pool pool,
973	struct work_struct *work)
974	{
975	struct worker *worker;
976
977	hash_for_each_possible(pool->busy_hash, worker, hentry,
978	(unsigned long)work)
979	if (worker->current_work == work &&
980	worker->current_func == work->func)
981	return worker;
982
983	return NULL;
984	}
985
986	/**
987	* move_linked_works - move linked works to a list
988	* @work: start of series of works to be scheduled
989	* @head: target list to append @work to
990	* @nextp: out paramter for nested worklist walking
991	*
992	* Schedule linked works starting from @work to @head. Work series to
993	* be scheduled starts at @work and includes any consecutive work with
994	* WORK_STRUCT_LINKED set in its predecessor.
995	*
996	* If @nextp is not NULL, it's updated to point to the next work of
997	* the last scheduled work. This allows move_linked_works() to be
998	* nested inside outer list_for_each_entry_safe().
999	*
1000	* CONTEXT:
1001	* spin_lock_irq(pool->lock).
1002	*/
1003	static void move_linked_works(struct work_struct work, struct list_head head,
1004	struct work_struct **nextp)
1005	{
1006	struct work_struct *n;
1007
1008	/*
1009	* Linked worklist will always end before the end of the list,
1010	* use NULL for list head.
1011	*/
1012	list_for_each_entry_safe_from(work, n, NULL, entry) {
1013	list_move_tail(&work->entry, head);
1014	if (!(*work_data_bits(work) & WORK_STRUCT_LINKED))
1015	break;
1016	}
1017
1018	/*
1019	* If we're already inside safe list traversal and have moved
1020	* multiple works to the scheduled queue, the next position
1021	* needs to be updated.
1022	*/
1023	if (nextp)
1024	*nextp = n;
1025	}
1026
1027	/**
1028	* get_pwq - get an extra reference on the specified pool_workqueue
1029	* @pwq: pool_workqueue to get
1030	*
1031	* Obtain an extra reference on @pwq. The caller should guarantee that
1032	* @pwq has positive refcnt and be holding the matching pool->lock.
1033	*/
1034	static void get_pwq(struct pool_workqueue *pwq)
1035	{
1036	lockdep_assert_held(&pwq->pool->lock);
1037	WARN_ON_ONCE(pwq->refcnt <= 0);
1038	pwq->refcnt++;
1039	}
1040
1041	/**
1042	* put_pwq - put a pool_workqueue reference
1043	* @pwq: pool_workqueue to put
1044	*
1045	* Drop a reference of @pwq. If its refcnt reaches zero, schedule its
1046	* destruction. The caller should be holding the matching pool->lock.
1047	*/
1048	static void put_pwq(struct pool_workqueue *pwq)
1049	{
1050	lockdep_assert_held(&pwq->pool->lock);
1051	if (likely(--pwq->refcnt))
1052	return;
1053	if (WARN_ON_ONCE(!(pwq->wq->flags & WQ_UNBOUND)))
1054	return;
1055	/*
1056	* @pwq can't be released under pool->lock, bounce to
1057	* pwq_unbound_release_workfn(). This never recurses on the same
1058	* pool->lock as this path is taken only for unbound workqueues and
1059	* the release work item is scheduled on a per-cpu workqueue. To
1060	* avoid lockdep warning, unbound pool->locks are given lockdep
1061	* subclass of 1 in get_unbound_pool().
1062	*/
1063	schedule_work(&pwq->unbound_release_work);
1064	}
1065
1066	/**
1067	* put_pwq_unlocked - put_pwq() with surrounding pool lock/unlock
1068	* @pwq: pool_workqueue to put (can be %NULL)
1069	*
1070	* put_pwq() with locking. This function also allows %NULL @pwq.
1071	*/
1072	static void put_pwq_unlocked(struct pool_workqueue *pwq)
1073	{
1074	if (pwq) {
1075	/*
1076	* As both pwqs and pools are sched-RCU protected, the
1077	* following lock operations are safe.
1078	*/
1079	spin_lock_irq(&pwq->pool->lock);
1080	put_pwq(pwq);
1081	spin_unlock_irq(&pwq->pool->lock);
1082	}
1083	}
1084
1085	static void pwq_activate_delayed_work(struct work_struct *work)
1086	{
1087	struct pool_workqueue *pwq = get_work_pwq(work);
1088
1089	trace_workqueue_activate_work(work);
1090	move_linked_works(work, &pwq->pool->worklist, NULL);
1091	__clear_bit(WORK_STRUCT_DELAYED_BIT, work_data_bits(work));
1092	pwq->nr_active++;
1093	}
1094
1095	static void pwq_activate_first_delayed(struct pool_workqueue *pwq)
1096	{
1097	struct work_struct *work = list_first_entry(&pwq->delayed_works,
1098	struct work_struct, entry);
1099
1100	pwq_activate_delayed_work(work);
1101	}
1102
1103	/**
1104	* pwq_dec_nr_in_flight - decrement pwq's nr_in_flight
1105	* @pwq: pwq of interest
1106	* @color: color of work which left the queue
1107	*
1108	* A work either has completed or is removed from pending queue,
1109	* decrement nr_in_flight of its pwq and handle workqueue flushing.
1110	*
1111	* CONTEXT:
1112	* spin_lock_irq(pool->lock).
1113	*/
1114	static void pwq_dec_nr_in_flight(struct pool_workqueue *pwq, int color)
1115	{
1116	/* uncolored work items don't participate in flushing or nr_active */
1117	if (color == WORK_NO_COLOR)
1118	goto out_put;
1119
1120	pwq->nr_in_flight[color]--;
1121
1122	pwq->nr_active--;
1123	if (!list_empty(&pwq->delayed_works)) {
1124	/* one down, submit a delayed one */
1125	if (pwq->nr_active < pwq->max_active)
1126	pwq_activate_first_delayed(pwq);
1127	}
1128
1129	/* is flush in progress and are we at the flushing tip? */
1130	if (likely(pwq->flush_color != color))
1131	goto out_put;
1132
1133	/* are there still in-flight works? */
1134	if (pwq->nr_in_flight[color])
1135	goto out_put;
1136
1137	/* this pwq is done, clear flush_color */
1138	pwq->flush_color = -1;
1139
1140	/*
1141	* If this was the last pwq, wake up the first flusher. It
1142	* will handle the rest.
1143	*/
1144	if (atomic_dec_and_test(&pwq->wq->nr_pwqs_to_flush))
1145	complete(&pwq->wq->first_flusher->done);
1146	out_put:
1147	put_pwq(pwq);
1148	}
1149
1150	/**
1151	* try_to_grab_pending - steal work item from worklist and disable irq
1152	* @work: work item to steal
1153	* @is_dwork: @work is a delayed_work
1154	* @flags: place to store irq state
1155	*
1156	* Try to grab PENDING bit of @work. This function can handle @work in any
1157	* stable state - idle, on timer or on worklist. Return values are
1158	*
1159	* 1 if @work was pending and we successfully stole PENDING
1160	* 0 if @work was idle and we claimed PENDING
1161	* -EAGAIN if PENDING couldn't be grabbed at the moment, safe to busy-retry
1162	* -ENOENT if someone else is canceling @work, this state may persist
1163	* for arbitrarily long
1164	*
1165	* On >= 0 return, the caller owns @work's PENDING bit. To avoid getting
1166	* interrupted while holding PENDING and @work off queue, irq must be
1167	* disabled on entry. This, combined with delayed_work->timer being
1168	* irqsafe, ensures that we return -EAGAIN for finite short period of time.
1169	*
1170	* On successful return, >= 0, irq is disabled and the caller is
1171	* responsible for releasing it using local_irq_restore(*@flags).
1172	*
1173	* This function is safe to call from any context including IRQ handler.
1174	*/
1175	static int try_to_grab_pending(struct work_struct *work, bool is_dwork,
1176	unsigned long *flags)
1177	{
1178	struct worker_pool *pool;
1179	struct pool_workqueue *pwq;
1180
1181	local_irq_save(*flags);
1182
1183	/* try to steal the timer if it exists */
1184	if (is_dwork) {
1185	struct delayed_work *dwork = to_delayed_work(work);
1186
1187	/*
1188	* dwork->timer is irqsafe. If del_timer() fails, it's
1189	* guaranteed that the timer is not queued anywhere and not
1190	* running on the local CPU.
1191	*/
1192	if (likely(del_timer(&dwork->timer)))
1193	return 1;
1194	}
1195
1196	/* try to claim PENDING the normal way */
1197	if (!test_and_set_bit(WORK_STRUCT_PENDING_BIT, work_data_bits(work)))
1198	return 0;
1199
1200	/*
1201	* The queueing is in progress, or it is already queued. Try to
1202	* steal it from ->worklist without clearing WORK_STRUCT_PENDING.
1203	*/
1204	pool = get_work_pool(work);
1205	if (!pool)
1206	goto fail;
1207
1208	spin_lock(&pool->lock);
1209	/*
1210	* work->data is guaranteed to point to pwq only while the work
1211	* item is queued on pwq->wq, and both updating work->data to point
1212	* to pwq on queueing and to pool on dequeueing are done under
1213	* pwq->pool->lock. This in turn guarantees that, if work->data
1214	* points to pwq which is associated with a locked pool, the work
1215	* item is currently queued on that pool.
1216	*/
1217	pwq = get_work_pwq(work);
1218	if (pwq && pwq->pool == pool) {
1219	debug_work_deactivate(work);
1220
1221	/*
1222	* A delayed work item cannot be grabbed directly because
1223	* it might have linked NO_COLOR work items which, if left
1224	* on the delayed_list, will confuse pwq->nr_active
1225	* management later on and cause stall. Make sure the work
1226	* item is activated before grabbing.
1227	*/
1228	if (*work_data_bits(work) & WORK_STRUCT_DELAYED)
1229	pwq_activate_delayed_work(work);
1230
1231	list_del_init(&work->entry);
1232	pwq_dec_nr_in_flight(get_work_pwq(work), get_work_color(work));
1233
1234	/* work->data points to pwq iff queued, point to pool */
1235	set_work_pool_and_keep_pending(work, pool->id);
1236
1237	spin_unlock(&pool->lock);
1238	return 1;
1239	}
1240	spin_unlock(&pool->lock);
1241	fail:
1242	local_irq_restore(*flags);
1243	if (work_is_canceling(work))
1244	return -ENOENT;
1245	cpu_relax();
1246	return -EAGAIN;
1247	}
1248
1249	/**
1250	* insert_work - insert a work into a pool
1251	* @pwq: pwq @work belongs to
1252	* @work: work to insert
1253	* @head: insertion point
1254	* @extra_flags: extra WORK_STRUCT_* flags to set
1255	*
1256	* Insert @work which belongs to @pwq after @head. @extra_flags is or'd to
1257	* work_struct flags.
1258	*
1259	* CONTEXT:
1260	* spin_lock_irq(pool->lock).
1261	*/
1262	static void insert_work(struct pool_workqueue pwq, struct work_struct work,
1263	struct list_head *head, unsigned int extra_flags)
1264	{
1265	struct worker_pool *pool = pwq->pool;
1266
1267	/* we own @work, set data and link */
1268	set_work_pwq(work, pwq, extra_flags);
1269	list_add_tail(&work->entry, head);
1270	get_pwq(pwq);
1271
1272	/*
1273	* Ensure either wq_worker_sleeping() sees the above
1274	* list_add_tail() or we see zero nr_running to avoid workers lying
1275	* around lazily while there are works to be processed.
1276	*/
1277	smp_mb();
1278
1279	if (__need_more_worker(pool))
1280	wake_up_worker(pool);
1281	}
1282
1283	/*
1284	* Test whether @work is being queued from another work executing on the
1285	* same workqueue.
1286	*/
1287	static bool is_chained_work(struct workqueue_struct *wq)
1288	{
1289	struct worker *worker;
1290
1291	worker = current_wq_worker();
1292	/*
1293	* Return %true iff I'm a worker execuing a work item on @wq. If
1294	* I'm @worker, it's safe to dereference it without locking.
1295	*/
1296	return worker && worker->current_pwq->wq == wq;
1297	}
1298
1299	static void __queue_work(int cpu, struct workqueue_struct *wq,
1300	struct work_struct *work)
1301	{
1302	struct pool_workqueue *pwq;
1303	struct worker_pool *last_pool;
1304	struct list_head *worklist;
1305	unsigned int work_flags;
1306	unsigned int req_cpu = cpu;
1307
1308	/*
1309	* While a work item is PENDING && off queue, a task trying to
1310	* steal the PENDING will busy-loop waiting for it to either get
1311	* queued or lose PENDING. Grabbing PENDING and queueing should
1312	* happen with IRQ disabled.
1313	*/
1314	WARN_ON_ONCE(!irqs_disabled());
1315
1316	debug_work_activate(work);
1317
1318	/* if dying, only works from the same workqueue are allowed */
1319	if (unlikely(wq->flags & __WQ_DRAINING) &&
1320	WARN_ON_ONCE(!is_chained_work(wq)))
1321	return;
1322	retry:
1323	if (req_cpu == WORK_CPU_UNBOUND)
1324	cpu = raw_smp_processor_id();
1325
1326	/* pwq which will be used unless @work is executing elsewhere */
1327	if (!(wq->flags & WQ_UNBOUND))
1328	pwq = per_cpu_ptr(wq->cpu_pwqs, cpu);
1329	else
1330	pwq = unbound_pwq_by_node(wq, cpu_to_node(cpu));
1331
1332	/*
1333	* If @work was previously on a different pool, it might still be
1334	* running there, in which case the work needs to be queued on that
1335	* pool to guarantee non-reentrancy.
1336	*/
1337	last_pool = get_work_pool(work);
1338	if (last_pool && last_pool != pwq->pool) {
1339	struct worker *worker;
1340
1341	spin_lock(&last_pool->lock);
1342
1343	worker = find_worker_executing_work(last_pool, work);
1344
1345	if (worker && worker->current_pwq->wq == wq) {
1346	pwq = worker->current_pwq;
1347	} else {
1348	/* meh... not running there, queue here */
1349	spin_unlock(&last_pool->lock);
1350	spin_lock(&pwq->pool->lock);
1351	}
1352	} else {
1353	spin_lock(&pwq->pool->lock);
1354	}
1355
1356	/*
1357	* pwq is determined and locked. For unbound pools, we could have
1358	* raced with pwq release and it could already be dead. If its
1359	* refcnt is zero, repeat pwq selection. Note that pwqs never die
1360	* without another pwq replacing it in the numa_pwq_tbl or while
1361	* work items are executing on it, so the retrying is guaranteed to
1362	* make forward-progress.
1363	*/
1364	if (unlikely(!pwq->refcnt)) {
1365	if (wq->flags & WQ_UNBOUND) {
1366	spin_unlock(&pwq->pool->lock);
1367	cpu_relax();
1368	goto retry;
1369	}
1370	/* oops */
1371	WARN_ONCE(true, "workqueue: per-cpu pwq for %s on cpu%d has 0 refcnt",
1372	wq->name, cpu);
1373	}
1374
1375	/* pwq determined, queue */
1376	trace_workqueue_queue_work(req_cpu, pwq, work);
1377
1378	if (WARN_ON(!list_empty(&work->entry))) {
1379	spin_unlock(&pwq->pool->lock);
1380	return;
1381	}
1382
1383	pwq->nr_in_flight[pwq->work_color]++;
1384	work_flags = work_color_to_flags(pwq->work_color);
1385
1386	if (likely(pwq->nr_active < pwq->max_active)) {
1387	trace_workqueue_activate_work(work);
1388	pwq->nr_active++;
1389	worklist = &pwq->pool->worklist;
1390	} else {
1391	work_flags \|= WORK_STRUCT_DELAYED;
1392	worklist = &pwq->delayed_works;
1393	}
1394
1395	insert_work(pwq, work, worklist, work_flags);
1396
1397	spin_unlock(&pwq->pool->lock);
1398	}
1399
1400	/**
1401	* queue_work_on - queue work on specific cpu
1402	* @cpu: CPU number to execute work on
1403	* @wq: workqueue to use
1404	* @work: work to queue
1405	*
1406	* Returns %false if @work was already on a queue, %true otherwise.
1407	*
1408	* We queue the work to a specific CPU, the caller must ensure it
1409	* can't go away.
1410	*/
1411	bool queue_work_on(int cpu, struct workqueue_struct *wq,
1412	struct work_struct *work)
1413	{
1414	bool ret = false;
1415	unsigned long flags;
1416
1417	local_irq_save(flags);
1418
1419	if (!test_and_set_bit(WORK_STRUCT_PENDING_BIT, work_data_bits(work))) {
1420	__queue_work(cpu, wq, work);
1421	ret = true;
1422	}
1423
1424	local_irq_restore(flags);
1425	return ret;
1426	}
1427	EXPORT_SYMBOL(queue_work_on);
1428
1429	void delayed_work_timer_fn(unsigned long __data)
1430	{
1431	struct delayed_work dwork = (struct delayed_work )__data;
1432
1433	/* should have been called from irqsafe timer with irq already off */
1434	__queue_work(dwork->cpu, dwork->wq, &dwork->work);
1435	}
1436	EXPORT_SYMBOL(delayed_work_timer_fn);
1437
1438	static void __queue_delayed_work(int cpu, struct workqueue_struct *wq,
1439	struct delayed_work *dwork, unsigned long delay)
1440	{
1441	struct timer_list *timer = &dwork->timer;
1442	struct work_struct *work = &dwork->work;
1443
1444	WARN_ON_ONCE(timer->function != delayed_work_timer_fn \|\|
1445	timer->data != (unsigned long)dwork);
1446	WARN_ON_ONCE(timer_pending(timer));
1447	WARN_ON_ONCE(!list_empty(&work->entry));
1448
1449	/*
1450	* If @delay is 0, queue @dwork->work immediately. This is for
1451	* both optimization and correctness. The earliest @timer can
1452	* expire is on the closest next tick and delayed_work users depend
1453	* on that there's no such delay when @delay is 0.
1454	*/
1455	if (!delay) {
1456	__queue_work(cpu, wq, &dwork->work);
1457	return;
1458	}
1459
1460	timer_stats_timer_set_start_info(&dwork->timer);
1461
1462	dwork->wq = wq;
1463	dwork->cpu = cpu;
1464	timer->expires = jiffies + delay;
1465
1466	if (unlikely(cpu != WORK_CPU_UNBOUND))
1467	add_timer_on(timer, cpu);
1468	else
1469	add_timer(timer);
1470	}
1471
1472	/**
1473	* queue_delayed_work_on - queue work on specific CPU after delay
1474	* @cpu: CPU number to execute work on
1475	* @wq: workqueue to use
1476	* @dwork: work to queue
1477	* @delay: number of jiffies to wait before queueing
1478	*
1479	* Returns %false if @work was already on a queue, %true otherwise. If
1480	* @delay is zero and @dwork is idle, it will be scheduled for immediate
1481	* execution.
1482	*/
1483	bool queue_delayed_work_on(int cpu, struct workqueue_struct *wq,
1484	struct delayed_work *dwork, unsigned long delay)
1485	{
1486	struct work_struct *work = &dwork->work;
1487	bool ret = false;
1488	unsigned long flags;
1489
1490	/* read the comment in __queue_work() */
1491	local_irq_save(flags);
1492
1493	if (!test_and_set_bit(WORK_STRUCT_PENDING_BIT, work_data_bits(work))) {
1494	__queue_delayed_work(cpu, wq, dwork, delay);
1495	ret = true;
1496	}
1497
1498	local_irq_restore(flags);
1499	return ret;
1500	}
1501	EXPORT_SYMBOL(queue_delayed_work_on);
1502
1503	/**
1504	* mod_delayed_work_on - modify delay of or queue a delayed work on specific CPU
1505	* @cpu: CPU number to execute work on
1506	* @wq: workqueue to use
1507	* @dwork: work to queue
1508	* @delay: number of jiffies to wait before queueing
1509	*
1510	* If @dwork is idle, equivalent to queue_delayed_work_on(); otherwise,
1511	* modify @dwork's timer so that it expires after @delay. If @delay is
1512	* zero, @work is guaranteed to be scheduled immediately regardless of its
1513	* current state.
1514	*
1515	* Returns %false if @dwork was idle and queued, %true if @dwork was
1516	* pending and its timer was modified.
1517	*
1518	* This function is safe to call from any context including IRQ handler.
1519	* See try_to_grab_pending() for details.
1520	*/
1521	bool mod_delayed_work_on(int cpu, struct workqueue_struct *wq,
1522	struct delayed_work *dwork, unsigned long delay)
1523	{
1524	unsigned long flags;
1525	int ret;
1526
1527	do {
1528	ret = try_to_grab_pending(&dwork->work, true, &flags);
1529	} while (unlikely(ret == -EAGAIN));
1530
1531	if (likely(ret >= 0)) {
1532	__queue_delayed_work(cpu, wq, dwork, delay);
1533	local_irq_restore(flags);
1534	}
1535
1536	/* -ENOENT from try_to_grab_pending() becomes %true */
1537	return ret;
1538	}
1539	EXPORT_SYMBOL_GPL(mod_delayed_work_on);
1540
1541	/**
1542	* worker_enter_idle - enter idle state
1543	* @worker: worker which is entering idle state
1544	*
1545	* @worker is entering idle state. Update stats and idle timer if
1546	* necessary.
1547	*
1548	* LOCKING:
1549	* spin_lock_irq(pool->lock).
1550	*/
1551	static void worker_enter_idle(struct worker *worker)
1552	{
1553	struct worker_pool *pool = worker->pool;
1554
1555	if (WARN_ON_ONCE(worker->flags & WORKER_IDLE) \|\|
1556	WARN_ON_ONCE(!list_empty(&worker->entry) &&
1557	(worker->hentry.next \|\| worker->hentry.pprev)))
1558	return;
1559
1560	/* can't use worker_set_flags(), also called from start_worker() */
1561	worker->flags \|= WORKER_IDLE;
1562	pool->nr_idle++;
1563	worker->last_active = jiffies;
1564
1565	/* idle_list is LIFO */
1566	list_add(&worker->entry, &pool->idle_list);
1567
1568	if (too_many_workers(pool) && !timer_pending(&pool->idle_timer))
1569	mod_timer(&pool->idle_timer, jiffies + IDLE_WORKER_TIMEOUT);
1570
1571	/*
1572	* Sanity check nr_running. Because wq_unbind_fn() releases
1573	* pool->lock between setting %WORKER_UNBOUND and zapping
1574	* nr_running, the warning may trigger spuriously. Check iff
1575	* unbind is not in progress.
1576	*/
1577	WARN_ON_ONCE(!(pool->flags & POOL_DISASSOCIATED) &&
1578	pool->nr_workers == pool->nr_idle &&
1579	atomic_read(&pool->nr_running));
1580	}
1581
1582	/**
1583	* worker_leave_idle - leave idle state
1584	* @worker: worker which is leaving idle state
1585	*
1586	* @worker is leaving idle state. Update stats.
1587	*
1588	* LOCKING:
1589	* spin_lock_irq(pool->lock).
1590	*/
1591	static void worker_leave_idle(struct worker *worker)
1592	{
1593	struct worker_pool *pool = worker->pool;
1594
1595	if (WARN_ON_ONCE(!(worker->flags & WORKER_IDLE)))
1596	return;
1597	worker_clr_flags(worker, WORKER_IDLE);
1598	pool->nr_idle--;
1599	list_del_init(&worker->entry);
1600	}
1601
1602	/**
1603	* worker_maybe_bind_and_lock - try to bind %current to worker_pool and lock it
1604	* @pool: target worker_pool
1605	*
1606	* Bind %current to the cpu of @pool if it is associated and lock @pool.
1607	*
1608	* Works which are scheduled while the cpu is online must at least be
1609	* scheduled to a worker which is bound to the cpu so that if they are
1610	* flushed from cpu callbacks while cpu is going down, they are
1611	* guaranteed to execute on the cpu.
1612	*
1613	* This function is to be used by unbound workers and rescuers to bind
1614	* themselves to the target cpu and may race with cpu going down or
1615	* coming online. kthread_bind() can't be used because it may put the
1616	* worker to already dead cpu and set_cpus_allowed_ptr() can't be used
1617	* verbatim as it's best effort and blocking and pool may be
1618	* [dis]associated in the meantime.
1619	*
1620	* This function tries set_cpus_allowed() and locks pool and verifies the
1621	* binding against %POOL_DISASSOCIATED which is set during
1622	* %CPU_DOWN_PREPARE and cleared during %CPU_ONLINE, so if the worker
1623	* enters idle state or fetches works without dropping lock, it can
1624	* guarantee the scheduling requirement described in the first paragraph.
1625	*
1626	* CONTEXT:
1627	* Might sleep. Called without any lock but returns with pool->lock
1628	* held.
1629	*
1630	* RETURNS:
1631	* %true if the associated pool is online (@worker is successfully
1632	* bound), %false if offline.
1633	*/
1634	static bool worker_maybe_bind_and_lock(struct worker_pool *pool)
1635	__acquires(&pool->lock)
1636	{
1637	while (true) {
1638	/*
1639	* The following call may fail, succeed or succeed
1640	* without actually migrating the task to the cpu if
1641	* it races with cpu hotunplug operation. Verify
1642	* against POOL_DISASSOCIATED.
1643	*/
1644	if (!(pool->flags & POOL_DISASSOCIATED))
1645	set_cpus_allowed_ptr(current, pool->attrs->cpumask);
1646
1647	spin_lock_irq(&pool->lock);
1648	if (pool->flags & POOL_DISASSOCIATED)
1649	return false;
1650	if (task_cpu(current) == pool->cpu &&
1651	cpumask_equal(&current->cpus_allowed, pool->attrs->cpumask))
1652	return true;
1653	spin_unlock_irq(&pool->lock);
1654
1655	/*
1656	* We've raced with CPU hot[un]plug. Give it a breather
1657	* and retry migration. cond_resched() is required here;
1658	* otherwise, we might deadlock against cpu_stop trying to
1659	* bring down the CPU on non-preemptive kernel.
1660	*/
1661	cpu_relax();
1662	cond_resched();
1663	}
1664	}
1665
1666	static struct worker *alloc_worker(void)
1667	{
1668	struct worker *worker;
1669
1670	worker = kzalloc(sizeof(*worker), GFP_KERNEL);
1671	if (worker) {
1672	INIT_LIST_HEAD(&worker->entry);
1673	INIT_LIST_HEAD(&worker->scheduled);
1674	/* on creation a worker is in !idle && prep state */
1675	worker->flags = WORKER_PREP;
1676	}
1677	return worker;
1678	}
1679
1680	/**
1681	* create_worker - create a new workqueue worker
1682	* @pool: pool the new worker will belong to
1683	*
1684	* Create a new worker which is bound to @pool. The returned worker
1685	* can be started by calling start_worker() or destroyed using
1686	* destroy_worker().
1687	*
1688	* CONTEXT:
1689	* Might sleep. Does GFP_KERNEL allocations.
1690	*
1691	* RETURNS:
1692	* Pointer to the newly created worker.
1693	*/
1694	static struct worker create_worker(struct worker_pool pool)
1695	{
1696	struct worker *worker = NULL;
1697	int id = -1;
1698	char id_buf[16];
1699
1700	lockdep_assert_held(&pool->manager_mutex);
1701
1702	/*
1703	* ID is needed to determine kthread name. Allocate ID first
1704	* without installing the pointer.
1705	*/
1706	idr_preload(GFP_KERNEL);
1707	spin_lock_irq(&pool->lock);
1708
1709	id = idr_alloc(&pool->worker_idr, NULL, 0, 0, GFP_NOWAIT);
1710
1711	spin_unlock_irq(&pool->lock);
1712	idr_preload_end();
1713	if (id < 0)
1714	goto fail;
1715
1716	worker = alloc_worker();
1717	if (!worker)
1718	goto fail;
1719
1720	worker->pool = pool;
1721	worker->id = id;
1722
1723	if (pool->cpu >= 0)
1724	snprintf(id_buf, sizeof(id_buf), "%d:%d%s", pool->cpu, id,
1725	pool->attrs->nice < 0 ? "H" : "");
1726	else
1727	snprintf(id_buf, sizeof(id_buf), "u%d:%d", pool->id, id);
1728
1729	worker->task = kthread_create_on_node(worker_thread, worker, pool->node,
1730	"kworker/%s", id_buf);
1731	if (IS_ERR(worker->task))
1732	goto fail;
1733
1734	/*
1735	* set_cpus_allowed_ptr() will fail if the cpumask doesn't have any
1736	* online CPUs. It'll be re-applied when any of the CPUs come up.
1737	*/
1738	set_user_nice(worker->task, pool->attrs->nice);
1739	set_cpus_allowed_ptr(worker->task, pool->attrs->cpumask);
1740
1741	/* prevent userland from meddling with cpumask of workqueue workers */
1742	worker->task->flags \|= PF_NO_SETAFFINITY;
1743
1744	/*
1745	* The caller is responsible for ensuring %POOL_DISASSOCIATED
1746	* remains stable across this function. See the comments above the
1747	* flag definition for details.
1748	*/
1749	if (pool->flags & POOL_DISASSOCIATED)
1750	worker->flags \|= WORKER_UNBOUND;
1751
1752	/* successful, commit the pointer to idr */
1753	spin_lock_irq(&pool->lock);
1754	idr_replace(&pool->worker_idr, worker, worker->id);
1755	spin_unlock_irq(&pool->lock);
1756
1757	return worker;
1758
1759	fail:
1760	if (id >= 0) {
1761	spin_lock_irq(&pool->lock);
1762	idr_remove(&pool->worker_idr, id);
1763	spin_unlock_irq(&pool->lock);
1764	}
1765	kfree(worker);
1766	return NULL;
1767	}
1768
1769	/**
1770	* start_worker - start a newly created worker
1771	* @worker: worker to start
1772	*
1773	* Make the pool aware of @worker and start it.
1774	*
1775	* CONTEXT:
1776	* spin_lock_irq(pool->lock).
1777	*/
1778	static void start_worker(struct worker *worker)
1779	{
1780	worker->flags \|= WORKER_STARTED;
1781	worker->pool->nr_workers++;
1782	worker_enter_idle(worker);
1783	wake_up_process(worker->task);
1784	}
1785
1786	/**
1787	* create_and_start_worker - create and start a worker for a pool
1788	* @pool: the target pool
1789	*
1790	* Grab the managership of @pool and create and start a new worker for it.
1791	*/
1792	static int create_and_start_worker(struct worker_pool *pool)
1793	{
1794	struct worker *worker;
1795
1796	mutex_lock(&pool->manager_mutex);
1797
1798	worker = create_worker(pool);
1799	if (worker) {
1800	spin_lock_irq(&pool->lock);
1801	start_worker(worker);
1802	spin_unlock_irq(&pool->lock);
1803	}
1804
1805	mutex_unlock(&pool->manager_mutex);
1806
1807	return worker ? 0 : -ENOMEM;
1808	}
1809
1810	/**
1811	* destroy_worker - destroy a workqueue worker
1812	* @worker: worker to be destroyed
1813	*
1814	* Destroy @worker and adjust @pool stats accordingly.
1815	*
1816	* CONTEXT:
1817	* spin_lock_irq(pool->lock) which is released and regrabbed.
1818	*/
1819	static void destroy_worker(struct worker *worker)
1820	{
1821	struct worker_pool *pool = worker->pool;
1822
1823	lockdep_assert_held(&pool->manager_mutex);
1824	lockdep_assert_held(&pool->lock);
1825
1826	/* sanity check frenzy */
1827	if (WARN_ON(worker->current_work) \|\|
1828	WARN_ON(!list_empty(&worker->scheduled)))
1829	return;
1830
1831	if (worker->flags & WORKER_STARTED)
1832	pool->nr_workers--;
1833	if (worker->flags & WORKER_IDLE)
1834	pool->nr_idle--;
1835
1836	list_del_init(&worker->entry);
1837	worker->flags \|= WORKER_DIE;
1838
1839	idr_remove(&pool->worker_idr, worker->id);
1840
1841	spin_unlock_irq(&pool->lock);
1842
1843	kthread_stop(worker->task);
1844	kfree(worker);
1845
1846	spin_lock_irq(&pool->lock);
1847	}
1848
1849	static void idle_worker_timeout(unsigned long __pool)
1850	{
1851	struct worker_pool pool = (void )__pool;
1852
1853	spin_lock_irq(&pool->lock);
1854
1855	if (too_many_workers(pool)) {
1856	struct worker *worker;
1857	unsigned long expires;
1858
1859	/* idle_list is kept in LIFO order, check the last one */
1860	worker = list_entry(pool->idle_list.prev, struct worker, entry);
1861	expires = worker->last_active + IDLE_WORKER_TIMEOUT;
1862
1863	if (time_before(jiffies, expires))
1864	mod_timer(&pool->idle_timer, expires);
1865	else {
1866	/* it's been idle for too long, wake up manager */
1867	pool->flags \|= POOL_MANAGE_WORKERS;
1868	wake_up_worker(pool);
1869	}
1870	}
1871
1872	spin_unlock_irq(&pool->lock);
1873	}
1874
1875	static void send_mayday(struct work_struct *work)
1876	{
1877	struct pool_workqueue *pwq = get_work_pwq(work);
1878	struct workqueue_struct *wq = pwq->wq;
1879
1880	lockdep_assert_held(&wq_mayday_lock);
1881
1882	if (!wq->rescuer)
1883	return;
1884
1885	/* mayday mayday mayday */
1886	if (list_empty(&pwq->mayday_node)) {
1887	list_add_tail(&pwq->mayday_node, &wq->maydays);
1888	wake_up_process(wq->rescuer->task);
1889	}
1890	}
1891
1892	static void pool_mayday_timeout(unsigned long __pool)
1893	{
1894	struct worker_pool pool = (void )__pool;
1895	struct work_struct *work;
1896
1897	spin_lock_irq(&wq_mayday_lock); /* for wq->maydays */
1898	spin_lock(&pool->lock);
1899
1900	if (need_to_create_worker(pool)) {
1901	/*
1902	* We've been trying to create a new worker but
1903	* haven't been successful. We might be hitting an
1904	* allocation deadlock. Send distress signals to
1905	* rescuers.
1906	*/
1907	list_for_each_entry(work, &pool->worklist, entry)
1908	send_mayday(work);
1909	}
1910
1911	spin_unlock(&pool->lock);
1912	spin_unlock_irq(&wq_mayday_lock);
1913
1914	mod_timer(&pool->mayday_timer, jiffies + MAYDAY_INTERVAL);
1915	}
1916
1917	/**
1918	* maybe_create_worker - create a new worker if necessary
1919	* @pool: pool to create a new worker for
1920	*
1921	* Create a new worker for @pool if necessary. @pool is guaranteed to
1922	* have at least one idle worker on return from this function. If
1923	* creating a new worker takes longer than MAYDAY_INTERVAL, mayday is
1924	* sent to all rescuers with works scheduled on @pool to resolve
1925	* possible allocation deadlock.
1926	*
1927	* On return, need_to_create_worker() is guaranteed to be %false and
1928	* may_start_working() %true.
1929	*
1930	* LOCKING:
1931	* spin_lock_irq(pool->lock) which may be released and regrabbed
1932	* multiple times. Does GFP_KERNEL allocations. Called only from
1933	* manager.
1934	*
1935	* RETURNS:
1936	* %false if no action was taken and pool->lock stayed locked, %true
1937	* otherwise.
1938	*/
1939	static bool maybe_create_worker(struct worker_pool *pool)
1940	__releases(&pool->lock)
1941	__acquires(&pool->lock)
1942	{
1943	if (!need_to_create_worker(pool))
1944	return false;
1945	restart:
1946	spin_unlock_irq(&pool->lock);
1947
1948	/* if we don't make progress in MAYDAY_INITIAL_TIMEOUT, call for help */
1949	mod_timer(&pool->mayday_timer, jiffies + MAYDAY_INITIAL_TIMEOUT);
1950
1951	while (true) {
1952	struct worker *worker;
1953
1954	worker = create_worker(pool);
1955	if (worker) {
1956	del_timer_sync(&pool->mayday_timer);
1957	spin_lock_irq(&pool->lock);
1958	start_worker(worker);
1959	if (WARN_ON_ONCE(need_to_create_worker(pool)))
1960	goto restart;
1961	return true;
1962	}
1963
1964	if (!need_to_create_worker(pool))
1965	break;
1966
1967	__set_current_state(TASK_INTERRUPTIBLE);
1968	schedule_timeout(CREATE_COOLDOWN);
1969
1970	if (!need_to_create_worker(pool))
1971	break;
1972	}
1973
1974	del_timer_sync(&pool->mayday_timer);
1975	spin_lock_irq(&pool->lock);
1976	if (need_to_create_worker(pool))
1977	goto restart;
1978	return true;
1979	}
1980
1981	/**
1982	* maybe_destroy_worker - destroy workers which have been idle for a while
1983	* @pool: pool to destroy workers for
1984	*
1985	* Destroy @pool workers which have been idle for longer than
1986	* IDLE_WORKER_TIMEOUT.
1987	*
1988	* LOCKING:
1989	* spin_lock_irq(pool->lock) which may be released and regrabbed
1990	* multiple times. Called only from manager.
1991	*
1992	* RETURNS:
1993	* %false if no action was taken and pool->lock stayed locked, %true
1994	* otherwise.
1995	*/
1996	static bool maybe_destroy_workers(struct worker_pool *pool)
1997	{
1998	bool ret = false;
1999
2000	while (too_many_workers(pool)) {
2001	struct worker *worker;
2002	unsigned long expires;
2003
2004	worker = list_entry(pool->idle_list.prev, struct worker, entry);
2005	expires = worker->last_active + IDLE_WORKER_TIMEOUT;
2006
2007	if (time_before(jiffies, expires)) {
2008	mod_timer(&pool->idle_timer, expires);
2009	break;
2010	}
2011
2012	destroy_worker(worker);
2013	ret = true;
2014	}
2015
2016	return ret;
2017	}
2018
2019	/**
2020	* manage_workers - manage worker pool
2021	* @worker: self
2022	*
2023	* Assume the manager role and manage the worker pool @worker belongs
2024	* to. At any given time, there can be only zero or one manager per
2025	* pool. The exclusion is handled automatically by this function.
2026	*
2027	* The caller can safely start processing works on false return. On
2028	* true return, it's guaranteed that need_to_create_worker() is false
2029	* and may_start_working() is true.
2030	*
2031	* CONTEXT:
2032	* spin_lock_irq(pool->lock) which may be released and regrabbed
2033	* multiple times. Does GFP_KERNEL allocations.
2034	*
2035	* RETURNS:
2036	* spin_lock_irq(pool->lock) which may be released and regrabbed
2037	* multiple times. Does GFP_KERNEL allocations.
2038	*/
2039	static bool manage_workers(struct worker *worker)
2040	{
2041	struct worker_pool *pool = worker->pool;
2042	bool ret = false;
2043
2044	/*
2045	* Managership is governed by two mutexes - manager_arb and
2046	* manager_mutex. manager_arb handles arbitration of manager role.
2047	* Anyone who successfully grabs manager_arb wins the arbitration
2048	* and becomes the manager. mutex_trylock() on pool->manager_arb
2049	* failure while holding pool->lock reliably indicates that someone
2050	* else is managing the pool and the worker which failed trylock
2051	* can proceed to executing work items. This means that anyone
2052	* grabbing manager_arb is responsible for actually performing
2053	* manager duties. If manager_arb is grabbed and released without
2054	* actual management, the pool may stall indefinitely.
2055	*
2056	* manager_mutex is used for exclusion of actual management
2057	* operations. The holder of manager_mutex can be sure that none
2058	* of management operations, including creation and destruction of
2059	* workers, won't take place until the mutex is released. Because
2060	* manager_mutex doesn't interfere with manager role arbitration,
2061	* it is guaranteed that the pool's management, while may be
2062	* delayed, won't be disturbed by someone else grabbing
2063	* manager_mutex.
2064	*/
2065	if (!mutex_trylock(&pool->manager_arb))
2066	return ret;
2067
2068	/*
2069	* With manager arbitration won, manager_mutex would be free in
2070	* most cases. trylock first without dropping @pool->lock.
2071	*/
2072	if (unlikely(!mutex_trylock(&pool->manager_mutex))) {
2073	spin_unlock_irq(&pool->lock);
2074	mutex_lock(&pool->manager_mutex);
2075	spin_lock_irq(&pool->lock);
2076	ret = true;
2077	}
2078
2079	pool->flags &= ~POOL_MANAGE_WORKERS;
2080
2081	/*
2082	* Destroy and then create so that may_start_working() is true
2083	* on return.
2084	*/
2085	ret \|= maybe_destroy_workers(pool);
2086	ret \|= maybe_create_worker(pool);
2087
2088	mutex_unlock(&pool->manager_mutex);
2089	mutex_unlock(&pool->manager_arb);
2090	return ret;
2091	}
2092
2093	/**
2094	* process_one_work - process single work
2095	* @worker: self
2096	* @work: work to process
2097	*
2098	* Process @work. This function contains all the logics necessary to
2099	* process a single work including synchronization against and
2100	* interaction with other workers on the same cpu, queueing and
2101	* flushing. As long as context requirement is met, any worker can
2102	* call this function to process a work.
2103	*
2104	* CONTEXT:
2105	* spin_lock_irq(pool->lock) which is released and regrabbed.
2106	*/
2107	static void process_one_work(struct worker worker, struct work_struct work)
2108	__releases(&pool->lock)
2109	__acquires(&pool->lock)
2110	{
2111	struct pool_workqueue *pwq = get_work_pwq(work);
2112	struct worker_pool *pool = worker->pool;
2113	bool cpu_intensive = pwq->wq->flags & WQ_CPU_INTENSIVE;
2114	int work_color;
2115	struct worker *collision;
2116	#ifdef CONFIG_LOCKDEP
2117	/*
2118	* It is permissible to free the struct work_struct from
2119	* inside the function that is called from it, this we need to
2120	* take into account for lockdep too. To avoid bogus "held
2121	* lock freed" warnings as well as problems when looking into
2122	* work->lockdep_map, make a copy and use that here.
2123	*/
2124	struct lockdep_map lockdep_map;
2125
2126	lockdep_copy_map(&lockdep_map, &work->lockdep_map);
2127	#endif
2128	/*
2129	* Ensure we're on the correct CPU. DISASSOCIATED test is
2130	* necessary to avoid spurious warnings from rescuers servicing the
2131	* unbound or a disassociated pool.
2132	*/
2133	WARN_ON_ONCE(!(worker->flags & WORKER_UNBOUND) &&
2134	!(pool->flags & POOL_DISASSOCIATED) &&
2135	raw_smp_processor_id() != pool->cpu);
2136
2137	/*
2138	* A single work shouldn't be executed concurrently by
2139	* multiple workers on a single cpu. Check whether anyone is
2140	* already processing the work. If so, defer the work to the
2141	* currently executing one.
2142	*/
2143	collision = find_worker_executing_work(pool, work);
2144	if (unlikely(collision)) {
2145	move_linked_works(work, &collision->scheduled, NULL);
2146	return;
2147	}
2148
2149	/* claim and dequeue */
2150	debug_work_deactivate(work);
2151	hash_add(pool->busy_hash, &worker->hentry, (unsigned long)work);
2152	worker->current_work = work;
2153	worker->current_func = work->func;
2154	worker->current_pwq = pwq;
2155	work_color = get_work_color(work);
2156
2157	list_del_init(&work->entry);
2158
2159	/*
2160	* CPU intensive works don't participate in concurrency
2161	* management. They're the scheduler's responsibility.
2162	*/
2163	if (unlikely(cpu_intensive))
2164	worker_set_flags(worker, WORKER_CPU_INTENSIVE, true);
2165
2166	/*
2167	* Unbound pool isn't concurrency managed and work items should be
2168	* executed ASAP. Wake up another worker if necessary.
2169	*/
2170	if ((worker->flags & WORKER_UNBOUND) && need_more_worker(pool))
2171	wake_up_worker(pool);
2172
2173	/*
2174	* Record the last pool and clear PENDING which should be the last
2175	* update to @work. Also, do this inside @pool->lock so that
2176	* PENDING and queued state changes happen together while IRQ is
2177	* disabled.
2178	*/
2179	set_work_pool_and_clear_pending(work, pool->id);
2180
2181	spin_unlock_irq(&pool->lock);
2182
2183	lock_map_acquire_read(&pwq->wq->lockdep_map);
2184	lock_map_acquire(&lockdep_map);
2185	trace_workqueue_execute_start(work);
2186	worker->current_func(work);
2187	/*
2188	* While we must be careful to not use "work" after this, the trace
2189	* point will only record its address.
2190	*/
2191	trace_workqueue_execute_end(work);
2192	lock_map_release(&lockdep_map);
2193	lock_map_release(&pwq->wq->lockdep_map);
2194
2195	if (unlikely(in_atomic() \|\| lockdep_depth(current) > 0)) {
2196	pr_err("BUG: workqueue leaked lock or atomic: %s/0x%08x/%d\n"
2197	" last function: %pf\n",
2198	current->comm, preempt_count(), task_pid_nr(current),
2199	worker->current_func);
2200	debug_show_held_locks(current);
2201	dump_stack();
2202	}
2203
2204	spin_lock_irq(&pool->lock);
2205
2206	/* clear cpu intensive status */
2207	if (unlikely(cpu_intensive))
2208	worker_clr_flags(worker, WORKER_CPU_INTENSIVE);
2209
2210	/* we're done with it, release */
2211	hash_del(&worker->hentry);
2212	worker->current_work = NULL;
2213	worker->current_func = NULL;
2214	worker->current_pwq = NULL;
2215	worker->desc_valid = false;
2216	pwq_dec_nr_in_flight(pwq, work_color);
2217	}
2218
2219	/**
2220	* process_scheduled_works - process scheduled works
2221	* @worker: self
2222	*
2223	* Process all scheduled works. Please note that the scheduled list
2224	* may change while processing a work, so this function repeatedly
2225	* fetches a work from the top and executes it.
2226	*
2227	* CONTEXT:
2228	* spin_lock_irq(pool->lock) which may be released and regrabbed
2229	* multiple times.
2230	*/
2231	static void process_scheduled_works(struct worker *worker)
2232	{
2233	while (!list_empty(&worker->scheduled)) {
2234	struct work_struct *work = list_first_entry(&worker->scheduled,
2235	struct work_struct, entry);
2236	process_one_work(worker, work);
2237	}
2238	}
2239
2240	/**
2241	* worker_thread - the worker thread function
2242	* @__worker: self
2243	*
2244	* The worker thread function. All workers belong to a worker_pool -
2245	* either a per-cpu one or dynamic unbound one. These workers process all
2246	* work items regardless of their specific target workqueue. The only
2247	* exception is work items which belong to workqueues with a rescuer which
2248	* will be explained in rescuer_thread().
2249	*/
2250	static int worker_thread(void *__worker)
2251	{
2252	struct worker *worker = __worker;
2253	struct worker_pool *pool = worker->pool;
2254
2255	/* tell the scheduler that this is a workqueue worker */
2256	worker->task->flags \|= PF_WQ_WORKER;
2257	woke_up:
2258	spin_lock_irq(&pool->lock);
2259
2260	/* am I supposed to die? */
2261	if (unlikely(worker->flags & WORKER_DIE)) {
2262	spin_unlock_irq(&pool->lock);
2263	WARN_ON_ONCE(!list_empty(&worker->entry));
2264	worker->task->flags &= ~PF_WQ_WORKER;
2265	return 0;
2266	}
2267
2268	worker_leave_idle(worker);
2269	recheck:
2270	/* no more worker necessary? */
2271	if (!need_more_worker(pool))
2272	goto sleep;
2273
2274	/* do we need to manage? */
2275	if (unlikely(!may_start_working(pool)) && manage_workers(worker))
2276	goto recheck;
2277
2278	/*
2279	* ->scheduled list can only be filled while a worker is
2280	* preparing to process a work or actually processing it.
2281	* Make sure nobody diddled with it while I was sleeping.
2282	*/
2283	WARN_ON_ONCE(!list_empty(&worker->scheduled));
2284
2285	/*
2286	* Finish PREP stage. We're guaranteed to have at least one idle
2287	* worker or that someone else has already assumed the manager
2288	* role. This is where @worker starts participating in concurrency
2289	* management if applicable and concurrency management is restored
2290	* after being rebound. See rebind_workers() for details.
2291	*/
2292	worker_clr_flags(worker, WORKER_PREP \| WORKER_REBOUND);
2293
2294	do {
2295	struct work_struct *work =
2296	list_first_entry(&pool->worklist,
2297	struct work_struct, entry);
2298
2299	if (likely(!(*work_data_bits(work) & WORK_STRUCT_LINKED))) {
2300	/* optimization path, not strictly necessary */
2301	process_one_work(worker, work);
2302	if (unlikely(!list_empty(&worker->scheduled)))
2303	process_scheduled_works(worker);
2304	} else {
2305	move_linked_works(work, &worker->scheduled, NULL);
2306	process_scheduled_works(worker);
2307	}
2308	} while (keep_working(pool));
2309
2310	worker_set_flags(worker, WORKER_PREP, false);
2311	sleep:
2312	if (unlikely(need_to_manage_workers(pool)) && manage_workers(worker))
2313	goto recheck;
2314
2315	/*
2316	* pool->lock is held and there's no work to process and no need to
2317	* manage, sleep. Workers are woken up only while holding
2318	* pool->lock or from local cpu, so setting the current state
2319	* before releasing pool->lock is enough to prevent losing any
2320	* event.
2321	*/
2322	worker_enter_idle(worker);
2323	__set_current_state(TASK_INTERRUPTIBLE);
2324	spin_unlock_irq(&pool->lock);
2325	schedule();
2326	goto woke_up;
2327	}
2328
2329	/**
2330	* rescuer_thread - the rescuer thread function
2331	* @__rescuer: self
2332	*
2333	* Workqueue rescuer thread function. There's one rescuer for each
2334	* workqueue which has WQ_MEM_RECLAIM set.
2335	*
2336	* Regular work processing on a pool may block trying to create a new
2337	* worker which uses GFP_KERNEL allocation which has slight chance of
2338	* developing into deadlock if some works currently on the same queue
2339	* need to be processed to satisfy the GFP_KERNEL allocation. This is
2340	* the problem rescuer solves.
2341	*
2342	* When such condition is possible, the pool summons rescuers of all
2343	* workqueues which have works queued on the pool and let them process
2344	* those works so that forward progress can be guaranteed.
2345	*
2346	* This should happen rarely.
2347	*/
2348	static int rescuer_thread(void *__rescuer)
2349	{
2350	struct worker *rescuer = __rescuer;
2351	struct workqueue_struct *wq = rescuer->rescue_wq;
2352	struct list_head *scheduled = &rescuer->scheduled;
2353
2354	set_user_nice(current, RESCUER_NICE_LEVEL);
2355
2356	/*
2357	* Mark rescuer as worker too. As WORKER_PREP is never cleared, it
2358	* doesn't participate in concurrency management.
2359	*/
2360	rescuer->task->flags \|= PF_WQ_WORKER;
2361	repeat:
2362	set_current_state(TASK_INTERRUPTIBLE);
2363
2364	if (kthread_should_stop()) {
2365	__set_current_state(TASK_RUNNING);
2366	rescuer->task->flags &= ~PF_WQ_WORKER;
2367	return 0;
2368	}
2369
2370	/* see whether any pwq is asking for help */
2371	spin_lock_irq(&wq_mayday_lock);
2372
2373	while (!list_empty(&wq->maydays)) {
2374	struct pool_workqueue *pwq = list_first_entry(&wq->maydays,
2375	struct pool_workqueue, mayday_node);
2376	struct worker_pool *pool = pwq->pool;
2377	struct work_struct work, n;
2378
2379	__set_current_state(TASK_RUNNING);
2380	list_del_init(&pwq->mayday_node);
2381
2382	spin_unlock_irq(&wq_mayday_lock);
2383
2384	/* migrate to the target cpu if possible */
2385	worker_maybe_bind_and_lock(pool);
2386	rescuer->pool = pool;
2387
2388	/*
2389	* Slurp in all works issued via this workqueue and
2390	* process'em.
2391	*/
2392	WARN_ON_ONCE(!list_empty(&rescuer->scheduled));
2393	list_for_each_entry_safe(work, n, &pool->worklist, entry)
2394	if (get_work_pwq(work) == pwq)
2395	move_linked_works(work, scheduled, &n);
2396
2397	process_scheduled_works(rescuer);
2398
2399	/*
2400	* Leave this pool. If keep_working() is %true, notify a
2401	* regular worker; otherwise, we end up with 0 concurrency
2402	* and stalling the execution.
2403	*/
2404	if (keep_working(pool))
2405	wake_up_worker(pool);
2406
2407	rescuer->pool = NULL;
2408	spin_unlock(&pool->lock);
2409	spin_lock(&wq_mayday_lock);
2410	}
2411
2412	spin_unlock_irq(&wq_mayday_lock);
2413
2414	/* rescuers should never participate in concurrency management */
2415	WARN_ON_ONCE(!(rescuer->flags & WORKER_NOT_RUNNING));
2416	schedule();
2417	goto repeat;
2418	}
2419
2420	struct wq_barrier {
2421	struct work_struct work;
2422	struct completion done;
2423	};
2424
2425	static void wq_barrier_func(struct work_struct *work)
2426	{
2427	struct wq_barrier *barr = container_of(work, struct wq_barrier, work);
2428	complete(&barr->done);
2429	}
2430
2431	/**
2432	* insert_wq_barrier - insert a barrier work
2433	* @pwq: pwq to insert barrier into
2434	* @barr: wq_barrier to insert
2435	* @target: target work to attach @barr to
2436	* @worker: worker currently executing @target, NULL if @target is not executing
2437	*
2438	* @barr is linked to @target such that @barr is completed only after
2439	* @target finishes execution. Please note that the ordering
2440	* guarantee is observed only with respect to @target and on the local
2441	* cpu.
2442	*
2443	* Currently, a queued barrier can't be canceled. This is because
2444	* try_to_grab_pending() can't determine whether the work to be
2445	* grabbed is at the head of the queue and thus can't clear LINKED
2446	* flag of the previous work while there must be a valid next work
2447	* after a work with LINKED flag set.
2448	*
2449	* Note that when @worker is non-NULL, @target may be modified
2450	* underneath us, so we can't reliably determine pwq from @target.
2451	*
2452	* CONTEXT:
2453	* spin_lock_irq(pool->lock).
2454	*/
2455	static void insert_wq_barrier(struct pool_workqueue *pwq,
2456	struct wq_barrier *barr,
2457	struct work_struct target, struct worker worker)
2458	{
2459	struct list_head *head;
2460	unsigned int linked = 0;
2461
2462	/*
2463	* debugobject calls are safe here even with pool->lock locked
2464	* as we know for sure that this will not trigger any of the
2465	* checks and call back into the fixup functions where we
2466	* might deadlock.
2467	*/
2468	INIT_WORK_ONSTACK(&barr->work, wq_barrier_func);
2469	__set_bit(WORK_STRUCT_PENDING_BIT, work_data_bits(&barr->work));
2470	init_completion(&barr->done);
2471
2472	/*
2473	* If @target is currently being executed, schedule the
2474	* barrier to the worker; otherwise, put it after @target.
2475	*/
2476	if (worker)
2477	head = worker->scheduled.next;
2478	else {
2479	unsigned long *bits = work_data_bits(target);
2480
2481	head = target->entry.next;
2482	/* there can already be other linked works, inherit and set */
2483	linked = *bits & WORK_STRUCT_LINKED;
2484	__set_bit(WORK_STRUCT_LINKED_BIT, bits);
2485	}
2486
2487	debug_work_activate(&barr->work);
2488	insert_work(pwq, &barr->work, head,
2489	work_color_to_flags(WORK_NO_COLOR) \| linked);
2490	}
2491
2492	/**
2493	* flush_workqueue_prep_pwqs - prepare pwqs for workqueue flushing
2494	* @wq: workqueue being flushed
2495	* @flush_color: new flush color, < 0 for no-op
2496	* @work_color: new work color, < 0 for no-op
2497	*
2498	* Prepare pwqs for workqueue flushing.
2499	*
2500	* If @flush_color is non-negative, flush_color on all pwqs should be
2501	* -1. If no pwq has in-flight commands at the specified color, all
2502	* pwq->flush_color's stay at -1 and %false is returned. If any pwq
2503	* has in flight commands, its pwq->flush_color is set to
2504	* @flush_color, @wq->nr_pwqs_to_flush is updated accordingly, pwq
2505	* wakeup logic is armed and %true is returned.
2506	*
2507	* The caller should have initialized @wq->first_flusher prior to
2508	* calling this function with non-negative @flush_color. If
2509	* @flush_color is negative, no flush color update is done and %false
2510	* is returned.
2511	*
2512	* If @work_color is non-negative, all pwqs should have the same
2513	* work_color which is previous to @work_color and all will be
2514	* advanced to @work_color.
2515	*
2516	* CONTEXT:
2517	* mutex_lock(wq->mutex).
2518	*
2519	* RETURNS:
2520	* %true if @flush_color >= 0 and there's something to flush. %false
2521	* otherwise.
2522	*/
2523	static bool flush_workqueue_prep_pwqs(struct workqueue_struct *wq,
2524	int flush_color, int work_color)
2525	{
2526	bool wait = false;
2527	struct pool_workqueue *pwq;
2528
2529	if (flush_color >= 0) {
2530	WARN_ON_ONCE(atomic_read(&wq->nr_pwqs_to_flush));
2531	atomic_set(&wq->nr_pwqs_to_flush, 1);
2532	}
2533
2534	for_each_pwq(pwq, wq) {
2535	struct worker_pool *pool = pwq->pool;
2536
2537	spin_lock_irq(&pool->lock);
2538
2539	if (flush_color >= 0) {
2540	WARN_ON_ONCE(pwq->flush_color != -1);
2541
2542	if (pwq->nr_in_flight[flush_color]) {
2543	pwq->flush_color = flush_color;
2544	atomic_inc(&wq->nr_pwqs_to_flush);
2545	wait = true;
2546	}
2547	}
2548
2549	if (work_color >= 0) {
2550	WARN_ON_ONCE(work_color != work_next_color(pwq->work_color));
2551	pwq->work_color = work_color;
2552	}
2553
2554	spin_unlock_irq(&pool->lock);
2555	}
2556
2557	if (flush_color >= 0 && atomic_dec_and_test(&wq->nr_pwqs_to_flush))
2558	complete(&wq->first_flusher->done);
2559
2560	return wait;
2561	}
2562
2563	/**
2564	* flush_workqueue - ensure that any scheduled work has run to completion.
2565	* @wq: workqueue to flush
2566	*
2567	* This function sleeps until all work items which were queued on entry
2568	* have finished execution, but it is not livelocked by new incoming ones.
2569	*/
2570	void flush_workqueue(struct workqueue_struct *wq)
2571	{
2572	struct wq_flusher this_flusher = {
2573	.list = LIST_HEAD_INIT(this_flusher.list),
2574	.flush_color = -1,
2575	.done = COMPLETION_INITIALIZER_ONSTACK(this_flusher.done),
2576	};
2577	int next_color;
2578
2579	lock_map_acquire(&wq->lockdep_map);
2580	lock_map_release(&wq->lockdep_map);
2581
2582	mutex_lock(&wq->mutex);
2583
2584	/*
2585	* Start-to-wait phase
2586	*/
2587	next_color = work_next_color(wq->work_color);
2588
2589	if (next_color != wq->flush_color) {
2590	/*
2591	* Color space is not full. The current work_color
2592	* becomes our flush_color and work_color is advanced
2593	* by one.
2594	*/
2595	WARN_ON_ONCE(!list_empty(&wq->flusher_overflow));
2596	this_flusher.flush_color = wq->work_color;
2597	wq->work_color = next_color;
2598
2599	if (!wq->first_flusher) {
2600	/* no flush in progress, become the first flusher */
2601	WARN_ON_ONCE(wq->flush_color != this_flusher.flush_color);
2602
2603	wq->first_flusher = &this_flusher;
2604
2605	if (!flush_workqueue_prep_pwqs(wq, wq->flush_color,
2606	wq->work_color)) {
2607	/* nothing to flush, done */
2608	wq->flush_color = next_color;
2609	wq->first_flusher = NULL;
2610	goto out_unlock;
2611	}
2612	} else {
2613	/* wait in queue */
2614	WARN_ON_ONCE(wq->flush_color == this_flusher.flush_color);
2615	list_add_tail(&this_flusher.list, &wq->flusher_queue);
2616	flush_workqueue_prep_pwqs(wq, -1, wq->work_color);
2617	}
2618	} else {
2619	/*
2620	* Oops, color space is full, wait on overflow queue.
2621	* The next flush completion will assign us
2622	* flush_color and transfer to flusher_queue.
2623	*/
2624	list_add_tail(&this_flusher.list, &wq->flusher_overflow);
2625	}
2626
2627	mutex_unlock(&wq->mutex);
2628
2629	wait_for_completion(&this_flusher.done);
2630
2631	/*
2632	* Wake-up-and-cascade phase
2633	*
2634	* First flushers are responsible for cascading flushes and
2635	* handling overflow. Non-first flushers can simply return.
2636	*/
2637	if (wq->first_flusher != &this_flusher)
2638	return;
2639
2640	mutex_lock(&wq->mutex);
2641
2642	/* we might have raced, check again with mutex held */
2643	if (wq->first_flusher != &this_flusher)
2644	goto out_unlock;
2645
2646	wq->first_flusher = NULL;
2647
2648	WARN_ON_ONCE(!list_empty(&this_flusher.list));
2649	WARN_ON_ONCE(wq->flush_color != this_flusher.flush_color);
2650
2651	while (true) {
2652	struct wq_flusher next, tmp;
2653
2654	/* complete all the flushers sharing the current flush color */
2655	list_for_each_entry_safe(next, tmp, &wq->flusher_queue, list) {
2656	if (next->flush_color != wq->flush_color)
2657	break;
2658	list_del_init(&next->list);
2659	complete(&next->done);
2660	}
2661
2662	WARN_ON_ONCE(!list_empty(&wq->flusher_overflow) &&
2663	wq->flush_color != work_next_color(wq->work_color));
2664
2665	/* this flush_color is finished, advance by one */
2666	wq->flush_color = work_next_color(wq->flush_color);
2667
2668	/* one color has been freed, handle overflow queue */
2669	if (!list_empty(&wq->flusher_overflow)) {
2670	/*
2671	* Assign the same color to all overflowed
2672	* flushers, advance work_color and append to
2673	* flusher_queue. This is the start-to-wait
2674	* phase for these overflowed flushers.
2675	*/
2676	list_for_each_entry(tmp, &wq->flusher_overflow, list)
2677	tmp->flush_color = wq->work_color;
2678
2679	wq->work_color = work_next_color(wq->work_color);
2680
2681	list_splice_tail_init(&wq->flusher_overflow,
2682	&wq->flusher_queue);
2683	flush_workqueue_prep_pwqs(wq, -1, wq->work_color);
2684	}
2685
2686	if (list_empty(&wq->flusher_queue)) {
2687	WARN_ON_ONCE(wq->flush_color != wq->work_color);
2688	break;
2689	}
2690
2691	/*
2692	* Need to flush more colors. Make the next flusher
2693	* the new first flusher and arm pwqs.
2694	*/
2695	WARN_ON_ONCE(wq->flush_color == wq->work_color);
2696	WARN_ON_ONCE(wq->flush_color != next->flush_color);
2697
2698	list_del_init(&next->list);
2699	wq->first_flusher = next;
2700
2701	if (flush_workqueue_prep_pwqs(wq, wq->flush_color, -1))
2702	break;
2703
2704	/*
2705	* Meh... this color is already done, clear first
2706	* flusher and repeat cascading.
2707	*/
2708	wq->first_flusher = NULL;
2709	}
2710
2711	out_unlock:
2712	mutex_unlock(&wq->mutex);
2713	}
2714	EXPORT_SYMBOL_GPL(flush_workqueue);
2715
2716	/**
2717	* drain_workqueue - drain a workqueue
2718	* @wq: workqueue to drain
2719	*
2720	* Wait until the workqueue becomes empty. While draining is in progress,
2721	* only chain queueing is allowed. IOW, only currently pending or running
2722	* work items on @wq can queue further work items on it. @wq is flushed
2723	* repeatedly until it becomes empty. The number of flushing is detemined
2724	* by the depth of chaining and should be relatively short. Whine if it
2725	* takes too long.
2726	*/
2727	void drain_workqueue(struct workqueue_struct *wq)
2728	{
2729	unsigned int flush_cnt = 0;
2730	struct pool_workqueue *pwq;
2731
2732	/*
2733	* __queue_work() needs to test whether there are drainers, is much
2734	* hotter than drain_workqueue() and already looks at @wq->flags.
2735	* Use __WQ_DRAINING so that queue doesn't have to check nr_drainers.
2736	*/
2737	mutex_lock(&wq->mutex);
2738	if (!wq->nr_drainers++)
2739	wq->flags \|= __WQ_DRAINING;
2740	mutex_unlock(&wq->mutex);
2741	reflush:
2742	flush_workqueue(wq);
2743
2744	mutex_lock(&wq->mutex);
2745
2746	for_each_pwq(pwq, wq) {
2747	bool drained;
2748
2749	spin_lock_irq(&pwq->pool->lock);
2750	drained = !pwq->nr_active && list_empty(&pwq->delayed_works);
2751	spin_unlock_irq(&pwq->pool->lock);
2752
2753	if (drained)
2754	continue;
2755
2756	if (++flush_cnt == 10 \|\|
2757	(flush_cnt % 100 == 0 && flush_cnt <= 1000))
2758	pr_warn("workqueue %s: drain_workqueue() isn't complete after %u tries\n",
2759	wq->name, flush_cnt);
2760
2761	mutex_unlock(&wq->mutex);
2762	goto reflush;
2763	}
2764
2765	if (!--wq->nr_drainers)
2766	wq->flags &= ~__WQ_DRAINING;
2767	mutex_unlock(&wq->mutex);
2768	}
2769	EXPORT_SYMBOL_GPL(drain_workqueue);
2770
2771	static bool start_flush_work(struct work_struct work, struct wq_barrier barr)
2772	{
2773	struct worker *worker = NULL;
2774	struct worker_pool *pool;
2775	struct pool_workqueue *pwq;
2776
2777	might_sleep();
2778
2779	local_irq_disable();
2780	pool = get_work_pool(work);
2781	if (!pool) {
2782	local_irq_enable();
2783	return false;
2784	}
2785
2786	spin_lock(&pool->lock);
2787	/* see the comment in try_to_grab_pending() with the same code */
2788	pwq = get_work_pwq(work);
2789	if (pwq) {
2790	if (unlikely(pwq->pool != pool))
2791	goto already_gone;
2792	} else {
2793	worker = find_worker_executing_work(pool, work);
2794	if (!worker)
2795	goto already_gone;
2796	pwq = worker->current_pwq;
2797	}
2798
2799	insert_wq_barrier(pwq, barr, work, worker);
2800	spin_unlock_irq(&pool->lock);
2801
2802	/*
2803	* If @max_active is 1 or rescuer is in use, flushing another work
2804	* item on the same workqueue may lead to deadlock. Make sure the
2805	* flusher is not running on the same workqueue by verifying write
2806	* access.
2807	*/
2808	if (pwq->wq->saved_max_active == 1 \|\| pwq->wq->rescuer)
2809	lock_map_acquire(&pwq->wq->lockdep_map);
2810	else
2811	lock_map_acquire_read(&pwq->wq->lockdep_map);
2812	lock_map_release(&pwq->wq->lockdep_map);
2813
2814	return true;
2815	already_gone:
2816	spin_unlock_irq(&pool->lock);
2817	return false;
2818	}
2819
2820	/**
2821	* flush_work - wait for a work to finish executing the last queueing instance
2822	* @work: the work to flush
2823	*
2824	* Wait until @work has finished execution. @work is guaranteed to be idle
2825	* on return if it hasn't been requeued since flush started.
2826	*
2827	* RETURNS:
2828	* %true if flush_work() waited for the work to finish execution,
2829	* %false if it was already idle.
2830	*/
2831	bool flush_work(struct work_struct *work)
2832	{
2833	struct wq_barrier barr;
2834
2835	lock_map_acquire(&work->lockdep_map);
2836	lock_map_release(&work->lockdep_map);
2837
2838	if (start_flush_work(work, &barr)) {
2839	wait_for_completion(&barr.done);
2840	destroy_work_on_stack(&barr.work);
2841	return true;
2842	} else {
2843	return false;
2844	}
2845	}
2846	EXPORT_SYMBOL_GPL(flush_work);
2847
2848	static bool __cancel_work_timer(struct work_struct *work, bool is_dwork)
2849	{
2850	unsigned long flags;
2851	int ret;
2852
2853	do {
2854	ret = try_to_grab_pending(work, is_dwork, &flags);
2855	/*
2856	* If someone else is canceling, wait for the same event it
2857	* would be waiting for before retrying.
2858	*/
2859	if (unlikely(ret == -ENOENT))
2860	flush_work(work);
2861	} while (unlikely(ret < 0));
2862
2863	/* tell other tasks trying to grab @work to back off */
2864	mark_work_canceling(work);
2865	local_irq_restore(flags);
2866
2867	flush_work(work);
2868	clear_work_data(work);
2869	return ret;
2870	}
2871
2872	/**
2873	* cancel_work_sync - cancel a work and wait for it to finish
2874	* @work: the work to cancel
2875	*
2876	* Cancel @work and wait for its execution to finish. This function
2877	* can be used even if the work re-queues itself or migrates to
2878	* another workqueue. On return from this function, @work is
2879	* guaranteed to be not pending or executing on any CPU.
2880	*
2881	* cancel_work_sync(&delayed_work->work) must not be used for
2882	* delayed_work's. Use cancel_delayed_work_sync() instead.
2883	*
2884	* The caller must ensure that the workqueue on which @work was last
2885	* queued can't be destroyed before this function returns.
2886	*
2887	* RETURNS:
2888	* %true if @work was pending, %false otherwise.
2889	*/
2890	bool cancel_work_sync(struct work_struct *work)
2891	{
2892	return __cancel_work_timer(work, false);
2893	}
2894	EXPORT_SYMBOL_GPL(cancel_work_sync);
2895
2896	/**
2897	* flush_delayed_work - wait for a dwork to finish executing the last queueing
2898	* @dwork: the delayed work to flush
2899	*
2900	* Delayed timer is cancelled and the pending work is queued for
2901	* immediate execution. Like flush_work(), this function only
2902	* considers the last queueing instance of @dwork.
2903	*
2904	* RETURNS:
2905	* %true if flush_work() waited for the work to finish execution,
2906	* %false if it was already idle.
2907	*/
2908	bool flush_delayed_work(struct delayed_work *dwork)
2909	{
2910	local_irq_disable();
2911	if (del_timer_sync(&dwork->timer))
2912	__queue_work(dwork->cpu, dwork->wq, &dwork->work);
2913	local_irq_enable();
2914	return flush_work(&dwork->work);
2915	}
2916	EXPORT_SYMBOL(flush_delayed_work);
2917
2918	/**
2919	* cancel_delayed_work - cancel a delayed work
2920	* @dwork: delayed_work to cancel
2921	*
2922	* Kill off a pending delayed_work. Returns %true if @dwork was pending
2923	* and canceled; %false if wasn't pending. Note that the work callback
2924	* function may still be running on return, unless it returns %true and the
2925	* work doesn't re-arm itself. Explicitly flush or use
2926	* cancel_delayed_work_sync() to wait on it.
2927	*
2928	* This function is safe to call from any context including IRQ handler.
2929	*/
2930	bool cancel_delayed_work(struct delayed_work *dwork)
2931	{
2932	unsigned long flags;
2933	int ret;
2934
2935	do {
2936	ret = try_to_grab_pending(&dwork->work, true, &flags);
2937	} while (unlikely(ret == -EAGAIN));
2938
2939	if (unlikely(ret < 0))
2940	return false;
2941
2942	set_work_pool_and_clear_pending(&dwork->work,
2943	get_work_pool_id(&dwork->work));
2944	local_irq_restore(flags);
2945	return ret;
2946	}
2947	EXPORT_SYMBOL(cancel_delayed_work);
2948
2949	/**
2950	* cancel_delayed_work_sync - cancel a delayed work and wait for it to finish
2951	* @dwork: the delayed work cancel
2952	*
2953	* This is cancel_work_sync() for delayed works.
2954	*
2955	* RETURNS:
2956	* %true if @dwork was pending, %false otherwise.
2957	*/
2958	bool cancel_delayed_work_sync(struct delayed_work *dwork)
2959	{
2960	return __cancel_work_timer(&dwork->work, true);
2961	}
2962	EXPORT_SYMBOL(cancel_delayed_work_sync);
2963
2964	/**
2965	* schedule_on_each_cpu - execute a function synchronously on each online CPU
2966	* @func: the function to call
2967	*
2968	* schedule_on_each_cpu() executes @func on each online CPU using the
2969	* system workqueue and blocks until all CPUs have completed.
2970	* schedule_on_each_cpu() is very slow.
2971	*
2972	* RETURNS:
2973	* 0 on success, -errno on failure.
2974	*/
2975	int schedule_on_each_cpu(work_func_t func)
2976	{
2977	int cpu;
2978	struct work_struct __percpu *works;
2979
2980	works = alloc_percpu(struct work_struct);
2981	if (!works)
2982	return -ENOMEM;
2983
2984	get_online_cpus();
2985
2986	for_each_online_cpu(cpu) {
2987	struct work_struct *work = per_cpu_ptr(works, cpu);
2988
2989	INIT_WORK(work, func);
2990	schedule_work_on(cpu, work);
2991	}
2992
2993	for_each_online_cpu(cpu)
2994	flush_work(per_cpu_ptr(works, cpu));
2995
2996	put_online_cpus();
2997	free_percpu(works);
2998	return 0;
2999	}
3000
3001	/**
3002	* flush_scheduled_work - ensure that any scheduled work has run to completion.
3003	*
3004	* Forces execution of the kernel-global workqueue and blocks until its
3005	* completion.
3006	*
3007	* Think twice before calling this function! It's very easy to get into
3008	* trouble if you don't take great care. Either of the following situations
3009	* will lead to deadlock:
3010	*
3011	* One of the work items currently on the workqueue needs to acquire
3012	* a lock held by your code or its caller.
3013	*
3014	* Your code is running in the context of a work routine.
3015	*
3016	* They will be detected by lockdep when they occur, but the first might not
3017	* occur very often. It depends on what work items are on the workqueue and
3018	* what locks they need, which you have no control over.
3019	*
3020	* In most situations flushing the entire workqueue is overkill; you merely
3021	* need to know that a particular work item isn't queued and isn't running.
3022	* In such cases you should use cancel_delayed_work_sync() or
3023	* cancel_work_sync() instead.
3024	*/
3025	void flush_scheduled_work(void)
3026	{
3027	flush_workqueue(system_wq);
3028	}
3029	EXPORT_SYMBOL(flush_scheduled_work);
3030
3031	/**
3032	* execute_in_process_context - reliably execute the routine with user context
3033	* @fn: the function to execute
3034	* @ew: guaranteed storage for the execute work structure (must
3035	* be available when the work executes)
3036	*
3037	* Executes the function immediately if process context is available,
3038	* otherwise schedules the function for delayed execution.
3039	*
3040	* Returns: 0 - function was executed
3041	* 1 - function was scheduled for execution
3042	*/
3043	int execute_in_process_context(work_func_t fn, struct execute_work *ew)
3044	{
3045	if (!in_interrupt()) {
3046	fn(&ew->work);
3047	return 0;
3048	}
3049
3050	INIT_WORK(&ew->work, fn);
3051	schedule_work(&ew->work);
3052
3053	return 1;
3054	}
3055	EXPORT_SYMBOL_GPL(execute_in_process_context);
3056
3057	#ifdef CONFIG_SYSFS
3058	/*
3059	* Workqueues with WQ_SYSFS flag set is visible to userland via
3060	* /sys/bus/workqueue/devices/WQ_NAME. All visible workqueues have the
3061	* following attributes.
3062	*
3063	* per_cpu RO bool : whether the workqueue is per-cpu or unbound
3064	* max_active RW int : maximum number of in-flight work items
3065	*
3066	* Unbound workqueues have the following extra attributes.
3067	*
3068	* id RO int : the associated pool ID
3069	* nice RW int : nice value of the workers
3070	* cpumask RW mask : bitmask of allowed CPUs for the workers
3071	*/
3072	struct wq_device {
3073	struct workqueue_struct *wq;
3074	struct device dev;
3075	};
3076
3077	static struct workqueue_struct dev_to_wq(struct device dev)
3078	{
3079	struct wq_device *wq_dev = container_of(dev, struct wq_device, dev);
3080
3081	return wq_dev->wq;
3082	}
3083
3084	static ssize_t wq_per_cpu_show(struct device *dev,
3085	struct device_attribute attr, char buf)
3086	{
3087	struct workqueue_struct *wq = dev_to_wq(dev);
3088
3089	return scnprintf(buf, PAGE_SIZE, "%d\n", (bool)!(wq->flags & WQ_UNBOUND));
3090	}
3091
3092	static ssize_t wq_max_active_show(struct device *dev,
3093	struct device_attribute attr, char buf)
3094	{
3095	struct workqueue_struct *wq = dev_to_wq(dev);
3096
3097	return scnprintf(buf, PAGE_SIZE, "%d\n", wq->saved_max_active);
3098	}
3099
3100	static ssize_t wq_max_active_store(struct device *dev,
3101	struct device_attribute *attr,
3102	const char *buf, size_t count)
3103	{
3104	struct workqueue_struct *wq = dev_to_wq(dev);
3105	int val;
3106
3107	if (sscanf(buf, "%d", &val) != 1 \|\| val <= 0)
3108	return -EINVAL;
3109
3110	workqueue_set_max_active(wq, val);
3111	return count;
3112	}
3113
3114	static struct device_attribute wq_sysfs_attrs[] = {
3115	__ATTR(per_cpu, 0444, wq_per_cpu_show, NULL),
3116	__ATTR(max_active, 0644, wq_max_active_show, wq_max_active_store),
3117	__ATTR_NULL,
3118	};
3119
3120	static ssize_t wq_pool_ids_show(struct device *dev,
3121	struct device_attribute attr, char buf)
3122	{
3123	struct workqueue_struct *wq = dev_to_wq(dev);
3124	const char *delim = "";
3125	int node, written = 0;
3126
3127	rcu_read_lock_sched();
3128	for_each_node(node) {
3129	written += scnprintf(buf + written, PAGE_SIZE - written,
3130	"%s%d:%d", delim, node,
3131	unbound_pwq_by_node(wq, node)->pool->id);
3132	delim = " ";
3133	}
3134	written += scnprintf(buf + written, PAGE_SIZE - written, "\n");
3135	rcu_read_unlock_sched();
3136
3137	return written;
3138	}
3139
3140	static ssize_t wq_nice_show(struct device dev, struct device_attribute attr,
3141	char *buf)
3142	{
3143	struct workqueue_struct *wq = dev_to_wq(dev);
3144	int written;
3145
3146	mutex_lock(&wq->mutex);
3147	written = scnprintf(buf, PAGE_SIZE, "%d\n", wq->unbound_attrs->nice);
3148	mutex_unlock(&wq->mutex);
3149
3150	return written;
3151	}
3152
3153	/* prepare workqueue_attrs for sysfs store operations */
3154	static struct workqueue_attrs wq_sysfs_prep_attrs(struct workqueue_struct wq)
3155	{
3156	struct workqueue_attrs *attrs;
3157
3158	attrs = alloc_workqueue_attrs(GFP_KERNEL);
3159	if (!attrs)
3160	return NULL;
3161
3162	mutex_lock(&wq->mutex);
3163	copy_workqueue_attrs(attrs, wq->unbound_attrs);
3164	mutex_unlock(&wq->mutex);
3165	return attrs;
3166	}
3167
3168	static ssize_t wq_nice_store(struct device dev, struct device_attribute attr,
3169	const char *buf, size_t count)
3170	{
3171	struct workqueue_struct *wq = dev_to_wq(dev);
3172	struct workqueue_attrs *attrs;
3173	int ret;
3174
3175	attrs = wq_sysfs_prep_attrs(wq);
3176	if (!attrs)
3177	return -ENOMEM;
3178
3179	if (sscanf(buf, "%d", &attrs->nice) == 1 &&
3180	attrs->nice >= -20 && attrs->nice <= 19)
3181	ret = apply_workqueue_attrs(wq, attrs);
3182	else
3183	ret = -EINVAL;
3184
3185	free_workqueue_attrs(attrs);
3186	return ret ?: count;
3187	}
3188
3189	static ssize_t wq_cpumask_show(struct device *dev,
3190	struct device_attribute attr, char buf)
3191	{
3192	struct workqueue_struct *wq = dev_to_wq(dev);
3193	int written;
3194
3195	mutex_lock(&wq->mutex);
3196	written = cpumask_scnprintf(buf, PAGE_SIZE, wq->unbound_attrs->cpumask);
3197	mutex_unlock(&wq->mutex);
3198
3199	written += scnprintf(buf + written, PAGE_SIZE - written, "\n");
3200	return written;
3201	}
3202
3203	static ssize_t wq_cpumask_store(struct device *dev,
3204	struct device_attribute *attr,
3205	const char *buf, size_t count)
3206	{
3207	struct workqueue_struct *wq = dev_to_wq(dev);
3208	struct workqueue_attrs *attrs;
3209	int ret;
3210
3211	attrs = wq_sysfs_prep_attrs(wq);
3212	if (!attrs)
3213	return -ENOMEM;
3214
3215	ret = cpumask_parse(buf, attrs->cpumask);
3216	if (!ret)
3217	ret = apply_workqueue_attrs(wq, attrs);
3218
3219	free_workqueue_attrs(attrs);
3220	return ret ?: count;
3221	}
3222
3223	static ssize_t wq_numa_show(struct device dev, struct device_attribute attr,
3224	char *buf)
3225	{
3226	struct workqueue_struct *wq = dev_to_wq(dev);
3227	int written;
3228
3229	mutex_lock(&wq->mutex);
3230	written = scnprintf(buf, PAGE_SIZE, "%d\n",
3231	!wq->unbound_attrs->no_numa);
3232	mutex_unlock(&wq->mutex);
3233
3234	return written;
3235	}
3236
3237	static ssize_t wq_numa_store(struct device dev, struct device_attribute attr,
3238	const char *buf, size_t count)
3239	{
3240	struct workqueue_struct *wq = dev_to_wq(dev);
3241	struct workqueue_attrs *attrs;
3242	int v, ret;
3243
3244	attrs = wq_sysfs_prep_attrs(wq);
3245	if (!attrs)
3246	return -ENOMEM;
3247
3248	ret = -EINVAL;
3249	if (sscanf(buf, "%d", &v) == 1) {
3250	attrs->no_numa = !v;
3251	ret = apply_workqueue_attrs(wq, attrs);
3252	}
3253
3254	free_workqueue_attrs(attrs);
3255	return ret ?: count;
3256	}
3257
3258	static struct device_attribute wq_sysfs_unbound_attrs[] = {
3259	__ATTR(pool_ids, 0444, wq_pool_ids_show, NULL),
3260	__ATTR(nice, 0644, wq_nice_show, wq_nice_store),
3261	__ATTR(cpumask, 0644, wq_cpumask_show, wq_cpumask_store),
3262	__ATTR(numa, 0644, wq_numa_show, wq_numa_store),
3263	__ATTR_NULL,
3264	};
3265
3266	static struct bus_type wq_subsys = {
3267	.name = "workqueue",
3268	.dev_attrs = wq_sysfs_attrs,
3269	};
3270
3271	static int __init wq_sysfs_init(void)
3272	{
3273	return subsys_virtual_register(&wq_subsys, NULL);
3274	}
3275	core_initcall(wq_sysfs_init);
3276
3277	static void wq_device_release(struct device *dev)
3278	{
3279	struct wq_device *wq_dev = container_of(dev, struct wq_device, dev);
3280
3281	kfree(wq_dev);
3282	}
3283
3284	/**
3285	* workqueue_sysfs_register - make a workqueue visible in sysfs
3286	* @wq: the workqueue to register
3287	*
3288	* Expose @wq in sysfs under /sys/bus/workqueue/devices.
3289	* alloc_workqueue*() automatically calls this function if WQ_SYSFS is set
3290	* which is the preferred method.
3291	*
3292	* Workqueue user should use this function directly iff it wants to apply
3293	* workqueue_attrs before making the workqueue visible in sysfs; otherwise,
3294	* apply_workqueue_attrs() may race against userland updating the
3295	* attributes.
3296	*
3297	* Returns 0 on success, -errno on failure.
3298	*/
3299	int workqueue_sysfs_register(struct workqueue_struct *wq)
3300	{
3301	struct wq_device *wq_dev;
3302	int ret;
3303
3304	/*
3305	* Adjusting max_active or creating new pwqs by applyting
3306	* attributes breaks ordering guarantee. Disallow exposing ordered
3307	* workqueues.
3308	*/
3309	if (WARN_ON(wq->flags & __WQ_ORDERED))
3310	return -EINVAL;
3311
3312	wq->wq_dev = wq_dev = kzalloc(sizeof(*wq_dev), GFP_KERNEL);
3313	if (!wq_dev)
3314	return -ENOMEM;
3315
3316	wq_dev->wq = wq;
3317	wq_dev->dev.bus = &wq_subsys;
3318	wq_dev->dev.init_name = wq->name;
3319	wq_dev->dev.release = wq_device_release;
3320
3321	/*
3322	* unbound_attrs are created separately. Suppress uevent until
3323	* everything is ready.
3324	*/
3325	dev_set_uevent_suppress(&wq_dev->dev, true);
3326
3327	ret = device_register(&wq_dev->dev);
3328	if (ret) {
3329	kfree(wq_dev);
3330	wq->wq_dev = NULL;
3331	return ret;
3332	}
3333
3334	if (wq->flags & WQ_UNBOUND) {
3335	struct device_attribute *attr;
3336
3337	for (attr = wq_sysfs_unbound_attrs; attr->attr.name; attr++) {
3338	ret = device_create_file(&wq_dev->dev, attr);
3339	if (ret) {
3340	device_unregister(&wq_dev->dev);
3341	wq->wq_dev = NULL;
3342	return ret;
3343	}
3344	}
3345	}
3346
3347	kobject_uevent(&wq_dev->dev.kobj, KOBJ_ADD);
3348	return 0;
3349	}
3350
3351	/**
3352	* workqueue_sysfs_unregister - undo workqueue_sysfs_register()
3353	* @wq: the workqueue to unregister
3354	*
3355	* If @wq is registered to sysfs by workqueue_sysfs_register(), unregister.
3356	*/
3357	static void workqueue_sysfs_unregister(struct workqueue_struct *wq)
3358	{
3359	struct wq_device *wq_dev = wq->wq_dev;
3360
3361	if (!wq->wq_dev)
3362	return;
3363
3364	wq->wq_dev = NULL;
3365	device_unregister(&wq_dev->dev);
3366	}
3367	#else /* CONFIG_SYSFS */
3368	static void workqueue_sysfs_unregister(struct workqueue_struct *wq) { }
3369	#endif /* CONFIG_SYSFS */
3370
3371	/**
3372	* free_workqueue_attrs - free a workqueue_attrs
3373	* @attrs: workqueue_attrs to free
3374	*
3375	* Undo alloc_workqueue_attrs().
3376	*/
3377	void free_workqueue_attrs(struct workqueue_attrs *attrs)
3378	{
3379	if (attrs) {
3380	free_cpumask_var(attrs->cpumask);
3381	kfree(attrs);
3382	}
3383	}
3384
3385	/**
3386	* alloc_workqueue_attrs - allocate a workqueue_attrs
3387	* @gfp_mask: allocation mask to use
3388	*
3389	* Allocate a new workqueue_attrs, initialize with default settings and
3390	* return it. Returns NULL on failure.
3391	*/
3392	struct workqueue_attrs *alloc_workqueue_attrs(gfp_t gfp_mask)
3393	{
3394	struct workqueue_attrs *attrs;
3395
3396	attrs = kzalloc(sizeof(*attrs), gfp_mask);
3397	if (!attrs)
3398	goto fail;
3399	if (!alloc_cpumask_var(&attrs->cpumask, gfp_mask))
3400	goto fail;
3401
3402	cpumask_copy(attrs->cpumask, cpu_possible_mask);
3403	return attrs;
3404	fail:
3405	free_workqueue_attrs(attrs);
3406	return NULL;
3407	}
3408
3409	static void copy_workqueue_attrs(struct workqueue_attrs *to,
3410	const struct workqueue_attrs *from)
3411	{
3412	to->nice = from->nice;
3413	cpumask_copy(to->cpumask, from->cpumask);
3414	}
3415
3416	/* hash value of the content of @attr */
3417	static u32 wqattrs_hash(const struct workqueue_attrs *attrs)
3418	{
3419	u32 hash = 0;
3420
3421	hash = jhash_1word(attrs->nice, hash);
3422	hash = jhash(cpumask_bits(attrs->cpumask),
3423	BITS_TO_LONGS(nr_cpumask_bits) * sizeof(long), hash);
3424	return hash;
3425	}
3426
3427	/* content equality test */
3428	static bool wqattrs_equal(const struct workqueue_attrs *a,
3429	const struct workqueue_attrs *b)
3430	{
3431	if (a->nice != b->nice)
3432	return false;
3433	if (!cpumask_equal(a->cpumask, b->cpumask))
3434	return false;
3435	return true;
3436	}
3437
3438	/**
3439	* init_worker_pool - initialize a newly zalloc'd worker_pool
3440	* @pool: worker_pool to initialize
3441	*
3442	* Initiailize a newly zalloc'd @pool. It also allocates @pool->attrs.
3443	* Returns 0 on success, -errno on failure. Even on failure, all fields
3444	* inside @pool proper are initialized and put_unbound_pool() can be called
3445	* on @pool safely to release it.
3446	*/
3447	static int init_worker_pool(struct worker_pool *pool)
3448	{
3449	spin_lock_init(&pool->lock);
3450	pool->id = -1;
3451	pool->cpu = -1;
3452	pool->node = NUMA_NO_NODE;
3453	pool->flags \|= POOL_DISASSOCIATED;
3454	INIT_LIST_HEAD(&pool->worklist);
3455	INIT_LIST_HEAD(&pool->idle_list);
3456	hash_init(pool->busy_hash);
3457
3458	init_timer_deferrable(&pool->idle_timer);
3459	pool->idle_timer.function = idle_worker_timeout;
3460	pool->idle_timer.data = (unsigned long)pool;
3461
3462	setup_timer(&pool->mayday_timer, pool_mayday_timeout,
3463	(unsigned long)pool);
3464
3465	mutex_init(&pool->manager_arb);
3466	mutex_init(&pool->manager_mutex);
3467	idr_init(&pool->worker_idr);
3468
3469	INIT_HLIST_NODE(&pool->hash_node);
3470	pool->refcnt = 1;
3471
3472	/* shouldn't fail above this point */
3473	pool->attrs = alloc_workqueue_attrs(GFP_KERNEL);
3474	if (!pool->attrs)
3475	return -ENOMEM;
3476	return 0;
3477	}
3478
3479	static void rcu_free_pool(struct rcu_head *rcu)
3480	{
3481	struct worker_pool *pool = container_of(rcu, struct worker_pool, rcu);
3482
3483	idr_destroy(&pool->worker_idr);
3484	free_workqueue_attrs(pool->attrs);
3485	kfree(pool);
3486	}
3487
3488	/**
3489	* put_unbound_pool - put a worker_pool
3490	* @pool: worker_pool to put
3491	*
3492	* Put @pool. If its refcnt reaches zero, it gets destroyed in sched-RCU
3493	* safe manner. get_unbound_pool() calls this function on its failure path
3494	* and this function should be able to release pools which went through,
3495	* successfully or not, init_worker_pool().
3496	*
3497	* Should be called with wq_pool_mutex held.
3498	*/
3499	static void put_unbound_pool(struct worker_pool *pool)
3500	{
3501	struct worker *worker;
3502
3503	lockdep_assert_held(&wq_pool_mutex);
3504
3505	if (--pool->refcnt)
3506	return;
3507
3508	/* sanity checks */
3509	if (WARN_ON(!(pool->flags & POOL_DISASSOCIATED)) \|\|
3510	WARN_ON(!list_empty(&pool->worklist)))
3511	return;
3512
3513	/* release id and unhash */
3514	if (pool->id >= 0)
3515	idr_remove(&worker_pool_idr, pool->id);
3516	hash_del(&pool->hash_node);
3517
3518	/*
3519	* Become the manager and destroy all workers. Grabbing
3520	* manager_arb prevents @pool's workers from blocking on
3521	* manager_mutex.
3522	*/
3523	mutex_lock(&pool->manager_arb);
3524	mutex_lock(&pool->manager_mutex);
3525	spin_lock_irq(&pool->lock);
3526
3527	while ((worker = first_worker(pool)))
3528	destroy_worker(worker);
3529	WARN_ON(pool->nr_workers \|\| pool->nr_idle);
3530
3531	spin_unlock_irq(&pool->lock);
3532	mutex_unlock(&pool->manager_mutex);
3533	mutex_unlock(&pool->manager_arb);
3534
3535	/* shut down the timers */
3536	del_timer_sync(&pool->idle_timer);
3537	del_timer_sync(&pool->mayday_timer);
3538
3539	/* sched-RCU protected to allow dereferences from get_work_pool() */
3540	call_rcu_sched(&pool->rcu, rcu_free_pool);
3541	}
3542
3543	/**
3544	* get_unbound_pool - get a worker_pool with the specified attributes
3545	* @attrs: the attributes of the worker_pool to get
3546	*
3547	* Obtain a worker_pool which has the same attributes as @attrs, bump the
3548	* reference count and return it. If there already is a matching
3549	* worker_pool, it will be used; otherwise, this function attempts to
3550	* create a new one. On failure, returns NULL.
3551	*
3552	* Should be called with wq_pool_mutex held.
3553	*/
3554	static struct worker_pool get_unbound_pool(const struct workqueue_attrs attrs)
3555	{
3556	u32 hash = wqattrs_hash(attrs);
3557	struct worker_pool *pool;
3558	int node;
3559
3560	lockdep_assert_held(&wq_pool_mutex);
3561
3562	/* do we already have a matching pool? */
3563	hash_for_each_possible(unbound_pool_hash, pool, hash_node, hash) {
3564	if (wqattrs_equal(pool->attrs, attrs)) {
3565	pool->refcnt++;
3566	goto out_unlock;
3567	}
3568	}
3569
3570	/* nope, create a new one */
3571	pool = kzalloc(sizeof(*pool), GFP_KERNEL);
3572	if (!pool \|\| init_worker_pool(pool) < 0)
3573	goto fail;
3574
3575	if (workqueue_freezing)
3576	pool->flags \|= POOL_FREEZING;
3577
3578	lockdep_set_subclass(&pool->lock, 1); /* see put_pwq() */
3579	copy_workqueue_attrs(pool->attrs, attrs);
3580
3581	/* if cpumask is contained inside a NUMA node, we belong to that node */
3582	if (wq_numa_enabled) {
3583	for_each_node(node) {
3584	if (cpumask_subset(pool->attrs->cpumask,
3585	wq_numa_possible_cpumask[node])) {
3586	pool->node = node;
3587	break;
3588	}
3589	}
3590	}
3591
3592	if (worker_pool_assign_id(pool) < 0)
3593	goto fail;
3594
3595	/* create and start the initial worker */
3596	if (create_and_start_worker(pool) < 0)
3597	goto fail;
3598
3599	/* install */
3600	hash_add(unbound_pool_hash, &pool->hash_node, hash);
3601	out_unlock:
3602	return pool;
3603	fail:
3604	if (pool)
3605	put_unbound_pool(pool);
3606	return NULL;
3607	}
3608
3609	static void rcu_free_pwq(struct rcu_head *rcu)
3610	{
3611	kmem_cache_free(pwq_cache,
3612	container_of(rcu, struct pool_workqueue, rcu));
3613	}
3614
3615	/*
3616	* Scheduled on system_wq by put_pwq() when an unbound pwq hits zero refcnt
3617	* and needs to be destroyed.
3618	*/
3619	static void pwq_unbound_release_workfn(struct work_struct *work)
3620	{
3621	struct pool_workqueue *pwq = container_of(work, struct pool_workqueue,
3622	unbound_release_work);
3623	struct workqueue_struct *wq = pwq->wq;
3624	struct worker_pool *pool = pwq->pool;
3625	bool is_last;
3626
3627	if (WARN_ON_ONCE(!(wq->flags & WQ_UNBOUND)))
3628	return;
3629
3630	/*
3631	* Unlink @pwq. Synchronization against wq->mutex isn't strictly
3632	* necessary on release but do it anyway. It's easier to verify
3633	* and consistent with the linking path.
3634	*/
3635	mutex_lock(&wq->mutex);
3636	list_del_rcu(&pwq->pwqs_node);
3637	is_last = list_empty(&wq->pwqs);
3638	mutex_unlock(&wq->mutex);
3639
3640	mutex_lock(&wq_pool_mutex);
3641	put_unbound_pool(pool);
3642	mutex_unlock(&wq_pool_mutex);
3643
3644	call_rcu_sched(&pwq->rcu, rcu_free_pwq);
3645
3646	/*
3647	* If we're the last pwq going away, @wq is already dead and no one
3648	* is gonna access it anymore. Free it.
3649	*/
3650	if (is_last) {
3651	free_workqueue_attrs(wq->unbound_attrs);
3652	kfree(wq);
3653	}
3654	}
3655
3656	/**
3657	* pwq_adjust_max_active - update a pwq's max_active to the current setting
3658	* @pwq: target pool_workqueue
3659	*
3660	* If @pwq isn't freezing, set @pwq->max_active to the associated
3661	* workqueue's saved_max_active and activate delayed work items
3662	* accordingly. If @pwq is freezing, clear @pwq->max_active to zero.
3663	*/
3664	static void pwq_adjust_max_active(struct pool_workqueue *pwq)
3665	{
3666	struct workqueue_struct *wq = pwq->wq;
3667	bool freezable = wq->flags & WQ_FREEZABLE;
3668
3669	/* for @wq->saved_max_active */
3670	lockdep_assert_held(&wq->mutex);
3671
3672	/* fast exit for non-freezable wqs */
3673	if (!freezable && pwq->max_active == wq->saved_max_active)
3674	return;
3675
3676	spin_lock_irq(&pwq->pool->lock);
3677
3678	if (!freezable \|\| !(pwq->pool->flags & POOL_FREEZING)) {
3679	pwq->max_active = wq->saved_max_active;
3680
3681	while (!list_empty(&pwq->delayed_works) &&
3682	pwq->nr_active < pwq->max_active)
3683	pwq_activate_first_delayed(pwq);
3684
3685	/*
3686	* Need to kick a worker after thawed or an unbound wq's
3687	* max_active is bumped. It's a slow path. Do it always.
3688	*/
3689	wake_up_worker(pwq->pool);
3690	} else {
3691	pwq->max_active = 0;
3692	}
3693
3694	spin_unlock_irq(&pwq->pool->lock);
3695	}
3696
3697	/* initialize newly alloced @pwq which is associated with @wq and @pool */
3698	static void init_pwq(struct pool_workqueue pwq, struct workqueue_struct wq,
3699	struct worker_pool *pool)
3700	{
3701	BUG_ON((unsigned long)pwq & WORK_STRUCT_FLAG_MASK);
3702
3703	memset(pwq, 0, sizeof(*pwq));
3704
3705	pwq->pool = pool;
3706	pwq->wq = wq;
3707	pwq->flush_color = -1;
3708	pwq->refcnt = 1;
3709	INIT_LIST_HEAD(&pwq->delayed_works);
3710	INIT_LIST_HEAD(&pwq->pwqs_node);
3711	INIT_LIST_HEAD(&pwq->mayday_node);
3712	INIT_WORK(&pwq->unbound_release_work, pwq_unbound_release_workfn);
3713	}
3714
3715	/* sync @pwq with the current state of its associated wq and link it */
3716	static void link_pwq(struct pool_workqueue *pwq)
3717	{
3718	struct workqueue_struct *wq = pwq->wq;
3719
3720	lockdep_assert_held(&wq->mutex);
3721
3722	/* may be called multiple times, ignore if already linked */
3723	if (!list_empty(&pwq->pwqs_node))
3724	return;
3725
3726	/*
3727	* Set the matching work_color. This is synchronized with
3728	* wq->mutex to avoid confusing flush_workqueue().
3729	*/
3730	pwq->work_color = wq->work_color;
3731
3732	/* sync max_active to the current setting */
3733	pwq_adjust_max_active(pwq);
3734
3735	/* link in @pwq */
3736	list_add_rcu(&pwq->pwqs_node, &wq->pwqs);
3737	}
3738
3739	/* obtain a pool matching @attr and create a pwq associating the pool and @wq */
3740	static struct pool_workqueue alloc_unbound_pwq(struct workqueue_struct wq,
3741	const struct workqueue_attrs *attrs)
3742	{
3743	struct worker_pool *pool;
3744	struct pool_workqueue *pwq;
3745
3746	lockdep_assert_held(&wq_pool_mutex);
3747
3748	pool = get_unbound_pool(attrs);
3749	if (!pool)
3750	return NULL;
3751
3752	pwq = kmem_cache_alloc_node(pwq_cache, GFP_KERNEL, pool->node);
3753	if (!pwq) {
3754	put_unbound_pool(pool);
3755	return NULL;
3756	}
3757
3758	init_pwq(pwq, wq, pool);
3759	return pwq;
3760	}
3761
3762	/* undo alloc_unbound_pwq(), used only in the error path */
3763	static void free_unbound_pwq(struct pool_workqueue *pwq)
3764	{
3765	lockdep_assert_held(&wq_pool_mutex);
3766
3767	if (pwq) {
3768	put_unbound_pool(pwq->pool);
3769	kmem_cache_free(pwq_cache, pwq);
3770	}
3771	}
3772
3773	/**
3774	* wq_calc_node_mask - calculate a wq_attrs' cpumask for the specified node
3775	* @attrs: the wq_attrs of interest
3776	* @node: the target NUMA node
3777	* @cpu_going_down: if >= 0, the CPU to consider as offline
3778	* @cpumask: outarg, the resulting cpumask
3779	*
3780	* Calculate the cpumask a workqueue with @attrs should use on @node. If
3781	* @cpu_going_down is >= 0, that cpu is considered offline during
3782	* calculation. The result is stored in @cpumask. This function returns
3783	* %true if the resulting @cpumask is different from @attrs->cpumask,
3784	* %false if equal.
3785	*
3786	* If NUMA affinity is not enabled, @attrs->cpumask is always used. If
3787	* enabled and @node has online CPUs requested by @attrs, the returned
3788	* cpumask is the intersection of the possible CPUs of @node and
3789	* @attrs->cpumask.
3790	*
3791	* The caller is responsible for ensuring that the cpumask of @node stays
3792	* stable.
3793	*/
3794	static bool wq_calc_node_cpumask(const struct workqueue_attrs *attrs, int node,
3795	int cpu_going_down, cpumask_t *cpumask)
3796	{
3797	if (!wq_numa_enabled \|\| attrs->no_numa)
3798	goto use_dfl;
3799
3800	/* does @node have any online CPUs @attrs wants? */
3801	cpumask_and(cpumask, cpumask_of_node(node), attrs->cpumask);
3802	if (cpu_going_down >= 0)
3803	cpumask_clear_cpu(cpu_going_down, cpumask);
3804
3805	if (cpumask_empty(cpumask))
3806	goto use_dfl;
3807
3808	/* yeap, return possible CPUs in @node that @attrs wants */
3809	cpumask_and(cpumask, attrs->cpumask, wq_numa_possible_cpumask[node]);
3810	return !cpumask_equal(cpumask, attrs->cpumask);
3811
3812	use_dfl:
3813	cpumask_copy(cpumask, attrs->cpumask);
3814	return false;
3815	}
3816
3817	/* install @pwq into @wq's numa_pwq_tbl[] for @node and return the old pwq */
3818	static struct pool_workqueue numa_pwq_tbl_install(struct workqueue_struct wq,
3819	int node,
3820	struct pool_workqueue *pwq)
3821	{
3822	struct pool_workqueue *old_pwq;
3823
3824	lockdep_assert_held(&wq->mutex);
3825
3826	/* link_pwq() can handle duplicate calls */
3827	link_pwq(pwq);
3828
3829	old_pwq = rcu_access_pointer(wq->numa_pwq_tbl[node]);
3830	rcu_assign_pointer(wq->numa_pwq_tbl[node], pwq);
3831	return old_pwq;
3832	}
3833
3834	/**
3835	* apply_workqueue_attrs - apply new workqueue_attrs to an unbound workqueue
3836	* @wq: the target workqueue
3837	* @attrs: the workqueue_attrs to apply, allocated with alloc_workqueue_attrs()
3838	*
3839	* Apply @attrs to an unbound workqueue @wq. Unless disabled, on NUMA
3840	* machines, this function maps a separate pwq to each NUMA node with
3841	* possibles CPUs in @attrs->cpumask so that work items are affine to the
3842	* NUMA node it was issued on. Older pwqs are released as in-flight work
3843	* items finish. Note that a work item which repeatedly requeues itself
3844	* back-to-back will stay on its current pwq.
3845	*
3846	* Performs GFP_KERNEL allocations. Returns 0 on success and -errno on
3847	* failure.
3848	*/
3849	int apply_workqueue_attrs(struct workqueue_struct *wq,
3850	const struct workqueue_attrs *attrs)
3851	{
3852	struct workqueue_attrs new_attrs, tmp_attrs;
3853	struct pool_workqueue *pwq_tbl, dfl_pwq;
3854	int node, ret;
3855
3856	/* only unbound workqueues can change attributes */
3857	if (WARN_ON(!(wq->flags & WQ_UNBOUND)))
3858	return -EINVAL;
3859
3860	/* creating multiple pwqs breaks ordering guarantee */
3861	if (WARN_ON((wq->flags & __WQ_ORDERED) && !list_empty(&wq->pwqs)))
3862	return -EINVAL;
3863
3864	pwq_tbl = kzalloc(wq_numa_tbl_len * sizeof(pwq_tbl[0]), GFP_KERNEL);
3865	new_attrs = alloc_workqueue_attrs(GFP_KERNEL);
3866	tmp_attrs = alloc_workqueue_attrs(GFP_KERNEL);
3867	if (!pwq_tbl \|\| !new_attrs \|\| !tmp_attrs)
3868	goto enomem;
3869
3870	/* make a copy of @attrs and sanitize it */
3871	copy_workqueue_attrs(new_attrs, attrs);
3872	cpumask_and(new_attrs->cpumask, new_attrs->cpumask, cpu_possible_mask);
3873
3874	/*
3875	* We may create multiple pwqs with differing cpumasks. Make a
3876	* copy of @new_attrs which will be modified and used to obtain
3877	* pools.
3878	*/
3879	copy_workqueue_attrs(tmp_attrs, new_attrs);
3880
3881	/*
3882	* CPUs should stay stable across pwq creations and installations.
3883	* Pin CPUs, determine the target cpumask for each node and create
3884	* pwqs accordingly.
3885	*/
3886	get_online_cpus();
3887
3888	mutex_lock(&wq_pool_mutex);
3889
3890	/*
3891	* If something goes wrong during CPU up/down, we'll fall back to
3892	* the default pwq covering whole @attrs->cpumask. Always create
3893	* it even if we don't use it immediately.
3894	*/
3895	dfl_pwq = alloc_unbound_pwq(wq, new_attrs);
3896	if (!dfl_pwq)
3897	goto enomem_pwq;
3898
3899	for_each_node(node) {
3900	if (wq_calc_node_cpumask(attrs, node, -1, tmp_attrs->cpumask)) {
3901	pwq_tbl[node] = alloc_unbound_pwq(wq, tmp_attrs);
3902	if (!pwq_tbl[node])
3903	goto enomem_pwq;
3904	} else {
3905	dfl_pwq->refcnt++;
3906	pwq_tbl[node] = dfl_pwq;
3907	}
3908	}
3909
3910	mutex_unlock(&wq_pool_mutex);
3911
3912	/* all pwqs have been created successfully, let's install'em */
3913	mutex_lock(&wq->mutex);
3914
3915	copy_workqueue_attrs(wq->unbound_attrs, new_attrs);
3916
3917	/* save the previous pwq and install the new one */
3918	for_each_node(node)
3919	pwq_tbl[node] = numa_pwq_tbl_install(wq, node, pwq_tbl[node]);
3920
3921	/* @dfl_pwq might not have been used, ensure it's linked */
3922	link_pwq(dfl_pwq);
3923	swap(wq->dfl_pwq, dfl_pwq);
3924
3925	mutex_unlock(&wq->mutex);
3926
3927	/* put the old pwqs */
3928	for_each_node(node)
3929	put_pwq_unlocked(pwq_tbl[node]);
3930	put_pwq_unlocked(dfl_pwq);
3931
3932	put_online_cpus();
3933	ret = 0;
3934	/* fall through */
3935	out_free:
3936	free_workqueue_attrs(tmp_attrs);
3937	free_workqueue_attrs(new_attrs);
3938	kfree(pwq_tbl);
3939	return ret;
3940
3941	enomem_pwq:
3942	free_unbound_pwq(dfl_pwq);
3943	for_each_node(node)
3944	if (pwq_tbl && pwq_tbl[node] != dfl_pwq)
3945	free_unbound_pwq(pwq_tbl[node]);
3946	mutex_unlock(&wq_pool_mutex);
3947	put_online_cpus();
3948	enomem:
3949	ret = -ENOMEM;
3950	goto out_free;
3951	}
3952
3953	/**
3954	* wq_update_unbound_numa - update NUMA affinity of a wq for CPU hot[un]plug
3955	* @wq: the target workqueue
3956	* @cpu: the CPU coming up or going down
3957	* @online: whether @cpu is coming up or going down
3958	*
3959	* This function is to be called from %CPU_DOWN_PREPARE, %CPU_ONLINE and
3960	* %CPU_DOWN_FAILED. @cpu is being hot[un]plugged, update NUMA affinity of
3961	* @wq accordingly.
3962	*
3963	* If NUMA affinity can't be adjusted due to memory allocation failure, it
3964	* falls back to @wq->dfl_pwq which may not be optimal but is always
3965	* correct.
3966	*
3967	* Note that when the last allowed CPU of a NUMA node goes offline for a
3968	* workqueue with a cpumask spanning multiple nodes, the workers which were
3969	* already executing the work items for the workqueue will lose their CPU
3970	* affinity and may execute on any CPU. This is similar to how per-cpu
3971	* workqueues behave on CPU_DOWN. If a workqueue user wants strict
3972	* affinity, it's the user's responsibility to flush the work item from
3973	* CPU_DOWN_PREPARE.
3974	*/
3975	static void wq_update_unbound_numa(struct workqueue_struct *wq, int cpu,
3976	bool online)
3977	{
3978	int node = cpu_to_node(cpu);
3979	int cpu_off = online ? -1 : cpu;
3980	struct pool_workqueue old_pwq = NULL, pwq;
3981	struct workqueue_attrs *target_attrs;
3982	cpumask_t *cpumask;
3983
3984	lockdep_assert_held(&wq_pool_mutex);
3985
3986	if (!wq_numa_enabled \|\| !(wq->flags & WQ_UNBOUND))
3987	return;
3988
3989	/*
3990	* We don't wanna alloc/free wq_attrs for each wq for each CPU.
3991	* Let's use a preallocated one. The following buf is protected by
3992	* CPU hotplug exclusion.
3993	*/
3994	target_attrs = wq_update_unbound_numa_attrs_buf;
3995	cpumask = target_attrs->cpumask;
3996
3997	mutex_lock(&wq->mutex);
3998	if (wq->unbound_attrs->no_numa)
3999	goto out_unlock;
4000
4001	copy_workqueue_attrs(target_attrs, wq->unbound_attrs);
4002	pwq = unbound_pwq_by_node(wq, node);
4003
4004	/*
4005	* Let's determine what needs to be done. If the target cpumask is
4006	* different from wq's, we need to compare it to @pwq's and create
4007	* a new one if they don't match. If the target cpumask equals
4008	* wq's, the default pwq should be used. If @pwq is already the
4009	* default one, nothing to do; otherwise, install the default one.
4010	*/
4011	if (wq_calc_node_cpumask(wq->unbound_attrs, node, cpu_off, cpumask)) {
4012	if (cpumask_equal(cpumask, pwq->pool->attrs->cpumask))
4013	goto out_unlock;
4014	} else {
4015	if (pwq == wq->dfl_pwq)
4016	goto out_unlock;
4017	else
4018	goto use_dfl_pwq;
4019	}
4020
4021	mutex_unlock(&wq->mutex);
4022
4023	/* create a new pwq */
4024	pwq = alloc_unbound_pwq(wq, target_attrs);
4025	if (!pwq) {
4026	pr_warning("workqueue: allocation failed while updating NUMA affinity of \"%s\"\n",
4027	wq->name);
4028	goto out_unlock;
4029	}
4030
4031	/*
4032	* Install the new pwq. As this function is called only from CPU
4033	* hotplug callbacks and applying a new attrs is wrapped with
4034	* get/put_online_cpus(), @wq->unbound_attrs couldn't have changed
4035	* inbetween.
4036	*/
4037	mutex_lock(&wq->mutex);
4038	old_pwq = numa_pwq_tbl_install(wq, node, pwq);
4039	goto out_unlock;
4040
4041	use_dfl_pwq:
4042	spin_lock_irq(&wq->dfl_pwq->pool->lock);
4043	get_pwq(wq->dfl_pwq);
4044	spin_unlock_irq(&wq->dfl_pwq->pool->lock);
4045	old_pwq = numa_pwq_tbl_install(wq, node, wq->dfl_pwq);
4046	out_unlock:
4047	mutex_unlock(&wq->mutex);
4048	put_pwq_unlocked(old_pwq);
4049	}
4050
4051	static int alloc_and_link_pwqs(struct workqueue_struct *wq)
4052	{
4053	bool highpri = wq->flags & WQ_HIGHPRI;
4054	int cpu;
4055
4056	if (!(wq->flags & WQ_UNBOUND)) {
4057	wq->cpu_pwqs = alloc_percpu(struct pool_workqueue);
4058	if (!wq->cpu_pwqs)
4059	return -ENOMEM;
4060
4061	for_each_possible_cpu(cpu) {
4062	struct pool_workqueue *pwq =
4063	per_cpu_ptr(wq->cpu_pwqs, cpu);
4064	struct worker_pool *cpu_pools =
4065	per_cpu(cpu_worker_pools, cpu);
4066
4067	init_pwq(pwq, wq, &cpu_pools[highpri]);
4068
4069	mutex_lock(&wq->mutex);
4070	link_pwq(pwq);
4071	mutex_unlock(&wq->mutex);
4072	}
4073	return 0;
4074	} else {
4075	return apply_workqueue_attrs(wq, unbound_std_wq_attrs[highpri]);
4076	}
4077	}
4078
4079	static int wq_clamp_max_active(int max_active, unsigned int flags,
4080	const char *name)
4081	{
4082	int lim = flags & WQ_UNBOUND ? WQ_UNBOUND_MAX_ACTIVE : WQ_MAX_ACTIVE;
4083
4084	if (max_active < 1 \|\| max_active > lim)
4085	pr_warn("workqueue: max_active %d requested for %s is out of range, clamping between %d and %d\n",
4086	max_active, name, 1, lim);
4087
4088	return clamp_val(max_active, 1, lim);
4089	}
4090
4091	struct workqueue_struct __alloc_workqueue_key(const char fmt,
4092	unsigned int flags,
4093	int max_active,
4094	struct lock_class_key *key,
4095	const char *lock_name, ...)
4096	{
4097	size_t tbl_size = 0;
4098	va_list args;
4099	struct workqueue_struct *wq;
4100	struct pool_workqueue *pwq;
4101
4102	/* see the comment above the definition of WQ_POWER_EFFICIENT */
4103	if ((flags & WQ_POWER_EFFICIENT) && wq_power_efficient)
4104	flags \|= WQ_UNBOUND;
4105
4106	/* allocate wq and format name */
4107	if (flags & WQ_UNBOUND)
4108	tbl_size = wq_numa_tbl_len * sizeof(wq->numa_pwq_tbl[0]);
4109
4110	wq = kzalloc(sizeof(*wq) + tbl_size, GFP_KERNEL);
4111	if (!wq)
4112	return NULL;
4113
4114	if (flags & WQ_UNBOUND) {
4115	wq->unbound_attrs = alloc_workqueue_attrs(GFP_KERNEL);
4116	if (!wq->unbound_attrs)
4117	goto err_free_wq;
4118	}
4119
4120	va_start(args, lock_name);
4121	vsnprintf(wq->name, sizeof(wq->name), fmt, args);
4122	va_end(args);
4123
4124	max_active = max_active ?: WQ_DFL_ACTIVE;
4125	max_active = wq_clamp_max_active(max_active, flags, wq->name);
4126
4127	/* init wq */
4128	wq->flags = flags;
4129	wq->saved_max_active = max_active;
4130	mutex_init(&wq->mutex);
4131	atomic_set(&wq->nr_pwqs_to_flush, 0);
4132	INIT_LIST_HEAD(&wq->pwqs);
4133	INIT_LIST_HEAD(&wq->flusher_queue);
4134	INIT_LIST_HEAD(&wq->flusher_overflow);
4135	INIT_LIST_HEAD(&wq->maydays);
4136
4137	lockdep_init_map(&wq->lockdep_map, lock_name, key, 0);
4138	INIT_LIST_HEAD(&wq->list);
4139
4140	if (alloc_and_link_pwqs(wq) < 0)
4141	goto err_free_wq;
4142
4143	/*
4144	* Workqueues which may be used during memory reclaim should
4145	* have a rescuer to guarantee forward progress.
4146	*/
4147	if (flags & WQ_MEM_RECLAIM) {
4148	struct worker *rescuer;
4149
4150	rescuer = alloc_worker();
4151	if (!rescuer)
4152	goto err_destroy;
4153
4154	rescuer->rescue_wq = wq;
4155	rescuer->task = kthread_create(rescuer_thread, rescuer, "%s",
4156	wq->name);
4157	if (IS_ERR(rescuer->task)) {
4158	kfree(rescuer);
4159	goto err_destroy;
4160	}
4161
4162	wq->rescuer = rescuer;
4163	rescuer->task->flags \|= PF_NO_SETAFFINITY;
4164	wake_up_process(rescuer->task);
4165	}
4166
4167	if ((wq->flags & WQ_SYSFS) && workqueue_sysfs_register(wq))
4168	goto err_destroy;
4169
4170	/*
4171	* wq_pool_mutex protects global freeze state and workqueues list.
4172	* Grab it, adjust max_active and add the new @wq to workqueues
4173	* list.
4174	*/
4175	mutex_lock(&wq_pool_mutex);
4176
4177	mutex_lock(&wq->mutex);
4178	for_each_pwq(pwq, wq)
4179	pwq_adjust_max_active(pwq);
4180	mutex_unlock(&wq->mutex);
4181
4182	list_add(&wq->list, &workqueues);
4183
4184	mutex_unlock(&wq_pool_mutex);
4185
4186	return wq;
4187
4188	err_free_wq:
4189	free_workqueue_attrs(wq->unbound_attrs);
4190	kfree(wq);
4191	return NULL;
4192	err_destroy:
4193	destroy_workqueue(wq);
4194	return NULL;
4195	}
4196	EXPORT_SYMBOL_GPL(__alloc_workqueue_key);
4197
4198	/**
4199	* destroy_workqueue - safely terminate a workqueue
4200	* @wq: target workqueue
4201	*
4202	* Safely destroy a workqueue. All work currently pending will be done first.
4203	*/
4204	void destroy_workqueue(struct workqueue_struct *wq)
4205	{
4206	struct pool_workqueue *pwq;
4207	int node;
4208
4209	/* drain it before proceeding with destruction */
4210	drain_workqueue(wq);
4211
4212	/* sanity checks */
4213	mutex_lock(&wq->mutex);
4214	for_each_pwq(pwq, wq) {
4215	int i;
4216
4217	for (i = 0; i < WORK_NR_COLORS; i++) {
4218	if (WARN_ON(pwq->nr_in_flight[i])) {
4219	mutex_unlock(&wq->mutex);
4220	return;
4221	}
4222	}
4223
4224	if (WARN_ON((pwq != wq->dfl_pwq) && (pwq->refcnt > 1)) \|\|
4225	WARN_ON(pwq->nr_active) \|\|
4226	WARN_ON(!list_empty(&pwq->delayed_works))) {
4227	mutex_unlock(&wq->mutex);
4228	return;
4229	}
4230	}
4231	mutex_unlock(&wq->mutex);
4232
4233	/*
4234	* wq list is used to freeze wq, remove from list after
4235	* flushing is complete in case freeze races us.
4236	*/
4237	mutex_lock(&wq_pool_mutex);
4238	list_del_init(&wq->list);
4239	mutex_unlock(&wq_pool_mutex);
4240
4241	workqueue_sysfs_unregister(wq);
4242
4243	if (wq->rescuer) {
4244	kthread_stop(wq->rescuer->task);
4245	kfree(wq->rescuer);
4246	wq->rescuer = NULL;
4247	}
4248
4249	if (!(wq->flags & WQ_UNBOUND)) {
4250	/*
4251	* The base ref is never dropped on per-cpu pwqs. Directly
4252	* free the pwqs and wq.
4253	*/
4254	free_percpu(wq->cpu_pwqs);
4255	kfree(wq);
4256	} else {
4257	/*
4258	* We're the sole accessor of @wq at this point. Directly
4259	* access numa_pwq_tbl[] and dfl_pwq to put the base refs.
4260	* @wq will be freed when the last pwq is released.
4261	*/
4262	for_each_node(node) {
4263	pwq = rcu_access_pointer(wq->numa_pwq_tbl[node]);
4264	RCU_INIT_POINTER(wq->numa_pwq_tbl[node], NULL);
4265	put_pwq_unlocked(pwq);
4266	}
4267
4268	/*
4269	* Put dfl_pwq. @wq may be freed any time after dfl_pwq is
4270	* put. Don't access it afterwards.
4271	*/
4272	pwq = wq->dfl_pwq;
4273	wq->dfl_pwq = NULL;
4274	put_pwq_unlocked(pwq);
4275	}
4276	}
4277	EXPORT_SYMBOL_GPL(destroy_workqueue);
4278
4279	/**
4280	* workqueue_set_max_active - adjust max_active of a workqueue
4281	* @wq: target workqueue
4282	* @max_active: new max_active value.
4283	*
4284	* Set max_active of @wq to @max_active.
4285	*
4286	* CONTEXT:
4287	* Don't call from IRQ context.
4288	*/
4289	void workqueue_set_max_active(struct workqueue_struct *wq, int max_active)
4290	{
4291	struct pool_workqueue *pwq;
4292
4293	/* disallow meddling with max_active for ordered workqueues */
4294	if (WARN_ON(wq->flags & __WQ_ORDERED))
4295	return;
4296
4297	max_active = wq_clamp_max_active(max_active, wq->flags, wq->name);
4298
4299	mutex_lock(&wq->mutex);
4300
4301	wq->saved_max_active = max_active;
4302
4303	for_each_pwq(pwq, wq)
4304	pwq_adjust_max_active(pwq);
4305
4306	mutex_unlock(&wq->mutex);
4307	}
4308	EXPORT_SYMBOL_GPL(workqueue_set_max_active);
4309
4310	/**
4311	* current_is_workqueue_rescuer - is %current workqueue rescuer?
4312	*
4313	* Determine whether %current is a workqueue rescuer. Can be used from
4314	* work functions to determine whether it's being run off the rescuer task.
4315	*/
4316	bool current_is_workqueue_rescuer(void)
4317	{
4318	struct worker *worker = current_wq_worker();
4319
4320	return worker && worker->rescue_wq;
4321	}
4322
4323	/**
4324	* workqueue_congested - test whether a workqueue is congested
4325	* @cpu: CPU in question
4326	* @wq: target workqueue
4327	*
4328	* Test whether @wq's cpu workqueue for @cpu is congested. There is
4329	* no synchronization around this function and the test result is
4330	* unreliable and only useful as advisory hints or for debugging.
4331	*
4332	* If @cpu is WORK_CPU_UNBOUND, the test is performed on the local CPU.
4333	* Note that both per-cpu and unbound workqueues may be associated with
4334	* multiple pool_workqueues which have separate congested states. A
4335	* workqueue being congested on one CPU doesn't mean the workqueue is also
4336	* contested on other CPUs / NUMA nodes.
4337	*
4338	* RETURNS:
4339	* %true if congested, %false otherwise.
4340	*/
4341	bool workqueue_congested(int cpu, struct workqueue_struct *wq)
4342	{
4343	struct pool_workqueue *pwq;
4344	bool ret;
4345
4346	rcu_read_lock_sched();
4347
4348	if (cpu == WORK_CPU_UNBOUND)
4349	cpu = smp_processor_id();
4350
4351	if (!(wq->flags & WQ_UNBOUND))
4352	pwq = per_cpu_ptr(wq->cpu_pwqs, cpu);
4353	else
4354	pwq = unbound_pwq_by_node(wq, cpu_to_node(cpu));
4355
4356	ret = !list_empty(&pwq->delayed_works);
4357	rcu_read_unlock_sched();
4358
4359	return ret;
4360	}
4361	EXPORT_SYMBOL_GPL(workqueue_congested);
4362
4363	/**
4364	* work_busy - test whether a work is currently pending or running
4365	* @work: the work to be tested
4366	*
4367	* Test whether @work is currently pending or running. There is no
4368	* synchronization around this function and the test result is
4369	* unreliable and only useful as advisory hints or for debugging.
4370	*
4371	* RETURNS:
4372	* OR'd bitmask of WORK_BUSY_* bits.
4373	*/
4374	unsigned int work_busy(struct work_struct *work)
4375	{
4376	struct worker_pool *pool;
4377	unsigned long flags;
4378	unsigned int ret = 0;
4379
4380	if (work_pending(work))
4381	ret \|= WORK_BUSY_PENDING;
4382
4383	local_irq_save(flags);
4384	pool = get_work_pool(work);
4385	if (pool) {
4386	spin_lock(&pool->lock);
4387	if (find_worker_executing_work(pool, work))
4388	ret \|= WORK_BUSY_RUNNING;
4389	spin_unlock(&pool->lock);
4390	}
4391	local_irq_restore(flags);
4392
4393	return ret;
4394	}
4395	EXPORT_SYMBOL_GPL(work_busy);
4396
4397	/**
4398	* set_worker_desc - set description for the current work item
4399	* @fmt: printf-style format string
4400	* @...: arguments for the format string
4401	*
4402	* This function can be called by a running work function to describe what
4403	* the work item is about. If the worker task gets dumped, this
4404	* information will be printed out together to help debugging. The
4405	* description can be at most WORKER_DESC_LEN including the trailing '\0'.
4406	*/
4407	void set_worker_desc(const char *fmt, ...)
4408	{
4409	struct worker *worker = current_wq_worker();
4410	va_list args;
4411
4412	if (worker) {
4413	va_start(args, fmt);
4414	vsnprintf(worker->desc, sizeof(worker->desc), fmt, args);
4415	va_end(args);
4416	worker->desc_valid = true;
4417	}
4418	}
4419
4420	/**
4421	* print_worker_info - print out worker information and description
4422	* @log_lvl: the log level to use when printing
4423	* @task: target task
4424	*
4425	* If @task is a worker and currently executing a work item, print out the
4426	* name of the workqueue being serviced and worker description set with
4427	* set_worker_desc() by the currently executing work item.
4428	*
4429	* This function can be safely called on any task as long as the
4430	* task_struct itself is accessible. While safe, this function isn't
4431	* synchronized and may print out mixups or garbages of limited length.
4432	*/
4433	void print_worker_info(const char log_lvl, struct task_struct task)
4434	{
4435	work_func_t *fn = NULL;
4436	char name[WQ_NAME_LEN] = { };
4437	char desc[WORKER_DESC_LEN] = { };
4438	struct pool_workqueue *pwq = NULL;
4439	struct workqueue_struct *wq = NULL;
4440	bool desc_valid = false;
4441	struct worker *worker;
4442
4443	if (!(task->flags & PF_WQ_WORKER))
4444	return;
4445
4446	/*
4447	* This function is called without any synchronization and @task
4448	* could be in any state. Be careful with dereferences.
4449	*/
4450	worker = probe_kthread_data(task);
4451
4452	/*
4453	* Carefully copy the associated workqueue's workfn and name. Keep
4454	* the original last '\0' in case the original contains garbage.
4455	*/
4456	probe_kernel_read(&fn, &worker->current_func, sizeof(fn));
4457	probe_kernel_read(&pwq, &worker->current_pwq, sizeof(pwq));
4458	probe_kernel_read(&wq, &pwq->wq, sizeof(wq));
4459	probe_kernel_read(name, wq->name, sizeof(name) - 1);
4460
4461	/* copy worker description */
4462	probe_kernel_read(&desc_valid, &worker->desc_valid, sizeof(desc_valid));
4463	if (desc_valid)
4464	probe_kernel_read(desc, worker->desc, sizeof(desc) - 1);
4465
4466	if (fn \|\| name[0] \|\| desc[0]) {
4467	printk("%sWorkqueue: %s %pf", log_lvl, name, fn);
4468	if (desc[0])
4469	pr_cont(" (%s)", desc);
4470	pr_cont("\n");
4471	}
4472	}
4473
4474	/*
4475	* CPU hotplug.
4476	*
4477	* There are two challenges in supporting CPU hotplug. Firstly, there
4478	* are a lot of assumptions on strong associations among work, pwq and
4479	* pool which make migrating pending and scheduled works very
4480	* difficult to implement without impacting hot paths. Secondly,
4481	* worker pools serve mix of short, long and very long running works making
4482	* blocked draining impractical.
4483	*
4484	* This is solved by allowing the pools to be disassociated from the CPU
4485	* running as an unbound one and allowing it to be reattached later if the
4486	* cpu comes back online.
4487	*/
4488
4489	static void wq_unbind_fn(struct work_struct *work)
4490	{
4491	int cpu = smp_processor_id();
4492	struct worker_pool *pool;
4493	struct worker *worker;
4494	int wi;
4495
4496	for_each_cpu_worker_pool(pool, cpu) {
4497	WARN_ON_ONCE(cpu != smp_processor_id());
4498
4499	mutex_lock(&pool->manager_mutex);
4500	spin_lock_irq(&pool->lock);
4501
4502	/*
4503	* We've blocked all manager operations. Make all workers
4504	* unbound and set DISASSOCIATED. Before this, all workers
4505	* except for the ones which are still executing works from
4506	* before the last CPU down must be on the cpu. After
4507	* this, they may become diasporas.
4508	*/
4509	for_each_pool_worker(worker, wi, pool)
4510	worker->flags \|= WORKER_UNBOUND;
4511
4512	pool->flags \|= POOL_DISASSOCIATED;
4513
4514	spin_unlock_irq(&pool->lock);
4515	mutex_unlock(&pool->manager_mutex);
4516
4517	/*
4518	* Call schedule() so that we cross rq->lock and thus can
4519	* guarantee sched callbacks see the %WORKER_UNBOUND flag.
4520	* This is necessary as scheduler callbacks may be invoked
4521	* from other cpus.
4522	*/
4523	schedule();
4524
4525	/*
4526	* Sched callbacks are disabled now. Zap nr_running.
4527	* After this, nr_running stays zero and need_more_worker()
4528	* and keep_working() are always true as long as the
4529	* worklist is not empty. This pool now behaves as an
4530	* unbound (in terms of concurrency management) pool which
4531	* are served by workers tied to the pool.
4532	*/
4533	atomic_set(&pool->nr_running, 0);
4534
4535	/*
4536	* With concurrency management just turned off, a busy
4537	* worker blocking could lead to lengthy stalls. Kick off
4538	* unbound chain execution of currently pending work items.
4539	*/
4540	spin_lock_irq(&pool->lock);
4541	wake_up_worker(pool);
4542	spin_unlock_irq(&pool->lock);
4543	}
4544	}
4545
4546	/**
4547	* rebind_workers - rebind all workers of a pool to the associated CPU
4548	* @pool: pool of interest
4549	*
4550	* @pool->cpu is coming online. Rebind all workers to the CPU.
4551	*/
4552	static void rebind_workers(struct worker_pool *pool)
4553	{
4554	struct worker *worker;
4555	int wi;
4556
4557	lockdep_assert_held(&pool->manager_mutex);
4558
4559	/*
4560	* Restore CPU affinity of all workers. As all idle workers should
4561	* be on the run-queue of the associated CPU before any local
4562	* wake-ups for concurrency management happen, restore CPU affinty
4563	* of all workers first and then clear UNBOUND. As we're called
4564	* from CPU_ONLINE, the following shouldn't fail.
4565	*/
4566	for_each_pool_worker(worker, wi, pool)
4567	WARN_ON_ONCE(set_cpus_allowed_ptr(worker->task,
4568	pool->attrs->cpumask) < 0);
4569
4570	spin_lock_irq(&pool->lock);
4571
4572	for_each_pool_worker(worker, wi, pool) {
4573	unsigned int worker_flags = worker->flags;
4574
4575	/*
4576	* A bound idle worker should actually be on the runqueue
4577	* of the associated CPU for local wake-ups targeting it to
4578	* work. Kick all idle workers so that they migrate to the
4579	* associated CPU. Doing this in the same loop as
4580	* replacing UNBOUND with REBOUND is safe as no worker will
4581	* be bound before @pool->lock is released.
4582	*/
4583	if (worker_flags & WORKER_IDLE)
4584	wake_up_process(worker->task);
4585
4586	/*
4587	* We want to clear UNBOUND but can't directly call
4588	* worker_clr_flags() or adjust nr_running. Atomically
4589	* replace UNBOUND with another NOT_RUNNING flag REBOUND.
4590	* @worker will clear REBOUND using worker_clr_flags() when
4591	* it initiates the next execution cycle thus restoring
4592	* concurrency management. Note that when or whether
4593	* @worker clears REBOUND doesn't affect correctness.
4594	*
4595	* ACCESS_ONCE() is necessary because @worker->flags may be
4596	* tested without holding any lock in
4597	* wq_worker_waking_up(). Without it, NOT_RUNNING test may
4598	* fail incorrectly leading to premature concurrency
4599	* management operations.
4600	*/
4601	WARN_ON_ONCE(!(worker_flags & WORKER_UNBOUND));
4602	worker_flags \|= WORKER_REBOUND;
4603	worker_flags &= ~WORKER_UNBOUND;
4604	ACCESS_ONCE(worker->flags) = worker_flags;
4605	}
4606
4607	spin_unlock_irq(&pool->lock);
4608	}
4609
4610	/**
4611	* restore_unbound_workers_cpumask - restore cpumask of unbound workers
4612	* @pool: unbound pool of interest
4613	* @cpu: the CPU which is coming up
4614	*
4615	* An unbound pool may end up with a cpumask which doesn't have any online
4616	* CPUs. When a worker of such pool get scheduled, the scheduler resets
4617	* its cpus_allowed. If @cpu is in @pool's cpumask which didn't have any
4618	* online CPU before, cpus_allowed of all its workers should be restored.
4619	*/
4620	static void restore_unbound_workers_cpumask(struct worker_pool *pool, int cpu)
4621	{
4622	static cpumask_t cpumask;
4623	struct worker *worker;
4624	int wi;
4625
4626	lockdep_assert_held(&pool->manager_mutex);
4627
4628	/* is @cpu allowed for @pool? */
4629	if (!cpumask_test_cpu(cpu, pool->attrs->cpumask))
4630	return;
4631
4632	/* is @cpu the only online CPU? */
4633	cpumask_and(&cpumask, pool->attrs->cpumask, cpu_online_mask);
4634	if (cpumask_weight(&cpumask) != 1)
4635	return;
4636
4637	/* as we're called from CPU_ONLINE, the following shouldn't fail */
4638	for_each_pool_worker(worker, wi, pool)
4639	WARN_ON_ONCE(set_cpus_allowed_ptr(worker->task,
4640	pool->attrs->cpumask) < 0);
4641	}
4642
4643	/*
4644	* Workqueues should be brought up before normal priority CPU notifiers.
4645	* This will be registered high priority CPU notifier.
4646	*/
4647	static int workqueue_cpu_up_callback(struct notifier_block *nfb,
4648	unsigned long action,
4649	void *hcpu)
4650	{
4651	int cpu = (unsigned long)hcpu;
4652	struct worker_pool *pool;
4653	struct workqueue_struct *wq;
4654	int pi;
4655
4656	switch (action & ~CPU_TASKS_FROZEN) {
4657	case CPU_UP_PREPARE:
4658	for_each_cpu_worker_pool(pool, cpu) {
4659	if (pool->nr_workers)
4660	continue;
4661	if (create_and_start_worker(pool) < 0)
4662	return NOTIFY_BAD;
4663	}
4664	break;
4665
4666	case CPU_DOWN_FAILED:
4667	case CPU_ONLINE:
4668	mutex_lock(&wq_pool_mutex);
4669
4670	for_each_pool(pool, pi) {
4671	mutex_lock(&pool->manager_mutex);
4672
4673	if (pool->cpu == cpu) {
4674	spin_lock_irq(&pool->lock);
4675	pool->flags &= ~POOL_DISASSOCIATED;
4676	spin_unlock_irq(&pool->lock);
4677
4678	rebind_workers(pool);
4679	} else if (pool->cpu < 0) {
4680	restore_unbound_workers_cpumask(pool, cpu);
4681	}
4682
4683	mutex_unlock(&pool->manager_mutex);
4684	}
4685
4686	/* update NUMA affinity of unbound workqueues */
4687	list_for_each_entry(wq, &workqueues, list)
4688	wq_update_unbound_numa(wq, cpu, true);
4689
4690	mutex_unlock(&wq_pool_mutex);
4691	break;
4692	}
4693	return NOTIFY_OK;
4694	}
4695
4696	/*
4697	* Workqueues should be brought down after normal priority CPU notifiers.
4698	* This will be registered as low priority CPU notifier.
4699	*/
4700	static int workqueue_cpu_down_callback(struct notifier_block *nfb,
4701	unsigned long action,
4702	void *hcpu)
4703	{
4704	int cpu = (unsigned long)hcpu;
4705	struct work_struct unbind_work;
4706	struct workqueue_struct *wq;
4707
4708	switch (action & ~CPU_TASKS_FROZEN) {
4709	case CPU_DOWN_PREPARE:
4710	/* unbinding per-cpu workers should happen on the local CPU */
4711	INIT_WORK_ONSTACK(&unbind_work, wq_unbind_fn);
4712	queue_work_on(cpu, system_highpri_wq, &unbind_work);
4713
4714	/* update NUMA affinity of unbound workqueues */
4715	mutex_lock(&wq_pool_mutex);
4716	list_for_each_entry(wq, &workqueues, list)
4717	wq_update_unbound_numa(wq, cpu, false);
4718	mutex_unlock(&wq_pool_mutex);
4719
4720	/* wait for per-cpu unbinding to finish */
4721	flush_work(&unbind_work);
4722	break;
4723	}
4724	return NOTIFY_OK;
4725	}
4726
4727	#ifdef CONFIG_SMP
4728
4729	struct work_for_cpu {
4730	struct work_struct work;
4731	long (fn)(void );
4732	void *arg;
4733	long ret;
4734	};
4735
4736	static void work_for_cpu_fn(struct work_struct *work)
4737	{
4738	struct work_for_cpu *wfc = container_of(work, struct work_for_cpu, work);
4739
4740	wfc->ret = wfc->fn(wfc->arg);
4741	}
4742
4743	/**
4744	* work_on_cpu - run a function in user context on a particular cpu
4745	* @cpu: the cpu to run on
4746	* @fn: the function to run
4747	* @arg: the function arg
4748	*
4749	* This will return the value @fn returns.
4750	* It is up to the caller to ensure that the cpu doesn't go offline.
4751	* The caller must not hold any locks which would prevent @fn from completing.
4752	*/
4753	long work_on_cpu(int cpu, long (fn)(void ), void *arg)
4754	{
4755	struct work_for_cpu wfc = { .fn = fn, .arg = arg };
4756
4757	INIT_WORK_ONSTACK(&wfc.work, work_for_cpu_fn);
4758	schedule_work_on(cpu, &wfc.work);
4759	flush_work(&wfc.work);
4760	return wfc.ret;
4761	}
4762	EXPORT_SYMBOL_GPL(work_on_cpu);
4763	#endif /* CONFIG_SMP */
4764
4765	#ifdef CONFIG_FREEZER
4766
4767	/**
4768	* freeze_workqueues_begin - begin freezing workqueues
4769	*
4770	* Start freezing workqueues. After this function returns, all freezable
4771	* workqueues will queue new works to their delayed_works list instead of
4772	* pool->worklist.
4773	*
4774	* CONTEXT:
4775	* Grabs and releases wq_pool_mutex, wq->mutex and pool->lock's.
4776	*/
4777	void freeze_workqueues_begin(void)
4778	{
4779	struct worker_pool *pool;
4780	struct workqueue_struct *wq;
4781	struct pool_workqueue *pwq;
4782	int pi;
4783
4784	mutex_lock(&wq_pool_mutex);
4785
4786	WARN_ON_ONCE(workqueue_freezing);
4787	workqueue_freezing = true;
4788
4789	/* set FREEZING */
4790	for_each_pool(pool, pi) {
4791	spin_lock_irq(&pool->lock);
4792	WARN_ON_ONCE(pool->flags & POOL_FREEZING);
4793	pool->flags \|= POOL_FREEZING;
4794	spin_unlock_irq(&pool->lock);
4795	}
4796
4797	list_for_each_entry(wq, &workqueues, list) {
4798	mutex_lock(&wq->mutex);
4799	for_each_pwq(pwq, wq)
4800	pwq_adjust_max_active(pwq);
4801	mutex_unlock(&wq->mutex);
4802	}
4803
4804	mutex_unlock(&wq_pool_mutex);
4805	}
4806
4807	/**
4808	* freeze_workqueues_busy - are freezable workqueues still busy?
4809	*
4810	* Check whether freezing is complete. This function must be called
4811	* between freeze_workqueues_begin() and thaw_workqueues().
4812	*
4813	* CONTEXT:
4814	* Grabs and releases wq_pool_mutex.
4815	*
4816	* RETURNS:
4817	* %true if some freezable workqueues are still busy. %false if freezing
4818	* is complete.
4819	*/
4820	bool freeze_workqueues_busy(void)
4821	{
4822	bool busy = false;
4823	struct workqueue_struct *wq;
4824	struct pool_workqueue *pwq;
4825
4826	mutex_lock(&wq_pool_mutex);
4827
4828	WARN_ON_ONCE(!workqueue_freezing);
4829
4830	list_for_each_entry(wq, &workqueues, list) {
4831	if (!(wq->flags & WQ_FREEZABLE))
4832	continue;
4833	/*
4834	* nr_active is monotonically decreasing. It's safe
4835	* to peek without lock.
4836	*/
4837	rcu_read_lock_sched();
4838	for_each_pwq(pwq, wq) {
4839	WARN_ON_ONCE(pwq->nr_active < 0);
4840	if (pwq->nr_active) {
4841	busy = true;
4842	rcu_read_unlock_sched();
4843	goto out_unlock;
4844	}
4845	}
4846	rcu_read_unlock_sched();
4847	}
4848	out_unlock:
4849	mutex_unlock(&wq_pool_mutex);
4850	return busy;
4851	}
4852
4853	/**
4854	* thaw_workqueues - thaw workqueues
4855	*
4856	* Thaw workqueues. Normal queueing is restored and all collected
4857	* frozen works are transferred to their respective pool worklists.
4858	*
4859	* CONTEXT:
4860	* Grabs and releases wq_pool_mutex, wq->mutex and pool->lock's.
4861	*/
4862	void thaw_workqueues(void)
4863	{
4864	struct workqueue_struct *wq;
4865	struct pool_workqueue *pwq;
4866	struct worker_pool *pool;
4867	int pi;
4868
4869	mutex_lock(&wq_pool_mutex);
4870
4871	if (!workqueue_freezing)
4872	goto out_unlock;
4873
4874	/* clear FREEZING */
4875	for_each_pool(pool, pi) {
4876	spin_lock_irq(&pool->lock);
4877	WARN_ON_ONCE(!(pool->flags & POOL_FREEZING));
4878	pool->flags &= ~POOL_FREEZING;
4879	spin_unlock_irq(&pool->lock);
4880	}
4881
4882	/* restore max_active and repopulate worklist */
4883	list_for_each_entry(wq, &workqueues, list) {
4884	mutex_lock(&wq->mutex);
4885	for_each_pwq(pwq, wq)
4886	pwq_adjust_max_active(pwq);
4887	mutex_unlock(&wq->mutex);
4888	}
4889
4890	workqueue_freezing = false;
4891	out_unlock:
4892	mutex_unlock(&wq_pool_mutex);
4893	}
4894	#endif /* CONFIG_FREEZER */
4895
4896	static void __init wq_numa_init(void)
4897	{
4898	cpumask_var_t *tbl;
4899	int node, cpu;
4900
4901	/* determine NUMA pwq table len - highest node id + 1 */
4902	for_each_node(node)
4903	wq_numa_tbl_len = max(wq_numa_tbl_len, node + 1);
4904
4905	if (num_possible_nodes() <= 1)
4906	return;
4907
4908	if (wq_disable_numa) {
4909	pr_info("workqueue: NUMA affinity support disabled\n");
4910	return;
4911	}
4912
4913	wq_update_unbound_numa_attrs_buf = alloc_workqueue_attrs(GFP_KERNEL);
4914	BUG_ON(!wq_update_unbound_numa_attrs_buf);
4915
4916	/*
4917	* We want masks of possible CPUs of each node which isn't readily
4918	* available. Build one from cpu_to_node() which should have been
4919	* fully initialized by now.
4920	*/
4921	tbl = kzalloc(wq_numa_tbl_len * sizeof(tbl[0]), GFP_KERNEL);
4922	BUG_ON(!tbl);
4923
4924	for_each_node(node)
4925	BUG_ON(!alloc_cpumask_var_node(&tbl[node], GFP_KERNEL,
4926	node_online(node) ? node : NUMA_NO_NODE));
4927
4928	for_each_possible_cpu(cpu) {
4929	node = cpu_to_node(cpu);
4930	if (WARN_ON(node == NUMA_NO_NODE)) {
4931	pr_warn("workqueue: NUMA node mapping not available for cpu%d, disabling NUMA support\n", cpu);
4932	/* happens iff arch is bonkers, let's just proceed */
4933	return;
4934	}
4935	cpumask_set_cpu(cpu, tbl[node]);
4936	}
4937
4938	wq_numa_possible_cpumask = tbl;
4939	wq_numa_enabled = true;
4940	}
4941
4942	static int __init init_workqueues(void)
4943	{
4944	int std_nice[NR_STD_WORKER_POOLS] = { 0, HIGHPRI_NICE_LEVEL };
4945	int i, cpu;
4946
4947	/* make sure we have enough bits for OFFQ pool ID */
4948	BUILD_BUG_ON((1LU << (BITS_PER_LONG - WORK_OFFQ_POOL_SHIFT)) <
4949	WORK_CPU_END * NR_STD_WORKER_POOLS);
4950
4951	WARN_ON(__alignof__(struct pool_workqueue) < __alignof__(long long));
4952
4953	pwq_cache = KMEM_CACHE(pool_workqueue, SLAB_PANIC);
4954
4955	cpu_notifier(workqueue_cpu_up_callback, CPU_PRI_WORKQUEUE_UP);
4956	hotcpu_notifier(workqueue_cpu_down_callback, CPU_PRI_WORKQUEUE_DOWN);
4957
4958	wq_numa_init();
4959
4960	/* initialize CPU pools */
4961	for_each_possible_cpu(cpu) {
4962	struct worker_pool *pool;
4963
4964	i = 0;
4965	for_each_cpu_worker_pool(pool, cpu) {
4966	BUG_ON(init_worker_pool(pool));
4967	pool->cpu = cpu;
4968	cpumask_copy(pool->attrs->cpumask, cpumask_of(cpu));
4969	pool->attrs->nice = std_nice[i++];
4970	pool->node = cpu_to_node(cpu);
4971
4972	/* alloc pool ID */
4973	mutex_lock(&wq_pool_mutex);
4974	BUG_ON(worker_pool_assign_id(pool));
4975	mutex_unlock(&wq_pool_mutex);
4976	}
4977	}
4978
4979	/* create the initial worker */
4980	for_each_online_cpu(cpu) {
4981	struct worker_pool *pool;
4982
4983	for_each_cpu_worker_pool(pool, cpu) {
4984	pool->flags &= ~POOL_DISASSOCIATED;
4985	BUG_ON(create_and_start_worker(pool) < 0);
4986	}
4987	}
4988
4989	/* create default unbound wq attrs */
4990	for (i = 0; i < NR_STD_WORKER_POOLS; i++) {
4991	struct workqueue_attrs *attrs;
4992
4993	BUG_ON(!(attrs = alloc_workqueue_attrs(GFP_KERNEL)));
4994	attrs->nice = std_nice[i];
4995	unbound_std_wq_attrs[i] = attrs;
4996	}
4997
4998	system_wq = alloc_workqueue("events", 0, 0);
4999	system_highpri_wq = alloc_workqueue("events_highpri", WQ_HIGHPRI, 0);
5000	system_long_wq = alloc_workqueue("events_long", 0, 0);
5001	system_unbound_wq = alloc_workqueue("events_unbound", WQ_UNBOUND,
5002	WQ_UNBOUND_MAX_ACTIVE);
5003	system_freezable_wq = alloc_workqueue("events_freezable",
5004	WQ_FREEZABLE, 0);
5005	system_power_efficient_wq = alloc_workqueue("events_power_efficient",
5006	WQ_POWER_EFFICIENT, 0);
5007	system_freezable_power_efficient_wq = alloc_workqueue("events_freezable_power_efficient",
5008	WQ_FREEZABLE \| WQ_POWER_EFFICIENT,
5009	0);
5010	BUG_ON(!system_wq \|\| !system_highpri_wq \|\| !system_long_wq \|\|
5011	!system_unbound_wq \|\| !system_freezable_wq \|\|
5012	!system_power_efficient_wq \|\|
5013	!system_freezable_power_efficient_wq);
5014	return 0;
5015	}
5016	early_initcall(init_workqueues);