linux - kernel/workqueue.c

3762 lines · 102.1 KiB

1	/*
2	* kernel/workqueue.c - generic async execution with shared worker pool
3	*
4	* Copyright (C) 2002 Ingo Molnar
5	*
6	* Derived from the taskqueue/keventd code by:
7	* David Woodhouse <[email protected]>
8	* Andrew Morton
9	* Kai Petzke <[email protected]>
10	* Theodore Ts'o <[email protected]>
11	*
12	* Made to use alloc_percpu by Christoph Lameter.
13	*
14	* Copyright (C) 2010 SUSE Linux Products GmbH
15	* Copyright (C) 2010 Tejun Heo <[email protected]>
16	*
17	* This is the generic async execution mechanism. Work items as are
18	* executed in process context. The worker pool is shared and
19	* automatically managed. There is one worker pool for each CPU and
20	* one extra for works which are better served by workers which are
21	* not bound to any specific CPU.
22	*
23	* Please read Documentation/workqueue.txt for details.
24	*/
25
26	#include <linux/export.h>
27	#include <linux/kernel.h>
28	#include <linux/sched.h>
29	#include <linux/init.h>
30	#include <linux/signal.h>
31	#include <linux/completion.h>
32	#include <linux/workqueue.h>
33	#include <linux/slab.h>
34	#include <linux/cpu.h>
35	#include <linux/notifier.h>
36	#include <linux/kthread.h>
37	#include <linux/hardirq.h>
38	#include <linux/mempolicy.h>
39	#include <linux/freezer.h>
40	#include <linux/kallsyms.h>
41	#include <linux/debug_locks.h>
42	#include <linux/lockdep.h>
43	#include <linux/idr.h>
44
45	#include "workqueue_sched.h"
46
47	enum {
48	/*
49	* global_cwq flags
50	*
51	* A bound gcwq is either associated or disassociated with its CPU.
52	* While associated (!DISASSOCIATED), all workers are bound to the
53	* CPU and none has %WORKER_UNBOUND set and concurrency management
54	* is in effect.
55	*
56	* While DISASSOCIATED, the cpu may be offline and all workers have
57	* %WORKER_UNBOUND set and concurrency management disabled, and may
58	* be executing on any CPU. The gcwq behaves as an unbound one.
59	*
60	* Note that DISASSOCIATED can be flipped only while holding
61	* managership of all pools on the gcwq to avoid changing binding
62	* state while create_worker() is in progress.
63	*/
64	GCWQ_DISASSOCIATED = 1 << 0, /* cpu can't serve workers */
65	GCWQ_FREEZING = 1 << 1, /* freeze in progress */
66
67	/* pool flags */
68	POOL_MANAGE_WORKERS = 1 << 0, /* need to manage workers */
69
70	/* worker flags */
71	WORKER_STARTED = 1 << 0, /* started */
72	WORKER_DIE = 1 << 1, /* die die die */
73	WORKER_IDLE = 1 << 2, /* is idle */
74	WORKER_PREP = 1 << 3, /* preparing to run works */
75	WORKER_REBIND = 1 << 5, /* mom is home, come back */
76	WORKER_CPU_INTENSIVE = 1 << 6, /* cpu intensive */
77	WORKER_UNBOUND = 1 << 7, /* worker is unbound */
78
79	WORKER_NOT_RUNNING = WORKER_PREP \| WORKER_REBIND \| WORKER_UNBOUND \|
80	WORKER_CPU_INTENSIVE,
81
82	NR_WORKER_POOLS = 2, /* # worker pools per gcwq */
83
84	BUSY_WORKER_HASH_ORDER = 6, /* 64 pointers */
85	BUSY_WORKER_HASH_SIZE = 1 << BUSY_WORKER_HASH_ORDER,
86	BUSY_WORKER_HASH_MASK = BUSY_WORKER_HASH_SIZE - 1,
87
88	MAX_IDLE_WORKERS_RATIO = 4, /* 1/4 of busy can be idle */
89	IDLE_WORKER_TIMEOUT = 300 * HZ, /* keep idle ones for 5 mins */
90
91	MAYDAY_INITIAL_TIMEOUT = HZ / 100 >= 2 ? HZ / 100 : 2,
92	/* call for help after 10ms
93	(min two ticks) */
94	MAYDAY_INTERVAL = HZ / 10, /* and then every 100ms */
95	CREATE_COOLDOWN = HZ, /* time to breath after fail */
96
97	/*
98	* Rescue workers are used only on emergencies and shared by
99	* all cpus. Give -20.
100	*/
101	RESCUER_NICE_LEVEL = -20,
102	HIGHPRI_NICE_LEVEL = -20,
103	};
104
105	/*
106	* Structure fields follow one of the following exclusion rules.
107	*
108	* I: Modifiable by initialization/destruction paths and read-only for
109	* everyone else.
110	*
111	* P: Preemption protected. Disabling preemption is enough and should
112	* only be modified and accessed from the local cpu.
113	*
114	* L: gcwq->lock protected. Access with gcwq->lock held.
115	*
116	* X: During normal operation, modification requires gcwq->lock and
117	* should be done only from local cpu. Either disabling preemption
118	* on local cpu or grabbing gcwq->lock is enough for read access.
119	* If GCWQ_DISASSOCIATED is set, it's identical to L.
120	*
121	* F: wq->flush_mutex protected.
122	*
123	* W: workqueue_lock protected.
124	*/
125
126	struct global_cwq;
127	struct worker_pool;
128	struct idle_rebind;
129
130	/*
131	* The poor guys doing the actual heavy lifting. All on-duty workers
132	* are either serving the manager role, on idle list or on busy hash.
133	*/
134	struct worker {
135	/* on idle list while idle, on busy hash table while busy */
136	union {
137	struct list_head entry; /* L: while idle */
138	struct hlist_node hentry; /* L: while busy */
139	};
140
141	struct work_struct current_work; / L: work being processed */
142	struct cpu_workqueue_struct current_cwq; / L: current_work's cwq */
143	struct list_head scheduled; /* L: scheduled works */
144	struct task_struct task; / I: worker task */
145	struct worker_pool pool; / I: the associated pool */
146	/* 64 bytes boundary on 64bit, 32 on 32bit */
147	unsigned long last_active; /* L: last active timestamp */
148	unsigned int flags; /* X: flags */
149	int id; /* I: worker id */
150
151	/* for rebinding worker to CPU */
152	struct idle_rebind idle_rebind; / L: for idle worker */
153	struct work_struct rebind_work; /* L: for busy worker */
154	};
155
156	struct worker_pool {
157	struct global_cwq gcwq; / I: the owning gcwq */
158	unsigned int flags; /* X: flags */
159
160	struct list_head worklist; /* L: list of pending works */
161	int nr_workers; /* L: total number of workers */
162	int nr_idle; /* L: currently idle ones */
163
164	struct list_head idle_list; /* X: list of idle workers */
165	struct timer_list idle_timer; /* L: worker idle timeout */
166	struct timer_list mayday_timer; /* L: SOS timer for workers */
167
168	struct mutex manager_mutex; /* mutex manager should hold */
169	struct ida worker_ida; /* L: for worker IDs */
170	};
171
172	/*
173	* Global per-cpu workqueue. There's one and only one for each cpu
174	* and all works are queued and processed here regardless of their
175	* target workqueues.
176	*/
177	struct global_cwq {
178	spinlock_t lock; /* the gcwq lock */
179	unsigned int cpu; /* I: the associated cpu */
180	unsigned int flags; /* L: GCWQ_* flags */
181
182	/* workers are chained either in busy_hash or pool idle_list */
183	struct hlist_head busy_hash[BUSY_WORKER_HASH_SIZE];
184	/* L: hash of busy workers */
185
186	struct worker_pool pools[2]; /* normal and highpri pools */
187
188	wait_queue_head_t rebind_hold; /* rebind hold wait */
189	} ____cacheline_aligned_in_smp;
190
191	/*
192	* The per-CPU workqueue. The lower WORK_STRUCT_FLAG_BITS of
193	* work_struct->data are used for flags and thus cwqs need to be
194	* aligned at two's power of the number of flag bits.
195	*/
196	struct cpu_workqueue_struct {
197	struct worker_pool pool; / I: the associated pool */
198	struct workqueue_struct wq; / I: the owning workqueue */
199	int work_color; /* L: current color */
200	int flush_color; /* L: flushing color */
201	int nr_in_flight[WORK_NR_COLORS];
202	/* L: nr of in_flight works */
203	int nr_active; /* L: nr of active works */
204	int max_active; /* L: max active works */
205	struct list_head delayed_works; /* L: delayed works */
206	};
207
208	/*
209	* Structure used to wait for workqueue flush.
210	*/
211	struct wq_flusher {
212	struct list_head list; /* F: list of flushers */
213	int flush_color; /* F: flush color waiting for */
214	struct completion done; /* flush completion */
215	};
216
217	/*
218	* All cpumasks are assumed to be always set on UP and thus can't be
219	* used to determine whether there's something to be done.
220	*/
221	#ifdef CONFIG_SMP
222	typedef cpumask_var_t mayday_mask_t;
223	#define mayday_test_and_set_cpu(cpu, mask) \
224	cpumask_test_and_set_cpu((cpu), (mask))
225	#define mayday_clear_cpu(cpu, mask) cpumask_clear_cpu((cpu), (mask))
226	#define for_each_mayday_cpu(cpu, mask) for_each_cpu((cpu), (mask))
227	#define alloc_mayday_mask(maskp, gfp) zalloc_cpumask_var((maskp), (gfp))
228	#define free_mayday_mask(mask) free_cpumask_var((mask))
229	#else
230	typedef unsigned long mayday_mask_t;
231	#define mayday_test_and_set_cpu(cpu, mask) test_and_set_bit(0, &(mask))
232	#define mayday_clear_cpu(cpu, mask) clear_bit(0, &(mask))
233	#define for_each_mayday_cpu(cpu, mask) if ((cpu) = 0, (mask))
234	#define alloc_mayday_mask(maskp, gfp) true
235	#define free_mayday_mask(mask) do { } while (0)
236	#endif
237
238	/*
239	* The externally visible workqueue abstraction is an array of
240	* per-CPU workqueues:
241	*/
242	struct workqueue_struct {
243	unsigned int flags; /* W: WQ_* flags */
244	union {
245	struct cpu_workqueue_struct __percpu *pcpu;
246	struct cpu_workqueue_struct *single;
247	unsigned long v;
248	} cpu_wq; /* I: cwq's */
249	struct list_head list; /* W: list of all workqueues */
250
251	struct mutex flush_mutex; /* protects wq flushing */
252	int work_color; /* F: current work color */
253	int flush_color; /* F: current flush color */
254	atomic_t nr_cwqs_to_flush; /* flush in progress */
255	struct wq_flusher first_flusher; / F: first flusher */
256	struct list_head flusher_queue; /* F: flush waiters */
257	struct list_head flusher_overflow; /* F: flush overflow list */
258
259	mayday_mask_t mayday_mask; /* cpus requesting rescue */
260	struct worker rescuer; / I: rescue worker */
261
262	int nr_drainers; /* W: drain in progress */
263	int saved_max_active; /* W: saved cwq max_active */
264	#ifdef CONFIG_LOCKDEP
265	struct lockdep_map lockdep_map;
266	#endif
267	char name[]; /* I: workqueue name */
268	};
269
270	struct workqueue_struct *system_wq __read_mostly;
271	struct workqueue_struct *system_long_wq __read_mostly;
272	struct workqueue_struct *system_nrt_wq __read_mostly;
273	struct workqueue_struct *system_unbound_wq __read_mostly;
274	struct workqueue_struct *system_freezable_wq __read_mostly;
275	struct workqueue_struct *system_nrt_freezable_wq __read_mostly;
276	EXPORT_SYMBOL_GPL(system_wq);
277	EXPORT_SYMBOL_GPL(system_long_wq);
278	EXPORT_SYMBOL_GPL(system_nrt_wq);
279	EXPORT_SYMBOL_GPL(system_unbound_wq);
280	EXPORT_SYMBOL_GPL(system_freezable_wq);
281	EXPORT_SYMBOL_GPL(system_nrt_freezable_wq);
282
283	#define CREATE_TRACE_POINTS
284	#include <trace/events/workqueue.h>
285
286	#define for_each_worker_pool(pool, gcwq) \
287	for ((pool) = &(gcwq)->pools[0]; \
288	(pool) < &(gcwq)->pools[NR_WORKER_POOLS]; (pool)++)
289
290	#define for_each_busy_worker(worker, i, pos, gcwq) \
291	for (i = 0; i < BUSY_WORKER_HASH_SIZE; i++) \
292	hlist_for_each_entry(worker, pos, &gcwq->busy_hash[i], hentry)
293
294	static inline int __next_gcwq_cpu(int cpu, const struct cpumask *mask,
295	unsigned int sw)
296	{
297	if (cpu < nr_cpu_ids) {
298	if (sw & 1) {
299	cpu = cpumask_next(cpu, mask);
300	if (cpu < nr_cpu_ids)
301	return cpu;
302	}
303	if (sw & 2)
304	return WORK_CPU_UNBOUND;
305	}
306	return WORK_CPU_NONE;
307	}
308
309	static inline int __next_wq_cpu(int cpu, const struct cpumask *mask,
310	struct workqueue_struct *wq)
311	{
312	return __next_gcwq_cpu(cpu, mask, !(wq->flags & WQ_UNBOUND) ? 1 : 2);
313	}
314
315	/*
316	* CPU iterators
317	*
318	* An extra gcwq is defined for an invalid cpu number
319	* (WORK_CPU_UNBOUND) to host workqueues which are not bound to any
320	* specific CPU. The following iterators are similar to
321	* for_each_*_cpu() iterators but also considers the unbound gcwq.
322	*
323	* for_each_gcwq_cpu() : possible CPUs + WORK_CPU_UNBOUND
324	* for_each_online_gcwq_cpu() : online CPUs + WORK_CPU_UNBOUND
325	* for_each_cwq_cpu() : possible CPUs for bound workqueues,
326	* WORK_CPU_UNBOUND for unbound workqueues
327	*/
328	#define for_each_gcwq_cpu(cpu) \
329	for ((cpu) = __next_gcwq_cpu(-1, cpu_possible_mask, 3); \
330	(cpu) < WORK_CPU_NONE; \
331	(cpu) = __next_gcwq_cpu((cpu), cpu_possible_mask, 3))
332
333	#define for_each_online_gcwq_cpu(cpu) \
334	for ((cpu) = __next_gcwq_cpu(-1, cpu_online_mask, 3); \
335	(cpu) < WORK_CPU_NONE; \
336	(cpu) = __next_gcwq_cpu((cpu), cpu_online_mask, 3))
337
338	#define for_each_cwq_cpu(cpu, wq) \
339	for ((cpu) = __next_wq_cpu(-1, cpu_possible_mask, (wq)); \
340	(cpu) < WORK_CPU_NONE; \
341	(cpu) = __next_wq_cpu((cpu), cpu_possible_mask, (wq)))
342
343	#ifdef CONFIG_DEBUG_OBJECTS_WORK
344
345	static struct debug_obj_descr work_debug_descr;
346
347	static void work_debug_hint(void addr)
348	{
349	return ((struct work_struct *) addr)->func;
350	}
351
352	/*
353	* fixup_init is called when:
354	* - an active object is initialized
355	*/
356	static int work_fixup_init(void *addr, enum debug_obj_state state)
357	{
358	struct work_struct *work = addr;
359
360	switch (state) {
361	case ODEBUG_STATE_ACTIVE:
362	cancel_work_sync(work);
363	debug_object_init(work, &work_debug_descr);
364	return 1;
365	default:
366	return 0;
367	}
368	}
369
370	/*
371	* fixup_activate is called when:
372	* - an active object is activated
373	* - an unknown object is activated (might be a statically initialized object)
374	*/
375	static int work_fixup_activate(void *addr, enum debug_obj_state state)
376	{
377	struct work_struct *work = addr;
378
379	switch (state) {
380
381	case ODEBUG_STATE_NOTAVAILABLE:
382	/*
383	* This is not really a fixup. The work struct was
384	* statically initialized. We just make sure that it
385	* is tracked in the object tracker.
386	*/
387	if (test_bit(WORK_STRUCT_STATIC_BIT, work_data_bits(work))) {
388	debug_object_init(work, &work_debug_descr);
389	debug_object_activate(work, &work_debug_descr);
390	return 0;
391	}
392	WARN_ON_ONCE(1);
393	return 0;
394
395	case ODEBUG_STATE_ACTIVE:
396	WARN_ON(1);
397
398	default:
399	return 0;
400	}
401	}
402
403	/*
404	* fixup_free is called when:
405	* - an active object is freed
406	*/
407	static int work_fixup_free(void *addr, enum debug_obj_state state)
408	{
409	struct work_struct *work = addr;
410
411	switch (state) {
412	case ODEBUG_STATE_ACTIVE:
413	cancel_work_sync(work);
414	debug_object_free(work, &work_debug_descr);
415	return 1;
416	default:
417	return 0;
418	}
419	}
420
421	static struct debug_obj_descr work_debug_descr = {
422	.name = "work_struct",
423	.debug_hint = work_debug_hint,
424	.fixup_init = work_fixup_init,
425	.fixup_activate = work_fixup_activate,
426	.fixup_free = work_fixup_free,
427	};
428
429	static inline void debug_work_activate(struct work_struct *work)
430	{
431	debug_object_activate(work, &work_debug_descr);
432	}
433
434	static inline void debug_work_deactivate(struct work_struct *work)
435	{
436	debug_object_deactivate(work, &work_debug_descr);
437	}
438
439	void __init_work(struct work_struct *work, int onstack)
440	{
441	if (onstack)
442	debug_object_init_on_stack(work, &work_debug_descr);
443	else
444	debug_object_init(work, &work_debug_descr);
445	}
446	EXPORT_SYMBOL_GPL(__init_work);
447
448	void destroy_work_on_stack(struct work_struct *work)
449	{
450	debug_object_free(work, &work_debug_descr);
451	}
452	EXPORT_SYMBOL_GPL(destroy_work_on_stack);
453
454	#else
455	static inline void debug_work_activate(struct work_struct *work) { }
456	static inline void debug_work_deactivate(struct work_struct *work) { }
457	#endif
458
459	/* Serializes the accesses to the list of workqueues. */
460	static DEFINE_SPINLOCK(workqueue_lock);
461	static LIST_HEAD(workqueues);
462	static bool workqueue_freezing; /* W: have wqs started freezing? */
463
464	/*
465	* The almighty global cpu workqueues. nr_running is the only field
466	* which is expected to be used frequently by other cpus via
467	* try_to_wake_up(). Put it in a separate cacheline.
468	*/
469	static DEFINE_PER_CPU(struct global_cwq, global_cwq);
470	static DEFINE_PER_CPU_SHARED_ALIGNED(atomic_t, pool_nr_running[NR_WORKER_POOLS]);
471
472	/*
473	* Global cpu workqueue and nr_running counter for unbound gcwq. The
474	* gcwq is always online, has GCWQ_DISASSOCIATED set, and all its
475	* workers have WORKER_UNBOUND set.
476	*/
477	static struct global_cwq unbound_global_cwq;
478	static atomic_t unbound_pool_nr_running[NR_WORKER_POOLS] = {
479	[0 ... NR_WORKER_POOLS - 1] = ATOMIC_INIT(0), /* always 0 */
480	};
481
482	static int worker_thread(void *__worker);
483
484	static int worker_pool_pri(struct worker_pool *pool)
485	{
486	return pool - pool->gcwq->pools;
487	}
488
489	static struct global_cwq *get_gcwq(unsigned int cpu)
490	{
491	if (cpu != WORK_CPU_UNBOUND)
492	return &per_cpu(global_cwq, cpu);
493	else
494	return &unbound_global_cwq;
495	}
496
497	static atomic_t get_pool_nr_running(struct worker_pool pool)
498	{
499	int cpu = pool->gcwq->cpu;
500	int idx = worker_pool_pri(pool);
501
502	if (cpu != WORK_CPU_UNBOUND)
503	return &per_cpu(pool_nr_running, cpu)[idx];
504	else
505	return &unbound_pool_nr_running[idx];
506	}
507
508	static struct cpu_workqueue_struct *get_cwq(unsigned int cpu,
509	struct workqueue_struct *wq)
510	{
511	if (!(wq->flags & WQ_UNBOUND)) {
512	if (likely(cpu < nr_cpu_ids))
513	return per_cpu_ptr(wq->cpu_wq.pcpu, cpu);
514	} else if (likely(cpu == WORK_CPU_UNBOUND))
515	return wq->cpu_wq.single;
516	return NULL;
517	}
518
519	static unsigned int work_color_to_flags(int color)
520	{
521	return color << WORK_STRUCT_COLOR_SHIFT;
522	}
523
524	static int get_work_color(struct work_struct *work)
525	{
526	return (*work_data_bits(work) >> WORK_STRUCT_COLOR_SHIFT) &
527	((1 << WORK_STRUCT_COLOR_BITS) - 1);
528	}
529
530	static int work_next_color(int color)
531	{
532	return (color + 1) % WORK_NR_COLORS;
533	}
534
535	/*
536	* A work's data points to the cwq with WORK_STRUCT_CWQ set while the
537	* work is on queue. Once execution starts, WORK_STRUCT_CWQ is
538	* cleared and the work data contains the cpu number it was last on.
539	*
540	* set_work_{cwq\|cpu}() and clear_work_data() can be used to set the
541	* cwq, cpu or clear work->data. These functions should only be
542	* called while the work is owned - ie. while the PENDING bit is set.
543	*
544	* get_work_[g]cwq() can be used to obtain the gcwq or cwq
545	* corresponding to a work. gcwq is available once the work has been
546	* queued anywhere after initialization. cwq is available only from
547	* queueing until execution starts.
548	*/
549	static inline void set_work_data(struct work_struct *work, unsigned long data,
550	unsigned long flags)
551	{
552	BUG_ON(!work_pending(work));
553	atomic_long_set(&work->data, data \| flags \| work_static(work));
554	}
555
556	static void set_work_cwq(struct work_struct *work,
557	struct cpu_workqueue_struct *cwq,
558	unsigned long extra_flags)
559	{
560	set_work_data(work, (unsigned long)cwq,
561	WORK_STRUCT_PENDING \| WORK_STRUCT_CWQ \| extra_flags);
562	}
563
564	static void set_work_cpu(struct work_struct *work, unsigned int cpu)
565	{
566	set_work_data(work, cpu << WORK_STRUCT_FLAG_BITS, WORK_STRUCT_PENDING);
567	}
568
569	static void clear_work_data(struct work_struct *work)
570	{
571	set_work_data(work, WORK_STRUCT_NO_CPU, 0);
572	}
573
574	static struct cpu_workqueue_struct get_work_cwq(struct work_struct work)
575	{
576	unsigned long data = atomic_long_read(&work->data);
577
578	if (data & WORK_STRUCT_CWQ)
579	return (void *)(data & WORK_STRUCT_WQ_DATA_MASK);
580	else
581	return NULL;
582	}
583
584	static struct global_cwq get_work_gcwq(struct work_struct work)
585	{
586	unsigned long data = atomic_long_read(&work->data);
587	unsigned int cpu;
588
589	if (data & WORK_STRUCT_CWQ)
590	return ((struct cpu_workqueue_struct *)
591	(data & WORK_STRUCT_WQ_DATA_MASK))->pool->gcwq;
592
593	cpu = data >> WORK_STRUCT_FLAG_BITS;
594	if (cpu == WORK_CPU_NONE)
595	return NULL;
596
597	BUG_ON(cpu >= nr_cpu_ids && cpu != WORK_CPU_UNBOUND);
598	return get_gcwq(cpu);
599	}
600
601	/*
602	* Policy functions. These define the policies on how the global worker
603	* pools are managed. Unless noted otherwise, these functions assume that
604	* they're being called with gcwq->lock held.
605	*/
606
607	static bool __need_more_worker(struct worker_pool *pool)
608	{
609	return !atomic_read(get_pool_nr_running(pool));
610	}
611
612	/*
613	* Need to wake up a worker? Called from anything but currently
614	* running workers.
615	*
616	* Note that, because unbound workers never contribute to nr_running, this
617	* function will always return %true for unbound gcwq as long as the
618	* worklist isn't empty.
619	*/
620	static bool need_more_worker(struct worker_pool *pool)
621	{
622	return !list_empty(&pool->worklist) && __need_more_worker(pool);
623	}
624
625	/* Can I start working? Called from busy but !running workers. */
626	static bool may_start_working(struct worker_pool *pool)
627	{
628	return pool->nr_idle;
629	}
630
631	/* Do I need to keep working? Called from currently running workers. */
632	static bool keep_working(struct worker_pool *pool)
633	{
634	atomic_t *nr_running = get_pool_nr_running(pool);
635
636	return !list_empty(&pool->worklist) && atomic_read(nr_running) <= 1;
637	}
638
639	/* Do we need a new worker? Called from manager. */
640	static bool need_to_create_worker(struct worker_pool *pool)
641	{
642	return need_more_worker(pool) && !may_start_working(pool);
643	}
644
645	/* Do I need to be the manager? */
646	static bool need_to_manage_workers(struct worker_pool *pool)
647	{
648	return need_to_create_worker(pool) \|\|
649	(pool->flags & POOL_MANAGE_WORKERS);
650	}
651
652	/* Do we have too many workers and should some go away? */
653	static bool too_many_workers(struct worker_pool *pool)
654	{
655	bool managing = mutex_is_locked(&pool->manager_mutex);
656	int nr_idle = pool->nr_idle + managing; /* manager is considered idle */
657	int nr_busy = pool->nr_workers - nr_idle;
658
659	return nr_idle > 2 && (nr_idle - 2) * MAX_IDLE_WORKERS_RATIO >= nr_busy;
660	}
661
662	/*
663	* Wake up functions.
664	*/
665
666	/* Return the first worker. Safe with preemption disabled */
667	static struct worker first_worker(struct worker_pool pool)
668	{
669	if (unlikely(list_empty(&pool->idle_list)))
670	return NULL;
671
672	return list_first_entry(&pool->idle_list, struct worker, entry);
673	}
674
675	/**
676	* wake_up_worker - wake up an idle worker
677	* @pool: worker pool to wake worker from
678	*
679	* Wake up the first idle worker of @pool.
680	*
681	* CONTEXT:
682	* spin_lock_irq(gcwq->lock).
683	*/
684	static void wake_up_worker(struct worker_pool *pool)
685	{
686	struct worker *worker = first_worker(pool);
687
688	if (likely(worker))
689	wake_up_process(worker->task);
690	}
691
692	/**
693	* wq_worker_waking_up - a worker is waking up
694	* @task: task waking up
695	* @cpu: CPU @task is waking up to
696	*
697	* This function is called during try_to_wake_up() when a worker is
698	* being awoken.
699	*
700	* CONTEXT:
701	* spin_lock_irq(rq->lock)
702	*/
703	void wq_worker_waking_up(struct task_struct *task, unsigned int cpu)
704	{
705	struct worker *worker = kthread_data(task);
706
707	if (!(worker->flags & WORKER_NOT_RUNNING))
708	atomic_inc(get_pool_nr_running(worker->pool));
709	}
710
711	/**
712	* wq_worker_sleeping - a worker is going to sleep
713	* @task: task going to sleep
714	* @cpu: CPU in question, must be the current CPU number
715	*
716	* This function is called during schedule() when a busy worker is
717	* going to sleep. Worker on the same cpu can be woken up by
718	* returning pointer to its task.
719	*
720	* CONTEXT:
721	* spin_lock_irq(rq->lock)
722	*
723	* RETURNS:
724	* Worker task on @cpu to wake up, %NULL if none.
725	*/
726	struct task_struct wq_worker_sleeping(struct task_struct task,
727	unsigned int cpu)
728	{
729	struct worker worker = kthread_data(task), to_wakeup = NULL;
730	struct worker_pool *pool = worker->pool;
731	atomic_t *nr_running = get_pool_nr_running(pool);
732
733	if (worker->flags & WORKER_NOT_RUNNING)
734	return NULL;
735
736	/* this can only happen on the local cpu */
737	BUG_ON(cpu != raw_smp_processor_id());
738
739	/*
740	* The counterpart of the following dec_and_test, implied mb,
741	* worklist not empty test sequence is in insert_work().
742	* Please read comment there.
743	*
744	* NOT_RUNNING is clear. This means that we're bound to and
745	* running on the local cpu w/ rq lock held and preemption
746	* disabled, which in turn means that none else could be
747	* manipulating idle_list, so dereferencing idle_list without gcwq
748	* lock is safe.
749	*/
750	if (atomic_dec_and_test(nr_running) && !list_empty(&pool->worklist))
751	to_wakeup = first_worker(pool);
752	return to_wakeup ? to_wakeup->task : NULL;
753	}
754
755	/**
756	* worker_set_flags - set worker flags and adjust nr_running accordingly
757	* @worker: self
758	* @flags: flags to set
759	* @wakeup: wakeup an idle worker if necessary
760	*
761	* Set @flags in @worker->flags and adjust nr_running accordingly. If
762	* nr_running becomes zero and @wakeup is %true, an idle worker is
763	* woken up.
764	*
765	* CONTEXT:
766	* spin_lock_irq(gcwq->lock)
767	*/
768	static inline void worker_set_flags(struct worker *worker, unsigned int flags,
769	bool wakeup)
770	{
771	struct worker_pool *pool = worker->pool;
772
773	WARN_ON_ONCE(worker->task != current);
774
775	/*
776	* If transitioning into NOT_RUNNING, adjust nr_running and
777	* wake up an idle worker as necessary if requested by
778	* @wakeup.
779	*/
780	if ((flags & WORKER_NOT_RUNNING) &&
781	!(worker->flags & WORKER_NOT_RUNNING)) {
782	atomic_t *nr_running = get_pool_nr_running(pool);
783
784	if (wakeup) {
785	if (atomic_dec_and_test(nr_running) &&
786	!list_empty(&pool->worklist))
787	wake_up_worker(pool);
788	} else
789	atomic_dec(nr_running);
790	}
791
792	worker->flags \|= flags;
793	}
794
795	/**
796	* worker_clr_flags - clear worker flags and adjust nr_running accordingly
797	* @worker: self
798	* @flags: flags to clear
799	*
800	* Clear @flags in @worker->flags and adjust nr_running accordingly.
801	*
802	* CONTEXT:
803	* spin_lock_irq(gcwq->lock)
804	*/
805	static inline void worker_clr_flags(struct worker *worker, unsigned int flags)
806	{
807	struct worker_pool *pool = worker->pool;
808	unsigned int oflags = worker->flags;
809
810	WARN_ON_ONCE(worker->task != current);
811
812	worker->flags &= ~flags;
813
814	/*
815	* If transitioning out of NOT_RUNNING, increment nr_running. Note
816	* that the nested NOT_RUNNING is not a noop. NOT_RUNNING is mask
817	* of multiple flags, not a single flag.
818	*/
819	if ((flags & WORKER_NOT_RUNNING) && (oflags & WORKER_NOT_RUNNING))
820	if (!(worker->flags & WORKER_NOT_RUNNING))
821	atomic_inc(get_pool_nr_running(pool));
822	}
823
824	/**
825	* busy_worker_head - return the busy hash head for a work
826	* @gcwq: gcwq of interest
827	* @work: work to be hashed
828	*
829	* Return hash head of @gcwq for @work.
830	*
831	* CONTEXT:
832	* spin_lock_irq(gcwq->lock).
833	*
834	* RETURNS:
835	* Pointer to the hash head.
836	*/
837	static struct hlist_head busy_worker_head(struct global_cwq gcwq,
838	struct work_struct *work)
839	{
840	const int base_shift = ilog2(sizeof(struct work_struct));
841	unsigned long v = (unsigned long)work;
842
843	/* simple shift and fold hash, do we need something better? */
844	v >>= base_shift;
845	v += v >> BUSY_WORKER_HASH_ORDER;
846	v &= BUSY_WORKER_HASH_MASK;
847
848	return &gcwq->busy_hash[v];
849	}
850
851	/**
852	* __find_worker_executing_work - find worker which is executing a work
853	* @gcwq: gcwq of interest
854	* @bwh: hash head as returned by busy_worker_head()
855	* @work: work to find worker for
856	*
857	* Find a worker which is executing @work on @gcwq. @bwh should be
858	* the hash head obtained by calling busy_worker_head() with the same
859	* work.
860	*
861	* CONTEXT:
862	* spin_lock_irq(gcwq->lock).
863	*
864	* RETURNS:
865	* Pointer to worker which is executing @work if found, NULL
866	* otherwise.
867	*/
868	static struct worker __find_worker_executing_work(struct global_cwq gcwq,
869	struct hlist_head *bwh,
870	struct work_struct *work)
871	{
872	struct worker *worker;
873	struct hlist_node *tmp;
874
875	hlist_for_each_entry(worker, tmp, bwh, hentry)
876	if (worker->current_work == work)
877	return worker;
878	return NULL;
879	}
880
881	/**
882	* find_worker_executing_work - find worker which is executing a work
883	* @gcwq: gcwq of interest
884	* @work: work to find worker for
885	*
886	* Find a worker which is executing @work on @gcwq. This function is
887	* identical to __find_worker_executing_work() except that this
888	* function calculates @bwh itself.
889	*
890	* CONTEXT:
891	* spin_lock_irq(gcwq->lock).
892	*
893	* RETURNS:
894	* Pointer to worker which is executing @work if found, NULL
895	* otherwise.
896	*/
897	static struct worker find_worker_executing_work(struct global_cwq gcwq,
898	struct work_struct *work)
899	{
900	return __find_worker_executing_work(gcwq, busy_worker_head(gcwq, work),
901	work);
902	}
903
904	/**
905	* insert_work - insert a work into gcwq
906	* @cwq: cwq @work belongs to
907	* @work: work to insert
908	* @head: insertion point
909	* @extra_flags: extra WORK_STRUCT_* flags to set
910	*
911	* Insert @work which belongs to @cwq into @gcwq after @head.
912	* @extra_flags is or'd to work_struct flags.
913	*
914	* CONTEXT:
915	* spin_lock_irq(gcwq->lock).
916	*/
917	static void insert_work(struct cpu_workqueue_struct *cwq,
918	struct work_struct work, struct list_head head,
919	unsigned int extra_flags)
920	{
921	struct worker_pool *pool = cwq->pool;
922
923	/* we own @work, set data and link */
924	set_work_cwq(work, cwq, extra_flags);
925
926	/*
927	* Ensure that we get the right work->data if we see the
928	* result of list_add() below, see try_to_grab_pending().
929	*/
930	smp_wmb();
931
932	list_add_tail(&work->entry, head);
933
934	/*
935	* Ensure either worker_sched_deactivated() sees the above
936	* list_add_tail() or we see zero nr_running to avoid workers
937	* lying around lazily while there are works to be processed.
938	*/
939	smp_mb();
940
941	if (__need_more_worker(pool))
942	wake_up_worker(pool);
943	}
944
945	/*
946	* Test whether @work is being queued from another work executing on the
947	* same workqueue. This is rather expensive and should only be used from
948	* cold paths.
949	*/
950	static bool is_chained_work(struct workqueue_struct *wq)
951	{
952	unsigned long flags;
953	unsigned int cpu;
954
955	for_each_gcwq_cpu(cpu) {
956	struct global_cwq *gcwq = get_gcwq(cpu);
957	struct worker *worker;
958	struct hlist_node *pos;
959	int i;
960
961	spin_lock_irqsave(&gcwq->lock, flags);
962	for_each_busy_worker(worker, i, pos, gcwq) {
963	if (worker->task != current)
964	continue;
965	spin_unlock_irqrestore(&gcwq->lock, flags);
966	/*
967	* I'm @worker, no locking necessary. See if @work
968	* is headed to the same workqueue.
969	*/
970	return worker->current_cwq->wq == wq;
971	}
972	spin_unlock_irqrestore(&gcwq->lock, flags);
973	}
974	return false;
975	}
976
977	static void __queue_work(unsigned int cpu, struct workqueue_struct *wq,
978	struct work_struct *work)
979	{
980	struct global_cwq *gcwq;
981	struct cpu_workqueue_struct *cwq;
982	struct list_head *worklist;
983	unsigned int work_flags;
984	unsigned long flags;
985
986	debug_work_activate(work);
987
988	/* if dying, only works from the same workqueue are allowed */
989	if (unlikely(wq->flags & WQ_DRAINING) &&
990	WARN_ON_ONCE(!is_chained_work(wq)))
991	return;
992
993	/* determine gcwq to use */
994	if (!(wq->flags & WQ_UNBOUND)) {
995	struct global_cwq *last_gcwq;
996
997	if (unlikely(cpu == WORK_CPU_UNBOUND))
998	cpu = raw_smp_processor_id();
999
1000	/*
1001	* It's multi cpu. If @wq is non-reentrant and @work
1002	* was previously on a different cpu, it might still
1003	* be running there, in which case the work needs to
1004	* be queued on that cpu to guarantee non-reentrance.
1005	*/
1006	gcwq = get_gcwq(cpu);
1007	if (wq->flags & WQ_NON_REENTRANT &&
1008	(last_gcwq = get_work_gcwq(work)) && last_gcwq != gcwq) {
1009	struct worker *worker;
1010
1011	spin_lock_irqsave(&last_gcwq->lock, flags);
1012
1013	worker = find_worker_executing_work(last_gcwq, work);
1014
1015	if (worker && worker->current_cwq->wq == wq)
1016	gcwq = last_gcwq;
1017	else {
1018	/* meh... not running there, queue here */
1019	spin_unlock_irqrestore(&last_gcwq->lock, flags);
1020	spin_lock_irqsave(&gcwq->lock, flags);
1021	}
1022	} else
1023	spin_lock_irqsave(&gcwq->lock, flags);
1024	} else {
1025	gcwq = get_gcwq(WORK_CPU_UNBOUND);
1026	spin_lock_irqsave(&gcwq->lock, flags);
1027	}
1028
1029	/* gcwq determined, get cwq and queue */
1030	cwq = get_cwq(gcwq->cpu, wq);
1031	trace_workqueue_queue_work(cpu, cwq, work);
1032
1033	if (WARN_ON(!list_empty(&work->entry))) {
1034	spin_unlock_irqrestore(&gcwq->lock, flags);
1035	return;
1036	}
1037
1038	cwq->nr_in_flight[cwq->work_color]++;
1039	work_flags = work_color_to_flags(cwq->work_color);
1040
1041	if (likely(cwq->nr_active < cwq->max_active)) {
1042	trace_workqueue_activate_work(work);
1043	cwq->nr_active++;
1044	worklist = &cwq->pool->worklist;
1045	} else {
1046	work_flags \|= WORK_STRUCT_DELAYED;
1047	worklist = &cwq->delayed_works;
1048	}
1049
1050	insert_work(cwq, work, worklist, work_flags);
1051
1052	spin_unlock_irqrestore(&gcwq->lock, flags);
1053	}
1054
1055	/**
1056	* queue_work - queue work on a workqueue
1057	* @wq: workqueue to use
1058	* @work: work to queue
1059	*
1060	* Returns 0 if @work was already on a queue, non-zero otherwise.
1061	*
1062	* We queue the work to the CPU on which it was submitted, but if the CPU dies
1063	* it can be processed by another CPU.
1064	*/
1065	int queue_work(struct workqueue_struct wq, struct work_struct work)
1066	{
1067	int ret;
1068
1069	ret = queue_work_on(get_cpu(), wq, work);
1070	put_cpu();
1071
1072	return ret;
1073	}
1074	EXPORT_SYMBOL_GPL(queue_work);
1075
1076	/**
1077	* queue_work_on - queue work on specific cpu
1078	* @cpu: CPU number to execute work on
1079	* @wq: workqueue to use
1080	* @work: work to queue
1081	*
1082	* Returns 0 if @work was already on a queue, non-zero otherwise.
1083	*
1084	* We queue the work to a specific CPU, the caller must ensure it
1085	* can't go away.
1086	*/
1087	int
1088	queue_work_on(int cpu, struct workqueue_struct wq, struct work_struct work)
1089	{
1090	int ret = 0;
1091
1092	if (!test_and_set_bit(WORK_STRUCT_PENDING_BIT, work_data_bits(work))) {
1093	__queue_work(cpu, wq, work);
1094	ret = 1;
1095	}
1096	return ret;
1097	}
1098	EXPORT_SYMBOL_GPL(queue_work_on);
1099
1100	static void delayed_work_timer_fn(unsigned long __data)
1101	{
1102	struct delayed_work dwork = (struct delayed_work )__data;
1103	struct cpu_workqueue_struct *cwq = get_work_cwq(&dwork->work);
1104
1105	__queue_work(smp_processor_id(), cwq->wq, &dwork->work);
1106	}
1107
1108	/**
1109	* queue_delayed_work - queue work on a workqueue after delay
1110	* @wq: workqueue to use
1111	* @dwork: delayable work to queue
1112	* @delay: number of jiffies to wait before queueing
1113	*
1114	* Returns 0 if @work was already on a queue, non-zero otherwise.
1115	*/
1116	int queue_delayed_work(struct workqueue_struct *wq,
1117	struct delayed_work *dwork, unsigned long delay)
1118	{
1119	if (delay == 0)
1120	return queue_work(wq, &dwork->work);
1121
1122	return queue_delayed_work_on(-1, wq, dwork, delay);
1123	}
1124	EXPORT_SYMBOL_GPL(queue_delayed_work);
1125
1126	/**
1127	* queue_delayed_work_on - queue work on specific CPU after delay
1128	* @cpu: CPU number to execute work on
1129	* @wq: workqueue to use
1130	* @dwork: work to queue
1131	* @delay: number of jiffies to wait before queueing
1132	*
1133	* Returns 0 if @work was already on a queue, non-zero otherwise.
1134	*/
1135	int queue_delayed_work_on(int cpu, struct workqueue_struct *wq,
1136	struct delayed_work *dwork, unsigned long delay)
1137	{
1138	int ret = 0;
1139	struct timer_list *timer = &dwork->timer;
1140	struct work_struct *work = &dwork->work;
1141
1142	if (!test_and_set_bit(WORK_STRUCT_PENDING_BIT, work_data_bits(work))) {
1143	unsigned int lcpu;
1144
1145	BUG_ON(timer_pending(timer));
1146	BUG_ON(!list_empty(&work->entry));
1147
1148	timer_stats_timer_set_start_info(&dwork->timer);
1149
1150	/*
1151	* This stores cwq for the moment, for the timer_fn.
1152	* Note that the work's gcwq is preserved to allow
1153	* reentrance detection for delayed works.
1154	*/
1155	if (!(wq->flags & WQ_UNBOUND)) {
1156	struct global_cwq *gcwq = get_work_gcwq(work);
1157
1158	if (gcwq && gcwq->cpu != WORK_CPU_UNBOUND)
1159	lcpu = gcwq->cpu;
1160	else
1161	lcpu = raw_smp_processor_id();
1162	} else
1163	lcpu = WORK_CPU_UNBOUND;
1164
1165	set_work_cwq(work, get_cwq(lcpu, wq), 0);
1166
1167	timer->expires = jiffies + delay;
1168	timer->data = (unsigned long)dwork;
1169	timer->function = delayed_work_timer_fn;
1170
1171	if (unlikely(cpu >= 0))
1172	add_timer_on(timer, cpu);
1173	else
1174	add_timer(timer);
1175	ret = 1;
1176	}
1177	return ret;
1178	}
1179	EXPORT_SYMBOL_GPL(queue_delayed_work_on);
1180
1181	/**
1182	* worker_enter_idle - enter idle state
1183	* @worker: worker which is entering idle state
1184	*
1185	* @worker is entering idle state. Update stats and idle timer if
1186	* necessary.
1187	*
1188	* LOCKING:
1189	* spin_lock_irq(gcwq->lock).
1190	*/
1191	static void worker_enter_idle(struct worker *worker)
1192	{
1193	struct worker_pool *pool = worker->pool;
1194	struct global_cwq *gcwq = pool->gcwq;
1195
1196	BUG_ON(worker->flags & WORKER_IDLE);
1197	BUG_ON(!list_empty(&worker->entry) &&
1198	(worker->hentry.next \|\| worker->hentry.pprev));
1199
1200	/* can't use worker_set_flags(), also called from start_worker() */
1201	worker->flags \|= WORKER_IDLE;
1202	pool->nr_idle++;
1203	worker->last_active = jiffies;
1204
1205	/* idle_list is LIFO */
1206	list_add(&worker->entry, &pool->idle_list);
1207
1208	if (too_many_workers(pool) && !timer_pending(&pool->idle_timer))
1209	mod_timer(&pool->idle_timer, jiffies + IDLE_WORKER_TIMEOUT);
1210
1211	/*
1212	* Sanity check nr_running. Because gcwq_unbind_fn() releases
1213	* gcwq->lock between setting %WORKER_UNBOUND and zapping
1214	* nr_running, the warning may trigger spuriously. Check iff
1215	* unbind is not in progress.
1216	*/
1217	WARN_ON_ONCE(!(gcwq->flags & GCWQ_DISASSOCIATED) &&
1218	pool->nr_workers == pool->nr_idle &&
1219	atomic_read(get_pool_nr_running(pool)));
1220	}
1221
1222	/**
1223	* worker_leave_idle - leave idle state
1224	* @worker: worker which is leaving idle state
1225	*
1226	* @worker is leaving idle state. Update stats.
1227	*
1228	* LOCKING:
1229	* spin_lock_irq(gcwq->lock).
1230	*/
1231	static void worker_leave_idle(struct worker *worker)
1232	{
1233	struct worker_pool *pool = worker->pool;
1234
1235	BUG_ON(!(worker->flags & WORKER_IDLE));
1236	worker_clr_flags(worker, WORKER_IDLE);
1237	pool->nr_idle--;
1238	list_del_init(&worker->entry);
1239	}
1240
1241	/**
1242	* worker_maybe_bind_and_lock - bind worker to its cpu if possible and lock gcwq
1243	* @worker: self
1244	*
1245	* Works which are scheduled while the cpu is online must at least be
1246	* scheduled to a worker which is bound to the cpu so that if they are
1247	* flushed from cpu callbacks while cpu is going down, they are
1248	* guaranteed to execute on the cpu.
1249	*
1250	* This function is to be used by rogue workers and rescuers to bind
1251	* themselves to the target cpu and may race with cpu going down or
1252	* coming online. kthread_bind() can't be used because it may put the
1253	* worker to already dead cpu and set_cpus_allowed_ptr() can't be used
1254	* verbatim as it's best effort and blocking and gcwq may be
1255	* [dis]associated in the meantime.
1256	*
1257	* This function tries set_cpus_allowed() and locks gcwq and verifies the
1258	* binding against %GCWQ_DISASSOCIATED which is set during
1259	* %CPU_DOWN_PREPARE and cleared during %CPU_ONLINE, so if the worker
1260	* enters idle state or fetches works without dropping lock, it can
1261	* guarantee the scheduling requirement described in the first paragraph.
1262	*
1263	* CONTEXT:
1264	* Might sleep. Called without any lock but returns with gcwq->lock
1265	* held.
1266	*
1267	* RETURNS:
1268	* %true if the associated gcwq is online (@worker is successfully
1269	* bound), %false if offline.
1270	*/
1271	static bool worker_maybe_bind_and_lock(struct worker *worker)
1272	__acquires(&gcwq->lock)
1273	{
1274	struct global_cwq *gcwq = worker->pool->gcwq;
1275	struct task_struct *task = worker->task;
1276
1277	while (true) {
1278	/*
1279	* The following call may fail, succeed or succeed
1280	* without actually migrating the task to the cpu if
1281	* it races with cpu hotunplug operation. Verify
1282	* against GCWQ_DISASSOCIATED.
1283	*/
1284	if (!(gcwq->flags & GCWQ_DISASSOCIATED))
1285	set_cpus_allowed_ptr(task, get_cpu_mask(gcwq->cpu));
1286
1287	spin_lock_irq(&gcwq->lock);
1288	if (gcwq->flags & GCWQ_DISASSOCIATED)
1289	return false;
1290	if (task_cpu(task) == gcwq->cpu &&
1291	cpumask_equal(&current->cpus_allowed,
1292	get_cpu_mask(gcwq->cpu)))
1293	return true;
1294	spin_unlock_irq(&gcwq->lock);
1295
1296	/*
1297	* We've raced with CPU hot[un]plug. Give it a breather
1298	* and retry migration. cond_resched() is required here;
1299	* otherwise, we might deadlock against cpu_stop trying to
1300	* bring down the CPU on non-preemptive kernel.
1301	*/
1302	cpu_relax();
1303	cond_resched();
1304	}
1305	}
1306
1307	struct idle_rebind {
1308	int cnt; /* # workers to be rebound */
1309	struct completion done; /* all workers rebound */
1310	};
1311
1312	/*
1313	* Rebind an idle @worker to its CPU. During CPU onlining, this has to
1314	* happen synchronously for idle workers. worker_thread() will test
1315	* %WORKER_REBIND before leaving idle and call this function.
1316	*/
1317	static void idle_worker_rebind(struct worker *worker)
1318	{
1319	struct global_cwq *gcwq = worker->pool->gcwq;
1320
1321	/* CPU must be online at this point */
1322	WARN_ON(!worker_maybe_bind_and_lock(worker));
1323	if (!--worker->idle_rebind->cnt)
1324	complete(&worker->idle_rebind->done);
1325	spin_unlock_irq(&worker->pool->gcwq->lock);
1326
1327	/* we did our part, wait for rebind_workers() to finish up */
1328	wait_event(gcwq->rebind_hold, !(worker->flags & WORKER_REBIND));
1329	}
1330
1331	/*
1332	* Function for @worker->rebind.work used to rebind unbound busy workers to
1333	* the associated cpu which is coming back online. This is scheduled by
1334	* cpu up but can race with other cpu hotplug operations and may be
1335	* executed twice without intervening cpu down.
1336	*/
1337	static void busy_worker_rebind_fn(struct work_struct *work)
1338	{
1339	struct worker *worker = container_of(work, struct worker, rebind_work);
1340	struct global_cwq *gcwq = worker->pool->gcwq;
1341
1342	if (worker_maybe_bind_and_lock(worker))
1343	worker_clr_flags(worker, WORKER_REBIND);
1344
1345	spin_unlock_irq(&gcwq->lock);
1346	}
1347
1348	/**
1349	* rebind_workers - rebind all workers of a gcwq to the associated CPU
1350	* @gcwq: gcwq of interest
1351	*
1352	* @gcwq->cpu is coming online. Rebind all workers to the CPU. Rebinding
1353	* is different for idle and busy ones.
1354	*
1355	* The idle ones should be rebound synchronously and idle rebinding should
1356	* be complete before any worker starts executing work items with
1357	* concurrency management enabled; otherwise, scheduler may oops trying to
1358	* wake up non-local idle worker from wq_worker_sleeping().
1359	*
1360	* This is achieved by repeatedly requesting rebinding until all idle
1361	* workers are known to have been rebound under @gcwq->lock and holding all
1362	* idle workers from becoming busy until idle rebinding is complete.
1363	*
1364	* Once idle workers are rebound, busy workers can be rebound as they
1365	* finish executing their current work items. Queueing the rebind work at
1366	* the head of their scheduled lists is enough. Note that nr_running will
1367	* be properbly bumped as busy workers rebind.
1368	*
1369	* On return, all workers are guaranteed to either be bound or have rebind
1370	* work item scheduled.
1371	*/
1372	static void rebind_workers(struct global_cwq *gcwq)
1373	__releases(&gcwq->lock) __acquires(&gcwq->lock)
1374	{
1375	struct idle_rebind idle_rebind;
1376	struct worker_pool *pool;
1377	struct worker *worker;
1378	struct hlist_node *pos;
1379	int i;
1380
1381	lockdep_assert_held(&gcwq->lock);
1382
1383	for_each_worker_pool(pool, gcwq)
1384	lockdep_assert_held(&pool->manager_mutex);
1385
1386	/*
1387	* Rebind idle workers. Interlocked both ways. We wait for
1388	* workers to rebind via @idle_rebind.done. Workers will wait for
1389	* us to finish up by watching %WORKER_REBIND.
1390	*/
1391	init_completion(&idle_rebind.done);
1392	retry:
1393	idle_rebind.cnt = 1;
1394	INIT_COMPLETION(idle_rebind.done);
1395
1396	/* set REBIND and kick idle ones, we'll wait for these later */
1397	for_each_worker_pool(pool, gcwq) {
1398	list_for_each_entry(worker, &pool->idle_list, entry) {
1399	if (worker->flags & WORKER_REBIND)
1400	continue;
1401
1402	/* morph UNBOUND to REBIND */
1403	worker->flags &= ~WORKER_UNBOUND;
1404	worker->flags \|= WORKER_REBIND;
1405
1406	idle_rebind.cnt++;
1407	worker->idle_rebind = &idle_rebind;
1408
1409	/* worker_thread() will call idle_worker_rebind() */
1410	wake_up_process(worker->task);
1411	}
1412	}
1413
1414	if (--idle_rebind.cnt) {
1415	spin_unlock_irq(&gcwq->lock);
1416	wait_for_completion(&idle_rebind.done);
1417	spin_lock_irq(&gcwq->lock);
1418	/* busy ones might have become idle while waiting, retry */
1419	goto retry;
1420	}
1421
1422	/*
1423	* All idle workers are rebound and waiting for %WORKER_REBIND to
1424	* be cleared inside idle_worker_rebind(). Clear and release.
1425	* Clearing %WORKER_REBIND from this foreign context is safe
1426	* because these workers are still guaranteed to be idle.
1427	*/
1428	for_each_worker_pool(pool, gcwq)
1429	list_for_each_entry(worker, &pool->idle_list, entry)
1430	worker->flags &= ~WORKER_REBIND;
1431
1432	wake_up_all(&gcwq->rebind_hold);
1433
1434	/* rebind busy workers */
1435	for_each_busy_worker(worker, i, pos, gcwq) {
1436	struct work_struct *rebind_work = &worker->rebind_work;
1437
1438	/* morph UNBOUND to REBIND */
1439	worker->flags &= ~WORKER_UNBOUND;
1440	worker->flags \|= WORKER_REBIND;
1441
1442	if (test_and_set_bit(WORK_STRUCT_PENDING_BIT,
1443	work_data_bits(rebind_work)))
1444	continue;
1445
1446	/* wq doesn't matter, use the default one */
1447	debug_work_activate(rebind_work);
1448	insert_work(get_cwq(gcwq->cpu, system_wq), rebind_work,
1449	worker->scheduled.next,
1450	work_color_to_flags(WORK_NO_COLOR));
1451	}
1452	}
1453
1454	static struct worker *alloc_worker(void)
1455	{
1456	struct worker *worker;
1457
1458	worker = kzalloc(sizeof(*worker), GFP_KERNEL);
1459	if (worker) {
1460	INIT_LIST_HEAD(&worker->entry);
1461	INIT_LIST_HEAD(&worker->scheduled);
1462	INIT_WORK(&worker->rebind_work, busy_worker_rebind_fn);
1463	/* on creation a worker is in !idle && prep state */
1464	worker->flags = WORKER_PREP;
1465	}
1466	return worker;
1467	}
1468
1469	/**
1470	* create_worker - create a new workqueue worker
1471	* @pool: pool the new worker will belong to
1472	*
1473	* Create a new worker which is bound to @pool. The returned worker
1474	* can be started by calling start_worker() or destroyed using
1475	* destroy_worker().
1476	*
1477	* CONTEXT:
1478	* Might sleep. Does GFP_KERNEL allocations.
1479	*
1480	* RETURNS:
1481	* Pointer to the newly created worker.
1482	*/
1483	static struct worker create_worker(struct worker_pool pool)
1484	{
1485	struct global_cwq *gcwq = pool->gcwq;
1486	const char *pri = worker_pool_pri(pool) ? "H" : "";
1487	struct worker *worker = NULL;
1488	int id = -1;
1489
1490	spin_lock_irq(&gcwq->lock);
1491	while (ida_get_new(&pool->worker_ida, &id)) {
1492	spin_unlock_irq(&gcwq->lock);
1493	if (!ida_pre_get(&pool->worker_ida, GFP_KERNEL))
1494	goto fail;
1495	spin_lock_irq(&gcwq->lock);
1496	}
1497	spin_unlock_irq(&gcwq->lock);
1498
1499	worker = alloc_worker();
1500	if (!worker)
1501	goto fail;
1502
1503	worker->pool = pool;
1504	worker->id = id;
1505
1506	if (gcwq->cpu != WORK_CPU_UNBOUND)
1507	worker->task = kthread_create_on_node(worker_thread,
1508	worker, cpu_to_node(gcwq->cpu),
1509	"kworker/%u:%d%s", gcwq->cpu, id, pri);
1510	else
1511	worker->task = kthread_create(worker_thread, worker,
1512	"kworker/u:%d%s", id, pri);
1513	if (IS_ERR(worker->task))
1514	goto fail;
1515
1516	if (worker_pool_pri(pool))
1517	set_user_nice(worker->task, HIGHPRI_NICE_LEVEL);
1518
1519	/*
1520	* Determine CPU binding of the new worker depending on
1521	* %GCWQ_DISASSOCIATED. The caller is responsible for ensuring the
1522	* flag remains stable across this function. See the comments
1523	* above the flag definition for details.
1524	*
1525	* As an unbound worker may later become a regular one if CPU comes
1526	* online, make sure every worker has %PF_THREAD_BOUND set.
1527	*/
1528	if (!(gcwq->flags & GCWQ_DISASSOCIATED)) {
1529	kthread_bind(worker->task, gcwq->cpu);
1530	} else {
1531	worker->task->flags \|= PF_THREAD_BOUND;
1532	worker->flags \|= WORKER_UNBOUND;
1533	}
1534
1535	return worker;
1536	fail:
1537	if (id >= 0) {
1538	spin_lock_irq(&gcwq->lock);
1539	ida_remove(&pool->worker_ida, id);
1540	spin_unlock_irq(&gcwq->lock);
1541	}
1542	kfree(worker);
1543	return NULL;
1544	}
1545
1546	/**
1547	* start_worker - start a newly created worker
1548	* @worker: worker to start
1549	*
1550	* Make the gcwq aware of @worker and start it.
1551	*
1552	* CONTEXT:
1553	* spin_lock_irq(gcwq->lock).
1554	*/
1555	static void start_worker(struct worker *worker)
1556	{
1557	worker->flags \|= WORKER_STARTED;
1558	worker->pool->nr_workers++;
1559	worker_enter_idle(worker);
1560	wake_up_process(worker->task);
1561	}
1562
1563	/**
1564	* destroy_worker - destroy a workqueue worker
1565	* @worker: worker to be destroyed
1566	*
1567	* Destroy @worker and adjust @gcwq stats accordingly.
1568	*
1569	* CONTEXT:
1570	* spin_lock_irq(gcwq->lock) which is released and regrabbed.
1571	*/
1572	static void destroy_worker(struct worker *worker)
1573	{
1574	struct worker_pool *pool = worker->pool;
1575	struct global_cwq *gcwq = pool->gcwq;
1576	int id = worker->id;
1577
1578	/* sanity check frenzy */
1579	BUG_ON(worker->current_work);
1580	BUG_ON(!list_empty(&worker->scheduled));
1581
1582	if (worker->flags & WORKER_STARTED)
1583	pool->nr_workers--;
1584	if (worker->flags & WORKER_IDLE)
1585	pool->nr_idle--;
1586
1587	list_del_init(&worker->entry);
1588	worker->flags \|= WORKER_DIE;
1589
1590	spin_unlock_irq(&gcwq->lock);
1591
1592	kthread_stop(worker->task);
1593	kfree(worker);
1594
1595	spin_lock_irq(&gcwq->lock);
1596	ida_remove(&pool->worker_ida, id);
1597	}
1598
1599	static void idle_worker_timeout(unsigned long __pool)
1600	{
1601	struct worker_pool pool = (void )__pool;
1602	struct global_cwq *gcwq = pool->gcwq;
1603
1604	spin_lock_irq(&gcwq->lock);
1605
1606	if (too_many_workers(pool)) {
1607	struct worker *worker;
1608	unsigned long expires;
1609
1610	/* idle_list is kept in LIFO order, check the last one */
1611	worker = list_entry(pool->idle_list.prev, struct worker, entry);
1612	expires = worker->last_active + IDLE_WORKER_TIMEOUT;
1613
1614	if (time_before(jiffies, expires))
1615	mod_timer(&pool->idle_timer, expires);
1616	else {
1617	/* it's been idle for too long, wake up manager */
1618	pool->flags \|= POOL_MANAGE_WORKERS;
1619	wake_up_worker(pool);
1620	}
1621	}
1622
1623	spin_unlock_irq(&gcwq->lock);
1624	}
1625
1626	static bool send_mayday(struct work_struct *work)
1627	{
1628	struct cpu_workqueue_struct *cwq = get_work_cwq(work);
1629	struct workqueue_struct *wq = cwq->wq;
1630	unsigned int cpu;
1631
1632	if (!(wq->flags & WQ_RESCUER))
1633	return false;
1634
1635	/* mayday mayday mayday */
1636	cpu = cwq->pool->gcwq->cpu;
1637	/* WORK_CPU_UNBOUND can't be set in cpumask, use cpu 0 instead */
1638	if (cpu == WORK_CPU_UNBOUND)
1639	cpu = 0;
1640	if (!mayday_test_and_set_cpu(cpu, wq->mayday_mask))
1641	wake_up_process(wq->rescuer->task);
1642	return true;
1643	}
1644
1645	static void gcwq_mayday_timeout(unsigned long __pool)
1646	{
1647	struct worker_pool pool = (void )__pool;
1648	struct global_cwq *gcwq = pool->gcwq;
1649	struct work_struct *work;
1650
1651	spin_lock_irq(&gcwq->lock);
1652
1653	if (need_to_create_worker(pool)) {
1654	/*
1655	* We've been trying to create a new worker but
1656	* haven't been successful. We might be hitting an
1657	* allocation deadlock. Send distress signals to
1658	* rescuers.
1659	*/
1660	list_for_each_entry(work, &pool->worklist, entry)
1661	send_mayday(work);
1662	}
1663
1664	spin_unlock_irq(&gcwq->lock);
1665
1666	mod_timer(&pool->mayday_timer, jiffies + MAYDAY_INTERVAL);
1667	}
1668
1669	/**
1670	* maybe_create_worker - create a new worker if necessary
1671	* @pool: pool to create a new worker for
1672	*
1673	* Create a new worker for @pool if necessary. @pool is guaranteed to
1674	* have at least one idle worker on return from this function. If
1675	* creating a new worker takes longer than MAYDAY_INTERVAL, mayday is
1676	* sent to all rescuers with works scheduled on @pool to resolve
1677	* possible allocation deadlock.
1678	*
1679	* On return, need_to_create_worker() is guaranteed to be false and
1680	* may_start_working() true.
1681	*
1682	* LOCKING:
1683	* spin_lock_irq(gcwq->lock) which may be released and regrabbed
1684	* multiple times. Does GFP_KERNEL allocations. Called only from
1685	* manager.
1686	*
1687	* RETURNS:
1688	* false if no action was taken and gcwq->lock stayed locked, true
1689	* otherwise.
1690	*/
1691	static bool maybe_create_worker(struct worker_pool *pool)
1692	__releases(&gcwq->lock)
1693	__acquires(&gcwq->lock)
1694	{
1695	struct global_cwq *gcwq = pool->gcwq;
1696
1697	if (!need_to_create_worker(pool))
1698	return false;
1699	restart:
1700	spin_unlock_irq(&gcwq->lock);
1701
1702	/* if we don't make progress in MAYDAY_INITIAL_TIMEOUT, call for help */
1703	mod_timer(&pool->mayday_timer, jiffies + MAYDAY_INITIAL_TIMEOUT);
1704
1705	while (true) {
1706	struct worker *worker;
1707
1708	worker = create_worker(pool);
1709	if (worker) {
1710	del_timer_sync(&pool->mayday_timer);
1711	spin_lock_irq(&gcwq->lock);
1712	start_worker(worker);
1713	BUG_ON(need_to_create_worker(pool));
1714	return true;
1715	}
1716
1717	if (!need_to_create_worker(pool))
1718	break;
1719
1720	__set_current_state(TASK_INTERRUPTIBLE);
1721	schedule_timeout(CREATE_COOLDOWN);
1722
1723	if (!need_to_create_worker(pool))
1724	break;
1725	}
1726
1727	del_timer_sync(&pool->mayday_timer);
1728	spin_lock_irq(&gcwq->lock);
1729	if (need_to_create_worker(pool))
1730	goto restart;
1731	return true;
1732	}
1733
1734	/**
1735	* maybe_destroy_worker - destroy workers which have been idle for a while
1736	* @pool: pool to destroy workers for
1737	*
1738	* Destroy @pool workers which have been idle for longer than
1739	* IDLE_WORKER_TIMEOUT.
1740	*
1741	* LOCKING:
1742	* spin_lock_irq(gcwq->lock) which may be released and regrabbed
1743	* multiple times. Called only from manager.
1744	*
1745	* RETURNS:
1746	* false if no action was taken and gcwq->lock stayed locked, true
1747	* otherwise.
1748	*/
1749	static bool maybe_destroy_workers(struct worker_pool *pool)
1750	{
1751	bool ret = false;
1752
1753	while (too_many_workers(pool)) {
1754	struct worker *worker;
1755	unsigned long expires;
1756
1757	worker = list_entry(pool->idle_list.prev, struct worker, entry);
1758	expires = worker->last_active + IDLE_WORKER_TIMEOUT;
1759
1760	if (time_before(jiffies, expires)) {
1761	mod_timer(&pool->idle_timer, expires);
1762	break;
1763	}
1764
1765	destroy_worker(worker);
1766	ret = true;
1767	}
1768
1769	return ret;
1770	}
1771
1772	/**
1773	* manage_workers - manage worker pool
1774	* @worker: self
1775	*
1776	* Assume the manager role and manage gcwq worker pool @worker belongs
1777	* to. At any given time, there can be only zero or one manager per
1778	* gcwq. The exclusion is handled automatically by this function.
1779	*
1780	* The caller can safely start processing works on false return. On
1781	* true return, it's guaranteed that need_to_create_worker() is false
1782	* and may_start_working() is true.
1783	*
1784	* CONTEXT:
1785	* spin_lock_irq(gcwq->lock) which may be released and regrabbed
1786	* multiple times. Does GFP_KERNEL allocations.
1787	*
1788	* RETURNS:
1789	* false if no action was taken and gcwq->lock stayed locked, true if
1790	* some action was taken.
1791	*/
1792	static bool manage_workers(struct worker *worker)
1793	{
1794	struct worker_pool *pool = worker->pool;
1795	bool ret = false;
1796
1797	if (!mutex_trylock(&pool->manager_mutex))
1798	return ret;
1799
1800	pool->flags &= ~POOL_MANAGE_WORKERS;
1801
1802	/*
1803	* Destroy and then create so that may_start_working() is true
1804	* on return.
1805	*/
1806	ret \|= maybe_destroy_workers(pool);
1807	ret \|= maybe_create_worker(pool);
1808
1809	mutex_unlock(&pool->manager_mutex);
1810	return ret;
1811	}
1812
1813	/**
1814	* move_linked_works - move linked works to a list
1815	* @work: start of series of works to be scheduled
1816	* @head: target list to append @work to
1817	* @nextp: out paramter for nested worklist walking
1818	*
1819	* Schedule linked works starting from @work to @head. Work series to
1820	* be scheduled starts at @work and includes any consecutive work with
1821	* WORK_STRUCT_LINKED set in its predecessor.
1822	*
1823	* If @nextp is not NULL, it's updated to point to the next work of
1824	* the last scheduled work. This allows move_linked_works() to be
1825	* nested inside outer list_for_each_entry_safe().
1826	*
1827	* CONTEXT:
1828	* spin_lock_irq(gcwq->lock).
1829	*/
1830	static void move_linked_works(struct work_struct work, struct list_head head,
1831	struct work_struct **nextp)
1832	{
1833	struct work_struct *n;
1834
1835	/*
1836	* Linked worklist will always end before the end of the list,
1837	* use NULL for list head.
1838	*/
1839	list_for_each_entry_safe_from(work, n, NULL, entry) {
1840	list_move_tail(&work->entry, head);
1841	if (!(*work_data_bits(work) & WORK_STRUCT_LINKED))
1842	break;
1843	}
1844
1845	/*
1846	* If we're already inside safe list traversal and have moved
1847	* multiple works to the scheduled queue, the next position
1848	* needs to be updated.
1849	*/
1850	if (nextp)
1851	*nextp = n;
1852	}
1853
1854	static void cwq_activate_first_delayed(struct cpu_workqueue_struct *cwq)
1855	{
1856	struct work_struct *work = list_first_entry(&cwq->delayed_works,
1857	struct work_struct, entry);
1858
1859	trace_workqueue_activate_work(work);
1860	move_linked_works(work, &cwq->pool->worklist, NULL);
1861	__clear_bit(WORK_STRUCT_DELAYED_BIT, work_data_bits(work));
1862	cwq->nr_active++;
1863	}
1864
1865	/**
1866	* cwq_dec_nr_in_flight - decrement cwq's nr_in_flight
1867	* @cwq: cwq of interest
1868	* @color: color of work which left the queue
1869	* @delayed: for a delayed work
1870	*
1871	* A work either has completed or is removed from pending queue,
1872	* decrement nr_in_flight of its cwq and handle workqueue flushing.
1873	*
1874	* CONTEXT:
1875	* spin_lock_irq(gcwq->lock).
1876	*/
1877	static void cwq_dec_nr_in_flight(struct cpu_workqueue_struct *cwq, int color,
1878	bool delayed)
1879	{
1880	/* ignore uncolored works */
1881	if (color == WORK_NO_COLOR)
1882	return;
1883
1884	cwq->nr_in_flight[color]--;
1885
1886	if (!delayed) {
1887	cwq->nr_active--;
1888	if (!list_empty(&cwq->delayed_works)) {
1889	/* one down, submit a delayed one */
1890	if (cwq->nr_active < cwq->max_active)
1891	cwq_activate_first_delayed(cwq);
1892	}
1893	}
1894
1895	/* is flush in progress and are we at the flushing tip? */
1896	if (likely(cwq->flush_color != color))
1897	return;
1898
1899	/* are there still in-flight works? */
1900	if (cwq->nr_in_flight[color])
1901	return;
1902
1903	/* this cwq is done, clear flush_color */
1904	cwq->flush_color = -1;
1905
1906	/*
1907	* If this was the last cwq, wake up the first flusher. It
1908	* will handle the rest.
1909	*/
1910	if (atomic_dec_and_test(&cwq->wq->nr_cwqs_to_flush))
1911	complete(&cwq->wq->first_flusher->done);
1912	}
1913
1914	/**
1915	* process_one_work - process single work
1916	* @worker: self
1917	* @work: work to process
1918	*
1919	* Process @work. This function contains all the logics necessary to
1920	* process a single work including synchronization against and
1921	* interaction with other workers on the same cpu, queueing and
1922	* flushing. As long as context requirement is met, any worker can
1923	* call this function to process a work.
1924	*
1925	* CONTEXT:
1926	* spin_lock_irq(gcwq->lock) which is released and regrabbed.
1927	*/
1928	static void process_one_work(struct worker worker, struct work_struct work)
1929	__releases(&gcwq->lock)
1930	__acquires(&gcwq->lock)
1931	{
1932	struct cpu_workqueue_struct *cwq = get_work_cwq(work);
1933	struct worker_pool *pool = worker->pool;
1934	struct global_cwq *gcwq = pool->gcwq;
1935	struct hlist_head *bwh = busy_worker_head(gcwq, work);
1936	bool cpu_intensive = cwq->wq->flags & WQ_CPU_INTENSIVE;
1937	work_func_t f = work->func;
1938	int work_color;
1939	struct worker *collision;
1940	#ifdef CONFIG_LOCKDEP
1941	/*
1942	* It is permissible to free the struct work_struct from
1943	* inside the function that is called from it, this we need to
1944	* take into account for lockdep too. To avoid bogus "held
1945	* lock freed" warnings as well as problems when looking into
1946	* work->lockdep_map, make a copy and use that here.
1947	*/
1948	struct lockdep_map lockdep_map;
1949
1950	lockdep_copy_map(&lockdep_map, &work->lockdep_map);
1951	#endif
1952	/*
1953	* Ensure we're on the correct CPU. DISASSOCIATED test is
1954	* necessary to avoid spurious warnings from rescuers servicing the
1955	* unbound or a disassociated gcwq.
1956	*/
1957	WARN_ON_ONCE(!(worker->flags & (WORKER_UNBOUND \| WORKER_REBIND)) &&
1958	!(gcwq->flags & GCWQ_DISASSOCIATED) &&
1959	raw_smp_processor_id() != gcwq->cpu);
1960
1961	/*
1962	* A single work shouldn't be executed concurrently by
1963	* multiple workers on a single cpu. Check whether anyone is
1964	* already processing the work. If so, defer the work to the
1965	* currently executing one.
1966	*/
1967	collision = __find_worker_executing_work(gcwq, bwh, work);
1968	if (unlikely(collision)) {
1969	move_linked_works(work, &collision->scheduled, NULL);
1970	return;
1971	}
1972
1973	/* claim and process */
1974	debug_work_deactivate(work);
1975	hlist_add_head(&worker->hentry, bwh);
1976	worker->current_work = work;
1977	worker->current_cwq = cwq;
1978	work_color = get_work_color(work);
1979
1980	/* record the current cpu number in the work data and dequeue */
1981	set_work_cpu(work, gcwq->cpu);
1982	list_del_init(&work->entry);
1983
1984	/*
1985	* CPU intensive works don't participate in concurrency
1986	* management. They're the scheduler's responsibility.
1987	*/
1988	if (unlikely(cpu_intensive))
1989	worker_set_flags(worker, WORKER_CPU_INTENSIVE, true);
1990
1991	/*
1992	* Unbound gcwq isn't concurrency managed and work items should be
1993	* executed ASAP. Wake up another worker if necessary.
1994	*/
1995	if ((worker->flags & WORKER_UNBOUND) && need_more_worker(pool))
1996	wake_up_worker(pool);
1997
1998	spin_unlock_irq(&gcwq->lock);
1999
2000	work_clear_pending(work);
2001	lock_map_acquire_read(&cwq->wq->lockdep_map);
2002	lock_map_acquire(&lockdep_map);
2003	trace_workqueue_execute_start(work);
2004	f(work);
2005	/*
2006	* While we must be careful to not use "work" after this, the trace
2007	* point will only record its address.
2008	*/
2009	trace_workqueue_execute_end(work);
2010	lock_map_release(&lockdep_map);
2011	lock_map_release(&cwq->wq->lockdep_map);
2012
2013	if (unlikely(in_atomic() \|\| lockdep_depth(current) > 0)) {
2014	printk(KERN_ERR "BUG: workqueue leaked lock or atomic: "
2015	"%s/0x%08x/%d\n",
2016	current->comm, preempt_count(), task_pid_nr(current));
2017	printk(KERN_ERR " last function: ");
2018	print_symbol("%s\n", (unsigned long)f);
2019	debug_show_held_locks(current);
2020	dump_stack();
2021	}
2022
2023	spin_lock_irq(&gcwq->lock);
2024
2025	/* clear cpu intensive status */
2026	if (unlikely(cpu_intensive))
2027	worker_clr_flags(worker, WORKER_CPU_INTENSIVE);
2028
2029	/* we're done with it, release */
2030	hlist_del_init(&worker->hentry);
2031	worker->current_work = NULL;
2032	worker->current_cwq = NULL;
2033	cwq_dec_nr_in_flight(cwq, work_color, false);
2034	}
2035
2036	/**
2037	* process_scheduled_works - process scheduled works
2038	* @worker: self
2039	*
2040	* Process all scheduled works. Please note that the scheduled list
2041	* may change while processing a work, so this function repeatedly
2042	* fetches a work from the top and executes it.
2043	*
2044	* CONTEXT:
2045	* spin_lock_irq(gcwq->lock) which may be released and regrabbed
2046	* multiple times.
2047	*/
2048	static void process_scheduled_works(struct worker *worker)
2049	{
2050	while (!list_empty(&worker->scheduled)) {
2051	struct work_struct *work = list_first_entry(&worker->scheduled,
2052	struct work_struct, entry);
2053	process_one_work(worker, work);
2054	}
2055	}
2056
2057	/**
2058	* worker_thread - the worker thread function
2059	* @__worker: self
2060	*
2061	* The gcwq worker thread function. There's a single dynamic pool of
2062	* these per each cpu. These workers process all works regardless of
2063	* their specific target workqueue. The only exception is works which
2064	* belong to workqueues with a rescuer which will be explained in
2065	* rescuer_thread().
2066	*/
2067	static int worker_thread(void *__worker)
2068	{
2069	struct worker *worker = __worker;
2070	struct worker_pool *pool = worker->pool;
2071	struct global_cwq *gcwq = pool->gcwq;
2072
2073	/* tell the scheduler that this is a workqueue worker */
2074	worker->task->flags \|= PF_WQ_WORKER;
2075	woke_up:
2076	spin_lock_irq(&gcwq->lock);
2077
2078	/*
2079	* DIE can be set only while idle and REBIND set while busy has
2080	* @worker->rebind_work scheduled. Checking here is enough.
2081	*/
2082	if (unlikely(worker->flags & (WORKER_REBIND \| WORKER_DIE))) {
2083	spin_unlock_irq(&gcwq->lock);
2084
2085	if (worker->flags & WORKER_DIE) {
2086	worker->task->flags &= ~PF_WQ_WORKER;
2087	return 0;
2088	}
2089
2090	idle_worker_rebind(worker);
2091	goto woke_up;
2092	}
2093
2094	worker_leave_idle(worker);
2095	recheck:
2096	/* no more worker necessary? */
2097	if (!need_more_worker(pool))
2098	goto sleep;
2099
2100	/* do we need to manage? */
2101	if (unlikely(!may_start_working(pool)) && manage_workers(worker))
2102	goto recheck;
2103
2104	/*
2105	* ->scheduled list can only be filled while a worker is
2106	* preparing to process a work or actually processing it.
2107	* Make sure nobody diddled with it while I was sleeping.
2108	*/
2109	BUG_ON(!list_empty(&worker->scheduled));
2110
2111	/*
2112	* When control reaches this point, we're guaranteed to have
2113	* at least one idle worker or that someone else has already
2114	* assumed the manager role.
2115	*/
2116	worker_clr_flags(worker, WORKER_PREP);
2117
2118	do {
2119	struct work_struct *work =
2120	list_first_entry(&pool->worklist,
2121	struct work_struct, entry);
2122
2123	if (likely(!(*work_data_bits(work) & WORK_STRUCT_LINKED))) {
2124	/* optimization path, not strictly necessary */
2125	process_one_work(worker, work);
2126	if (unlikely(!list_empty(&worker->scheduled)))
2127	process_scheduled_works(worker);
2128	} else {
2129	move_linked_works(work, &worker->scheduled, NULL);
2130	process_scheduled_works(worker);
2131	}
2132	} while (keep_working(pool));
2133
2134	worker_set_flags(worker, WORKER_PREP, false);
2135	sleep:
2136	if (unlikely(need_to_manage_workers(pool)) && manage_workers(worker))
2137	goto recheck;
2138
2139	/*
2140	* gcwq->lock is held and there's no work to process and no
2141	* need to manage, sleep. Workers are woken up only while
2142	* holding gcwq->lock or from local cpu, so setting the
2143	* current state before releasing gcwq->lock is enough to
2144	* prevent losing any event.
2145	*/
2146	worker_enter_idle(worker);
2147	__set_current_state(TASK_INTERRUPTIBLE);
2148	spin_unlock_irq(&gcwq->lock);
2149	schedule();
2150	goto woke_up;
2151	}
2152
2153	/**
2154	* rescuer_thread - the rescuer thread function
2155	* @__wq: the associated workqueue
2156	*
2157	* Workqueue rescuer thread function. There's one rescuer for each
2158	* workqueue which has WQ_RESCUER set.
2159	*
2160	* Regular work processing on a gcwq may block trying to create a new
2161	* worker which uses GFP_KERNEL allocation which has slight chance of
2162	* developing into deadlock if some works currently on the same queue
2163	* need to be processed to satisfy the GFP_KERNEL allocation. This is
2164	* the problem rescuer solves.
2165	*
2166	* When such condition is possible, the gcwq summons rescuers of all
2167	* workqueues which have works queued on the gcwq and let them process
2168	* those works so that forward progress can be guaranteed.
2169	*
2170	* This should happen rarely.
2171	*/
2172	static int rescuer_thread(void *__wq)
2173	{
2174	struct workqueue_struct *wq = __wq;
2175	struct worker *rescuer = wq->rescuer;
2176	struct list_head *scheduled = &rescuer->scheduled;
2177	bool is_unbound = wq->flags & WQ_UNBOUND;
2178	unsigned int cpu;
2179
2180	set_user_nice(current, RESCUER_NICE_LEVEL);
2181	repeat:
2182	set_current_state(TASK_INTERRUPTIBLE);
2183
2184	if (kthread_should_stop())
2185	return 0;
2186
2187	/*
2188	* See whether any cpu is asking for help. Unbounded
2189	* workqueues use cpu 0 in mayday_mask for CPU_UNBOUND.
2190	*/
2191	for_each_mayday_cpu(cpu, wq->mayday_mask) {
2192	unsigned int tcpu = is_unbound ? WORK_CPU_UNBOUND : cpu;
2193	struct cpu_workqueue_struct *cwq = get_cwq(tcpu, wq);
2194	struct worker_pool *pool = cwq->pool;
2195	struct global_cwq *gcwq = pool->gcwq;
2196	struct work_struct work, n;
2197
2198	__set_current_state(TASK_RUNNING);
2199	mayday_clear_cpu(cpu, wq->mayday_mask);
2200
2201	/* migrate to the target cpu if possible */
2202	rescuer->pool = pool;
2203	worker_maybe_bind_and_lock(rescuer);
2204
2205	/*
2206	* Slurp in all works issued via this workqueue and
2207	* process'em.
2208	*/
2209	BUG_ON(!list_empty(&rescuer->scheduled));
2210	list_for_each_entry_safe(work, n, &pool->worklist, entry)
2211	if (get_work_cwq(work) == cwq)
2212	move_linked_works(work, scheduled, &n);
2213
2214	process_scheduled_works(rescuer);
2215
2216	/*
2217	* Leave this gcwq. If keep_working() is %true, notify a
2218	* regular worker; otherwise, we end up with 0 concurrency
2219	* and stalling the execution.
2220	*/
2221	if (keep_working(pool))
2222	wake_up_worker(pool);
2223
2224	spin_unlock_irq(&gcwq->lock);
2225	}
2226
2227	schedule();
2228	goto repeat;
2229	}
2230
2231	struct wq_barrier {
2232	struct work_struct work;
2233	struct completion done;
2234	};
2235
2236	static void wq_barrier_func(struct work_struct *work)
2237	{
2238	struct wq_barrier *barr = container_of(work, struct wq_barrier, work);
2239	complete(&barr->done);
2240	}
2241
2242	/**
2243	* insert_wq_barrier - insert a barrier work
2244	* @cwq: cwq to insert barrier into
2245	* @barr: wq_barrier to insert
2246	* @target: target work to attach @barr to
2247	* @worker: worker currently executing @target, NULL if @target is not executing
2248	*
2249	* @barr is linked to @target such that @barr is completed only after
2250	* @target finishes execution. Please note that the ordering
2251	* guarantee is observed only with respect to @target and on the local
2252	* cpu.
2253	*
2254	* Currently, a queued barrier can't be canceled. This is because
2255	* try_to_grab_pending() can't determine whether the work to be
2256	* grabbed is at the head of the queue and thus can't clear LINKED
2257	* flag of the previous work while there must be a valid next work
2258	* after a work with LINKED flag set.
2259	*
2260	* Note that when @worker is non-NULL, @target may be modified
2261	* underneath us, so we can't reliably determine cwq from @target.
2262	*
2263	* CONTEXT:
2264	* spin_lock_irq(gcwq->lock).
2265	*/
2266	static void insert_wq_barrier(struct cpu_workqueue_struct *cwq,
2267	struct wq_barrier *barr,
2268	struct work_struct target, struct worker worker)
2269	{
2270	struct list_head *head;
2271	unsigned int linked = 0;
2272
2273	/*
2274	* debugobject calls are safe here even with gcwq->lock locked
2275	* as we know for sure that this will not trigger any of the
2276	* checks and call back into the fixup functions where we
2277	* might deadlock.
2278	*/
2279	INIT_WORK_ONSTACK(&barr->work, wq_barrier_func);
2280	__set_bit(WORK_STRUCT_PENDING_BIT, work_data_bits(&barr->work));
2281	init_completion(&barr->done);
2282
2283	/*
2284	* If @target is currently being executed, schedule the
2285	* barrier to the worker; otherwise, put it after @target.
2286	*/
2287	if (worker)
2288	head = worker->scheduled.next;
2289	else {
2290	unsigned long *bits = work_data_bits(target);
2291
2292	head = target->entry.next;
2293	/* there can already be other linked works, inherit and set */
2294	linked = *bits & WORK_STRUCT_LINKED;
2295	__set_bit(WORK_STRUCT_LINKED_BIT, bits);
2296	}
2297
2298	debug_work_activate(&barr->work);
2299	insert_work(cwq, &barr->work, head,
2300	work_color_to_flags(WORK_NO_COLOR) \| linked);
2301	}
2302
2303	/**
2304	* flush_workqueue_prep_cwqs - prepare cwqs for workqueue flushing
2305	* @wq: workqueue being flushed
2306	* @flush_color: new flush color, < 0 for no-op
2307	* @work_color: new work color, < 0 for no-op
2308	*
2309	* Prepare cwqs for workqueue flushing.
2310	*
2311	* If @flush_color is non-negative, flush_color on all cwqs should be
2312	* -1. If no cwq has in-flight commands at the specified color, all
2313	* cwq->flush_color's stay at -1 and %false is returned. If any cwq
2314	* has in flight commands, its cwq->flush_color is set to
2315	* @flush_color, @wq->nr_cwqs_to_flush is updated accordingly, cwq
2316	* wakeup logic is armed and %true is returned.
2317	*
2318	* The caller should have initialized @wq->first_flusher prior to
2319	* calling this function with non-negative @flush_color. If
2320	* @flush_color is negative, no flush color update is done and %false
2321	* is returned.
2322	*
2323	* If @work_color is non-negative, all cwqs should have the same
2324	* work_color which is previous to @work_color and all will be
2325	* advanced to @work_color.
2326	*
2327	* CONTEXT:
2328	* mutex_lock(wq->flush_mutex).
2329	*
2330	* RETURNS:
2331	* %true if @flush_color >= 0 and there's something to flush. %false
2332	* otherwise.
2333	*/
2334	static bool flush_workqueue_prep_cwqs(struct workqueue_struct *wq,
2335	int flush_color, int work_color)
2336	{
2337	bool wait = false;
2338	unsigned int cpu;
2339
2340	if (flush_color >= 0) {
2341	BUG_ON(atomic_read(&wq->nr_cwqs_to_flush));
2342	atomic_set(&wq->nr_cwqs_to_flush, 1);
2343	}
2344
2345	for_each_cwq_cpu(cpu, wq) {
2346	struct cpu_workqueue_struct *cwq = get_cwq(cpu, wq);
2347	struct global_cwq *gcwq = cwq->pool->gcwq;
2348
2349	spin_lock_irq(&gcwq->lock);
2350
2351	if (flush_color >= 0) {
2352	BUG_ON(cwq->flush_color != -1);
2353
2354	if (cwq->nr_in_flight[flush_color]) {
2355	cwq->flush_color = flush_color;
2356	atomic_inc(&wq->nr_cwqs_to_flush);
2357	wait = true;
2358	}
2359	}
2360
2361	if (work_color >= 0) {
2362	BUG_ON(work_color != work_next_color(cwq->work_color));
2363	cwq->work_color = work_color;
2364	}
2365
2366	spin_unlock_irq(&gcwq->lock);
2367	}
2368
2369	if (flush_color >= 0 && atomic_dec_and_test(&wq->nr_cwqs_to_flush))
2370	complete(&wq->first_flusher->done);
2371
2372	return wait;
2373	}
2374
2375	/**
2376	* flush_workqueue - ensure that any scheduled work has run to completion.
2377	* @wq: workqueue to flush
2378	*
2379	* Forces execution of the workqueue and blocks until its completion.
2380	* This is typically used in driver shutdown handlers.
2381	*
2382	* We sleep until all works which were queued on entry have been handled,
2383	* but we are not livelocked by new incoming ones.
2384	*/
2385	void flush_workqueue(struct workqueue_struct *wq)
2386	{
2387	struct wq_flusher this_flusher = {
2388	.list = LIST_HEAD_INIT(this_flusher.list),
2389	.flush_color = -1,
2390	.done = COMPLETION_INITIALIZER_ONSTACK(this_flusher.done),
2391	};
2392	int next_color;
2393
2394	lock_map_acquire(&wq->lockdep_map);
2395	lock_map_release(&wq->lockdep_map);
2396
2397	mutex_lock(&wq->flush_mutex);
2398
2399	/*
2400	* Start-to-wait phase
2401	*/
2402	next_color = work_next_color(wq->work_color);
2403
2404	if (next_color != wq->flush_color) {
2405	/*
2406	* Color space is not full. The current work_color
2407	* becomes our flush_color and work_color is advanced
2408	* by one.
2409	*/
2410	BUG_ON(!list_empty(&wq->flusher_overflow));
2411	this_flusher.flush_color = wq->work_color;
2412	wq->work_color = next_color;
2413
2414	if (!wq->first_flusher) {
2415	/* no flush in progress, become the first flusher */
2416	BUG_ON(wq->flush_color != this_flusher.flush_color);
2417
2418	wq->first_flusher = &this_flusher;
2419
2420	if (!flush_workqueue_prep_cwqs(wq, wq->flush_color,
2421	wq->work_color)) {
2422	/* nothing to flush, done */
2423	wq->flush_color = next_color;
2424	wq->first_flusher = NULL;
2425	goto out_unlock;
2426	}
2427	} else {
2428	/* wait in queue */
2429	BUG_ON(wq->flush_color == this_flusher.flush_color);
2430	list_add_tail(&this_flusher.list, &wq->flusher_queue);
2431	flush_workqueue_prep_cwqs(wq, -1, wq->work_color);
2432	}
2433	} else {
2434	/*
2435	* Oops, color space is full, wait on overflow queue.
2436	* The next flush completion will assign us
2437	* flush_color and transfer to flusher_queue.
2438	*/
2439	list_add_tail(&this_flusher.list, &wq->flusher_overflow);
2440	}
2441
2442	mutex_unlock(&wq->flush_mutex);
2443
2444	wait_for_completion(&this_flusher.done);
2445
2446	/*
2447	* Wake-up-and-cascade phase
2448	*
2449	* First flushers are responsible for cascading flushes and
2450	* handling overflow. Non-first flushers can simply return.
2451	*/
2452	if (wq->first_flusher != &this_flusher)
2453	return;
2454
2455	mutex_lock(&wq->flush_mutex);
2456
2457	/* we might have raced, check again with mutex held */
2458	if (wq->first_flusher != &this_flusher)
2459	goto out_unlock;
2460
2461	wq->first_flusher = NULL;
2462
2463	BUG_ON(!list_empty(&this_flusher.list));
2464	BUG_ON(wq->flush_color != this_flusher.flush_color);
2465
2466	while (true) {
2467	struct wq_flusher next, tmp;
2468
2469	/* complete all the flushers sharing the current flush color */
2470	list_for_each_entry_safe(next, tmp, &wq->flusher_queue, list) {
2471	if (next->flush_color != wq->flush_color)
2472	break;
2473	list_del_init(&next->list);
2474	complete(&next->done);
2475	}
2476
2477	BUG_ON(!list_empty(&wq->flusher_overflow) &&
2478	wq->flush_color != work_next_color(wq->work_color));
2479
2480	/* this flush_color is finished, advance by one */
2481	wq->flush_color = work_next_color(wq->flush_color);
2482
2483	/* one color has been freed, handle overflow queue */
2484	if (!list_empty(&wq->flusher_overflow)) {
2485	/*
2486	* Assign the same color to all overflowed
2487	* flushers, advance work_color and append to
2488	* flusher_queue. This is the start-to-wait
2489	* phase for these overflowed flushers.
2490	*/
2491	list_for_each_entry(tmp, &wq->flusher_overflow, list)
2492	tmp->flush_color = wq->work_color;
2493
2494	wq->work_color = work_next_color(wq->work_color);
2495
2496	list_splice_tail_init(&wq->flusher_overflow,
2497	&wq->flusher_queue);
2498	flush_workqueue_prep_cwqs(wq, -1, wq->work_color);
2499	}
2500
2501	if (list_empty(&wq->flusher_queue)) {
2502	BUG_ON(wq->flush_color != wq->work_color);
2503	break;
2504	}
2505
2506	/*
2507	* Need to flush more colors. Make the next flusher
2508	* the new first flusher and arm cwqs.
2509	*/
2510	BUG_ON(wq->flush_color == wq->work_color);
2511	BUG_ON(wq->flush_color != next->flush_color);
2512
2513	list_del_init(&next->list);
2514	wq->first_flusher = next;
2515
2516	if (flush_workqueue_prep_cwqs(wq, wq->flush_color, -1))
2517	break;
2518
2519	/*
2520	* Meh... this color is already done, clear first
2521	* flusher and repeat cascading.
2522	*/
2523	wq->first_flusher = NULL;
2524	}
2525
2526	out_unlock:
2527	mutex_unlock(&wq->flush_mutex);
2528	}
2529	EXPORT_SYMBOL_GPL(flush_workqueue);
2530
2531	/**
2532	* drain_workqueue - drain a workqueue
2533	* @wq: workqueue to drain
2534	*
2535	* Wait until the workqueue becomes empty. While draining is in progress,
2536	* only chain queueing is allowed. IOW, only currently pending or running
2537	* work items on @wq can queue further work items on it. @wq is flushed
2538	* repeatedly until it becomes empty. The number of flushing is detemined
2539	* by the depth of chaining and should be relatively short. Whine if it
2540	* takes too long.
2541	*/
2542	void drain_workqueue(struct workqueue_struct *wq)
2543	{
2544	unsigned int flush_cnt = 0;
2545	unsigned int cpu;
2546
2547	/*
2548	* __queue_work() needs to test whether there are drainers, is much
2549	* hotter than drain_workqueue() and already looks at @wq->flags.
2550	* Use WQ_DRAINING so that queue doesn't have to check nr_drainers.
2551	*/
2552	spin_lock(&workqueue_lock);
2553	if (!wq->nr_drainers++)
2554	wq->flags \|= WQ_DRAINING;
2555	spin_unlock(&workqueue_lock);
2556	reflush:
2557	flush_workqueue(wq);
2558
2559	for_each_cwq_cpu(cpu, wq) {
2560	struct cpu_workqueue_struct *cwq = get_cwq(cpu, wq);
2561	bool drained;
2562
2563	spin_lock_irq(&cwq->pool->gcwq->lock);
2564	drained = !cwq->nr_active && list_empty(&cwq->delayed_works);
2565	spin_unlock_irq(&cwq->pool->gcwq->lock);
2566
2567	if (drained)
2568	continue;
2569
2570	if (++flush_cnt == 10 \|\|
2571	(flush_cnt % 100 == 0 && flush_cnt <= 1000))
2572	pr_warning("workqueue %s: flush on destruction isn't complete after %u tries\n",
2573	wq->name, flush_cnt);
2574	goto reflush;
2575	}
2576
2577	spin_lock(&workqueue_lock);
2578	if (!--wq->nr_drainers)
2579	wq->flags &= ~WQ_DRAINING;
2580	spin_unlock(&workqueue_lock);
2581	}
2582	EXPORT_SYMBOL_GPL(drain_workqueue);
2583
2584	static bool start_flush_work(struct work_struct work, struct wq_barrier barr,
2585	bool wait_executing)
2586	{
2587	struct worker *worker = NULL;
2588	struct global_cwq *gcwq;
2589	struct cpu_workqueue_struct *cwq;
2590
2591	might_sleep();
2592	gcwq = get_work_gcwq(work);
2593	if (!gcwq)
2594	return false;
2595
2596	spin_lock_irq(&gcwq->lock);
2597	if (!list_empty(&work->entry)) {
2598	/*
2599	* See the comment near try_to_grab_pending()->smp_rmb().
2600	* If it was re-queued to a different gcwq under us, we
2601	* are not going to wait.
2602	*/
2603	smp_rmb();
2604	cwq = get_work_cwq(work);
2605	if (unlikely(!cwq \|\| gcwq != cwq->pool->gcwq))
2606	goto already_gone;
2607	} else if (wait_executing) {
2608	worker = find_worker_executing_work(gcwq, work);
2609	if (!worker)
2610	goto already_gone;
2611	cwq = worker->current_cwq;
2612	} else
2613	goto already_gone;
2614
2615	insert_wq_barrier(cwq, barr, work, worker);
2616	spin_unlock_irq(&gcwq->lock);
2617
2618	/*
2619	* If @max_active is 1 or rescuer is in use, flushing another work
2620	* item on the same workqueue may lead to deadlock. Make sure the
2621	* flusher is not running on the same workqueue by verifying write
2622	* access.
2623	*/
2624	if (cwq->wq->saved_max_active == 1 \|\| cwq->wq->flags & WQ_RESCUER)
2625	lock_map_acquire(&cwq->wq->lockdep_map);
2626	else
2627	lock_map_acquire_read(&cwq->wq->lockdep_map);
2628	lock_map_release(&cwq->wq->lockdep_map);
2629
2630	return true;
2631	already_gone:
2632	spin_unlock_irq(&gcwq->lock);
2633	return false;
2634	}
2635
2636	/**
2637	* flush_work - wait for a work to finish executing the last queueing instance
2638	* @work: the work to flush
2639	*
2640	* Wait until @work has finished execution. This function considers
2641	* only the last queueing instance of @work. If @work has been
2642	* enqueued across different CPUs on a non-reentrant workqueue or on
2643	* multiple workqueues, @work might still be executing on return on
2644	* some of the CPUs from earlier queueing.
2645	*
2646	* If @work was queued only on a non-reentrant, ordered or unbound
2647	* workqueue, @work is guaranteed to be idle on return if it hasn't
2648	* been requeued since flush started.
2649	*
2650	* RETURNS:
2651	* %true if flush_work() waited for the work to finish execution,
2652	* %false if it was already idle.
2653	*/
2654	bool flush_work(struct work_struct *work)
2655	{
2656	struct wq_barrier barr;
2657
2658	lock_map_acquire(&work->lockdep_map);
2659	lock_map_release(&work->lockdep_map);
2660
2661	if (start_flush_work(work, &barr, true)) {
2662	wait_for_completion(&barr.done);
2663	destroy_work_on_stack(&barr.work);
2664	return true;
2665	} else
2666	return false;
2667	}
2668	EXPORT_SYMBOL_GPL(flush_work);
2669
2670	static bool wait_on_cpu_work(struct global_cwq gcwq, struct work_struct work)
2671	{
2672	struct wq_barrier barr;
2673	struct worker *worker;
2674
2675	spin_lock_irq(&gcwq->lock);
2676
2677	worker = find_worker_executing_work(gcwq, work);
2678	if (unlikely(worker))
2679	insert_wq_barrier(worker->current_cwq, &barr, work, worker);
2680
2681	spin_unlock_irq(&gcwq->lock);
2682
2683	if (unlikely(worker)) {
2684	wait_for_completion(&barr.done);
2685	destroy_work_on_stack(&barr.work);
2686	return true;
2687	} else
2688	return false;
2689	}
2690
2691	static bool wait_on_work(struct work_struct *work)
2692	{
2693	bool ret = false;
2694	int cpu;
2695
2696	might_sleep();
2697
2698	lock_map_acquire(&work->lockdep_map);
2699	lock_map_release(&work->lockdep_map);
2700
2701	for_each_gcwq_cpu(cpu)
2702	ret \|= wait_on_cpu_work(get_gcwq(cpu), work);
2703	return ret;
2704	}
2705
2706	/**
2707	* flush_work_sync - wait until a work has finished execution
2708	* @work: the work to flush
2709	*
2710	* Wait until @work has finished execution. On return, it's
2711	* guaranteed that all queueing instances of @work which happened
2712	* before this function is called are finished. In other words, if
2713	* @work hasn't been requeued since this function was called, @work is
2714	* guaranteed to be idle on return.
2715	*
2716	* RETURNS:
2717	* %true if flush_work_sync() waited for the work to finish execution,
2718	* %false if it was already idle.
2719	*/
2720	bool flush_work_sync(struct work_struct *work)
2721	{
2722	struct wq_barrier barr;
2723	bool pending, waited;
2724
2725	/* we'll wait for executions separately, queue barr only if pending */
2726	pending = start_flush_work(work, &barr, false);
2727
2728	/* wait for executions to finish */
2729	waited = wait_on_work(work);
2730
2731	/* wait for the pending one */
2732	if (pending) {
2733	wait_for_completion(&barr.done);
2734	destroy_work_on_stack(&barr.work);
2735	}
2736
2737	return pending \|\| waited;
2738	}
2739	EXPORT_SYMBOL_GPL(flush_work_sync);
2740
2741	/*
2742	* Upon a successful return (>= 0), the caller "owns" WORK_STRUCT_PENDING bit,
2743	* so this work can't be re-armed in any way.
2744	*/
2745	static int try_to_grab_pending(struct work_struct *work)
2746	{
2747	struct global_cwq *gcwq;
2748	int ret = -1;
2749
2750	if (!test_and_set_bit(WORK_STRUCT_PENDING_BIT, work_data_bits(work)))
2751	return 0;
2752
2753	/*
2754	* The queueing is in progress, or it is already queued. Try to
2755	* steal it from ->worklist without clearing WORK_STRUCT_PENDING.
2756	*/
2757	gcwq = get_work_gcwq(work);
2758	if (!gcwq)
2759	return ret;
2760
2761	spin_lock_irq(&gcwq->lock);
2762	if (!list_empty(&work->entry)) {
2763	/*
2764	* This work is queued, but perhaps we locked the wrong gcwq.
2765	* In that case we must see the new value after rmb(), see
2766	* insert_work()->wmb().
2767	*/
2768	smp_rmb();
2769	if (gcwq == get_work_gcwq(work)) {
2770	debug_work_deactivate(work);
2771	list_del_init(&work->entry);
2772	cwq_dec_nr_in_flight(get_work_cwq(work),
2773	get_work_color(work),
2774	*work_data_bits(work) & WORK_STRUCT_DELAYED);
2775	ret = 1;
2776	}
2777	}
2778	spin_unlock_irq(&gcwq->lock);
2779
2780	return ret;
2781	}
2782
2783	static bool __cancel_work_timer(struct work_struct *work,
2784	struct timer_list* timer)
2785	{
2786	int ret;
2787
2788	do {
2789	ret = (timer && likely(del_timer(timer)));
2790	if (!ret)
2791	ret = try_to_grab_pending(work);
2792	wait_on_work(work);
2793	} while (unlikely(ret < 0));
2794
2795	clear_work_data(work);
2796	return ret;
2797	}
2798
2799	/**
2800	* cancel_work_sync - cancel a work and wait for it to finish
2801	* @work: the work to cancel
2802	*
2803	* Cancel @work and wait for its execution to finish. This function
2804	* can be used even if the work re-queues itself or migrates to
2805	* another workqueue. On return from this function, @work is
2806	* guaranteed to be not pending or executing on any CPU.
2807	*
2808	* cancel_work_sync(&delayed_work->work) must not be used for
2809	* delayed_work's. Use cancel_delayed_work_sync() instead.
2810	*
2811	* The caller must ensure that the workqueue on which @work was last
2812	* queued can't be destroyed before this function returns.
2813	*
2814	* RETURNS:
2815	* %true if @work was pending, %false otherwise.
2816	*/
2817	bool cancel_work_sync(struct work_struct *work)
2818	{
2819	return __cancel_work_timer(work, NULL);
2820	}
2821	EXPORT_SYMBOL_GPL(cancel_work_sync);
2822
2823	/**
2824	* flush_delayed_work - wait for a dwork to finish executing the last queueing
2825	* @dwork: the delayed work to flush
2826	*
2827	* Delayed timer is cancelled and the pending work is queued for
2828	* immediate execution. Like flush_work(), this function only
2829	* considers the last queueing instance of @dwork.
2830	*
2831	* RETURNS:
2832	* %true if flush_work() waited for the work to finish execution,
2833	* %false if it was already idle.
2834	*/
2835	bool flush_delayed_work(struct delayed_work *dwork)
2836	{
2837	if (del_timer_sync(&dwork->timer))
2838	__queue_work(raw_smp_processor_id(),
2839	get_work_cwq(&dwork->work)->wq, &dwork->work);
2840	return flush_work(&dwork->work);
2841	}
2842	EXPORT_SYMBOL(flush_delayed_work);
2843
2844	/**
2845	* flush_delayed_work_sync - wait for a dwork to finish
2846	* @dwork: the delayed work to flush
2847	*
2848	* Delayed timer is cancelled and the pending work is queued for
2849	* execution immediately. Other than timer handling, its behavior
2850	* is identical to flush_work_sync().
2851	*
2852	* RETURNS:
2853	* %true if flush_work_sync() waited for the work to finish execution,
2854	* %false if it was already idle.
2855	*/
2856	bool flush_delayed_work_sync(struct delayed_work *dwork)
2857	{
2858	if (del_timer_sync(&dwork->timer))
2859	__queue_work(raw_smp_processor_id(),
2860	get_work_cwq(&dwork->work)->wq, &dwork->work);
2861	return flush_work_sync(&dwork->work);
2862	}
2863	EXPORT_SYMBOL(flush_delayed_work_sync);
2864
2865	/**
2866	* cancel_delayed_work_sync - cancel a delayed work and wait for it to finish
2867	* @dwork: the delayed work cancel
2868	*
2869	* This is cancel_work_sync() for delayed works.
2870	*
2871	* RETURNS:
2872	* %true if @dwork was pending, %false otherwise.
2873	*/
2874	bool cancel_delayed_work_sync(struct delayed_work *dwork)
2875	{
2876	return __cancel_work_timer(&dwork->work, &dwork->timer);
2877	}
2878	EXPORT_SYMBOL(cancel_delayed_work_sync);
2879
2880	/**
2881	* schedule_work - put work task in global workqueue
2882	* @work: job to be done
2883	*
2884	* Returns zero if @work was already on the kernel-global workqueue and
2885	* non-zero otherwise.
2886	*
2887	* This puts a job in the kernel-global workqueue if it was not already
2888	* queued and leaves it in the same position on the kernel-global
2889	* workqueue otherwise.
2890	*/
2891	int schedule_work(struct work_struct *work)
2892	{
2893	return queue_work(system_wq, work);
2894	}
2895	EXPORT_SYMBOL(schedule_work);
2896
2897	/*
2898	* schedule_work_on - put work task on a specific cpu
2899	* @cpu: cpu to put the work task on
2900	* @work: job to be done
2901	*
2902	* This puts a job on a specific cpu
2903	*/
2904	int schedule_work_on(int cpu, struct work_struct *work)
2905	{
2906	return queue_work_on(cpu, system_wq, work);
2907	}
2908	EXPORT_SYMBOL(schedule_work_on);
2909
2910	/**
2911	* schedule_delayed_work - put work task in global workqueue after delay
2912	* @dwork: job to be done
2913	* @delay: number of jiffies to wait or 0 for immediate execution
2914	*
2915	* After waiting for a given time this puts a job in the kernel-global
2916	* workqueue.
2917	*/
2918	int schedule_delayed_work(struct delayed_work *dwork,
2919	unsigned long delay)
2920	{
2921	return queue_delayed_work(system_wq, dwork, delay);
2922	}
2923	EXPORT_SYMBOL(schedule_delayed_work);
2924
2925	/**
2926	* schedule_delayed_work_on - queue work in global workqueue on CPU after delay
2927	* @cpu: cpu to use
2928	* @dwork: job to be done
2929	* @delay: number of jiffies to wait
2930	*
2931	* After waiting for a given time this puts a job in the kernel-global
2932	* workqueue on the specified CPU.
2933	*/
2934	int schedule_delayed_work_on(int cpu,
2935	struct delayed_work *dwork, unsigned long delay)
2936	{
2937	return queue_delayed_work_on(cpu, system_wq, dwork, delay);
2938	}
2939	EXPORT_SYMBOL(schedule_delayed_work_on);
2940
2941	/**
2942	* schedule_on_each_cpu - execute a function synchronously on each online CPU
2943	* @func: the function to call
2944	*
2945	* schedule_on_each_cpu() executes @func on each online CPU using the
2946	* system workqueue and blocks until all CPUs have completed.
2947	* schedule_on_each_cpu() is very slow.
2948	*
2949	* RETURNS:
2950	* 0 on success, -errno on failure.
2951	*/
2952	int schedule_on_each_cpu(work_func_t func)
2953	{
2954	int cpu;
2955	struct work_struct __percpu *works;
2956
2957	works = alloc_percpu(struct work_struct);
2958	if (!works)
2959	return -ENOMEM;
2960
2961	get_online_cpus();
2962
2963	for_each_online_cpu(cpu) {
2964	struct work_struct *work = per_cpu_ptr(works, cpu);
2965
2966	INIT_WORK(work, func);
2967	schedule_work_on(cpu, work);
2968	}
2969
2970	for_each_online_cpu(cpu)
2971	flush_work(per_cpu_ptr(works, cpu));
2972
2973	put_online_cpus();
2974	free_percpu(works);
2975	return 0;
2976	}
2977
2978	/**
2979	* flush_scheduled_work - ensure that any scheduled work has run to completion.
2980	*
2981	* Forces execution of the kernel-global workqueue and blocks until its
2982	* completion.
2983	*
2984	* Think twice before calling this function! It's very easy to get into
2985	* trouble if you don't take great care. Either of the following situations
2986	* will lead to deadlock:
2987	*
2988	* One of the work items currently on the workqueue needs to acquire
2989	* a lock held by your code or its caller.
2990	*
2991	* Your code is running in the context of a work routine.
2992	*
2993	* They will be detected by lockdep when they occur, but the first might not
2994	* occur very often. It depends on what work items are on the workqueue and
2995	* what locks they need, which you have no control over.
2996	*
2997	* In most situations flushing the entire workqueue is overkill; you merely
2998	* need to know that a particular work item isn't queued and isn't running.
2999	* In such cases you should use cancel_delayed_work_sync() or
3000	* cancel_work_sync() instead.
3001	*/
3002	void flush_scheduled_work(void)
3003	{
3004	flush_workqueue(system_wq);
3005	}
3006	EXPORT_SYMBOL(flush_scheduled_work);
3007
3008	/**
3009	* execute_in_process_context - reliably execute the routine with user context
3010	* @fn: the function to execute
3011	* @ew: guaranteed storage for the execute work structure (must
3012	* be available when the work executes)
3013	*
3014	* Executes the function immediately if process context is available,
3015	* otherwise schedules the function for delayed execution.
3016	*
3017	* Returns: 0 - function was executed
3018	* 1 - function was scheduled for execution
3019	*/
3020	int execute_in_process_context(work_func_t fn, struct execute_work *ew)
3021	{
3022	if (!in_interrupt()) {
3023	fn(&ew->work);
3024	return 0;
3025	}
3026
3027	INIT_WORK(&ew->work, fn);
3028	schedule_work(&ew->work);
3029
3030	return 1;
3031	}
3032	EXPORT_SYMBOL_GPL(execute_in_process_context);
3033
3034	int keventd_up(void)
3035	{
3036	return system_wq != NULL;
3037	}
3038
3039	static int alloc_cwqs(struct workqueue_struct *wq)
3040	{
3041	/*
3042	* cwqs are forced aligned according to WORK_STRUCT_FLAG_BITS.
3043	* Make sure that the alignment isn't lower than that of
3044	* unsigned long long.
3045	*/
3046	const size_t size = sizeof(struct cpu_workqueue_struct);
3047	const size_t align = max_t(size_t, 1 << WORK_STRUCT_FLAG_BITS,
3048	__alignof__(unsigned long long));
3049
3050	if (!(wq->flags & WQ_UNBOUND))
3051	wq->cpu_wq.pcpu = __alloc_percpu(size, align);
3052	else {
3053	void *ptr;
3054
3055	/*
3056	* Allocate enough room to align cwq and put an extra
3057	* pointer at the end pointing back to the originally
3058	* allocated pointer which will be used for free.
3059	*/
3060	ptr = kzalloc(size + align + sizeof(void *), GFP_KERNEL);
3061	if (ptr) {
3062	wq->cpu_wq.single = PTR_ALIGN(ptr, align);
3063	(void *)(wq->cpu_wq.single + 1) = ptr;
3064	}
3065	}
3066
3067	/* just in case, make sure it's actually aligned */
3068	BUG_ON(!IS_ALIGNED(wq->cpu_wq.v, align));
3069	return wq->cpu_wq.v ? 0 : -ENOMEM;
3070	}
3071
3072	static void free_cwqs(struct workqueue_struct *wq)
3073	{
3074	if (!(wq->flags & WQ_UNBOUND))
3075	free_percpu(wq->cpu_wq.pcpu);
3076	else if (wq->cpu_wq.single) {
3077	/* the pointer to free is stored right after the cwq */
3078	kfree((void *)(wq->cpu_wq.single + 1));
3079	}
3080	}
3081
3082	static int wq_clamp_max_active(int max_active, unsigned int flags,
3083	const char *name)
3084	{
3085	int lim = flags & WQ_UNBOUND ? WQ_UNBOUND_MAX_ACTIVE : WQ_MAX_ACTIVE;
3086
3087	if (max_active < 1 \|\| max_active > lim)
3088	printk(KERN_WARNING "workqueue: max_active %d requested for %s "
3089	"is out of range, clamping between %d and %d\n",
3090	max_active, name, 1, lim);
3091
3092	return clamp_val(max_active, 1, lim);
3093	}
3094
3095	struct workqueue_struct __alloc_workqueue_key(const char fmt,
3096	unsigned int flags,
3097	int max_active,
3098	struct lock_class_key *key,
3099	const char *lock_name, ...)
3100	{
3101	va_list args, args1;
3102	struct workqueue_struct *wq;
3103	unsigned int cpu;
3104	size_t namelen;
3105
3106	/* determine namelen, allocate wq and format name */
3107	va_start(args, lock_name);
3108	va_copy(args1, args);
3109	namelen = vsnprintf(NULL, 0, fmt, args) + 1;
3110
3111	wq = kzalloc(sizeof(*wq) + namelen, GFP_KERNEL);
3112	if (!wq)
3113	goto err;
3114
3115	vsnprintf(wq->name, namelen, fmt, args1);
3116	va_end(args);
3117	va_end(args1);
3118
3119	/*
3120	* Workqueues which may be used during memory reclaim should
3121	* have a rescuer to guarantee forward progress.
3122	*/
3123	if (flags & WQ_MEM_RECLAIM)
3124	flags \|= WQ_RESCUER;
3125
3126	max_active = max_active ?: WQ_DFL_ACTIVE;
3127	max_active = wq_clamp_max_active(max_active, flags, wq->name);
3128
3129	/* init wq */
3130	wq->flags = flags;
3131	wq->saved_max_active = max_active;
3132	mutex_init(&wq->flush_mutex);
3133	atomic_set(&wq->nr_cwqs_to_flush, 0);
3134	INIT_LIST_HEAD(&wq->flusher_queue);
3135	INIT_LIST_HEAD(&wq->flusher_overflow);
3136
3137	lockdep_init_map(&wq->lockdep_map, lock_name, key, 0);
3138	INIT_LIST_HEAD(&wq->list);
3139
3140	if (alloc_cwqs(wq) < 0)
3141	goto err;
3142
3143	for_each_cwq_cpu(cpu, wq) {
3144	struct cpu_workqueue_struct *cwq = get_cwq(cpu, wq);
3145	struct global_cwq *gcwq = get_gcwq(cpu);
3146	int pool_idx = (bool)(flags & WQ_HIGHPRI);
3147
3148	BUG_ON((unsigned long)cwq & WORK_STRUCT_FLAG_MASK);
3149	cwq->pool = &gcwq->pools[pool_idx];
3150	cwq->wq = wq;
3151	cwq->flush_color = -1;
3152	cwq->max_active = max_active;
3153	INIT_LIST_HEAD(&cwq->delayed_works);
3154	}
3155
3156	if (flags & WQ_RESCUER) {
3157	struct worker *rescuer;
3158
3159	if (!alloc_mayday_mask(&wq->mayday_mask, GFP_KERNEL))
3160	goto err;
3161
3162	wq->rescuer = rescuer = alloc_worker();
3163	if (!rescuer)
3164	goto err;
3165
3166	rescuer->task = kthread_create(rescuer_thread, wq, "%s",
3167	wq->name);
3168	if (IS_ERR(rescuer->task))
3169	goto err;
3170
3171	rescuer->task->flags \|= PF_THREAD_BOUND;
3172	wake_up_process(rescuer->task);
3173	}
3174
3175	/*
3176	* workqueue_lock protects global freeze state and workqueues
3177	* list. Grab it, set max_active accordingly and add the new
3178	* workqueue to workqueues list.
3179	*/
3180	spin_lock(&workqueue_lock);
3181
3182	if (workqueue_freezing && wq->flags & WQ_FREEZABLE)
3183	for_each_cwq_cpu(cpu, wq)
3184	get_cwq(cpu, wq)->max_active = 0;
3185
3186	list_add(&wq->list, &workqueues);
3187
3188	spin_unlock(&workqueue_lock);
3189
3190	return wq;
3191	err:
3192	if (wq) {
3193	free_cwqs(wq);
3194	free_mayday_mask(wq->mayday_mask);
3195	kfree(wq->rescuer);
3196	kfree(wq);
3197	}
3198	return NULL;
3199	}
3200	EXPORT_SYMBOL_GPL(__alloc_workqueue_key);
3201
3202	/**
3203	* destroy_workqueue - safely terminate a workqueue
3204	* @wq: target workqueue
3205	*
3206	* Safely destroy a workqueue. All work currently pending will be done first.
3207	*/
3208	void destroy_workqueue(struct workqueue_struct *wq)
3209	{
3210	unsigned int cpu;
3211
3212	/* drain it before proceeding with destruction */
3213	drain_workqueue(wq);
3214
3215	/*
3216	* wq list is used to freeze wq, remove from list after
3217	* flushing is complete in case freeze races us.
3218	*/
3219	spin_lock(&workqueue_lock);
3220	list_del(&wq->list);
3221	spin_unlock(&workqueue_lock);
3222
3223	/* sanity check */
3224	for_each_cwq_cpu(cpu, wq) {
3225	struct cpu_workqueue_struct *cwq = get_cwq(cpu, wq);
3226	int i;
3227
3228	for (i = 0; i < WORK_NR_COLORS; i++)
3229	BUG_ON(cwq->nr_in_flight[i]);
3230	BUG_ON(cwq->nr_active);
3231	BUG_ON(!list_empty(&cwq->delayed_works));
3232	}
3233
3234	if (wq->flags & WQ_RESCUER) {
3235	kthread_stop(wq->rescuer->task);
3236	free_mayday_mask(wq->mayday_mask);
3237	kfree(wq->rescuer);
3238	}
3239
3240	free_cwqs(wq);
3241	kfree(wq);
3242	}
3243	EXPORT_SYMBOL_GPL(destroy_workqueue);
3244
3245	/**
3246	* workqueue_set_max_active - adjust max_active of a workqueue
3247	* @wq: target workqueue
3248	* @max_active: new max_active value.
3249	*
3250	* Set max_active of @wq to @max_active.
3251	*
3252	* CONTEXT:
3253	* Don't call from IRQ context.
3254	*/
3255	void workqueue_set_max_active(struct workqueue_struct *wq, int max_active)
3256	{
3257	unsigned int cpu;
3258
3259	max_active = wq_clamp_max_active(max_active, wq->flags, wq->name);
3260
3261	spin_lock(&workqueue_lock);
3262
3263	wq->saved_max_active = max_active;
3264
3265	for_each_cwq_cpu(cpu, wq) {
3266	struct global_cwq *gcwq = get_gcwq(cpu);
3267
3268	spin_lock_irq(&gcwq->lock);
3269
3270	if (!(wq->flags & WQ_FREEZABLE) \|\|
3271	!(gcwq->flags & GCWQ_FREEZING))
3272	get_cwq(gcwq->cpu, wq)->max_active = max_active;
3273
3274	spin_unlock_irq(&gcwq->lock);
3275	}
3276
3277	spin_unlock(&workqueue_lock);
3278	}
3279	EXPORT_SYMBOL_GPL(workqueue_set_max_active);
3280
3281	/**
3282	* workqueue_congested - test whether a workqueue is congested
3283	* @cpu: CPU in question
3284	* @wq: target workqueue
3285	*
3286	* Test whether @wq's cpu workqueue for @cpu is congested. There is
3287	* no synchronization around this function and the test result is
3288	* unreliable and only useful as advisory hints or for debugging.
3289	*
3290	* RETURNS:
3291	* %true if congested, %false otherwise.
3292	*/
3293	bool workqueue_congested(unsigned int cpu, struct workqueue_struct *wq)
3294	{
3295	struct cpu_workqueue_struct *cwq = get_cwq(cpu, wq);
3296
3297	return !list_empty(&cwq->delayed_works);
3298	}
3299	EXPORT_SYMBOL_GPL(workqueue_congested);
3300
3301	/**
3302	* work_cpu - return the last known associated cpu for @work
3303	* @work: the work of interest
3304	*
3305	* RETURNS:
3306	* CPU number if @work was ever queued. WORK_CPU_NONE otherwise.
3307	*/
3308	unsigned int work_cpu(struct work_struct *work)
3309	{
3310	struct global_cwq *gcwq = get_work_gcwq(work);
3311
3312	return gcwq ? gcwq->cpu : WORK_CPU_NONE;
3313	}
3314	EXPORT_SYMBOL_GPL(work_cpu);
3315
3316	/**
3317	* work_busy - test whether a work is currently pending or running
3318	* @work: the work to be tested
3319	*
3320	* Test whether @work is currently pending or running. There is no
3321	* synchronization around this function and the test result is
3322	* unreliable and only useful as advisory hints or for debugging.
3323	* Especially for reentrant wqs, the pending state might hide the
3324	* running state.
3325	*
3326	* RETURNS:
3327	* OR'd bitmask of WORK_BUSY_* bits.
3328	*/
3329	unsigned int work_busy(struct work_struct *work)
3330	{
3331	struct global_cwq *gcwq = get_work_gcwq(work);
3332	unsigned long flags;
3333	unsigned int ret = 0;
3334
3335	if (!gcwq)
3336	return false;
3337
3338	spin_lock_irqsave(&gcwq->lock, flags);
3339
3340	if (work_pending(work))
3341	ret \|= WORK_BUSY_PENDING;
3342	if (find_worker_executing_work(gcwq, work))
3343	ret \|= WORK_BUSY_RUNNING;
3344
3345	spin_unlock_irqrestore(&gcwq->lock, flags);
3346
3347	return ret;
3348	}
3349	EXPORT_SYMBOL_GPL(work_busy);
3350
3351	/*
3352	* CPU hotplug.
3353	*
3354	* There are two challenges in supporting CPU hotplug. Firstly, there
3355	* are a lot of assumptions on strong associations among work, cwq and
3356	* gcwq which make migrating pending and scheduled works very
3357	* difficult to implement without impacting hot paths. Secondly,
3358	* gcwqs serve mix of short, long and very long running works making
3359	* blocked draining impractical.
3360	*
3361	* This is solved by allowing a gcwq to be disassociated from the CPU
3362	* running as an unbound one and allowing it to be reattached later if the
3363	* cpu comes back online.
3364	*/
3365
3366	/* claim manager positions of all pools */
3367	static void gcwq_claim_management_and_lock(struct global_cwq *gcwq)
3368	{
3369	struct worker_pool *pool;
3370
3371	for_each_worker_pool(pool, gcwq)
3372	mutex_lock_nested(&pool->manager_mutex, pool - gcwq->pools);
3373	spin_lock_irq(&gcwq->lock);
3374	}
3375
3376	/* release manager positions */
3377	static void gcwq_release_management_and_unlock(struct global_cwq *gcwq)
3378	{
3379	struct worker_pool *pool;
3380
3381	spin_unlock_irq(&gcwq->lock);
3382	for_each_worker_pool(pool, gcwq)
3383	mutex_unlock(&pool->manager_mutex);
3384	}
3385
3386	static void gcwq_unbind_fn(struct work_struct *work)
3387	{
3388	struct global_cwq *gcwq = get_gcwq(smp_processor_id());
3389	struct worker_pool *pool;
3390	struct worker *worker;
3391	struct hlist_node *pos;
3392	int i;
3393
3394	BUG_ON(gcwq->cpu != smp_processor_id());
3395
3396	gcwq_claim_management_and_lock(gcwq);
3397
3398	/*
3399	* We've claimed all manager positions. Make all workers unbound
3400	* and set DISASSOCIATED. Before this, all workers except for the
3401	* ones which are still executing works from before the last CPU
3402	* down must be on the cpu. After this, they may become diasporas.
3403	*/
3404	for_each_worker_pool(pool, gcwq)
3405	list_for_each_entry(worker, &pool->idle_list, entry)
3406	worker->flags \|= WORKER_UNBOUND;
3407
3408	for_each_busy_worker(worker, i, pos, gcwq)
3409	worker->flags \|= WORKER_UNBOUND;
3410
3411	gcwq->flags \|= GCWQ_DISASSOCIATED;
3412
3413	gcwq_release_management_and_unlock(gcwq);
3414
3415	/*
3416	* Call schedule() so that we cross rq->lock and thus can guarantee
3417	* sched callbacks see the %WORKER_UNBOUND flag. This is necessary
3418	* as scheduler callbacks may be invoked from other cpus.
3419	*/
3420	schedule();
3421
3422	/*
3423	* Sched callbacks are disabled now. Zap nr_running. After this,
3424	* nr_running stays zero and need_more_worker() and keep_working()
3425	* are always true as long as the worklist is not empty. @gcwq now
3426	* behaves as unbound (in terms of concurrency management) gcwq
3427	* which is served by workers tied to the CPU.
3428	*
3429	* On return from this function, the current worker would trigger
3430	* unbound chain execution of pending work items if other workers
3431	* didn't already.
3432	*/
3433	for_each_worker_pool(pool, gcwq)
3434	atomic_set(get_pool_nr_running(pool), 0);
3435	}
3436
3437	/*
3438	* Workqueues should be brought up before normal priority CPU notifiers.
3439	* This will be registered high priority CPU notifier.
3440	*/
3441	static int __devinit workqueue_cpu_up_callback(struct notifier_block *nfb,
3442	unsigned long action,
3443	void *hcpu)
3444	{
3445	unsigned int cpu = (unsigned long)hcpu;
3446	struct global_cwq *gcwq = get_gcwq(cpu);
3447	struct worker_pool *pool;
3448
3449	switch (action & ~CPU_TASKS_FROZEN) {
3450	case CPU_UP_PREPARE:
3451	for_each_worker_pool(pool, gcwq) {
3452	struct worker *worker;
3453
3454	if (pool->nr_workers)
3455	continue;
3456
3457	worker = create_worker(pool);
3458	if (!worker)
3459	return NOTIFY_BAD;
3460
3461	spin_lock_irq(&gcwq->lock);
3462	start_worker(worker);
3463	spin_unlock_irq(&gcwq->lock);
3464	}
3465	break;
3466
3467	case CPU_DOWN_FAILED:
3468	case CPU_ONLINE:
3469	gcwq_claim_management_and_lock(gcwq);
3470	gcwq->flags &= ~GCWQ_DISASSOCIATED;
3471	rebind_workers(gcwq);
3472	gcwq_release_management_and_unlock(gcwq);
3473	break;
3474	}
3475	return NOTIFY_OK;
3476	}
3477
3478	/*
3479	* Workqueues should be brought down after normal priority CPU notifiers.
3480	* This will be registered as low priority CPU notifier.
3481	*/
3482	static int __devinit workqueue_cpu_down_callback(struct notifier_block *nfb,
3483	unsigned long action,
3484	void *hcpu)
3485	{
3486	unsigned int cpu = (unsigned long)hcpu;
3487	struct work_struct unbind_work;
3488
3489	switch (action & ~CPU_TASKS_FROZEN) {
3490	case CPU_DOWN_PREPARE:
3491	/* unbinding should happen on the local CPU */
3492	INIT_WORK_ONSTACK(&unbind_work, gcwq_unbind_fn);
3493	schedule_work_on(cpu, &unbind_work);
3494	flush_work(&unbind_work);
3495	break;
3496	}
3497	return NOTIFY_OK;
3498	}
3499
3500	#ifdef CONFIG_SMP
3501
3502	struct work_for_cpu {
3503	struct completion completion;
3504	long (fn)(void );
3505	void *arg;
3506	long ret;
3507	};
3508
3509	static int do_work_for_cpu(void *_wfc)
3510	{
3511	struct work_for_cpu *wfc = _wfc;
3512	wfc->ret = wfc->fn(wfc->arg);
3513	complete(&wfc->completion);
3514	return 0;
3515	}
3516
3517	/**
3518	* work_on_cpu - run a function in user context on a particular cpu
3519	* @cpu: the cpu to run on
3520	* @fn: the function to run
3521	* @arg: the function arg
3522	*
3523	* This will return the value @fn returns.
3524	* It is up to the caller to ensure that the cpu doesn't go offline.
3525	* The caller must not hold any locks which would prevent @fn from completing.
3526	*/
3527	long work_on_cpu(unsigned int cpu, long (fn)(void ), void *arg)
3528	{
3529	struct task_struct *sub_thread;
3530	struct work_for_cpu wfc = {
3531	.completion = COMPLETION_INITIALIZER_ONSTACK(wfc.completion),
3532	.fn = fn,
3533	.arg = arg,
3534	};
3535
3536	sub_thread = kthread_create(do_work_for_cpu, &wfc, "work_for_cpu");
3537	if (IS_ERR(sub_thread))
3538	return PTR_ERR(sub_thread);
3539	kthread_bind(sub_thread, cpu);
3540	wake_up_process(sub_thread);
3541	wait_for_completion(&wfc.completion);
3542	return wfc.ret;
3543	}
3544	EXPORT_SYMBOL_GPL(work_on_cpu);
3545	#endif /* CONFIG_SMP */
3546
3547	#ifdef CONFIG_FREEZER
3548
3549	/**
3550	* freeze_workqueues_begin - begin freezing workqueues
3551	*
3552	* Start freezing workqueues. After this function returns, all freezable
3553	* workqueues will queue new works to their frozen_works list instead of
3554	* gcwq->worklist.
3555	*
3556	* CONTEXT:
3557	* Grabs and releases workqueue_lock and gcwq->lock's.
3558	*/
3559	void freeze_workqueues_begin(void)
3560	{
3561	unsigned int cpu;
3562
3563	spin_lock(&workqueue_lock);
3564
3565	BUG_ON(workqueue_freezing);
3566	workqueue_freezing = true;
3567
3568	for_each_gcwq_cpu(cpu) {
3569	struct global_cwq *gcwq = get_gcwq(cpu);
3570	struct workqueue_struct *wq;
3571
3572	spin_lock_irq(&gcwq->lock);
3573
3574	BUG_ON(gcwq->flags & GCWQ_FREEZING);
3575	gcwq->flags \|= GCWQ_FREEZING;
3576
3577	list_for_each_entry(wq, &workqueues, list) {
3578	struct cpu_workqueue_struct *cwq = get_cwq(cpu, wq);
3579
3580	if (cwq && wq->flags & WQ_FREEZABLE)
3581	cwq->max_active = 0;
3582	}
3583
3584	spin_unlock_irq(&gcwq->lock);
3585	}
3586
3587	spin_unlock(&workqueue_lock);
3588	}
3589
3590	/**
3591	* freeze_workqueues_busy - are freezable workqueues still busy?
3592	*
3593	* Check whether freezing is complete. This function must be called
3594	* between freeze_workqueues_begin() and thaw_workqueues().
3595	*
3596	* CONTEXT:
3597	* Grabs and releases workqueue_lock.
3598	*
3599	* RETURNS:
3600	* %true if some freezable workqueues are still busy. %false if freezing
3601	* is complete.
3602	*/
3603	bool freeze_workqueues_busy(void)
3604	{
3605	unsigned int cpu;
3606	bool busy = false;
3607
3608	spin_lock(&workqueue_lock);
3609
3610	BUG_ON(!workqueue_freezing);
3611
3612	for_each_gcwq_cpu(cpu) {
3613	struct workqueue_struct *wq;
3614	/*
3615	* nr_active is monotonically decreasing. It's safe
3616	* to peek without lock.
3617	*/
3618	list_for_each_entry(wq, &workqueues, list) {
3619	struct cpu_workqueue_struct *cwq = get_cwq(cpu, wq);
3620
3621	if (!cwq \|\| !(wq->flags & WQ_FREEZABLE))
3622	continue;
3623
3624	BUG_ON(cwq->nr_active < 0);
3625	if (cwq->nr_active) {
3626	busy = true;
3627	goto out_unlock;
3628	}
3629	}
3630	}
3631	out_unlock:
3632	spin_unlock(&workqueue_lock);
3633	return busy;
3634	}
3635
3636	/**
3637	* thaw_workqueues - thaw workqueues
3638	*
3639	* Thaw workqueues. Normal queueing is restored and all collected
3640	* frozen works are transferred to their respective gcwq worklists.
3641	*
3642	* CONTEXT:
3643	* Grabs and releases workqueue_lock and gcwq->lock's.
3644	*/
3645	void thaw_workqueues(void)
3646	{
3647	unsigned int cpu;
3648
3649	spin_lock(&workqueue_lock);
3650
3651	if (!workqueue_freezing)
3652	goto out_unlock;
3653
3654	for_each_gcwq_cpu(cpu) {
3655	struct global_cwq *gcwq = get_gcwq(cpu);
3656	struct worker_pool *pool;
3657	struct workqueue_struct *wq;
3658
3659	spin_lock_irq(&gcwq->lock);
3660
3661	BUG_ON(!(gcwq->flags & GCWQ_FREEZING));
3662	gcwq->flags &= ~GCWQ_FREEZING;
3663
3664	list_for_each_entry(wq, &workqueues, list) {
3665	struct cpu_workqueue_struct *cwq = get_cwq(cpu, wq);
3666
3667	if (!cwq \|\| !(wq->flags & WQ_FREEZABLE))
3668	continue;
3669
3670	/* restore max_active and repopulate worklist */
3671	cwq->max_active = wq->saved_max_active;
3672
3673	while (!list_empty(&cwq->delayed_works) &&
3674	cwq->nr_active < cwq->max_active)
3675	cwq_activate_first_delayed(cwq);
3676	}
3677
3678	for_each_worker_pool(pool, gcwq)
3679	wake_up_worker(pool);
3680
3681	spin_unlock_irq(&gcwq->lock);
3682	}
3683
3684	workqueue_freezing = false;
3685	out_unlock:
3686	spin_unlock(&workqueue_lock);
3687	}
3688	#endif /* CONFIG_FREEZER */
3689
3690	static int __init init_workqueues(void)
3691	{
3692	unsigned int cpu;
3693	int i;
3694
3695	cpu_notifier(workqueue_cpu_up_callback, CPU_PRI_WORKQUEUE_UP);
3696	cpu_notifier(workqueue_cpu_down_callback, CPU_PRI_WORKQUEUE_DOWN);
3697
3698	/* initialize gcwqs */
3699	for_each_gcwq_cpu(cpu) {
3700	struct global_cwq *gcwq = get_gcwq(cpu);
3701	struct worker_pool *pool;
3702
3703	spin_lock_init(&gcwq->lock);
3704	gcwq->cpu = cpu;
3705	gcwq->flags \|= GCWQ_DISASSOCIATED;
3706
3707	for (i = 0; i < BUSY_WORKER_HASH_SIZE; i++)
3708	INIT_HLIST_HEAD(&gcwq->busy_hash[i]);
3709
3710	for_each_worker_pool(pool, gcwq) {
3711	pool->gcwq = gcwq;
3712	INIT_LIST_HEAD(&pool->worklist);
3713	INIT_LIST_HEAD(&pool->idle_list);
3714
3715	init_timer_deferrable(&pool->idle_timer);
3716	pool->idle_timer.function = idle_worker_timeout;
3717	pool->idle_timer.data = (unsigned long)pool;
3718
3719	setup_timer(&pool->mayday_timer, gcwq_mayday_timeout,
3720	(unsigned long)pool);
3721
3722	mutex_init(&pool->manager_mutex);
3723	ida_init(&pool->worker_ida);
3724	}
3725
3726	init_waitqueue_head(&gcwq->rebind_hold);
3727	}
3728
3729	/* create the initial worker */
3730	for_each_online_gcwq_cpu(cpu) {
3731	struct global_cwq *gcwq = get_gcwq(cpu);
3732	struct worker_pool *pool;
3733
3734	if (cpu != WORK_CPU_UNBOUND)
3735	gcwq->flags &= ~GCWQ_DISASSOCIATED;
3736
3737	for_each_worker_pool(pool, gcwq) {
3738	struct worker *worker;
3739
3740	worker = create_worker(pool);
3741	BUG_ON(!worker);
3742	spin_lock_irq(&gcwq->lock);
3743	start_worker(worker);
3744	spin_unlock_irq(&gcwq->lock);
3745	}
3746	}
3747
3748	system_wq = alloc_workqueue("events", 0, 0);
3749	system_long_wq = alloc_workqueue("events_long", 0, 0);
3750	system_nrt_wq = alloc_workqueue("events_nrt", WQ_NON_REENTRANT, 0);
3751	system_unbound_wq = alloc_workqueue("events_unbound", WQ_UNBOUND,
3752	WQ_UNBOUND_MAX_ACTIVE);
3753	system_freezable_wq = alloc_workqueue("events_freezable",
3754	WQ_FREEZABLE, 0);
3755	system_nrt_freezable_wq = alloc_workqueue("events_nrt_freezable",
3756	WQ_NON_REENTRANT \| WQ_FREEZABLE, 0);
3757	BUG_ON(!system_wq \|\| !system_long_wq \|\| !system_nrt_wq \|\|
3758	!system_unbound_wq \|\| !system_freezable_wq \|\|
3759	!system_nrt_freezable_wq);
3760	return 0;
3761	}
3762	early_initcall(init_workqueues);