linux - kernel/workqueue.c

3757 lines · 101.7 KiB

1	/*
2	* kernel/workqueue.c - generic async execution with shared worker pool
3	*
4	* Copyright (C) 2002 Ingo Molnar
5	*
6	* Derived from the taskqueue/keventd code by:
7	* David Woodhouse <[email protected]>
8	* Andrew Morton
9	* Kai Petzke <[email protected]>
10	* Theodore Ts'o <[email protected]>
11	*
12	* Made to use alloc_percpu by Christoph Lameter.
13	*
14	* Copyright (C) 2010 SUSE Linux Products GmbH
15	* Copyright (C) 2010 Tejun Heo <[email protected]>
16	*
17	* This is the generic async execution mechanism. Work items as are
18	* executed in process context. The worker pool is shared and
19	* automatically managed. There is one worker pool for each CPU and
20	* one extra for works which are better served by workers which are
21	* not bound to any specific CPU.
22	*
23	* Please read Documentation/workqueue.txt for details.
24	*/
25
26	#include <linux/module.h>
27	#include <linux/kernel.h>
28	#include <linux/sched.h>
29	#include <linux/init.h>
30	#include <linux/signal.h>
31	#include <linux/completion.h>
32	#include <linux/workqueue.h>
33	#include <linux/slab.h>
34	#include <linux/cpu.h>
35	#include <linux/notifier.h>
36	#include <linux/kthread.h>
37	#include <linux/hardirq.h>
38	#include <linux/mempolicy.h>
39	#include <linux/freezer.h>
40	#include <linux/kallsyms.h>
41	#include <linux/debug_locks.h>
42	#include <linux/lockdep.h>
43	#include <linux/idr.h>
44
45	#include "workqueue_sched.h"
46
47	enum {
48	/* global_cwq flags */
49	GCWQ_MANAGE_WORKERS = 1 << 0, /* need to manage workers */
50	GCWQ_MANAGING_WORKERS = 1 << 1, /* managing workers */
51	GCWQ_DISASSOCIATED = 1 << 2, /* cpu can't serve workers */
52	GCWQ_FREEZING = 1 << 3, /* freeze in progress */
53	GCWQ_HIGHPRI_PENDING = 1 << 4, /* highpri works on queue */
54
55	/* worker flags */
56	WORKER_STARTED = 1 << 0, /* started */
57	WORKER_DIE = 1 << 1, /* die die die */
58	WORKER_IDLE = 1 << 2, /* is idle */
59	WORKER_PREP = 1 << 3, /* preparing to run works */
60	WORKER_ROGUE = 1 << 4, /* not bound to any cpu */
61	WORKER_REBIND = 1 << 5, /* mom is home, come back */
62	WORKER_CPU_INTENSIVE = 1 << 6, /* cpu intensive */
63	WORKER_UNBOUND = 1 << 7, /* worker is unbound */
64
65	WORKER_NOT_RUNNING = WORKER_PREP \| WORKER_ROGUE \| WORKER_REBIND \|
66	WORKER_CPU_INTENSIVE \| WORKER_UNBOUND,
67
68	/* gcwq->trustee_state */
69	TRUSTEE_START = 0, /* start */
70	TRUSTEE_IN_CHARGE = 1, /* trustee in charge of gcwq */
71	TRUSTEE_BUTCHER = 2, /* butcher workers */
72	TRUSTEE_RELEASE = 3, /* release workers */
73	TRUSTEE_DONE = 4, /* trustee is done */
74
75	BUSY_WORKER_HASH_ORDER = 6, /* 64 pointers */
76	BUSY_WORKER_HASH_SIZE = 1 << BUSY_WORKER_HASH_ORDER,
77	BUSY_WORKER_HASH_MASK = BUSY_WORKER_HASH_SIZE - 1,
78
79	MAX_IDLE_WORKERS_RATIO = 4, /* 1/4 of busy can be idle */
80	IDLE_WORKER_TIMEOUT = 300 * HZ, /* keep idle ones for 5 mins */
81
82	MAYDAY_INITIAL_TIMEOUT = HZ / 100, /* call for help after 10ms */
83	MAYDAY_INTERVAL = HZ / 10, /* and then every 100ms */
84	CREATE_COOLDOWN = HZ, /* time to breath after fail */
85	TRUSTEE_COOLDOWN = HZ / 10, /* for trustee draining */
86
87	/*
88	* Rescue workers are used only on emergencies and shared by
89	* all cpus. Give -20.
90	*/
91	RESCUER_NICE_LEVEL = -20,
92	};
93
94	/*
95	* Structure fields follow one of the following exclusion rules.
96	*
97	* I: Modifiable by initialization/destruction paths and read-only for
98	* everyone else.
99	*
100	* P: Preemption protected. Disabling preemption is enough and should
101	* only be modified and accessed from the local cpu.
102	*
103	* L: gcwq->lock protected. Access with gcwq->lock held.
104	*
105	* X: During normal operation, modification requires gcwq->lock and
106	* should be done only from local cpu. Either disabling preemption
107	* on local cpu or grabbing gcwq->lock is enough for read access.
108	* If GCWQ_DISASSOCIATED is set, it's identical to L.
109	*
110	* F: wq->flush_mutex protected.
111	*
112	* W: workqueue_lock protected.
113	*/
114
115	struct global_cwq;
116
117	/*
118	* The poor guys doing the actual heavy lifting. All on-duty workers
119	* are either serving the manager role, on idle list or on busy hash.
120	*/
121	struct worker {
122	/* on idle list while idle, on busy hash table while busy */
123	union {
124	struct list_head entry; /* L: while idle */
125	struct hlist_node hentry; /* L: while busy */
126	};
127
128	struct work_struct current_work; / L: work being processed */
129	struct cpu_workqueue_struct current_cwq; / L: current_work's cwq */
130	struct list_head scheduled; /* L: scheduled works */
131	struct task_struct task; / I: worker task */
132	struct global_cwq gcwq; / I: the associated gcwq */
133	/* 64 bytes boundary on 64bit, 32 on 32bit */
134	unsigned long last_active; /* L: last active timestamp */
135	unsigned int flags; /* X: flags */
136	int id; /* I: worker id */
137	struct work_struct rebind_work; /* L: rebind worker to cpu */
138	};
139
140	/*
141	* Global per-cpu workqueue. There's one and only one for each cpu
142	* and all works are queued and processed here regardless of their
143	* target workqueues.
144	*/
145	struct global_cwq {
146	spinlock_t lock; /* the gcwq lock */
147	struct list_head worklist; /* L: list of pending works */
148	unsigned int cpu; /* I: the associated cpu */
149	unsigned int flags; /* L: GCWQ_* flags */
150
151	int nr_workers; /* L: total number of workers */
152	int nr_idle; /* L: currently idle ones */
153
154	/* workers are chained either in the idle_list or busy_hash */
155	struct list_head idle_list; /* X: list of idle workers */
156	struct hlist_head busy_hash[BUSY_WORKER_HASH_SIZE];
157	/* L: hash of busy workers */
158
159	struct timer_list idle_timer; /* L: worker idle timeout */
160	struct timer_list mayday_timer; /* L: SOS timer for dworkers */
161
162	struct ida worker_ida; /* L: for worker IDs */
163
164	struct task_struct trustee; / L: for gcwq shutdown */
165	unsigned int trustee_state; /* L: trustee state */
166	wait_queue_head_t trustee_wait; /* trustee wait */
167	struct worker first_idle; / L: first idle worker */
168	} ____cacheline_aligned_in_smp;
169
170	/*
171	* The per-CPU workqueue. The lower WORK_STRUCT_FLAG_BITS of
172	* work_struct->data are used for flags and thus cwqs need to be
173	* aligned at two's power of the number of flag bits.
174	*/
175	struct cpu_workqueue_struct {
176	struct global_cwq gcwq; / I: the associated gcwq */
177	struct workqueue_struct wq; / I: the owning workqueue */
178	int work_color; /* L: current color */
179	int flush_color; /* L: flushing color */
180	int nr_in_flight[WORK_NR_COLORS];
181	/* L: nr of in_flight works */
182	int nr_active; /* L: nr of active works */
183	int max_active; /* L: max active works */
184	struct list_head delayed_works; /* L: delayed works */
185	};
186
187	/*
188	* Structure used to wait for workqueue flush.
189	*/
190	struct wq_flusher {
191	struct list_head list; /* F: list of flushers */
192	int flush_color; /* F: flush color waiting for */
193	struct completion done; /* flush completion */
194	};
195
196	/*
197	* All cpumasks are assumed to be always set on UP and thus can't be
198	* used to determine whether there's something to be done.
199	*/
200	#ifdef CONFIG_SMP
201	typedef cpumask_var_t mayday_mask_t;
202	#define mayday_test_and_set_cpu(cpu, mask) \
203	cpumask_test_and_set_cpu((cpu), (mask))
204	#define mayday_clear_cpu(cpu, mask) cpumask_clear_cpu((cpu), (mask))
205	#define for_each_mayday_cpu(cpu, mask) for_each_cpu((cpu), (mask))
206	#define alloc_mayday_mask(maskp, gfp) zalloc_cpumask_var((maskp), (gfp))
207	#define free_mayday_mask(mask) free_cpumask_var((mask))
208	#else
209	typedef unsigned long mayday_mask_t;
210	#define mayday_test_and_set_cpu(cpu, mask) test_and_set_bit(0, &(mask))
211	#define mayday_clear_cpu(cpu, mask) clear_bit(0, &(mask))
212	#define for_each_mayday_cpu(cpu, mask) if ((cpu) = 0, (mask))
213	#define alloc_mayday_mask(maskp, gfp) true
214	#define free_mayday_mask(mask) do { } while (0)
215	#endif
216
217	/*
218	* The externally visible workqueue abstraction is an array of
219	* per-CPU workqueues:
220	*/
221	struct workqueue_struct {
222	unsigned int flags; /* I: WQ_* flags */
223	union {
224	struct cpu_workqueue_struct __percpu *pcpu;
225	struct cpu_workqueue_struct *single;
226	unsigned long v;
227	} cpu_wq; /* I: cwq's */
228	struct list_head list; /* W: list of all workqueues */
229
230	struct mutex flush_mutex; /* protects wq flushing */
231	int work_color; /* F: current work color */
232	int flush_color; /* F: current flush color */
233	atomic_t nr_cwqs_to_flush; /* flush in progress */
234	struct wq_flusher first_flusher; / F: first flusher */
235	struct list_head flusher_queue; /* F: flush waiters */
236	struct list_head flusher_overflow; /* F: flush overflow list */
237
238	mayday_mask_t mayday_mask; /* cpus requesting rescue */
239	struct worker rescuer; / I: rescue worker */
240
241	int saved_max_active; /* W: saved cwq max_active */
242	const char name; / I: workqueue name */
243	#ifdef CONFIG_LOCKDEP
244	struct lockdep_map lockdep_map;
245	#endif
246	};
247
248	struct workqueue_struct *system_wq __read_mostly;
249	struct workqueue_struct *system_long_wq __read_mostly;
250	struct workqueue_struct *system_nrt_wq __read_mostly;
251	struct workqueue_struct *system_unbound_wq __read_mostly;
252	EXPORT_SYMBOL_GPL(system_wq);
253	EXPORT_SYMBOL_GPL(system_long_wq);
254	EXPORT_SYMBOL_GPL(system_nrt_wq);
255	EXPORT_SYMBOL_GPL(system_unbound_wq);
256
257	#define CREATE_TRACE_POINTS
258	#include <trace/events/workqueue.h>
259
260	#define for_each_busy_worker(worker, i, pos, gcwq) \
261	for (i = 0; i < BUSY_WORKER_HASH_SIZE; i++) \
262	hlist_for_each_entry(worker, pos, &gcwq->busy_hash[i], hentry)
263
264	static inline int __next_gcwq_cpu(int cpu, const struct cpumask *mask,
265	unsigned int sw)
266	{
267	if (cpu < nr_cpu_ids) {
268	if (sw & 1) {
269	cpu = cpumask_next(cpu, mask);
270	if (cpu < nr_cpu_ids)
271	return cpu;
272	}
273	if (sw & 2)
274	return WORK_CPU_UNBOUND;
275	}
276	return WORK_CPU_NONE;
277	}
278
279	static inline int __next_wq_cpu(int cpu, const struct cpumask *mask,
280	struct workqueue_struct *wq)
281	{
282	return __next_gcwq_cpu(cpu, mask, !(wq->flags & WQ_UNBOUND) ? 1 : 2);
283	}
284
285	/*
286	* CPU iterators
287	*
288	* An extra gcwq is defined for an invalid cpu number
289	* (WORK_CPU_UNBOUND) to host workqueues which are not bound to any
290	* specific CPU. The following iterators are similar to
291	* for_each_*_cpu() iterators but also considers the unbound gcwq.
292	*
293	* for_each_gcwq_cpu() : possible CPUs + WORK_CPU_UNBOUND
294	* for_each_online_gcwq_cpu() : online CPUs + WORK_CPU_UNBOUND
295	* for_each_cwq_cpu() : possible CPUs for bound workqueues,
296	* WORK_CPU_UNBOUND for unbound workqueues
297	*/
298	#define for_each_gcwq_cpu(cpu) \
299	for ((cpu) = __next_gcwq_cpu(-1, cpu_possible_mask, 3); \
300	(cpu) < WORK_CPU_NONE; \
301	(cpu) = __next_gcwq_cpu((cpu), cpu_possible_mask, 3))
302
303	#define for_each_online_gcwq_cpu(cpu) \
304	for ((cpu) = __next_gcwq_cpu(-1, cpu_online_mask, 3); \
305	(cpu) < WORK_CPU_NONE; \
306	(cpu) = __next_gcwq_cpu((cpu), cpu_online_mask, 3))
307
308	#define for_each_cwq_cpu(cpu, wq) \
309	for ((cpu) = __next_wq_cpu(-1, cpu_possible_mask, (wq)); \
310	(cpu) < WORK_CPU_NONE; \
311	(cpu) = __next_wq_cpu((cpu), cpu_possible_mask, (wq)))
312
313	#ifdef CONFIG_DEBUG_OBJECTS_WORK
314
315	static struct debug_obj_descr work_debug_descr;
316
317	/*
318	* fixup_init is called when:
319	* - an active object is initialized
320	*/
321	static int work_fixup_init(void *addr, enum debug_obj_state state)
322	{
323	struct work_struct *work = addr;
324
325	switch (state) {
326	case ODEBUG_STATE_ACTIVE:
327	cancel_work_sync(work);
328	debug_object_init(work, &work_debug_descr);
329	return 1;
330	default:
331	return 0;
332	}
333	}
334
335	/*
336	* fixup_activate is called when:
337	* - an active object is activated
338	* - an unknown object is activated (might be a statically initialized object)
339	*/
340	static int work_fixup_activate(void *addr, enum debug_obj_state state)
341	{
342	struct work_struct *work = addr;
343
344	switch (state) {
345
346	case ODEBUG_STATE_NOTAVAILABLE:
347	/*
348	* This is not really a fixup. The work struct was
349	* statically initialized. We just make sure that it
350	* is tracked in the object tracker.
351	*/
352	if (test_bit(WORK_STRUCT_STATIC_BIT, work_data_bits(work))) {
353	debug_object_init(work, &work_debug_descr);
354	debug_object_activate(work, &work_debug_descr);
355	return 0;
356	}
357	WARN_ON_ONCE(1);
358	return 0;
359
360	case ODEBUG_STATE_ACTIVE:
361	WARN_ON(1);
362
363	default:
364	return 0;
365	}
366	}
367
368	/*
369	* fixup_free is called when:
370	* - an active object is freed
371	*/
372	static int work_fixup_free(void *addr, enum debug_obj_state state)
373	{
374	struct work_struct *work = addr;
375
376	switch (state) {
377	case ODEBUG_STATE_ACTIVE:
378	cancel_work_sync(work);
379	debug_object_free(work, &work_debug_descr);
380	return 1;
381	default:
382	return 0;
383	}
384	}
385
386	static struct debug_obj_descr work_debug_descr = {
387	.name = "work_struct",
388	.fixup_init = work_fixup_init,
389	.fixup_activate = work_fixup_activate,
390	.fixup_free = work_fixup_free,
391	};
392
393	static inline void debug_work_activate(struct work_struct *work)
394	{
395	debug_object_activate(work, &work_debug_descr);
396	}
397
398	static inline void debug_work_deactivate(struct work_struct *work)
399	{
400	debug_object_deactivate(work, &work_debug_descr);
401	}
402
403	void __init_work(struct work_struct *work, int onstack)
404	{
405	if (onstack)
406	debug_object_init_on_stack(work, &work_debug_descr);
407	else
408	debug_object_init(work, &work_debug_descr);
409	}
410	EXPORT_SYMBOL_GPL(__init_work);
411
412	void destroy_work_on_stack(struct work_struct *work)
413	{
414	debug_object_free(work, &work_debug_descr);
415	}
416	EXPORT_SYMBOL_GPL(destroy_work_on_stack);
417
418	#else
419	static inline void debug_work_activate(struct work_struct *work) { }
420	static inline void debug_work_deactivate(struct work_struct *work) { }
421	#endif
422
423	/* Serializes the accesses to the list of workqueues. */
424	static DEFINE_SPINLOCK(workqueue_lock);
425	static LIST_HEAD(workqueues);
426	static bool workqueue_freezing; /* W: have wqs started freezing? */
427
428	/*
429	* The almighty global cpu workqueues. nr_running is the only field
430	* which is expected to be used frequently by other cpus via
431	* try_to_wake_up(). Put it in a separate cacheline.
432	*/
433	static DEFINE_PER_CPU(struct global_cwq, global_cwq);
434	static DEFINE_PER_CPU_SHARED_ALIGNED(atomic_t, gcwq_nr_running);
435
436	/*
437	* Global cpu workqueue and nr_running counter for unbound gcwq. The
438	* gcwq is always online, has GCWQ_DISASSOCIATED set, and all its
439	* workers have WORKER_UNBOUND set.
440	*/
441	static struct global_cwq unbound_global_cwq;
442	static atomic_t unbound_gcwq_nr_running = ATOMIC_INIT(0); /* always 0 */
443
444	static int worker_thread(void *__worker);
445
446	static struct global_cwq *get_gcwq(unsigned int cpu)
447	{
448	if (cpu != WORK_CPU_UNBOUND)
449	return &per_cpu(global_cwq, cpu);
450	else
451	return &unbound_global_cwq;
452	}
453
454	static atomic_t *get_gcwq_nr_running(unsigned int cpu)
455	{
456	if (cpu != WORK_CPU_UNBOUND)
457	return &per_cpu(gcwq_nr_running, cpu);
458	else
459	return &unbound_gcwq_nr_running;
460	}
461
462	static struct cpu_workqueue_struct *get_cwq(unsigned int cpu,
463	struct workqueue_struct *wq)
464	{
465	if (!(wq->flags & WQ_UNBOUND)) {
466	if (likely(cpu < nr_cpu_ids)) {
467	#ifdef CONFIG_SMP
468	return per_cpu_ptr(wq->cpu_wq.pcpu, cpu);
469	#else
470	return wq->cpu_wq.single;
471	#endif
472	}
473	} else if (likely(cpu == WORK_CPU_UNBOUND))
474	return wq->cpu_wq.single;
475	return NULL;
476	}
477
478	static unsigned int work_color_to_flags(int color)
479	{
480	return color << WORK_STRUCT_COLOR_SHIFT;
481	}
482
483	static int get_work_color(struct work_struct *work)
484	{
485	return (*work_data_bits(work) >> WORK_STRUCT_COLOR_SHIFT) &
486	((1 << WORK_STRUCT_COLOR_BITS) - 1);
487	}
488
489	static int work_next_color(int color)
490	{
491	return (color + 1) % WORK_NR_COLORS;
492	}
493
494	/*
495	* A work's data points to the cwq with WORK_STRUCT_CWQ set while the
496	* work is on queue. Once execution starts, WORK_STRUCT_CWQ is
497	* cleared and the work data contains the cpu number it was last on.
498	*
499	* set_work_{cwq\|cpu}() and clear_work_data() can be used to set the
500	* cwq, cpu or clear work->data. These functions should only be
501	* called while the work is owned - ie. while the PENDING bit is set.
502	*
503	* get_work_[g]cwq() can be used to obtain the gcwq or cwq
504	* corresponding to a work. gcwq is available once the work has been
505	* queued anywhere after initialization. cwq is available only from
506	* queueing until execution starts.
507	*/
508	static inline void set_work_data(struct work_struct *work, unsigned long data,
509	unsigned long flags)
510	{
511	BUG_ON(!work_pending(work));
512	atomic_long_set(&work->data, data \| flags \| work_static(work));
513	}
514
515	static void set_work_cwq(struct work_struct *work,
516	struct cpu_workqueue_struct *cwq,
517	unsigned long extra_flags)
518	{
519	set_work_data(work, (unsigned long)cwq,
520	WORK_STRUCT_PENDING \| WORK_STRUCT_CWQ \| extra_flags);
521	}
522
523	static void set_work_cpu(struct work_struct *work, unsigned int cpu)
524	{
525	set_work_data(work, cpu << WORK_STRUCT_FLAG_BITS, WORK_STRUCT_PENDING);
526	}
527
528	static void clear_work_data(struct work_struct *work)
529	{
530	set_work_data(work, WORK_STRUCT_NO_CPU, 0);
531	}
532
533	static struct cpu_workqueue_struct get_work_cwq(struct work_struct work)
534	{
535	unsigned long data = atomic_long_read(&work->data);
536
537	if (data & WORK_STRUCT_CWQ)
538	return (void *)(data & WORK_STRUCT_WQ_DATA_MASK);
539	else
540	return NULL;
541	}
542
543	static struct global_cwq get_work_gcwq(struct work_struct work)
544	{
545	unsigned long data = atomic_long_read(&work->data);
546	unsigned int cpu;
547
548	if (data & WORK_STRUCT_CWQ)
549	return ((struct cpu_workqueue_struct *)
550	(data & WORK_STRUCT_WQ_DATA_MASK))->gcwq;
551
552	cpu = data >> WORK_STRUCT_FLAG_BITS;
553	if (cpu == WORK_CPU_NONE)
554	return NULL;
555
556	BUG_ON(cpu >= nr_cpu_ids && cpu != WORK_CPU_UNBOUND);
557	return get_gcwq(cpu);
558	}
559
560	/*
561	* Policy functions. These define the policies on how the global
562	* worker pool is managed. Unless noted otherwise, these functions
563	* assume that they're being called with gcwq->lock held.
564	*/
565
566	static bool __need_more_worker(struct global_cwq *gcwq)
567	{
568	return !atomic_read(get_gcwq_nr_running(gcwq->cpu)) \|\|
569	gcwq->flags & GCWQ_HIGHPRI_PENDING;
570	}
571
572	/*
573	* Need to wake up a worker? Called from anything but currently
574	* running workers.
575	*/
576	static bool need_more_worker(struct global_cwq *gcwq)
577	{
578	return !list_empty(&gcwq->worklist) && __need_more_worker(gcwq);
579	}
580
581	/* Can I start working? Called from busy but !running workers. */
582	static bool may_start_working(struct global_cwq *gcwq)
583	{
584	return gcwq->nr_idle;
585	}
586
587	/* Do I need to keep working? Called from currently running workers. */
588	static bool keep_working(struct global_cwq *gcwq)
589	{
590	atomic_t *nr_running = get_gcwq_nr_running(gcwq->cpu);
591
592	return !list_empty(&gcwq->worklist) &&
593	(atomic_read(nr_running) <= 1 \|\|
594	gcwq->flags & GCWQ_HIGHPRI_PENDING);
595	}
596
597	/* Do we need a new worker? Called from manager. */
598	static bool need_to_create_worker(struct global_cwq *gcwq)
599	{
600	return need_more_worker(gcwq) && !may_start_working(gcwq);
601	}
602
603	/* Do I need to be the manager? */
604	static bool need_to_manage_workers(struct global_cwq *gcwq)
605	{
606	return need_to_create_worker(gcwq) \|\| gcwq->flags & GCWQ_MANAGE_WORKERS;
607	}
608
609	/* Do we have too many workers and should some go away? */
610	static bool too_many_workers(struct global_cwq *gcwq)
611	{
612	bool managing = gcwq->flags & GCWQ_MANAGING_WORKERS;
613	int nr_idle = gcwq->nr_idle + managing; /* manager is considered idle */
614	int nr_busy = gcwq->nr_workers - nr_idle;
615
616	return nr_idle > 2 && (nr_idle - 2) * MAX_IDLE_WORKERS_RATIO >= nr_busy;
617	}
618
619	/*
620	* Wake up functions.
621	*/
622
623	/* Return the first worker. Safe with preemption disabled */
624	static struct worker first_worker(struct global_cwq gcwq)
625	{
626	if (unlikely(list_empty(&gcwq->idle_list)))
627	return NULL;
628
629	return list_first_entry(&gcwq->idle_list, struct worker, entry);
630	}
631
632	/**
633	* wake_up_worker - wake up an idle worker
634	* @gcwq: gcwq to wake worker for
635	*
636	* Wake up the first idle worker of @gcwq.
637	*
638	* CONTEXT:
639	* spin_lock_irq(gcwq->lock).
640	*/
641	static void wake_up_worker(struct global_cwq *gcwq)
642	{
643	struct worker *worker = first_worker(gcwq);
644
645	if (likely(worker))
646	wake_up_process(worker->task);
647	}
648
649	/**
650	* wq_worker_waking_up - a worker is waking up
651	* @task: task waking up
652	* @cpu: CPU @task is waking up to
653	*
654	* This function is called during try_to_wake_up() when a worker is
655	* being awoken.
656	*
657	* CONTEXT:
658	* spin_lock_irq(rq->lock)
659	*/
660	void wq_worker_waking_up(struct task_struct *task, unsigned int cpu)
661	{
662	struct worker *worker = kthread_data(task);
663
664	if (!(worker->flags & WORKER_NOT_RUNNING))
665	atomic_inc(get_gcwq_nr_running(cpu));
666	}
667
668	/**
669	* wq_worker_sleeping - a worker is going to sleep
670	* @task: task going to sleep
671	* @cpu: CPU in question, must be the current CPU number
672	*
673	* This function is called during schedule() when a busy worker is
674	* going to sleep. Worker on the same cpu can be woken up by
675	* returning pointer to its task.
676	*
677	* CONTEXT:
678	* spin_lock_irq(rq->lock)
679	*
680	* RETURNS:
681	* Worker task on @cpu to wake up, %NULL if none.
682	*/
683	struct task_struct wq_worker_sleeping(struct task_struct task,
684	unsigned int cpu)
685	{
686	struct worker worker = kthread_data(task), to_wakeup = NULL;
687	struct global_cwq *gcwq = get_gcwq(cpu);
688	atomic_t *nr_running = get_gcwq_nr_running(cpu);
689
690	if (worker->flags & WORKER_NOT_RUNNING)
691	return NULL;
692
693	/* this can only happen on the local cpu */
694	BUG_ON(cpu != raw_smp_processor_id());
695
696	/*
697	* The counterpart of the following dec_and_test, implied mb,
698	* worklist not empty test sequence is in insert_work().
699	* Please read comment there.
700	*
701	* NOT_RUNNING is clear. This means that trustee is not in
702	* charge and we're running on the local cpu w/ rq lock held
703	* and preemption disabled, which in turn means that none else
704	* could be manipulating idle_list, so dereferencing idle_list
705	* without gcwq lock is safe.
706	*/
707	if (atomic_dec_and_test(nr_running) && !list_empty(&gcwq->worklist))
708	to_wakeup = first_worker(gcwq);
709	return to_wakeup ? to_wakeup->task : NULL;
710	}
711
712	/**
713	* worker_set_flags - set worker flags and adjust nr_running accordingly
714	* @worker: self
715	* @flags: flags to set
716	* @wakeup: wakeup an idle worker if necessary
717	*
718	* Set @flags in @worker->flags and adjust nr_running accordingly. If
719	* nr_running becomes zero and @wakeup is %true, an idle worker is
720	* woken up.
721	*
722	* CONTEXT:
723	* spin_lock_irq(gcwq->lock)
724	*/
725	static inline void worker_set_flags(struct worker *worker, unsigned int flags,
726	bool wakeup)
727	{
728	struct global_cwq *gcwq = worker->gcwq;
729
730	WARN_ON_ONCE(worker->task != current);
731
732	/*
733	* If transitioning into NOT_RUNNING, adjust nr_running and
734	* wake up an idle worker as necessary if requested by
735	* @wakeup.
736	*/
737	if ((flags & WORKER_NOT_RUNNING) &&
738	!(worker->flags & WORKER_NOT_RUNNING)) {
739	atomic_t *nr_running = get_gcwq_nr_running(gcwq->cpu);
740
741	if (wakeup) {
742	if (atomic_dec_and_test(nr_running) &&
743	!list_empty(&gcwq->worklist))
744	wake_up_worker(gcwq);
745	} else
746	atomic_dec(nr_running);
747	}
748
749	worker->flags \|= flags;
750	}
751
752	/**
753	* worker_clr_flags - clear worker flags and adjust nr_running accordingly
754	* @worker: self
755	* @flags: flags to clear
756	*
757	* Clear @flags in @worker->flags and adjust nr_running accordingly.
758	*
759	* CONTEXT:
760	* spin_lock_irq(gcwq->lock)
761	*/
762	static inline void worker_clr_flags(struct worker *worker, unsigned int flags)
763	{
764	struct global_cwq *gcwq = worker->gcwq;
765	unsigned int oflags = worker->flags;
766
767	WARN_ON_ONCE(worker->task != current);
768
769	worker->flags &= ~flags;
770
771	/* if transitioning out of NOT_RUNNING, increment nr_running */
772	if ((flags & WORKER_NOT_RUNNING) && (oflags & WORKER_NOT_RUNNING))
773	if (!(worker->flags & WORKER_NOT_RUNNING))
774	atomic_inc(get_gcwq_nr_running(gcwq->cpu));
775	}
776
777	/**
778	* busy_worker_head - return the busy hash head for a work
779	* @gcwq: gcwq of interest
780	* @work: work to be hashed
781	*
782	* Return hash head of @gcwq for @work.
783	*
784	* CONTEXT:
785	* spin_lock_irq(gcwq->lock).
786	*
787	* RETURNS:
788	* Pointer to the hash head.
789	*/
790	static struct hlist_head busy_worker_head(struct global_cwq gcwq,
791	struct work_struct *work)
792	{
793	const int base_shift = ilog2(sizeof(struct work_struct));
794	unsigned long v = (unsigned long)work;
795
796	/* simple shift and fold hash, do we need something better? */
797	v >>= base_shift;
798	v += v >> BUSY_WORKER_HASH_ORDER;
799	v &= BUSY_WORKER_HASH_MASK;
800
801	return &gcwq->busy_hash[v];
802	}
803
804	/**
805	* __find_worker_executing_work - find worker which is executing a work
806	* @gcwq: gcwq of interest
807	* @bwh: hash head as returned by busy_worker_head()
808	* @work: work to find worker for
809	*
810	* Find a worker which is executing @work on @gcwq. @bwh should be
811	* the hash head obtained by calling busy_worker_head() with the same
812	* work.
813	*
814	* CONTEXT:
815	* spin_lock_irq(gcwq->lock).
816	*
817	* RETURNS:
818	* Pointer to worker which is executing @work if found, NULL
819	* otherwise.
820	*/
821	static struct worker __find_worker_executing_work(struct global_cwq gcwq,
822	struct hlist_head *bwh,
823	struct work_struct *work)
824	{
825	struct worker *worker;
826	struct hlist_node *tmp;
827
828	hlist_for_each_entry(worker, tmp, bwh, hentry)
829	if (worker->current_work == work)
830	return worker;
831	return NULL;
832	}
833
834	/**
835	* find_worker_executing_work - find worker which is executing a work
836	* @gcwq: gcwq of interest
837	* @work: work to find worker for
838	*
839	* Find a worker which is executing @work on @gcwq. This function is
840	* identical to __find_worker_executing_work() except that this
841	* function calculates @bwh itself.
842	*
843	* CONTEXT:
844	* spin_lock_irq(gcwq->lock).
845	*
846	* RETURNS:
847	* Pointer to worker which is executing @work if found, NULL
848	* otherwise.
849	*/
850	static struct worker find_worker_executing_work(struct global_cwq gcwq,
851	struct work_struct *work)
852	{
853	return __find_worker_executing_work(gcwq, busy_worker_head(gcwq, work),
854	work);
855	}
856
857	/**
858	* gcwq_determine_ins_pos - find insertion position
859	* @gcwq: gcwq of interest
860	* @cwq: cwq a work is being queued for
861	*
862	* A work for @cwq is about to be queued on @gcwq, determine insertion
863	* position for the work. If @cwq is for HIGHPRI wq, the work is
864	* queued at the head of the queue but in FIFO order with respect to
865	* other HIGHPRI works; otherwise, at the end of the queue. This
866	* function also sets GCWQ_HIGHPRI_PENDING flag to hint @gcwq that
867	* there are HIGHPRI works pending.
868	*
869	* CONTEXT:
870	* spin_lock_irq(gcwq->lock).
871	*
872	* RETURNS:
873	* Pointer to inserstion position.
874	*/
875	static inline struct list_head gcwq_determine_ins_pos(struct global_cwq gcwq,
876	struct cpu_workqueue_struct *cwq)
877	{
878	struct work_struct *twork;
879
880	if (likely(!(cwq->wq->flags & WQ_HIGHPRI)))
881	return &gcwq->worklist;
882
883	list_for_each_entry(twork, &gcwq->worklist, entry) {
884	struct cpu_workqueue_struct *tcwq = get_work_cwq(twork);
885
886	if (!(tcwq->wq->flags & WQ_HIGHPRI))
887	break;
888	}
889
890	gcwq->flags \|= GCWQ_HIGHPRI_PENDING;
891	return &twork->entry;
892	}
893
894	/**
895	* insert_work - insert a work into gcwq
896	* @cwq: cwq @work belongs to
897	* @work: work to insert
898	* @head: insertion point
899	* @extra_flags: extra WORK_STRUCT_* flags to set
900	*
901	* Insert @work which belongs to @cwq into @gcwq after @head.
902	* @extra_flags is or'd to work_struct flags.
903	*
904	* CONTEXT:
905	* spin_lock_irq(gcwq->lock).
906	*/
907	static void insert_work(struct cpu_workqueue_struct *cwq,
908	struct work_struct work, struct list_head head,
909	unsigned int extra_flags)
910	{
911	struct global_cwq *gcwq = cwq->gcwq;
912
913	/* we own @work, set data and link */
914	set_work_cwq(work, cwq, extra_flags);
915
916	/*
917	* Ensure that we get the right work->data if we see the
918	* result of list_add() below, see try_to_grab_pending().
919	*/
920	smp_wmb();
921
922	list_add_tail(&work->entry, head);
923
924	/*
925	* Ensure either worker_sched_deactivated() sees the above
926	* list_add_tail() or we see zero nr_running to avoid workers
927	* lying around lazily while there are works to be processed.
928	*/
929	smp_mb();
930
931	if (__need_more_worker(gcwq))
932	wake_up_worker(gcwq);
933	}
934
935	/*
936	* Test whether @work is being queued from another work executing on the
937	* same workqueue. This is rather expensive and should only be used from
938	* cold paths.
939	*/
940	static bool is_chained_work(struct workqueue_struct *wq)
941	{
942	unsigned long flags;
943	unsigned int cpu;
944
945	for_each_gcwq_cpu(cpu) {
946	struct global_cwq *gcwq = get_gcwq(cpu);
947	struct worker *worker;
948	struct hlist_node *pos;
949	int i;
950
951	spin_lock_irqsave(&gcwq->lock, flags);
952	for_each_busy_worker(worker, i, pos, gcwq) {
953	if (worker->task != current)
954	continue;
955	spin_unlock_irqrestore(&gcwq->lock, flags);
956	/*
957	* I'm @worker, no locking necessary. See if @work
958	* is headed to the same workqueue.
959	*/
960	return worker->current_cwq->wq == wq;
961	}
962	spin_unlock_irqrestore(&gcwq->lock, flags);
963	}
964	return false;
965	}
966
967	static void __queue_work(unsigned int cpu, struct workqueue_struct *wq,
968	struct work_struct *work)
969	{
970	struct global_cwq *gcwq;
971	struct cpu_workqueue_struct *cwq;
972	struct list_head *worklist;
973	unsigned int work_flags;
974	unsigned long flags;
975
976	debug_work_activate(work);
977
978	/* if dying, only works from the same workqueue are allowed */
979	if (unlikely(wq->flags & WQ_DYING) &&
980	WARN_ON_ONCE(!is_chained_work(wq)))
981	return;
982
983	/* determine gcwq to use */
984	if (!(wq->flags & WQ_UNBOUND)) {
985	struct global_cwq *last_gcwq;
986
987	if (unlikely(cpu == WORK_CPU_UNBOUND))
988	cpu = raw_smp_processor_id();
989
990	/*
991	* It's multi cpu. If @wq is non-reentrant and @work
992	* was previously on a different cpu, it might still
993	* be running there, in which case the work needs to
994	* be queued on that cpu to guarantee non-reentrance.
995	*/
996	gcwq = get_gcwq(cpu);
997	if (wq->flags & WQ_NON_REENTRANT &&
998	(last_gcwq = get_work_gcwq(work)) && last_gcwq != gcwq) {
999	struct worker *worker;
1000
1001	spin_lock_irqsave(&last_gcwq->lock, flags);
1002
1003	worker = find_worker_executing_work(last_gcwq, work);
1004
1005	if (worker && worker->current_cwq->wq == wq)
1006	gcwq = last_gcwq;
1007	else {
1008	/* meh... not running there, queue here */
1009	spin_unlock_irqrestore(&last_gcwq->lock, flags);
1010	spin_lock_irqsave(&gcwq->lock, flags);
1011	}
1012	} else
1013	spin_lock_irqsave(&gcwq->lock, flags);
1014	} else {
1015	gcwq = get_gcwq(WORK_CPU_UNBOUND);
1016	spin_lock_irqsave(&gcwq->lock, flags);
1017	}
1018
1019	/* gcwq determined, get cwq and queue */
1020	cwq = get_cwq(gcwq->cpu, wq);
1021	trace_workqueue_queue_work(cpu, cwq, work);
1022
1023	BUG_ON(!list_empty(&work->entry));
1024
1025	cwq->nr_in_flight[cwq->work_color]++;
1026	work_flags = work_color_to_flags(cwq->work_color);
1027
1028	if (likely(cwq->nr_active < cwq->max_active)) {
1029	trace_workqueue_activate_work(work);
1030	cwq->nr_active++;
1031	worklist = gcwq_determine_ins_pos(gcwq, cwq);
1032	} else {
1033	work_flags \|= WORK_STRUCT_DELAYED;
1034	worklist = &cwq->delayed_works;
1035	}
1036
1037	insert_work(cwq, work, worklist, work_flags);
1038
1039	spin_unlock_irqrestore(&gcwq->lock, flags);
1040	}
1041
1042	/**
1043	* queue_work - queue work on a workqueue
1044	* @wq: workqueue to use
1045	* @work: work to queue
1046	*
1047	* Returns 0 if @work was already on a queue, non-zero otherwise.
1048	*
1049	* We queue the work to the CPU on which it was submitted, but if the CPU dies
1050	* it can be processed by another CPU.
1051	*/
1052	int queue_work(struct workqueue_struct wq, struct work_struct work)
1053	{
1054	int ret;
1055
1056	ret = queue_work_on(get_cpu(), wq, work);
1057	put_cpu();
1058
1059	return ret;
1060	}
1061	EXPORT_SYMBOL_GPL(queue_work);
1062
1063	/**
1064	* queue_work_on - queue work on specific cpu
1065	* @cpu: CPU number to execute work on
1066	* @wq: workqueue to use
1067	* @work: work to queue
1068	*
1069	* Returns 0 if @work was already on a queue, non-zero otherwise.
1070	*
1071	* We queue the work to a specific CPU, the caller must ensure it
1072	* can't go away.
1073	*/
1074	int
1075	queue_work_on(int cpu, struct workqueue_struct wq, struct work_struct work)
1076	{
1077	int ret = 0;
1078
1079	if (!test_and_set_bit(WORK_STRUCT_PENDING_BIT, work_data_bits(work))) {
1080	__queue_work(cpu, wq, work);
1081	ret = 1;
1082	}
1083	return ret;
1084	}
1085	EXPORT_SYMBOL_GPL(queue_work_on);
1086
1087	static void delayed_work_timer_fn(unsigned long __data)
1088	{
1089	struct delayed_work dwork = (struct delayed_work )__data;
1090	struct cpu_workqueue_struct *cwq = get_work_cwq(&dwork->work);
1091
1092	__queue_work(smp_processor_id(), cwq->wq, &dwork->work);
1093	}
1094
1095	/**
1096	* queue_delayed_work - queue work on a workqueue after delay
1097	* @wq: workqueue to use
1098	* @dwork: delayable work to queue
1099	* @delay: number of jiffies to wait before queueing
1100	*
1101	* Returns 0 if @work was already on a queue, non-zero otherwise.
1102	*/
1103	int queue_delayed_work(struct workqueue_struct *wq,
1104	struct delayed_work *dwork, unsigned long delay)
1105	{
1106	if (delay == 0)
1107	return queue_work(wq, &dwork->work);
1108
1109	return queue_delayed_work_on(-1, wq, dwork, delay);
1110	}
1111	EXPORT_SYMBOL_GPL(queue_delayed_work);
1112
1113	/**
1114	* queue_delayed_work_on - queue work on specific CPU after delay
1115	* @cpu: CPU number to execute work on
1116	* @wq: workqueue to use
1117	* @dwork: work to queue
1118	* @delay: number of jiffies to wait before queueing
1119	*
1120	* Returns 0 if @work was already on a queue, non-zero otherwise.
1121	*/
1122	int queue_delayed_work_on(int cpu, struct workqueue_struct *wq,
1123	struct delayed_work *dwork, unsigned long delay)
1124	{
1125	int ret = 0;
1126	struct timer_list *timer = &dwork->timer;
1127	struct work_struct *work = &dwork->work;
1128
1129	if (!test_and_set_bit(WORK_STRUCT_PENDING_BIT, work_data_bits(work))) {
1130	unsigned int lcpu;
1131
1132	BUG_ON(timer_pending(timer));
1133	BUG_ON(!list_empty(&work->entry));
1134
1135	timer_stats_timer_set_start_info(&dwork->timer);
1136
1137	/*
1138	* This stores cwq for the moment, for the timer_fn.
1139	* Note that the work's gcwq is preserved to allow
1140	* reentrance detection for delayed works.
1141	*/
1142	if (!(wq->flags & WQ_UNBOUND)) {
1143	struct global_cwq *gcwq = get_work_gcwq(work);
1144
1145	if (gcwq && gcwq->cpu != WORK_CPU_UNBOUND)
1146	lcpu = gcwq->cpu;
1147	else
1148	lcpu = raw_smp_processor_id();
1149	} else
1150	lcpu = WORK_CPU_UNBOUND;
1151
1152	set_work_cwq(work, get_cwq(lcpu, wq), 0);
1153
1154	timer->expires = jiffies + delay;
1155	timer->data = (unsigned long)dwork;
1156	timer->function = delayed_work_timer_fn;
1157
1158	if (unlikely(cpu >= 0))
1159	add_timer_on(timer, cpu);
1160	else
1161	add_timer(timer);
1162	ret = 1;
1163	}
1164	return ret;
1165	}
1166	EXPORT_SYMBOL_GPL(queue_delayed_work_on);
1167
1168	/**
1169	* worker_enter_idle - enter idle state
1170	* @worker: worker which is entering idle state
1171	*
1172	* @worker is entering idle state. Update stats and idle timer if
1173	* necessary.
1174	*
1175	* LOCKING:
1176	* spin_lock_irq(gcwq->lock).
1177	*/
1178	static void worker_enter_idle(struct worker *worker)
1179	{
1180	struct global_cwq *gcwq = worker->gcwq;
1181
1182	BUG_ON(worker->flags & WORKER_IDLE);
1183	BUG_ON(!list_empty(&worker->entry) &&
1184	(worker->hentry.next \|\| worker->hentry.pprev));
1185
1186	/* can't use worker_set_flags(), also called from start_worker() */
1187	worker->flags \|= WORKER_IDLE;
1188	gcwq->nr_idle++;
1189	worker->last_active = jiffies;
1190
1191	/* idle_list is LIFO */
1192	list_add(&worker->entry, &gcwq->idle_list);
1193
1194	if (likely(!(worker->flags & WORKER_ROGUE))) {
1195	if (too_many_workers(gcwq) && !timer_pending(&gcwq->idle_timer))
1196	mod_timer(&gcwq->idle_timer,
1197	jiffies + IDLE_WORKER_TIMEOUT);
1198	} else
1199	wake_up_all(&gcwq->trustee_wait);
1200
1201	/* sanity check nr_running */
1202	WARN_ON_ONCE(gcwq->nr_workers == gcwq->nr_idle &&
1203	atomic_read(get_gcwq_nr_running(gcwq->cpu)));
1204	}
1205
1206	/**
1207	* worker_leave_idle - leave idle state
1208	* @worker: worker which is leaving idle state
1209	*
1210	* @worker is leaving idle state. Update stats.
1211	*
1212	* LOCKING:
1213	* spin_lock_irq(gcwq->lock).
1214	*/
1215	static void worker_leave_idle(struct worker *worker)
1216	{
1217	struct global_cwq *gcwq = worker->gcwq;
1218
1219	BUG_ON(!(worker->flags & WORKER_IDLE));
1220	worker_clr_flags(worker, WORKER_IDLE);
1221	gcwq->nr_idle--;
1222	list_del_init(&worker->entry);
1223	}
1224
1225	/**
1226	* worker_maybe_bind_and_lock - bind worker to its cpu if possible and lock gcwq
1227	* @worker: self
1228	*
1229	* Works which are scheduled while the cpu is online must at least be
1230	* scheduled to a worker which is bound to the cpu so that if they are
1231	* flushed from cpu callbacks while cpu is going down, they are
1232	* guaranteed to execute on the cpu.
1233	*
1234	* This function is to be used by rogue workers and rescuers to bind
1235	* themselves to the target cpu and may race with cpu going down or
1236	* coming online. kthread_bind() can't be used because it may put the
1237	* worker to already dead cpu and set_cpus_allowed_ptr() can't be used
1238	* verbatim as it's best effort and blocking and gcwq may be
1239	* [dis]associated in the meantime.
1240	*
1241	* This function tries set_cpus_allowed() and locks gcwq and verifies
1242	* the binding against GCWQ_DISASSOCIATED which is set during
1243	* CPU_DYING and cleared during CPU_ONLINE, so if the worker enters
1244	* idle state or fetches works without dropping lock, it can guarantee
1245	* the scheduling requirement described in the first paragraph.
1246	*
1247	* CONTEXT:
1248	* Might sleep. Called without any lock but returns with gcwq->lock
1249	* held.
1250	*
1251	* RETURNS:
1252	* %true if the associated gcwq is online (@worker is successfully
1253	* bound), %false if offline.
1254	*/
1255	static bool worker_maybe_bind_and_lock(struct worker *worker)
1256	__acquires(&gcwq->lock)
1257	{
1258	struct global_cwq *gcwq = worker->gcwq;
1259	struct task_struct *task = worker->task;
1260
1261	while (true) {
1262	/*
1263	* The following call may fail, succeed or succeed
1264	* without actually migrating the task to the cpu if
1265	* it races with cpu hotunplug operation. Verify
1266	* against GCWQ_DISASSOCIATED.
1267	*/
1268	if (!(gcwq->flags & GCWQ_DISASSOCIATED))
1269	set_cpus_allowed_ptr(task, get_cpu_mask(gcwq->cpu));
1270
1271	spin_lock_irq(&gcwq->lock);
1272	if (gcwq->flags & GCWQ_DISASSOCIATED)
1273	return false;
1274	if (task_cpu(task) == gcwq->cpu &&
1275	cpumask_equal(&current->cpus_allowed,
1276	get_cpu_mask(gcwq->cpu)))
1277	return true;
1278	spin_unlock_irq(&gcwq->lock);
1279
1280	/* CPU has come up inbetween, retry migration */
1281	cpu_relax();
1282	}
1283	}
1284
1285	/*
1286	* Function for worker->rebind_work used to rebind rogue busy workers
1287	* to the associated cpu which is coming back online. This is
1288	* scheduled by cpu up but can race with other cpu hotplug operations
1289	* and may be executed twice without intervening cpu down.
1290	*/
1291	static void worker_rebind_fn(struct work_struct *work)
1292	{
1293	struct worker *worker = container_of(work, struct worker, rebind_work);
1294	struct global_cwq *gcwq = worker->gcwq;
1295
1296	if (worker_maybe_bind_and_lock(worker))
1297	worker_clr_flags(worker, WORKER_REBIND);
1298
1299	spin_unlock_irq(&gcwq->lock);
1300	}
1301
1302	static struct worker *alloc_worker(void)
1303	{
1304	struct worker *worker;
1305
1306	worker = kzalloc(sizeof(*worker), GFP_KERNEL);
1307	if (worker) {
1308	INIT_LIST_HEAD(&worker->entry);
1309	INIT_LIST_HEAD(&worker->scheduled);
1310	INIT_WORK(&worker->rebind_work, worker_rebind_fn);
1311	/* on creation a worker is in !idle && prep state */
1312	worker->flags = WORKER_PREP;
1313	}
1314	return worker;
1315	}
1316
1317	/**
1318	* create_worker - create a new workqueue worker
1319	* @gcwq: gcwq the new worker will belong to
1320	* @bind: whether to set affinity to @cpu or not
1321	*
1322	* Create a new worker which is bound to @gcwq. The returned worker
1323	* can be started by calling start_worker() or destroyed using
1324	* destroy_worker().
1325	*
1326	* CONTEXT:
1327	* Might sleep. Does GFP_KERNEL allocations.
1328	*
1329	* RETURNS:
1330	* Pointer to the newly created worker.
1331	*/
1332	static struct worker create_worker(struct global_cwq gcwq, bool bind)
1333	{
1334	bool on_unbound_cpu = gcwq->cpu == WORK_CPU_UNBOUND;
1335	struct worker *worker = NULL;
1336	int id = -1;
1337
1338	spin_lock_irq(&gcwq->lock);
1339	while (ida_get_new(&gcwq->worker_ida, &id)) {
1340	spin_unlock_irq(&gcwq->lock);
1341	if (!ida_pre_get(&gcwq->worker_ida, GFP_KERNEL))
1342	goto fail;
1343	spin_lock_irq(&gcwq->lock);
1344	}
1345	spin_unlock_irq(&gcwq->lock);
1346
1347	worker = alloc_worker();
1348	if (!worker)
1349	goto fail;
1350
1351	worker->gcwq = gcwq;
1352	worker->id = id;
1353
1354	if (!on_unbound_cpu)
1355	worker->task = kthread_create(worker_thread, worker,
1356	"kworker/%u:%d", gcwq->cpu, id);
1357	else
1358	worker->task = kthread_create(worker_thread, worker,
1359	"kworker/u:%d", id);
1360	if (IS_ERR(worker->task))
1361	goto fail;
1362
1363	/*
1364	* A rogue worker will become a regular one if CPU comes
1365	* online later on. Make sure every worker has
1366	* PF_THREAD_BOUND set.
1367	*/
1368	if (bind && !on_unbound_cpu)
1369	kthread_bind(worker->task, gcwq->cpu);
1370	else {
1371	worker->task->flags \|= PF_THREAD_BOUND;
1372	if (on_unbound_cpu)
1373	worker->flags \|= WORKER_UNBOUND;
1374	}
1375
1376	return worker;
1377	fail:
1378	if (id >= 0) {
1379	spin_lock_irq(&gcwq->lock);
1380	ida_remove(&gcwq->worker_ida, id);
1381	spin_unlock_irq(&gcwq->lock);
1382	}
1383	kfree(worker);
1384	return NULL;
1385	}
1386
1387	/**
1388	* start_worker - start a newly created worker
1389	* @worker: worker to start
1390	*
1391	* Make the gcwq aware of @worker and start it.
1392	*
1393	* CONTEXT:
1394	* spin_lock_irq(gcwq->lock).
1395	*/
1396	static void start_worker(struct worker *worker)
1397	{
1398	worker->flags \|= WORKER_STARTED;
1399	worker->gcwq->nr_workers++;
1400	worker_enter_idle(worker);
1401	wake_up_process(worker->task);
1402	}
1403
1404	/**
1405	* destroy_worker - destroy a workqueue worker
1406	* @worker: worker to be destroyed
1407	*
1408	* Destroy @worker and adjust @gcwq stats accordingly.
1409	*
1410	* CONTEXT:
1411	* spin_lock_irq(gcwq->lock) which is released and regrabbed.
1412	*/
1413	static void destroy_worker(struct worker *worker)
1414	{
1415	struct global_cwq *gcwq = worker->gcwq;
1416	int id = worker->id;
1417
1418	/* sanity check frenzy */
1419	BUG_ON(worker->current_work);
1420	BUG_ON(!list_empty(&worker->scheduled));
1421
1422	if (worker->flags & WORKER_STARTED)
1423	gcwq->nr_workers--;
1424	if (worker->flags & WORKER_IDLE)
1425	gcwq->nr_idle--;
1426
1427	list_del_init(&worker->entry);
1428	worker->flags \|= WORKER_DIE;
1429
1430	spin_unlock_irq(&gcwq->lock);
1431
1432	kthread_stop(worker->task);
1433	kfree(worker);
1434
1435	spin_lock_irq(&gcwq->lock);
1436	ida_remove(&gcwq->worker_ida, id);
1437	}
1438
1439	static void idle_worker_timeout(unsigned long __gcwq)
1440	{
1441	struct global_cwq gcwq = (void )__gcwq;
1442
1443	spin_lock_irq(&gcwq->lock);
1444
1445	if (too_many_workers(gcwq)) {
1446	struct worker *worker;
1447	unsigned long expires;
1448
1449	/* idle_list is kept in LIFO order, check the last one */
1450	worker = list_entry(gcwq->idle_list.prev, struct worker, entry);
1451	expires = worker->last_active + IDLE_WORKER_TIMEOUT;
1452
1453	if (time_before(jiffies, expires))
1454	mod_timer(&gcwq->idle_timer, expires);
1455	else {
1456	/* it's been idle for too long, wake up manager */
1457	gcwq->flags \|= GCWQ_MANAGE_WORKERS;
1458	wake_up_worker(gcwq);
1459	}
1460	}
1461
1462	spin_unlock_irq(&gcwq->lock);
1463	}
1464
1465	static bool send_mayday(struct work_struct *work)
1466	{
1467	struct cpu_workqueue_struct *cwq = get_work_cwq(work);
1468	struct workqueue_struct *wq = cwq->wq;
1469	unsigned int cpu;
1470
1471	if (!(wq->flags & WQ_RESCUER))
1472	return false;
1473
1474	/* mayday mayday mayday */
1475	cpu = cwq->gcwq->cpu;
1476	/* WORK_CPU_UNBOUND can't be set in cpumask, use cpu 0 instead */
1477	if (cpu == WORK_CPU_UNBOUND)
1478	cpu = 0;
1479	if (!mayday_test_and_set_cpu(cpu, wq->mayday_mask))
1480	wake_up_process(wq->rescuer->task);
1481	return true;
1482	}
1483
1484	static void gcwq_mayday_timeout(unsigned long __gcwq)
1485	{
1486	struct global_cwq gcwq = (void )__gcwq;
1487	struct work_struct *work;
1488
1489	spin_lock_irq(&gcwq->lock);
1490
1491	if (need_to_create_worker(gcwq)) {
1492	/*
1493	* We've been trying to create a new worker but
1494	* haven't been successful. We might be hitting an
1495	* allocation deadlock. Send distress signals to
1496	* rescuers.
1497	*/
1498	list_for_each_entry(work, &gcwq->worklist, entry)
1499	send_mayday(work);
1500	}
1501
1502	spin_unlock_irq(&gcwq->lock);
1503
1504	mod_timer(&gcwq->mayday_timer, jiffies + MAYDAY_INTERVAL);
1505	}
1506
1507	/**
1508	* maybe_create_worker - create a new worker if necessary
1509	* @gcwq: gcwq to create a new worker for
1510	*
1511	* Create a new worker for @gcwq if necessary. @gcwq is guaranteed to
1512	* have at least one idle worker on return from this function. If
1513	* creating a new worker takes longer than MAYDAY_INTERVAL, mayday is
1514	* sent to all rescuers with works scheduled on @gcwq to resolve
1515	* possible allocation deadlock.
1516	*
1517	* On return, need_to_create_worker() is guaranteed to be false and
1518	* may_start_working() true.
1519	*
1520	* LOCKING:
1521	* spin_lock_irq(gcwq->lock) which may be released and regrabbed
1522	* multiple times. Does GFP_KERNEL allocations. Called only from
1523	* manager.
1524	*
1525	* RETURNS:
1526	* false if no action was taken and gcwq->lock stayed locked, true
1527	* otherwise.
1528	*/
1529	static bool maybe_create_worker(struct global_cwq *gcwq)
1530	__releases(&gcwq->lock)
1531	__acquires(&gcwq->lock)
1532	{
1533	if (!need_to_create_worker(gcwq))
1534	return false;
1535	restart:
1536	spin_unlock_irq(&gcwq->lock);
1537
1538	/* if we don't make progress in MAYDAY_INITIAL_TIMEOUT, call for help */
1539	mod_timer(&gcwq->mayday_timer, jiffies + MAYDAY_INITIAL_TIMEOUT);
1540
1541	while (true) {
1542	struct worker *worker;
1543
1544	worker = create_worker(gcwq, true);
1545	if (worker) {
1546	del_timer_sync(&gcwq->mayday_timer);
1547	spin_lock_irq(&gcwq->lock);
1548	start_worker(worker);
1549	BUG_ON(need_to_create_worker(gcwq));
1550	return true;
1551	}
1552
1553	if (!need_to_create_worker(gcwq))
1554	break;
1555
1556	__set_current_state(TASK_INTERRUPTIBLE);
1557	schedule_timeout(CREATE_COOLDOWN);
1558
1559	if (!need_to_create_worker(gcwq))
1560	break;
1561	}
1562
1563	del_timer_sync(&gcwq->mayday_timer);
1564	spin_lock_irq(&gcwq->lock);
1565	if (need_to_create_worker(gcwq))
1566	goto restart;
1567	return true;
1568	}
1569
1570	/**
1571	* maybe_destroy_worker - destroy workers which have been idle for a while
1572	* @gcwq: gcwq to destroy workers for
1573	*
1574	* Destroy @gcwq workers which have been idle for longer than
1575	* IDLE_WORKER_TIMEOUT.
1576	*
1577	* LOCKING:
1578	* spin_lock_irq(gcwq->lock) which may be released and regrabbed
1579	* multiple times. Called only from manager.
1580	*
1581	* RETURNS:
1582	* false if no action was taken and gcwq->lock stayed locked, true
1583	* otherwise.
1584	*/
1585	static bool maybe_destroy_workers(struct global_cwq *gcwq)
1586	{
1587	bool ret = false;
1588
1589	while (too_many_workers(gcwq)) {
1590	struct worker *worker;
1591	unsigned long expires;
1592
1593	worker = list_entry(gcwq->idle_list.prev, struct worker, entry);
1594	expires = worker->last_active + IDLE_WORKER_TIMEOUT;
1595
1596	if (time_before(jiffies, expires)) {
1597	mod_timer(&gcwq->idle_timer, expires);
1598	break;
1599	}
1600
1601	destroy_worker(worker);
1602	ret = true;
1603	}
1604
1605	return ret;
1606	}
1607
1608	/**
1609	* manage_workers - manage worker pool
1610	* @worker: self
1611	*
1612	* Assume the manager role and manage gcwq worker pool @worker belongs
1613	* to. At any given time, there can be only zero or one manager per
1614	* gcwq. The exclusion is handled automatically by this function.
1615	*
1616	* The caller can safely start processing works on false return. On
1617	* true return, it's guaranteed that need_to_create_worker() is false
1618	* and may_start_working() is true.
1619	*
1620	* CONTEXT:
1621	* spin_lock_irq(gcwq->lock) which may be released and regrabbed
1622	* multiple times. Does GFP_KERNEL allocations.
1623	*
1624	* RETURNS:
1625	* false if no action was taken and gcwq->lock stayed locked, true if
1626	* some action was taken.
1627	*/
1628	static bool manage_workers(struct worker *worker)
1629	{
1630	struct global_cwq *gcwq = worker->gcwq;
1631	bool ret = false;
1632
1633	if (gcwq->flags & GCWQ_MANAGING_WORKERS)
1634	return ret;
1635
1636	gcwq->flags &= ~GCWQ_MANAGE_WORKERS;
1637	gcwq->flags \|= GCWQ_MANAGING_WORKERS;
1638
1639	/*
1640	* Destroy and then create so that may_start_working() is true
1641	* on return.
1642	*/
1643	ret \|= maybe_destroy_workers(gcwq);
1644	ret \|= maybe_create_worker(gcwq);
1645
1646	gcwq->flags &= ~GCWQ_MANAGING_WORKERS;
1647
1648	/*
1649	* The trustee might be waiting to take over the manager
1650	* position, tell it we're done.
1651	*/
1652	if (unlikely(gcwq->trustee))
1653	wake_up_all(&gcwq->trustee_wait);
1654
1655	return ret;
1656	}
1657
1658	/**
1659	* move_linked_works - move linked works to a list
1660	* @work: start of series of works to be scheduled
1661	* @head: target list to append @work to
1662	* @nextp: out paramter for nested worklist walking
1663	*
1664	* Schedule linked works starting from @work to @head. Work series to
1665	* be scheduled starts at @work and includes any consecutive work with
1666	* WORK_STRUCT_LINKED set in its predecessor.
1667	*
1668	* If @nextp is not NULL, it's updated to point to the next work of
1669	* the last scheduled work. This allows move_linked_works() to be
1670	* nested inside outer list_for_each_entry_safe().
1671	*
1672	* CONTEXT:
1673	* spin_lock_irq(gcwq->lock).
1674	*/
1675	static void move_linked_works(struct work_struct work, struct list_head head,
1676	struct work_struct **nextp)
1677	{
1678	struct work_struct *n;
1679
1680	/*
1681	* Linked worklist will always end before the end of the list,
1682	* use NULL for list head.
1683	*/
1684	list_for_each_entry_safe_from(work, n, NULL, entry) {
1685	list_move_tail(&work->entry, head);
1686	if (!(*work_data_bits(work) & WORK_STRUCT_LINKED))
1687	break;
1688	}
1689
1690	/*
1691	* If we're already inside safe list traversal and have moved
1692	* multiple works to the scheduled queue, the next position
1693	* needs to be updated.
1694	*/
1695	if (nextp)
1696	*nextp = n;
1697	}
1698
1699	static void cwq_activate_first_delayed(struct cpu_workqueue_struct *cwq)
1700	{
1701	struct work_struct *work = list_first_entry(&cwq->delayed_works,
1702	struct work_struct, entry);
1703	struct list_head *pos = gcwq_determine_ins_pos(cwq->gcwq, cwq);
1704
1705	trace_workqueue_activate_work(work);
1706	move_linked_works(work, pos, NULL);
1707	__clear_bit(WORK_STRUCT_DELAYED_BIT, work_data_bits(work));
1708	cwq->nr_active++;
1709	}
1710
1711	/**
1712	* cwq_dec_nr_in_flight - decrement cwq's nr_in_flight
1713	* @cwq: cwq of interest
1714	* @color: color of work which left the queue
1715	* @delayed: for a delayed work
1716	*
1717	* A work either has completed or is removed from pending queue,
1718	* decrement nr_in_flight of its cwq and handle workqueue flushing.
1719	*
1720	* CONTEXT:
1721	* spin_lock_irq(gcwq->lock).
1722	*/
1723	static void cwq_dec_nr_in_flight(struct cpu_workqueue_struct *cwq, int color,
1724	bool delayed)
1725	{
1726	/* ignore uncolored works */
1727	if (color == WORK_NO_COLOR)
1728	return;
1729
1730	cwq->nr_in_flight[color]--;
1731
1732	if (!delayed) {
1733	cwq->nr_active--;
1734	if (!list_empty(&cwq->delayed_works)) {
1735	/* one down, submit a delayed one */
1736	if (cwq->nr_active < cwq->max_active)
1737	cwq_activate_first_delayed(cwq);
1738	}
1739	}
1740
1741	/* is flush in progress and are we at the flushing tip? */
1742	if (likely(cwq->flush_color != color))
1743	return;
1744
1745	/* are there still in-flight works? */
1746	if (cwq->nr_in_flight[color])
1747	return;
1748
1749	/* this cwq is done, clear flush_color */
1750	cwq->flush_color = -1;
1751
1752	/*
1753	* If this was the last cwq, wake up the first flusher. It
1754	* will handle the rest.
1755	*/
1756	if (atomic_dec_and_test(&cwq->wq->nr_cwqs_to_flush))
1757	complete(&cwq->wq->first_flusher->done);
1758	}
1759
1760	/**
1761	* process_one_work - process single work
1762	* @worker: self
1763	* @work: work to process
1764	*
1765	* Process @work. This function contains all the logics necessary to
1766	* process a single work including synchronization against and
1767	* interaction with other workers on the same cpu, queueing and
1768	* flushing. As long as context requirement is met, any worker can
1769	* call this function to process a work.
1770	*
1771	* CONTEXT:
1772	* spin_lock_irq(gcwq->lock) which is released and regrabbed.
1773	*/
1774	static void process_one_work(struct worker worker, struct work_struct work)
1775	__releases(&gcwq->lock)
1776	__acquires(&gcwq->lock)
1777	{
1778	struct cpu_workqueue_struct *cwq = get_work_cwq(work);
1779	struct global_cwq *gcwq = cwq->gcwq;
1780	struct hlist_head *bwh = busy_worker_head(gcwq, work);
1781	bool cpu_intensive = cwq->wq->flags & WQ_CPU_INTENSIVE;
1782	work_func_t f = work->func;
1783	int work_color;
1784	struct worker *collision;
1785	#ifdef CONFIG_LOCKDEP
1786	/*
1787	* It is permissible to free the struct work_struct from
1788	* inside the function that is called from it, this we need to
1789	* take into account for lockdep too. To avoid bogus "held
1790	* lock freed" warnings as well as problems when looking into
1791	* work->lockdep_map, make a copy and use that here.
1792	*/
1793	struct lockdep_map lockdep_map = work->lockdep_map;
1794	#endif
1795	/*
1796	* A single work shouldn't be executed concurrently by
1797	* multiple workers on a single cpu. Check whether anyone is
1798	* already processing the work. If so, defer the work to the
1799	* currently executing one.
1800	*/
1801	collision = __find_worker_executing_work(gcwq, bwh, work);
1802	if (unlikely(collision)) {
1803	move_linked_works(work, &collision->scheduled, NULL);
1804	return;
1805	}
1806
1807	/* claim and process */
1808	debug_work_deactivate(work);
1809	hlist_add_head(&worker->hentry, bwh);
1810	worker->current_work = work;
1811	worker->current_cwq = cwq;
1812	work_color = get_work_color(work);
1813
1814	/* record the current cpu number in the work data and dequeue */
1815	set_work_cpu(work, gcwq->cpu);
1816	list_del_init(&work->entry);
1817
1818	/*
1819	* If HIGHPRI_PENDING, check the next work, and, if HIGHPRI,
1820	* wake up another worker; otherwise, clear HIGHPRI_PENDING.
1821	*/
1822	if (unlikely(gcwq->flags & GCWQ_HIGHPRI_PENDING)) {
1823	struct work_struct *nwork = list_first_entry(&gcwq->worklist,
1824	struct work_struct, entry);
1825
1826	if (!list_empty(&gcwq->worklist) &&
1827	get_work_cwq(nwork)->wq->flags & WQ_HIGHPRI)
1828	wake_up_worker(gcwq);
1829	else
1830	gcwq->flags &= ~GCWQ_HIGHPRI_PENDING;
1831	}
1832
1833	/*
1834	* CPU intensive works don't participate in concurrency
1835	* management. They're the scheduler's responsibility.
1836	*/
1837	if (unlikely(cpu_intensive))
1838	worker_set_flags(worker, WORKER_CPU_INTENSIVE, true);
1839
1840	spin_unlock_irq(&gcwq->lock);
1841
1842	work_clear_pending(work);
1843	lock_map_acquire(&cwq->wq->lockdep_map);
1844	lock_map_acquire(&lockdep_map);
1845	trace_workqueue_execute_start(work);
1846	f(work);
1847	/*
1848	* While we must be careful to not use "work" after this, the trace
1849	* point will only record its address.
1850	*/
1851	trace_workqueue_execute_end(work);
1852	lock_map_release(&lockdep_map);
1853	lock_map_release(&cwq->wq->lockdep_map);
1854
1855	if (unlikely(in_atomic() \|\| lockdep_depth(current) > 0)) {
1856	printk(KERN_ERR "BUG: workqueue leaked lock or atomic: "
1857	"%s/0x%08x/%d\n",
1858	current->comm, preempt_count(), task_pid_nr(current));
1859	printk(KERN_ERR " last function: ");
1860	print_symbol("%s\n", (unsigned long)f);
1861	debug_show_held_locks(current);
1862	dump_stack();
1863	}
1864
1865	spin_lock_irq(&gcwq->lock);
1866
1867	/* clear cpu intensive status */
1868	if (unlikely(cpu_intensive))
1869	worker_clr_flags(worker, WORKER_CPU_INTENSIVE);
1870
1871	/* we're done with it, release */
1872	hlist_del_init(&worker->hentry);
1873	worker->current_work = NULL;
1874	worker->current_cwq = NULL;
1875	cwq_dec_nr_in_flight(cwq, work_color, false);
1876	}
1877
1878	/**
1879	* process_scheduled_works - process scheduled works
1880	* @worker: self
1881	*
1882	* Process all scheduled works. Please note that the scheduled list
1883	* may change while processing a work, so this function repeatedly
1884	* fetches a work from the top and executes it.
1885	*
1886	* CONTEXT:
1887	* spin_lock_irq(gcwq->lock) which may be released and regrabbed
1888	* multiple times.
1889	*/
1890	static void process_scheduled_works(struct worker *worker)
1891	{
1892	while (!list_empty(&worker->scheduled)) {
1893	struct work_struct *work = list_first_entry(&worker->scheduled,
1894	struct work_struct, entry);
1895	process_one_work(worker, work);
1896	}
1897	}
1898
1899	/**
1900	* worker_thread - the worker thread function
1901	* @__worker: self
1902	*
1903	* The gcwq worker thread function. There's a single dynamic pool of
1904	* these per each cpu. These workers process all works regardless of
1905	* their specific target workqueue. The only exception is works which
1906	* belong to workqueues with a rescuer which will be explained in
1907	* rescuer_thread().
1908	*/
1909	static int worker_thread(void *__worker)
1910	{
1911	struct worker *worker = __worker;
1912	struct global_cwq *gcwq = worker->gcwq;
1913
1914	/* tell the scheduler that this is a workqueue worker */
1915	worker->task->flags \|= PF_WQ_WORKER;
1916	woke_up:
1917	spin_lock_irq(&gcwq->lock);
1918
1919	/* DIE can be set only while we're idle, checking here is enough */
1920	if (worker->flags & WORKER_DIE) {
1921	spin_unlock_irq(&gcwq->lock);
1922	worker->task->flags &= ~PF_WQ_WORKER;
1923	return 0;
1924	}
1925
1926	worker_leave_idle(worker);
1927	recheck:
1928	/* no more worker necessary? */
1929	if (!need_more_worker(gcwq))
1930	goto sleep;
1931
1932	/* do we need to manage? */
1933	if (unlikely(!may_start_working(gcwq)) && manage_workers(worker))
1934	goto recheck;
1935
1936	/*
1937	* ->scheduled list can only be filled while a worker is
1938	* preparing to process a work or actually processing it.
1939	* Make sure nobody diddled with it while I was sleeping.
1940	*/
1941	BUG_ON(!list_empty(&worker->scheduled));
1942
1943	/*
1944	* When control reaches this point, we're guaranteed to have
1945	* at least one idle worker or that someone else has already
1946	* assumed the manager role.
1947	*/
1948	worker_clr_flags(worker, WORKER_PREP);
1949
1950	do {
1951	struct work_struct *work =
1952	list_first_entry(&gcwq->worklist,
1953	struct work_struct, entry);
1954
1955	if (likely(!(*work_data_bits(work) & WORK_STRUCT_LINKED))) {
1956	/* optimization path, not strictly necessary */
1957	process_one_work(worker, work);
1958	if (unlikely(!list_empty(&worker->scheduled)))
1959	process_scheduled_works(worker);
1960	} else {
1961	move_linked_works(work, &worker->scheduled, NULL);
1962	process_scheduled_works(worker);
1963	}
1964	} while (keep_working(gcwq));
1965
1966	worker_set_flags(worker, WORKER_PREP, false);
1967	sleep:
1968	if (unlikely(need_to_manage_workers(gcwq)) && manage_workers(worker))
1969	goto recheck;
1970
1971	/*
1972	* gcwq->lock is held and there's no work to process and no
1973	* need to manage, sleep. Workers are woken up only while
1974	* holding gcwq->lock or from local cpu, so setting the
1975	* current state before releasing gcwq->lock is enough to
1976	* prevent losing any event.
1977	*/
1978	worker_enter_idle(worker);
1979	__set_current_state(TASK_INTERRUPTIBLE);
1980	spin_unlock_irq(&gcwq->lock);
1981	schedule();
1982	goto woke_up;
1983	}
1984
1985	/**
1986	* rescuer_thread - the rescuer thread function
1987	* @__wq: the associated workqueue
1988	*
1989	* Workqueue rescuer thread function. There's one rescuer for each
1990	* workqueue which has WQ_RESCUER set.
1991	*
1992	* Regular work processing on a gcwq may block trying to create a new
1993	* worker which uses GFP_KERNEL allocation which has slight chance of
1994	* developing into deadlock if some works currently on the same queue
1995	* need to be processed to satisfy the GFP_KERNEL allocation. This is
1996	* the problem rescuer solves.
1997	*
1998	* When such condition is possible, the gcwq summons rescuers of all
1999	* workqueues which have works queued on the gcwq and let them process
2000	* those works so that forward progress can be guaranteed.
2001	*
2002	* This should happen rarely.
2003	*/
2004	static int rescuer_thread(void *__wq)
2005	{
2006	struct workqueue_struct *wq = __wq;
2007	struct worker *rescuer = wq->rescuer;
2008	struct list_head *scheduled = &rescuer->scheduled;
2009	bool is_unbound = wq->flags & WQ_UNBOUND;
2010	unsigned int cpu;
2011
2012	set_user_nice(current, RESCUER_NICE_LEVEL);
2013	repeat:
2014	set_current_state(TASK_INTERRUPTIBLE);
2015
2016	if (kthread_should_stop())
2017	return 0;
2018
2019	/*
2020	* See whether any cpu is asking for help. Unbounded
2021	* workqueues use cpu 0 in mayday_mask for CPU_UNBOUND.
2022	*/
2023	for_each_mayday_cpu(cpu, wq->mayday_mask) {
2024	unsigned int tcpu = is_unbound ? WORK_CPU_UNBOUND : cpu;
2025	struct cpu_workqueue_struct *cwq = get_cwq(tcpu, wq);
2026	struct global_cwq *gcwq = cwq->gcwq;
2027	struct work_struct work, n;
2028
2029	__set_current_state(TASK_RUNNING);
2030	mayday_clear_cpu(cpu, wq->mayday_mask);
2031
2032	/* migrate to the target cpu if possible */
2033	rescuer->gcwq = gcwq;
2034	worker_maybe_bind_and_lock(rescuer);
2035
2036	/*
2037	* Slurp in all works issued via this workqueue and
2038	* process'em.
2039	*/
2040	BUG_ON(!list_empty(&rescuer->scheduled));
2041	list_for_each_entry_safe(work, n, &gcwq->worklist, entry)
2042	if (get_work_cwq(work) == cwq)
2043	move_linked_works(work, scheduled, &n);
2044
2045	process_scheduled_works(rescuer);
2046	spin_unlock_irq(&gcwq->lock);
2047	}
2048
2049	schedule();
2050	goto repeat;
2051	}
2052
2053	struct wq_barrier {
2054	struct work_struct work;
2055	struct completion done;
2056	};
2057
2058	static void wq_barrier_func(struct work_struct *work)
2059	{
2060	struct wq_barrier *barr = container_of(work, struct wq_barrier, work);
2061	complete(&barr->done);
2062	}
2063
2064	/**
2065	* insert_wq_barrier - insert a barrier work
2066	* @cwq: cwq to insert barrier into
2067	* @barr: wq_barrier to insert
2068	* @target: target work to attach @barr to
2069	* @worker: worker currently executing @target, NULL if @target is not executing
2070	*
2071	* @barr is linked to @target such that @barr is completed only after
2072	* @target finishes execution. Please note that the ordering
2073	* guarantee is observed only with respect to @target and on the local
2074	* cpu.
2075	*
2076	* Currently, a queued barrier can't be canceled. This is because
2077	* try_to_grab_pending() can't determine whether the work to be
2078	* grabbed is at the head of the queue and thus can't clear LINKED
2079	* flag of the previous work while there must be a valid next work
2080	* after a work with LINKED flag set.
2081	*
2082	* Note that when @worker is non-NULL, @target may be modified
2083	* underneath us, so we can't reliably determine cwq from @target.
2084	*
2085	* CONTEXT:
2086	* spin_lock_irq(gcwq->lock).
2087	*/
2088	static void insert_wq_barrier(struct cpu_workqueue_struct *cwq,
2089	struct wq_barrier *barr,
2090	struct work_struct target, struct worker worker)
2091	{
2092	struct list_head *head;
2093	unsigned int linked = 0;
2094
2095	/*
2096	* debugobject calls are safe here even with gcwq->lock locked
2097	* as we know for sure that this will not trigger any of the
2098	* checks and call back into the fixup functions where we
2099	* might deadlock.
2100	*/
2101	INIT_WORK_ONSTACK(&barr->work, wq_barrier_func);
2102	__set_bit(WORK_STRUCT_PENDING_BIT, work_data_bits(&barr->work));
2103	init_completion(&barr->done);
2104
2105	/*
2106	* If @target is currently being executed, schedule the
2107	* barrier to the worker; otherwise, put it after @target.
2108	*/
2109	if (worker)
2110	head = worker->scheduled.next;
2111	else {
2112	unsigned long *bits = work_data_bits(target);
2113
2114	head = target->entry.next;
2115	/* there can already be other linked works, inherit and set */
2116	linked = *bits & WORK_STRUCT_LINKED;
2117	__set_bit(WORK_STRUCT_LINKED_BIT, bits);
2118	}
2119
2120	debug_work_activate(&barr->work);
2121	insert_work(cwq, &barr->work, head,
2122	work_color_to_flags(WORK_NO_COLOR) \| linked);
2123	}
2124
2125	/**
2126	* flush_workqueue_prep_cwqs - prepare cwqs for workqueue flushing
2127	* @wq: workqueue being flushed
2128	* @flush_color: new flush color, < 0 for no-op
2129	* @work_color: new work color, < 0 for no-op
2130	*
2131	* Prepare cwqs for workqueue flushing.
2132	*
2133	* If @flush_color is non-negative, flush_color on all cwqs should be
2134	* -1. If no cwq has in-flight commands at the specified color, all
2135	* cwq->flush_color's stay at -1 and %false is returned. If any cwq
2136	* has in flight commands, its cwq->flush_color is set to
2137	* @flush_color, @wq->nr_cwqs_to_flush is updated accordingly, cwq
2138	* wakeup logic is armed and %true is returned.
2139	*
2140	* The caller should have initialized @wq->first_flusher prior to
2141	* calling this function with non-negative @flush_color. If
2142	* @flush_color is negative, no flush color update is done and %false
2143	* is returned.
2144	*
2145	* If @work_color is non-negative, all cwqs should have the same
2146	* work_color which is previous to @work_color and all will be
2147	* advanced to @work_color.
2148	*
2149	* CONTEXT:
2150	* mutex_lock(wq->flush_mutex).
2151	*
2152	* RETURNS:
2153	* %true if @flush_color >= 0 and there's something to flush. %false
2154	* otherwise.
2155	*/
2156	static bool flush_workqueue_prep_cwqs(struct workqueue_struct *wq,
2157	int flush_color, int work_color)
2158	{
2159	bool wait = false;
2160	unsigned int cpu;
2161
2162	if (flush_color >= 0) {
2163	BUG_ON(atomic_read(&wq->nr_cwqs_to_flush));
2164	atomic_set(&wq->nr_cwqs_to_flush, 1);
2165	}
2166
2167	for_each_cwq_cpu(cpu, wq) {
2168	struct cpu_workqueue_struct *cwq = get_cwq(cpu, wq);
2169	struct global_cwq *gcwq = cwq->gcwq;
2170
2171	spin_lock_irq(&gcwq->lock);
2172
2173	if (flush_color >= 0) {
2174	BUG_ON(cwq->flush_color != -1);
2175
2176	if (cwq->nr_in_flight[flush_color]) {
2177	cwq->flush_color = flush_color;
2178	atomic_inc(&wq->nr_cwqs_to_flush);
2179	wait = true;
2180	}
2181	}
2182
2183	if (work_color >= 0) {
2184	BUG_ON(work_color != work_next_color(cwq->work_color));
2185	cwq->work_color = work_color;
2186	}
2187
2188	spin_unlock_irq(&gcwq->lock);
2189	}
2190
2191	if (flush_color >= 0 && atomic_dec_and_test(&wq->nr_cwqs_to_flush))
2192	complete(&wq->first_flusher->done);
2193
2194	return wait;
2195	}
2196
2197	/**
2198	* flush_workqueue - ensure that any scheduled work has run to completion.
2199	* @wq: workqueue to flush
2200	*
2201	* Forces execution of the workqueue and blocks until its completion.
2202	* This is typically used in driver shutdown handlers.
2203	*
2204	* We sleep until all works which were queued on entry have been handled,
2205	* but we are not livelocked by new incoming ones.
2206	*/
2207	void flush_workqueue(struct workqueue_struct *wq)
2208	{
2209	struct wq_flusher this_flusher = {
2210	.list = LIST_HEAD_INIT(this_flusher.list),
2211	.flush_color = -1,
2212	.done = COMPLETION_INITIALIZER_ONSTACK(this_flusher.done),
2213	};
2214	int next_color;
2215
2216	lock_map_acquire(&wq->lockdep_map);
2217	lock_map_release(&wq->lockdep_map);
2218
2219	mutex_lock(&wq->flush_mutex);
2220
2221	/*
2222	* Start-to-wait phase
2223	*/
2224	next_color = work_next_color(wq->work_color);
2225
2226	if (next_color != wq->flush_color) {
2227	/*
2228	* Color space is not full. The current work_color
2229	* becomes our flush_color and work_color is advanced
2230	* by one.
2231	*/
2232	BUG_ON(!list_empty(&wq->flusher_overflow));
2233	this_flusher.flush_color = wq->work_color;
2234	wq->work_color = next_color;
2235
2236	if (!wq->first_flusher) {
2237	/* no flush in progress, become the first flusher */
2238	BUG_ON(wq->flush_color != this_flusher.flush_color);
2239
2240	wq->first_flusher = &this_flusher;
2241
2242	if (!flush_workqueue_prep_cwqs(wq, wq->flush_color,
2243	wq->work_color)) {
2244	/* nothing to flush, done */
2245	wq->flush_color = next_color;
2246	wq->first_flusher = NULL;
2247	goto out_unlock;
2248	}
2249	} else {
2250	/* wait in queue */
2251	BUG_ON(wq->flush_color == this_flusher.flush_color);
2252	list_add_tail(&this_flusher.list, &wq->flusher_queue);
2253	flush_workqueue_prep_cwqs(wq, -1, wq->work_color);
2254	}
2255	} else {
2256	/*
2257	* Oops, color space is full, wait on overflow queue.
2258	* The next flush completion will assign us
2259	* flush_color and transfer to flusher_queue.
2260	*/
2261	list_add_tail(&this_flusher.list, &wq->flusher_overflow);
2262	}
2263
2264	mutex_unlock(&wq->flush_mutex);
2265
2266	wait_for_completion(&this_flusher.done);
2267
2268	/*
2269	* Wake-up-and-cascade phase
2270	*
2271	* First flushers are responsible for cascading flushes and
2272	* handling overflow. Non-first flushers can simply return.
2273	*/
2274	if (wq->first_flusher != &this_flusher)
2275	return;
2276
2277	mutex_lock(&wq->flush_mutex);
2278
2279	/* we might have raced, check again with mutex held */
2280	if (wq->first_flusher != &this_flusher)
2281	goto out_unlock;
2282
2283	wq->first_flusher = NULL;
2284
2285	BUG_ON(!list_empty(&this_flusher.list));
2286	BUG_ON(wq->flush_color != this_flusher.flush_color);
2287
2288	while (true) {
2289	struct wq_flusher next, tmp;
2290
2291	/* complete all the flushers sharing the current flush color */
2292	list_for_each_entry_safe(next, tmp, &wq->flusher_queue, list) {
2293	if (next->flush_color != wq->flush_color)
2294	break;
2295	list_del_init(&next->list);
2296	complete(&next->done);
2297	}
2298
2299	BUG_ON(!list_empty(&wq->flusher_overflow) &&
2300	wq->flush_color != work_next_color(wq->work_color));
2301
2302	/* this flush_color is finished, advance by one */
2303	wq->flush_color = work_next_color(wq->flush_color);
2304
2305	/* one color has been freed, handle overflow queue */
2306	if (!list_empty(&wq->flusher_overflow)) {
2307	/*
2308	* Assign the same color to all overflowed
2309	* flushers, advance work_color and append to
2310	* flusher_queue. This is the start-to-wait
2311	* phase for these overflowed flushers.
2312	*/
2313	list_for_each_entry(tmp, &wq->flusher_overflow, list)
2314	tmp->flush_color = wq->work_color;
2315
2316	wq->work_color = work_next_color(wq->work_color);
2317
2318	list_splice_tail_init(&wq->flusher_overflow,
2319	&wq->flusher_queue);
2320	flush_workqueue_prep_cwqs(wq, -1, wq->work_color);
2321	}
2322
2323	if (list_empty(&wq->flusher_queue)) {
2324	BUG_ON(wq->flush_color != wq->work_color);
2325	break;
2326	}
2327
2328	/*
2329	* Need to flush more colors. Make the next flusher
2330	* the new first flusher and arm cwqs.
2331	*/
2332	BUG_ON(wq->flush_color == wq->work_color);
2333	BUG_ON(wq->flush_color != next->flush_color);
2334
2335	list_del_init(&next->list);
2336	wq->first_flusher = next;
2337
2338	if (flush_workqueue_prep_cwqs(wq, wq->flush_color, -1))
2339	break;
2340
2341	/*
2342	* Meh... this color is already done, clear first
2343	* flusher and repeat cascading.
2344	*/
2345	wq->first_flusher = NULL;
2346	}
2347
2348	out_unlock:
2349	mutex_unlock(&wq->flush_mutex);
2350	}
2351	EXPORT_SYMBOL_GPL(flush_workqueue);
2352
2353	static bool start_flush_work(struct work_struct work, struct wq_barrier barr,
2354	bool wait_executing)
2355	{
2356	struct worker *worker = NULL;
2357	struct global_cwq *gcwq;
2358	struct cpu_workqueue_struct *cwq;
2359
2360	might_sleep();
2361	gcwq = get_work_gcwq(work);
2362	if (!gcwq)
2363	return false;
2364
2365	spin_lock_irq(&gcwq->lock);
2366	if (!list_empty(&work->entry)) {
2367	/*
2368	* See the comment near try_to_grab_pending()->smp_rmb().
2369	* If it was re-queued to a different gcwq under us, we
2370	* are not going to wait.
2371	*/
2372	smp_rmb();
2373	cwq = get_work_cwq(work);
2374	if (unlikely(!cwq \|\| gcwq != cwq->gcwq))
2375	goto already_gone;
2376	} else if (wait_executing) {
2377	worker = find_worker_executing_work(gcwq, work);
2378	if (!worker)
2379	goto already_gone;
2380	cwq = worker->current_cwq;
2381	} else
2382	goto already_gone;
2383
2384	insert_wq_barrier(cwq, barr, work, worker);
2385	spin_unlock_irq(&gcwq->lock);
2386
2387	lock_map_acquire(&cwq->wq->lockdep_map);
2388	lock_map_release(&cwq->wq->lockdep_map);
2389	return true;
2390	already_gone:
2391	spin_unlock_irq(&gcwq->lock);
2392	return false;
2393	}
2394
2395	/**
2396	* flush_work - wait for a work to finish executing the last queueing instance
2397	* @work: the work to flush
2398	*
2399	* Wait until @work has finished execution. This function considers
2400	* only the last queueing instance of @work. If @work has been
2401	* enqueued across different CPUs on a non-reentrant workqueue or on
2402	* multiple workqueues, @work might still be executing on return on
2403	* some of the CPUs from earlier queueing.
2404	*
2405	* If @work was queued only on a non-reentrant, ordered or unbound
2406	* workqueue, @work is guaranteed to be idle on return if it hasn't
2407	* been requeued since flush started.
2408	*
2409	* RETURNS:
2410	* %true if flush_work() waited for the work to finish execution,
2411	* %false if it was already idle.
2412	*/
2413	bool flush_work(struct work_struct *work)
2414	{
2415	struct wq_barrier barr;
2416
2417	if (start_flush_work(work, &barr, true)) {
2418	wait_for_completion(&barr.done);
2419	destroy_work_on_stack(&barr.work);
2420	return true;
2421	} else
2422	return false;
2423	}
2424	EXPORT_SYMBOL_GPL(flush_work);
2425
2426	static bool wait_on_cpu_work(struct global_cwq gcwq, struct work_struct work)
2427	{
2428	struct wq_barrier barr;
2429	struct worker *worker;
2430
2431	spin_lock_irq(&gcwq->lock);
2432
2433	worker = find_worker_executing_work(gcwq, work);
2434	if (unlikely(worker))
2435	insert_wq_barrier(worker->current_cwq, &barr, work, worker);
2436
2437	spin_unlock_irq(&gcwq->lock);
2438
2439	if (unlikely(worker)) {
2440	wait_for_completion(&barr.done);
2441	destroy_work_on_stack(&barr.work);
2442	return true;
2443	} else
2444	return false;
2445	}
2446
2447	static bool wait_on_work(struct work_struct *work)
2448	{
2449	bool ret = false;
2450	int cpu;
2451
2452	might_sleep();
2453
2454	lock_map_acquire(&work->lockdep_map);
2455	lock_map_release(&work->lockdep_map);
2456
2457	for_each_gcwq_cpu(cpu)
2458	ret \|= wait_on_cpu_work(get_gcwq(cpu), work);
2459	return ret;
2460	}
2461
2462	/**
2463	* flush_work_sync - wait until a work has finished execution
2464	* @work: the work to flush
2465	*
2466	* Wait until @work has finished execution. On return, it's
2467	* guaranteed that all queueing instances of @work which happened
2468	* before this function is called are finished. In other words, if
2469	* @work hasn't been requeued since this function was called, @work is
2470	* guaranteed to be idle on return.
2471	*
2472	* RETURNS:
2473	* %true if flush_work_sync() waited for the work to finish execution,
2474	* %false if it was already idle.
2475	*/
2476	bool flush_work_sync(struct work_struct *work)
2477	{
2478	struct wq_barrier barr;
2479	bool pending, waited;
2480
2481	/* we'll wait for executions separately, queue barr only if pending */
2482	pending = start_flush_work(work, &barr, false);
2483
2484	/* wait for executions to finish */
2485	waited = wait_on_work(work);
2486
2487	/* wait for the pending one */
2488	if (pending) {
2489	wait_for_completion(&barr.done);
2490	destroy_work_on_stack(&barr.work);
2491	}
2492
2493	return pending \|\| waited;
2494	}
2495	EXPORT_SYMBOL_GPL(flush_work_sync);
2496
2497	/*
2498	* Upon a successful return (>= 0), the caller "owns" WORK_STRUCT_PENDING bit,
2499	* so this work can't be re-armed in any way.
2500	*/
2501	static int try_to_grab_pending(struct work_struct *work)
2502	{
2503	struct global_cwq *gcwq;
2504	int ret = -1;
2505
2506	if (!test_and_set_bit(WORK_STRUCT_PENDING_BIT, work_data_bits(work)))
2507	return 0;
2508
2509	/*
2510	* The queueing is in progress, or it is already queued. Try to
2511	* steal it from ->worklist without clearing WORK_STRUCT_PENDING.
2512	*/
2513	gcwq = get_work_gcwq(work);
2514	if (!gcwq)
2515	return ret;
2516
2517	spin_lock_irq(&gcwq->lock);
2518	if (!list_empty(&work->entry)) {
2519	/*
2520	* This work is queued, but perhaps we locked the wrong gcwq.
2521	* In that case we must see the new value after rmb(), see
2522	* insert_work()->wmb().
2523	*/
2524	smp_rmb();
2525	if (gcwq == get_work_gcwq(work)) {
2526	debug_work_deactivate(work);
2527	list_del_init(&work->entry);
2528	cwq_dec_nr_in_flight(get_work_cwq(work),
2529	get_work_color(work),
2530	*work_data_bits(work) & WORK_STRUCT_DELAYED);
2531	ret = 1;
2532	}
2533	}
2534	spin_unlock_irq(&gcwq->lock);
2535
2536	return ret;
2537	}
2538
2539	static bool __cancel_work_timer(struct work_struct *work,
2540	struct timer_list* timer)
2541	{
2542	int ret;
2543
2544	do {
2545	ret = (timer && likely(del_timer(timer)));
2546	if (!ret)
2547	ret = try_to_grab_pending(work);
2548	wait_on_work(work);
2549	} while (unlikely(ret < 0));
2550
2551	clear_work_data(work);
2552	return ret;
2553	}
2554
2555	/**
2556	* cancel_work_sync - cancel a work and wait for it to finish
2557	* @work: the work to cancel
2558	*
2559	* Cancel @work and wait for its execution to finish. This function
2560	* can be used even if the work re-queues itself or migrates to
2561	* another workqueue. On return from this function, @work is
2562	* guaranteed to be not pending or executing on any CPU.
2563	*
2564	* cancel_work_sync(&delayed_work->work) must not be used for
2565	* delayed_work's. Use cancel_delayed_work_sync() instead.
2566	*
2567	* The caller must ensure that the workqueue on which @work was last
2568	* queued can't be destroyed before this function returns.
2569	*
2570	* RETURNS:
2571	* %true if @work was pending, %false otherwise.
2572	*/
2573	bool cancel_work_sync(struct work_struct *work)
2574	{
2575	return __cancel_work_timer(work, NULL);
2576	}
2577	EXPORT_SYMBOL_GPL(cancel_work_sync);
2578
2579	/**
2580	* flush_delayed_work - wait for a dwork to finish executing the last queueing
2581	* @dwork: the delayed work to flush
2582	*
2583	* Delayed timer is cancelled and the pending work is queued for
2584	* immediate execution. Like flush_work(), this function only
2585	* considers the last queueing instance of @dwork.
2586	*
2587	* RETURNS:
2588	* %true if flush_work() waited for the work to finish execution,
2589	* %false if it was already idle.
2590	*/
2591	bool flush_delayed_work(struct delayed_work *dwork)
2592	{
2593	if (del_timer_sync(&dwork->timer))
2594	__queue_work(raw_smp_processor_id(),
2595	get_work_cwq(&dwork->work)->wq, &dwork->work);
2596	return flush_work(&dwork->work);
2597	}
2598	EXPORT_SYMBOL(flush_delayed_work);
2599
2600	/**
2601	* flush_delayed_work_sync - wait for a dwork to finish
2602	* @dwork: the delayed work to flush
2603	*
2604	* Delayed timer is cancelled and the pending work is queued for
2605	* execution immediately. Other than timer handling, its behavior
2606	* is identical to flush_work_sync().
2607	*
2608	* RETURNS:
2609	* %true if flush_work_sync() waited for the work to finish execution,
2610	* %false if it was already idle.
2611	*/
2612	bool flush_delayed_work_sync(struct delayed_work *dwork)
2613	{
2614	if (del_timer_sync(&dwork->timer))
2615	__queue_work(raw_smp_processor_id(),
2616	get_work_cwq(&dwork->work)->wq, &dwork->work);
2617	return flush_work_sync(&dwork->work);
2618	}
2619	EXPORT_SYMBOL(flush_delayed_work_sync);
2620
2621	/**
2622	* cancel_delayed_work_sync - cancel a delayed work and wait for it to finish
2623	* @dwork: the delayed work cancel
2624	*
2625	* This is cancel_work_sync() for delayed works.
2626	*
2627	* RETURNS:
2628	* %true if @dwork was pending, %false otherwise.
2629	*/
2630	bool cancel_delayed_work_sync(struct delayed_work *dwork)
2631	{
2632	return __cancel_work_timer(&dwork->work, &dwork->timer);
2633	}
2634	EXPORT_SYMBOL(cancel_delayed_work_sync);
2635
2636	/**
2637	* schedule_work - put work task in global workqueue
2638	* @work: job to be done
2639	*
2640	* Returns zero if @work was already on the kernel-global workqueue and
2641	* non-zero otherwise.
2642	*
2643	* This puts a job in the kernel-global workqueue if it was not already
2644	* queued and leaves it in the same position on the kernel-global
2645	* workqueue otherwise.
2646	*/
2647	int schedule_work(struct work_struct *work)
2648	{
2649	return queue_work(system_wq, work);
2650	}
2651	EXPORT_SYMBOL(schedule_work);
2652
2653	/*
2654	* schedule_work_on - put work task on a specific cpu
2655	* @cpu: cpu to put the work task on
2656	* @work: job to be done
2657	*
2658	* This puts a job on a specific cpu
2659	*/
2660	int schedule_work_on(int cpu, struct work_struct *work)
2661	{
2662	return queue_work_on(cpu, system_wq, work);
2663	}
2664	EXPORT_SYMBOL(schedule_work_on);
2665
2666	/**
2667	* schedule_delayed_work - put work task in global workqueue after delay
2668	* @dwork: job to be done
2669	* @delay: number of jiffies to wait or 0 for immediate execution
2670	*
2671	* After waiting for a given time this puts a job in the kernel-global
2672	* workqueue.
2673	*/
2674	int schedule_delayed_work(struct delayed_work *dwork,
2675	unsigned long delay)
2676	{
2677	return queue_delayed_work(system_wq, dwork, delay);
2678	}
2679	EXPORT_SYMBOL(schedule_delayed_work);
2680
2681	/**
2682	* schedule_delayed_work_on - queue work in global workqueue on CPU after delay
2683	* @cpu: cpu to use
2684	* @dwork: job to be done
2685	* @delay: number of jiffies to wait
2686	*
2687	* After waiting for a given time this puts a job in the kernel-global
2688	* workqueue on the specified CPU.
2689	*/
2690	int schedule_delayed_work_on(int cpu,
2691	struct delayed_work *dwork, unsigned long delay)
2692	{
2693	return queue_delayed_work_on(cpu, system_wq, dwork, delay);
2694	}
2695	EXPORT_SYMBOL(schedule_delayed_work_on);
2696
2697	/**
2698	* schedule_on_each_cpu - execute a function synchronously on each online CPU
2699	* @func: the function to call
2700	*
2701	* schedule_on_each_cpu() executes @func on each online CPU using the
2702	* system workqueue and blocks until all CPUs have completed.
2703	* schedule_on_each_cpu() is very slow.
2704	*
2705	* RETURNS:
2706	* 0 on success, -errno on failure.
2707	*/
2708	int schedule_on_each_cpu(work_func_t func)
2709	{
2710	int cpu;
2711	struct work_struct __percpu *works;
2712
2713	works = alloc_percpu(struct work_struct);
2714	if (!works)
2715	return -ENOMEM;
2716
2717	get_online_cpus();
2718
2719	for_each_online_cpu(cpu) {
2720	struct work_struct *work = per_cpu_ptr(works, cpu);
2721
2722	INIT_WORK(work, func);
2723	schedule_work_on(cpu, work);
2724	}
2725
2726	for_each_online_cpu(cpu)
2727	flush_work(per_cpu_ptr(works, cpu));
2728
2729	put_online_cpus();
2730	free_percpu(works);
2731	return 0;
2732	}
2733
2734	/**
2735	* flush_scheduled_work - ensure that any scheduled work has run to completion.
2736	*
2737	* Forces execution of the kernel-global workqueue and blocks until its
2738	* completion.
2739	*
2740	* Think twice before calling this function! It's very easy to get into
2741	* trouble if you don't take great care. Either of the following situations
2742	* will lead to deadlock:
2743	*
2744	* One of the work items currently on the workqueue needs to acquire
2745	* a lock held by your code or its caller.
2746	*
2747	* Your code is running in the context of a work routine.
2748	*
2749	* They will be detected by lockdep when they occur, but the first might not
2750	* occur very often. It depends on what work items are on the workqueue and
2751	* what locks they need, which you have no control over.
2752	*
2753	* In most situations flushing the entire workqueue is overkill; you merely
2754	* need to know that a particular work item isn't queued and isn't running.
2755	* In such cases you should use cancel_delayed_work_sync() or
2756	* cancel_work_sync() instead.
2757	*/
2758	void flush_scheduled_work(void)
2759	{
2760	flush_workqueue(system_wq);
2761	}
2762	EXPORT_SYMBOL(flush_scheduled_work);
2763
2764	/**
2765	* execute_in_process_context - reliably execute the routine with user context
2766	* @fn: the function to execute
2767	* @ew: guaranteed storage for the execute work structure (must
2768	* be available when the work executes)
2769	*
2770	* Executes the function immediately if process context is available,
2771	* otherwise schedules the function for delayed execution.
2772	*
2773	* Returns: 0 - function was executed
2774	* 1 - function was scheduled for execution
2775	*/
2776	int execute_in_process_context(work_func_t fn, struct execute_work *ew)
2777	{
2778	if (!in_interrupt()) {
2779	fn(&ew->work);
2780	return 0;
2781	}
2782
2783	INIT_WORK(&ew->work, fn);
2784	schedule_work(&ew->work);
2785
2786	return 1;
2787	}
2788	EXPORT_SYMBOL_GPL(execute_in_process_context);
2789
2790	int keventd_up(void)
2791	{
2792	return system_wq != NULL;
2793	}
2794
2795	static int alloc_cwqs(struct workqueue_struct *wq)
2796	{
2797	/*
2798	* cwqs are forced aligned according to WORK_STRUCT_FLAG_BITS.
2799	* Make sure that the alignment isn't lower than that of
2800	* unsigned long long.
2801	*/
2802	const size_t size = sizeof(struct cpu_workqueue_struct);
2803	const size_t align = max_t(size_t, 1 << WORK_STRUCT_FLAG_BITS,
2804	__alignof__(unsigned long long));
2805	#ifdef CONFIG_SMP
2806	bool percpu = !(wq->flags & WQ_UNBOUND);
2807	#else
2808	bool percpu = false;
2809	#endif
2810
2811	if (percpu)
2812	wq->cpu_wq.pcpu = __alloc_percpu(size, align);
2813	else {
2814	void *ptr;
2815
2816	/*
2817	* Allocate enough room to align cwq and put an extra
2818	* pointer at the end pointing back to the originally
2819	* allocated pointer which will be used for free.
2820	*/
2821	ptr = kzalloc(size + align + sizeof(void *), GFP_KERNEL);
2822	if (ptr) {
2823	wq->cpu_wq.single = PTR_ALIGN(ptr, align);
2824	(void *)(wq->cpu_wq.single + 1) = ptr;
2825	}
2826	}
2827
2828	/* just in case, make sure it's actually aligned
2829	* - this is affected by PERCPU() alignment in vmlinux.lds.S
2830	*/
2831	BUG_ON(!IS_ALIGNED(wq->cpu_wq.v, align));
2832	return wq->cpu_wq.v ? 0 : -ENOMEM;
2833	}
2834
2835	static void free_cwqs(struct workqueue_struct *wq)
2836	{
2837	#ifdef CONFIG_SMP
2838	bool percpu = !(wq->flags & WQ_UNBOUND);
2839	#else
2840	bool percpu = false;
2841	#endif
2842
2843	if (percpu)
2844	free_percpu(wq->cpu_wq.pcpu);
2845	else if (wq->cpu_wq.single) {
2846	/* the pointer to free is stored right after the cwq */
2847	kfree((void *)(wq->cpu_wq.single + 1));
2848	}
2849	}
2850
2851	static int wq_clamp_max_active(int max_active, unsigned int flags,
2852	const char *name)
2853	{
2854	int lim = flags & WQ_UNBOUND ? WQ_UNBOUND_MAX_ACTIVE : WQ_MAX_ACTIVE;
2855
2856	if (max_active < 1 \|\| max_active > lim)
2857	printk(KERN_WARNING "workqueue: max_active %d requested for %s "
2858	"is out of range, clamping between %d and %d\n",
2859	max_active, name, 1, lim);
2860
2861	return clamp_val(max_active, 1, lim);
2862	}
2863
2864	struct workqueue_struct __alloc_workqueue_key(const char name,
2865	unsigned int flags,
2866	int max_active,
2867	struct lock_class_key *key,
2868	const char *lock_name)
2869	{
2870	struct workqueue_struct *wq;
2871	unsigned int cpu;
2872
2873	/*
2874	* Workqueues which may be used during memory reclaim should
2875	* have a rescuer to guarantee forward progress.
2876	*/
2877	if (flags & WQ_MEM_RECLAIM)
2878	flags \|= WQ_RESCUER;
2879
2880	/*
2881	* Unbound workqueues aren't concurrency managed and should be
2882	* dispatched to workers immediately.
2883	*/
2884	if (flags & WQ_UNBOUND)
2885	flags \|= WQ_HIGHPRI;
2886
2887	max_active = max_active ?: WQ_DFL_ACTIVE;
2888	max_active = wq_clamp_max_active(max_active, flags, name);
2889
2890	wq = kzalloc(sizeof(*wq), GFP_KERNEL);
2891	if (!wq)
2892	goto err;
2893
2894	wq->flags = flags;
2895	wq->saved_max_active = max_active;
2896	mutex_init(&wq->flush_mutex);
2897	atomic_set(&wq->nr_cwqs_to_flush, 0);
2898	INIT_LIST_HEAD(&wq->flusher_queue);
2899	INIT_LIST_HEAD(&wq->flusher_overflow);
2900
2901	wq->name = name;
2902	lockdep_init_map(&wq->lockdep_map, lock_name, key, 0);
2903	INIT_LIST_HEAD(&wq->list);
2904
2905	if (alloc_cwqs(wq) < 0)
2906	goto err;
2907
2908	for_each_cwq_cpu(cpu, wq) {
2909	struct cpu_workqueue_struct *cwq = get_cwq(cpu, wq);
2910	struct global_cwq *gcwq = get_gcwq(cpu);
2911
2912	BUG_ON((unsigned long)cwq & WORK_STRUCT_FLAG_MASK);
2913	cwq->gcwq = gcwq;
2914	cwq->wq = wq;
2915	cwq->flush_color = -1;
2916	cwq->max_active = max_active;
2917	INIT_LIST_HEAD(&cwq->delayed_works);
2918	}
2919
2920	if (flags & WQ_RESCUER) {
2921	struct worker *rescuer;
2922
2923	if (!alloc_mayday_mask(&wq->mayday_mask, GFP_KERNEL))
2924	goto err;
2925
2926	wq->rescuer = rescuer = alloc_worker();
2927	if (!rescuer)
2928	goto err;
2929
2930	rescuer->task = kthread_create(rescuer_thread, wq, "%s", name);
2931	if (IS_ERR(rescuer->task))
2932	goto err;
2933
2934	rescuer->task->flags \|= PF_THREAD_BOUND;
2935	wake_up_process(rescuer->task);
2936	}
2937
2938	/*
2939	* workqueue_lock protects global freeze state and workqueues
2940	* list. Grab it, set max_active accordingly and add the new
2941	* workqueue to workqueues list.
2942	*/
2943	spin_lock(&workqueue_lock);
2944
2945	if (workqueue_freezing && wq->flags & WQ_FREEZEABLE)
2946	for_each_cwq_cpu(cpu, wq)
2947	get_cwq(cpu, wq)->max_active = 0;
2948
2949	list_add(&wq->list, &workqueues);
2950
2951	spin_unlock(&workqueue_lock);
2952
2953	return wq;
2954	err:
2955	if (wq) {
2956	free_cwqs(wq);
2957	free_mayday_mask(wq->mayday_mask);
2958	kfree(wq->rescuer);
2959	kfree(wq);
2960	}
2961	return NULL;
2962	}
2963	EXPORT_SYMBOL_GPL(__alloc_workqueue_key);
2964
2965	/**
2966	* destroy_workqueue - safely terminate a workqueue
2967	* @wq: target workqueue
2968	*
2969	* Safely destroy a workqueue. All work currently pending will be done first.
2970	*/
2971	void destroy_workqueue(struct workqueue_struct *wq)
2972	{
2973	unsigned int flush_cnt = 0;
2974	unsigned int cpu;
2975
2976	/*
2977	* Mark @wq dying and drain all pending works. Once WQ_DYING is
2978	* set, only chain queueing is allowed. IOW, only currently
2979	* pending or running work items on @wq can queue further work
2980	* items on it. @wq is flushed repeatedly until it becomes empty.
2981	* The number of flushing is detemined by the depth of chaining and
2982	* should be relatively short. Whine if it takes too long.
2983	*/
2984	wq->flags \|= WQ_DYING;
2985	reflush:
2986	flush_workqueue(wq);
2987
2988	for_each_cwq_cpu(cpu, wq) {
2989	struct cpu_workqueue_struct *cwq = get_cwq(cpu, wq);
2990
2991	if (!cwq->nr_active && list_empty(&cwq->delayed_works))
2992	continue;
2993
2994	if (++flush_cnt == 10 \|\|
2995	(flush_cnt % 100 == 0 && flush_cnt <= 1000))
2996	printk(KERN_WARNING "workqueue %s: flush on "
2997	"destruction isn't complete after %u tries\n",
2998	wq->name, flush_cnt);
2999	goto reflush;
3000	}
3001
3002	/*
3003	* wq list is used to freeze wq, remove from list after
3004	* flushing is complete in case freeze races us.
3005	*/
3006	spin_lock(&workqueue_lock);
3007	list_del(&wq->list);
3008	spin_unlock(&workqueue_lock);
3009
3010	/* sanity check */
3011	for_each_cwq_cpu(cpu, wq) {
3012	struct cpu_workqueue_struct *cwq = get_cwq(cpu, wq);
3013	int i;
3014
3015	for (i = 0; i < WORK_NR_COLORS; i++)
3016	BUG_ON(cwq->nr_in_flight[i]);
3017	BUG_ON(cwq->nr_active);
3018	BUG_ON(!list_empty(&cwq->delayed_works));
3019	}
3020
3021	if (wq->flags & WQ_RESCUER) {
3022	kthread_stop(wq->rescuer->task);
3023	free_mayday_mask(wq->mayday_mask);
3024	kfree(wq->rescuer);
3025	}
3026
3027	free_cwqs(wq);
3028	kfree(wq);
3029	}
3030	EXPORT_SYMBOL_GPL(destroy_workqueue);
3031
3032	/**
3033	* workqueue_set_max_active - adjust max_active of a workqueue
3034	* @wq: target workqueue
3035	* @max_active: new max_active value.
3036	*
3037	* Set max_active of @wq to @max_active.
3038	*
3039	* CONTEXT:
3040	* Don't call from IRQ context.
3041	*/
3042	void workqueue_set_max_active(struct workqueue_struct *wq, int max_active)
3043	{
3044	unsigned int cpu;
3045
3046	max_active = wq_clamp_max_active(max_active, wq->flags, wq->name);
3047
3048	spin_lock(&workqueue_lock);
3049
3050	wq->saved_max_active = max_active;
3051
3052	for_each_cwq_cpu(cpu, wq) {
3053	struct global_cwq *gcwq = get_gcwq(cpu);
3054
3055	spin_lock_irq(&gcwq->lock);
3056
3057	if (!(wq->flags & WQ_FREEZEABLE) \|\|
3058	!(gcwq->flags & GCWQ_FREEZING))
3059	get_cwq(gcwq->cpu, wq)->max_active = max_active;
3060
3061	spin_unlock_irq(&gcwq->lock);
3062	}
3063
3064	spin_unlock(&workqueue_lock);
3065	}
3066	EXPORT_SYMBOL_GPL(workqueue_set_max_active);
3067
3068	/**
3069	* workqueue_congested - test whether a workqueue is congested
3070	* @cpu: CPU in question
3071	* @wq: target workqueue
3072	*
3073	* Test whether @wq's cpu workqueue for @cpu is congested. There is
3074	* no synchronization around this function and the test result is
3075	* unreliable and only useful as advisory hints or for debugging.
3076	*
3077	* RETURNS:
3078	* %true if congested, %false otherwise.
3079	*/
3080	bool workqueue_congested(unsigned int cpu, struct workqueue_struct *wq)
3081	{
3082	struct cpu_workqueue_struct *cwq = get_cwq(cpu, wq);
3083
3084	return !list_empty(&cwq->delayed_works);
3085	}
3086	EXPORT_SYMBOL_GPL(workqueue_congested);
3087
3088	/**
3089	* work_cpu - return the last known associated cpu for @work
3090	* @work: the work of interest
3091	*
3092	* RETURNS:
3093	* CPU number if @work was ever queued. WORK_CPU_NONE otherwise.
3094	*/
3095	unsigned int work_cpu(struct work_struct *work)
3096	{
3097	struct global_cwq *gcwq = get_work_gcwq(work);
3098
3099	return gcwq ? gcwq->cpu : WORK_CPU_NONE;
3100	}
3101	EXPORT_SYMBOL_GPL(work_cpu);
3102
3103	/**
3104	* work_busy - test whether a work is currently pending or running
3105	* @work: the work to be tested
3106	*
3107	* Test whether @work is currently pending or running. There is no
3108	* synchronization around this function and the test result is
3109	* unreliable and only useful as advisory hints or for debugging.
3110	* Especially for reentrant wqs, the pending state might hide the
3111	* running state.
3112	*
3113	* RETURNS:
3114	* OR'd bitmask of WORK_BUSY_* bits.
3115	*/
3116	unsigned int work_busy(struct work_struct *work)
3117	{
3118	struct global_cwq *gcwq = get_work_gcwq(work);
3119	unsigned long flags;
3120	unsigned int ret = 0;
3121
3122	if (!gcwq)
3123	return false;
3124
3125	spin_lock_irqsave(&gcwq->lock, flags);
3126
3127	if (work_pending(work))
3128	ret \|= WORK_BUSY_PENDING;
3129	if (find_worker_executing_work(gcwq, work))
3130	ret \|= WORK_BUSY_RUNNING;
3131
3132	spin_unlock_irqrestore(&gcwq->lock, flags);
3133
3134	return ret;
3135	}
3136	EXPORT_SYMBOL_GPL(work_busy);
3137
3138	/*
3139	* CPU hotplug.
3140	*
3141	* There are two challenges in supporting CPU hotplug. Firstly, there
3142	* are a lot of assumptions on strong associations among work, cwq and
3143	* gcwq which make migrating pending and scheduled works very
3144	* difficult to implement without impacting hot paths. Secondly,
3145	* gcwqs serve mix of short, long and very long running works making
3146	* blocked draining impractical.
3147	*
3148	* This is solved by allowing a gcwq to be detached from CPU, running
3149	* it with unbound (rogue) workers and allowing it to be reattached
3150	* later if the cpu comes back online. A separate thread is created
3151	* to govern a gcwq in such state and is called the trustee of the
3152	* gcwq.
3153	*
3154	* Trustee states and their descriptions.
3155	*
3156	* START Command state used on startup. On CPU_DOWN_PREPARE, a
3157	* new trustee is started with this state.
3158	*
3159	* IN_CHARGE Once started, trustee will enter this state after
3160	* assuming the manager role and making all existing
3161	* workers rogue. DOWN_PREPARE waits for trustee to
3162	* enter this state. After reaching IN_CHARGE, trustee
3163	* tries to execute the pending worklist until it's empty
3164	* and the state is set to BUTCHER, or the state is set
3165	* to RELEASE.
3166	*
3167	* BUTCHER Command state which is set by the cpu callback after
3168	* the cpu has went down. Once this state is set trustee
3169	* knows that there will be no new works on the worklist
3170	* and once the worklist is empty it can proceed to
3171	* killing idle workers.
3172	*
3173	* RELEASE Command state which is set by the cpu callback if the
3174	* cpu down has been canceled or it has come online
3175	* again. After recognizing this state, trustee stops
3176	* trying to drain or butcher and clears ROGUE, rebinds
3177	* all remaining workers back to the cpu and releases
3178	* manager role.
3179	*
3180	* DONE Trustee will enter this state after BUTCHER or RELEASE
3181	* is complete.
3182	*
3183	* trustee CPU draining
3184	* took over down complete
3185	* START -----------> IN_CHARGE -----------> BUTCHER -----------> DONE
3186	* \| \| ^
3187	* \| CPU is back online v return workers \|
3188	* ----------------> RELEASE --------------
3189	*/
3190
3191	/**
3192	* trustee_wait_event_timeout - timed event wait for trustee
3193	* @cond: condition to wait for
3194	* @timeout: timeout in jiffies
3195	*
3196	* wait_event_timeout() for trustee to use. Handles locking and
3197	* checks for RELEASE request.
3198	*
3199	* CONTEXT:
3200	* spin_lock_irq(gcwq->lock) which may be released and regrabbed
3201	* multiple times. To be used by trustee.
3202	*
3203	* RETURNS:
3204	* Positive indicating left time if @cond is satisfied, 0 if timed
3205	* out, -1 if canceled.
3206	*/
3207	#define trustee_wait_event_timeout(cond, timeout) ({ \
3208	long __ret = (timeout); \
3209	while (!((cond) \|\| (gcwq->trustee_state == TRUSTEE_RELEASE)) && \
3210	__ret) { \
3211	spin_unlock_irq(&gcwq->lock); \
3212	__wait_event_timeout(gcwq->trustee_wait, (cond) \|\| \
3213	(gcwq->trustee_state == TRUSTEE_RELEASE), \
3214	__ret); \
3215	spin_lock_irq(&gcwq->lock); \
3216	} \
3217	gcwq->trustee_state == TRUSTEE_RELEASE ? -1 : (__ret); \
3218	})
3219
3220	/**
3221	* trustee_wait_event - event wait for trustee
3222	* @cond: condition to wait for
3223	*
3224	* wait_event() for trustee to use. Automatically handles locking and
3225	* checks for CANCEL request.
3226	*
3227	* CONTEXT:
3228	* spin_lock_irq(gcwq->lock) which may be released and regrabbed
3229	* multiple times. To be used by trustee.
3230	*
3231	* RETURNS:
3232	* 0 if @cond is satisfied, -1 if canceled.
3233	*/
3234	#define trustee_wait_event(cond) ({ \
3235	long __ret1; \
3236	__ret1 = trustee_wait_event_timeout(cond, MAX_SCHEDULE_TIMEOUT);\
3237	__ret1 < 0 ? -1 : 0; \
3238	})
3239
3240	static int __cpuinit trustee_thread(void *__gcwq)
3241	{
3242	struct global_cwq *gcwq = __gcwq;
3243	struct worker *worker;
3244	struct work_struct *work;
3245	struct hlist_node *pos;
3246	long rc;
3247	int i;
3248
3249	BUG_ON(gcwq->cpu != smp_processor_id());
3250
3251	spin_lock_irq(&gcwq->lock);
3252	/*
3253	* Claim the manager position and make all workers rogue.
3254	* Trustee must be bound to the target cpu and can't be
3255	* cancelled.
3256	*/
3257	BUG_ON(gcwq->cpu != smp_processor_id());
3258	rc = trustee_wait_event(!(gcwq->flags & GCWQ_MANAGING_WORKERS));
3259	BUG_ON(rc < 0);
3260
3261	gcwq->flags \|= GCWQ_MANAGING_WORKERS;
3262
3263	list_for_each_entry(worker, &gcwq->idle_list, entry)
3264	worker->flags \|= WORKER_ROGUE;
3265
3266	for_each_busy_worker(worker, i, pos, gcwq)
3267	worker->flags \|= WORKER_ROGUE;
3268
3269	/*
3270	* Call schedule() so that we cross rq->lock and thus can
3271	* guarantee sched callbacks see the rogue flag. This is
3272	* necessary as scheduler callbacks may be invoked from other
3273	* cpus.
3274	*/
3275	spin_unlock_irq(&gcwq->lock);
3276	schedule();
3277	spin_lock_irq(&gcwq->lock);
3278
3279	/*
3280	* Sched callbacks are disabled now. Zap nr_running. After
3281	* this, nr_running stays zero and need_more_worker() and
3282	* keep_working() are always true as long as the worklist is
3283	* not empty.
3284	*/
3285	atomic_set(get_gcwq_nr_running(gcwq->cpu), 0);
3286
3287	spin_unlock_irq(&gcwq->lock);
3288	del_timer_sync(&gcwq->idle_timer);
3289	spin_lock_irq(&gcwq->lock);
3290
3291	/*
3292	* We're now in charge. Notify and proceed to drain. We need
3293	* to keep the gcwq running during the whole CPU down
3294	* procedure as other cpu hotunplug callbacks may need to
3295	* flush currently running tasks.
3296	*/
3297	gcwq->trustee_state = TRUSTEE_IN_CHARGE;
3298	wake_up_all(&gcwq->trustee_wait);
3299
3300	/*
3301	* The original cpu is in the process of dying and may go away
3302	* anytime now. When that happens, we and all workers would
3303	* be migrated to other cpus. Try draining any left work. We
3304	* want to get it over with ASAP - spam rescuers, wake up as
3305	* many idlers as necessary and create new ones till the
3306	* worklist is empty. Note that if the gcwq is frozen, there
3307	* may be frozen works in freezeable cwqs. Don't declare
3308	* completion while frozen.
3309	*/
3310	while (gcwq->nr_workers != gcwq->nr_idle \|\|
3311	gcwq->flags & GCWQ_FREEZING \|\|
3312	gcwq->trustee_state == TRUSTEE_IN_CHARGE) {
3313	int nr_works = 0;
3314
3315	list_for_each_entry(work, &gcwq->worklist, entry) {
3316	send_mayday(work);
3317	nr_works++;
3318	}
3319
3320	list_for_each_entry(worker, &gcwq->idle_list, entry) {
3321	if (!nr_works--)
3322	break;
3323	wake_up_process(worker->task);
3324	}
3325
3326	if (need_to_create_worker(gcwq)) {
3327	spin_unlock_irq(&gcwq->lock);
3328	worker = create_worker(gcwq, false);
3329	spin_lock_irq(&gcwq->lock);
3330	if (worker) {
3331	worker->flags \|= WORKER_ROGUE;
3332	start_worker(worker);
3333	}
3334	}
3335
3336	/* give a breather */
3337	if (trustee_wait_event_timeout(false, TRUSTEE_COOLDOWN) < 0)
3338	break;
3339	}
3340
3341	/*
3342	* Either all works have been scheduled and cpu is down, or
3343	* cpu down has already been canceled. Wait for and butcher
3344	* all workers till we're canceled.
3345	*/
3346	do {
3347	rc = trustee_wait_event(!list_empty(&gcwq->idle_list));
3348	while (!list_empty(&gcwq->idle_list))
3349	destroy_worker(list_first_entry(&gcwq->idle_list,
3350	struct worker, entry));
3351	} while (gcwq->nr_workers && rc >= 0);
3352
3353	/*
3354	* At this point, either draining has completed and no worker
3355	* is left, or cpu down has been canceled or the cpu is being
3356	* brought back up. There shouldn't be any idle one left.
3357	* Tell the remaining busy ones to rebind once it finishes the
3358	* currently scheduled works by scheduling the rebind_work.
3359	*/
3360	WARN_ON(!list_empty(&gcwq->idle_list));
3361
3362	for_each_busy_worker(worker, i, pos, gcwq) {
3363	struct work_struct *rebind_work = &worker->rebind_work;
3364
3365	/*
3366	* Rebind_work may race with future cpu hotplug
3367	* operations. Use a separate flag to mark that
3368	* rebinding is scheduled.
3369	*/
3370	worker->flags \|= WORKER_REBIND;
3371	worker->flags &= ~WORKER_ROGUE;
3372
3373	/* queue rebind_work, wq doesn't matter, use the default one */
3374	if (test_and_set_bit(WORK_STRUCT_PENDING_BIT,
3375	work_data_bits(rebind_work)))
3376	continue;
3377
3378	debug_work_activate(rebind_work);
3379	insert_work(get_cwq(gcwq->cpu, system_wq), rebind_work,
3380	worker->scheduled.next,
3381	work_color_to_flags(WORK_NO_COLOR));
3382	}
3383
3384	/* relinquish manager role */
3385	gcwq->flags &= ~GCWQ_MANAGING_WORKERS;
3386
3387	/* notify completion */
3388	gcwq->trustee = NULL;
3389	gcwq->trustee_state = TRUSTEE_DONE;
3390	wake_up_all(&gcwq->trustee_wait);
3391	spin_unlock_irq(&gcwq->lock);
3392	return 0;
3393	}
3394
3395	/**
3396	* wait_trustee_state - wait for trustee to enter the specified state
3397	* @gcwq: gcwq the trustee of interest belongs to
3398	* @state: target state to wait for
3399	*
3400	* Wait for the trustee to reach @state. DONE is already matched.
3401	*
3402	* CONTEXT:
3403	* spin_lock_irq(gcwq->lock) which may be released and regrabbed
3404	* multiple times. To be used by cpu_callback.
3405	*/
3406	static void __cpuinit wait_trustee_state(struct global_cwq *gcwq, int state)
3407	__releases(&gcwq->lock)
3408	__acquires(&gcwq->lock)
3409	{
3410	if (!(gcwq->trustee_state == state \|\|
3411	gcwq->trustee_state == TRUSTEE_DONE)) {
3412	spin_unlock_irq(&gcwq->lock);
3413	__wait_event(gcwq->trustee_wait,
3414	gcwq->trustee_state == state \|\|
3415	gcwq->trustee_state == TRUSTEE_DONE);
3416	spin_lock_irq(&gcwq->lock);
3417	}
3418	}
3419
3420	static int __devinit workqueue_cpu_callback(struct notifier_block *nfb,
3421	unsigned long action,
3422	void *hcpu)
3423	{
3424	unsigned int cpu = (unsigned long)hcpu;
3425	struct global_cwq *gcwq = get_gcwq(cpu);
3426	struct task_struct *new_trustee = NULL;
3427	struct worker *uninitialized_var(new_worker);
3428	unsigned long flags;
3429
3430	action &= ~CPU_TASKS_FROZEN;
3431
3432	switch (action) {
3433	case CPU_DOWN_PREPARE:
3434	new_trustee = kthread_create(trustee_thread, gcwq,
3435	"workqueue_trustee/%d\n", cpu);
3436	if (IS_ERR(new_trustee))
3437	return notifier_from_errno(PTR_ERR(new_trustee));
3438	kthread_bind(new_trustee, cpu);
3439	/* fall through */
3440	case CPU_UP_PREPARE:
3441	BUG_ON(gcwq->first_idle);
3442	new_worker = create_worker(gcwq, false);
3443	if (!new_worker) {
3444	if (new_trustee)
3445	kthread_stop(new_trustee);
3446	return NOTIFY_BAD;
3447	}
3448	}
3449
3450	/* some are called w/ irq disabled, don't disturb irq status */
3451	spin_lock_irqsave(&gcwq->lock, flags);
3452
3453	switch (action) {
3454	case CPU_DOWN_PREPARE:
3455	/* initialize trustee and tell it to acquire the gcwq */
3456	BUG_ON(gcwq->trustee \|\| gcwq->trustee_state != TRUSTEE_DONE);
3457	gcwq->trustee = new_trustee;
3458	gcwq->trustee_state = TRUSTEE_START;
3459	wake_up_process(gcwq->trustee);
3460	wait_trustee_state(gcwq, TRUSTEE_IN_CHARGE);
3461	/* fall through */
3462	case CPU_UP_PREPARE:
3463	BUG_ON(gcwq->first_idle);
3464	gcwq->first_idle = new_worker;
3465	break;
3466
3467	case CPU_DYING:
3468	/*
3469	* Before this, the trustee and all workers except for
3470	* the ones which are still executing works from
3471	* before the last CPU down must be on the cpu. After
3472	* this, they'll all be diasporas.
3473	*/
3474	gcwq->flags \|= GCWQ_DISASSOCIATED;
3475	break;
3476
3477	case CPU_POST_DEAD:
3478	gcwq->trustee_state = TRUSTEE_BUTCHER;
3479	/* fall through */
3480	case CPU_UP_CANCELED:
3481	destroy_worker(gcwq->first_idle);
3482	gcwq->first_idle = NULL;
3483	break;
3484
3485	case CPU_DOWN_FAILED:
3486	case CPU_ONLINE:
3487	gcwq->flags &= ~GCWQ_DISASSOCIATED;
3488	if (gcwq->trustee_state != TRUSTEE_DONE) {
3489	gcwq->trustee_state = TRUSTEE_RELEASE;
3490	wake_up_process(gcwq->trustee);
3491	wait_trustee_state(gcwq, TRUSTEE_DONE);
3492	}
3493
3494	/*
3495	* Trustee is done and there might be no worker left.
3496	* Put the first_idle in and request a real manager to
3497	* take a look.
3498	*/
3499	spin_unlock_irq(&gcwq->lock);
3500	kthread_bind(gcwq->first_idle->task, cpu);
3501	spin_lock_irq(&gcwq->lock);
3502	gcwq->flags \|= GCWQ_MANAGE_WORKERS;
3503	start_worker(gcwq->first_idle);
3504	gcwq->first_idle = NULL;
3505	break;
3506	}
3507
3508	spin_unlock_irqrestore(&gcwq->lock, flags);
3509
3510	return notifier_from_errno(0);
3511	}
3512
3513	#ifdef CONFIG_SMP
3514
3515	struct work_for_cpu {
3516	struct completion completion;
3517	long (fn)(void );
3518	void *arg;
3519	long ret;
3520	};
3521
3522	static int do_work_for_cpu(void *_wfc)
3523	{
3524	struct work_for_cpu *wfc = _wfc;
3525	wfc->ret = wfc->fn(wfc->arg);
3526	complete(&wfc->completion);
3527	return 0;
3528	}
3529
3530	/**
3531	* work_on_cpu - run a function in user context on a particular cpu
3532	* @cpu: the cpu to run on
3533	* @fn: the function to run
3534	* @arg: the function arg
3535	*
3536	* This will return the value @fn returns.
3537	* It is up to the caller to ensure that the cpu doesn't go offline.
3538	* The caller must not hold any locks which would prevent @fn from completing.
3539	*/
3540	long work_on_cpu(unsigned int cpu, long (fn)(void ), void *arg)
3541	{
3542	struct task_struct *sub_thread;
3543	struct work_for_cpu wfc = {
3544	.completion = COMPLETION_INITIALIZER_ONSTACK(wfc.completion),
3545	.fn = fn,
3546	.arg = arg,
3547	};
3548
3549	sub_thread = kthread_create(do_work_for_cpu, &wfc, "work_for_cpu");
3550	if (IS_ERR(sub_thread))
3551	return PTR_ERR(sub_thread);
3552	kthread_bind(sub_thread, cpu);
3553	wake_up_process(sub_thread);
3554	wait_for_completion(&wfc.completion);
3555	return wfc.ret;
3556	}
3557	EXPORT_SYMBOL_GPL(work_on_cpu);
3558	#endif /* CONFIG_SMP */
3559
3560	#ifdef CONFIG_FREEZER
3561
3562	/**
3563	* freeze_workqueues_begin - begin freezing workqueues
3564	*
3565	* Start freezing workqueues. After this function returns, all
3566	* freezeable workqueues will queue new works to their frozen_works
3567	* list instead of gcwq->worklist.
3568	*
3569	* CONTEXT:
3570	* Grabs and releases workqueue_lock and gcwq->lock's.
3571	*/
3572	void freeze_workqueues_begin(void)
3573	{
3574	unsigned int cpu;
3575
3576	spin_lock(&workqueue_lock);
3577
3578	BUG_ON(workqueue_freezing);
3579	workqueue_freezing = true;
3580
3581	for_each_gcwq_cpu(cpu) {
3582	struct global_cwq *gcwq = get_gcwq(cpu);
3583	struct workqueue_struct *wq;
3584
3585	spin_lock_irq(&gcwq->lock);
3586
3587	BUG_ON(gcwq->flags & GCWQ_FREEZING);
3588	gcwq->flags \|= GCWQ_FREEZING;
3589
3590	list_for_each_entry(wq, &workqueues, list) {
3591	struct cpu_workqueue_struct *cwq = get_cwq(cpu, wq);
3592
3593	if (cwq && wq->flags & WQ_FREEZEABLE)
3594	cwq->max_active = 0;
3595	}
3596
3597	spin_unlock_irq(&gcwq->lock);
3598	}
3599
3600	spin_unlock(&workqueue_lock);
3601	}
3602
3603	/**
3604	* freeze_workqueues_busy - are freezeable workqueues still busy?
3605	*
3606	* Check whether freezing is complete. This function must be called
3607	* between freeze_workqueues_begin() and thaw_workqueues().
3608	*
3609	* CONTEXT:
3610	* Grabs and releases workqueue_lock.
3611	*
3612	* RETURNS:
3613	* %true if some freezeable workqueues are still busy. %false if
3614	* freezing is complete.
3615	*/
3616	bool freeze_workqueues_busy(void)
3617	{
3618	unsigned int cpu;
3619	bool busy = false;
3620
3621	spin_lock(&workqueue_lock);
3622
3623	BUG_ON(!workqueue_freezing);
3624
3625	for_each_gcwq_cpu(cpu) {
3626	struct workqueue_struct *wq;
3627	/*
3628	* nr_active is monotonically decreasing. It's safe
3629	* to peek without lock.
3630	*/
3631	list_for_each_entry(wq, &workqueues, list) {
3632	struct cpu_workqueue_struct *cwq = get_cwq(cpu, wq);
3633
3634	if (!cwq \|\| !(wq->flags & WQ_FREEZEABLE))
3635	continue;
3636
3637	BUG_ON(cwq->nr_active < 0);
3638	if (cwq->nr_active) {
3639	busy = true;
3640	goto out_unlock;
3641	}
3642	}
3643	}
3644	out_unlock:
3645	spin_unlock(&workqueue_lock);
3646	return busy;
3647	}
3648
3649	/**
3650	* thaw_workqueues - thaw workqueues
3651	*
3652	* Thaw workqueues. Normal queueing is restored and all collected
3653	* frozen works are transferred to their respective gcwq worklists.
3654	*
3655	* CONTEXT:
3656	* Grabs and releases workqueue_lock and gcwq->lock's.
3657	*/
3658	void thaw_workqueues(void)
3659	{
3660	unsigned int cpu;
3661
3662	spin_lock(&workqueue_lock);
3663
3664	if (!workqueue_freezing)
3665	goto out_unlock;
3666
3667	for_each_gcwq_cpu(cpu) {
3668	struct global_cwq *gcwq = get_gcwq(cpu);
3669	struct workqueue_struct *wq;
3670
3671	spin_lock_irq(&gcwq->lock);
3672
3673	BUG_ON(!(gcwq->flags & GCWQ_FREEZING));
3674	gcwq->flags &= ~GCWQ_FREEZING;
3675
3676	list_for_each_entry(wq, &workqueues, list) {
3677	struct cpu_workqueue_struct *cwq = get_cwq(cpu, wq);
3678
3679	if (!cwq \|\| !(wq->flags & WQ_FREEZEABLE))
3680	continue;
3681
3682	/* restore max_active and repopulate worklist */
3683	cwq->max_active = wq->saved_max_active;
3684
3685	while (!list_empty(&cwq->delayed_works) &&
3686	cwq->nr_active < cwq->max_active)
3687	cwq_activate_first_delayed(cwq);
3688	}
3689
3690	wake_up_worker(gcwq);
3691
3692	spin_unlock_irq(&gcwq->lock);
3693	}
3694
3695	workqueue_freezing = false;
3696	out_unlock:
3697	spin_unlock(&workqueue_lock);
3698	}
3699	#endif /* CONFIG_FREEZER */
3700
3701	static int __init init_workqueues(void)
3702	{
3703	unsigned int cpu;
3704	int i;
3705
3706	cpu_notifier(workqueue_cpu_callback, CPU_PRI_WORKQUEUE);
3707
3708	/* initialize gcwqs */
3709	for_each_gcwq_cpu(cpu) {
3710	struct global_cwq *gcwq = get_gcwq(cpu);
3711
3712	spin_lock_init(&gcwq->lock);
3713	INIT_LIST_HEAD(&gcwq->worklist);
3714	gcwq->cpu = cpu;
3715	gcwq->flags \|= GCWQ_DISASSOCIATED;
3716
3717	INIT_LIST_HEAD(&gcwq->idle_list);
3718	for (i = 0; i < BUSY_WORKER_HASH_SIZE; i++)
3719	INIT_HLIST_HEAD(&gcwq->busy_hash[i]);
3720
3721	init_timer_deferrable(&gcwq->idle_timer);
3722	gcwq->idle_timer.function = idle_worker_timeout;
3723	gcwq->idle_timer.data = (unsigned long)gcwq;
3724
3725	setup_timer(&gcwq->mayday_timer, gcwq_mayday_timeout,
3726	(unsigned long)gcwq);
3727
3728	ida_init(&gcwq->worker_ida);
3729
3730	gcwq->trustee_state = TRUSTEE_DONE;
3731	init_waitqueue_head(&gcwq->trustee_wait);
3732	}
3733
3734	/* create the initial worker */
3735	for_each_online_gcwq_cpu(cpu) {
3736	struct global_cwq *gcwq = get_gcwq(cpu);
3737	struct worker *worker;
3738
3739	if (cpu != WORK_CPU_UNBOUND)
3740	gcwq->flags &= ~GCWQ_DISASSOCIATED;
3741	worker = create_worker(gcwq, true);
3742	BUG_ON(!worker);
3743	spin_lock_irq(&gcwq->lock);
3744	start_worker(worker);
3745	spin_unlock_irq(&gcwq->lock);
3746	}
3747
3748	system_wq = alloc_workqueue("events", 0, 0);
3749	system_long_wq = alloc_workqueue("events_long", 0, 0);
3750	system_nrt_wq = alloc_workqueue("events_nrt", WQ_NON_REENTRANT, 0);
3751	system_unbound_wq = alloc_workqueue("events_unbound", WQ_UNBOUND,
3752	WQ_UNBOUND_MAX_ACTIVE);
3753	BUG_ON(!system_wq \|\| !system_long_wq \|\| !system_nrt_wq \|\|
3754	!system_unbound_wq);
3755	return 0;
3756	}
3757	early_initcall(init_workqueues);