projects/linux·blob v4.1-rc5 · refs

1155 lines · 38.8 KiB

1#ifndef _LINUX_WAIT_H
2#define _LINUX_WAIT_H
3/*
4 * Linux wait queue related types and methods
5 */
6#include <linux/list.h>
7#include <linux/stddef.h>
8#include <linux/spinlock.h>
9#include <asm/current.h>
10#include <uapi/linux/wait.h>
11
12typedef struct __wait_queue wait_queue_t;
13typedef int (*wait_queue_func_t)(wait_queue_t *wait, unsigned mode, int flags, void *key);
14int default_wake_function(wait_queue_t *wait, unsigned mode, int flags, void *key);
15
16/* __wait_queue::flags */
17#define WQ_FLAG_EXCLUSIVE 0x01
18#define WQ_FLAG_WOKEN 0x02
19
20struct __wait_queue {
21 unsigned int flags;
22 void *private;
23 wait_queue_func_t func;
24 struct list_head task_list;
25};
26
27struct wait_bit_key {
28 void *flags;
29 int bit_nr;
30#define WAIT_ATOMIC_T_BIT_NR -1
31 unsigned long timeout;
32};
33
34struct wait_bit_queue {
35 struct wait_bit_key key;
36 wait_queue_t wait;
37};
38
39struct __wait_queue_head {
40 spinlock_t lock;
41 struct list_head task_list;
42};
43typedef struct __wait_queue_head wait_queue_head_t;
44
45struct task_struct;
46
47/*
48 * Macros for declaration and initialisaton of the datatypes
49 */
50
51#define __WAITQUEUE_INITIALIZER(name, tsk) { \
52 .private = tsk, \
53 .func = default_wake_function, \
54 .task_list = { NULL, NULL } }
55
56#define DECLARE_WAITQUEUE(name, tsk) \
57 wait_queue_t name = __WAITQUEUE_INITIALIZER(name, tsk)
58
59#define __WAIT_QUEUE_HEAD_INITIALIZER(name) { \
60 .lock = __SPIN_LOCK_UNLOCKED(name.lock), \
61 .task_list = { &(name).task_list, &(name).task_list } }
62
63#define DECLARE_WAIT_QUEUE_HEAD(name) \
64 wait_queue_head_t name = __WAIT_QUEUE_HEAD_INITIALIZER(name)
65
66#define __WAIT_BIT_KEY_INITIALIZER(word, bit) \
67 { .flags = word, .bit_nr = bit, }
68
69#define __WAIT_ATOMIC_T_KEY_INITIALIZER(p) \
70 { .flags = p, .bit_nr = WAIT_ATOMIC_T_BIT_NR, }
71
72extern void __init_waitqueue_head(wait_queue_head_t *q, const char *name, struct lock_class_key *);
73
74#define init_waitqueue_head(q) \
75 do { \
76 static struct lock_class_key __key; \
77 \
78 __init_waitqueue_head((q), #q, &__key); \
79 } while (0)
80
81#ifdef CONFIG_LOCKDEP
82# define __WAIT_QUEUE_HEAD_INIT_ONSTACK(name) \
83 ({ init_waitqueue_head(&name); name; })
84# define DECLARE_WAIT_QUEUE_HEAD_ONSTACK(name) \
85 wait_queue_head_t name = __WAIT_QUEUE_HEAD_INIT_ONSTACK(name)
86#else
87# define DECLARE_WAIT_QUEUE_HEAD_ONSTACK(name) DECLARE_WAIT_QUEUE_HEAD(name)
88#endif
89
90static inline void init_waitqueue_entry(wait_queue_t *q, struct task_struct *p)
91{
92 q->flags = 0;
93 q->private = p;
94 q->func = default_wake_function;
95}
96
97static inline void
98init_waitqueue_func_entry(wait_queue_t *q, wait_queue_func_t func)
99{
100 q->flags = 0;
101 q->private = NULL;
102 q->func = func;
103}
104
105static inline int waitqueue_active(wait_queue_head_t *q)
106{
107 return !list_empty(&q->task_list);
108}
109
110extern void add_wait_queue(wait_queue_head_t *q, wait_queue_t *wait);
111extern void add_wait_queue_exclusive(wait_queue_head_t *q, wait_queue_t *wait);
112extern void remove_wait_queue(wait_queue_head_t *q, wait_queue_t *wait);
113
114static inline void __add_wait_queue(wait_queue_head_t *head, wait_queue_t *new)
115{
116 list_add(&new->task_list, &head->task_list);
117}
118
119/*
120 * Used for wake-one threads:
121 */
122static inline void
123__add_wait_queue_exclusive(wait_queue_head_t *q, wait_queue_t *wait)
124{
125 wait->flags |= WQ_FLAG_EXCLUSIVE;
126 __add_wait_queue(q, wait);
127}
128
129static inline void __add_wait_queue_tail(wait_queue_head_t *head,
130 wait_queue_t *new)
131{
132 list_add_tail(&new->task_list, &head->task_list);
133}
134
135static inline void
136__add_wait_queue_tail_exclusive(wait_queue_head_t *q, wait_queue_t *wait)
137{
138 wait->flags |= WQ_FLAG_EXCLUSIVE;
139 __add_wait_queue_tail(q, wait);
140}
141
142static inline void
143__remove_wait_queue(wait_queue_head_t *head, wait_queue_t *old)
144{
145 list_del(&old->task_list);
146}
147
148typedef int wait_bit_action_f(struct wait_bit_key *);
149void __wake_up(wait_queue_head_t *q, unsigned int mode, int nr, void *key);
150void __wake_up_locked_key(wait_queue_head_t *q, unsigned int mode, void *key);
151void __wake_up_sync_key(wait_queue_head_t *q, unsigned int mode, int nr, void *key);
152void __wake_up_locked(wait_queue_head_t *q, unsigned int mode, int nr);
153void __wake_up_sync(wait_queue_head_t *q, unsigned int mode, int nr);
154void __wake_up_bit(wait_queue_head_t *, void *, int);
155int __wait_on_bit(wait_queue_head_t *, struct wait_bit_queue *, wait_bit_action_f *, unsigned);
156int __wait_on_bit_lock(wait_queue_head_t *, struct wait_bit_queue *, wait_bit_action_f *, unsigned);
157void wake_up_bit(void *, int);
158void wake_up_atomic_t(atomic_t *);
159int out_of_line_wait_on_bit(void *, int, wait_bit_action_f *, unsigned);
160int out_of_line_wait_on_bit_timeout(void *, int, wait_bit_action_f *, unsigned, unsigned long);
161int out_of_line_wait_on_bit_lock(void *, int, wait_bit_action_f *, unsigned);
162int out_of_line_wait_on_atomic_t(atomic_t *, int (*)(atomic_t *), unsigned);
163wait_queue_head_t *bit_waitqueue(void *, int);
164
165#define wake_up(x) __wake_up(x, TASK_NORMAL, 1, NULL)
166#define wake_up_nr(x, nr) __wake_up(x, TASK_NORMAL, nr, NULL)
167#define wake_up_all(x) __wake_up(x, TASK_NORMAL, 0, NULL)
168#define wake_up_locked(x) __wake_up_locked((x), TASK_NORMAL, 1)
169#define wake_up_all_locked(x) __wake_up_locked((x), TASK_NORMAL, 0)
170
171#define wake_up_interruptible(x) __wake_up(x, TASK_INTERRUPTIBLE, 1, NULL)
172#define wake_up_interruptible_nr(x, nr) __wake_up(x, TASK_INTERRUPTIBLE, nr, NULL)
173#define wake_up_interruptible_all(x) __wake_up(x, TASK_INTERRUPTIBLE, 0, NULL)
174#define wake_up_interruptible_sync(x) __wake_up_sync((x), TASK_INTERRUPTIBLE, 1)
175
176/*
177 * Wakeup macros to be used to report events to the targets.
178 */
179#define wake_up_poll(x, m) \
180 __wake_up(x, TASK_NORMAL, 1, (void *) (m))
181#define wake_up_locked_poll(x, m) \
182 __wake_up_locked_key((x), TASK_NORMAL, (void *) (m))
183#define wake_up_interruptible_poll(x, m) \
184 __wake_up(x, TASK_INTERRUPTIBLE, 1, (void *) (m))
185#define wake_up_interruptible_sync_poll(x, m) \
186 __wake_up_sync_key((x), TASK_INTERRUPTIBLE, 1, (void *) (m))
187
188#define ___wait_cond_timeout(condition) \
189({ \
190 bool __cond = (condition); \
191 if (__cond && !__ret) \
192 __ret = 1; \
193 __cond || !__ret; \
194})
195
196#define ___wait_is_interruptible(state) \
197 (!__builtin_constant_p(state) || \
198 state == TASK_INTERRUPTIBLE || state == TASK_KILLABLE) \
199
200/*
201 * The below macro ___wait_event() has an explicit shadow of the __ret
202 * variable when used from the wait_event_*() macros.
203 *
204 * This is so that both can use the ___wait_cond_timeout() construct
205 * to wrap the condition.
206 *
207 * The type inconsistency of the wait_event_*() __ret variable is also
208 * on purpose; we use long where we can return timeout values and int
209 * otherwise.
210 */
211
212#define ___wait_event(wq, condition, state, exclusive, ret, cmd) \
213({ \
214 __label__ __out; \
215 wait_queue_t __wait; \
216 long __ret = ret; /* explicit shadow */ \
217 \
218 INIT_LIST_HEAD(&__wait.task_list); \
219 if (exclusive) \
220 __wait.flags = WQ_FLAG_EXCLUSIVE; \
221 else \
222 __wait.flags = 0; \
223 \
224 for (;;) { \
225 long __int = prepare_to_wait_event(&wq, &__wait, state);\
226 \
227 if (condition) \
228 break; \
229 \
230 if (___wait_is_interruptible(state) && __int) { \
231 __ret = __int; \
232 if (exclusive) { \
233 abort_exclusive_wait(&wq, &__wait, \
234 state, NULL); \
235 goto __out; \
236 } \
237 break; \
238 } \
239 \
240 cmd; \
241 } \
242 finish_wait(&wq, &__wait); \
243__out: __ret; \
244})
245
246#define __wait_event(wq, condition) \
247 (void)___wait_event(wq, condition, TASK_UNINTERRUPTIBLE, 0, 0, \
248 schedule())
249
250/**
251 * wait_event - sleep until a condition gets true
252 * @wq: the waitqueue to wait on
253 * @condition: a C expression for the event to wait for
254 *
255 * The process is put to sleep (TASK_UNINTERRUPTIBLE) until the
256 * @condition evaluates to true. The @condition is checked each time
257 * the waitqueue @wq is woken up.
258 *
259 * wake_up() has to be called after changing any variable that could
260 * change the result of the wait condition.
261 */
262#define wait_event(wq, condition) \
263do { \
264 might_sleep(); \
265 if (condition) \
266 break; \
267 __wait_event(wq, condition); \
268} while (0)
269
270#define __io_wait_event(wq, condition) \
271 (void)___wait_event(wq, condition, TASK_UNINTERRUPTIBLE, 0, 0, \
272 io_schedule())
273
274/*
275 * io_wait_event() -- like wait_event() but with io_schedule()
276 */
277#define io_wait_event(wq, condition) \
278do { \
279 might_sleep(); \
280 if (condition) \
281 break; \
282 __io_wait_event(wq, condition); \
283} while (0)
284
285#define __wait_event_freezable(wq, condition) \
286 ___wait_event(wq, condition, TASK_INTERRUPTIBLE, 0, 0, \
287 schedule(); try_to_freeze())
288
289/**
290 * wait_event - sleep (or freeze) until a condition gets true
291 * @wq: the waitqueue to wait on
292 * @condition: a C expression for the event to wait for
293 *
294 * The process is put to sleep (TASK_INTERRUPTIBLE -- so as not to contribute
295 * to system load) until the @condition evaluates to true. The
296 * @condition is checked each time the waitqueue @wq is woken up.
297 *
298 * wake_up() has to be called after changing any variable that could
299 * change the result of the wait condition.
300 */
301#define wait_event_freezable(wq, condition) \
302({ \
303 int __ret = 0; \
304 might_sleep(); \
305 if (!(condition)) \
306 __ret = __wait_event_freezable(wq, condition); \
307 __ret; \
308})
309
310#define __wait_event_timeout(wq, condition, timeout) \
311 ___wait_event(wq, ___wait_cond_timeout(condition), \
312 TASK_UNINTERRUPTIBLE, 0, timeout, \
313 __ret = schedule_timeout(__ret))
314
315/**
316 * wait_event_timeout - sleep until a condition gets true or a timeout elapses
317 * @wq: the waitqueue to wait on
318 * @condition: a C expression for the event to wait for
319 * @timeout: timeout, in jiffies
320 *
321 * The process is put to sleep (TASK_UNINTERRUPTIBLE) until the
322 * @condition evaluates to true. The @condition is checked each time
323 * the waitqueue @wq is woken up.
324 *
325 * wake_up() has to be called after changing any variable that could
326 * change the result of the wait condition.
327 *
328 * Returns:
329 * 0 if the @condition evaluated to %false after the @timeout elapsed,
330 * 1 if the @condition evaluated to %true after the @timeout elapsed,
331 * or the remaining jiffies (at least 1) if the @condition evaluated
332 * to %true before the @timeout elapsed.
333 */
334#define wait_event_timeout(wq, condition, timeout) \
335({ \
336 long __ret = timeout; \
337 might_sleep(); \
338 if (!___wait_cond_timeout(condition)) \
339 __ret = __wait_event_timeout(wq, condition, timeout); \
340 __ret; \
341})
342
343#define __wait_event_freezable_timeout(wq, condition, timeout) \
344 ___wait_event(wq, ___wait_cond_timeout(condition), \
345 TASK_INTERRUPTIBLE, 0, timeout, \
346 __ret = schedule_timeout(__ret); try_to_freeze())
347
348/*
349 * like wait_event_timeout() -- except it uses TASK_INTERRUPTIBLE to avoid
350 * increasing load and is freezable.
351 */
352#define wait_event_freezable_timeout(wq, condition, timeout) \
353({ \
354 long __ret = timeout; \
355 might_sleep(); \
356 if (!___wait_cond_timeout(condition)) \
357 __ret = __wait_event_freezable_timeout(wq, condition, timeout); \
358 __ret; \
359})
360
361#define __wait_event_cmd(wq, condition, cmd1, cmd2) \
362 (void)___wait_event(wq, condition, TASK_UNINTERRUPTIBLE, 0, 0, \
363 cmd1; schedule(); cmd2)
364
365/**
366 * wait_event_cmd - sleep until a condition gets true
367 * @wq: the waitqueue to wait on
368 * @condition: a C expression for the event to wait for
369 * @cmd1: the command will be executed before sleep
370 * @cmd2: the command will be executed after sleep
371 *
372 * The process is put to sleep (TASK_UNINTERRUPTIBLE) until the
373 * @condition evaluates to true. The @condition is checked each time
374 * the waitqueue @wq is woken up.
375 *
376 * wake_up() has to be called after changing any variable that could
377 * change the result of the wait condition.
378 */
379#define wait_event_cmd(wq, condition, cmd1, cmd2) \
380do { \
381 if (condition) \
382 break; \
383 __wait_event_cmd(wq, condition, cmd1, cmd2); \
384} while (0)
385
386#define __wait_event_interruptible(wq, condition) \
387 ___wait_event(wq, condition, TASK_INTERRUPTIBLE, 0, 0, \
388 schedule())
389
390/**
391 * wait_event_interruptible - sleep until a condition gets true
392 * @wq: the waitqueue to wait on
393 * @condition: a C expression for the event to wait for
394 *
395 * The process is put to sleep (TASK_INTERRUPTIBLE) until the
396 * @condition evaluates to true or a signal is received.
397 * The @condition is checked each time the waitqueue @wq is woken up.
398 *
399 * wake_up() has to be called after changing any variable that could
400 * change the result of the wait condition.
401 *
402 * The function will return -ERESTARTSYS if it was interrupted by a
403 * signal and 0 if @condition evaluated to true.
404 */
405#define wait_event_interruptible(wq, condition) \
406({ \
407 int __ret = 0; \
408 might_sleep(); \
409 if (!(condition)) \
410 __ret = __wait_event_interruptible(wq, condition); \
411 __ret; \
412})
413
414#define __wait_event_interruptible_timeout(wq, condition, timeout) \
415 ___wait_event(wq, ___wait_cond_timeout(condition), \
416 TASK_INTERRUPTIBLE, 0, timeout, \
417 __ret = schedule_timeout(__ret))
418
419/**
420 * wait_event_interruptible_timeout - sleep until a condition gets true or a timeout elapses
421 * @wq: the waitqueue to wait on
422 * @condition: a C expression for the event to wait for
423 * @timeout: timeout, in jiffies
424 *
425 * The process is put to sleep (TASK_INTERRUPTIBLE) until the
426 * @condition evaluates to true or a signal is received.
427 * The @condition is checked each time the waitqueue @wq is woken up.
428 *
429 * wake_up() has to be called after changing any variable that could
430 * change the result of the wait condition.
431 *
432 * Returns:
433 * 0 if the @condition evaluated to %false after the @timeout elapsed,
434 * 1 if the @condition evaluated to %true after the @timeout elapsed,
435 * the remaining jiffies (at least 1) if the @condition evaluated
436 * to %true before the @timeout elapsed, or -%ERESTARTSYS if it was
437 * interrupted by a signal.
438 */
439#define wait_event_interruptible_timeout(wq, condition, timeout) \
440({ \
441 long __ret = timeout; \
442 might_sleep(); \
443 if (!___wait_cond_timeout(condition)) \
444 __ret = __wait_event_interruptible_timeout(wq, \
445 condition, timeout); \
446 __ret; \
447})
448
449#define __wait_event_hrtimeout(wq, condition, timeout, state) \
450({ \
451 int __ret = 0; \
452 struct hrtimer_sleeper __t; \
453 \
454 hrtimer_init_on_stack(&__t.timer, CLOCK_MONOTONIC, \
455 HRTIMER_MODE_REL); \
456 hrtimer_init_sleeper(&__t, current); \
457 if ((timeout).tv64 != KTIME_MAX) \
458 hrtimer_start_range_ns(&__t.timer, timeout, \
459 current->timer_slack_ns, \
460 HRTIMER_MODE_REL); \
461 \
462 __ret = ___wait_event(wq, condition, state, 0, 0, \
463 if (!__t.task) { \
464 __ret = -ETIME; \
465 break; \
466 } \
467 schedule()); \
468 \
469 hrtimer_cancel(&__t.timer); \
470 destroy_hrtimer_on_stack(&__t.timer); \
471 __ret; \
472})
473
474/**
475 * wait_event_hrtimeout - sleep until a condition gets true or a timeout elapses
476 * @wq: the waitqueue to wait on
477 * @condition: a C expression for the event to wait for
478 * @timeout: timeout, as a ktime_t
479 *
480 * The process is put to sleep (TASK_UNINTERRUPTIBLE) until the
481 * @condition evaluates to true or a signal is received.
482 * The @condition is checked each time the waitqueue @wq is woken up.
483 *
484 * wake_up() has to be called after changing any variable that could
485 * change the result of the wait condition.
486 *
487 * The function returns 0 if @condition became true, or -ETIME if the timeout
488 * elapsed.
489 */
490#define wait_event_hrtimeout(wq, condition, timeout) \
491({ \
492 int __ret = 0; \
493 might_sleep(); \
494 if (!(condition)) \
495 __ret = __wait_event_hrtimeout(wq, condition, timeout, \
496 TASK_UNINTERRUPTIBLE); \
497 __ret; \
498})
499
500/**
501 * wait_event_interruptible_hrtimeout - sleep until a condition gets true or a timeout elapses
502 * @wq: the waitqueue to wait on
503 * @condition: a C expression for the event to wait for
504 * @timeout: timeout, as a ktime_t
505 *
506 * The process is put to sleep (TASK_INTERRUPTIBLE) until the
507 * @condition evaluates to true or a signal is received.
508 * The @condition is checked each time the waitqueue @wq is woken up.
509 *
510 * wake_up() has to be called after changing any variable that could
511 * change the result of the wait condition.
512 *
513 * The function returns 0 if @condition became true, -ERESTARTSYS if it was
514 * interrupted by a signal, or -ETIME if the timeout elapsed.
515 */
516#define wait_event_interruptible_hrtimeout(wq, condition, timeout) \
517({ \
518 long __ret = 0; \
519 might_sleep(); \
520 if (!(condition)) \
521 __ret = __wait_event_hrtimeout(wq, condition, timeout, \
522 TASK_INTERRUPTIBLE); \
523 __ret; \
524})
525
526#define __wait_event_interruptible_exclusive(wq, condition) \
527 ___wait_event(wq, condition, TASK_INTERRUPTIBLE, 1, 0, \
528 schedule())
529
530#define wait_event_interruptible_exclusive(wq, condition) \
531({ \
532 int __ret = 0; \
533 might_sleep(); \
534 if (!(condition)) \
535 __ret = __wait_event_interruptible_exclusive(wq, condition);\
536 __ret; \
537})
538
539
540#define __wait_event_freezable_exclusive(wq, condition) \
541 ___wait_event(wq, condition, TASK_INTERRUPTIBLE, 1, 0, \
542 schedule(); try_to_freeze())
543
544#define wait_event_freezable_exclusive(wq, condition) \
545({ \
546 int __ret = 0; \
547 might_sleep(); \
548 if (!(condition)) \
549 __ret = __wait_event_freezable_exclusive(wq, condition);\
550 __ret; \
551})
552
553
554#define __wait_event_interruptible_locked(wq, condition, exclusive, irq) \
555({ \
556 int __ret = 0; \
557 DEFINE_WAIT(__wait); \
558 if (exclusive) \
559 __wait.flags |= WQ_FLAG_EXCLUSIVE; \
560 do { \
561 if (likely(list_empty(&__wait.task_list))) \
562 __add_wait_queue_tail(&(wq), &__wait); \
563 set_current_state(TASK_INTERRUPTIBLE); \
564 if (signal_pending(current)) { \
565 __ret = -ERESTARTSYS; \
566 break; \
567 } \
568 if (irq) \
569 spin_unlock_irq(&(wq).lock); \
570 else \
571 spin_unlock(&(wq).lock); \
572 schedule(); \
573 if (irq) \
574 spin_lock_irq(&(wq).lock); \
575 else \
576 spin_lock(&(wq).lock); \
577 } while (!(condition)); \
578 __remove_wait_queue(&(wq), &__wait); \
579 __set_current_state(TASK_RUNNING); \
580 __ret; \
581})
582
583
584/**
585 * wait_event_interruptible_locked - sleep until a condition gets true
586 * @wq: the waitqueue to wait on
587 * @condition: a C expression for the event to wait for
588 *
589 * The process is put to sleep (TASK_INTERRUPTIBLE) until the
590 * @condition evaluates to true or a signal is received.
591 * The @condition is checked each time the waitqueue @wq is woken up.
592 *
593 * It must be called with wq.lock being held. This spinlock is
594 * unlocked while sleeping but @condition testing is done while lock
595 * is held and when this macro exits the lock is held.
596 *
597 * The lock is locked/unlocked using spin_lock()/spin_unlock()
598 * functions which must match the way they are locked/unlocked outside
599 * of this macro.
600 *
601 * wake_up_locked() has to be called after changing any variable that could
602 * change the result of the wait condition.
603 *
604 * The function will return -ERESTARTSYS if it was interrupted by a
605 * signal and 0 if @condition evaluated to true.
606 */
607#define wait_event_interruptible_locked(wq, condition) \
608 ((condition) \
609 ? 0 : __wait_event_interruptible_locked(wq, condition, 0, 0))
610
611/**
612 * wait_event_interruptible_locked_irq - sleep until a condition gets true
613 * @wq: the waitqueue to wait on
614 * @condition: a C expression for the event to wait for
615 *
616 * The process is put to sleep (TASK_INTERRUPTIBLE) until the
617 * @condition evaluates to true or a signal is received.
618 * The @condition is checked each time the waitqueue @wq is woken up.
619 *
620 * It must be called with wq.lock being held. This spinlock is
621 * unlocked while sleeping but @condition testing is done while lock
622 * is held and when this macro exits the lock is held.
623 *
624 * The lock is locked/unlocked using spin_lock_irq()/spin_unlock_irq()
625 * functions which must match the way they are locked/unlocked outside
626 * of this macro.
627 *
628 * wake_up_locked() has to be called after changing any variable that could
629 * change the result of the wait condition.
630 *
631 * The function will return -ERESTARTSYS if it was interrupted by a
632 * signal and 0 if @condition evaluated to true.
633 */
634#define wait_event_interruptible_locked_irq(wq, condition) \
635 ((condition) \
636 ? 0 : __wait_event_interruptible_locked(wq, condition, 0, 1))
637
638/**
639 * wait_event_interruptible_exclusive_locked - sleep exclusively until a condition gets true
640 * @wq: the waitqueue to wait on
641 * @condition: a C expression for the event to wait for
642 *
643 * The process is put to sleep (TASK_INTERRUPTIBLE) until the
644 * @condition evaluates to true or a signal is received.
645 * The @condition is checked each time the waitqueue @wq is woken up.
646 *
647 * It must be called with wq.lock being held. This spinlock is
648 * unlocked while sleeping but @condition testing is done while lock
649 * is held and when this macro exits the lock is held.
650 *
651 * The lock is locked/unlocked using spin_lock()/spin_unlock()
652 * functions which must match the way they are locked/unlocked outside
653 * of this macro.
654 *
655 * The process is put on the wait queue with an WQ_FLAG_EXCLUSIVE flag
656 * set thus when other process waits process on the list if this
657 * process is awaken further processes are not considered.
658 *
659 * wake_up_locked() has to be called after changing any variable that could
660 * change the result of the wait condition.
661 *
662 * The function will return -ERESTARTSYS if it was interrupted by a
663 * signal and 0 if @condition evaluated to true.
664 */
665#define wait_event_interruptible_exclusive_locked(wq, condition) \
666 ((condition) \
667 ? 0 : __wait_event_interruptible_locked(wq, condition, 1, 0))
668
669/**
670 * wait_event_interruptible_exclusive_locked_irq - sleep until a condition gets true
671 * @wq: the waitqueue to wait on
672 * @condition: a C expression for the event to wait for
673 *
674 * The process is put to sleep (TASK_INTERRUPTIBLE) until the
675 * @condition evaluates to true or a signal is received.
676 * The @condition is checked each time the waitqueue @wq is woken up.
677 *
678 * It must be called with wq.lock being held. This spinlock is
679 * unlocked while sleeping but @condition testing is done while lock
680 * is held and when this macro exits the lock is held.
681 *
682 * The lock is locked/unlocked using spin_lock_irq()/spin_unlock_irq()
683 * functions which must match the way they are locked/unlocked outside
684 * of this macro.
685 *
686 * The process is put on the wait queue with an WQ_FLAG_EXCLUSIVE flag
687 * set thus when other process waits process on the list if this
688 * process is awaken further processes are not considered.
689 *
690 * wake_up_locked() has to be called after changing any variable that could
691 * change the result of the wait condition.
692 *
693 * The function will return -ERESTARTSYS if it was interrupted by a
694 * signal and 0 if @condition evaluated to true.
695 */
696#define wait_event_interruptible_exclusive_locked_irq(wq, condition) \
697 ((condition) \
698 ? 0 : __wait_event_interruptible_locked(wq, condition, 1, 1))
699
700
701#define __wait_event_killable(wq, condition) \
702 ___wait_event(wq, condition, TASK_KILLABLE, 0, 0, schedule())
703
704/**
705 * wait_event_killable - sleep until a condition gets true
706 * @wq: the waitqueue to wait on
707 * @condition: a C expression for the event to wait for
708 *
709 * The process is put to sleep (TASK_KILLABLE) until the
710 * @condition evaluates to true or a signal is received.
711 * The @condition is checked each time the waitqueue @wq is woken up.
712 *
713 * wake_up() has to be called after changing any variable that could
714 * change the result of the wait condition.
715 *
716 * The function will return -ERESTARTSYS if it was interrupted by a
717 * signal and 0 if @condition evaluated to true.
718 */
719#define wait_event_killable(wq, condition) \
720({ \
721 int __ret = 0; \
722 might_sleep(); \
723 if (!(condition)) \
724 __ret = __wait_event_killable(wq, condition); \
725 __ret; \
726})
727
728
729#define __wait_event_lock_irq(wq, condition, lock, cmd) \
730 (void)___wait_event(wq, condition, TASK_UNINTERRUPTIBLE, 0, 0, \
731 spin_unlock_irq(&lock); \
732 cmd; \
733 schedule(); \
734 spin_lock_irq(&lock))
735
736/**
737 * wait_event_lock_irq_cmd - sleep until a condition gets true. The
738 * condition is checked under the lock. This
739 * is expected to be called with the lock
740 * taken.
741 * @wq: the waitqueue to wait on
742 * @condition: a C expression for the event to wait for
743 * @lock: a locked spinlock_t, which will be released before cmd
744 * and schedule() and reacquired afterwards.
745 * @cmd: a command which is invoked outside the critical section before
746 * sleep
747 *
748 * The process is put to sleep (TASK_UNINTERRUPTIBLE) until the
749 * @condition evaluates to true. The @condition is checked each time
750 * the waitqueue @wq is woken up.
751 *
752 * wake_up() has to be called after changing any variable that could
753 * change the result of the wait condition.
754 *
755 * This is supposed to be called while holding the lock. The lock is
756 * dropped before invoking the cmd and going to sleep and is reacquired
757 * afterwards.
758 */
759#define wait_event_lock_irq_cmd(wq, condition, lock, cmd) \
760do { \
761 if (condition) \
762 break; \
763 __wait_event_lock_irq(wq, condition, lock, cmd); \
764} while (0)
765
766/**
767 * wait_event_lock_irq - sleep until a condition gets true. The
768 * condition is checked under the lock. This
769 * is expected to be called with the lock
770 * taken.
771 * @wq: the waitqueue to wait on
772 * @condition: a C expression for the event to wait for
773 * @lock: a locked spinlock_t, which will be released before schedule()
774 * and reacquired afterwards.
775 *
776 * The process is put to sleep (TASK_UNINTERRUPTIBLE) until the
777 * @condition evaluates to true. The @condition is checked each time
778 * the waitqueue @wq is woken up.
779 *
780 * wake_up() has to be called after changing any variable that could
781 * change the result of the wait condition.
782 *
783 * This is supposed to be called while holding the lock. The lock is
784 * dropped before going to sleep and is reacquired afterwards.
785 */
786#define wait_event_lock_irq(wq, condition, lock) \
787do { \
788 if (condition) \
789 break; \
790 __wait_event_lock_irq(wq, condition, lock, ); \
791} while (0)
792
793
794#define __wait_event_interruptible_lock_irq(wq, condition, lock, cmd) \
795 ___wait_event(wq, condition, TASK_INTERRUPTIBLE, 0, 0, \
796 spin_unlock_irq(&lock); \
797 cmd; \
798 schedule(); \
799 spin_lock_irq(&lock))
800
801/**
802 * wait_event_interruptible_lock_irq_cmd - sleep until a condition gets true.
803 * The condition is checked under the lock. This is expected to
804 * be called with the lock taken.
805 * @wq: the waitqueue to wait on
806 * @condition: a C expression for the event to wait for
807 * @lock: a locked spinlock_t, which will be released before cmd and
808 * schedule() and reacquired afterwards.
809 * @cmd: a command which is invoked outside the critical section before
810 * sleep
811 *
812 * The process is put to sleep (TASK_INTERRUPTIBLE) until the
813 * @condition evaluates to true or a signal is received. The @condition is
814 * checked each time the waitqueue @wq is woken up.
815 *
816 * wake_up() has to be called after changing any variable that could
817 * change the result of the wait condition.
818 *
819 * This is supposed to be called while holding the lock. The lock is
820 * dropped before invoking the cmd and going to sleep and is reacquired
821 * afterwards.
822 *
823 * The macro will return -ERESTARTSYS if it was interrupted by a signal
824 * and 0 if @condition evaluated to true.
825 */
826#define wait_event_interruptible_lock_irq_cmd(wq, condition, lock, cmd) \
827({ \
828 int __ret = 0; \
829 if (!(condition)) \
830 __ret = __wait_event_interruptible_lock_irq(wq, \
831 condition, lock, cmd); \
832 __ret; \
833})
834
835/**
836 * wait_event_interruptible_lock_irq - sleep until a condition gets true.
837 * The condition is checked under the lock. This is expected
838 * to be called with the lock taken.
839 * @wq: the waitqueue to wait on
840 * @condition: a C expression for the event to wait for
841 * @lock: a locked spinlock_t, which will be released before schedule()
842 * and reacquired afterwards.
843 *
844 * The process is put to sleep (TASK_INTERRUPTIBLE) until the
845 * @condition evaluates to true or signal is received. The @condition is
846 * checked each time the waitqueue @wq is woken up.
847 *
848 * wake_up() has to be called after changing any variable that could
849 * change the result of the wait condition.
850 *
851 * This is supposed to be called while holding the lock. The lock is
852 * dropped before going to sleep and is reacquired afterwards.
853 *
854 * The macro will return -ERESTARTSYS if it was interrupted by a signal
855 * and 0 if @condition evaluated to true.
856 */
857#define wait_event_interruptible_lock_irq(wq, condition, lock) \
858({ \
859 int __ret = 0; \
860 if (!(condition)) \
861 __ret = __wait_event_interruptible_lock_irq(wq, \
862 condition, lock,); \
863 __ret; \
864})
865
866#define __wait_event_interruptible_lock_irq_timeout(wq, condition, \
867 lock, timeout) \
868 ___wait_event(wq, ___wait_cond_timeout(condition), \
869 TASK_INTERRUPTIBLE, 0, timeout, \
870 spin_unlock_irq(&lock); \
871 __ret = schedule_timeout(__ret); \
872 spin_lock_irq(&lock));
873
874/**
875 * wait_event_interruptible_lock_irq_timeout - sleep until a condition gets
876 * true or a timeout elapses. The condition is checked under
877 * the lock. This is expected to be called with the lock taken.
878 * @wq: the waitqueue to wait on
879 * @condition: a C expression for the event to wait for
880 * @lock: a locked spinlock_t, which will be released before schedule()
881 * and reacquired afterwards.
882 * @timeout: timeout, in jiffies
883 *
884 * The process is put to sleep (TASK_INTERRUPTIBLE) until the
885 * @condition evaluates to true or signal is received. The @condition is
886 * checked each time the waitqueue @wq is woken up.
887 *
888 * wake_up() has to be called after changing any variable that could
889 * change the result of the wait condition.
890 *
891 * This is supposed to be called while holding the lock. The lock is
892 * dropped before going to sleep and is reacquired afterwards.
893 *
894 * The function returns 0 if the @timeout elapsed, -ERESTARTSYS if it
895 * was interrupted by a signal, and the remaining jiffies otherwise
896 * if the condition evaluated to true before the timeout elapsed.
897 */
898#define wait_event_interruptible_lock_irq_timeout(wq, condition, lock, \
899 timeout) \
900({ \
901 long __ret = timeout; \
902 if (!___wait_cond_timeout(condition)) \
903 __ret = __wait_event_interruptible_lock_irq_timeout( \
904 wq, condition, lock, timeout); \
905 __ret; \
906})
907
908/*
909 * Waitqueues which are removed from the waitqueue_head at wakeup time
910 */
911void prepare_to_wait(wait_queue_head_t *q, wait_queue_t *wait, int state);
912void prepare_to_wait_exclusive(wait_queue_head_t *q, wait_queue_t *wait, int state);
913long prepare_to_wait_event(wait_queue_head_t *q, wait_queue_t *wait, int state);
914void finish_wait(wait_queue_head_t *q, wait_queue_t *wait);
915void abort_exclusive_wait(wait_queue_head_t *q, wait_queue_t *wait, unsigned int mode, void *key);
916long wait_woken(wait_queue_t *wait, unsigned mode, long timeout);
917int woken_wake_function(wait_queue_t *wait, unsigned mode, int sync, void *key);
918int autoremove_wake_function(wait_queue_t *wait, unsigned mode, int sync, void *key);
919int wake_bit_function(wait_queue_t *wait, unsigned mode, int sync, void *key);
920
921#define DEFINE_WAIT_FUNC(name, function) \
922 wait_queue_t name = { \
923 .private = current, \
924 .func = function, \
925 .task_list = LIST_HEAD_INIT((name).task_list), \
926 }
927
928#define DEFINE_WAIT(name) DEFINE_WAIT_FUNC(name, autoremove_wake_function)
929
930#define DEFINE_WAIT_BIT(name, word, bit) \
931 struct wait_bit_queue name = { \
932 .key = __WAIT_BIT_KEY_INITIALIZER(word, bit), \
933 .wait = { \
934 .private = current, \
935 .func = wake_bit_function, \
936 .task_list = \
937 LIST_HEAD_INIT((name).wait.task_list), \
938 }, \
939 }
940
941#define init_wait(wait) \
942 do { \
943 (wait)->private = current; \
944 (wait)->func = autoremove_wake_function; \
945 INIT_LIST_HEAD(&(wait)->task_list); \
946 (wait)->flags = 0; \
947 } while (0)
948
949
950extern int bit_wait(struct wait_bit_key *);
951extern int bit_wait_io(struct wait_bit_key *);
952extern int bit_wait_timeout(struct wait_bit_key *);
953extern int bit_wait_io_timeout(struct wait_bit_key *);
954
955/**
956 * wait_on_bit - wait for a bit to be cleared
957 * @word: the word being waited on, a kernel virtual address
958 * @bit: the bit of the word being waited on
959 * @mode: the task state to sleep in
960 *
961 * There is a standard hashed waitqueue table for generic use. This
962 * is the part of the hashtable's accessor API that waits on a bit.
963 * For instance, if one were to have waiters on a bitflag, one would
964 * call wait_on_bit() in threads waiting for the bit to clear.
965 * One uses wait_on_bit() where one is waiting for the bit to clear,
966 * but has no intention of setting it.
967 * Returned value will be zero if the bit was cleared, or non-zero
968 * if the process received a signal and the mode permitted wakeup
969 * on that signal.
970 */
971static inline int
972wait_on_bit(void *word, int bit, unsigned mode)
973{
974 might_sleep();
975 if (!test_bit(bit, word))
976 return 0;
977 return out_of_line_wait_on_bit(word, bit,
978 bit_wait,
979 mode);
980}
981
982/**
983 * wait_on_bit_io - wait for a bit to be cleared
984 * @word: the word being waited on, a kernel virtual address
985 * @bit: the bit of the word being waited on
986 * @mode: the task state to sleep in
987 *
988 * Use the standard hashed waitqueue table to wait for a bit
989 * to be cleared. This is similar to wait_on_bit(), but calls
990 * io_schedule() instead of schedule() for the actual waiting.
991 *
992 * Returned value will be zero if the bit was cleared, or non-zero
993 * if the process received a signal and the mode permitted wakeup
994 * on that signal.
995 */
996static inline int
997wait_on_bit_io(void *word, int bit, unsigned mode)
998{
999 might_sleep();
1000 if (!test_bit(bit, word))
1001 return 0;
1002 return out_of_line_wait_on_bit(word, bit,
1003 bit_wait_io,
1004 mode);
1005}
1006
1007/**
1008 * wait_on_bit_timeout - wait for a bit to be cleared or a timeout elapses
1009 * @word: the word being waited on, a kernel virtual address
1010 * @bit: the bit of the word being waited on
1011 * @mode: the task state to sleep in
1012 * @timeout: timeout, in jiffies
1013 *
1014 * Use the standard hashed waitqueue table to wait for a bit
1015 * to be cleared. This is similar to wait_on_bit(), except also takes a
1016 * timeout parameter.
1017 *
1018 * Returned value will be zero if the bit was cleared before the
1019 * @timeout elapsed, or non-zero if the @timeout elapsed or process
1020 * received a signal and the mode permitted wakeup on that signal.
1021 */
1022static inline int
1023wait_on_bit_timeout(void *word, int bit, unsigned mode, unsigned long timeout)
1024{
1025 might_sleep();
1026 if (!test_bit(bit, word))
1027 return 0;
1028 return out_of_line_wait_on_bit_timeout(word, bit,
1029 bit_wait_timeout,
1030 mode, timeout);
1031}
1032
1033/**
1034 * wait_on_bit_action - wait for a bit to be cleared
1035 * @word: the word being waited on, a kernel virtual address
1036 * @bit: the bit of the word being waited on
1037 * @action: the function used to sleep, which may take special actions
1038 * @mode: the task state to sleep in
1039 *
1040 * Use the standard hashed waitqueue table to wait for a bit
1041 * to be cleared, and allow the waiting action to be specified.
1042 * This is like wait_on_bit() but allows fine control of how the waiting
1043 * is done.
1044 *
1045 * Returned value will be zero if the bit was cleared, or non-zero
1046 * if the process received a signal and the mode permitted wakeup
1047 * on that signal.
1048 */
1049static inline int
1050wait_on_bit_action(void *word, int bit, wait_bit_action_f *action, unsigned mode)
1051{
1052 might_sleep();
1053 if (!test_bit(bit, word))
1054 return 0;
1055 return out_of_line_wait_on_bit(word, bit, action, mode);
1056}
1057
1058/**
1059 * wait_on_bit_lock - wait for a bit to be cleared, when wanting to set it
1060 * @word: the word being waited on, a kernel virtual address
1061 * @bit: the bit of the word being waited on
1062 * @mode: the task state to sleep in
1063 *
1064 * There is a standard hashed waitqueue table for generic use. This
1065 * is the part of the hashtable's accessor API that waits on a bit
1066 * when one intends to set it, for instance, trying to lock bitflags.
1067 * For instance, if one were to have waiters trying to set bitflag
1068 * and waiting for it to clear before setting it, one would call
1069 * wait_on_bit() in threads waiting to be able to set the bit.
1070 * One uses wait_on_bit_lock() where one is waiting for the bit to
1071 * clear with the intention of setting it, and when done, clearing it.
1072 *
1073 * Returns zero if the bit was (eventually) found to be clear and was
1074 * set. Returns non-zero if a signal was delivered to the process and
1075 * the @mode allows that signal to wake the process.
1076 */
1077static inline int
1078wait_on_bit_lock(void *word, int bit, unsigned mode)
1079{
1080 might_sleep();
1081 if (!test_and_set_bit(bit, word))
1082 return 0;
1083 return out_of_line_wait_on_bit_lock(word, bit, bit_wait, mode);
1084}
1085
1086/**
1087 * wait_on_bit_lock_io - wait for a bit to be cleared, when wanting to set it
1088 * @word: the word being waited on, a kernel virtual address
1089 * @bit: the bit of the word being waited on
1090 * @mode: the task state to sleep in
1091 *
1092 * Use the standard hashed waitqueue table to wait for a bit
1093 * to be cleared and then to atomically set it. This is similar
1094 * to wait_on_bit(), but calls io_schedule() instead of schedule()
1095 * for the actual waiting.
1096 *
1097 * Returns zero if the bit was (eventually) found to be clear and was
1098 * set. Returns non-zero if a signal was delivered to the process and
1099 * the @mode allows that signal to wake the process.
1100 */
1101static inline int
1102wait_on_bit_lock_io(void *word, int bit, unsigned mode)
1103{
1104 might_sleep();
1105 if (!test_and_set_bit(bit, word))
1106 return 0;
1107 return out_of_line_wait_on_bit_lock(word, bit, bit_wait_io, mode);
1108}
1109
1110/**
1111 * wait_on_bit_lock_action - wait for a bit to be cleared, when wanting to set it
1112 * @word: the word being waited on, a kernel virtual address
1113 * @bit: the bit of the word being waited on
1114 * @action: the function used to sleep, which may take special actions
1115 * @mode: the task state to sleep in
1116 *
1117 * Use the standard hashed waitqueue table to wait for a bit
1118 * to be cleared and then to set it, and allow the waiting action
1119 * to be specified.
1120 * This is like wait_on_bit() but allows fine control of how the waiting
1121 * is done.
1122 *
1123 * Returns zero if the bit was (eventually) found to be clear and was
1124 * set. Returns non-zero if a signal was delivered to the process and
1125 * the @mode allows that signal to wake the process.
1126 */
1127static inline int
1128wait_on_bit_lock_action(void *word, int bit, wait_bit_action_f *action, unsigned mode)
1129{
1130 might_sleep();
1131 if (!test_and_set_bit(bit, word))
1132 return 0;
1133 return out_of_line_wait_on_bit_lock(word, bit, action, mode);
1134}
1135
1136/**
1137 * wait_on_atomic_t - Wait for an atomic_t to become 0
1138 * @val: The atomic value being waited on, a kernel virtual address
1139 * @action: the function used to sleep, which may take special actions
1140 * @mode: the task state to sleep in
1141 *
1142 * Wait for an atomic_t to become 0. We abuse the bit-wait waitqueue table for
1143 * the purpose of getting a waitqueue, but we set the key to a bit number
1144 * outside of the target 'word'.
1145 */
1146static inline
1147int wait_on_atomic_t(atomic_t *val, int (*action)(atomic_t *), unsigned mode)
1148{
1149 might_sleep();
1150 if (atomic_read(val) == 0)
1151 return 0;
1152 return out_of_line_wait_on_atomic_t(val, action, mode);
1153}
1154
1155#endif /* _LINUX_WAIT_H */