projects/linux·blob master · refs

467 lines · 14.0 KiB

1// SPDX-License-Identifier: GPL-2.0-only
2/*
3 * Generic waiting primitives.
4 *
5 * (C) 2004 Nadia Yvette Chambers, Oracle
6 */
7#include "sched.h"
8#include <linux/wait_bit.h>
9
10void __init_waitqueue_head(struct wait_queue_head *wq_head, const char *name, struct lock_class_key *key)
11{
12 spin_lock_init(&wq_head->lock);
13 lockdep_set_class_and_name(&wq_head->lock, key, name);
14 INIT_LIST_HEAD(&wq_head->head);
15}
16
17EXPORT_SYMBOL(__init_waitqueue_head);
18
19void add_wait_queue(struct wait_queue_head *wq_head, struct wait_queue_entry *wq_entry)
20{
21 unsigned long flags;
22
23 wq_entry->flags &= ~WQ_FLAG_EXCLUSIVE;
24 spin_lock_irqsave(&wq_head->lock, flags);
25 __add_wait_queue(wq_head, wq_entry);
26 spin_unlock_irqrestore(&wq_head->lock, flags);
27}
28EXPORT_SYMBOL(add_wait_queue);
29
30void add_wait_queue_exclusive(struct wait_queue_head *wq_head, struct wait_queue_entry *wq_entry)
31{
32 unsigned long flags;
33
34 wq_entry->flags |= WQ_FLAG_EXCLUSIVE;
35 spin_lock_irqsave(&wq_head->lock, flags);
36 __add_wait_queue_entry_tail(wq_head, wq_entry);
37 spin_unlock_irqrestore(&wq_head->lock, flags);
38}
39EXPORT_SYMBOL(add_wait_queue_exclusive);
40
41void add_wait_queue_priority(struct wait_queue_head *wq_head, struct wait_queue_entry *wq_entry)
42{
43 unsigned long flags;
44
45 wq_entry->flags |= WQ_FLAG_PRIORITY;
46 spin_lock_irqsave(&wq_head->lock, flags);
47 __add_wait_queue(wq_head, wq_entry);
48 spin_unlock_irqrestore(&wq_head->lock, flags);
49}
50EXPORT_SYMBOL_GPL(add_wait_queue_priority);
51
52int add_wait_queue_priority_exclusive(struct wait_queue_head *wq_head,
53 struct wait_queue_entry *wq_entry)
54{
55 struct list_head *head = &wq_head->head;
56
57 guard(spinlock_irqsave)(&wq_head->lock);
58
59 if (!list_empty(head) &&
60 (list_first_entry(head, typeof(*wq_entry), entry)->flags & WQ_FLAG_PRIORITY))
61 return -EBUSY;
62
63 wq_entry->flags |= WQ_FLAG_EXCLUSIVE | WQ_FLAG_PRIORITY;
64 list_add(&wq_entry->entry, head);
65 return 0;
66}
67EXPORT_SYMBOL_GPL(add_wait_queue_priority_exclusive);
68
69void remove_wait_queue(struct wait_queue_head *wq_head, struct wait_queue_entry *wq_entry)
70{
71 unsigned long flags;
72
73 spin_lock_irqsave(&wq_head->lock, flags);
74 __remove_wait_queue(wq_head, wq_entry);
75 spin_unlock_irqrestore(&wq_head->lock, flags);
76}
77EXPORT_SYMBOL(remove_wait_queue);
78
79/*
80 * The core wakeup function. Non-exclusive wakeups (nr_exclusive == 0) just
81 * wake everything up. If it's an exclusive wakeup (nr_exclusive == small +ve
82 * number) then we wake that number of exclusive tasks, and potentially all
83 * the non-exclusive tasks. Normally, exclusive tasks will be at the end of
84 * the list and any non-exclusive tasks will be woken first. A priority task
85 * may be at the head of the list, and can consume the event without any other
86 * tasks being woken if it's also an exclusive task.
87 *
88 * There are circumstances in which we can try to wake a task which has already
89 * started to run but is not in state TASK_RUNNING. try_to_wake_up() returns
90 * zero in this (rare) case, and we handle it by continuing to scan the queue.
91 */
92static int __wake_up_common(struct wait_queue_head *wq_head, unsigned int mode,
93 int nr_exclusive, int wake_flags, void *key)
94{
95 wait_queue_entry_t *curr, *next;
96
97 lockdep_assert_held(&wq_head->lock);
98
99 curr = list_first_entry(&wq_head->head, wait_queue_entry_t, entry);
100
101 if (&curr->entry == &wq_head->head)
102 return nr_exclusive;
103
104 list_for_each_entry_safe_from(curr, next, &wq_head->head, entry) {
105 unsigned flags = curr->flags;
106 int ret;
107
108 ret = curr->func(curr, mode, wake_flags, key);
109 if (ret < 0)
110 break;
111 if (ret && (flags & WQ_FLAG_EXCLUSIVE) && !--nr_exclusive)
112 break;
113 }
114
115 return nr_exclusive;
116}
117
118static int __wake_up_common_lock(struct wait_queue_head *wq_head, unsigned int mode,
119 int nr_exclusive, int wake_flags, void *key)
120{
121 unsigned long flags;
122 int remaining;
123
124 spin_lock_irqsave(&wq_head->lock, flags);
125 remaining = __wake_up_common(wq_head, mode, nr_exclusive, wake_flags,
126 key);
127 spin_unlock_irqrestore(&wq_head->lock, flags);
128
129 return nr_exclusive - remaining;
130}
131
132/**
133 * __wake_up - wake up threads blocked on a waitqueue.
134 * @wq_head: the waitqueue
135 * @mode: which threads
136 * @nr_exclusive: how many wake-one or wake-many threads to wake up
137 * @key: is directly passed to the wakeup function
138 *
139 * If this function wakes up a task, it executes a full memory barrier
140 * before accessing the task state. Returns the number of exclusive
141 * tasks that were awaken.
142 */
143int __wake_up(struct wait_queue_head *wq_head, unsigned int mode,
144 int nr_exclusive, void *key)
145{
146 return __wake_up_common_lock(wq_head, mode, nr_exclusive, 0, key);
147}
148EXPORT_SYMBOL(__wake_up);
149
150void __wake_up_on_current_cpu(struct wait_queue_head *wq_head, unsigned int mode, void *key)
151{
152 __wake_up_common_lock(wq_head, mode, 1, WF_CURRENT_CPU, key);
153}
154
155/*
156 * Same as __wake_up but called with the spinlock in wait_queue_head_t held.
157 */
158void __wake_up_locked(struct wait_queue_head *wq_head, unsigned int mode, int nr)
159{
160 __wake_up_common(wq_head, mode, nr, 0, NULL);
161}
162EXPORT_SYMBOL_GPL(__wake_up_locked);
163
164void __wake_up_locked_key(struct wait_queue_head *wq_head, unsigned int mode, void *key)
165{
166 __wake_up_common(wq_head, mode, 1, 0, key);
167}
168EXPORT_SYMBOL_GPL(__wake_up_locked_key);
169
170/**
171 * __wake_up_sync_key - wake up threads blocked on a waitqueue.
172 * @wq_head: the waitqueue
173 * @mode: which threads
174 * @key: opaque value to be passed to wakeup targets
175 *
176 * Passes WF_SYNC to waitqueue wake functions. The default wake function
177 * forwards it to the scheduler; see WF_SYNC for the hint's semantics.
178 *
179 * If this function wakes up a task, it executes a full memory barrier
180 * before accessing the task state.
181 */
182void __wake_up_sync_key(struct wait_queue_head *wq_head, unsigned int mode,
183 void *key)
184{
185 if (unlikely(!wq_head))
186 return;
187
188 __wake_up_common_lock(wq_head, mode, 1, WF_SYNC, key);
189}
190EXPORT_SYMBOL_GPL(__wake_up_sync_key);
191
192/**
193 * __wake_up_locked_sync_key - wake up a thread blocked on a locked waitqueue.
194 * @wq_head: the waitqueue
195 * @mode: which threads
196 * @key: opaque value to be passed to wakeup targets
197 *
198 * Same as __wake_up_sync_key(), but called with @wq_head->lock held.
199 */
200void __wake_up_locked_sync_key(struct wait_queue_head *wq_head,
201 unsigned int mode, void *key)
202{
203 __wake_up_common(wq_head, mode, 1, WF_SYNC, key);
204}
205EXPORT_SYMBOL_GPL(__wake_up_locked_sync_key);
206
207/*
208 * __wake_up_sync - see __wake_up_sync_key()
209 */
210void __wake_up_sync(struct wait_queue_head *wq_head, unsigned int mode)
211{
212 __wake_up_sync_key(wq_head, mode, NULL);
213}
214EXPORT_SYMBOL_GPL(__wake_up_sync); /* For internal use only */
215
216void __wake_up_pollfree(struct wait_queue_head *wq_head)
217{
218 __wake_up(wq_head, TASK_NORMAL, 0, poll_to_key(EPOLLHUP | POLLFREE));
219 /* POLLFREE must have cleared the queue. */
220 WARN_ON_ONCE(waitqueue_active(wq_head));
221}
222
223/*
224 * Note: we use "set_current_state()" _after_ the wait-queue add,
225 * because we need a memory barrier there on SMP, so that any
226 * wake-function that tests for the wait-queue being active
227 * will be guaranteed to see waitqueue addition _or_ subsequent
228 * tests in this thread will see the wakeup having taken place.
229 *
230 * The spin_unlock() itself is semi-permeable and only protects
231 * one way (it only protects stuff inside the critical region and
232 * stops them from bleeding out - it would still allow subsequent
233 * loads to move into the critical region).
234 */
235void
236prepare_to_wait(struct wait_queue_head *wq_head, struct wait_queue_entry *wq_entry, int state)
237{
238 unsigned long flags;
239
240 wq_entry->flags &= ~WQ_FLAG_EXCLUSIVE;
241 spin_lock_irqsave(&wq_head->lock, flags);
242 if (list_empty(&wq_entry->entry))
243 __add_wait_queue(wq_head, wq_entry);
244 set_current_state(state);
245 spin_unlock_irqrestore(&wq_head->lock, flags);
246}
247EXPORT_SYMBOL(prepare_to_wait);
248
249/* Returns true if we are the first waiter in the queue, false otherwise. */
250bool
251prepare_to_wait_exclusive(struct wait_queue_head *wq_head, struct wait_queue_entry *wq_entry, int state)
252{
253 unsigned long flags;
254 bool was_empty = false;
255
256 wq_entry->flags |= WQ_FLAG_EXCLUSIVE;
257 spin_lock_irqsave(&wq_head->lock, flags);
258 if (list_empty(&wq_entry->entry)) {
259 was_empty = list_empty(&wq_head->head);
260 __add_wait_queue_entry_tail(wq_head, wq_entry);
261 }
262 set_current_state(state);
263 spin_unlock_irqrestore(&wq_head->lock, flags);
264 return was_empty;
265}
266EXPORT_SYMBOL(prepare_to_wait_exclusive);
267
268void init_wait_entry(struct wait_queue_entry *wq_entry, int flags)
269{
270 wq_entry->flags = flags;
271 wq_entry->private = current;
272 wq_entry->func = autoremove_wake_function;
273 INIT_LIST_HEAD(&wq_entry->entry);
274}
275EXPORT_SYMBOL(init_wait_entry);
276
277long prepare_to_wait_event(struct wait_queue_head *wq_head, struct wait_queue_entry *wq_entry, int state)
278{
279 unsigned long flags;
280 long ret = 0;
281
282 spin_lock_irqsave(&wq_head->lock, flags);
283 if (signal_pending_state(state, current)) {
284 /*
285 * Exclusive waiter must not fail if it was selected by wakeup,
286 * it should "consume" the condition we were waiting for.
287 *
288 * The caller will recheck the condition and return success if
289 * we were already woken up, we can not miss the event because
290 * wakeup locks/unlocks the same wq_head->lock.
291 *
292 * But we need to ensure that set-condition + wakeup after that
293 * can't see us, it should wake up another exclusive waiter if
294 * we fail.
295 */
296 list_del_init(&wq_entry->entry);
297 ret = -ERESTARTSYS;
298 } else {
299 if (list_empty(&wq_entry->entry)) {
300 if (wq_entry->flags & WQ_FLAG_EXCLUSIVE)
301 __add_wait_queue_entry_tail(wq_head, wq_entry);
302 else
303 __add_wait_queue(wq_head, wq_entry);
304 }
305 set_current_state(state);
306 }
307 spin_unlock_irqrestore(&wq_head->lock, flags);
308
309 return ret;
310}
311EXPORT_SYMBOL(prepare_to_wait_event);
312
313/*
314 * Note! These two wait functions are entered with the
315 * wait-queue lock held (and interrupts off in the _irq
316 * case), so there is no race with testing the wakeup
317 * condition in the caller before they add the wait
318 * entry to the wake queue.
319 */
320int do_wait_intr(wait_queue_head_t *wq, wait_queue_entry_t *wait)
321{
322 if (likely(list_empty(&wait->entry)))
323 __add_wait_queue_entry_tail(wq, wait);
324
325 set_current_state(TASK_INTERRUPTIBLE);
326 if (signal_pending(current))
327 return -ERESTARTSYS;
328
329 spin_unlock(&wq->lock);
330 schedule();
331 spin_lock(&wq->lock);
332
333 return 0;
334}
335EXPORT_SYMBOL(do_wait_intr);
336
337int do_wait_intr_irq(wait_queue_head_t *wq, wait_queue_entry_t *wait)
338{
339 if (likely(list_empty(&wait->entry)))
340 __add_wait_queue_entry_tail(wq, wait);
341
342 set_current_state(TASK_INTERRUPTIBLE);
343 if (signal_pending(current))
344 return -ERESTARTSYS;
345
346 spin_unlock_irq(&wq->lock);
347 schedule();
348 spin_lock_irq(&wq->lock);
349
350 return 0;
351}
352EXPORT_SYMBOL(do_wait_intr_irq);
353
354/**
355 * finish_wait - clean up after waiting in a queue
356 * @wq_head: waitqueue waited on
357 * @wq_entry: wait descriptor
358 *
359 * Sets current thread back to running state and removes
360 * the wait descriptor from the given waitqueue if still
361 * queued.
362 */
363void finish_wait(struct wait_queue_head *wq_head, struct wait_queue_entry *wq_entry)
364{
365 unsigned long flags;
366
367 __set_current_state(TASK_RUNNING);
368 /*
369 * We can check for list emptiness outside the lock
370 * IFF:
371 * - we use the "careful" check that verifies both
372 * the next and prev pointers, so that there cannot
373 * be any half-pending updates in progress on other
374 * CPU's that we haven't seen yet (and that might
375 * still change the stack area.
376 * and
377 * - all other users take the lock (ie we can only
378 * have _one_ other CPU that looks at or modifies
379 * the list).
380 */
381 if (!list_empty_careful(&wq_entry->entry)) {
382 spin_lock_irqsave(&wq_head->lock, flags);
383 list_del_init(&wq_entry->entry);
384 spin_unlock_irqrestore(&wq_head->lock, flags);
385 }
386}
387EXPORT_SYMBOL(finish_wait);
388
389int autoremove_wake_function(struct wait_queue_entry *wq_entry, unsigned mode, int sync, void *key)
390{
391 int ret = default_wake_function(wq_entry, mode, sync, key);
392
393 if (ret)
394 list_del_init_careful(&wq_entry->entry);
395
396 return ret;
397}
398EXPORT_SYMBOL(autoremove_wake_function);
399
400/*
401 * DEFINE_WAIT_FUNC(wait, woken_wake_func);
402 *
403 * add_wait_queue(&wq_head, &wait);
404 * for (;;) {
405 * if (condition)
406 * break;
407 *
408 * // in wait_woken() // in woken_wake_function()
409 *
410 * p->state = mode; wq_entry->flags |= WQ_FLAG_WOKEN;
411 * smp_mb(); // A try_to_wake_up():
412 * if (!(wq_entry->flags & WQ_FLAG_WOKEN)) <full barrier>
413 * schedule() if (p->state & mode)
414 * p->state = TASK_RUNNING; p->state = TASK_RUNNING;
415 * wq_entry->flags &= ~WQ_FLAG_WOKEN; ~~~~~~~~~~~~~~~~~~
416 * smp_mb(); // B condition = true;
417 * } smp_mb(); // C
418 * remove_wait_queue(&wq_head, &wait); wq_entry->flags |= WQ_FLAG_WOKEN;
419 */
420long wait_woken(struct wait_queue_entry *wq_entry, unsigned mode, long timeout)
421{
422 /*
423 * The below executes an smp_mb(), which matches with the full barrier
424 * executed by the try_to_wake_up() in woken_wake_function() such that
425 * either we see the store to wq_entry->flags in woken_wake_function()
426 * or woken_wake_function() sees our store to current->state.
427 */
428 set_current_state(mode); /* A */
429 if (!(wq_entry->flags & WQ_FLAG_WOKEN) && !kthread_should_stop_or_park())
430 timeout = schedule_timeout(timeout);
431 __set_current_state(TASK_RUNNING);
432
433 /*
434 * The below executes an smp_mb(), which matches with the smp_mb() (C)
435 * in woken_wake_function() such that either we see the wait condition
436 * being true or the store to wq_entry->flags in woken_wake_function()
437 * follows ours in the coherence order.
438 */
439 smp_store_mb(wq_entry->flags, wq_entry->flags & ~WQ_FLAG_WOKEN); /* B */
440
441 return timeout;
442}
443EXPORT_SYMBOL(wait_woken);
444
445int woken_wake_function(struct wait_queue_entry *wq_entry, unsigned mode, int sync, void *key)
446{
447 /* Pairs with the smp_store_mb() in wait_woken(). */
448 smp_mb(); /* C */
449 wq_entry->flags |= WQ_FLAG_WOKEN;
450
451 return default_wake_function(wq_entry, mode, sync, key);
452}
453EXPORT_SYMBOL(woken_wake_function);
454
455int woken_wake_bit_function(struct wait_queue_entry *wq_entry, unsigned mode, int sync, void *arg)
456{
457 struct wait_bit_key *key = __var_wake_key(wq_entry, arg);
458 if (!key)
459 return 0;
460
461 /* Pairs with the smp_store_mb() in wait_woken(). */
462 smp_mb(); /* C */
463 wq_entry->flags |= WQ_FLAG_WOKEN;
464
465 return default_wake_function(wq_entry, mode, sync, key);
466}
467EXPORT_SYMBOL(woken_wake_bit_function);