projects/linux·blob v6.2-rc8 · refs

607 lines · 17.9 KiB

1// SPDX-License-Identifier: GPL-2.0-only
2#include <linux/kdebug.h>
3#include <linux/kprobes.h>
4#include <linux/export.h>
5#include <linux/notifier.h>
6#include <linux/rcupdate.h>
7#include <linux/vmalloc.h>
8#include <linux/reboot.h>
9
10/*
11 * Notifier list for kernel code which wants to be called
12 * at shutdown. This is used to stop any idling DMA operations
13 * and the like.
14 */
15BLOCKING_NOTIFIER_HEAD(reboot_notifier_list);
16
17/*
18 * Notifier chain core routines. The exported routines below
19 * are layered on top of these, with appropriate locking added.
20 */
21
22static int notifier_chain_register(struct notifier_block **nl,
23 struct notifier_block *n,
24 bool unique_priority)
25{
26 while ((*nl) != NULL) {
27 if (unlikely((*nl) == n)) {
28 WARN(1, "notifier callback %ps already registered",
29 n->notifier_call);
30 return -EEXIST;
31 }
32 if (n->priority > (*nl)->priority)
33 break;
34 if (n->priority == (*nl)->priority && unique_priority)
35 return -EBUSY;
36 nl = &((*nl)->next);
37 }
38 n->next = *nl;
39 rcu_assign_pointer(*nl, n);
40 return 0;
41}
42
43static int notifier_chain_unregister(struct notifier_block **nl,
44 struct notifier_block *n)
45{
46 while ((*nl) != NULL) {
47 if ((*nl) == n) {
48 rcu_assign_pointer(*nl, n->next);
49 return 0;
50 }
51 nl = &((*nl)->next);
52 }
53 return -ENOENT;
54}
55
56/**
57 * notifier_call_chain - Informs the registered notifiers about an event.
58 * @nl: Pointer to head of the blocking notifier chain
59 * @val: Value passed unmodified to notifier function
60 * @v: Pointer passed unmodified to notifier function
61 * @nr_to_call: Number of notifier functions to be called. Don't care
62 * value of this parameter is -1.
63 * @nr_calls: Records the number of notifications sent. Don't care
64 * value of this field is NULL.
65 * Return: notifier_call_chain returns the value returned by the
66 * last notifier function called.
67 */
68static int notifier_call_chain(struct notifier_block **nl,
69 unsigned long val, void *v,
70 int nr_to_call, int *nr_calls)
71{
72 int ret = NOTIFY_DONE;
73 struct notifier_block *nb, *next_nb;
74
75 nb = rcu_dereference_raw(*nl);
76
77 while (nb && nr_to_call) {
78 next_nb = rcu_dereference_raw(nb->next);
79
80#ifdef CONFIG_DEBUG_NOTIFIERS
81 if (unlikely(!func_ptr_is_kernel_text(nb->notifier_call))) {
82 WARN(1, "Invalid notifier called!");
83 nb = next_nb;
84 continue;
85 }
86#endif
87 ret = nb->notifier_call(nb, val, v);
88
89 if (nr_calls)
90 (*nr_calls)++;
91
92 if (ret & NOTIFY_STOP_MASK)
93 break;
94 nb = next_nb;
95 nr_to_call--;
96 }
97 return ret;
98}
99NOKPROBE_SYMBOL(notifier_call_chain);
100
101/**
102 * notifier_call_chain_robust - Inform the registered notifiers about an event
103 * and rollback on error.
104 * @nl: Pointer to head of the blocking notifier chain
105 * @val_up: Value passed unmodified to the notifier function
106 * @val_down: Value passed unmodified to the notifier function when recovering
107 * from an error on @val_up
108 * @v: Pointer passed unmodified to the notifier function
109 *
110 * NOTE: It is important the @nl chain doesn't change between the two
111 * invocations of notifier_call_chain() such that we visit the
112 * exact same notifier callbacks; this rules out any RCU usage.
113 *
114 * Return: the return value of the @val_up call.
115 */
116static int notifier_call_chain_robust(struct notifier_block **nl,
117 unsigned long val_up, unsigned long val_down,
118 void *v)
119{
120 int ret, nr = 0;
121
122 ret = notifier_call_chain(nl, val_up, v, -1, &nr);
123 if (ret & NOTIFY_STOP_MASK)
124 notifier_call_chain(nl, val_down, v, nr-1, NULL);
125
126 return ret;
127}
128
129/*
130 * Atomic notifier chain routines. Registration and unregistration
131 * use a spinlock, and call_chain is synchronized by RCU (no locks).
132 */
133
134/**
135 * atomic_notifier_chain_register - Add notifier to an atomic notifier chain
136 * @nh: Pointer to head of the atomic notifier chain
137 * @n: New entry in notifier chain
138 *
139 * Adds a notifier to an atomic notifier chain.
140 *
141 * Returns 0 on success, %-EEXIST on error.
142 */
143int atomic_notifier_chain_register(struct atomic_notifier_head *nh,
144 struct notifier_block *n)
145{
146 unsigned long flags;
147 int ret;
148
149 spin_lock_irqsave(&nh->lock, flags);
150 ret = notifier_chain_register(&nh->head, n, false);
151 spin_unlock_irqrestore(&nh->lock, flags);
152 return ret;
153}
154EXPORT_SYMBOL_GPL(atomic_notifier_chain_register);
155
156/**
157 * atomic_notifier_chain_register_unique_prio - Add notifier to an atomic notifier chain
158 * @nh: Pointer to head of the atomic notifier chain
159 * @n: New entry in notifier chain
160 *
161 * Adds a notifier to an atomic notifier chain if there is no other
162 * notifier registered using the same priority.
163 *
164 * Returns 0 on success, %-EEXIST or %-EBUSY on error.
165 */
166int atomic_notifier_chain_register_unique_prio(struct atomic_notifier_head *nh,
167 struct notifier_block *n)
168{
169 unsigned long flags;
170 int ret;
171
172 spin_lock_irqsave(&nh->lock, flags);
173 ret = notifier_chain_register(&nh->head, n, true);
174 spin_unlock_irqrestore(&nh->lock, flags);
175 return ret;
176}
177EXPORT_SYMBOL_GPL(atomic_notifier_chain_register_unique_prio);
178
179/**
180 * atomic_notifier_chain_unregister - Remove notifier from an atomic notifier chain
181 * @nh: Pointer to head of the atomic notifier chain
182 * @n: Entry to remove from notifier chain
183 *
184 * Removes a notifier from an atomic notifier chain.
185 *
186 * Returns zero on success or %-ENOENT on failure.
187 */
188int atomic_notifier_chain_unregister(struct atomic_notifier_head *nh,
189 struct notifier_block *n)
190{
191 unsigned long flags;
192 int ret;
193
194 spin_lock_irqsave(&nh->lock, flags);
195 ret = notifier_chain_unregister(&nh->head, n);
196 spin_unlock_irqrestore(&nh->lock, flags);
197 synchronize_rcu();
198 return ret;
199}
200EXPORT_SYMBOL_GPL(atomic_notifier_chain_unregister);
201
202/**
203 * atomic_notifier_call_chain - Call functions in an atomic notifier chain
204 * @nh: Pointer to head of the atomic notifier chain
205 * @val: Value passed unmodified to notifier function
206 * @v: Pointer passed unmodified to notifier function
207 *
208 * Calls each function in a notifier chain in turn. The functions
209 * run in an atomic context, so they must not block.
210 * This routine uses RCU to synchronize with changes to the chain.
211 *
212 * If the return value of the notifier can be and'ed
213 * with %NOTIFY_STOP_MASK then atomic_notifier_call_chain()
214 * will return immediately, with the return value of
215 * the notifier function which halted execution.
216 * Otherwise the return value is the return value
217 * of the last notifier function called.
218 */
219int atomic_notifier_call_chain(struct atomic_notifier_head *nh,
220 unsigned long val, void *v)
221{
222 int ret;
223
224 rcu_read_lock();
225 ret = notifier_call_chain(&nh->head, val, v, -1, NULL);
226 rcu_read_unlock();
227
228 return ret;
229}
230EXPORT_SYMBOL_GPL(atomic_notifier_call_chain);
231NOKPROBE_SYMBOL(atomic_notifier_call_chain);
232
233/**
234 * atomic_notifier_call_chain_is_empty - Check whether notifier chain is empty
235 * @nh: Pointer to head of the atomic notifier chain
236 *
237 * Checks whether notifier chain is empty.
238 *
239 * Returns true is notifier chain is empty, false otherwise.
240 */
241bool atomic_notifier_call_chain_is_empty(struct atomic_notifier_head *nh)
242{
243 return !rcu_access_pointer(nh->head);
244}
245
246/*
247 * Blocking notifier chain routines. All access to the chain is
248 * synchronized by an rwsem.
249 */
250
251static int __blocking_notifier_chain_register(struct blocking_notifier_head *nh,
252 struct notifier_block *n,
253 bool unique_priority)
254{
255 int ret;
256
257 /*
258 * This code gets used during boot-up, when task switching is
259 * not yet working and interrupts must remain disabled. At
260 * such times we must not call down_write().
261 */
262 if (unlikely(system_state == SYSTEM_BOOTING))
263 return notifier_chain_register(&nh->head, n, unique_priority);
264
265 down_write(&nh->rwsem);
266 ret = notifier_chain_register(&nh->head, n, unique_priority);
267 up_write(&nh->rwsem);
268 return ret;
269}
270
271/**
272 * blocking_notifier_chain_register - Add notifier to a blocking notifier chain
273 * @nh: Pointer to head of the blocking notifier chain
274 * @n: New entry in notifier chain
275 *
276 * Adds a notifier to a blocking notifier chain.
277 * Must be called in process context.
278 *
279 * Returns 0 on success, %-EEXIST on error.
280 */
281int blocking_notifier_chain_register(struct blocking_notifier_head *nh,
282 struct notifier_block *n)
283{
284 return __blocking_notifier_chain_register(nh, n, false);
285}
286EXPORT_SYMBOL_GPL(blocking_notifier_chain_register);
287
288/**
289 * blocking_notifier_chain_register_unique_prio - Add notifier to a blocking notifier chain
290 * @nh: Pointer to head of the blocking notifier chain
291 * @n: New entry in notifier chain
292 *
293 * Adds a notifier to an blocking notifier chain if there is no other
294 * notifier registered using the same priority.
295 *
296 * Returns 0 on success, %-EEXIST or %-EBUSY on error.
297 */
298int blocking_notifier_chain_register_unique_prio(struct blocking_notifier_head *nh,
299 struct notifier_block *n)
300{
301 return __blocking_notifier_chain_register(nh, n, true);
302}
303EXPORT_SYMBOL_GPL(blocking_notifier_chain_register_unique_prio);
304
305/**
306 * blocking_notifier_chain_unregister - Remove notifier from a blocking notifier chain
307 * @nh: Pointer to head of the blocking notifier chain
308 * @n: Entry to remove from notifier chain
309 *
310 * Removes a notifier from a blocking notifier chain.
311 * Must be called from process context.
312 *
313 * Returns zero on success or %-ENOENT on failure.
314 */
315int blocking_notifier_chain_unregister(struct blocking_notifier_head *nh,
316 struct notifier_block *n)
317{
318 int ret;
319
320 /*
321 * This code gets used during boot-up, when task switching is
322 * not yet working and interrupts must remain disabled. At
323 * such times we must not call down_write().
324 */
325 if (unlikely(system_state == SYSTEM_BOOTING))
326 return notifier_chain_unregister(&nh->head, n);
327
328 down_write(&nh->rwsem);
329 ret = notifier_chain_unregister(&nh->head, n);
330 up_write(&nh->rwsem);
331 return ret;
332}
333EXPORT_SYMBOL_GPL(blocking_notifier_chain_unregister);
334
335int blocking_notifier_call_chain_robust(struct blocking_notifier_head *nh,
336 unsigned long val_up, unsigned long val_down, void *v)
337{
338 int ret = NOTIFY_DONE;
339
340 /*
341 * We check the head outside the lock, but if this access is
342 * racy then it does not matter what the result of the test
343 * is, we re-check the list after having taken the lock anyway:
344 */
345 if (rcu_access_pointer(nh->head)) {
346 down_read(&nh->rwsem);
347 ret = notifier_call_chain_robust(&nh->head, val_up, val_down, v);
348 up_read(&nh->rwsem);
349 }
350 return ret;
351}
352EXPORT_SYMBOL_GPL(blocking_notifier_call_chain_robust);
353
354/**
355 * blocking_notifier_call_chain - Call functions in a blocking notifier chain
356 * @nh: Pointer to head of the blocking notifier chain
357 * @val: Value passed unmodified to notifier function
358 * @v: Pointer passed unmodified to notifier function
359 *
360 * Calls each function in a notifier chain in turn. The functions
361 * run in a process context, so they are allowed to block.
362 *
363 * If the return value of the notifier can be and'ed
364 * with %NOTIFY_STOP_MASK then blocking_notifier_call_chain()
365 * will return immediately, with the return value of
366 * the notifier function which halted execution.
367 * Otherwise the return value is the return value
368 * of the last notifier function called.
369 */
370int blocking_notifier_call_chain(struct blocking_notifier_head *nh,
371 unsigned long val, void *v)
372{
373 int ret = NOTIFY_DONE;
374
375 /*
376 * We check the head outside the lock, but if this access is
377 * racy then it does not matter what the result of the test
378 * is, we re-check the list after having taken the lock anyway:
379 */
380 if (rcu_access_pointer(nh->head)) {
381 down_read(&nh->rwsem);
382 ret = notifier_call_chain(&nh->head, val, v, -1, NULL);
383 up_read(&nh->rwsem);
384 }
385 return ret;
386}
387EXPORT_SYMBOL_GPL(blocking_notifier_call_chain);
388
389/*
390 * Raw notifier chain routines. There is no protection;
391 * the caller must provide it. Use at your own risk!
392 */
393
394/**
395 * raw_notifier_chain_register - Add notifier to a raw notifier chain
396 * @nh: Pointer to head of the raw notifier chain
397 * @n: New entry in notifier chain
398 *
399 * Adds a notifier to a raw notifier chain.
400 * All locking must be provided by the caller.
401 *
402 * Returns 0 on success, %-EEXIST on error.
403 */
404int raw_notifier_chain_register(struct raw_notifier_head *nh,
405 struct notifier_block *n)
406{
407 return notifier_chain_register(&nh->head, n, false);
408}
409EXPORT_SYMBOL_GPL(raw_notifier_chain_register);
410
411/**
412 * raw_notifier_chain_unregister - Remove notifier from a raw notifier chain
413 * @nh: Pointer to head of the raw notifier chain
414 * @n: Entry to remove from notifier chain
415 *
416 * Removes a notifier from a raw notifier chain.
417 * All locking must be provided by the caller.
418 *
419 * Returns zero on success or %-ENOENT on failure.
420 */
421int raw_notifier_chain_unregister(struct raw_notifier_head *nh,
422 struct notifier_block *n)
423{
424 return notifier_chain_unregister(&nh->head, n);
425}
426EXPORT_SYMBOL_GPL(raw_notifier_chain_unregister);
427
428int raw_notifier_call_chain_robust(struct raw_notifier_head *nh,
429 unsigned long val_up, unsigned long val_down, void *v)
430{
431 return notifier_call_chain_robust(&nh->head, val_up, val_down, v);
432}
433EXPORT_SYMBOL_GPL(raw_notifier_call_chain_robust);
434
435/**
436 * raw_notifier_call_chain - Call functions in a raw notifier chain
437 * @nh: Pointer to head of the raw notifier chain
438 * @val: Value passed unmodified to notifier function
439 * @v: Pointer passed unmodified to notifier function
440 *
441 * Calls each function in a notifier chain in turn. The functions
442 * run in an undefined context.
443 * All locking must be provided by the caller.
444 *
445 * If the return value of the notifier can be and'ed
446 * with %NOTIFY_STOP_MASK then raw_notifier_call_chain()
447 * will return immediately, with the return value of
448 * the notifier function which halted execution.
449 * Otherwise the return value is the return value
450 * of the last notifier function called.
451 */
452int raw_notifier_call_chain(struct raw_notifier_head *nh,
453 unsigned long val, void *v)
454{
455 return notifier_call_chain(&nh->head, val, v, -1, NULL);
456}
457EXPORT_SYMBOL_GPL(raw_notifier_call_chain);
458
459#ifdef CONFIG_SRCU
460/*
461 * SRCU notifier chain routines. Registration and unregistration
462 * use a mutex, and call_chain is synchronized by SRCU (no locks).
463 */
464
465/**
466 * srcu_notifier_chain_register - Add notifier to an SRCU notifier chain
467 * @nh: Pointer to head of the SRCU notifier chain
468 * @n: New entry in notifier chain
469 *
470 * Adds a notifier to an SRCU notifier chain.
471 * Must be called in process context.
472 *
473 * Returns 0 on success, %-EEXIST on error.
474 */
475int srcu_notifier_chain_register(struct srcu_notifier_head *nh,
476 struct notifier_block *n)
477{
478 int ret;
479
480 /*
481 * This code gets used during boot-up, when task switching is
482 * not yet working and interrupts must remain disabled. At
483 * such times we must not call mutex_lock().
484 */
485 if (unlikely(system_state == SYSTEM_BOOTING))
486 return notifier_chain_register(&nh->head, n, false);
487
488 mutex_lock(&nh->mutex);
489 ret = notifier_chain_register(&nh->head, n, false);
490 mutex_unlock(&nh->mutex);
491 return ret;
492}
493EXPORT_SYMBOL_GPL(srcu_notifier_chain_register);
494
495/**
496 * srcu_notifier_chain_unregister - Remove notifier from an SRCU notifier chain
497 * @nh: Pointer to head of the SRCU notifier chain
498 * @n: Entry to remove from notifier chain
499 *
500 * Removes a notifier from an SRCU notifier chain.
501 * Must be called from process context.
502 *
503 * Returns zero on success or %-ENOENT on failure.
504 */
505int srcu_notifier_chain_unregister(struct srcu_notifier_head *nh,
506 struct notifier_block *n)
507{
508 int ret;
509
510 /*
511 * This code gets used during boot-up, when task switching is
512 * not yet working and interrupts must remain disabled. At
513 * such times we must not call mutex_lock().
514 */
515 if (unlikely(system_state == SYSTEM_BOOTING))
516 return notifier_chain_unregister(&nh->head, n);
517
518 mutex_lock(&nh->mutex);
519 ret = notifier_chain_unregister(&nh->head, n);
520 mutex_unlock(&nh->mutex);
521 synchronize_srcu(&nh->srcu);
522 return ret;
523}
524EXPORT_SYMBOL_GPL(srcu_notifier_chain_unregister);
525
526/**
527 * srcu_notifier_call_chain - Call functions in an SRCU notifier chain
528 * @nh: Pointer to head of the SRCU notifier chain
529 * @val: Value passed unmodified to notifier function
530 * @v: Pointer passed unmodified to notifier function
531 *
532 * Calls each function in a notifier chain in turn. The functions
533 * run in a process context, so they are allowed to block.
534 *
535 * If the return value of the notifier can be and'ed
536 * with %NOTIFY_STOP_MASK then srcu_notifier_call_chain()
537 * will return immediately, with the return value of
538 * the notifier function which halted execution.
539 * Otherwise the return value is the return value
540 * of the last notifier function called.
541 */
542int srcu_notifier_call_chain(struct srcu_notifier_head *nh,
543 unsigned long val, void *v)
544{
545 int ret;
546 int idx;
547
548 idx = srcu_read_lock(&nh->srcu);
549 ret = notifier_call_chain(&nh->head, val, v, -1, NULL);
550 srcu_read_unlock(&nh->srcu, idx);
551 return ret;
552}
553EXPORT_SYMBOL_GPL(srcu_notifier_call_chain);
554
555/**
556 * srcu_init_notifier_head - Initialize an SRCU notifier head
557 * @nh: Pointer to head of the srcu notifier chain
558 *
559 * Unlike other sorts of notifier heads, SRCU notifier heads require
560 * dynamic initialization. Be sure to call this routine before
561 * calling any of the other SRCU notifier routines for this head.
562 *
563 * If an SRCU notifier head is deallocated, it must first be cleaned
564 * up by calling srcu_cleanup_notifier_head(). Otherwise the head's
565 * per-cpu data (used by the SRCU mechanism) will leak.
566 */
567void srcu_init_notifier_head(struct srcu_notifier_head *nh)
568{
569 mutex_init(&nh->mutex);
570 if (init_srcu_struct(&nh->srcu) < 0)
571 BUG();
572 nh->head = NULL;
573}
574EXPORT_SYMBOL_GPL(srcu_init_notifier_head);
575
576#endif /* CONFIG_SRCU */
577
578static ATOMIC_NOTIFIER_HEAD(die_chain);
579
580int notrace notify_die(enum die_val val, const char *str,
581 struct pt_regs *regs, long err, int trap, int sig)
582{
583 struct die_args args = {
584 .regs = regs,
585 .str = str,
586 .err = err,
587 .trapnr = trap,
588 .signr = sig,
589
590 };
591 RCU_LOCKDEP_WARN(!rcu_is_watching(),
592 "notify_die called but RCU thinks we're quiescent");
593 return atomic_notifier_call_chain(&die_chain, val, &args);
594}
595NOKPROBE_SYMBOL(notify_die);
596
597int register_die_notifier(struct notifier_block *nb)
598{
599 return atomic_notifier_chain_register(&die_chain, nb);
600}
601EXPORT_SYMBOL_GPL(register_die_notifier);
602
603int unregister_die_notifier(struct notifier_block *nb)
604{
605 return atomic_notifier_chain_unregister(&die_chain, nb);
606}
607EXPORT_SYMBOL_GPL(unregister_die_notifier);