projects/linux·blob v5.18-rc4 · refs

544 lines · 15.9 KiB

1// SPDX-License-Identifier: GPL-2.0-only
2#include <linux/kdebug.h>
3#include <linux/kprobes.h>
4#include <linux/export.h>
5#include <linux/notifier.h>
6#include <linux/rcupdate.h>
7#include <linux/vmalloc.h>
8#include <linux/reboot.h>
9
10/*
11 * Notifier list for kernel code which wants to be called
12 * at shutdown. This is used to stop any idling DMA operations
13 * and the like.
14 */
15BLOCKING_NOTIFIER_HEAD(reboot_notifier_list);
16
17/*
18 * Notifier chain core routines. The exported routines below
19 * are layered on top of these, with appropriate locking added.
20 */
21
22static int notifier_chain_register(struct notifier_block **nl,
23 struct notifier_block *n)
24{
25 while ((*nl) != NULL) {
26 if (unlikely((*nl) == n)) {
27 WARN(1, "notifier callback %ps already registered",
28 n->notifier_call);
29 return -EEXIST;
30 }
31 if (n->priority > (*nl)->priority)
32 break;
33 nl = &((*nl)->next);
34 }
35 n->next = *nl;
36 rcu_assign_pointer(*nl, n);
37 return 0;
38}
39
40static int notifier_chain_unregister(struct notifier_block **nl,
41 struct notifier_block *n)
42{
43 while ((*nl) != NULL) {
44 if ((*nl) == n) {
45 rcu_assign_pointer(*nl, n->next);
46 return 0;
47 }
48 nl = &((*nl)->next);
49 }
50 return -ENOENT;
51}
52
53/**
54 * notifier_call_chain - Informs the registered notifiers about an event.
55 * @nl: Pointer to head of the blocking notifier chain
56 * @val: Value passed unmodified to notifier function
57 * @v: Pointer passed unmodified to notifier function
58 * @nr_to_call: Number of notifier functions to be called. Don't care
59 * value of this parameter is -1.
60 * @nr_calls: Records the number of notifications sent. Don't care
61 * value of this field is NULL.
62 * @returns: notifier_call_chain returns the value returned by the
63 * last notifier function called.
64 */
65static int notifier_call_chain(struct notifier_block **nl,
66 unsigned long val, void *v,
67 int nr_to_call, int *nr_calls)
68{
69 int ret = NOTIFY_DONE;
70 struct notifier_block *nb, *next_nb;
71
72 nb = rcu_dereference_raw(*nl);
73
74 while (nb && nr_to_call) {
75 next_nb = rcu_dereference_raw(nb->next);
76
77#ifdef CONFIG_DEBUG_NOTIFIERS
78 if (unlikely(!func_ptr_is_kernel_text(nb->notifier_call))) {
79 WARN(1, "Invalid notifier called!");
80 nb = next_nb;
81 continue;
82 }
83#endif
84 ret = nb->notifier_call(nb, val, v);
85
86 if (nr_calls)
87 (*nr_calls)++;
88
89 if (ret & NOTIFY_STOP_MASK)
90 break;
91 nb = next_nb;
92 nr_to_call--;
93 }
94 return ret;
95}
96NOKPROBE_SYMBOL(notifier_call_chain);
97
98/**
99 * notifier_call_chain_robust - Inform the registered notifiers about an event
100 * and rollback on error.
101 * @nl: Pointer to head of the blocking notifier chain
102 * @val_up: Value passed unmodified to the notifier function
103 * @val_down: Value passed unmodified to the notifier function when recovering
104 * from an error on @val_up
105 * @v Pointer passed unmodified to the notifier function
106 *
107 * NOTE: It is important the @nl chain doesn't change between the two
108 * invocations of notifier_call_chain() such that we visit the
109 * exact same notifier callbacks; this rules out any RCU usage.
110 *
111 * Returns: the return value of the @val_up call.
112 */
113static int notifier_call_chain_robust(struct notifier_block **nl,
114 unsigned long val_up, unsigned long val_down,
115 void *v)
116{
117 int ret, nr = 0;
118
119 ret = notifier_call_chain(nl, val_up, v, -1, &nr);
120 if (ret & NOTIFY_STOP_MASK)
121 notifier_call_chain(nl, val_down, v, nr-1, NULL);
122
123 return ret;
124}
125
126/*
127 * Atomic notifier chain routines. Registration and unregistration
128 * use a spinlock, and call_chain is synchronized by RCU (no locks).
129 */
130
131/**
132 * atomic_notifier_chain_register - Add notifier to an atomic notifier chain
133 * @nh: Pointer to head of the atomic notifier chain
134 * @n: New entry in notifier chain
135 *
136 * Adds a notifier to an atomic notifier chain.
137 *
138 * Returns 0 on success, %-EEXIST on error.
139 */
140int atomic_notifier_chain_register(struct atomic_notifier_head *nh,
141 struct notifier_block *n)
142{
143 unsigned long flags;
144 int ret;
145
146 spin_lock_irqsave(&nh->lock, flags);
147 ret = notifier_chain_register(&nh->head, n);
148 spin_unlock_irqrestore(&nh->lock, flags);
149 return ret;
150}
151EXPORT_SYMBOL_GPL(atomic_notifier_chain_register);
152
153/**
154 * atomic_notifier_chain_unregister - Remove notifier from an atomic notifier chain
155 * @nh: Pointer to head of the atomic notifier chain
156 * @n: Entry to remove from notifier chain
157 *
158 * Removes a notifier from an atomic notifier chain.
159 *
160 * Returns zero on success or %-ENOENT on failure.
161 */
162int atomic_notifier_chain_unregister(struct atomic_notifier_head *nh,
163 struct notifier_block *n)
164{
165 unsigned long flags;
166 int ret;
167
168 spin_lock_irqsave(&nh->lock, flags);
169 ret = notifier_chain_unregister(&nh->head, n);
170 spin_unlock_irqrestore(&nh->lock, flags);
171 synchronize_rcu();
172 return ret;
173}
174EXPORT_SYMBOL_GPL(atomic_notifier_chain_unregister);
175
176/**
177 * atomic_notifier_call_chain - Call functions in an atomic notifier chain
178 * @nh: Pointer to head of the atomic notifier chain
179 * @val: Value passed unmodified to notifier function
180 * @v: Pointer passed unmodified to notifier function
181 *
182 * Calls each function in a notifier chain in turn. The functions
183 * run in an atomic context, so they must not block.
184 * This routine uses RCU to synchronize with changes to the chain.
185 *
186 * If the return value of the notifier can be and'ed
187 * with %NOTIFY_STOP_MASK then atomic_notifier_call_chain()
188 * will return immediately, with the return value of
189 * the notifier function which halted execution.
190 * Otherwise the return value is the return value
191 * of the last notifier function called.
192 */
193int atomic_notifier_call_chain(struct atomic_notifier_head *nh,
194 unsigned long val, void *v)
195{
196 int ret;
197
198 rcu_read_lock();
199 ret = notifier_call_chain(&nh->head, val, v, -1, NULL);
200 rcu_read_unlock();
201
202 return ret;
203}
204EXPORT_SYMBOL_GPL(atomic_notifier_call_chain);
205NOKPROBE_SYMBOL(atomic_notifier_call_chain);
206
207/*
208 * Blocking notifier chain routines. All access to the chain is
209 * synchronized by an rwsem.
210 */
211
212/**
213 * blocking_notifier_chain_register - Add notifier to a blocking notifier chain
214 * @nh: Pointer to head of the blocking notifier chain
215 * @n: New entry in notifier chain
216 *
217 * Adds a notifier to a blocking notifier chain.
218 * Must be called in process context.
219 *
220 * Returns 0 on success, %-EEXIST on error.
221 */
222int blocking_notifier_chain_register(struct blocking_notifier_head *nh,
223 struct notifier_block *n)
224{
225 int ret;
226
227 /*
228 * This code gets used during boot-up, when task switching is
229 * not yet working and interrupts must remain disabled. At
230 * such times we must not call down_write().
231 */
232 if (unlikely(system_state == SYSTEM_BOOTING))
233 return notifier_chain_register(&nh->head, n);
234
235 down_write(&nh->rwsem);
236 ret = notifier_chain_register(&nh->head, n);
237 up_write(&nh->rwsem);
238 return ret;
239}
240EXPORT_SYMBOL_GPL(blocking_notifier_chain_register);
241
242/**
243 * blocking_notifier_chain_unregister - Remove notifier from a blocking notifier chain
244 * @nh: Pointer to head of the blocking notifier chain
245 * @n: Entry to remove from notifier chain
246 *
247 * Removes a notifier from a blocking notifier chain.
248 * Must be called from process context.
249 *
250 * Returns zero on success or %-ENOENT on failure.
251 */
252int blocking_notifier_chain_unregister(struct blocking_notifier_head *nh,
253 struct notifier_block *n)
254{
255 int ret;
256
257 /*
258 * This code gets used during boot-up, when task switching is
259 * not yet working and interrupts must remain disabled. At
260 * such times we must not call down_write().
261 */
262 if (unlikely(system_state == SYSTEM_BOOTING))
263 return notifier_chain_unregister(&nh->head, n);
264
265 down_write(&nh->rwsem);
266 ret = notifier_chain_unregister(&nh->head, n);
267 up_write(&nh->rwsem);
268 return ret;
269}
270EXPORT_SYMBOL_GPL(blocking_notifier_chain_unregister);
271
272int blocking_notifier_call_chain_robust(struct blocking_notifier_head *nh,
273 unsigned long val_up, unsigned long val_down, void *v)
274{
275 int ret = NOTIFY_DONE;
276
277 /*
278 * We check the head outside the lock, but if this access is
279 * racy then it does not matter what the result of the test
280 * is, we re-check the list after having taken the lock anyway:
281 */
282 if (rcu_access_pointer(nh->head)) {
283 down_read(&nh->rwsem);
284 ret = notifier_call_chain_robust(&nh->head, val_up, val_down, v);
285 up_read(&nh->rwsem);
286 }
287 return ret;
288}
289EXPORT_SYMBOL_GPL(blocking_notifier_call_chain_robust);
290
291/**
292 * blocking_notifier_call_chain - Call functions in a blocking notifier chain
293 * @nh: Pointer to head of the blocking notifier chain
294 * @val: Value passed unmodified to notifier function
295 * @v: Pointer passed unmodified to notifier function
296 *
297 * Calls each function in a notifier chain in turn. The functions
298 * run in a process context, so they are allowed to block.
299 *
300 * If the return value of the notifier can be and'ed
301 * with %NOTIFY_STOP_MASK then blocking_notifier_call_chain()
302 * will return immediately, with the return value of
303 * the notifier function which halted execution.
304 * Otherwise the return value is the return value
305 * of the last notifier function called.
306 */
307int blocking_notifier_call_chain(struct blocking_notifier_head *nh,
308 unsigned long val, void *v)
309{
310 int ret = NOTIFY_DONE;
311
312 /*
313 * We check the head outside the lock, but if this access is
314 * racy then it does not matter what the result of the test
315 * is, we re-check the list after having taken the lock anyway:
316 */
317 if (rcu_access_pointer(nh->head)) {
318 down_read(&nh->rwsem);
319 ret = notifier_call_chain(&nh->head, val, v, -1, NULL);
320 up_read(&nh->rwsem);
321 }
322 return ret;
323}
324EXPORT_SYMBOL_GPL(blocking_notifier_call_chain);
325
326/*
327 * Raw notifier chain routines. There is no protection;
328 * the caller must provide it. Use at your own risk!
329 */
330
331/**
332 * raw_notifier_chain_register - Add notifier to a raw notifier chain
333 * @nh: Pointer to head of the raw notifier chain
334 * @n: New entry in notifier chain
335 *
336 * Adds a notifier to a raw notifier chain.
337 * All locking must be provided by the caller.
338 *
339 * Returns 0 on success, %-EEXIST on error.
340 */
341int raw_notifier_chain_register(struct raw_notifier_head *nh,
342 struct notifier_block *n)
343{
344 return notifier_chain_register(&nh->head, n);
345}
346EXPORT_SYMBOL_GPL(raw_notifier_chain_register);
347
348/**
349 * raw_notifier_chain_unregister - Remove notifier from a raw notifier chain
350 * @nh: Pointer to head of the raw notifier chain
351 * @n: Entry to remove from notifier chain
352 *
353 * Removes a notifier from a raw notifier chain.
354 * All locking must be provided by the caller.
355 *
356 * Returns zero on success or %-ENOENT on failure.
357 */
358int raw_notifier_chain_unregister(struct raw_notifier_head *nh,
359 struct notifier_block *n)
360{
361 return notifier_chain_unregister(&nh->head, n);
362}
363EXPORT_SYMBOL_GPL(raw_notifier_chain_unregister);
364
365int raw_notifier_call_chain_robust(struct raw_notifier_head *nh,
366 unsigned long val_up, unsigned long val_down, void *v)
367{
368 return notifier_call_chain_robust(&nh->head, val_up, val_down, v);
369}
370EXPORT_SYMBOL_GPL(raw_notifier_call_chain_robust);
371
372/**
373 * raw_notifier_call_chain - Call functions in a raw notifier chain
374 * @nh: Pointer to head of the raw notifier chain
375 * @val: Value passed unmodified to notifier function
376 * @v: Pointer passed unmodified to notifier function
377 *
378 * Calls each function in a notifier chain in turn. The functions
379 * run in an undefined context.
380 * All locking must be provided by the caller.
381 *
382 * If the return value of the notifier can be and'ed
383 * with %NOTIFY_STOP_MASK then raw_notifier_call_chain()
384 * will return immediately, with the return value of
385 * the notifier function which halted execution.
386 * Otherwise the return value is the return value
387 * of the last notifier function called.
388 */
389int raw_notifier_call_chain(struct raw_notifier_head *nh,
390 unsigned long val, void *v)
391{
392 return notifier_call_chain(&nh->head, val, v, -1, NULL);
393}
394EXPORT_SYMBOL_GPL(raw_notifier_call_chain);
395
396#ifdef CONFIG_SRCU
397/*
398 * SRCU notifier chain routines. Registration and unregistration
399 * use a mutex, and call_chain is synchronized by SRCU (no locks).
400 */
401
402/**
403 * srcu_notifier_chain_register - Add notifier to an SRCU notifier chain
404 * @nh: Pointer to head of the SRCU notifier chain
405 * @n: New entry in notifier chain
406 *
407 * Adds a notifier to an SRCU notifier chain.
408 * Must be called in process context.
409 *
410 * Returns 0 on success, %-EEXIST on error.
411 */
412int srcu_notifier_chain_register(struct srcu_notifier_head *nh,
413 struct notifier_block *n)
414{
415 int ret;
416
417 /*
418 * This code gets used during boot-up, when task switching is
419 * not yet working and interrupts must remain disabled. At
420 * such times we must not call mutex_lock().
421 */
422 if (unlikely(system_state == SYSTEM_BOOTING))
423 return notifier_chain_register(&nh->head, n);
424
425 mutex_lock(&nh->mutex);
426 ret = notifier_chain_register(&nh->head, n);
427 mutex_unlock(&nh->mutex);
428 return ret;
429}
430EXPORT_SYMBOL_GPL(srcu_notifier_chain_register);
431
432/**
433 * srcu_notifier_chain_unregister - Remove notifier from an SRCU notifier chain
434 * @nh: Pointer to head of the SRCU notifier chain
435 * @n: Entry to remove from notifier chain
436 *
437 * Removes a notifier from an SRCU notifier chain.
438 * Must be called from process context.
439 *
440 * Returns zero on success or %-ENOENT on failure.
441 */
442int srcu_notifier_chain_unregister(struct srcu_notifier_head *nh,
443 struct notifier_block *n)
444{
445 int ret;
446
447 /*
448 * This code gets used during boot-up, when task switching is
449 * not yet working and interrupts must remain disabled. At
450 * such times we must not call mutex_lock().
451 */
452 if (unlikely(system_state == SYSTEM_BOOTING))
453 return notifier_chain_unregister(&nh->head, n);
454
455 mutex_lock(&nh->mutex);
456 ret = notifier_chain_unregister(&nh->head, n);
457 mutex_unlock(&nh->mutex);
458 synchronize_srcu(&nh->srcu);
459 return ret;
460}
461EXPORT_SYMBOL_GPL(srcu_notifier_chain_unregister);
462
463/**
464 * srcu_notifier_call_chain - Call functions in an SRCU notifier chain
465 * @nh: Pointer to head of the SRCU notifier chain
466 * @val: Value passed unmodified to notifier function
467 * @v: Pointer passed unmodified to notifier function
468 *
469 * Calls each function in a notifier chain in turn. The functions
470 * run in a process context, so they are allowed to block.
471 *
472 * If the return value of the notifier can be and'ed
473 * with %NOTIFY_STOP_MASK then srcu_notifier_call_chain()
474 * will return immediately, with the return value of
475 * the notifier function which halted execution.
476 * Otherwise the return value is the return value
477 * of the last notifier function called.
478 */
479int srcu_notifier_call_chain(struct srcu_notifier_head *nh,
480 unsigned long val, void *v)
481{
482 int ret;
483 int idx;
484
485 idx = srcu_read_lock(&nh->srcu);
486 ret = notifier_call_chain(&nh->head, val, v, -1, NULL);
487 srcu_read_unlock(&nh->srcu, idx);
488 return ret;
489}
490EXPORT_SYMBOL_GPL(srcu_notifier_call_chain);
491
492/**
493 * srcu_init_notifier_head - Initialize an SRCU notifier head
494 * @nh: Pointer to head of the srcu notifier chain
495 *
496 * Unlike other sorts of notifier heads, SRCU notifier heads require
497 * dynamic initialization. Be sure to call this routine before
498 * calling any of the other SRCU notifier routines for this head.
499 *
500 * If an SRCU notifier head is deallocated, it must first be cleaned
501 * up by calling srcu_cleanup_notifier_head(). Otherwise the head's
502 * per-cpu data (used by the SRCU mechanism) will leak.
503 */
504void srcu_init_notifier_head(struct srcu_notifier_head *nh)
505{
506 mutex_init(&nh->mutex);
507 if (init_srcu_struct(&nh->srcu) < 0)
508 BUG();
509 nh->head = NULL;
510}
511EXPORT_SYMBOL_GPL(srcu_init_notifier_head);
512
513#endif /* CONFIG_SRCU */
514
515static ATOMIC_NOTIFIER_HEAD(die_chain);
516
517int notrace notify_die(enum die_val val, const char *str,
518 struct pt_regs *regs, long err, int trap, int sig)
519{
520 struct die_args args = {
521 .regs = regs,
522 .str = str,
523 .err = err,
524 .trapnr = trap,
525 .signr = sig,
526
527 };
528 RCU_LOCKDEP_WARN(!rcu_is_watching(),
529 "notify_die called but RCU thinks we're quiescent");
530 return atomic_notifier_call_chain(&die_chain, val, &args);
531}
532NOKPROBE_SYMBOL(notify_die);
533
534int register_die_notifier(struct notifier_block *nb)
535{
536 return atomic_notifier_chain_register(&die_chain, nb);
537}
538EXPORT_SYMBOL_GPL(register_die_notifier);
539
540int unregister_die_notifier(struct notifier_block *nb)
541{
542 return atomic_notifier_chain_unregister(&die_chain, nb);
543}
544EXPORT_SYMBOL_GPL(unregister_die_notifier);