Line data Source code
1 : // SPDX-License-Identifier: GPL-2.0-only
2 : #include <linux/kdebug.h>
3 : #include <linux/kprobes.h>
4 : #include <linux/export.h>
5 : #include <linux/notifier.h>
6 : #include <linux/rcupdate.h>
7 : #include <linux/vmalloc.h>
8 : #include <linux/reboot.h>
9 :
10 : /*
11 : * Notifier list for kernel code which wants to be called
12 : * at shutdown. This is used to stop any idling DMA operations
13 : * and the like.
14 : */
15 : BLOCKING_NOTIFIER_HEAD(reboot_notifier_list);
16 :
17 : /*
18 : * Notifier chain core routines. The exported routines below
19 : * are layered on top of these, with appropriate locking added.
20 : */
21 :
22 16 : static int notifier_chain_register(struct notifier_block **nl,
23 : struct notifier_block *n)
24 : {
25 42 : while ((*nl) != NULL) {
26 28 : if (unlikely((*nl) == n)) {
27 0 : WARN(1, "double register detected");
28 0 : return 0;
29 : }
30 28 : if (n->priority > (*nl)->priority)
31 : break;
32 26 : nl = &((*nl)->next);
33 : }
34 16 : n->next = *nl;
35 16 : rcu_assign_pointer(*nl, n);
36 16 : return 0;
37 : }
38 :
39 1 : static int notifier_chain_unregister(struct notifier_block **nl,
40 : struct notifier_block *n)
41 : {
42 2 : while ((*nl) != NULL) {
43 2 : if ((*nl) == n) {
44 1 : rcu_assign_pointer(*nl, n->next);
45 1 : return 0;
46 : }
47 1 : nl = &((*nl)->next);
48 : }
49 : return -ENOENT;
50 : }
51 :
52 : /**
53 : * notifier_call_chain - Informs the registered notifiers about an event.
54 : * @nl: Pointer to head of the blocking notifier chain
55 : * @val: Value passed unmodified to notifier function
56 : * @v: Pointer passed unmodified to notifier function
57 : * @nr_to_call: Number of notifier functions to be called. Don't care
58 : * value of this parameter is -1.
59 : * @nr_calls: Records the number of notifications sent. Don't care
60 : * value of this field is NULL.
61 : * @returns: notifier_call_chain returns the value returned by the
62 : * last notifier function called.
63 : */
64 7091 : static int notifier_call_chain(struct notifier_block **nl,
65 : unsigned long val, void *v,
66 : int nr_to_call, int *nr_calls)
67 : {
68 7091 : int ret = NOTIFY_DONE;
69 7091 : struct notifier_block *nb, *next_nb;
70 :
71 7091 : nb = rcu_dereference_raw(*nl);
72 :
73 7134 : while (nb && nr_to_call) {
74 44 : next_nb = rcu_dereference_raw(nb->next);
75 :
76 : #ifdef CONFIG_DEBUG_NOTIFIERS
77 : if (unlikely(!func_ptr_is_kernel_text(nb->notifier_call))) {
78 : WARN(1, "Invalid notifier called!");
79 : nb = next_nb;
80 : continue;
81 : }
82 : #endif
83 44 : ret = nb->notifier_call(nb, val, v);
84 :
85 44 : if (nr_calls)
86 0 : (*nr_calls)++;
87 :
88 44 : if (ret & NOTIFY_STOP_MASK)
89 : break;
90 43 : nb = next_nb;
91 43 : nr_to_call--;
92 : }
93 7091 : return ret;
94 : }
95 : NOKPROBE_SYMBOL(notifier_call_chain);
96 :
97 : /**
98 : * notifier_call_chain_robust - Inform the registered notifiers about an event
99 : * and rollback on error.
100 : * @nl: Pointer to head of the blocking notifier chain
101 : * @val_up: Value passed unmodified to the notifier function
102 : * @val_down: Value passed unmodified to the notifier function when recovering
103 : * from an error on @val_up
104 : * @v Pointer passed unmodified to the notifier function
105 : *
106 : * NOTE: It is important the @nl chain doesn't change between the two
107 : * invocations of notifier_call_chain() such that we visit the
108 : * exact same notifier callbacks; this rules out any RCU usage.
109 : *
110 : * Returns: the return value of the @val_up call.
111 : */
112 0 : static int notifier_call_chain_robust(struct notifier_block **nl,
113 : unsigned long val_up, unsigned long val_down,
114 : void *v)
115 : {
116 0 : int ret, nr = 0;
117 :
118 0 : ret = notifier_call_chain(nl, val_up, v, -1, &nr);
119 0 : if (ret & NOTIFY_STOP_MASK)
120 0 : notifier_call_chain(nl, val_down, v, nr-1, NULL);
121 :
122 0 : return ret;
123 : }
124 :
125 : /*
126 : * Atomic notifier chain routines. Registration and unregistration
127 : * use a spinlock, and call_chain is synchronized by RCU (no locks).
128 : */
129 :
130 : /**
131 : * atomic_notifier_chain_register - Add notifier to an atomic notifier chain
132 : * @nh: Pointer to head of the atomic notifier chain
133 : * @n: New entry in notifier chain
134 : *
135 : * Adds a notifier to an atomic notifier chain.
136 : *
137 : * Currently always returns zero.
138 : */
139 6 : int atomic_notifier_chain_register(struct atomic_notifier_head *nh,
140 : struct notifier_block *n)
141 : {
142 6 : unsigned long flags;
143 6 : int ret;
144 :
145 6 : spin_lock_irqsave(&nh->lock, flags);
146 6 : ret = notifier_chain_register(&nh->head, n);
147 6 : spin_unlock_irqrestore(&nh->lock, flags);
148 6 : return ret;
149 : }
150 : EXPORT_SYMBOL_GPL(atomic_notifier_chain_register);
151 :
152 : /**
153 : * atomic_notifier_chain_unregister - Remove notifier from an atomic notifier chain
154 : * @nh: Pointer to head of the atomic notifier chain
155 : * @n: Entry to remove from notifier chain
156 : *
157 : * Removes a notifier from an atomic notifier chain.
158 : *
159 : * Returns zero on success or %-ENOENT on failure.
160 : */
161 1 : int atomic_notifier_chain_unregister(struct atomic_notifier_head *nh,
162 : struct notifier_block *n)
163 : {
164 1 : unsigned long flags;
165 1 : int ret;
166 :
167 1 : spin_lock_irqsave(&nh->lock, flags);
168 1 : ret = notifier_chain_unregister(&nh->head, n);
169 1 : spin_unlock_irqrestore(&nh->lock, flags);
170 1 : synchronize_rcu();
171 1 : return ret;
172 : }
173 : EXPORT_SYMBOL_GPL(atomic_notifier_chain_unregister);
174 :
175 0 : int atomic_notifier_call_chain_robust(struct atomic_notifier_head *nh,
176 : unsigned long val_up, unsigned long val_down, void *v)
177 : {
178 0 : unsigned long flags;
179 0 : int ret;
180 :
181 : /*
182 : * Musn't use RCU; because then the notifier list can
183 : * change between the up and down traversal.
184 : */
185 0 : spin_lock_irqsave(&nh->lock, flags);
186 0 : ret = notifier_call_chain_robust(&nh->head, val_up, val_down, v);
187 0 : spin_unlock_irqrestore(&nh->lock, flags);
188 :
189 0 : return ret;
190 : }
191 : EXPORT_SYMBOL_GPL(atomic_notifier_call_chain_robust);
192 : NOKPROBE_SYMBOL(atomic_notifier_call_chain_robust);
193 :
194 : /**
195 : * atomic_notifier_call_chain - Call functions in an atomic notifier chain
196 : * @nh: Pointer to head of the atomic notifier chain
197 : * @val: Value passed unmodified to notifier function
198 : * @v: Pointer passed unmodified to notifier function
199 : *
200 : * Calls each function in a notifier chain in turn. The functions
201 : * run in an atomic context, so they must not block.
202 : * This routine uses RCU to synchronize with changes to the chain.
203 : *
204 : * If the return value of the notifier can be and'ed
205 : * with %NOTIFY_STOP_MASK then atomic_notifier_call_chain()
206 : * will return immediately, with the return value of
207 : * the notifier function which halted execution.
208 : * Otherwise the return value is the return value
209 : * of the last notifier function called.
210 : */
211 170 : int atomic_notifier_call_chain(struct atomic_notifier_head *nh,
212 : unsigned long val, void *v)
213 : {
214 170 : int ret;
215 :
216 170 : rcu_read_lock();
217 170 : ret = notifier_call_chain(&nh->head, val, v, -1, NULL);
218 170 : rcu_read_unlock();
219 :
220 170 : return ret;
221 : }
222 : EXPORT_SYMBOL_GPL(atomic_notifier_call_chain);
223 : NOKPROBE_SYMBOL(atomic_notifier_call_chain);
224 :
225 : /*
226 : * Blocking notifier chain routines. All access to the chain is
227 : * synchronized by an rwsem.
228 : */
229 :
230 : /**
231 : * blocking_notifier_chain_register - Add notifier to a blocking notifier chain
232 : * @nh: Pointer to head of the blocking notifier chain
233 : * @n: New entry in notifier chain
234 : *
235 : * Adds a notifier to a blocking notifier chain.
236 : * Must be called in process context.
237 : *
238 : * Currently always returns zero.
239 : */
240 3 : int blocking_notifier_chain_register(struct blocking_notifier_head *nh,
241 : struct notifier_block *n)
242 : {
243 3 : int ret;
244 :
245 : /*
246 : * This code gets used during boot-up, when task switching is
247 : * not yet working and interrupts must remain disabled. At
248 : * such times we must not call down_write().
249 : */
250 3 : if (unlikely(system_state == SYSTEM_BOOTING))
251 2 : return notifier_chain_register(&nh->head, n);
252 :
253 1 : down_write(&nh->rwsem);
254 1 : ret = notifier_chain_register(&nh->head, n);
255 1 : up_write(&nh->rwsem);
256 1 : return ret;
257 : }
258 : EXPORT_SYMBOL_GPL(blocking_notifier_chain_register);
259 :
260 : /**
261 : * blocking_notifier_chain_unregister - Remove notifier from a blocking notifier chain
262 : * @nh: Pointer to head of the blocking notifier chain
263 : * @n: Entry to remove from notifier chain
264 : *
265 : * Removes a notifier from a blocking notifier chain.
266 : * Must be called from process context.
267 : *
268 : * Returns zero on success or %-ENOENT on failure.
269 : */
270 0 : int blocking_notifier_chain_unregister(struct blocking_notifier_head *nh,
271 : struct notifier_block *n)
272 : {
273 0 : int ret;
274 :
275 : /*
276 : * This code gets used during boot-up, when task switching is
277 : * not yet working and interrupts must remain disabled. At
278 : * such times we must not call down_write().
279 : */
280 0 : if (unlikely(system_state == SYSTEM_BOOTING))
281 0 : return notifier_chain_unregister(&nh->head, n);
282 :
283 0 : down_write(&nh->rwsem);
284 0 : ret = notifier_chain_unregister(&nh->head, n);
285 0 : up_write(&nh->rwsem);
286 0 : return ret;
287 : }
288 : EXPORT_SYMBOL_GPL(blocking_notifier_chain_unregister);
289 :
290 0 : int blocking_notifier_call_chain_robust(struct blocking_notifier_head *nh,
291 : unsigned long val_up, unsigned long val_down, void *v)
292 : {
293 0 : int ret = NOTIFY_DONE;
294 :
295 : /*
296 : * We check the head outside the lock, but if this access is
297 : * racy then it does not matter what the result of the test
298 : * is, we re-check the list after having taken the lock anyway:
299 : */
300 0 : if (rcu_access_pointer(nh->head)) {
301 0 : down_read(&nh->rwsem);
302 0 : ret = notifier_call_chain_robust(&nh->head, val_up, val_down, v);
303 0 : up_read(&nh->rwsem);
304 : }
305 0 : return ret;
306 : }
307 : EXPORT_SYMBOL_GPL(blocking_notifier_call_chain_robust);
308 :
309 : /**
310 : * blocking_notifier_call_chain - Call functions in a blocking notifier chain
311 : * @nh: Pointer to head of the blocking notifier chain
312 : * @val: Value passed unmodified to notifier function
313 : * @v: Pointer passed unmodified to notifier function
314 : *
315 : * Calls each function in a notifier chain in turn. The functions
316 : * run in a process context, so they are allowed to block.
317 : *
318 : * If the return value of the notifier can be and'ed
319 : * with %NOTIFY_STOP_MASK then blocking_notifier_call_chain()
320 : * will return immediately, with the return value of
321 : * the notifier function which halted execution.
322 : * Otherwise the return value is the return value
323 : * of the last notifier function called.
324 : */
325 109 : int blocking_notifier_call_chain(struct blocking_notifier_head *nh,
326 : unsigned long val, void *v)
327 : {
328 109 : int ret = NOTIFY_DONE;
329 :
330 : /*
331 : * We check the head outside the lock, but if this access is
332 : * racy then it does not matter what the result of the test
333 : * is, we re-check the list after having taken the lock anyway:
334 : */
335 109 : if (rcu_access_pointer(nh->head)) {
336 2 : down_read(&nh->rwsem);
337 2 : ret = notifier_call_chain(&nh->head, val, v, -1, NULL);
338 2 : up_read(&nh->rwsem);
339 : }
340 109 : return ret;
341 : }
342 : EXPORT_SYMBOL_GPL(blocking_notifier_call_chain);
343 :
344 : /*
345 : * Raw notifier chain routines. There is no protection;
346 : * the caller must provide it. Use at your own risk!
347 : */
348 :
349 : /**
350 : * raw_notifier_chain_register - Add notifier to a raw notifier chain
351 : * @nh: Pointer to head of the raw notifier chain
352 : * @n: New entry in notifier chain
353 : *
354 : * Adds a notifier to a raw notifier chain.
355 : * All locking must be provided by the caller.
356 : *
357 : * Currently always returns zero.
358 : */
359 7 : int raw_notifier_chain_register(struct raw_notifier_head *nh,
360 : struct notifier_block *n)
361 : {
362 7 : return notifier_chain_register(&nh->head, n);
363 : }
364 : EXPORT_SYMBOL_GPL(raw_notifier_chain_register);
365 :
366 : /**
367 : * raw_notifier_chain_unregister - Remove notifier from a raw notifier chain
368 : * @nh: Pointer to head of the raw notifier chain
369 : * @n: Entry to remove from notifier chain
370 : *
371 : * Removes a notifier from a raw notifier chain.
372 : * All locking must be provided by the caller.
373 : *
374 : * Returns zero on success or %-ENOENT on failure.
375 : */
376 0 : int raw_notifier_chain_unregister(struct raw_notifier_head *nh,
377 : struct notifier_block *n)
378 : {
379 0 : return notifier_chain_unregister(&nh->head, n);
380 : }
381 : EXPORT_SYMBOL_GPL(raw_notifier_chain_unregister);
382 :
383 0 : int raw_notifier_call_chain_robust(struct raw_notifier_head *nh,
384 : unsigned long val_up, unsigned long val_down, void *v)
385 : {
386 0 : return notifier_call_chain_robust(&nh->head, val_up, val_down, v);
387 : }
388 : EXPORT_SYMBOL_GPL(raw_notifier_call_chain_robust);
389 :
390 : /**
391 : * raw_notifier_call_chain - Call functions in a raw notifier chain
392 : * @nh: Pointer to head of the raw notifier chain
393 : * @val: Value passed unmodified to notifier function
394 : * @v: Pointer passed unmodified to notifier function
395 : *
396 : * Calls each function in a notifier chain in turn. The functions
397 : * run in an undefined context.
398 : * All locking must be provided by the caller.
399 : *
400 : * If the return value of the notifier can be and'ed
401 : * with %NOTIFY_STOP_MASK then raw_notifier_call_chain()
402 : * will return immediately, with the return value of
403 : * the notifier function which halted execution.
404 : * Otherwise the return value is the return value
405 : * of the last notifier function called.
406 : */
407 6919 : int raw_notifier_call_chain(struct raw_notifier_head *nh,
408 : unsigned long val, void *v)
409 : {
410 6919 : return notifier_call_chain(&nh->head, val, v, -1, NULL);
411 : }
412 : EXPORT_SYMBOL_GPL(raw_notifier_call_chain);
413 :
414 : #ifdef CONFIG_SRCU
415 : /*
416 : * SRCU notifier chain routines. Registration and unregistration
417 : * use a mutex, and call_chain is synchronized by SRCU (no locks).
418 : */
419 :
420 : /**
421 : * srcu_notifier_chain_register - Add notifier to an SRCU notifier chain
422 : * @nh: Pointer to head of the SRCU notifier chain
423 : * @n: New entry in notifier chain
424 : *
425 : * Adds a notifier to an SRCU notifier chain.
426 : * Must be called in process context.
427 : *
428 : * Currently always returns zero.
429 : */
430 0 : int srcu_notifier_chain_register(struct srcu_notifier_head *nh,
431 : struct notifier_block *n)
432 : {
433 0 : int ret;
434 :
435 : /*
436 : * This code gets used during boot-up, when task switching is
437 : * not yet working and interrupts must remain disabled. At
438 : * such times we must not call mutex_lock().
439 : */
440 0 : if (unlikely(system_state == SYSTEM_BOOTING))
441 0 : return notifier_chain_register(&nh->head, n);
442 :
443 0 : mutex_lock(&nh->mutex);
444 0 : ret = notifier_chain_register(&nh->head, n);
445 0 : mutex_unlock(&nh->mutex);
446 0 : return ret;
447 : }
448 : EXPORT_SYMBOL_GPL(srcu_notifier_chain_register);
449 :
450 : /**
451 : * srcu_notifier_chain_unregister - Remove notifier from an SRCU notifier chain
452 : * @nh: Pointer to head of the SRCU notifier chain
453 : * @n: Entry to remove from notifier chain
454 : *
455 : * Removes a notifier from an SRCU notifier chain.
456 : * Must be called from process context.
457 : *
458 : * Returns zero on success or %-ENOENT on failure.
459 : */
460 0 : int srcu_notifier_chain_unregister(struct srcu_notifier_head *nh,
461 : struct notifier_block *n)
462 : {
463 0 : int ret;
464 :
465 : /*
466 : * This code gets used during boot-up, when task switching is
467 : * not yet working and interrupts must remain disabled. At
468 : * such times we must not call mutex_lock().
469 : */
470 0 : if (unlikely(system_state == SYSTEM_BOOTING))
471 0 : return notifier_chain_unregister(&nh->head, n);
472 :
473 0 : mutex_lock(&nh->mutex);
474 0 : ret = notifier_chain_unregister(&nh->head, n);
475 0 : mutex_unlock(&nh->mutex);
476 0 : synchronize_srcu(&nh->srcu);
477 0 : return ret;
478 : }
479 : EXPORT_SYMBOL_GPL(srcu_notifier_chain_unregister);
480 :
481 : /**
482 : * srcu_notifier_call_chain - Call functions in an SRCU notifier chain
483 : * @nh: Pointer to head of the SRCU notifier chain
484 : * @val: Value passed unmodified to notifier function
485 : * @v: Pointer passed unmodified to notifier function
486 : *
487 : * Calls each function in a notifier chain in turn. The functions
488 : * run in a process context, so they are allowed to block.
489 : *
490 : * If the return value of the notifier can be and'ed
491 : * with %NOTIFY_STOP_MASK then srcu_notifier_call_chain()
492 : * will return immediately, with the return value of
493 : * the notifier function which halted execution.
494 : * Otherwise the return value is the return value
495 : * of the last notifier function called.
496 : */
497 0 : int srcu_notifier_call_chain(struct srcu_notifier_head *nh,
498 : unsigned long val, void *v)
499 : {
500 0 : int ret;
501 0 : int idx;
502 :
503 0 : idx = srcu_read_lock(&nh->srcu);
504 0 : ret = notifier_call_chain(&nh->head, val, v, -1, NULL);
505 0 : srcu_read_unlock(&nh->srcu, idx);
506 0 : return ret;
507 : }
508 : EXPORT_SYMBOL_GPL(srcu_notifier_call_chain);
509 :
510 : /**
511 : * srcu_init_notifier_head - Initialize an SRCU notifier head
512 : * @nh: Pointer to head of the srcu notifier chain
513 : *
514 : * Unlike other sorts of notifier heads, SRCU notifier heads require
515 : * dynamic initialization. Be sure to call this routine before
516 : * calling any of the other SRCU notifier routines for this head.
517 : *
518 : * If an SRCU notifier head is deallocated, it must first be cleaned
519 : * up by calling srcu_cleanup_notifier_head(). Otherwise the head's
520 : * per-cpu data (used by the SRCU mechanism) will leak.
521 : */
522 1 : void srcu_init_notifier_head(struct srcu_notifier_head *nh)
523 : {
524 1 : mutex_init(&nh->mutex);
525 1 : if (init_srcu_struct(&nh->srcu) < 0)
526 0 : BUG();
527 1 : nh->head = NULL;
528 1 : }
529 : EXPORT_SYMBOL_GPL(srcu_init_notifier_head);
530 :
531 : #endif /* CONFIG_SRCU */
532 :
533 : static ATOMIC_NOTIFIER_HEAD(die_chain);
534 :
535 1 : int notrace notify_die(enum die_val val, const char *str,
536 : struct pt_regs *regs, long err, int trap, int sig)
537 : {
538 1 : struct die_args args = {
539 : .regs = regs,
540 : .str = str,
541 : .err = err,
542 : .trapnr = trap,
543 : .signr = sig,
544 :
545 : };
546 1 : RCU_LOCKDEP_WARN(!rcu_is_watching(),
547 : "notify_die called but RCU thinks we're quiescent");
548 1 : return atomic_notifier_call_chain(&die_chain, val, &args);
549 : }
550 : NOKPROBE_SYMBOL(notify_die);
551 :
552 3 : int register_die_notifier(struct notifier_block *nb)
553 : {
554 3 : return atomic_notifier_chain_register(&die_chain, nb);
555 : }
556 : EXPORT_SYMBOL_GPL(register_die_notifier);
557 :
558 1 : int unregister_die_notifier(struct notifier_block *nb)
559 : {
560 1 : return atomic_notifier_chain_unregister(&die_chain, nb);
561 : }
562 : EXPORT_SYMBOL_GPL(unregister_die_notifier);
|
