projects/linux·blob uaccess-cleanups · refs

1305 lines · 36.0 KiB

1// SPDX-License-Identifier: GPL-2.0-only
2/*
3 * Generic helpers for smp ipi calls
4 *
5 * (C) Jens Axboe <[email protected]> 2008
6 */
7
8#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
9
10#include <linux/irq_work.h>
11#include <linux/rcupdate.h>
12#include <linux/rculist.h>
13#include <linux/kernel.h>
14#include <linux/export.h>
15#include <linux/percpu.h>
16#include <linux/init.h>
17#include <linux/interrupt.h>
18#include <linux/gfp.h>
19#include <linux/slab.h>
20#include <linux/smp.h>
21#include <linux/cpu.h>
22#include <linux/sched.h>
23#include <linux/sched/idle.h>
24#include <linux/hypervisor.h>
25#include <linux/sched/clock.h>
26#include <linux/nmi.h>
27#include <linux/sched/debug.h>
28#include <linux/jump_label.h>
29#include <linux/string_choices.h>
30
31#include <trace/events/ipi.h>
32#define CREATE_TRACE_POINTS
33#include <trace/events/csd.h>
34#undef CREATE_TRACE_POINTS
35
36#include "smpboot.h"
37#include "sched/smp.h"
38
39#define CSD_TYPE(_csd) ((_csd)->node.u_flags & CSD_FLAG_TYPE_MASK)
40
41struct call_function_data {
42 call_single_data_t __percpu *csd;
43 cpumask_var_t cpumask;
44 cpumask_var_t cpumask_ipi;
45};
46
47static DEFINE_PER_CPU_ALIGNED(struct call_function_data, cfd_data);
48
49static DEFINE_PER_CPU_SHARED_ALIGNED(struct llist_head, call_single_queue);
50
51static DEFINE_PER_CPU(atomic_t, trigger_backtrace) = ATOMIC_INIT(1);
52
53static void __flush_smp_call_function_queue(bool warn_cpu_offline);
54
55int smpcfd_prepare_cpu(unsigned int cpu)
56{
57 struct call_function_data *cfd = &per_cpu(cfd_data, cpu);
58
59 if (!zalloc_cpumask_var_node(&cfd->cpumask, GFP_KERNEL,
60 cpu_to_node(cpu)))
61 return -ENOMEM;
62 if (!zalloc_cpumask_var_node(&cfd->cpumask_ipi, GFP_KERNEL,
63 cpu_to_node(cpu))) {
64 free_cpumask_var(cfd->cpumask);
65 return -ENOMEM;
66 }
67
68 /*
69 * Allocate the per-CPU CSD the first time a CPU comes up. It is
70 * not freed when the CPU is offlined, so csd_lock_wait() can access
71 * it even when the CPU was offlined after preemption was re-enabled.
72 */
73 if (!cfd->csd)
74 cfd->csd = alloc_percpu(call_single_data_t);
75 if (!cfd->csd) {
76 free_cpumask_var(cfd->cpumask);
77 free_cpumask_var(cfd->cpumask_ipi);
78 return -ENOMEM;
79 }
80
81 return 0;
82}
83
84int smpcfd_dead_cpu(unsigned int cpu)
85{
86 struct call_function_data *cfd = &per_cpu(cfd_data, cpu);
87
88 free_cpumask_var(cfd->cpumask);
89 free_cpumask_var(cfd->cpumask_ipi);
90 return 0;
91}
92
93int smpcfd_dying_cpu(unsigned int cpu)
94{
95 /*
96 * The IPIs for the smp-call-function callbacks queued by other CPUs
97 * might arrive late, either due to hardware latencies or because this
98 * CPU disabled interrupts (inside stop-machine) before the IPIs were
99 * sent. So flush out any pending callbacks explicitly (without waiting
100 * for the IPIs to arrive), to ensure that the outgoing CPU doesn't go
101 * offline with work still pending.
102 *
103 * This runs with interrupts disabled inside the stopper task invoked by
104 * stop_machine(), ensuring mutually exclusive CPU offlining and IPI flush.
105 */
106 __flush_smp_call_function_queue(false);
107 irq_work_run();
108 return 0;
109}
110
111void __init call_function_init(void)
112{
113 int i;
114
115 for_each_possible_cpu(i)
116 init_llist_head(&per_cpu(call_single_queue, i));
117
118 smpcfd_prepare_cpu(smp_processor_id());
119}
120
121static __always_inline void
122send_call_function_single_ipi(int cpu)
123{
124 if (call_function_single_prep_ipi(cpu)) {
125 trace_ipi_send_cpu(cpu, _RET_IP_,
126 generic_smp_call_function_single_interrupt);
127 arch_send_call_function_single_ipi(cpu);
128 }
129}
130
131static __always_inline void
132send_call_function_ipi_mask(struct cpumask *mask)
133{
134 trace_ipi_send_cpumask(mask, _RET_IP_,
135 generic_smp_call_function_single_interrupt);
136 arch_send_call_function_ipi_mask(mask);
137}
138
139static __always_inline void
140csd_do_func(smp_call_func_t func, void *info, call_single_data_t *csd)
141{
142 trace_csd_function_entry(func, csd);
143 func(info);
144 trace_csd_function_exit(func, csd);
145}
146
147static DEFINE_STATIC_KEY_MAYBE(CONFIG_CSD_LOCK_WAIT_DEBUG_DEFAULT, csdlock_debug_enabled);
148
149#ifdef CONFIG_CSD_LOCK_WAIT_DEBUG
150
151/*
152 * Parse the csdlock_debug= kernel boot parameter.
153 *
154 * If you need to restore the old "ext" value that once provided
155 * additional debugging information, reapply the following commits:
156 *
157 * de7b09ef658d ("locking/csd_lock: Prepare more CSD lock debugging")
158 * a5aabace5fb8 ("locking/csd_lock: Add more data to CSD lock debugging")
159 */
160static int __init csdlock_debug(char *str)
161{
162 int ret;
163 unsigned int val = 0;
164
165 ret = get_option(&str, &val);
166 if (ret) {
167 if (val)
168 static_branch_enable(&csdlock_debug_enabled);
169 else
170 static_branch_disable(&csdlock_debug_enabled);
171 }
172
173 return 1;
174}
175__setup("csdlock_debug=", csdlock_debug);
176
177static DEFINE_PER_CPU(call_single_data_t *, cur_csd);
178static DEFINE_PER_CPU(smp_call_func_t, cur_csd_func);
179static DEFINE_PER_CPU(void *, cur_csd_info);
180
181static ulong csd_lock_timeout = 5000; /* CSD lock timeout in milliseconds. */
182module_param(csd_lock_timeout, ulong, 0644);
183static int panic_on_ipistall; /* CSD panic timeout in milliseconds, 300000 for five minutes. */
184module_param(panic_on_ipistall, int, 0644);
185
186static atomic_t csd_bug_count = ATOMIC_INIT(0);
187
188/* Record current CSD work for current CPU, NULL to erase. */
189static void __csd_lock_record(call_single_data_t *csd)
190{
191 if (!csd) {
192 /*
193 * Pairs with smp_load_acquire() of cur_csd in
194 * csd_lock_wait_toolong(): orders any preceding CSD
195 * callback/unlock before a remote reader observes NULL.
196 */
197 smp_store_release(this_cpu_ptr(&cur_csd), NULL);
198 return;
199 }
200 __this_cpu_write(cur_csd_func, csd->func);
201 __this_cpu_write(cur_csd_info, csd->info);
202 /*
203 * Pairs with smp_load_acquire() of cur_csd in
204 * csd_lock_wait_toolong(): publishes cur_csd_func and
205 * cur_csd_info before the non-NULL pointer becomes visible.
206 */
207 smp_store_release(this_cpu_ptr(&cur_csd), csd);
208}
209
210static __always_inline void csd_lock_record(call_single_data_t *csd)
211{
212 if (static_branch_unlikely(&csdlock_debug_enabled))
213 __csd_lock_record(csd);
214}
215
216static int csd_lock_wait_getcpu(call_single_data_t *csd)
217{
218 unsigned int csd_type;
219
220 csd_type = CSD_TYPE(csd);
221 if (csd_type == CSD_TYPE_ASYNC || csd_type == CSD_TYPE_SYNC)
222 return csd->node.dst; /* Other CSD_TYPE_ values might not have ->dst. */
223 return -1;
224}
225
226static atomic_t n_csd_lock_stuck;
227
228/**
229 * csd_lock_is_stuck - Has a CSD-lock acquisition been stuck too long?
230 *
231 * Returns: @true if a CSD-lock acquisition is stuck and has been stuck
232 * long enough for a "non-responsive CSD lock" message to be printed.
233 */
234bool csd_lock_is_stuck(void)
235{
236 return !!atomic_read(&n_csd_lock_stuck);
237}
238
239/*
240 * Complain if too much time spent waiting. Note that only
241 * the CSD_TYPE_SYNC/ASYNC types provide the destination CPU,
242 * so waiting on other types gets much less information.
243 */
244static bool csd_lock_wait_toolong(call_single_data_t *csd, u64 ts0, u64 *ts1, int *bug_id, unsigned long *nmessages)
245{
246 int cpu = -1;
247 int cpux;
248 bool firsttime;
249 u64 ts2, ts_delta;
250 call_single_data_t *cpu_cur_csd;
251 unsigned int flags = READ_ONCE(csd->node.u_flags);
252 unsigned long long csd_lock_timeout_ns = csd_lock_timeout * NSEC_PER_MSEC;
253
254 if (!(flags & CSD_FLAG_LOCK)) {
255 if (!unlikely(*bug_id))
256 return true;
257 cpu = csd_lock_wait_getcpu(csd);
258 pr_alert("csd: CSD lock (#%d) got unstuck on CPU#%02d, CPU#%02d released the lock.\n",
259 *bug_id, raw_smp_processor_id(), cpu);
260 atomic_dec(&n_csd_lock_stuck);
261 return true;
262 }
263
264 ts2 = ktime_get_mono_fast_ns();
265 /* How long since we last checked for a stuck CSD lock.*/
266 ts_delta = ts2 - *ts1;
267 if (likely(ts_delta <= csd_lock_timeout_ns * (*nmessages + 1) *
268 (!*nmessages ? 1 : (ilog2(num_online_cpus()) / 2 + 1)) ||
269 csd_lock_timeout_ns == 0))
270 return false;
271
272 if (ts0 > ts2) {
273 /* Our own sched_clock went backward; don't blame another CPU. */
274 ts_delta = ts0 - ts2;
275 pr_alert("sched_clock on CPU %d went backward by %llu ns\n", raw_smp_processor_id(), ts_delta);
276 *ts1 = ts2;
277 return false;
278 }
279
280 firsttime = !*bug_id;
281 if (firsttime)
282 *bug_id = atomic_inc_return(&csd_bug_count);
283 cpu = csd_lock_wait_getcpu(csd);
284 if (WARN_ONCE(cpu < 0 || cpu >= nr_cpu_ids, "%s: cpu = %d\n", __func__, cpu))
285 cpux = 0;
286 else
287 cpux = cpu;
288 /*
289 * Pairs with smp_store_release() of cur_csd in __csd_lock_record():
290 * a non-NULL cur_csd here implies cur_csd_func and cur_csd_info
291 * are the matching publication; a NULL value is ordered after any
292 * preceding CSD callback/unlock on the remote CPU.
293 */
294 cpu_cur_csd = smp_load_acquire(&per_cpu(cur_csd, cpux));
295 /* How long since this CSD lock was stuck. */
296 ts_delta = ts2 - ts0;
297 pr_alert("csd: %s non-responsive CSD lock (#%d) on CPU#%d, waiting %lld ns for CPU#%02d %pS(%ps).\n",
298 firsttime ? "Detected" : "Continued", *bug_id, raw_smp_processor_id(), (s64)ts_delta,
299 cpu, csd->func, csd->info);
300 (*nmessages)++;
301 if (firsttime)
302 atomic_inc(&n_csd_lock_stuck);
303 /*
304 * If the CSD lock is still stuck after 5 minutes, it is unlikely
305 * to become unstuck. Use a signed comparison to avoid triggering
306 * on underflows when the TSC is out of sync between sockets.
307 */
308 BUG_ON(panic_on_ipistall > 0 && (s64)ts_delta > ((s64)panic_on_ipistall * NSEC_PER_MSEC));
309 if (cpu_cur_csd && csd != cpu_cur_csd) {
310 pr_alert("\tcsd: CSD lock (#%d) handling prior %pS(%ps) request.\n",
311 *bug_id, READ_ONCE(per_cpu(cur_csd_func, cpux)),
312 READ_ONCE(per_cpu(cur_csd_info, cpux)));
313 } else {
314 pr_alert("\tcsd: CSD lock (#%d) %s.\n",
315 *bug_id, !cpu_cur_csd ? "unresponsive" : "handling this request");
316 }
317 if (cpu >= 0) {
318 if (atomic_cmpxchg_acquire(&per_cpu(trigger_backtrace, cpu), 1, 0))
319 dump_cpu_task(cpu);
320 if (!cpu_cur_csd) {
321 pr_alert("csd: Re-sending CSD lock (#%d) IPI from CPU#%02d to CPU#%02d\n", *bug_id, raw_smp_processor_id(), cpu);
322 arch_send_call_function_single_ipi(cpu);
323 }
324 }
325 if (firsttime)
326 dump_stack();
327 *ts1 = ts2;
328
329 return false;
330}
331
332/*
333 * csd_lock/csd_unlock used to serialize access to per-cpu csd resources
334 *
335 * For non-synchronous ipi calls the csd can still be in use by the
336 * previous function call. For multi-cpu calls its even more interesting
337 * as we'll have to ensure no other cpu is observing our csd.
338 */
339static void __csd_lock_wait(call_single_data_t *csd)
340{
341 unsigned long nmessages = 0;
342 int bug_id = 0;
343 u64 ts0, ts1;
344
345 guard(preempt)();
346
347 ts1 = ts0 = ktime_get_mono_fast_ns();
348 for (;;) {
349 if (csd_lock_wait_toolong(csd, ts0, &ts1, &bug_id, &nmessages))
350 break;
351 cpu_relax();
352 }
353 smp_acquire__after_ctrl_dep();
354}
355
356static __always_inline void csd_lock_wait(call_single_data_t *csd)
357{
358 if (static_branch_unlikely(&csdlock_debug_enabled)) {
359 __csd_lock_wait(csd);
360 return;
361 }
362
363 smp_cond_load_acquire(&csd->node.u_flags, !(VAL & CSD_FLAG_LOCK));
364}
365#else
366static __always_inline void __csd_lock_wait(call_single_data_t *csd)
367{
368}
369
370static void csd_lock_record(call_single_data_t *csd)
371{
372}
373
374static __always_inline void csd_lock_wait(call_single_data_t *csd)
375{
376 smp_cond_load_acquire(&csd->node.u_flags, !(VAL & CSD_FLAG_LOCK));
377}
378#endif
379
380static __always_inline void csd_lock(call_single_data_t *csd)
381{
382 if (IS_ENABLED(CONFIG_CSD_LOCK_WAIT_DEBUG) &&
383 static_branch_unlikely(&csdlock_debug_enabled)) {
384
385 for (;;) {
386 unsigned int flags;
387
388 __csd_lock_wait(csd);
389 flags = READ_ONCE(csd->node.u_flags);
390
391 if (!(flags & CSD_FLAG_LOCK) &&
392 try_cmpxchg_acquire(&csd->node.u_flags, &flags, flags | CSD_FLAG_LOCK))
393 break;
394 }
395 } else {
396 csd_lock_wait(csd);
397 csd->node.u_flags |= CSD_FLAG_LOCK;
398 }
399
400 /*
401 * prevent CPU from reordering the above assignment
402 * to ->flags with any subsequent assignments to other
403 * fields of the specified call_single_data_t structure:
404 */
405 smp_wmb();
406}
407
408static __always_inline void csd_unlock(call_single_data_t *csd)
409{
410 WARN_ON(!(csd->node.u_flags & CSD_FLAG_LOCK));
411
412 /*
413 * ensure we're all done before releasing data:
414 */
415 smp_store_release(&csd->node.u_flags, 0);
416}
417
418static DEFINE_PER_CPU_SHARED_ALIGNED(call_single_data_t, csd_data);
419
420#ifdef CONFIG_CSD_LOCK_WAIT_DEBUG
421static call_single_data_t *get_single_csd_data(int cpu)
422{
423 if (static_branch_unlikely(&csdlock_debug_enabled) &&
424 (unsigned int)cpu < nr_cpu_ids)
425 return per_cpu_ptr(&csd_data, cpu);
426 return this_cpu_ptr(&csd_data);
427}
428#else
429static call_single_data_t *get_single_csd_data(int cpu)
430{
431 return this_cpu_ptr(&csd_data);
432}
433#endif
434
435void __smp_call_single_queue(int cpu, struct llist_node *node)
436{
437 /*
438 * We have to check the type of the CSD before queueing it, because
439 * once queued it can have its flags cleared by
440 * flush_smp_call_function_queue()
441 * even if we haven't sent the smp_call IPI yet (e.g. the stopper
442 * executes migration_cpu_stop() on the remote CPU).
443 */
444 if (trace_csd_queue_cpu_enabled()) {
445 call_single_data_t *csd;
446 smp_call_func_t func;
447
448 csd = container_of(node, call_single_data_t, node.llist);
449 func = CSD_TYPE(csd) == CSD_TYPE_TTWU ?
450 sched_ttwu_pending : csd->func;
451
452 trace_call__csd_queue_cpu(cpu, _RET_IP_, func, csd);
453 }
454
455 /*
456 * The list addition should be visible to the target CPU when it pops
457 * the head of the list to pull the entry off it in the IPI handler
458 * because of normal cache coherency rules implied by the underlying
459 * llist ops.
460 *
461 * If IPIs can go out of order to the cache coherency protocol
462 * in an architecture, sufficient synchronisation should be added
463 * to arch code to make it appear to obey cache coherency WRT
464 * locking and barrier primitives. Generic code isn't really
465 * equipped to do the right thing...
466 */
467 if (llist_add(node, &per_cpu(call_single_queue, cpu)))
468 send_call_function_single_ipi(cpu);
469}
470
471/*
472 * Insert a previously allocated call_single_data_t element
473 * for execution on the given CPU. data must already have
474 * ->func, ->info, and ->flags set.
475 */
476static int generic_exec_single(int cpu, call_single_data_t *csd)
477{
478 /*
479 * Preemption already disabled here so stopper cannot run on this CPU,
480 * ensuring mutually exclusive CPU offlining and last IPI flush.
481 */
482 if (cpu == smp_processor_id()) {
483 smp_call_func_t func = csd->func;
484 void *info = csd->info;
485 unsigned long flags;
486
487 /*
488 * We can unlock early even for the synchronous on-stack case,
489 * since we're doing this from the same CPU..
490 */
491 csd_lock_record(csd);
492 csd_unlock(csd);
493 local_irq_save(flags);
494 csd_do_func(func, info, NULL);
495 csd_lock_record(NULL);
496 local_irq_restore(flags);
497 return 0;
498 }
499
500 if ((unsigned)cpu >= nr_cpu_ids || !cpu_online(cpu)) {
501 csd_unlock(csd);
502 return -ENXIO;
503 }
504
505 __smp_call_single_queue(cpu, &csd->node.llist);
506
507 return 0;
508}
509
510/**
511 * generic_smp_call_function_single_interrupt - Execute SMP IPI callbacks
512 *
513 * Invoked by arch to handle an IPI for call function single.
514 * Must be called with interrupts disabled.
515 */
516void generic_smp_call_function_single_interrupt(void)
517{
518 __flush_smp_call_function_queue(true);
519}
520
521/**
522 * __flush_smp_call_function_queue - Flush pending smp-call-function callbacks
523 *
524 * @warn_cpu_offline: If set to 'true', warn if callbacks were queued on an
525 * offline CPU. Skip this check if set to 'false'.
526 *
527 * Flush any pending smp-call-function callbacks queued on this CPU. This is
528 * invoked by the generic IPI handler, as well as by a CPU about to go offline,
529 * to ensure that all pending IPI callbacks are run before it goes completely
530 * offline.
531 *
532 * Loop through the call_single_queue and run all the queued callbacks.
533 * Must be called with interrupts disabled.
534 */
535static void __flush_smp_call_function_queue(bool warn_cpu_offline)
536{
537 call_single_data_t *csd, *csd_next;
538 struct llist_node *entry, *prev;
539 struct llist_head *head;
540 static bool warned;
541 atomic_t *tbt;
542
543 lockdep_assert_irqs_disabled();
544
545 /* Allow waiters to send backtrace NMI from here onwards */
546 tbt = this_cpu_ptr(&trigger_backtrace);
547 atomic_set_release(tbt, 1);
548
549 head = this_cpu_ptr(&call_single_queue);
550 entry = llist_del_all(head);
551 entry = llist_reverse_order(entry);
552
553 /* There shouldn't be any pending callbacks on an offline CPU. */
554 if (unlikely(warn_cpu_offline && !cpu_online(smp_processor_id()) &&
555 !warned && entry != NULL)) {
556 warned = true;
557 WARN(1, "IPI on offline CPU %d\n", smp_processor_id());
558
559 /*
560 * We don't have to use the _safe() variant here
561 * because we are not invoking the IPI handlers yet.
562 */
563 llist_for_each_entry(csd, entry, node.llist) {
564 switch (CSD_TYPE(csd)) {
565 case CSD_TYPE_ASYNC:
566 case CSD_TYPE_SYNC:
567 case CSD_TYPE_IRQ_WORK:
568 pr_warn("IPI callback %pS sent to offline CPU\n",
569 csd->func);
570 break;
571
572 case CSD_TYPE_TTWU:
573 pr_warn("IPI task-wakeup sent to offline CPU\n");
574 break;
575
576 default:
577 pr_warn("IPI callback, unknown type %d, sent to offline CPU\n",
578 CSD_TYPE(csd));
579 break;
580 }
581 }
582 }
583
584 /*
585 * First; run all SYNC callbacks, people are waiting for us.
586 */
587 prev = NULL;
588 llist_for_each_entry_safe(csd, csd_next, entry, node.llist) {
589 /* Do we wait until *after* callback? */
590 if (CSD_TYPE(csd) == CSD_TYPE_SYNC) {
591 smp_call_func_t func = csd->func;
592 void *info = csd->info;
593
594 if (prev) {
595 prev->next = &csd_next->node.llist;
596 } else {
597 entry = &csd_next->node.llist;
598 }
599
600 csd_lock_record(csd);
601 csd_do_func(func, info, csd);
602 csd_unlock(csd);
603 csd_lock_record(NULL);
604 } else {
605 prev = &csd->node.llist;
606 }
607 }
608
609 if (!entry)
610 return;
611
612 /*
613 * Second; run all !SYNC callbacks.
614 */
615 prev = NULL;
616 llist_for_each_entry_safe(csd, csd_next, entry, node.llist) {
617 int type = CSD_TYPE(csd);
618
619 if (type != CSD_TYPE_TTWU) {
620 if (prev) {
621 prev->next = &csd_next->node.llist;
622 } else {
623 entry = &csd_next->node.llist;
624 }
625
626 if (type == CSD_TYPE_ASYNC) {
627 smp_call_func_t func = csd->func;
628 void *info = csd->info;
629
630 csd_lock_record(csd);
631 csd_unlock(csd);
632 csd_do_func(func, info, csd);
633 csd_lock_record(NULL);
634 } else if (type == CSD_TYPE_IRQ_WORK) {
635 irq_work_single(csd);
636 }
637
638 } else {
639 prev = &csd->node.llist;
640 }
641 }
642
643 /*
644 * Third; only CSD_TYPE_TTWU is left, issue those.
645 */
646 if (entry) {
647 csd = llist_entry(entry, typeof(*csd), node.llist);
648 csd_do_func(sched_ttwu_pending, entry, csd);
649 }
650}
651
652
653/**
654 * flush_smp_call_function_queue - Flush pending smp-call-function callbacks
655 * from task context (idle, migration thread)
656 *
657 * When TIF_POLLING_NRFLAG is supported and a CPU is in idle and has it
658 * set, then remote CPUs can avoid sending IPIs and wake the idle CPU by
659 * setting TIF_NEED_RESCHED. The idle task on the woken up CPU has to
660 * handle queued SMP function calls before scheduling.
661 *
662 * The migration thread has to ensure that an eventually pending wakeup has
663 * been handled before it migrates a task.
664 */
665void flush_smp_call_function_queue(void)
666{
667 unsigned int was_pending;
668 unsigned long flags;
669
670 if (llist_empty(this_cpu_ptr(&call_single_queue)))
671 return;
672
673 local_irq_save(flags);
674 /* Get the already pending soft interrupts for RT enabled kernels */
675 was_pending = local_softirq_pending();
676 __flush_smp_call_function_queue(true);
677 if (local_softirq_pending())
678 do_softirq_post_smp_call_flush(was_pending);
679
680 local_irq_restore(flags);
681}
682
683static int __smp_call_function_single(int cpu, smp_call_func_t func,
684 void *info, const struct cpumask *mask,
685 bool wait)
686{
687 call_single_data_t *csd;
688 call_single_data_t csd_stack = {
689 .node = { .u_flags = CSD_FLAG_LOCK | CSD_TYPE_SYNC, },
690 };
691 int this_cpu;
692 int err;
693
694 /*
695 * Prevent preemption and reschedule on another CPU, as well as CPU
696 * removal. This prevents stopper from running on this CPU, thus
697 * providing mutual exclusion of the below cpu_online() check and
698 * IPI sending ensuring IPI are not missed by CPU going offline.
699 */
700 this_cpu = get_cpu();
701
702 if (mask) {
703 /* Try for same CPU (cheapest) */
704 if (!cpumask_test_cpu(this_cpu, mask))
705 cpu = sched_numa_find_nth_cpu(mask, 0, cpu_to_node(this_cpu));
706 else
707 cpu = this_cpu;
708 }
709
710 /*
711 * Can deadlock when called with interrupts disabled.
712 * We allow cpu's that are not yet online though, as no one else can
713 * send smp call function interrupt to this cpu and as such deadlocks
714 * can't happen.
715 */
716 WARN_ON_ONCE(cpu_online(this_cpu) && irqs_disabled()
717 && !oops_in_progress);
718
719 /*
720 * When @wait we can deadlock when we interrupt between llist_add() and
721 * arch_send_call_function_ipi*(); when !@wait we can deadlock due to
722 * csd_lock() on because the interrupt context uses the same csd
723 * storage.
724 */
725 WARN_ON_ONCE(!in_task());
726
727 csd = &csd_stack;
728 if (!wait) {
729 csd = get_single_csd_data(cpu);
730 csd_lock(csd);
731 }
732
733 csd->func = func;
734 csd->info = info;
735#ifdef CONFIG_CSD_LOCK_WAIT_DEBUG
736 csd->node.src = this_cpu;
737 csd->node.dst = cpu;
738#endif
739
740 err = generic_exec_single(cpu, csd);
741
742 /*
743 * @csd is stack-allocated when @wait is true. No concurrent access
744 * except from the IPI completion path, so we can re-enable preemption
745 * early to reduce latency.
746 */
747 put_cpu();
748
749 if (wait)
750 csd_lock_wait(csd);
751
752 return err;
753}
754
755/**
756 * smp_call_function_single - Run a function on a specific CPU
757 * @cpu: Specific target CPU for this function.
758 * @func: The function to run. This must be fast and non-blocking.
759 * @info: An arbitrary pointer to pass to the function.
760 * @wait: If true, wait until function has completed on other CPUs.
761 *
762 * Returns: %0 on success, else a negative status code.
763 */
764int smp_call_function_single(int cpu, smp_call_func_t func, void *info, bool wait)
765{
766 return __smp_call_function_single(cpu, func, info, NULL, wait);
767}
768EXPORT_SYMBOL(smp_call_function_single);
769
770/**
771 * smp_call_function_single_async() - Run an asynchronous function on a
772 * specific CPU.
773 * @cpu: The CPU to run on.
774 * @csd: Pre-allocated and setup data structure
775 *
776 * Like smp_call_function_single(), but the call is asynchonous and
777 * can thus be done from contexts with disabled interrupts.
778 *
779 * The caller passes his own pre-allocated data structure
780 * (ie: embedded in an object) and is responsible for synchronizing it
781 * such that the IPIs performed on the @csd are strictly serialized.
782 *
783 * If the function is called with one csd which has not yet been
784 * processed by previous call to smp_call_function_single_async(), the
785 * function will return immediately with -EBUSY showing that the csd
786 * object is still in progress.
787 *
788 * NOTE: Be careful, there is unfortunately no current debugging facility to
789 * validate the correctness of this serialization.
790 *
791 * Return: %0 on success or negative errno value on error
792 */
793int smp_call_function_single_async(int cpu, call_single_data_t *csd)
794{
795 int err = 0;
796
797 preempt_disable();
798
799 if (csd->node.u_flags & CSD_FLAG_LOCK) {
800 err = -EBUSY;
801 goto out;
802 }
803
804 csd->node.u_flags = CSD_FLAG_LOCK;
805 smp_wmb();
806
807 err = generic_exec_single(cpu, csd);
808
809out:
810 preempt_enable();
811
812 return err;
813}
814EXPORT_SYMBOL_GPL(smp_call_function_single_async);
815
816/**
817 * smp_call_function_any - Run a function on any of the given cpus
818 * @mask: The mask of cpus it can run on.
819 * @func: The function to run. This must be fast and non-blocking.
820 * @info: An arbitrary pointer to pass to the function.
821 * @wait: If true, wait until function has completed.
822 *
823 * Selection preference:
824 * 1) current cpu if in @mask
825 * 2) nearest cpu in @mask, based on NUMA topology
826 *
827 * Returns: %0 on success, else a negative status code (if no cpus were online).
828 */
829int smp_call_function_any(const struct cpumask *mask,
830 smp_call_func_t func, void *info, int wait)
831{
832 return __smp_call_function_single(-1, func, info, mask, wait);
833}
834EXPORT_SYMBOL_GPL(smp_call_function_any);
835
836static DEFINE_STATIC_KEY_FALSE(ipi_mask_inlined);
837
838#ifdef CONFIG_PREEMPTION
839
840int smp_task_ipi_mask_alloc(struct task_struct *task)
841{
842 if (static_branch_unlikely(&ipi_mask_inlined))
843 return 0;
844
845 ACCESS_PRIVATE(task, ipi_mask).ipi_mask_ptr =
846 kmalloc(cpumask_size(), GFP_KERNEL);
847 if (!ACCESS_PRIVATE(task, ipi_mask).ipi_mask_ptr)
848 return -ENOMEM;
849
850 return 0;
851}
852
853void smp_task_ipi_mask_free(struct task_struct *task)
854{
855 if (static_branch_unlikely(&ipi_mask_inlined))
856 return;
857
858 kfree(ACCESS_PRIVATE(task, ipi_mask).ipi_mask_ptr);
859}
860
861static cpumask_t *smp_task_ipi_mask(struct task_struct *cur)
862{
863 /*
864 * If cpumask_size() is smaller than or equal to the pointer
865 * size, it stashes the cpumask in the pointer itself to
866 * avoid extra memory allocations.
867 */
868 if (static_branch_unlikely(&ipi_mask_inlined))
869 return (cpumask_t *)&ACCESS_PRIVATE(cur, ipi_mask).ipi_mask_val;
870
871 return ACCESS_PRIVATE(cur, ipi_mask).ipi_mask_ptr;
872}
873#else
874static cpumask_t *smp_task_ipi_mask(struct task_struct *cur)
875{
876 return NULL;
877}
878#endif
879
880/*
881 * Flags to be used as scf_flags argument of smp_call_function_many_cond().
882 *
883 * %SCF_WAIT: Wait until function execution is completed
884 * %SCF_RUN_LOCAL: Run also locally if local cpu is set in cpumask
885 */
886#define SCF_WAIT (1U << 0)
887#define SCF_RUN_LOCAL (1U << 1)
888
889static void smp_call_function_many_cond(const struct cpumask *mask,
890 smp_call_func_t func, void *info,
891 unsigned int scf_flags,
892 smp_cond_func_t cond_func)
893{
894 struct cpumask *cpumask, *task_mask;
895 bool wait = scf_flags & SCF_WAIT;
896 struct call_function_data *cfd;
897 int cpu, last_cpu, this_cpu;
898 bool run_remote = false;
899 int nr_cpus = 0;
900
901 this_cpu = get_cpu();
902 cfd = this_cpu_ptr(&cfd_data);
903 task_mask = smp_task_ipi_mask(current);
904 if (task_mask)
905 cpumask = task_mask;
906 else
907 cpumask = cfd->cpumask;
908
909 /*
910 * Can deadlock when called with interrupts disabled.
911 * We allow cpu's that are not yet online though, as no one else can
912 * send smp call function interrupt to this cpu and as such deadlocks
913 * can't happen.
914 */
915 if (cpu_online(this_cpu) && !oops_in_progress &&
916 !early_boot_irqs_disabled)
917 lockdep_assert_irqs_enabled();
918
919 /*
920 * When @wait we can deadlock when we interrupt between llist_add() and
921 * arch_send_call_function_ipi*(); when !@wait we can deadlock due to
922 * csd_lock() on because the interrupt context uses the same csd
923 * storage.
924 */
925 WARN_ON_ONCE(!in_task());
926
927 /* Check if we need remote execution, i.e., any CPU excluding this one. */
928 if (cpumask_any_and_but(mask, cpu_online_mask, this_cpu) < nr_cpu_ids) {
929 cpumask_and(cpumask, mask, cpu_online_mask);
930 __cpumask_clear_cpu(this_cpu, cpumask);
931
932 cpumask_clear(cfd->cpumask_ipi);
933 for_each_cpu(cpu, cpumask) {
934 call_single_data_t *csd = per_cpu_ptr(cfd->csd, cpu);
935
936 if (cond_func && !cond_func(cpu, info)) {
937 __cpumask_clear_cpu(cpu, cpumask);
938 continue;
939 }
940
941 /* Work is enqueued on a remote CPU. */
942 run_remote = true;
943
944 csd_lock(csd);
945 if (wait)
946 csd->node.u_flags |= CSD_TYPE_SYNC;
947 csd->func = func;
948 csd->info = info;
949#ifdef CONFIG_CSD_LOCK_WAIT_DEBUG
950 csd->node.src = this_cpu;
951 csd->node.dst = cpu;
952#endif
953 trace_csd_queue_cpu(cpu, _RET_IP_, func, csd);
954
955 /*
956 * Kick the remote CPU if this is the first work
957 * item enqueued.
958 */
959 if (llist_add(&csd->node.llist, &per_cpu(call_single_queue, cpu))) {
960 __cpumask_set_cpu(cpu, cfd->cpumask_ipi);
961 nr_cpus++;
962 last_cpu = cpu;
963 }
964 }
965
966 /*
967 * Choose the most efficient way to send an IPI. Note that the
968 * number of CPUs might be zero due to concurrent changes to the
969 * provided mask.
970 */
971 if (nr_cpus == 1)
972 send_call_function_single_ipi(last_cpu);
973 else if (likely(nr_cpus > 1))
974 send_call_function_ipi_mask(cfd->cpumask_ipi);
975 }
976
977 /* Check if we need local execution. */
978 if ((scf_flags & SCF_RUN_LOCAL) && cpumask_test_cpu(this_cpu, mask) &&
979 (!cond_func || cond_func(this_cpu, info))) {
980 unsigned long flags;
981
982 local_irq_save(flags);
983 csd_do_func(func, info, NULL);
984 local_irq_restore(flags);
985 }
986
987 /*
988 * The IPI work has been queued and dispatched. On PREEMPT kernels,
989 * tasks created through dup_task_struct() have task-local wait masks.
990 * The boot init_task can fall back to cfd->cpumask when the mask is
991 * not inlined, but other tasks still use task-local masks and cannot
992 * overwrite it. On !PREEMPT kernels, preempt_enable() cannot schedule
993 * another task, so the per-CPU mask remains protected.
994 */
995 put_cpu();
996
997 if (run_remote && wait) {
998 for_each_cpu(cpu, cpumask) {
999 call_single_data_t *csd;
1000
1001 csd = per_cpu_ptr(cfd->csd, cpu);
1002 csd_lock_wait(csd);
1003 }
1004 }
1005}
1006
1007/**
1008 * smp_call_function_many() - Run a function on a set of CPUs.
1009 * @mask: The set of cpus to run on (only runs on online subset).
1010 * @func: The function to run. This must be fast and non-blocking.
1011 * @info: An arbitrary pointer to pass to the function.
1012 * @wait: If true, wait (atomically) until function has completed
1013 * on other CPUs.
1014 *
1015 * You must not call this function with disabled interrupts or from a
1016 * hardware interrupt handler or from a bottom half handler.
1017 *
1018 * @func is not called on the local CPU even if @mask contains it. Consider
1019 * using on_each_cpu_cond_mask() instead if this is not desirable.
1020 */
1021void smp_call_function_many(const struct cpumask *mask,
1022 smp_call_func_t func, void *info, bool wait)
1023{
1024 smp_call_function_many_cond(mask, func, info, wait * SCF_WAIT, NULL);
1025}
1026EXPORT_SYMBOL(smp_call_function_many);
1027
1028/**
1029 * smp_call_function() - Run a function on all other CPUs.
1030 * @func: The function to run. This must be fast and non-blocking.
1031 * @info: An arbitrary pointer to pass to the function.
1032 * @wait: If true, wait (atomically) until function has completed
1033 * on other CPUs.
1034 *
1035 * If @wait is true, then returns once @func has returned; otherwise
1036 * it returns just before the target cpu calls @func.
1037 *
1038 * You must not call this function with disabled interrupts or from a
1039 * hardware interrupt handler or from a bottom half handler.
1040 */
1041void smp_call_function(smp_call_func_t func, void *info, int wait)
1042{
1043 smp_call_function_many_cond(cpu_online_mask, func, info,
1044 wait ? SCF_WAIT : 0, NULL);
1045}
1046EXPORT_SYMBOL(smp_call_function);
1047
1048/* Setup configured maximum number of CPUs to activate */
1049unsigned int setup_max_cpus = NR_CPUS;
1050EXPORT_SYMBOL(setup_max_cpus);
1051
1052
1053/*
1054 * Setup routine for controlling SMP activation
1055 *
1056 * Command-line option of "nosmp" or "maxcpus=0" will disable SMP
1057 * activation entirely (the MPS table probe still happens, though).
1058 *
1059 * Command-line option of "maxcpus=<NUM>", where <NUM> is an integer
1060 * greater than 0, limits the maximum number of CPUs activated in
1061 * SMP mode to <NUM>.
1062 */
1063
1064void __weak __init arch_disable_smp_support(void) { }
1065
1066static int __init nosmp(char *str)
1067{
1068 setup_max_cpus = 0;
1069 arch_disable_smp_support();
1070
1071 return 0;
1072}
1073
1074early_param("nosmp", nosmp);
1075
1076/* this is hard limit */
1077static int __init nrcpus(char *str)
1078{
1079 int nr_cpus;
1080
1081 if (get_option(&str, &nr_cpus) && nr_cpus > 0 && nr_cpus < nr_cpu_ids)
1082 set_nr_cpu_ids(nr_cpus);
1083
1084 return 0;
1085}
1086
1087early_param("nr_cpus", nrcpus);
1088
1089static int __init maxcpus(char *str)
1090{
1091 get_option(&str, &setup_max_cpus);
1092 if (setup_max_cpus == 0)
1093 arch_disable_smp_support();
1094
1095 return 0;
1096}
1097
1098early_param("maxcpus", maxcpus);
1099
1100#if (NR_CPUS > 1) && !defined(CONFIG_FORCE_NR_CPUS)
1101/* Setup number of possible processor ids */
1102unsigned int nr_cpu_ids __read_mostly = NR_CPUS;
1103EXPORT_SYMBOL(nr_cpu_ids);
1104#endif
1105
1106/* An arch may set nr_cpu_ids earlier if needed, so this would be redundant */
1107void __init setup_nr_cpu_ids(void)
1108{
1109 set_nr_cpu_ids(find_last_bit(cpumask_bits(cpu_possible_mask), NR_CPUS) + 1);
1110
1111 if (IS_ENABLED(CONFIG_PREEMPTION) && cpumask_size() <= sizeof(unsigned long))
1112 static_branch_enable(&ipi_mask_inlined);
1113}
1114
1115/* Called by boot processor to activate the rest. */
1116void __init smp_init(void)
1117{
1118 int num_nodes, num_cpus;
1119
1120 idle_threads_init();
1121 cpuhp_threads_init();
1122
1123 pr_info("Bringing up secondary CPUs ...\n");
1124
1125 bringup_nonboot_cpus(setup_max_cpus);
1126
1127 num_nodes = num_online_nodes();
1128 num_cpus = num_online_cpus();
1129 pr_info("Brought up %d node%s, %d CPU%s\n",
1130 num_nodes, str_plural(num_nodes), num_cpus, str_plural(num_cpus));
1131
1132 /* Any cleanup work */
1133 smp_cpus_done(setup_max_cpus);
1134}
1135
1136/**
1137 * on_each_cpu_cond_mask() - Call a function on each processor for which
1138 * the supplied function cond_func returns true, optionally waiting
1139 * for all the required CPUs to finish. This may include the local
1140 * processor.
1141 * @cond_func: A callback function that is passed a cpu id and
1142 * the info parameter. The function is called
1143 * with preemption disabled. The function should
1144 * return a boolean value indicating whether to IPI
1145 * the specified CPU.
1146 * @func: The function to run on all applicable CPUs.
1147 * This must be fast and non-blocking.
1148 * @info: An arbitrary pointer to pass to both functions.
1149 * @wait: If true, wait until function has completed on other CPUs.
1150 * @mask: The set of cpus to run on (only runs on online subset).
1151 *
1152 * Target CPU selection and work queueing are done with preemption
1153 * disabled. This protects against CPUs going offline, but not against
1154 * CPUs coming online concurrently; newly online CPUs are not guaranteed
1155 * to be seen or sent an IPI. If @wait is true, the final wait for remote
1156 * completion happens after that preemption-disabled section.
1157 *
1158 * You must not call this function with disabled interrupts or
1159 * from a hardware interrupt handler or from a bottom half handler.
1160 */
1161void on_each_cpu_cond_mask(smp_cond_func_t cond_func, smp_call_func_t func,
1162 void *info, bool wait, const struct cpumask *mask)
1163{
1164 unsigned int scf_flags = SCF_RUN_LOCAL;
1165
1166 if (wait)
1167 scf_flags |= SCF_WAIT;
1168
1169 smp_call_function_many_cond(mask, func, info, scf_flags, cond_func);
1170}
1171EXPORT_SYMBOL(on_each_cpu_cond_mask);
1172
1173static void do_nothing(void *unused)
1174{
1175}
1176
1177/**
1178 * kick_all_cpus_sync - Force all cpus out of idle
1179 *
1180 * Used to synchronize the update of pm_idle function pointer. It's
1181 * called after the pointer is updated and returns after the dummy
1182 * callback function has been executed on all cpus. The execution of
1183 * the function can only happen on the remote cpus after they have
1184 * left the idle function which had been called via pm_idle function
1185 * pointer. So it's guaranteed that nothing uses the previous pointer
1186 * anymore.
1187 */
1188void kick_all_cpus_sync(void)
1189{
1190 /* Make sure the change is visible before we kick the cpus */
1191 smp_mb();
1192 smp_call_function(do_nothing, NULL, 1);
1193}
1194EXPORT_SYMBOL_GPL(kick_all_cpus_sync);
1195
1196/**
1197 * wake_up_all_idle_cpus - break all cpus out of idle
1198 * wake_up_all_idle_cpus try to break all cpus which is in idle state even
1199 * including idle polling cpus, for non-idle cpus, we will do nothing
1200 * for them.
1201 */
1202void wake_up_all_idle_cpus(void)
1203{
1204 int cpu;
1205
1206 for_each_possible_cpu(cpu) {
1207 preempt_disable();
1208 if (cpu != smp_processor_id() && cpu_online(cpu))
1209 wake_up_if_idle(cpu);
1210 preempt_enable();
1211 }
1212}
1213EXPORT_SYMBOL_GPL(wake_up_all_idle_cpus);
1214
1215/**
1216 * cpus_peek_for_pending_ipi - Check for pending IPI for CPUs
1217 * @mask: The CPU mask for the CPUs to check.
1218 *
1219 * This function walks through the @mask to check if there are any pending IPIs
1220 * scheduled, for any of the CPUs in the @mask. It does not guarantee
1221 * correctness as it only provides a racy snapshot.
1222 *
1223 * Returns: true if there is a pending IPI scheduled and false otherwise.
1224 */
1225bool cpus_peek_for_pending_ipi(const struct cpumask *mask)
1226{
1227 unsigned int cpu;
1228
1229 for_each_cpu(cpu, mask) {
1230 if (!llist_empty(per_cpu_ptr(&call_single_queue, cpu)))
1231 return true;
1232 }
1233
1234 return false;
1235}
1236
1237/**
1238 * struct smp_call_on_cpu_struct - Call a function on a specific CPU
1239 * @work: &work_struct
1240 * @done: &completion to signal
1241 * @func: function to call
1242 * @data: function's data argument
1243 * @ret: return value from @func
1244 * @cpu: target CPU (%-1 for any CPU)
1245 *
1246 * Used to call a function on a specific cpu and wait for it to return.
1247 * Optionally make sure the call is done on a specified physical cpu via vcpu
1248 * pinning in order to support virtualized environments.
1249 */
1250struct smp_call_on_cpu_struct {
1251 struct work_struct work;
1252 struct completion done;
1253 int (*func)(void *);
1254 void *data;
1255 int ret;
1256 int cpu;
1257};
1258
1259static void smp_call_on_cpu_callback(struct work_struct *work)
1260{
1261 struct smp_call_on_cpu_struct *sscs;
1262
1263 sscs = container_of(work, struct smp_call_on_cpu_struct, work);
1264 if (sscs->cpu >= 0)
1265 hypervisor_pin_vcpu(sscs->cpu);
1266 sscs->ret = sscs->func(sscs->data);
1267 if (sscs->cpu >= 0)
1268 hypervisor_pin_vcpu(-1);
1269
1270 complete(&sscs->done);
1271}
1272
1273/**
1274 * smp_call_on_cpu() - Call a function on a specific CPU and wait
1275 * for it to return.
1276 * @cpu: The CPU to run on.
1277 * @func: The function to run
1278 * @par: An arbitrary pointer parameter for @func.
1279 * @phys: If @true, force to run on physical @cpu. See
1280 * &struct smp_call_on_cpu_struct for more info.
1281 *
1282 * Returns: %-ENXIO if the @cpu is invalid; otherwise the return value
1283 * from @func.
1284 */
1285int smp_call_on_cpu(unsigned int cpu, int (*func)(void *), void *par, bool phys)
1286{
1287 struct smp_call_on_cpu_struct sscs = {
1288 .done = COMPLETION_INITIALIZER_ONSTACK(sscs.done),
1289 .func = func,
1290 .data = par,
1291 .cpu = phys ? cpu : -1,
1292 };
1293
1294 INIT_WORK_ONSTACK(&sscs.work, smp_call_on_cpu_callback);
1295
1296 if (cpu >= nr_cpu_ids || !cpu_online(cpu))
1297 return -ENXIO;
1298
1299 queue_work_on(cpu, system_percpu_wq, &sscs.work);
1300 wait_for_completion(&sscs.done);
1301 destroy_work_on_stack(&sscs.work);
1302
1303 return sscs.ret;
1304}
1305EXPORT_SYMBOL_GPL(smp_call_on_cpu);