projects/linux·blob v7.2-rc6 · refs

1214 lines · 33.3 KiB

1// SPDX-License-Identifier: GPL-2.0-only
2/*
3 * Generic helpers for smp ipi calls
4 *
5 * (C) Jens Axboe <[email protected]> 2008
6 */
7
8#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
9
10#include <linux/irq_work.h>
11#include <linux/rcupdate.h>
12#include <linux/rculist.h>
13#include <linux/kernel.h>
14#include <linux/export.h>
15#include <linux/percpu.h>
16#include <linux/init.h>
17#include <linux/interrupt.h>
18#include <linux/gfp.h>
19#include <linux/smp.h>
20#include <linux/cpu.h>
21#include <linux/sched.h>
22#include <linux/sched/idle.h>
23#include <linux/hypervisor.h>
24#include <linux/sched/clock.h>
25#include <linux/nmi.h>
26#include <linux/sched/debug.h>
27#include <linux/jump_label.h>
28#include <linux/string_choices.h>
29
30#include <trace/events/ipi.h>
31#define CREATE_TRACE_POINTS
32#include <trace/events/csd.h>
33#undef CREATE_TRACE_POINTS
34
35#include "smpboot.h"
36#include "sched/smp.h"
37
38#define CSD_TYPE(_csd) ((_csd)->node.u_flags & CSD_FLAG_TYPE_MASK)
39
40struct call_function_data {
41 call_single_data_t __percpu *csd;
42 cpumask_var_t cpumask;
43 cpumask_var_t cpumask_ipi;
44};
45
46static DEFINE_PER_CPU_ALIGNED(struct call_function_data, cfd_data);
47
48static DEFINE_PER_CPU_SHARED_ALIGNED(struct llist_head, call_single_queue);
49
50static DEFINE_PER_CPU(atomic_t, trigger_backtrace) = ATOMIC_INIT(1);
51
52static void __flush_smp_call_function_queue(bool warn_cpu_offline);
53
54int smpcfd_prepare_cpu(unsigned int cpu)
55{
56 struct call_function_data *cfd = &per_cpu(cfd_data, cpu);
57
58 if (!zalloc_cpumask_var_node(&cfd->cpumask, GFP_KERNEL,
59 cpu_to_node(cpu)))
60 return -ENOMEM;
61 if (!zalloc_cpumask_var_node(&cfd->cpumask_ipi, GFP_KERNEL,
62 cpu_to_node(cpu))) {
63 free_cpumask_var(cfd->cpumask);
64 return -ENOMEM;
65 }
66 cfd->csd = alloc_percpu(call_single_data_t);
67 if (!cfd->csd) {
68 free_cpumask_var(cfd->cpumask);
69 free_cpumask_var(cfd->cpumask_ipi);
70 return -ENOMEM;
71 }
72
73 return 0;
74}
75
76int smpcfd_dead_cpu(unsigned int cpu)
77{
78 struct call_function_data *cfd = &per_cpu(cfd_data, cpu);
79
80 free_cpumask_var(cfd->cpumask);
81 free_cpumask_var(cfd->cpumask_ipi);
82 free_percpu(cfd->csd);
83 return 0;
84}
85
86int smpcfd_dying_cpu(unsigned int cpu)
87{
88 /*
89 * The IPIs for the smp-call-function callbacks queued by other CPUs
90 * might arrive late, either due to hardware latencies or because this
91 * CPU disabled interrupts (inside stop-machine) before the IPIs were
92 * sent. So flush out any pending callbacks explicitly (without waiting
93 * for the IPIs to arrive), to ensure that the outgoing CPU doesn't go
94 * offline with work still pending.
95 *
96 * This runs with interrupts disabled inside the stopper task invoked by
97 * stop_machine(), ensuring mutually exclusive CPU offlining and IPI flush.
98 */
99 __flush_smp_call_function_queue(false);
100 irq_work_run();
101 return 0;
102}
103
104void __init call_function_init(void)
105{
106 int i;
107
108 for_each_possible_cpu(i)
109 init_llist_head(&per_cpu(call_single_queue, i));
110
111 smpcfd_prepare_cpu(smp_processor_id());
112}
113
114static __always_inline void
115send_call_function_single_ipi(int cpu)
116{
117 if (call_function_single_prep_ipi(cpu)) {
118 trace_ipi_send_cpu(cpu, _RET_IP_,
119 generic_smp_call_function_single_interrupt);
120 arch_send_call_function_single_ipi(cpu);
121 }
122}
123
124static __always_inline void
125send_call_function_ipi_mask(struct cpumask *mask)
126{
127 trace_ipi_send_cpumask(mask, _RET_IP_,
128 generic_smp_call_function_single_interrupt);
129 arch_send_call_function_ipi_mask(mask);
130}
131
132static __always_inline void
133csd_do_func(smp_call_func_t func, void *info, call_single_data_t *csd)
134{
135 trace_csd_function_entry(func, csd);
136 func(info);
137 trace_csd_function_exit(func, csd);
138}
139
140static DEFINE_STATIC_KEY_MAYBE(CONFIG_CSD_LOCK_WAIT_DEBUG_DEFAULT, csdlock_debug_enabled);
141
142#ifdef CONFIG_CSD_LOCK_WAIT_DEBUG
143
144/*
145 * Parse the csdlock_debug= kernel boot parameter.
146 *
147 * If you need to restore the old "ext" value that once provided
148 * additional debugging information, reapply the following commits:
149 *
150 * de7b09ef658d ("locking/csd_lock: Prepare more CSD lock debugging")
151 * a5aabace5fb8 ("locking/csd_lock: Add more data to CSD lock debugging")
152 */
153static int __init csdlock_debug(char *str)
154{
155 int ret;
156 unsigned int val = 0;
157
158 ret = get_option(&str, &val);
159 if (ret) {
160 if (val)
161 static_branch_enable(&csdlock_debug_enabled);
162 else
163 static_branch_disable(&csdlock_debug_enabled);
164 }
165
166 return 1;
167}
168__setup("csdlock_debug=", csdlock_debug);
169
170static DEFINE_PER_CPU(call_single_data_t *, cur_csd);
171static DEFINE_PER_CPU(smp_call_func_t, cur_csd_func);
172static DEFINE_PER_CPU(void *, cur_csd_info);
173
174static ulong csd_lock_timeout = 5000; /* CSD lock timeout in milliseconds. */
175module_param(csd_lock_timeout, ulong, 0644);
176static int panic_on_ipistall; /* CSD panic timeout in milliseconds, 300000 for five minutes. */
177module_param(panic_on_ipistall, int, 0644);
178
179static atomic_t csd_bug_count = ATOMIC_INIT(0);
180
181/* Record current CSD work for current CPU, NULL to erase. */
182static void __csd_lock_record(call_single_data_t *csd)
183{
184 if (!csd) {
185 smp_mb(); /* NULL cur_csd after unlock. */
186 __this_cpu_write(cur_csd, NULL);
187 return;
188 }
189 __this_cpu_write(cur_csd_func, csd->func);
190 __this_cpu_write(cur_csd_info, csd->info);
191 smp_wmb(); /* func and info before csd. */
192 __this_cpu_write(cur_csd, csd);
193 smp_mb(); /* Update cur_csd before function call. */
194 /* Or before unlock, as the case may be. */
195}
196
197static __always_inline void csd_lock_record(call_single_data_t *csd)
198{
199 if (static_branch_unlikely(&csdlock_debug_enabled))
200 __csd_lock_record(csd);
201}
202
203static int csd_lock_wait_getcpu(call_single_data_t *csd)
204{
205 unsigned int csd_type;
206
207 csd_type = CSD_TYPE(csd);
208 if (csd_type == CSD_TYPE_ASYNC || csd_type == CSD_TYPE_SYNC)
209 return csd->node.dst; /* Other CSD_TYPE_ values might not have ->dst. */
210 return -1;
211}
212
213static atomic_t n_csd_lock_stuck;
214
215/**
216 * csd_lock_is_stuck - Has a CSD-lock acquisition been stuck too long?
217 *
218 * Returns: @true if a CSD-lock acquisition is stuck and has been stuck
219 * long enough for a "non-responsive CSD lock" message to be printed.
220 */
221bool csd_lock_is_stuck(void)
222{
223 return !!atomic_read(&n_csd_lock_stuck);
224}
225
226/*
227 * Complain if too much time spent waiting. Note that only
228 * the CSD_TYPE_SYNC/ASYNC types provide the destination CPU,
229 * so waiting on other types gets much less information.
230 */
231static bool csd_lock_wait_toolong(call_single_data_t *csd, u64 ts0, u64 *ts1, int *bug_id, unsigned long *nmessages)
232{
233 int cpu = -1;
234 int cpux;
235 bool firsttime;
236 u64 ts2, ts_delta;
237 call_single_data_t *cpu_cur_csd;
238 unsigned int flags = READ_ONCE(csd->node.u_flags);
239 unsigned long long csd_lock_timeout_ns = csd_lock_timeout * NSEC_PER_MSEC;
240
241 if (!(flags & CSD_FLAG_LOCK)) {
242 if (!unlikely(*bug_id))
243 return true;
244 cpu = csd_lock_wait_getcpu(csd);
245 pr_alert("csd: CSD lock (#%d) got unstuck on CPU#%02d, CPU#%02d released the lock.\n",
246 *bug_id, raw_smp_processor_id(), cpu);
247 atomic_dec(&n_csd_lock_stuck);
248 return true;
249 }
250
251 ts2 = ktime_get_mono_fast_ns();
252 /* How long since we last checked for a stuck CSD lock.*/
253 ts_delta = ts2 - *ts1;
254 if (likely(ts_delta <= csd_lock_timeout_ns * (*nmessages + 1) *
255 (!*nmessages ? 1 : (ilog2(num_online_cpus()) / 2 + 1)) ||
256 csd_lock_timeout_ns == 0))
257 return false;
258
259 if (ts0 > ts2) {
260 /* Our own sched_clock went backward; don't blame another CPU. */
261 ts_delta = ts0 - ts2;
262 pr_alert("sched_clock on CPU %d went backward by %llu ns\n", raw_smp_processor_id(), ts_delta);
263 *ts1 = ts2;
264 return false;
265 }
266
267 firsttime = !*bug_id;
268 if (firsttime)
269 *bug_id = atomic_inc_return(&csd_bug_count);
270 cpu = csd_lock_wait_getcpu(csd);
271 if (WARN_ONCE(cpu < 0 || cpu >= nr_cpu_ids, "%s: cpu = %d\n", __func__, cpu))
272 cpux = 0;
273 else
274 cpux = cpu;
275 cpu_cur_csd = smp_load_acquire(&per_cpu(cur_csd, cpux)); /* Before func and info. */
276 /* How long since this CSD lock was stuck. */
277 ts_delta = ts2 - ts0;
278 pr_alert("csd: %s non-responsive CSD lock (#%d) on CPU#%d, waiting %lld ns for CPU#%02d %pS(%ps).\n",
279 firsttime ? "Detected" : "Continued", *bug_id, raw_smp_processor_id(), (s64)ts_delta,
280 cpu, csd->func, csd->info);
281 (*nmessages)++;
282 if (firsttime)
283 atomic_inc(&n_csd_lock_stuck);
284 /*
285 * If the CSD lock is still stuck after 5 minutes, it is unlikely
286 * to become unstuck. Use a signed comparison to avoid triggering
287 * on underflows when the TSC is out of sync between sockets.
288 */
289 BUG_ON(panic_on_ipistall > 0 && (s64)ts_delta > ((s64)panic_on_ipistall * NSEC_PER_MSEC));
290 if (cpu_cur_csd && csd != cpu_cur_csd) {
291 pr_alert("\tcsd: CSD lock (#%d) handling prior %pS(%ps) request.\n",
292 *bug_id, READ_ONCE(per_cpu(cur_csd_func, cpux)),
293 READ_ONCE(per_cpu(cur_csd_info, cpux)));
294 } else {
295 pr_alert("\tcsd: CSD lock (#%d) %s.\n",
296 *bug_id, !cpu_cur_csd ? "unresponsive" : "handling this request");
297 }
298 if (cpu >= 0) {
299 if (atomic_cmpxchg_acquire(&per_cpu(trigger_backtrace, cpu), 1, 0))
300 dump_cpu_task(cpu);
301 if (!cpu_cur_csd) {
302 pr_alert("csd: Re-sending CSD lock (#%d) IPI from CPU#%02d to CPU#%02d\n", *bug_id, raw_smp_processor_id(), cpu);
303 arch_send_call_function_single_ipi(cpu);
304 }
305 }
306 if (firsttime)
307 dump_stack();
308 *ts1 = ts2;
309
310 return false;
311}
312
313/*
314 * csd_lock/csd_unlock used to serialize access to per-cpu csd resources
315 *
316 * For non-synchronous ipi calls the csd can still be in use by the
317 * previous function call. For multi-cpu calls its even more interesting
318 * as we'll have to ensure no other cpu is observing our csd.
319 */
320static void __csd_lock_wait(call_single_data_t *csd)
321{
322 unsigned long nmessages = 0;
323 int bug_id = 0;
324 u64 ts0, ts1;
325
326 ts1 = ts0 = ktime_get_mono_fast_ns();
327 for (;;) {
328 if (csd_lock_wait_toolong(csd, ts0, &ts1, &bug_id, &nmessages))
329 break;
330 cpu_relax();
331 }
332 smp_acquire__after_ctrl_dep();
333}
334
335static __always_inline void csd_lock_wait(call_single_data_t *csd)
336{
337 if (static_branch_unlikely(&csdlock_debug_enabled)) {
338 __csd_lock_wait(csd);
339 return;
340 }
341
342 smp_cond_load_acquire(&csd->node.u_flags, !(VAL & CSD_FLAG_LOCK));
343}
344#else
345static __always_inline void __csd_lock_wait(call_single_data_t *csd)
346{
347}
348
349static void csd_lock_record(call_single_data_t *csd)
350{
351}
352
353static __always_inline void csd_lock_wait(call_single_data_t *csd)
354{
355 smp_cond_load_acquire(&csd->node.u_flags, !(VAL & CSD_FLAG_LOCK));
356}
357#endif
358
359static __always_inline void csd_lock(call_single_data_t *csd)
360{
361 if (IS_ENABLED(CONFIG_CSD_LOCK_WAIT_DEBUG) &&
362 static_branch_unlikely(&csdlock_debug_enabled)) {
363
364 for (;;) {
365 unsigned int flags;
366
367 __csd_lock_wait(csd);
368 flags = READ_ONCE(csd->node.u_flags);
369
370 if (!(flags & CSD_FLAG_LOCK) &&
371 try_cmpxchg_acquire(&csd->node.u_flags, &flags, flags | CSD_FLAG_LOCK))
372 break;
373 }
374 } else {
375 csd_lock_wait(csd);
376 csd->node.u_flags |= CSD_FLAG_LOCK;
377 }
378
379 /*
380 * prevent CPU from reordering the above assignment
381 * to ->flags with any subsequent assignments to other
382 * fields of the specified call_single_data_t structure:
383 */
384 smp_wmb();
385}
386
387static __always_inline void csd_unlock(call_single_data_t *csd)
388{
389 WARN_ON(!(csd->node.u_flags & CSD_FLAG_LOCK));
390
391 /*
392 * ensure we're all done before releasing data:
393 */
394 smp_store_release(&csd->node.u_flags, 0);
395}
396
397static DEFINE_PER_CPU_SHARED_ALIGNED(call_single_data_t, csd_data);
398
399#ifdef CONFIG_CSD_LOCK_WAIT_DEBUG
400static call_single_data_t *get_single_csd_data(int cpu)
401{
402 if (static_branch_unlikely(&csdlock_debug_enabled) &&
403 (unsigned int)cpu < nr_cpu_ids)
404 return per_cpu_ptr(&csd_data, cpu);
405 return this_cpu_ptr(&csd_data);
406}
407#else
408static call_single_data_t *get_single_csd_data(int cpu)
409{
410 return this_cpu_ptr(&csd_data);
411}
412#endif
413
414void __smp_call_single_queue(int cpu, struct llist_node *node)
415{
416 /*
417 * We have to check the type of the CSD before queueing it, because
418 * once queued it can have its flags cleared by
419 * flush_smp_call_function_queue()
420 * even if we haven't sent the smp_call IPI yet (e.g. the stopper
421 * executes migration_cpu_stop() on the remote CPU).
422 */
423 if (trace_csd_queue_cpu_enabled()) {
424 call_single_data_t *csd;
425 smp_call_func_t func;
426
427 csd = container_of(node, call_single_data_t, node.llist);
428 func = CSD_TYPE(csd) == CSD_TYPE_TTWU ?
429 sched_ttwu_pending : csd->func;
430
431 trace_call__csd_queue_cpu(cpu, _RET_IP_, func, csd);
432 }
433
434 /*
435 * The list addition should be visible to the target CPU when it pops
436 * the head of the list to pull the entry off it in the IPI handler
437 * because of normal cache coherency rules implied by the underlying
438 * llist ops.
439 *
440 * If IPIs can go out of order to the cache coherency protocol
441 * in an architecture, sufficient synchronisation should be added
442 * to arch code to make it appear to obey cache coherency WRT
443 * locking and barrier primitives. Generic code isn't really
444 * equipped to do the right thing...
445 */
446 if (llist_add(node, &per_cpu(call_single_queue, cpu)))
447 send_call_function_single_ipi(cpu);
448}
449
450/*
451 * Insert a previously allocated call_single_data_t element
452 * for execution on the given CPU. data must already have
453 * ->func, ->info, and ->flags set.
454 */
455static int generic_exec_single(int cpu, call_single_data_t *csd)
456{
457 /*
458 * Preemption already disabled here so stopper cannot run on this CPU,
459 * ensuring mutually exclusive CPU offlining and last IPI flush.
460 */
461 if (cpu == smp_processor_id()) {
462 smp_call_func_t func = csd->func;
463 void *info = csd->info;
464 unsigned long flags;
465
466 /*
467 * We can unlock early even for the synchronous on-stack case,
468 * since we're doing this from the same CPU..
469 */
470 csd_lock_record(csd);
471 csd_unlock(csd);
472 local_irq_save(flags);
473 csd_do_func(func, info, NULL);
474 csd_lock_record(NULL);
475 local_irq_restore(flags);
476 return 0;
477 }
478
479 if ((unsigned)cpu >= nr_cpu_ids || !cpu_online(cpu)) {
480 csd_unlock(csd);
481 return -ENXIO;
482 }
483
484 __smp_call_single_queue(cpu, &csd->node.llist);
485
486 return 0;
487}
488
489/**
490 * generic_smp_call_function_single_interrupt - Execute SMP IPI callbacks
491 *
492 * Invoked by arch to handle an IPI for call function single.
493 * Must be called with interrupts disabled.
494 */
495void generic_smp_call_function_single_interrupt(void)
496{
497 __flush_smp_call_function_queue(true);
498}
499
500/**
501 * __flush_smp_call_function_queue - Flush pending smp-call-function callbacks
502 *
503 * @warn_cpu_offline: If set to 'true', warn if callbacks were queued on an
504 * offline CPU. Skip this check if set to 'false'.
505 *
506 * Flush any pending smp-call-function callbacks queued on this CPU. This is
507 * invoked by the generic IPI handler, as well as by a CPU about to go offline,
508 * to ensure that all pending IPI callbacks are run before it goes completely
509 * offline.
510 *
511 * Loop through the call_single_queue and run all the queued callbacks.
512 * Must be called with interrupts disabled.
513 */
514static void __flush_smp_call_function_queue(bool warn_cpu_offline)
515{
516 call_single_data_t *csd, *csd_next;
517 struct llist_node *entry, *prev;
518 struct llist_head *head;
519 static bool warned;
520 atomic_t *tbt;
521
522 lockdep_assert_irqs_disabled();
523
524 /* Allow waiters to send backtrace NMI from here onwards */
525 tbt = this_cpu_ptr(&trigger_backtrace);
526 atomic_set_release(tbt, 1);
527
528 head = this_cpu_ptr(&call_single_queue);
529 entry = llist_del_all(head);
530 entry = llist_reverse_order(entry);
531
532 /* There shouldn't be any pending callbacks on an offline CPU. */
533 if (unlikely(warn_cpu_offline && !cpu_online(smp_processor_id()) &&
534 !warned && entry != NULL)) {
535 warned = true;
536 WARN(1, "IPI on offline CPU %d\n", smp_processor_id());
537
538 /*
539 * We don't have to use the _safe() variant here
540 * because we are not invoking the IPI handlers yet.
541 */
542 llist_for_each_entry(csd, entry, node.llist) {
543 switch (CSD_TYPE(csd)) {
544 case CSD_TYPE_ASYNC:
545 case CSD_TYPE_SYNC:
546 case CSD_TYPE_IRQ_WORK:
547 pr_warn("IPI callback %pS sent to offline CPU\n",
548 csd->func);
549 break;
550
551 case CSD_TYPE_TTWU:
552 pr_warn("IPI task-wakeup sent to offline CPU\n");
553 break;
554
555 default:
556 pr_warn("IPI callback, unknown type %d, sent to offline CPU\n",
557 CSD_TYPE(csd));
558 break;
559 }
560 }
561 }
562
563 /*
564 * First; run all SYNC callbacks, people are waiting for us.
565 */
566 prev = NULL;
567 llist_for_each_entry_safe(csd, csd_next, entry, node.llist) {
568 /* Do we wait until *after* callback? */
569 if (CSD_TYPE(csd) == CSD_TYPE_SYNC) {
570 smp_call_func_t func = csd->func;
571 void *info = csd->info;
572
573 if (prev) {
574 prev->next = &csd_next->node.llist;
575 } else {
576 entry = &csd_next->node.llist;
577 }
578
579 csd_lock_record(csd);
580 csd_do_func(func, info, csd);
581 csd_unlock(csd);
582 csd_lock_record(NULL);
583 } else {
584 prev = &csd->node.llist;
585 }
586 }
587
588 if (!entry)
589 return;
590
591 /*
592 * Second; run all !SYNC callbacks.
593 */
594 prev = NULL;
595 llist_for_each_entry_safe(csd, csd_next, entry, node.llist) {
596 int type = CSD_TYPE(csd);
597
598 if (type != CSD_TYPE_TTWU) {
599 if (prev) {
600 prev->next = &csd_next->node.llist;
601 } else {
602 entry = &csd_next->node.llist;
603 }
604
605 if (type == CSD_TYPE_ASYNC) {
606 smp_call_func_t func = csd->func;
607 void *info = csd->info;
608
609 csd_lock_record(csd);
610 csd_unlock(csd);
611 csd_do_func(func, info, csd);
612 csd_lock_record(NULL);
613 } else if (type == CSD_TYPE_IRQ_WORK) {
614 irq_work_single(csd);
615 }
616
617 } else {
618 prev = &csd->node.llist;
619 }
620 }
621
622 /*
623 * Third; only CSD_TYPE_TTWU is left, issue those.
624 */
625 if (entry) {
626 csd = llist_entry(entry, typeof(*csd), node.llist);
627 csd_do_func(sched_ttwu_pending, entry, csd);
628 }
629}
630
631
632/**
633 * flush_smp_call_function_queue - Flush pending smp-call-function callbacks
634 * from task context (idle, migration thread)
635 *
636 * When TIF_POLLING_NRFLAG is supported and a CPU is in idle and has it
637 * set, then remote CPUs can avoid sending IPIs and wake the idle CPU by
638 * setting TIF_NEED_RESCHED. The idle task on the woken up CPU has to
639 * handle queued SMP function calls before scheduling.
640 *
641 * The migration thread has to ensure that an eventually pending wakeup has
642 * been handled before it migrates a task.
643 */
644void flush_smp_call_function_queue(void)
645{
646 unsigned int was_pending;
647 unsigned long flags;
648
649 if (llist_empty(this_cpu_ptr(&call_single_queue)))
650 return;
651
652 local_irq_save(flags);
653 /* Get the already pending soft interrupts for RT enabled kernels */
654 was_pending = local_softirq_pending();
655 __flush_smp_call_function_queue(true);
656 if (local_softirq_pending())
657 do_softirq_post_smp_call_flush(was_pending);
658
659 local_irq_restore(flags);
660}
661
662/**
663 * smp_call_function_single - Run a function on a specific CPU
664 * @cpu: Specific target CPU for this function.
665 * @func: The function to run. This must be fast and non-blocking.
666 * @info: An arbitrary pointer to pass to the function.
667 * @wait: If true, wait until function has completed on other CPUs.
668 *
669 * Returns: %0 on success, else a negative status code.
670 */
671int smp_call_function_single(int cpu, smp_call_func_t func, void *info,
672 int wait)
673{
674 call_single_data_t *csd;
675 call_single_data_t csd_stack = {
676 .node = { .u_flags = CSD_FLAG_LOCK | CSD_TYPE_SYNC, },
677 };
678 int this_cpu;
679 int err;
680
681 /*
682 * Prevent preemption and reschedule on another CPU, as well as CPU
683 * removal. This prevents stopper from running on this CPU, thus
684 * providing mutual exclusion of the below cpu_online() check and
685 * IPI sending ensuring IPI are not missed by CPU going offline.
686 */
687 this_cpu = get_cpu();
688
689 /*
690 * Can deadlock when called with interrupts disabled.
691 * We allow cpu's that are not yet online though, as no one else can
692 * send smp call function interrupt to this cpu and as such deadlocks
693 * can't happen.
694 */
695 WARN_ON_ONCE(cpu_online(this_cpu) && irqs_disabled()
696 && !oops_in_progress);
697
698 /*
699 * When @wait we can deadlock when we interrupt between llist_add() and
700 * arch_send_call_function_ipi*(); when !@wait we can deadlock due to
701 * csd_lock() on because the interrupt context uses the same csd
702 * storage.
703 */
704 WARN_ON_ONCE(!in_task());
705
706 csd = &csd_stack;
707 if (!wait) {
708 csd = get_single_csd_data(cpu);
709 csd_lock(csd);
710 }
711
712 csd->func = func;
713 csd->info = info;
714#ifdef CONFIG_CSD_LOCK_WAIT_DEBUG
715 csd->node.src = this_cpu;
716 csd->node.dst = cpu;
717#endif
718
719 err = generic_exec_single(cpu, csd);
720
721 if (wait)
722 csd_lock_wait(csd);
723
724 put_cpu();
725
726 return err;
727}
728EXPORT_SYMBOL(smp_call_function_single);
729
730/**
731 * smp_call_function_single_async() - Run an asynchronous function on a
732 * specific CPU.
733 * @cpu: The CPU to run on.
734 * @csd: Pre-allocated and setup data structure
735 *
736 * Like smp_call_function_single(), but the call is asynchonous and
737 * can thus be done from contexts with disabled interrupts.
738 *
739 * The caller passes his own pre-allocated data structure
740 * (ie: embedded in an object) and is responsible for synchronizing it
741 * such that the IPIs performed on the @csd are strictly serialized.
742 *
743 * If the function is called with one csd which has not yet been
744 * processed by previous call to smp_call_function_single_async(), the
745 * function will return immediately with -EBUSY showing that the csd
746 * object is still in progress.
747 *
748 * NOTE: Be careful, there is unfortunately no current debugging facility to
749 * validate the correctness of this serialization.
750 *
751 * Return: %0 on success or negative errno value on error
752 */
753int smp_call_function_single_async(int cpu, call_single_data_t *csd)
754{
755 int err = 0;
756
757 preempt_disable();
758
759 if (csd->node.u_flags & CSD_FLAG_LOCK) {
760 err = -EBUSY;
761 goto out;
762 }
763
764 csd->node.u_flags = CSD_FLAG_LOCK;
765 smp_wmb();
766
767 err = generic_exec_single(cpu, csd);
768
769out:
770 preempt_enable();
771
772 return err;
773}
774EXPORT_SYMBOL_GPL(smp_call_function_single_async);
775
776/**
777 * smp_call_function_any - Run a function on any of the given cpus
778 * @mask: The mask of cpus it can run on.
779 * @func: The function to run. This must be fast and non-blocking.
780 * @info: An arbitrary pointer to pass to the function.
781 * @wait: If true, wait until function has completed.
782 *
783 * Selection preference:
784 * 1) current cpu if in @mask
785 * 2) nearest cpu in @mask, based on NUMA topology
786 *
787 * Returns: %0 on success, else a negative status code (if no cpus were online).
788 */
789int smp_call_function_any(const struct cpumask *mask,
790 smp_call_func_t func, void *info, int wait)
791{
792 unsigned int cpu;
793 int ret;
794
795 /* Try for same CPU (cheapest) */
796 cpu = get_cpu();
797 if (!cpumask_test_cpu(cpu, mask))
798 cpu = sched_numa_find_nth_cpu(mask, 0, cpu_to_node(cpu));
799
800 ret = smp_call_function_single(cpu, func, info, wait);
801 put_cpu();
802 return ret;
803}
804EXPORT_SYMBOL_GPL(smp_call_function_any);
805
806/*
807 * Flags to be used as scf_flags argument of smp_call_function_many_cond().
808 *
809 * %SCF_WAIT: Wait until function execution is completed
810 * %SCF_RUN_LOCAL: Run also locally if local cpu is set in cpumask
811 */
812#define SCF_WAIT (1U << 0)
813#define SCF_RUN_LOCAL (1U << 1)
814
815static void smp_call_function_many_cond(const struct cpumask *mask,
816 smp_call_func_t func, void *info,
817 unsigned int scf_flags,
818 smp_cond_func_t cond_func)
819{
820 int cpu, last_cpu, this_cpu = smp_processor_id();
821 struct call_function_data *cfd;
822 bool wait = scf_flags & SCF_WAIT;
823 int nr_cpus = 0;
824 bool run_remote = false;
825
826 lockdep_assert_preemption_disabled();
827
828 /*
829 * Can deadlock when called with interrupts disabled.
830 * We allow cpu's that are not yet online though, as no one else can
831 * send smp call function interrupt to this cpu and as such deadlocks
832 * can't happen.
833 */
834 if (cpu_online(this_cpu) && !oops_in_progress &&
835 !early_boot_irqs_disabled)
836 lockdep_assert_irqs_enabled();
837
838 /*
839 * When @wait we can deadlock when we interrupt between llist_add() and
840 * arch_send_call_function_ipi*(); when !@wait we can deadlock due to
841 * csd_lock() on because the interrupt context uses the same csd
842 * storage.
843 */
844 WARN_ON_ONCE(!in_task());
845
846 /* Check if we need remote execution, i.e., any CPU excluding this one. */
847 if (cpumask_any_and_but(mask, cpu_online_mask, this_cpu) < nr_cpu_ids) {
848 cfd = this_cpu_ptr(&cfd_data);
849 cpumask_and(cfd->cpumask, mask, cpu_online_mask);
850 __cpumask_clear_cpu(this_cpu, cfd->cpumask);
851
852 cpumask_clear(cfd->cpumask_ipi);
853 for_each_cpu(cpu, cfd->cpumask) {
854 call_single_data_t *csd = per_cpu_ptr(cfd->csd, cpu);
855
856 if (cond_func && !cond_func(cpu, info)) {
857 __cpumask_clear_cpu(cpu, cfd->cpumask);
858 continue;
859 }
860
861 /* Work is enqueued on a remote CPU. */
862 run_remote = true;
863
864 csd_lock(csd);
865 if (wait)
866 csd->node.u_flags |= CSD_TYPE_SYNC;
867 csd->func = func;
868 csd->info = info;
869#ifdef CONFIG_CSD_LOCK_WAIT_DEBUG
870 csd->node.src = this_cpu;
871 csd->node.dst = cpu;
872#endif
873 trace_csd_queue_cpu(cpu, _RET_IP_, func, csd);
874
875 /*
876 * Kick the remote CPU if this is the first work
877 * item enqueued.
878 */
879 if (llist_add(&csd->node.llist, &per_cpu(call_single_queue, cpu))) {
880 __cpumask_set_cpu(cpu, cfd->cpumask_ipi);
881 nr_cpus++;
882 last_cpu = cpu;
883 }
884 }
885
886 /*
887 * Choose the most efficient way to send an IPI. Note that the
888 * number of CPUs might be zero due to concurrent changes to the
889 * provided mask.
890 */
891 if (nr_cpus == 1)
892 send_call_function_single_ipi(last_cpu);
893 else if (likely(nr_cpus > 1))
894 send_call_function_ipi_mask(cfd->cpumask_ipi);
895 }
896
897 /* Check if we need local execution. */
898 if ((scf_flags & SCF_RUN_LOCAL) && cpumask_test_cpu(this_cpu, mask) &&
899 (!cond_func || cond_func(this_cpu, info))) {
900 unsigned long flags;
901
902 local_irq_save(flags);
903 csd_do_func(func, info, NULL);
904 local_irq_restore(flags);
905 }
906
907 if (run_remote && wait) {
908 for_each_cpu(cpu, cfd->cpumask) {
909 call_single_data_t *csd;
910
911 csd = per_cpu_ptr(cfd->csd, cpu);
912 csd_lock_wait(csd);
913 }
914 }
915}
916
917/**
918 * smp_call_function_many() - Run a function on a set of CPUs.
919 * @mask: The set of cpus to run on (only runs on online subset).
920 * @func: The function to run. This must be fast and non-blocking.
921 * @info: An arbitrary pointer to pass to the function.
922 * @wait: If true, wait (atomically) until function has completed
923 * on other CPUs.
924 *
925 * You must not call this function with disabled interrupts or from a
926 * hardware interrupt handler or from a bottom half handler. Preemption
927 * must be disabled when calling this function.
928 *
929 * @func is not called on the local CPU even if @mask contains it. Consider
930 * using on_each_cpu_cond_mask() instead if this is not desirable.
931 */
932void smp_call_function_many(const struct cpumask *mask,
933 smp_call_func_t func, void *info, bool wait)
934{
935 smp_call_function_many_cond(mask, func, info, wait * SCF_WAIT, NULL);
936}
937EXPORT_SYMBOL(smp_call_function_many);
938
939/**
940 * smp_call_function() - Run a function on all other CPUs.
941 * @func: The function to run. This must be fast and non-blocking.
942 * @info: An arbitrary pointer to pass to the function.
943 * @wait: If true, wait (atomically) until function has completed
944 * on other CPUs.
945 *
946 * If @wait is true, then returns once @func has returned; otherwise
947 * it returns just before the target cpu calls @func.
948 *
949 * You must not call this function with disabled interrupts or from a
950 * hardware interrupt handler or from a bottom half handler.
951 */
952void smp_call_function(smp_call_func_t func, void *info, int wait)
953{
954 preempt_disable();
955 smp_call_function_many(cpu_online_mask, func, info, wait);
956 preempt_enable();
957}
958EXPORT_SYMBOL(smp_call_function);
959
960/* Setup configured maximum number of CPUs to activate */
961unsigned int setup_max_cpus = NR_CPUS;
962EXPORT_SYMBOL(setup_max_cpus);
963
964
965/*
966 * Setup routine for controlling SMP activation
967 *
968 * Command-line option of "nosmp" or "maxcpus=0" will disable SMP
969 * activation entirely (the MPS table probe still happens, though).
970 *
971 * Command-line option of "maxcpus=<NUM>", where <NUM> is an integer
972 * greater than 0, limits the maximum number of CPUs activated in
973 * SMP mode to <NUM>.
974 */
975
976void __weak __init arch_disable_smp_support(void) { }
977
978static int __init nosmp(char *str)
979{
980 setup_max_cpus = 0;
981 arch_disable_smp_support();
982
983 return 0;
984}
985
986early_param("nosmp", nosmp);
987
988/* this is hard limit */
989static int __init nrcpus(char *str)
990{
991 int nr_cpus;
992
993 if (get_option(&str, &nr_cpus) && nr_cpus > 0 && nr_cpus < nr_cpu_ids)
994 set_nr_cpu_ids(nr_cpus);
995
996 return 0;
997}
998
999early_param("nr_cpus", nrcpus);
1000
1001static int __init maxcpus(char *str)
1002{
1003 get_option(&str, &setup_max_cpus);
1004 if (setup_max_cpus == 0)
1005 arch_disable_smp_support();
1006
1007 return 0;
1008}
1009
1010early_param("maxcpus", maxcpus);
1011
1012#if (NR_CPUS > 1) && !defined(CONFIG_FORCE_NR_CPUS)
1013/* Setup number of possible processor ids */
1014unsigned int nr_cpu_ids __read_mostly = NR_CPUS;
1015EXPORT_SYMBOL(nr_cpu_ids);
1016#endif
1017
1018/* An arch may set nr_cpu_ids earlier if needed, so this would be redundant */
1019void __init setup_nr_cpu_ids(void)
1020{
1021 set_nr_cpu_ids(find_last_bit(cpumask_bits(cpu_possible_mask), NR_CPUS) + 1);
1022}
1023
1024/* Called by boot processor to activate the rest. */
1025void __init smp_init(void)
1026{
1027 int num_nodes, num_cpus;
1028
1029 idle_threads_init();
1030 cpuhp_threads_init();
1031
1032 pr_info("Bringing up secondary CPUs ...\n");
1033
1034 bringup_nonboot_cpus(setup_max_cpus);
1035
1036 num_nodes = num_online_nodes();
1037 num_cpus = num_online_cpus();
1038 pr_info("Brought up %d node%s, %d CPU%s\n",
1039 num_nodes, str_plural(num_nodes), num_cpus, str_plural(num_cpus));
1040
1041 /* Any cleanup work */
1042 smp_cpus_done(setup_max_cpus);
1043}
1044
1045/**
1046 * on_each_cpu_cond_mask() - Call a function on each processor for which
1047 * the supplied function cond_func returns true, optionally waiting
1048 * for all the required CPUs to finish. This may include the local
1049 * processor.
1050 * @cond_func: A callback function that is passed a cpu id and
1051 * the info parameter. The function is called
1052 * with preemption disabled. The function should
1053 * return a boolean value indicating whether to IPI
1054 * the specified CPU.
1055 * @func: The function to run on all applicable CPUs.
1056 * This must be fast and non-blocking.
1057 * @info: An arbitrary pointer to pass to both functions.
1058 * @wait: If true, wait (atomically) until function has
1059 * completed on other CPUs.
1060 * @mask: The set of cpus to run on (only runs on online subset).
1061 *
1062 * Preemption is disabled to protect against CPUs going offline but not online.
1063 * CPUs going online during the call will not be seen or sent an IPI.
1064 *
1065 * You must not call this function with disabled interrupts or
1066 * from a hardware interrupt handler or from a bottom half handler.
1067 */
1068void on_each_cpu_cond_mask(smp_cond_func_t cond_func, smp_call_func_t func,
1069 void *info, bool wait, const struct cpumask *mask)
1070{
1071 unsigned int scf_flags = SCF_RUN_LOCAL;
1072
1073 if (wait)
1074 scf_flags |= SCF_WAIT;
1075
1076 preempt_disable();
1077 smp_call_function_many_cond(mask, func, info, scf_flags, cond_func);
1078 preempt_enable();
1079}
1080EXPORT_SYMBOL(on_each_cpu_cond_mask);
1081
1082static void do_nothing(void *unused)
1083{
1084}
1085
1086/**
1087 * kick_all_cpus_sync - Force all cpus out of idle
1088 *
1089 * Used to synchronize the update of pm_idle function pointer. It's
1090 * called after the pointer is updated and returns after the dummy
1091 * callback function has been executed on all cpus. The execution of
1092 * the function can only happen on the remote cpus after they have
1093 * left the idle function which had been called via pm_idle function
1094 * pointer. So it's guaranteed that nothing uses the previous pointer
1095 * anymore.
1096 */
1097void kick_all_cpus_sync(void)
1098{
1099 /* Make sure the change is visible before we kick the cpus */
1100 smp_mb();
1101 smp_call_function(do_nothing, NULL, 1);
1102}
1103EXPORT_SYMBOL_GPL(kick_all_cpus_sync);
1104
1105/**
1106 * wake_up_all_idle_cpus - break all cpus out of idle
1107 * wake_up_all_idle_cpus try to break all cpus which is in idle state even
1108 * including idle polling cpus, for non-idle cpus, we will do nothing
1109 * for them.
1110 */
1111void wake_up_all_idle_cpus(void)
1112{
1113 int cpu;
1114
1115 for_each_possible_cpu(cpu) {
1116 preempt_disable();
1117 if (cpu != smp_processor_id() && cpu_online(cpu))
1118 wake_up_if_idle(cpu);
1119 preempt_enable();
1120 }
1121}
1122EXPORT_SYMBOL_GPL(wake_up_all_idle_cpus);
1123
1124/**
1125 * cpus_peek_for_pending_ipi - Check for pending IPI for CPUs
1126 * @mask: The CPU mask for the CPUs to check.
1127 *
1128 * This function walks through the @mask to check if there are any pending IPIs
1129 * scheduled, for any of the CPUs in the @mask. It does not guarantee
1130 * correctness as it only provides a racy snapshot.
1131 *
1132 * Returns: true if there is a pending IPI scheduled and false otherwise.
1133 */
1134bool cpus_peek_for_pending_ipi(const struct cpumask *mask)
1135{
1136 unsigned int cpu;
1137
1138 for_each_cpu(cpu, mask) {
1139 if (!llist_empty(per_cpu_ptr(&call_single_queue, cpu)))
1140 return true;
1141 }
1142
1143 return false;
1144}
1145
1146/**
1147 * struct smp_call_on_cpu_struct - Call a function on a specific CPU
1148 * @work: &work_struct
1149 * @done: &completion to signal
1150 * @func: function to call
1151 * @data: function's data argument
1152 * @ret: return value from @func
1153 * @cpu: target CPU (%-1 for any CPU)
1154 *
1155 * Used to call a function on a specific cpu and wait for it to return.
1156 * Optionally make sure the call is done on a specified physical cpu via vcpu
1157 * pinning in order to support virtualized environments.
1158 */
1159struct smp_call_on_cpu_struct {
1160 struct work_struct work;
1161 struct completion done;
1162 int (*func)(void *);
1163 void *data;
1164 int ret;
1165 int cpu;
1166};
1167
1168static void smp_call_on_cpu_callback(struct work_struct *work)
1169{
1170 struct smp_call_on_cpu_struct *sscs;
1171
1172 sscs = container_of(work, struct smp_call_on_cpu_struct, work);
1173 if (sscs->cpu >= 0)
1174 hypervisor_pin_vcpu(sscs->cpu);
1175 sscs->ret = sscs->func(sscs->data);
1176 if (sscs->cpu >= 0)
1177 hypervisor_pin_vcpu(-1);
1178
1179 complete(&sscs->done);
1180}
1181
1182/**
1183 * smp_call_on_cpu() - Call a function on a specific CPU and wait
1184 * for it to return.
1185 * @cpu: The CPU to run on.
1186 * @func: The function to run
1187 * @par: An arbitrary pointer parameter for @func.
1188 * @phys: If @true, force to run on physical @cpu. See
1189 * &struct smp_call_on_cpu_struct for more info.
1190 *
1191 * Returns: %-ENXIO if the @cpu is invalid; otherwise the return value
1192 * from @func.
1193 */
1194int smp_call_on_cpu(unsigned int cpu, int (*func)(void *), void *par, bool phys)
1195{
1196 struct smp_call_on_cpu_struct sscs = {
1197 .done = COMPLETION_INITIALIZER_ONSTACK(sscs.done),
1198 .func = func,
1199 .data = par,
1200 .cpu = phys ? cpu : -1,
1201 };
1202
1203 INIT_WORK_ONSTACK(&sscs.work, smp_call_on_cpu_callback);
1204
1205 if (cpu >= nr_cpu_ids || !cpu_online(cpu))
1206 return -ENXIO;
1207
1208 queue_work_on(cpu, system_percpu_wq, &sscs.work);
1209 wait_for_completion(&sscs.done);
1210 destroy_work_on_stack(&sscs.work);
1211
1212 return sscs.ret;
1213}
1214EXPORT_SYMBOL_GPL(smp_call_on_cpu);