projects/linux·blob master · refs

1364 lines · 33.3 KiB

1// SPDX-License-Identifier: GPL-2.0-only
2/*
3 * SMP initialisation and IPI support
4 * Based on arch/arm/kernel/smp.c
5 *
6 * Copyright (C) 2012 ARM Ltd.
7 */
8
9#include <linux/acpi.h>
10#include <linux/arm_sdei.h>
11#include <linux/delay.h>
12#include <linux/init.h>
13#include <linux/spinlock.h>
14#include <linux/sched/mm.h>
15#include <linux/sched/hotplug.h>
16#include <linux/sched/task_stack.h>
17#include <linux/interrupt.h>
18#include <linux/cache.h>
19#include <linux/profile.h>
20#include <linux/errno.h>
21#include <linux/mm.h>
22#include <linux/err.h>
23#include <linux/cpu.h>
24#include <linux/smp.h>
25#include <linux/seq_file.h>
26#include <linux/irq.h>
27#include <linux/irqchip/arm-gic-v3.h>
28#include <linux/percpu.h>
29#include <linux/clockchips.h>
30#include <linux/completion.h>
31#include <linux/of.h>
32#include <linux/irq_work.h>
33#include <linux/kernel_stat.h>
34#include <linux/kexec.h>
35#include <linux/kgdb.h>
36#include <linux/kprobes.h>
37#include <linux/kvm_host.h>
38#include <linux/nmi.h>
39
40#include <asm/alternative.h>
41#include <asm/atomic.h>
42#include <asm/cacheflush.h>
43#include <asm/cpu.h>
44#include <asm/cputype.h>
45#include <asm/cpu_ops.h>
46#include <asm/daifflags.h>
47#include <asm/kvm_mmu.h>
48#include <asm/mmu_context.h>
49#include <asm/nmi.h>
50#include <asm/numa.h>
51#include <asm/processor.h>
52#include <asm/smp_plat.h>
53#include <asm/sections.h>
54#include <asm/tlbflush.h>
55#include <asm/ptrace.h>
56#include <asm/virt.h>
57
58#include <trace/events/ipi.h>
59
60/*
61 * as from 2.5, kernels no longer have an init_tasks structure
62 * so we need some other way of telling a new secondary core
63 * where to place its SVC stack
64 */
65struct secondary_data secondary_data;
66/* Number of CPUs which aren't online, but looping in kernel text. */
67static int cpus_stuck_in_kernel;
68
69static int ipi_irq_base __ro_after_init;
70static int nr_ipi __ro_after_init = NR_IPI;
71
72struct ipi_descs {
73 struct irq_desc *descs[MAX_IPI];
74};
75
76static DEFINE_PER_CPU_READ_MOSTLY(struct ipi_descs, pcpu_ipi_desc);
77
78#define get_ipi_desc(__cpu, __ipi) (per_cpu_ptr(&pcpu_ipi_desc, __cpu)->descs[__ipi])
79
80static bool percpu_ipi_descs __ro_after_init;
81
82static bool crash_stop;
83
84static void ipi_setup(int cpu);
85
86#ifdef CONFIG_HOTPLUG_CPU
87static void ipi_teardown(int cpu);
88static int op_cpu_kill(unsigned int cpu);
89#else
90static inline int op_cpu_kill(unsigned int cpu)
91{
92 return -ENOSYS;
93}
94#endif
95
96
97/*
98 * Boot a secondary CPU, and assign it the specified idle task.
99 * This also gives us the initial stack to use for this CPU.
100 */
101static int boot_secondary(unsigned int cpu, struct task_struct *idle)
102{
103 const struct cpu_operations *ops = get_cpu_ops(cpu);
104
105 if (ops->cpu_boot)
106 return ops->cpu_boot(cpu);
107
108 return -EOPNOTSUPP;
109}
110
111static DECLARE_COMPLETION(cpu_running);
112
113int __cpu_up(unsigned int cpu, struct task_struct *idle)
114{
115 int ret;
116 long status;
117
118 /*
119 * We need to tell the secondary core where to find its stack and the
120 * page tables.
121 */
122 secondary_data.task = idle;
123 update_cpu_boot_status(CPU_MMU_OFF);
124
125 /* Now bring the CPU into our world */
126 ret = boot_secondary(cpu, idle);
127 if (ret) {
128 if (ret != -EPERM)
129 pr_err("CPU%u: failed to boot: %d\n", cpu, ret);
130 return ret;
131 }
132
133 /*
134 * CPU was successfully started, wait for it to come online or
135 * time out.
136 */
137 wait_for_completion_timeout(&cpu_running,
138 msecs_to_jiffies(5000));
139 if (cpu_online(cpu))
140 return 0;
141
142 pr_crit("CPU%u: failed to come online\n", cpu);
143 secondary_data.task = NULL;
144 status = READ_ONCE(secondary_data.status);
145 if (status == CPU_MMU_OFF)
146 status = READ_ONCE(__early_cpu_boot_status);
147
148 switch (status & CPU_BOOT_STATUS_MASK) {
149 default:
150 pr_err("CPU%u: failed in unknown state : 0x%lx\n",
151 cpu, status);
152 cpus_stuck_in_kernel++;
153 break;
154 case CPU_KILL_ME:
155 if (!op_cpu_kill(cpu)) {
156 pr_crit("CPU%u: died during early boot\n", cpu);
157 break;
158 }
159 pr_crit("CPU%u: may not have shut down cleanly\n", cpu);
160 fallthrough;
161 case CPU_STUCK_IN_KERNEL:
162 pr_crit("CPU%u: is stuck in kernel\n", cpu);
163 if (status & CPU_STUCK_REASON_52_BIT_VA)
164 pr_crit("CPU%u: does not support 52-bit VAs\n", cpu);
165 if (status & CPU_STUCK_REASON_NO_GRAN) {
166 pr_crit("CPU%u: does not support %luK granule\n",
167 cpu, PAGE_SIZE / SZ_1K);
168 }
169 cpus_stuck_in_kernel++;
170 break;
171 case CPU_PANIC_KERNEL:
172 panic("CPU%u detected unsupported configuration\n", cpu);
173 }
174
175 return -EIO;
176}
177
178static void init_gic_priority_masking(void)
179{
180 u32 cpuflags;
181
182 if (WARN_ON(!gic_enable_sre()))
183 return;
184
185 cpuflags = read_sysreg(daif);
186
187 WARN_ON(!(cpuflags & PSR_I_BIT));
188 WARN_ON(!(cpuflags & PSR_F_BIT));
189
190 gic_write_pmr(GIC_PRIO_IRQON | GIC_PRIO_PSR_I_SET);
191}
192
193/*
194 * This is the secondary CPU boot entry. We're using this CPUs
195 * idle thread stack, but a set of temporary page tables.
196 */
197asmlinkage notrace void secondary_start_kernel(void)
198{
199 u64 mpidr = read_cpuid_mpidr() & MPIDR_HWID_BITMASK;
200 struct mm_struct *mm = &init_mm;
201 const struct cpu_operations *ops;
202 unsigned int cpu = smp_processor_id();
203
204 /*
205 * All kernel threads share the same mm context; grab a
206 * reference and switch to it.
207 */
208 mmgrab(mm);
209 current->active_mm = mm;
210
211 /*
212 * TTBR0 is only used for the identity mapping at this stage. Make it
213 * point to zero page to avoid speculatively fetching new entries.
214 */
215 cpu_uninstall_idmap();
216
217 if (system_uses_irq_prio_masking())
218 init_gic_priority_masking();
219
220 lockdep_hardirqs_off(CALLER_ADDR0);
221
222 /*
223 * If the system has established the capabilities, make sure
224 * this CPU ticks all of those. If it doesn't, the CPU will
225 * fail to come online.
226 */
227 check_local_cpu_capabilities();
228
229 ops = get_cpu_ops(cpu);
230 if (ops->cpu_postboot)
231 ops->cpu_postboot();
232
233 /*
234 * Log the CPU info before it is marked online and might get read.
235 */
236 cpuinfo_store_cpu();
237 rcutree_report_cpu_starting(cpu);
238 update_cpu_features(cpu);
239 store_cpu_topology(cpu);
240
241 /*
242 * Enable GIC and timers.
243 */
244 notify_cpu_starting(cpu);
245 trace_hardirqs_off_finish();
246
247 ipi_setup(cpu);
248
249 numa_add_cpu(cpu);
250
251 /*
252 * OK, now it's safe to let the boot CPU continue. Wait for
253 * the CPU migration code to notice that the CPU is online
254 * before we continue.
255 */
256 pr_info("CPU%u: Booted secondary processor 0x%010lx [0x%08x]\n",
257 cpu, (unsigned long)mpidr,
258 read_cpuid_id());
259 update_cpu_boot_status(CPU_BOOT_SUCCESS);
260 set_cpu_online(cpu, true);
261 complete(&cpu_running);
262
263 /*
264 * Secondary CPUs enter the kernel with all DAIF exceptions masked.
265 *
266 * As with setup_arch() we must unmask Debug and SError exceptions, and
267 * as the root irqchip has already been detected and initialized we can
268 * unmask IRQ and FIQ at the same time.
269 */
270 local_daif_restore(DAIF_PROCCTX);
271
272 /*
273 * OK, it's off to the idle thread for us
274 */
275 cpu_startup_entry(CPUHP_AP_ONLINE_IDLE);
276}
277
278#ifdef CONFIG_HOTPLUG_CPU
279static int op_cpu_disable(unsigned int cpu)
280{
281 const struct cpu_operations *ops = get_cpu_ops(cpu);
282
283 /*
284 * If we don't have a cpu_die method, abort before we reach the point
285 * of no return. CPU0 may not have an cpu_ops, so test for it.
286 */
287 if (!ops || !ops->cpu_die)
288 return -EOPNOTSUPP;
289
290 /*
291 * We may need to abort a hot unplug for some other mechanism-specific
292 * reason.
293 */
294 if (ops->cpu_disable)
295 return ops->cpu_disable(cpu);
296
297 return 0;
298}
299
300/*
301 * __cpu_disable runs on the processor to be shutdown.
302 */
303int __cpu_disable(void)
304{
305 unsigned int cpu = smp_processor_id();
306 int ret;
307
308 ret = op_cpu_disable(cpu);
309 if (ret)
310 return ret;
311
312 remove_cpu_topology(cpu);
313 numa_remove_cpu(cpu);
314
315 /*
316 * Take this CPU offline. Once we clear this, we can't return,
317 * and we must not schedule until we're ready to give up the cpu.
318 */
319 set_cpu_online(cpu, false);
320 ipi_teardown(cpu);
321
322 /*
323 * OK - migrate IRQs away from this CPU
324 */
325 irq_migrate_all_off_this_cpu();
326
327 return 0;
328}
329
330static int op_cpu_kill(unsigned int cpu)
331{
332 const struct cpu_operations *ops = get_cpu_ops(cpu);
333
334 /*
335 * If we have no means of synchronising with the dying CPU, then assume
336 * that it is really dead. We can only wait for an arbitrary length of
337 * time and hope that it's dead, so let's skip the wait and just hope.
338 */
339 if (!ops->cpu_kill)
340 return 0;
341
342 return ops->cpu_kill(cpu);
343}
344
345/*
346 * Called on the thread which is asking for a CPU to be shutdown after the
347 * shutdown completed.
348 */
349void arch_cpuhp_cleanup_dead_cpu(unsigned int cpu)
350{
351 int err;
352
353 pr_debug("CPU%u: shutdown\n", cpu);
354
355 /*
356 * Now that the dying CPU is beyond the point of no return w.r.t.
357 * in-kernel synchronisation, try to get the firmware to help us to
358 * verify that it has really left the kernel before we consider
359 * clobbering anything it might still be using.
360 */
361 err = op_cpu_kill(cpu);
362 if (err)
363 pr_warn("CPU%d may not have shut down cleanly: %d\n", cpu, err);
364}
365
366/*
367 * Called from the idle thread for the CPU which has been shutdown.
368 *
369 */
370void __noreturn cpu_die(void)
371{
372 unsigned int cpu = smp_processor_id();
373 const struct cpu_operations *ops = get_cpu_ops(cpu);
374
375 idle_task_exit();
376
377 local_daif_mask();
378
379 /* Tell cpuhp_bp_sync_dead() that this CPU is now safe to dispose of */
380 cpuhp_ap_report_dead();
381
382 /*
383 * Actually shutdown the CPU. This must never fail. The specific hotplug
384 * mechanism must perform all required cache maintenance to ensure that
385 * no dirty lines are lost in the process of shutting down the CPU.
386 */
387 ops->cpu_die(cpu);
388
389 BUG();
390}
391#endif
392
393static void __cpu_try_die(int cpu)
394{
395#ifdef CONFIG_HOTPLUG_CPU
396 const struct cpu_operations *ops = get_cpu_ops(cpu);
397
398 if (ops && ops->cpu_die)
399 ops->cpu_die(cpu);
400#endif
401}
402
403/*
404 * Kill the calling secondary CPU, early in bringup before it is turned
405 * online.
406 */
407void __noreturn cpu_die_early(void)
408{
409 int cpu = smp_processor_id();
410
411 pr_crit("CPU%d: will not boot\n", cpu);
412
413 /* Mark this CPU absent */
414 set_cpu_present(cpu, 0);
415
416 if (IS_ENABLED(CONFIG_HOTPLUG_CPU)) {
417 update_cpu_boot_status(CPU_KILL_ME);
418 __cpu_try_die(cpu);
419 }
420
421 update_cpu_boot_status(CPU_STUCK_IN_KERNEL);
422
423 cpu_park_loop();
424}
425
426static void __init hyp_mode_check(void)
427{
428 if (is_hyp_mode_available())
429 pr_info("CPU: All CPU(s) started at EL2\n");
430 else if (is_hyp_mode_mismatched())
431 WARN_TAINT(1, TAINT_CPU_OUT_OF_SPEC,
432 "CPU: CPUs started in inconsistent modes");
433 else
434 pr_info("CPU: All CPU(s) started at EL1\n");
435 if (IS_ENABLED(CONFIG_KVM) && !is_kernel_in_hyp_mode()) {
436 kvm_compute_layout();
437 kvm_apply_hyp_relocations();
438 }
439}
440
441void __init smp_cpus_done(unsigned int max_cpus)
442{
443 pr_info("SMP: Total of %d processors activated.\n", num_online_cpus());
444 hyp_mode_check();
445 setup_system_features();
446 setup_user_features();
447 mark_linear_text_alias_ro();
448}
449
450void __init smp_prepare_boot_cpu(void)
451{
452 /*
453 * The runtime per-cpu areas have been allocated by
454 * setup_per_cpu_areas(), and CPU0's boot time per-cpu area will be
455 * freed shortly, so we must move over to the runtime per-cpu area.
456 */
457 set_my_cpu_offset(per_cpu_offset(smp_processor_id()));
458
459 /* Check arm64_ftr_regs[] before anything looks registers up in it. */
460 validate_ftr_regs();
461 cpuinfo_store_cpu();
462 setup_boot_cpu_features();
463
464 /* Conditionally switch to GIC PMR for interrupt masking */
465 if (system_uses_irq_prio_masking())
466 init_gic_priority_masking();
467
468 kasan_init_hw_tags();
469 /* Init percpu seeds for random tags after cpus are set up. */
470 kasan_init_sw_tags();
471}
472
473/*
474 * Duplicate MPIDRs are a recipe for disaster. Scan all initialized
475 * entries and check for duplicates. If any is found just ignore the
476 * cpu. cpu_logical_map was initialized to INVALID_HWID to avoid
477 * matching valid MPIDR values.
478 */
479static bool __init is_mpidr_duplicate(unsigned int cpu, u64 hwid)
480{
481 unsigned int i;
482
483 for (i = 1; (i < cpu) && (i < NR_CPUS); i++)
484 if (cpu_logical_map(i) == hwid)
485 return true;
486 return false;
487}
488
489/*
490 * Initialize cpu operations for a logical cpu and
491 * set it in the possible mask on success
492 */
493static int __init smp_cpu_setup(int cpu)
494{
495 const struct cpu_operations *ops;
496
497 if (init_cpu_ops(cpu))
498 return -ENODEV;
499
500 ops = get_cpu_ops(cpu);
501 if (ops->cpu_init(cpu))
502 return -ENODEV;
503
504 set_cpu_possible(cpu, true);
505
506 return 0;
507}
508
509static bool bootcpu_valid __initdata;
510static unsigned int cpu_count = 1;
511
512int arch_register_cpu(int cpu)
513{
514 acpi_handle acpi_handle = acpi_get_processor_handle(cpu);
515 struct cpu *c = &per_cpu(cpu_devices, cpu);
516
517 if (!acpi_disabled && !acpi_handle &&
518 IS_ENABLED(CONFIG_ACPI_HOTPLUG_CPU))
519 return -EPROBE_DEFER;
520
521#ifdef CONFIG_ACPI_HOTPLUG_CPU
522 /* For now block anything that looks like physical CPU Hotplug */
523 if (invalid_logical_cpuid(cpu) || !cpu_present(cpu)) {
524 pr_err_once("Changing CPU present bit is not supported\n");
525 return -ENODEV;
526 }
527#endif
528
529 /*
530 * Availability of the acpi handle is sufficient to establish
531 * that _STA has already been checked. No need to recheck here.
532 */
533 c->hotpluggable = arch_cpu_is_hotpluggable(cpu);
534
535 return register_cpu(c, cpu);
536}
537
538#ifdef CONFIG_ACPI_HOTPLUG_CPU
539void arch_unregister_cpu(int cpu)
540{
541 acpi_handle acpi_handle = acpi_get_processor_handle(cpu);
542 struct cpu *c = &per_cpu(cpu_devices, cpu);
543 unsigned long long sta;
544 acpi_status status;
545
546 status = acpi_evaluate_integer(acpi_handle, "_STA", NULL, &sta);
547 if (!ACPI_FAILURE(status) &&
548 cpu_present(cpu) && !(sta & ACPI_STA_DEVICE_PRESENT))
549 pr_err_once("Changing CPU present bit is not supported\n");
550
551 unregister_cpu(c);
552}
553#endif /* CONFIG_ACPI_HOTPLUG_CPU */
554
555#ifdef CONFIG_ACPI
556static struct acpi_madt_generic_interrupt cpu_madt_gicc[NR_CPUS];
557
558struct acpi_madt_generic_interrupt *acpi_cpu_get_madt_gicc(int cpu)
559{
560 return &cpu_madt_gicc[cpu];
561}
562EXPORT_SYMBOL_GPL(acpi_cpu_get_madt_gicc);
563
564static bool acpi_cpu_is_present(int cpu)
565{
566 return acpi_cpu_get_madt_gicc(cpu)->flags & ACPI_MADT_ENABLED;
567}
568
569/*
570 * acpi_map_gic_cpu_interface - parse processor MADT entry
571 *
572 * Carry out sanity checks on MADT processor entry and initialize
573 * cpu_logical_map on success
574 */
575static void __init
576acpi_map_gic_cpu_interface(struct acpi_madt_generic_interrupt *processor)
577{
578 u64 hwid = processor->arm_mpidr;
579
580 if (!(processor->flags &
581 (ACPI_MADT_ENABLED | ACPI_MADT_GICC_ONLINE_CAPABLE))) {
582 pr_debug("skipping disabled CPU entry with 0x%llx MPIDR\n", hwid);
583 return;
584 }
585
586 if (hwid & ~MPIDR_HWID_BITMASK || hwid == INVALID_HWID) {
587 pr_err("skipping CPU entry with invalid MPIDR 0x%llx\n", hwid);
588 return;
589 }
590
591 if (is_mpidr_duplicate(cpu_count, hwid)) {
592 pr_err("duplicate CPU MPIDR 0x%llx in MADT\n", hwid);
593 return;
594 }
595
596 /* Check if GICC structure of boot CPU is available in the MADT */
597 if (cpu_logical_map(0) == hwid) {
598 if (bootcpu_valid) {
599 pr_err("duplicate boot CPU MPIDR: 0x%llx in MADT\n",
600 hwid);
601 return;
602 }
603 bootcpu_valid = true;
604 cpu_madt_gicc[0] = *processor;
605 return;
606 }
607
608 if (cpu_count >= NR_CPUS)
609 return;
610
611 /* map the logical cpu id to cpu MPIDR */
612 set_cpu_logical_map(cpu_count, hwid);
613
614 cpu_madt_gicc[cpu_count] = *processor;
615
616 /*
617 * Set-up the ACPI parking protocol cpu entries
618 * while initializing the cpu_logical_map to
619 * avoid parsing MADT entries multiple times for
620 * nothing (ie a valid cpu_logical_map entry should
621 * contain a valid parking protocol data set to
622 * initialize the cpu if the parking protocol is
623 * the only available enable method).
624 */
625 acpi_set_mailbox_entry(cpu_count, processor);
626
627 cpu_count++;
628}
629
630static int __init
631acpi_parse_gic_cpu_interface(union acpi_subtable_headers *header,
632 const unsigned long end)
633{
634 struct acpi_madt_generic_interrupt *processor;
635
636 processor = (struct acpi_madt_generic_interrupt *)header;
637 if (BAD_MADT_GICC_ENTRY(processor, end))
638 return -EINVAL;
639
640 acpi_table_print_madt_entry(&header->common);
641
642 acpi_map_gic_cpu_interface(processor);
643
644 return 0;
645}
646
647static void __init acpi_parse_and_init_cpus(void)
648{
649 int i;
650
651 /*
652 * do a walk of MADT to determine how many CPUs
653 * we have including disabled CPUs, and get information
654 * we need for SMP init.
655 */
656 acpi_table_parse_madt(ACPI_MADT_TYPE_GENERIC_INTERRUPT,
657 acpi_parse_gic_cpu_interface, 0);
658
659 /*
660 * In ACPI, SMP and CPU NUMA information is provided in separate
661 * static tables, namely the MADT and the SRAT.
662 *
663 * Thus, it is simpler to first create the cpu logical map through
664 * an MADT walk and then map the logical cpus to their node ids
665 * as separate steps.
666 */
667 acpi_map_cpus_to_nodes();
668
669 for (i = 0; i < nr_cpu_ids; i++)
670 early_map_cpu_to_node(i, acpi_numa_get_nid(i));
671}
672#else
673static bool acpi_cpu_is_present(int cpu)
674{
675 return false;
676}
677#define acpi_parse_and_init_cpus(...) do { } while (0)
678#endif
679
680/*
681 * Enumerate the possible CPU set from the device tree and build the
682 * cpu logical map array containing MPIDR values related to logical
683 * cpus. Assumes that cpu_logical_map(0) has already been initialized.
684 */
685static void __init of_parse_and_init_cpus(void)
686{
687 struct device_node *dn;
688
689 for_each_of_cpu_node(dn) {
690 u64 hwid = of_get_cpu_hwid(dn, 0);
691
692 if (hwid & ~MPIDR_HWID_BITMASK)
693 goto next;
694
695 if (is_mpidr_duplicate(cpu_count, hwid)) {
696 pr_err("%pOF: duplicate cpu reg properties in the DT\n",
697 dn);
698 goto next;
699 }
700
701 /*
702 * The numbering scheme requires that the boot CPU
703 * must be assigned logical id 0. Record it so that
704 * the logical map built from DT is validated and can
705 * be used.
706 */
707 if (hwid == cpu_logical_map(0)) {
708 if (bootcpu_valid) {
709 pr_err("%pOF: duplicate boot cpu reg property in DT\n",
710 dn);
711 goto next;
712 }
713
714 bootcpu_valid = true;
715 early_map_cpu_to_node(0, of_node_to_nid(dn));
716
717 /*
718 * cpu_logical_map has already been
719 * initialized and the boot cpu doesn't need
720 * the enable-method so continue without
721 * incrementing cpu.
722 */
723 continue;
724 }
725
726 if (cpu_count >= NR_CPUS)
727 goto next;
728
729 pr_debug("cpu logical map 0x%llx\n", hwid);
730 set_cpu_logical_map(cpu_count, hwid);
731
732 early_map_cpu_to_node(cpu_count, of_node_to_nid(dn));
733next:
734 cpu_count++;
735 }
736}
737
738/*
739 * Enumerate the possible CPU set from the device tree or ACPI and build the
740 * cpu logical map array containing MPIDR values related to logical
741 * cpus. Assumes that cpu_logical_map(0) has already been initialized.
742 */
743void __init smp_init_cpus(void)
744{
745 int i;
746
747 if (acpi_disabled)
748 of_parse_and_init_cpus();
749 else
750 acpi_parse_and_init_cpus();
751
752 if (!bootcpu_valid) {
753 pr_err("missing boot CPU MPIDR, not enabling secondaries\n");
754 return;
755 }
756
757 /*
758 * For the nosmp/maxcpus=0 case, do not mark the secondary CPUs
759 * possible.
760 */
761 if (!setup_max_cpus)
762 return;
763
764 if (cpu_count > nr_cpu_ids)
765 pr_warn("Number of cores (%d) exceeds configured maximum of %u - clipping\n",
766 cpu_count, nr_cpu_ids);
767 /*
768 * We need to set the cpu_logical_map entries before enabling
769 * the cpus so that cpu processor description entries (DT cpu nodes
770 * and ACPI MADT entries) can be retrieved by matching the cpu hwid
771 * with entries in cpu_logical_map while initializing the cpus.
772 * If the cpu set-up fails, invalidate the cpu_logical_map entry.
773 */
774 for (i = 1; i < nr_cpu_ids; i++) {
775 if (cpu_logical_map(i) != INVALID_HWID) {
776 if (smp_cpu_setup(i))
777 set_cpu_logical_map(i, INVALID_HWID);
778 }
779 }
780}
781
782void __init smp_prepare_cpus(unsigned int max_cpus)
783{
784 const struct cpu_operations *ops;
785 unsigned int cpu;
786 int err;
787
788 init_cpu_topology();
789
790 store_cpu_topology(0);
791 numa_store_cpu_info(0);
792 numa_add_cpu(0);
793
794 /*
795 * If UP is mandated by "nosmp" (which implies "maxcpus=0"), don't set
796 * secondary CPUs present.
797 */
798 if (max_cpus == 0)
799 return;
800
801 /*
802 * Initialise the present map (which describes the set of CPUs
803 * actually populated at the present time) and release the
804 * secondaries from the bootloader.
805 */
806 for_each_possible_cpu(cpu) {
807 if (cpu == 0)
808 continue;
809
810 ops = get_cpu_ops(cpu);
811 if (!ops)
812 continue;
813
814 err = ops->cpu_prepare(cpu);
815 if (err)
816 continue;
817
818 if (acpi_disabled || acpi_cpu_is_present(cpu))
819 set_cpu_present(cpu, true);
820 numa_store_cpu_info(cpu);
821 }
822}
823
824static const char *ipi_types[MAX_IPI] __tracepoint_string = {
825 [IPI_RESCHEDULE] = "Rescheduling interrupts",
826 [IPI_CALL_FUNC] = "Function call interrupts",
827 [IPI_CPU_STOP] = "CPU stop interrupts",
828 [IPI_CPU_STOP_NMI] = "CPU stop NMIs",
829 [IPI_TIMER] = "Timer broadcast interrupts",
830 [IPI_IRQ_WORK] = "IRQ work interrupts",
831 [IPI_CPU_BACKTRACE] = "CPU backtrace interrupts",
832 [IPI_KGDB_ROUNDUP] = "KGDB roundup interrupts",
833};
834
835static void smp_cross_call(const struct cpumask *target, unsigned int ipinr);
836
837unsigned long irq_err_count;
838
839int arch_show_interrupts(struct seq_file *p, int prec)
840{
841 unsigned int cpu, i;
842
843 for (i = 0; i < MAX_IPI; i++) {
844 seq_printf(p, "%*s%u: ", prec - 1, "IPI", i);
845 for_each_online_cpu(cpu)
846 seq_printf(p, "%10u ", irq_desc_kstat_cpu(get_ipi_desc(cpu, i), cpu));
847 seq_printf(p, " %s\n", ipi_types[i]);
848 }
849
850 seq_printf(p, "%*s: %10lu\n", prec, "Err", irq_err_count);
851 return 0;
852}
853
854void arch_send_call_function_ipi_mask(const struct cpumask *mask)
855{
856 smp_cross_call(mask, IPI_CALL_FUNC);
857}
858
859void arch_send_call_function_single_ipi(int cpu)
860{
861 smp_cross_call(cpumask_of(cpu), IPI_CALL_FUNC);
862}
863
864#ifdef CONFIG_IRQ_WORK
865void arch_irq_work_raise(void)
866{
867 smp_cross_call(cpumask_of(smp_processor_id()), IPI_IRQ_WORK);
868}
869#endif
870
871/**
872 * arm64_nmi_cpu_stop() - stop the local CPU after it is told to stop.
873 * @regs: register state to record in the vmcore on a crash stop, or NULL for
874 * panic_smp_self_stop(), which has no interrupted context to save.
875 * @die_on_crash: on the kdump crash path, power the CPU off via PSCI CPU_OFF
876 * (so a capture kernel can reclaim it) rather than parking it.
877 *
878 * The single point every arm64 stop path funnels through, keeping the
879 * bookkeeping (mask interrupts, save the crash context, mark offline, mask
880 * SDEI, optionally power off) in one place:
881 *
882 * - the regular IPI_CPU_STOP and pseudo-NMI IPI_CPU_STOP_NMI handlers;
883 * - panic_smp_self_stop(), a CPU parking itself on a parallel panic();
884 * - the SDEI cross-CPU NMI handler (drivers/firmware/arm_sdei_nmi.c),
885 * which reaches CPUs the stop IPIs could not.
886 *
887 * The IPI stop handlers pass @die_on_crash true. The SDEI handler and
888 * panic_smp_self_stop() pass false and only park. For SDEI that is required,
889 * not just conservative: it runs inside an SDEI event that is deliberately
890 * never completed (completing it has firmware resume the wedged context), and
891 * a CPU_OFF from that not-yet-completed context wedges EL3 on some firmware --
892 * a documented follow-up. Parking also matches this path's own fallback when
893 * CPU_OFF is unavailable.
894 */
895void __noreturn arm64_nmi_cpu_stop(struct pt_regs *regs, bool die_on_crash)
896{
897 unsigned int cpu = smp_processor_id();
898 bool crash = IS_ENABLED(CONFIG_KEXEC_CORE) && crash_stop;
899
900 /*
901 * Use local_daif_mask() instead of local_irq_disable() to make sure
902 * that pseudo-NMIs are disabled. The "stop" code starts with an IRQ
903 * and falls back to NMI (which might be pseudo). If the IRQ finally
904 * goes through right as we're timing out then the NMI could interrupt
905 * us. It's better to prevent the NMI and let the IRQ finish since the
906 * pt_regs will be better.
907 */
908 local_daif_mask();
909
910#ifdef CONFIG_KEXEC_CORE
911 if (crash && regs)
912 crash_save_cpu(regs, cpu);
913#endif
914
915 /* the ack a stop requester (e.g. smp_send_stop()) polls for */
916 set_cpu_online(cpu, false);
917
918 sdei_mask_local_cpu();
919
920 if (crash && die_on_crash)
921 __cpu_try_die(cpu);
922
923 /* just in case */
924 cpu_park_loop();
925}
926NOKPROBE_SYMBOL(arm64_nmi_cpu_stop);
927
928/*
929 * We need to implement panic_smp_self_stop() for parallel panic() calls, so
930 * that cpu_online_mask gets correctly updated and smp_send_stop() can skip
931 * CPUs that have already stopped themselves.
932 */
933void __noreturn panic_smp_self_stop(void)
934{
935 arm64_nmi_cpu_stop(NULL, false);
936}
937
938static void arm64_send_ipi(const cpumask_t *mask, unsigned int nr)
939{
940 unsigned int cpu;
941
942 if (!percpu_ipi_descs)
943 __ipi_send_mask(get_ipi_desc(0, nr), mask);
944 else
945 for_each_cpu(cpu, mask)
946 __ipi_send_single(get_ipi_desc(cpu, nr), cpu);
947}
948
949static void arm64_backtrace_ipi(cpumask_t *mask)
950{
951 arm64_send_ipi(mask, IPI_CPU_BACKTRACE);
952}
953
954void arch_trigger_cpumask_backtrace(const cpumask_t *mask, int exclude_cpu)
955{
956 /*
957 * Prefer the SDEI cross-CPU NMI provider when active: firmware
958 * dispatches the event out of EL3 and reaches CPUs that have
959 * interrupts locally masked, without the per-IRQ-mask cost that
960 * pseudo-NMI pays for the same reach. The plain IPI path below
961 * can't reach such a CPU unless pseudo-NMI is enabled.
962 */
963 if (sdei_nmi_trigger_cpumask_backtrace(mask, exclude_cpu))
964 return;
965
966 /*
967 * NOTE: though nmi_trigger_cpumask_backtrace() has "nmi_" in the name,
968 * nothing about it truly needs to be implemented using an NMI, it's
969 * just that it's _allowed_ to work with NMIs. If ipi_should_be_nmi()
970 * returned false our backtrace attempt will just use a regular IPI.
971 */
972 nmi_trigger_cpumask_backtrace(mask, exclude_cpu, arm64_backtrace_ipi);
973}
974
975#ifdef CONFIG_KGDB
976void kgdb_roundup_cpus(void)
977{
978 int this_cpu = raw_smp_processor_id();
979 int cpu;
980
981 for_each_online_cpu(cpu) {
982 /* No need to roundup ourselves */
983 if (cpu == this_cpu)
984 continue;
985
986 __ipi_send_single(get_ipi_desc(cpu, IPI_KGDB_ROUNDUP), cpu);
987 }
988}
989#endif
990
991/*
992 * Main handler for inter-processor interrupts
993 */
994static void do_handle_IPI(int ipinr)
995{
996 unsigned int cpu = smp_processor_id();
997
998 if ((unsigned)ipinr < NR_IPI)
999 trace_ipi_entry(ipi_types[ipinr]);
1000
1001 switch (ipinr) {
1002 case IPI_RESCHEDULE:
1003 scheduler_ipi();
1004 break;
1005
1006 case IPI_CALL_FUNC:
1007 generic_smp_call_function_interrupt();
1008 break;
1009
1010 case IPI_CPU_STOP:
1011 case IPI_CPU_STOP_NMI:
1012 arm64_nmi_cpu_stop(get_irq_regs(), true);
1013 break;
1014
1015#ifdef CONFIG_GENERIC_CLOCKEVENTS_BROADCAST
1016 case IPI_TIMER:
1017 tick_receive_broadcast();
1018 break;
1019#endif
1020
1021#ifdef CONFIG_IRQ_WORK
1022 case IPI_IRQ_WORK:
1023 irq_work_run();
1024 break;
1025#endif
1026
1027 case IPI_CPU_BACKTRACE:
1028 /*
1029 * NOTE: in some cases this _won't_ be NMI context. See the
1030 * comment in arch_trigger_cpumask_backtrace().
1031 */
1032 nmi_cpu_backtrace(get_irq_regs());
1033 break;
1034
1035 case IPI_KGDB_ROUNDUP:
1036 kgdb_nmicallback(cpu, get_irq_regs());
1037 break;
1038
1039 default:
1040 pr_crit("CPU%u: Unknown IPI message 0x%x\n", cpu, ipinr);
1041 break;
1042 }
1043
1044 if ((unsigned)ipinr < NR_IPI)
1045 trace_ipi_exit(ipi_types[ipinr]);
1046}
1047
1048static irqreturn_t ipi_handler(int irq, void *data)
1049{
1050 unsigned int ipi = (irq - ipi_irq_base) % nr_ipi;
1051
1052 do_handle_IPI(ipi);
1053 return IRQ_HANDLED;
1054}
1055
1056static void smp_cross_call(const struct cpumask *target, unsigned int ipinr)
1057{
1058 trace_ipi_raise(target, ipi_types[ipinr]);
1059 arm64_send_ipi(target, ipinr);
1060}
1061
1062static bool ipi_should_be_nmi(enum ipi_msg_type ipi)
1063{
1064 if (!system_uses_irq_prio_masking())
1065 return false;
1066
1067 switch (ipi) {
1068 case IPI_CPU_STOP_NMI:
1069 case IPI_CPU_BACKTRACE:
1070 case IPI_KGDB_ROUNDUP:
1071 return true;
1072 default:
1073 return false;
1074 }
1075}
1076
1077static void ipi_setup(int cpu)
1078{
1079 int i;
1080
1081 if (WARN_ON_ONCE(!ipi_irq_base))
1082 return;
1083
1084 for (i = 0; i < nr_ipi; i++) {
1085 if (!percpu_ipi_descs) {
1086 if (ipi_should_be_nmi(i)) {
1087 prepare_percpu_nmi(ipi_irq_base + i);
1088 enable_percpu_nmi(ipi_irq_base + i, 0);
1089 } else {
1090 enable_percpu_irq(ipi_irq_base + i, 0);
1091 }
1092 } else {
1093 enable_irq(irq_desc_get_irq(get_ipi_desc(cpu, i)));
1094 }
1095 }
1096}
1097
1098#ifdef CONFIG_HOTPLUG_CPU
1099static void ipi_teardown(int cpu)
1100{
1101 int i;
1102
1103 if (WARN_ON_ONCE(!ipi_irq_base))
1104 return;
1105
1106 for (i = 0; i < nr_ipi; i++) {
1107 if (!percpu_ipi_descs) {
1108 if (ipi_should_be_nmi(i)) {
1109 disable_percpu_nmi(ipi_irq_base + i);
1110 teardown_percpu_nmi(ipi_irq_base + i);
1111 } else {
1112 disable_percpu_irq(ipi_irq_base + i);
1113 }
1114 } else {
1115 disable_irq_nosync(irq_desc_get_irq(get_ipi_desc(cpu, i)));
1116 }
1117 }
1118}
1119#endif
1120
1121static void ipi_setup_sgi(int ipi)
1122{
1123 int err, irq, cpu;
1124
1125 irq = ipi_irq_base + ipi;
1126
1127 if (ipi_should_be_nmi(ipi)) {
1128 err = request_percpu_nmi(irq, ipi_handler, "IPI", NULL, &irq_stat);
1129 WARN(err, "Could not request IRQ %d as NMI, err=%d\n", irq, err);
1130 } else {
1131 err = request_percpu_irq(irq, ipi_handler, "IPI", &irq_stat);
1132 WARN(err, "Could not request IRQ %d as IRQ, err=%d\n", irq, err);
1133 }
1134
1135 for_each_possible_cpu(cpu)
1136 get_ipi_desc(cpu, ipi) = irq_to_desc(irq);
1137
1138 irq_set_status_flags(irq, IRQ_HIDDEN);
1139}
1140
1141static void ipi_setup_lpi(int ipi, int ncpus)
1142{
1143 for (int cpu = 0; cpu < ncpus; cpu++) {
1144 int err, irq;
1145
1146 irq = ipi_irq_base + (cpu * nr_ipi) + ipi;
1147
1148 err = irq_force_affinity(irq, cpumask_of(cpu));
1149 WARN(err, "Could not force affinity IRQ %d, err=%d\n", irq, err);
1150
1151 err = request_irq(irq, ipi_handler, IRQF_NO_AUTOEN, "IPI",
1152 NULL);
1153 WARN(err, "Could not request IRQ %d, err=%d\n", irq, err);
1154
1155 irq_set_status_flags(irq, (IRQ_HIDDEN | IRQ_NO_BALANCING_MASK));
1156
1157 get_ipi_desc(cpu, ipi) = irq_to_desc(irq);
1158 }
1159}
1160
1161void __init set_smp_ipi_range_percpu(int ipi_base, int n, int ncpus)
1162{
1163 int i;
1164
1165 WARN_ON(n < MAX_IPI);
1166 nr_ipi = min(n, MAX_IPI);
1167
1168 percpu_ipi_descs = !!ncpus;
1169 ipi_irq_base = ipi_base;
1170
1171 for (i = 0; i < nr_ipi; i++) {
1172 if (!percpu_ipi_descs)
1173 ipi_setup_sgi(i);
1174 else
1175 ipi_setup_lpi(i, ncpus);
1176 }
1177
1178 /* Setup the boot CPU immediately */
1179 ipi_setup(smp_processor_id());
1180}
1181
1182void arch_smp_send_reschedule(int cpu)
1183{
1184 smp_cross_call(cpumask_of(cpu), IPI_RESCHEDULE);
1185}
1186
1187#ifdef CONFIG_ARM64_ACPI_PARKING_PROTOCOL
1188void arch_send_wakeup_ipi(unsigned int cpu)
1189{
1190 /*
1191 * We use a scheduler IPI to wake the CPU as this avoids the need for a
1192 * dedicated IPI and we can safely handle spurious scheduler IPIs.
1193 */
1194 smp_send_reschedule(cpu);
1195}
1196#endif
1197
1198#ifdef CONFIG_GENERIC_CLOCKEVENTS_BROADCAST
1199void tick_broadcast(const struct cpumask *mask)
1200{
1201 smp_cross_call(mask, IPI_TIMER);
1202}
1203#endif
1204
1205/*
1206 * The number of CPUs online, not counting this CPU (which may not be
1207 * fully online and so not counted in num_online_cpus()).
1208 */
1209static inline unsigned int num_other_online_cpus(void)
1210{
1211 unsigned int this_cpu_online = cpu_online(smp_processor_id());
1212
1213 return num_online_cpus() - this_cpu_online;
1214}
1215
1216void smp_send_stop(void)
1217{
1218 static unsigned long stop_in_progress;
1219 static cpumask_t mask;
1220 unsigned long timeout;
1221
1222 /*
1223 * If this cpu is the only one alive at this point in time, online or
1224 * not, there are no stop messages to be sent around, so just back out.
1225 */
1226 if (num_other_online_cpus() == 0)
1227 goto skip_ipi;
1228
1229 /* Only proceed if this is the first CPU to reach this code */
1230 if (test_and_set_bit(0, &stop_in_progress))
1231 return;
1232
1233 /*
1234 * Send an IPI to all currently online CPUs except the CPU running
1235 * this code.
1236 *
1237 * NOTE: we don't do anything here to prevent other CPUs from coming
1238 * online after we snapshot `cpu_online_mask`. Ideally, the calling code
1239 * should do something to prevent other CPUs from coming up. This code
1240 * can be called in the panic path and thus it doesn't seem wise to
1241 * grab the CPU hotplug mutex ourselves. Worst case:
1242 * - If a CPU comes online as we're running, we'll likely notice it
1243 * during the 1 second wait below and then we'll catch it when we try
1244 * with an NMI (assuming NMIs are enabled) since we re-snapshot the
1245 * mask before sending an NMI.
1246 * - If we leave the function and see that CPUs are still online we'll
1247 * at least print a warning. Especially without NMIs this function
1248 * isn't foolproof anyway so calling code will just have to accept
1249 * the fact that there could be cases where a CPU can't be stopped.
1250 */
1251 cpumask_copy(&mask, cpu_online_mask);
1252 cpumask_clear_cpu(smp_processor_id(), &mask);
1253
1254 if (system_state <= SYSTEM_RUNNING)
1255 pr_crit("SMP: stopping secondary CPUs\n");
1256
1257 /*
1258 * Start with a normal IPI and wait up to one second for other CPUs to
1259 * stop. We do this first because it gives other processors a chance
1260 * to exit critical sections / drop locks and makes the rest of the
1261 * stop process (especially console flush) more robust.
1262 */
1263 smp_cross_call(&mask, IPI_CPU_STOP);
1264 timeout = USEC_PER_SEC;
1265 while (num_other_online_cpus() && timeout--)
1266 udelay(1);
1267
1268 /*
1269 * If CPUs are still online, try an NMI. There's no excuse for this to
1270 * be slow, so we only give them an extra 10 ms to respond.
1271 */
1272 if (num_other_online_cpus() && ipi_should_be_nmi(IPI_CPU_STOP_NMI)) {
1273 smp_rmb();
1274 cpumask_copy(&mask, cpu_online_mask);
1275 cpumask_clear_cpu(smp_processor_id(), &mask);
1276
1277 pr_info("SMP: retry stop with NMI for CPUs %*pbl\n",
1278 cpumask_pr_args(&mask));
1279
1280 smp_cross_call(&mask, IPI_CPU_STOP_NMI);
1281 timeout = USEC_PER_MSEC * 10;
1282 while (num_other_online_cpus() && timeout--)
1283 udelay(1);
1284 }
1285
1286 /*
1287 * If CPUs are *still* online, try the SDEI cross-CPU NMI. Firmware
1288 * delivers it regardless of the target's DAIF state, so it reaches
1289 * a CPU spinning with interrupts masked, which neither rung above
1290 * could (without pseudo-NMI there is no NMI rung at all). Allow
1291 * 100ms: a firmware round-trip per CPU, with headroom.
1292 */
1293 if (num_other_online_cpus() && sdei_nmi_active()) {
1294 /* re-snapshot after the rungs above took CPUs offline */
1295 smp_rmb();
1296 cpumask_copy(&mask, cpu_online_mask);
1297 cpumask_clear_cpu(smp_processor_id(), &mask);
1298
1299 pr_info("SMP: retry stop with SDEI NMI for CPUs %*pbl\n",
1300 cpumask_pr_args(&mask));
1301
1302 sdei_nmi_stop_cpus(&mask);
1303 timeout = USEC_PER_MSEC * 100;
1304 while (num_other_online_cpus() && timeout--)
1305 udelay(1);
1306 }
1307
1308 if (num_other_online_cpus()) {
1309 smp_rmb();
1310 cpumask_copy(&mask, cpu_online_mask);
1311 cpumask_clear_cpu(smp_processor_id(), &mask);
1312
1313 pr_warn("SMP: failed to stop secondary CPUs %*pbl\n",
1314 cpumask_pr_args(&mask));
1315 }
1316
1317skip_ipi:
1318 sdei_mask_local_cpu();
1319}
1320
1321#ifdef CONFIG_KEXEC_CORE
1322void crash_smp_send_stop(void)
1323{
1324 /*
1325 * This function can be called twice in panic path, but obviously
1326 * we execute this only once.
1327 *
1328 * We use this same boolean to tell whether the IPI we send was a
1329 * stop or a "crash stop".
1330 */
1331 if (crash_stop)
1332 return;
1333 crash_stop = 1;
1334
1335 smp_send_stop();
1336
1337 sdei_handler_abort();
1338}
1339
1340bool smp_crash_stop_failed(void)
1341{
1342 return num_other_online_cpus() != 0;
1343}
1344#endif
1345
1346static bool have_cpu_die(void)
1347{
1348#ifdef CONFIG_HOTPLUG_CPU
1349 int any_cpu = raw_smp_processor_id();
1350 const struct cpu_operations *ops = get_cpu_ops(any_cpu);
1351
1352 if (ops && ops->cpu_die)
1353 return true;
1354#endif
1355 return false;
1356}
1357
1358bool cpus_are_stuck_in_kernel(void)
1359{
1360 bool smp_spin_tables = (num_possible_cpus() > 1 && !have_cpu_die());
1361
1362 return !!cpus_stuck_in_kernel || smp_spin_tables ||
1363 is_protected_kvm_enabled();
1364}