projects/linux·blob master · refs

769 lines · 16.4 KiB

1// SPDX-License-Identifier: GPL-2.0
2/*
3 * linux/arch/alpha/kernel/smp.c
4 *
5 * 2001-07-09 Phil Ezolt ([email protected])
6 * Renamed modified smp_call_function to smp_call_function_on_cpu()
7 * Created an function that conforms to the old calling convention
8 * of smp_call_function().
9 *
10 * This is helpful for DCPI.
11 *
12 */
13
14#include <linux/errno.h>
15#include <linux/kernel.h>
16#include <linux/kernel_stat.h>
17#include <linux/module.h>
18#include <linux/sched/mm.h>
19#include <linux/mm.h>
20#include <linux/err.h>
21#include <linux/threads.h>
22#include <linux/smp.h>
23#include <linux/interrupt.h>
24#include <linux/init.h>
25#include <linux/delay.h>
26#include <linux/spinlock.h>
27#include <linux/irq.h>
28#include <linux/cache.h>
29#include <linux/profile.h>
30#include <linux/bitops.h>
31#include <linux/cpu.h>
32
33#include <asm/hwrpb.h>
34#include <asm/ptrace.h>
35#include <linux/atomic.h>
36
37#include <asm/io.h>
38#include <asm/irq.h>
39#include <asm/mmu_context.h>
40#include <asm/tlbflush.h>
41#include <asm/cacheflush.h>
42
43#include "proto.h"
44#include "irq_impl.h"
45
46
47#define DEBUG_SMP 0
48#if DEBUG_SMP
49#define DBGS(args) printk args
50#else
51#define DBGS(args)
52#endif
53
54/* A collection of per-processor data. */
55struct cpuinfo_alpha cpu_data[NR_CPUS];
56EXPORT_SYMBOL(cpu_data);
57
58/* A collection of single bit ipi messages. */
59static struct {
60 unsigned long bits ____cacheline_aligned;
61} ipi_data[NR_CPUS] __cacheline_aligned;
62
63enum ipi_message_type {
64 IPI_RESCHEDULE,
65 IPI_CALL_FUNC,
66 IPI_CPU_STOP,
67};
68
69/* Set to a secondary's cpuid when it comes online. */
70static int smp_secondary_alive = 0;
71
72int smp_num_probed; /* Internal processor count */
73int smp_num_cpus = 1; /* Number that came online. */
74EXPORT_SYMBOL(smp_num_cpus);
75
76/*
77 * Called by both boot and secondaries to move global data into
78 * per-processor storage.
79 */
80static inline void __init
81smp_store_cpu_info(int cpuid)
82{
83 cpu_data[cpuid].loops_per_jiffy = loops_per_jiffy;
84 cpu_data[cpuid].last_asn = ASN_FIRST_VERSION;
85 cpu_data[cpuid].need_new_asn = 0;
86 cpu_data[cpuid].asn_lock = 0;
87}
88
89/*
90 * Ideally sets up per-cpu profiling hooks. Doesn't do much now...
91 */
92static inline void __init
93smp_setup_percpu_timer(int cpuid)
94{
95 cpu_data[cpuid].prof_counter = 1;
96 cpu_data[cpuid].prof_multiplier = 1;
97}
98
99static void __init
100wait_boot_cpu_to_stop(int cpuid)
101{
102 unsigned long stop = jiffies + 10*HZ;
103
104 while (time_before(jiffies, stop)) {
105 if (!smp_secondary_alive)
106 return;
107 barrier();
108 }
109
110 printk("wait_boot_cpu_to_stop: FAILED on CPU %d, hanging now\n", cpuid);
111 for (;;)
112 barrier();
113}
114
115/*
116 * Where secondaries begin a life of C.
117 */
118void __init
119smp_callin(void)
120{
121 int cpuid = hard_smp_processor_id();
122
123 if (cpu_online(cpuid)) {
124 printk("??, cpu 0x%x already present??\n", cpuid);
125 BUG();
126 }
127 set_cpu_online(cpuid, true);
128
129 /* Turn on machine checks. */
130 wrmces(7);
131
132 /* Set trap vectors. */
133 trap_init();
134
135 /* Set interrupt vector. */
136 wrent(entInt, 0);
137
138 /* Get our local ticker going. */
139 smp_setup_percpu_timer(cpuid);
140 init_clockevent();
141
142 /* Call platform-specific callin, if specified */
143 if (alpha_mv.smp_callin)
144 alpha_mv.smp_callin();
145
146 /* All kernel threads share the same mm context. */
147 mmgrab(&init_mm);
148 current->active_mm = &init_mm;
149
150 /* inform the notifiers about the new cpu */
151 notify_cpu_starting(cpuid);
152
153 /* Must have completely accurate bogos. */
154 local_irq_enable();
155
156 /* Wait boot CPU to stop with irq enabled before running
157 calibrate_delay. */
158 wait_boot_cpu_to_stop(cpuid);
159 mb();
160 calibrate_delay();
161
162 smp_store_cpu_info(cpuid);
163 /* Allow master to continue only after we written loops_per_jiffy. */
164 wmb();
165 smp_secondary_alive = 1;
166
167 DBGS(("smp_callin: commencing CPU %d current %p active_mm %p\n",
168 cpuid, current, current->active_mm));
169
170 cpu_startup_entry(CPUHP_AP_ONLINE_IDLE);
171}
172
173/* Wait until hwrpb->txrdy is clear for cpu. Return -1 on timeout. */
174static int
175wait_for_txrdy (unsigned long cpumask)
176{
177 unsigned long timeout;
178
179 if (!(hwrpb->txrdy & cpumask))
180 return 0;
181
182 timeout = jiffies + 10*HZ;
183 while (time_before(jiffies, timeout)) {
184 if (!(hwrpb->txrdy & cpumask))
185 return 0;
186 udelay(10);
187 barrier();
188 }
189
190 return -1;
191}
192
193/*
194 * Send a message to a secondary's console. "START" is one such
195 * interesting message. ;-)
196 */
197static void
198send_secondary_console_msg(char *str, int cpuid)
199{
200 struct percpu_struct *cpu;
201 register char *cp1, *cp2;
202 unsigned long cpumask;
203 size_t len;
204
205 cpu = (struct percpu_struct *)
206 ((char*)hwrpb
207 + hwrpb->processor_offset
208 + cpuid * hwrpb->processor_size);
209
210 cpumask = (1UL << cpuid);
211 if (wait_for_txrdy(cpumask))
212 goto timeout;
213
214 cp2 = str;
215 len = strlen(cp2);
216 *(unsigned int *)&cpu->ipc_buffer[0] = len;
217 cp1 = (char *) &cpu->ipc_buffer[1];
218 memcpy(cp1, cp2, len);
219
220 /* atomic test and set */
221 wmb();
222 set_bit(cpuid, &hwrpb->rxrdy);
223
224 if (wait_for_txrdy(cpumask))
225 goto timeout;
226 return;
227
228 timeout:
229 printk("Processor %x not ready\n", cpuid);
230}
231
232/*
233 * A secondary console wants to send a message. Receive it.
234 */
235static void
236recv_secondary_console_msg(void)
237{
238 int mycpu, i, cnt;
239 unsigned long txrdy = hwrpb->txrdy;
240 char *cp1, *cp2, buf[80];
241 struct percpu_struct *cpu;
242
243 DBGS(("recv_secondary_console_msg: TXRDY 0x%lx.\n", txrdy));
244
245 mycpu = hard_smp_processor_id();
246
247 for (i = 0; i < NR_CPUS; i++) {
248 if (!(txrdy & (1UL << i)))
249 continue;
250
251 DBGS(("recv_secondary_console_msg: "
252 "TXRDY contains CPU %d.\n", i));
253
254 cpu = (struct percpu_struct *)
255 ((char*)hwrpb
256 + hwrpb->processor_offset
257 + i * hwrpb->processor_size);
258
259 DBGS(("recv_secondary_console_msg: on %d from %d"
260 " HALT_REASON 0x%lx FLAGS 0x%lx\n",
261 mycpu, i, cpu->halt_reason, cpu->flags));
262
263 cnt = cpu->ipc_buffer[0] >> 32;
264 if (cnt <= 0 || cnt >= 80)
265 strcpy(buf, "<<< BOGUS MSG >>>");
266 else {
267 cp1 = (char *) &cpu->ipc_buffer[1];
268 cp2 = buf;
269 memcpy(cp2, cp1, cnt);
270 cp2[cnt] = '\0';
271
272 while ((cp2 = strchr(cp2, '\r')) != 0) {
273 *cp2 = ' ';
274 if (cp2[1] == '\n')
275 cp2[1] = ' ';
276 }
277 }
278
279 DBGS((KERN_INFO "recv_secondary_console_msg: on %d "
280 "message is '%s'\n", mycpu, buf));
281 }
282
283 hwrpb->txrdy = 0;
284}
285
286/*
287 * Convince the console to have a secondary cpu begin execution.
288 */
289static int
290secondary_cpu_start(int cpuid, struct task_struct *idle)
291{
292 struct percpu_struct *cpu;
293 struct pcb_struct *hwpcb, *ipcb;
294 unsigned long timeout;
295
296 cpu = (struct percpu_struct *)
297 ((char*)hwrpb
298 + hwrpb->processor_offset
299 + cpuid * hwrpb->processor_size);
300 hwpcb = (struct pcb_struct *) cpu->hwpcb;
301 ipcb = &task_thread_info(idle)->pcb;
302
303 /* Initialize the CPU's HWPCB to something just good enough for
304 us to get started. Immediately after starting, we'll swpctx
305 to the target idle task's pcb. Reuse the stack in the mean
306 time. Precalculate the target PCBB. */
307 hwpcb->ksp = (unsigned long)ipcb + sizeof(union thread_union) - 16;
308 hwpcb->usp = 0;
309 hwpcb->ptbr = ipcb->ptbr;
310 hwpcb->pcc = 0;
311 hwpcb->asn = 0;
312 hwpcb->unique = virt_to_phys(ipcb);
313 hwpcb->flags = ipcb->flags;
314 hwpcb->res1 = hwpcb->res2 = 0;
315
316#if 0
317 DBGS(("KSP 0x%lx PTBR 0x%lx VPTBR 0x%lx UNIQUE 0x%lx\n",
318 hwpcb->ksp, hwpcb->ptbr, hwrpb->vptb, hwpcb->unique));
319#endif
320 DBGS(("Starting secondary cpu %d: state 0x%lx pal_flags 0x%lx\n",
321 cpuid, idle->state, ipcb->flags));
322
323 /* Setup HWRPB fields that SRM uses to activate secondary CPU */
324 hwrpb->CPU_restart = __smp_callin;
325 hwrpb->CPU_restart_data = (unsigned long) __smp_callin;
326
327 /* Recalculate and update the HWRPB checksum */
328 hwrpb_update_checksum(hwrpb);
329
330 /*
331 * Send a "start" command to the specified processor.
332 */
333
334 /* SRM III 3.4.1.3 */
335 cpu->flags |= 0x22; /* turn on Context Valid and Restart Capable */
336 cpu->flags &= ~1; /* turn off Bootstrap In Progress */
337 wmb();
338
339 send_secondary_console_msg("START\r\n", cpuid);
340
341 /* Wait 10 seconds for an ACK from the console. */
342 timeout = jiffies + 10*HZ;
343 while (time_before(jiffies, timeout)) {
344 if (cpu->flags & 1)
345 goto started;
346 udelay(10);
347 barrier();
348 }
349 printk(KERN_ERR "SMP: Processor %d failed to start.\n", cpuid);
350 return -1;
351
352 started:
353 DBGS(("secondary_cpu_start: SUCCESS for CPU %d!!!\n", cpuid));
354 return 0;
355}
356
357/*
358 * Bring one cpu online.
359 */
360static int
361smp_boot_one_cpu(int cpuid, struct task_struct *idle)
362{
363 unsigned long timeout;
364
365 /* Signal the secondary to wait a moment. */
366 smp_secondary_alive = -1;
367
368 /* Whirrr, whirrr, whirrrrrrrrr... */
369 if (secondary_cpu_start(cpuid, idle))
370 return -1;
371
372 /* Notify the secondary CPU it can run calibrate_delay. */
373 mb();
374 smp_secondary_alive = 0;
375
376 /* We've been acked by the console; wait one second for
377 the task to start up for real. */
378 timeout = jiffies + 1*HZ;
379 while (time_before(jiffies, timeout)) {
380 if (smp_secondary_alive == 1)
381 goto alive;
382 udelay(10);
383 barrier();
384 }
385
386 /* We failed to boot the CPU. */
387
388 printk(KERN_ERR "SMP: Processor %d is stuck.\n", cpuid);
389 return -1;
390
391 alive:
392 /* Another "Red Snapper". */
393 return 0;
394}
395
396/*
397 * Called from setup_arch. Detect an SMP system and which processors
398 * are present.
399 */
400void __init
401setup_smp(void)
402{
403 struct percpu_struct *cpubase, *cpu;
404 unsigned long i;
405
406 if (boot_cpuid != 0) {
407 printk(KERN_WARNING "SMP: Booting off cpu %d instead of 0?\n",
408 boot_cpuid);
409 }
410
411 if (hwrpb->nr_processors > 1) {
412 int boot_cpu_palrev;
413
414 DBGS(("setup_smp: nr_processors %ld\n",
415 hwrpb->nr_processors));
416
417 cpubase = (struct percpu_struct *)
418 ((char*)hwrpb + hwrpb->processor_offset);
419 boot_cpu_palrev = cpubase->pal_revision;
420
421 for (i = 0; i < hwrpb->nr_processors; i++) {
422 cpu = (struct percpu_struct *)
423 ((char *)cpubase + i*hwrpb->processor_size);
424 if ((cpu->flags & 0x1cc) == 0x1cc) {
425 smp_num_probed++;
426 set_cpu_possible(i, true);
427 set_cpu_present(i, true);
428 cpu->pal_revision = boot_cpu_palrev;
429 }
430
431 DBGS(("setup_smp: CPU %d: flags 0x%lx type 0x%lx\n",
432 i, cpu->flags, cpu->type));
433 DBGS(("setup_smp: CPU %d: PAL rev 0x%lx\n",
434 i, cpu->pal_revision));
435 }
436 } else {
437 smp_num_probed = 1;
438 }
439
440 printk(KERN_INFO "SMP: %d CPUs probed -- cpu_present_mask = %lx\n",
441 smp_num_probed, cpumask_bits(cpu_present_mask)[0]);
442}
443
444/*
445 * Called by smp_init prepare the secondaries
446 */
447void __init
448smp_prepare_cpus(unsigned int max_cpus)
449{
450 /* Take care of some initial bookkeeping. */
451 memset(ipi_data, 0, sizeof(ipi_data));
452
453 current_thread_info()->cpu = boot_cpuid;
454
455 smp_store_cpu_info(boot_cpuid);
456 smp_setup_percpu_timer(boot_cpuid);
457
458 /* Nothing to do on a UP box, or when told not to. */
459 if (smp_num_probed == 1 || max_cpus == 0) {
460 init_cpu_possible(cpumask_of(boot_cpuid));
461 init_cpu_present(cpumask_of(boot_cpuid));
462 printk(KERN_INFO "SMP mode deactivated.\n");
463 return;
464 }
465
466 printk(KERN_INFO "SMP starting up secondaries.\n");
467
468 smp_num_cpus = smp_num_probed;
469}
470
471int
472__cpu_up(unsigned int cpu, struct task_struct *tidle)
473{
474 smp_boot_one_cpu(cpu, tidle);
475
476 return cpu_online(cpu) ? 0 : -ENOSYS;
477}
478
479void __init
480smp_cpus_done(unsigned int max_cpus)
481{
482 int cpu;
483 unsigned long bogosum = 0;
484
485 for(cpu = 0; cpu < NR_CPUS; cpu++)
486 if (cpu_online(cpu))
487 bogosum += cpu_data[cpu].loops_per_jiffy;
488
489 printk(KERN_INFO "SMP: Total of %d processors activated "
490 "(%lu.%02lu BogoMIPS).\n",
491 num_online_cpus(),
492 (bogosum + 2500) / (500000/HZ),
493 ((bogosum + 2500) / (5000/HZ)) % 100);
494}
495
496static void
497send_ipi_message(const struct cpumask *to_whom, enum ipi_message_type operation)
498{
499 int i;
500
501 mb();
502 for_each_cpu(i, to_whom)
503 set_bit(operation, &ipi_data[i].bits);
504
505 mb();
506 for_each_cpu(i, to_whom)
507 wripir(i);
508}
509
510void
511handle_ipi(struct pt_regs *regs)
512{
513 int this_cpu = smp_processor_id();
514 unsigned long *pending_ipis = &ipi_data[this_cpu].bits;
515 unsigned long ops;
516
517#if 0
518 DBGS(("handle_ipi: on CPU %d ops 0x%lx PC 0x%lx\n",
519 this_cpu, *pending_ipis, regs->pc));
520#endif
521
522 mb(); /* Order interrupt and bit testing. */
523 while ((ops = xchg(pending_ipis, 0)) != 0) {
524 mb(); /* Order bit clearing and data access. */
525 do {
526 unsigned long which;
527
528 which = ops & -ops;
529 ops &= ~which;
530 which = __ffs(which);
531
532 switch (which) {
533 case IPI_RESCHEDULE:
534 scheduler_ipi();
535 break;
536
537 case IPI_CALL_FUNC:
538 generic_smp_call_function_interrupt();
539 break;
540
541 case IPI_CPU_STOP:
542 halt();
543
544 default:
545 printk(KERN_CRIT "Unknown IPI on CPU %d: %lu\n",
546 this_cpu, which);
547 break;
548 }
549 } while (ops);
550
551 mb(); /* Order data access and bit testing. */
552 }
553
554 cpu_data[this_cpu].ipi_count++;
555
556 if (hwrpb->txrdy)
557 recv_secondary_console_msg();
558}
559
560void
561arch_smp_send_reschedule(int cpu)
562{
563#ifdef DEBUG_IPI_MSG
564 if (cpu == hard_smp_processor_id())
565 printk(KERN_WARNING
566 "smp_send_reschedule: Sending IPI to self.\n");
567#endif
568 send_ipi_message(cpumask_of(cpu), IPI_RESCHEDULE);
569}
570
571void
572smp_send_stop(void)
573{
574 cpumask_t to_whom;
575 cpumask_copy(&to_whom, cpu_online_mask);
576 cpumask_clear_cpu(smp_processor_id(), &to_whom);
577#ifdef DEBUG_IPI_MSG
578 if (hard_smp_processor_id() != boot_cpu_id)
579 printk(KERN_WARNING "smp_send_stop: Not on boot cpu.\n");
580#endif
581 send_ipi_message(&to_whom, IPI_CPU_STOP);
582}
583
584void arch_send_call_function_ipi_mask(const struct cpumask *mask)
585{
586 send_ipi_message(mask, IPI_CALL_FUNC);
587}
588
589void arch_send_call_function_single_ipi(int cpu)
590{
591 send_ipi_message(cpumask_of(cpu), IPI_CALL_FUNC);
592}
593
594static void
595ipi_imb(void *ignored)
596{
597 imb();
598}
599
600void
601smp_imb(void)
602{
603 /* Must wait other processors to flush their icache before continue. */
604 on_each_cpu(ipi_imb, NULL, 1);
605}
606EXPORT_SYMBOL(smp_imb);
607
608static void
609ipi_flush_tlb_all(void *ignored)
610{
611 tbia();
612}
613
614void
615flush_tlb_all(void)
616{
617 /* Although we don't have any data to pass, we do want to
618 synchronize with the other processors. */
619 on_each_cpu(ipi_flush_tlb_all, NULL, 1);
620}
621
622#define asn_locked() (cpu_data[smp_processor_id()].asn_lock)
623
624static void
625ipi_flush_tlb_mm(void *x)
626{
627 struct mm_struct *mm = x;
628 if (mm == current->mm && !asn_locked())
629 flush_tlb_current(mm);
630 else
631 flush_tlb_other(mm);
632}
633
634/* True if a CPU other than this one holds an ASN for MM. */
635static bool
636mm_context_elsewhere(struct mm_struct *mm)
637{
638 int cpu, this_cpu = smp_processor_id();
639
640 /* Pairs with the barrier the publishing CPU issues before using MM. */
641 smp_mb();
642
643 for_each_online_cpu(cpu) {
644 if (cpu == this_cpu)
645 continue;
646 if (READ_ONCE(mm->context[cpu]))
647 return true;
648 }
649 return false;
650}
651
652void
653flush_tlb_mm(struct mm_struct *mm)
654{
655 preempt_disable();
656
657 if (mm == current->active_mm) {
658 flush_tlb_current(mm);
659 if (atomic_read(&mm->mm_users) <= 1 &&
660 !mm_context_elsewhere(mm)) {
661 preempt_enable();
662 return;
663 }
664 } else {
665 /* smp_call_function() does not call back into this CPU. */
666 flush_tlb_other(mm);
667 }
668
669 smp_call_function(ipi_flush_tlb_mm, mm, 1);
670
671 preempt_enable();
672}
673EXPORT_SYMBOL(flush_tlb_mm);
674
675struct flush_tlb_page_struct {
676 struct vm_area_struct *vma;
677 struct mm_struct *mm;
678 unsigned long addr;
679};
680
681static void
682ipi_flush_tlb_page(void *x)
683{
684 struct flush_tlb_page_struct *data = x;
685 struct mm_struct * mm = data->mm;
686
687 /* A targeted tbi() needs a thread of MM to be current. */
688 if (mm == current->mm && !asn_locked())
689 flush_tlb_current_page(mm, data->vma, data->addr);
690 else
691 flush_tlb_other(mm);
692}
693
694void
695flush_tlb_page(struct vm_area_struct *vma, unsigned long addr)
696{
697 struct flush_tlb_page_struct data;
698 struct mm_struct *mm = vma->vm_mm;
699
700 preempt_disable();
701
702 /* As in ipi_flush_tlb_page(): a targeted tbi() needs MM current. */
703 if (mm == current->mm) {
704 flush_tlb_current_page(mm, vma, addr);
705 if (atomic_read(&mm->mm_users) <= 1 &&
706 !mm_context_elsewhere(mm)) {
707 preempt_enable();
708 return;
709 }
710 } else {
711 /* smp_call_function() does not call back into this CPU. */
712 flush_tlb_other(mm);
713 }
714
715 data.vma = vma;
716 data.mm = mm;
717 data.addr = addr;
718
719 smp_call_function(ipi_flush_tlb_page, &data, 1);
720
721 preempt_enable();
722}
723EXPORT_SYMBOL(flush_tlb_page);
724
725void
726flush_tlb_range(struct vm_area_struct *vma, unsigned long start, unsigned long end)
727{
728 /* On the Alpha we always flush the whole user tlb. */
729 flush_tlb_mm(vma->vm_mm);
730}
731EXPORT_SYMBOL(flush_tlb_range);
732
733static void
734ipi_flush_icache_page(void *x)
735{
736 struct mm_struct *mm = (struct mm_struct *) x;
737 if (mm == current->mm && !asn_locked())
738 __load_new_mm_context(mm);
739 else
740 flush_tlb_other(mm);
741}
742
743void
744flush_icache_user_page(struct vm_area_struct *vma, struct page *page,
745 unsigned long addr, int len)
746{
747 struct mm_struct *mm = vma->vm_mm;
748
749 if ((vma->vm_flags & VM_EXEC) == 0)
750 return;
751
752 preempt_disable();
753
754 if (mm == current->active_mm) {
755 __load_new_mm_context(mm);
756 if (atomic_read(&mm->mm_users) <= 1 &&
757 !mm_context_elsewhere(mm)) {
758 preempt_enable();
759 return;
760 }
761 } else {
762 /* smp_call_function() does not call back into this CPU. */
763 flush_tlb_other(mm);
764 }
765
766 smp_call_function(ipi_flush_icache_page, mm, 1);
767
768 preempt_enable();
769}