projects/linux·blob master · refs

369 lines · 7.5 KiB

1/*
2 * Copyright (C) 2014 Stefan Kristiansson <[email protected]>
3 * Copyright (C) 2017 Stafford Horne <[email protected]>
4 *
5 * Based on arm64 and arc implementations
6 * Copyright (C) 2013 ARM Ltd.
7 * Copyright (C) 2004, 2007-2010, 2011-2012 Synopsys, Inc. (www.synopsys.com)
8 *
9 * This file is licensed under the terms of the GNU General Public License
10 * version 2. This program is licensed "as is" without any warranty of any
11 * kind, whether express or implied.
12 */
13
14#include <linux/smp.h>
15#include <linux/cpu.h>
16#include <linux/interrupt.h>
17#include <linux/sched.h>
18#include <linux/sched/mm.h>
19#include <linux/irq.h>
20#include <linux/of.h>
21#include <asm/cpuinfo.h>
22#include <asm/mmu_context.h>
23#include <asm/tlbflush.h>
24#include <asm/cacheflush.h>
25#include <asm/time.h>
26
27asmlinkage __init void secondary_start_kernel(void);
28
29static unsigned int ipi_irq __ro_after_init;
30static void (*smp_cross_call)(const struct cpumask *, unsigned int);
31
32unsigned long secondary_release = -1;
33struct thread_info *secondary_thread_info;
34
35enum ipi_msg_type {
36 IPI_WAKEUP,
37 IPI_RESCHEDULE,
38 IPI_CALL_FUNC,
39 IPI_CALL_FUNC_SINGLE,
40};
41
42static DEFINE_SPINLOCK(boot_lock);
43
44static void or1k_ipi_enable(void)
45{
46 if (WARN_ON_ONCE(!ipi_irq))
47 return;
48
49 enable_percpu_irq(ipi_irq, 0);
50}
51
52static void boot_secondary(unsigned int cpu, struct task_struct *idle)
53{
54 /*
55 * set synchronisation state between this boot processor
56 * and the secondary one
57 */
58 spin_lock(&boot_lock);
59
60 secondary_release = cpu;
61 smp_cross_call(cpumask_of(cpu), IPI_WAKEUP);
62
63 /*
64 * now the secondary core is starting up let it run its
65 * calibrations, then wait for it to finish
66 */
67 spin_unlock(&boot_lock);
68}
69
70void __init smp_init_cpus(void)
71{
72 struct device_node *cpu;
73 u32 cpu_id;
74
75 for_each_of_cpu_node(cpu) {
76 cpu_id = of_get_cpu_hwid(cpu, 0);
77 if (cpu_id < NR_CPUS)
78 set_cpu_possible(cpu_id, true);
79 }
80}
81
82void __init smp_prepare_cpus(unsigned int max_cpus)
83{
84 unsigned int cpu;
85
86 /*
87 * Initialise the present map, which describes the set of CPUs
88 * actually populated at the present time.
89 */
90 for_each_possible_cpu(cpu) {
91 if (cpu < max_cpus)
92 set_cpu_present(cpu, true);
93 }
94}
95
96void __init smp_cpus_done(unsigned int max_cpus)
97{
98}
99
100static DECLARE_COMPLETION(cpu_running);
101
102int __cpu_up(unsigned int cpu, struct task_struct *idle)
103{
104 if (smp_cross_call == NULL) {
105 pr_warn("CPU%u: failed to start, IPI controller missing",
106 cpu);
107 return -EIO;
108 }
109
110 secondary_thread_info = task_thread_info(idle);
111 current_pgd[cpu] = init_mm.pgd;
112
113 boot_secondary(cpu, idle);
114 if (!wait_for_completion_timeout(&cpu_running,
115 msecs_to_jiffies(1000))) {
116 pr_crit("CPU%u: failed to start\n", cpu);
117 return -EIO;
118 }
119 synchronise_count_master(cpu);
120
121 return 0;
122}
123
124asmlinkage __init void secondary_start_kernel(void)
125{
126 struct mm_struct *mm = &init_mm;
127 unsigned int cpu = smp_processor_id();
128 /*
129 * All kernel threads share the same mm context; grab a
130 * reference and switch to it.
131 */
132 mmgrab(mm);
133 current->active_mm = mm;
134 cpumask_set_cpu(cpu, mm_cpumask(mm));
135
136 pr_info("CPU%u: Booted secondary processor\n", cpu);
137
138 setup_cpuinfo();
139 openrisc_clockevent_init();
140
141 notify_cpu_starting(cpu);
142
143 /*
144 * OK, now it's safe to let the boot CPU continue
145 */
146 complete(&cpu_running);
147
148 synchronise_count_slave(cpu);
149 or1k_ipi_enable();
150 set_cpu_online(cpu, true);
151
152 local_irq_enable();
153 /*
154 * OK, it's off to the idle thread for us
155 */
156 cpu_startup_entry(CPUHP_AP_ONLINE_IDLE);
157}
158
159void handle_IPI(unsigned int ipi_msg)
160{
161 unsigned int cpu = smp_processor_id();
162
163 switch (ipi_msg) {
164 case IPI_WAKEUP:
165 break;
166
167 case IPI_RESCHEDULE:
168 scheduler_ipi();
169 break;
170
171 case IPI_CALL_FUNC:
172 generic_smp_call_function_interrupt();
173 break;
174
175 case IPI_CALL_FUNC_SINGLE:
176 generic_smp_call_function_single_interrupt();
177 break;
178
179 default:
180 WARN(1, "CPU%u: Unknown IPI message 0x%x\n", cpu, ipi_msg);
181 break;
182 }
183}
184
185void arch_smp_send_reschedule(int cpu)
186{
187 smp_cross_call(cpumask_of(cpu), IPI_RESCHEDULE);
188}
189
190static void stop_this_cpu(void *dummy)
191{
192 /* Remove this CPU */
193 set_cpu_online(smp_processor_id(), false);
194
195 local_irq_disable();
196 /* CPU Doze */
197 if (mfspr(SPR_UPR) & SPR_UPR_PMP)
198 mtspr(SPR_PMR, mfspr(SPR_PMR) | SPR_PMR_DME);
199 /* If that didn't work, infinite loop */
200 while (1)
201 ;
202}
203
204void smp_send_stop(void)
205{
206 smp_call_function(stop_this_cpu, NULL, 0);
207}
208
209void __init set_smp_cross_call(void (*fn)(const struct cpumask *, unsigned int),
210 unsigned int irq)
211{
212 if (WARN_ON(ipi_irq))
213 return;
214
215 smp_cross_call = fn;
216
217 ipi_irq = irq;
218
219 /* Enabled IPIs for boot CPU immediately */
220 or1k_ipi_enable();
221}
222
223void arch_send_call_function_single_ipi(int cpu)
224{
225 smp_cross_call(cpumask_of(cpu), IPI_CALL_FUNC_SINGLE);
226}
227
228void arch_send_call_function_ipi_mask(const struct cpumask *mask)
229{
230 smp_cross_call(mask, IPI_CALL_FUNC);
231}
232
233/* TLB flush operations - Performed on each CPU*/
234static inline void ipi_flush_tlb_all(void *ignored)
235{
236 local_flush_tlb_all();
237}
238
239static inline void ipi_flush_tlb_mm(void *info)
240{
241 struct mm_struct *mm = (struct mm_struct *)info;
242
243 local_flush_tlb_mm(mm);
244}
245
246static void smp_flush_tlb_mm(struct cpumask *cmask, struct mm_struct *mm)
247{
248 unsigned int cpuid;
249
250 if (cpumask_empty(cmask))
251 return;
252
253 cpuid = get_cpu();
254
255 if (cpumask_any_but(cmask, cpuid) >= nr_cpu_ids) {
256 /* local cpu is the only cpu present in cpumask */
257 local_flush_tlb_mm(mm);
258 } else {
259 on_each_cpu_mask(cmask, ipi_flush_tlb_mm, mm, 1);
260 }
261 put_cpu();
262}
263
264struct flush_tlb_data {
265 unsigned long addr1;
266 unsigned long addr2;
267};
268
269static inline void ipi_flush_tlb_page(void *info)
270{
271 struct flush_tlb_data *fd = (struct flush_tlb_data *)info;
272
273 local_flush_tlb_page(NULL, fd->addr1);
274}
275
276static inline void ipi_flush_tlb_range(void *info)
277{
278 struct flush_tlb_data *fd = (struct flush_tlb_data *)info;
279
280 local_flush_tlb_range(NULL, fd->addr1, fd->addr2);
281}
282
283static void smp_flush_tlb_range(const struct cpumask *cmask, unsigned long start,
284 unsigned long end)
285{
286 unsigned int cpuid;
287
288 if (cpumask_empty(cmask))
289 return;
290
291 cpuid = get_cpu();
292
293 if (cpumask_any_but(cmask, cpuid) >= nr_cpu_ids) {
294 /* local cpu is the only cpu present in cpumask */
295 if ((end - start) <= PAGE_SIZE)
296 local_flush_tlb_page(NULL, start);
297 else
298 local_flush_tlb_range(NULL, start, end);
299 } else {
300 struct flush_tlb_data fd;
301
302 fd.addr1 = start;
303 fd.addr2 = end;
304
305 if ((end - start) <= PAGE_SIZE)
306 on_each_cpu_mask(cmask, ipi_flush_tlb_page, &fd, 1);
307 else
308 on_each_cpu_mask(cmask, ipi_flush_tlb_range, &fd, 1);
309 }
310 put_cpu();
311}
312
313void flush_tlb_all(void)
314{
315 on_each_cpu(ipi_flush_tlb_all, NULL, 1);
316}
317
318void flush_tlb_mm(struct mm_struct *mm)
319{
320 smp_flush_tlb_mm(mm_cpumask(mm), mm);
321}
322
323void flush_tlb_page(struct vm_area_struct *vma, unsigned long uaddr)
324{
325 smp_flush_tlb_range(mm_cpumask(vma->vm_mm), uaddr, uaddr + PAGE_SIZE);
326}
327
328void flush_tlb_range(struct vm_area_struct *vma,
329 unsigned long start, unsigned long end)
330{
331 const struct cpumask *cmask = vma ? mm_cpumask(vma->vm_mm)
332 : cpu_online_mask;
333 smp_flush_tlb_range(cmask, start, end);
334}
335
336/* Instruction cache invalidate - performed on each cpu */
337static void ipi_icache_page_inv(void *arg)
338{
339 struct page *page = arg;
340
341 local_icache_page_inv(page);
342}
343
344void smp_icache_page_inv(struct page *page)
345{
346 on_each_cpu(ipi_icache_page_inv, page, 1);
347}
348EXPORT_SYMBOL(smp_icache_page_inv);
349
350static void ipi_icache_all_inv(void *arg)
351{
352 local_icache_all_inv();
353}
354
355void smp_icache_all_inv(void)
356{
357 if (num_online_cpus() < 2) {
358 local_icache_all_inv();
359 return;
360 }
361
362 /*
363 * Ensure stores complete before we request remote icaches
364 * to invalidate.
365 */
366 mb();
367
368 on_each_cpu(ipi_icache_all_inv, NULL, 1);
369}