projects/linux·blob master · refs

616 lines · 12.0 KiB

1/* SPDX-License-Identifier: GPL-2.0-or-later */
2/*
3 * Copyright (C) 2013 Imagination Technologies
4 * Author: Paul Burton <paul.burton@mips.com>
5 */
6
7#include <linux/init.h>
8#include <asm/addrspace.h>
9#include <asm/asm.h>
10#include <asm/asm-offsets.h>
11#include <asm/asmmacro.h>
12#include <asm/cacheops.h>
13#include <asm/eva.h>
14#include <asm/mipsregs.h>
15#include <asm/mipsmtregs.h>
16#include <asm/pm.h>
17#include <asm/smp-cps.h>
18
19#define GCR_CPC_BASE_OFS 0x0088
20#define GCR_CL_COHERENCE_OFS 0x2008
21#define GCR_CL_ID_OFS 0x2028
22#define CM3_GCR_Cx_ID_CLUSTER_SHF 8
23#define CM3_GCR_Cx_ID_CLUSTER_MSK (0xff << 8)
24#define CM3_GCR_Cx_ID_CORENUM_SHF 0
25#define CM3_GCR_Cx_ID_CORENUM_MSK (0xff << 0)
26
27#define CPC_CL_VC_STOP_OFS 0x2020
28#define CPC_CL_VC_RUN_OFS 0x2028
29
30.extern mips_cm_base
31
32.set noreorder
33
34#ifdef CONFIG_64BIT
35# define STATUS_BITDEPS ST0_KX
36#else
37# define STATUS_BITDEPS 0
38#endif
39
40#ifdef CONFIG_MIPS_CPS_NS16550
41
42#define DUMP_EXCEP(name) \
43 PTR_LA a0, 8f; \
44 jal mips_cps_bev_dump; \
45 nop; \
46 TEXT(name)
47
48#else /* !CONFIG_MIPS_CPS_NS16550 */
49
50#define DUMP_EXCEP(name)
51
52#endif /* !CONFIG_MIPS_CPS_NS16550 */
53
54 /*
55 * Set dest to non-zero if the core supports the MT ASE, else zero. If
56 * MT is not supported then branch to nomt.
57 */
58 .macro has_mt dest, nomt
59 mfc0 \dest, CP0_CONFIG, 1
60 bgez \dest, \nomt
61 mfc0 \dest, CP0_CONFIG, 2
62 bgez \dest, \nomt
63 mfc0 \dest, CP0_CONFIG, 3
64 andi \dest, \dest, MIPS_CONF3_MT
65 beqz \dest, \nomt
66 nop
67 .endm
68
69 /*
70 * Set dest to non-zero if the core supports MIPSr6 multithreading
71 * (ie. VPs), else zero. If MIPSr6 multithreading is not supported then
72 * branch to nomt.
73 */
74 .macro has_vp dest, nomt
75 mfc0 \dest, CP0_CONFIG, 1
76 bgez \dest, \nomt
77 mfc0 \dest, CP0_CONFIG, 2
78 bgez \dest, \nomt
79 mfc0 \dest, CP0_CONFIG, 3
80 bgez \dest, \nomt
81 mfc0 \dest, CP0_CONFIG, 4
82 bgez \dest, \nomt
83 mfc0 \dest, CP0_CONFIG, 5
84 andi \dest, \dest, MIPS_CONF5_VP
85 beqz \dest, \nomt
86 nop
87 .endm
88
89
90LEAF(mips_cps_core_boot)
91 /* Save CCA and GCR base */
92 move s0, a0
93 move s1, a1
94
95 /* We don't know how to do coherence setup on earlier ISA */
96#if MIPS_ISA_REV > 0
97 /* Skip cache & coherence setup if we're already coherent */
98 lw s7, GCR_CL_COHERENCE_OFS(s1)
99 bnez s7, 1f
100 nop
101
102 /* Initialize the L1 caches */
103 jal mips_cps_cache_init
104 nop
105
106 /* Enter the coherent domain */
107 li t0, 0xff
108 sw t0, GCR_CL_COHERENCE_OFS(s1)
109 ehb
110#endif /* MIPS_ISA_REV > 0 */
111
112 /* Set Kseg0 CCA to that in s0 */
1131: mfc0 t0, CP0_CONFIG
114 ori t0, 0x7
115 xori t0, 0x7
116 or t0, t0, s0
117 mtc0 t0, CP0_CONFIG
118 ehb
119
120 /* Jump to kseg0 */
121 PTR_LA t0, 1f
122 jr t0
123 nop
124
125 /*
126 * We're up, cached & coherent. Perform any EVA initialization necessary
127 * before we access memory.
128 */
1291: eva_init
130
131 /* Retrieve boot configuration pointers */
132 jal mips_cps_get_bootcfg
133 nop
134
135 /* Skip core-level init if we started up coherent */
136 bnez s7, 1f
137 nop
138
139 /* Perform any further required core-level initialisation */
140 jal mips_cps_core_init
141 nop
142
143 /*
144 * Boot any other VPEs within this core that should be online, and
145 * deactivate this VPE if it should be offline.
146 */
147 move a1, t9
148 jal mips_cps_boot_vpes
149 move a0, v0
150
151 /* Off we go! */
1521: PTR_L t1, VPEBOOTCFG_PC(v1)
153 PTR_L gp, VPEBOOTCFG_GP(v1)
154 PTR_L sp, VPEBOOTCFG_SP(v1)
155 jr t1
156 nop
157 END(mips_cps_core_boot)
158
159 __INIT
160LEAF(excep_tlbfill)
161 DUMP_EXCEP("TLB Fill")
162 b .
163 nop
164 END(excep_tlbfill)
165
166LEAF(excep_xtlbfill)
167 DUMP_EXCEP("XTLB Fill")
168 b .
169 nop
170 END(excep_xtlbfill)
171
172LEAF(excep_cache)
173 DUMP_EXCEP("Cache")
174 b .
175 nop
176 END(excep_cache)
177
178LEAF(excep_genex)
179 DUMP_EXCEP("General")
180 b .
181 nop
182 END(excep_genex)
183
184LEAF(excep_intex)
185 DUMP_EXCEP("Interrupt")
186 b .
187 nop
188 END(excep_intex)
189
190LEAF(excep_ejtag)
191 PTR_LA k0, ejtag_debug_handler
192 jr k0
193 nop
194 END(excep_ejtag)
195 __FINIT
196
197LEAF(mips_cps_core_init)
198#ifdef CONFIG_MIPS_MT_SMP
199 /* Check that the core implements the MT ASE */
200 has_mt t0, 3f
201
202 .set push
203 .set MIPS_ISA_LEVEL_RAW
204 .set mt
205
206 /* Only allow 1 TC per VPE to execute... */
207 dmt
208
209 /* ...and for the moment only 1 VPE */
210 dvpe
211 PTR_LA t1, 1f
212 jr.hb t1
213 nop
214
215 /* Enter VPE configuration state */
2161: mfc0 t0, CP0_MVPCONTROL
217 ori t0, t0, MVPCONTROL_VPC
218 mtc0 t0, CP0_MVPCONTROL
219
220 /* Retrieve the number of VPEs within the core */
221 mfc0 t0, CP0_MVPCONF0
222 srl t0, t0, MVPCONF0_PVPE_SHIFT
223 andi t0, t0, (MVPCONF0_PVPE >> MVPCONF0_PVPE_SHIFT)
224 addiu ta3, t0, 1
225
226 /* If there's only 1, we're done */
227 beqz t0, 2f
228 nop
229
230 /* Loop through each VPE within this core */
231 li ta1, 1
232
2331: /* Operate on the appropriate TC */
234 mtc0 ta1, CP0_VPECONTROL
235 ehb
236
237 /* Bind TC to VPE (1:1 TC:VPE mapping) */
238 mttc0 ta1, CP0_TCBIND
239
240 /* Set exclusive TC, non-active, master */
241 li t0, VPECONF0_MVP
242 sll t1, ta1, VPECONF0_XTC_SHIFT
243 or t0, t0, t1
244 mttc0 t0, CP0_VPECONF0
245
246 /* Set TC non-active, non-allocatable */
247 mttc0 zero, CP0_TCSTATUS
248
249 /* Set TC halted */
250 li t0, TCHALT_H
251 mttc0 t0, CP0_TCHALT
252
253 /* Next VPE */
254 addiu ta1, ta1, 1
255 slt t0, ta1, ta3
256 bnez t0, 1b
257 nop
258
259 /* Leave VPE configuration state */
2602: mfc0 t0, CP0_MVPCONTROL
261 xori t0, t0, MVPCONTROL_VPC
262 mtc0 t0, CP0_MVPCONTROL
263
2643: .set pop
265#endif
266 jr ra
267 nop
268 END(mips_cps_core_init)
269
270/**
271 * mips_cps_get_bootcfg() - retrieve boot configuration pointers
272 *
273 * Returns: pointer to struct core_boot_config in v0, pointer to
274 * struct vpe_boot_config in v1, VPE ID in t9
275 */
276LEAF(mips_cps_get_bootcfg)
277 /* Calculate a pointer to this cores struct core_boot_config */
278 PTR_LA v0, mips_cps_cluster_bootcfg
279 PTR_L v0, 0(v0)
280 lw t0, GCR_CL_ID_OFS(s1)
281#ifdef CONFIG_CPU_MIPSR6
282 ext t1, t0, CM3_GCR_Cx_ID_CLUSTER_SHF, 8
283 li t2, CLUSTERBOOTCFG_SIZE
284 mul t1, t1, t2
285 PTR_ADDU \
286 v0, v0, t1
287#endif
288 PTR_L v0, CLUSTERBOOTCFG_CORECONFIG(v0)
289 andi t0, t0, CM3_GCR_Cx_ID_CORENUM_MSK
290 li t1, COREBOOTCFG_SIZE
291 mul t0, t0, t1
292 PTR_ADDU v0, v0, t0
293
294 /* Calculate this VPEs ID. If the core doesn't support MT use 0 */
295 li t9, 0
296#if defined(CONFIG_CPU_MIPSR6)
297 has_vp ta2, 1f
298
299 /*
300 * Assume non-contiguous numbering. Perhaps some day we'll need
301 * to handle contiguous VP numbering, but no such systems yet
302 * exist.
303 */
304 mfc0 t9, CP0_GLOBALNUMBER
305 andi t9, t9, MIPS_GLOBALNUMBER_VP
306#elif defined(CONFIG_MIPS_MT_SMP)
307 has_mt ta2, 1f
308
309 /* Find the number of VPEs present in the core */
310 mfc0 t1, CP0_MVPCONF0
311 srl t1, t1, MVPCONF0_PVPE_SHIFT
312 andi t1, t1, MVPCONF0_PVPE >> MVPCONF0_PVPE_SHIFT
313 addiu t1, t1, 1
314
315 /* Calculate a mask for the VPE ID from EBase.CPUNum */
316 clz t1, t1
317 li t2, 31
318 subu t1, t2, t1
319 li t2, 1
320 sll t1, t2, t1
321 addiu t1, t1, -1
322
323 /* Retrieve the VPE ID from EBase.CPUNum */
324 mfc0 t9, $15, 1
325 and t9, t9, t1
326#endif
327
3281: /* Calculate a pointer to this VPEs struct vpe_boot_config */
329 li t1, VPEBOOTCFG_SIZE
330 mul v1, t9, t1
331 PTR_L ta3, COREBOOTCFG_VPECONFIG(v0)
332 PTR_ADDU v1, v1, ta3
333
334 jr ra
335 nop
336 END(mips_cps_get_bootcfg)
337
338LEAF(mips_cps_boot_vpes)
339 lw ta2, COREBOOTCFG_VPEMASK(a0)
340 PTR_L ta3, COREBOOTCFG_VPECONFIG(a0)
341
342#if defined(CONFIG_CPU_MIPSR6)
343
344 has_vp t0, 5f
345
346 /* Find base address of CPC */
347 PTR_LA t1, mips_gcr_base
348 PTR_L t1, 0(t1)
349 PTR_L t1, GCR_CPC_BASE_OFS(t1)
350 PTR_LI t2, ~0x7fff
351 and t1, t1, t2
352 PTR_LI t2, UNCAC_BASE
353 PTR_ADD t1, t1, t2
354
355 /* Start any other VPs that ought to be running */
356 PTR_S ta2, CPC_CL_VC_RUN_OFS(t1)
357
358 /* Ensure this VP stops running if it shouldn't be */
359 not ta2
360 PTR_S ta2, CPC_CL_VC_STOP_OFS(t1)
361 ehb
362
363#elif defined(CONFIG_MIPS_MT)
364
365 /* If the core doesn't support MT then return */
366 has_mt t0, 5f
367
368 /* Enter VPE configuration state */
369 .set push
370 .set MIPS_ISA_LEVEL_RAW
371 .set mt
372 dvpe
373 .set pop
374
375 PTR_LA t1, 1f
376 .set push
377 .set MIPS_ISA_LEVEL_RAW
378 jr.hb t1
379 nop
380 .set pop
3811: mfc0 t1, CP0_MVPCONTROL
382 ori t1, t1, MVPCONTROL_VPC
383 mtc0 t1, CP0_MVPCONTROL
384 ehb
385
386 /* Loop through each VPE */
387 move t8, ta2
388 li ta1, 0
389
390 /* Check whether the VPE should be running. If not, skip it */
3911: andi t0, ta2, 1
392 beqz t0, 2f
393 nop
394
395 /* Operate on the appropriate TC */
396 mfc0 t0, CP0_VPECONTROL
397 ori t0, t0, VPECONTROL_TARGTC
398 xori t0, t0, VPECONTROL_TARGTC
399 or t0, t0, ta1
400 mtc0 t0, CP0_VPECONTROL
401 ehb
402
403 .set push
404 .set MIPS_ISA_LEVEL_RAW
405 .set mt
406
407 /* Skip the VPE if its TC is not halted */
408 mftc0 t0, CP0_TCHALT
409 beqz t0, 2f
410 nop
411
412 /* Calculate a pointer to the VPEs struct vpe_boot_config */
413 li t0, VPEBOOTCFG_SIZE
414 mul t0, t0, ta1
415 PTR_ADDU t0, t0, ta3
416
417 /* Set the TC restart PC */
418 lw t1, VPEBOOTCFG_PC(t0)
419 mttc0 t1, CP0_TCRESTART
420
421 /* Set the TC stack pointer */
422 lw t1, VPEBOOTCFG_SP(t0)
423 mttgpr t1, sp
424
425 /* Set the TC global pointer */
426 lw t1, VPEBOOTCFG_GP(t0)
427 mttgpr t1, gp
428
429 /* Copy config from this VPE */
430 mfc0 t0, CP0_CONFIG
431 mttc0 t0, CP0_CONFIG
432
433 /*
434 * Copy the EVA config from this VPE if the CPU supports it.
435 * CONFIG3 must exist to be running MT startup - just read it.
436 */
437 mfc0 t0, CP0_CONFIG, 3
438 and t0, t0, MIPS_CONF3_SC
439 beqz t0, 3f
440 nop
441 mfc0 t0, CP0_SEGCTL0
442 mttc0 t0, CP0_SEGCTL0
443 mfc0 t0, CP0_SEGCTL1
444 mttc0 t0, CP0_SEGCTL1
445 mfc0 t0, CP0_SEGCTL2
446 mttc0 t0, CP0_SEGCTL2
4473:
448 /* Ensure no software interrupts are pending */
449 mttc0 zero, CP0_CAUSE
450 mttc0 zero, CP0_STATUS
451
452 /* Set TC active, not interrupt exempt */
453 mftc0 t0, CP0_TCSTATUS
454 li t1, ~TCSTATUS_IXMT
455 and t0, t0, t1
456 ori t0, t0, TCSTATUS_A
457 mttc0 t0, CP0_TCSTATUS
458
459 /* Clear the TC halt bit */
460 mttc0 zero, CP0_TCHALT
461
462 /* Set VPE active */
463 mftc0 t0, CP0_VPECONF0
464 ori t0, t0, VPECONF0_VPA
465 mttc0 t0, CP0_VPECONF0
466
467 /* Next VPE */
4682: srl ta2, ta2, 1
469 addiu ta1, ta1, 1
470 bnez ta2, 1b
471 nop
472
473 /* Leave VPE configuration state */
474 mfc0 t1, CP0_MVPCONTROL
475 xori t1, t1, MVPCONTROL_VPC
476 mtc0 t1, CP0_MVPCONTROL
477 ehb
478 evpe
479
480 .set pop
481
482 /* Check whether this VPE is meant to be running */
483 li t0, 1
484 sll t0, t0, a1
485 and t0, t0, t8
486 bnez t0, 2f
487 nop
488
489 /* This VPE should be offline, halt the TC */
490 li t0, TCHALT_H
491 mtc0 t0, CP0_TCHALT
492 PTR_LA t0, 1f
493 .set push
494 .set MIPS_ISA_LEVEL_RAW
4951: jr.hb t0
496 nop
497 .set pop
498
4992:
500
501#endif /* CONFIG_MIPS_MT_SMP */
502
503 /* Return */
5045: jr ra
505 nop
506 END(mips_cps_boot_vpes)
507
508#if MIPS_ISA_REV > 0
509LEAF(mips_cps_cache_init)
510 /*
511 * Clear the bits used to index the caches. Note that the architecture
512 * dictates that writing to any of TagLo or TagHi selects 0 or 2 should
513 * be valid for all MIPS32 CPUs, even those for which said writes are
514 * unnecessary.
515 */
516 mtc0 zero, CP0_TAGLO, 0
517 mtc0 zero, CP0_TAGHI, 0
518 mtc0 zero, CP0_TAGLO, 2
519 mtc0 zero, CP0_TAGHI, 2
520 ehb
521
522 /* Primary cache configuration is indicated by Config1 */
523 mfc0 v0, CP0_CONFIG, 1
524
525 /* Detect I-cache line size */
526 _EXT t0, v0, MIPS_CONF1_IL_SHF, MIPS_CONF1_IL_SZ
527 beqz t0, icache_done
528 li t1, 2
529 sllv t0, t1, t0
530
531 /* Detect I-cache size */
532 _EXT t1, v0, MIPS_CONF1_IS_SHF, MIPS_CONF1_IS_SZ
533 xori t2, t1, 0x7
534 beqz t2, 1f
535 li t3, 32
536 addiu t1, t1, 1
537 sllv t1, t3, t1
5381: /* At this point t1 == I-cache sets per way */
539 _EXT t2, v0, MIPS_CONF1_IA_SHF, MIPS_CONF1_IA_SZ
540 addiu t2, t2, 1
541 mul t1, t1, t0
542 mul t1, t1, t2
543
544 li a0, CKSEG0
545 PTR_ADD a1, a0, t1
5461: cache Index_Store_Tag_I, 0(a0)
547 PTR_ADD a0, a0, t0
548 bne a0, a1, 1b
549 nop
550icache_done:
551
552 /* Detect D-cache line size */
553 _EXT t0, v0, MIPS_CONF1_DL_SHF, MIPS_CONF1_DL_SZ
554 beqz t0, dcache_done
555 li t1, 2
556 sllv t0, t1, t0
557
558 /* Detect D-cache size */
559 _EXT t1, v0, MIPS_CONF1_DS_SHF, MIPS_CONF1_DS_SZ
560 xori t2, t1, 0x7
561 beqz t2, 1f
562 li t3, 32
563 addiu t1, t1, 1
564 sllv t1, t3, t1
5651: /* At this point t1 == D-cache sets per way */
566 _EXT t2, v0, MIPS_CONF1_DA_SHF, MIPS_CONF1_DA_SZ
567 addiu t2, t2, 1
568 mul t1, t1, t0
569 mul t1, t1, t2
570
571 li a0, CKSEG0
572 PTR_ADDU a1, a0, t1
573 PTR_SUBU a1, a1, t0
5741: cache Index_Store_Tag_D, 0(a0)
575 bne a0, a1, 1b
576 PTR_ADD a0, a0, t0
577dcache_done:
578
579 jr ra
580 nop
581 END(mips_cps_cache_init)
582#endif /* MIPS_ISA_REV > 0 */
583
584#if defined(CONFIG_MIPS_CPS_PM) && defined(CONFIG_CPU_PM)
585
586 /* Calculate a pointer to this CPUs struct mips_static_suspend_state */
587 .macro psstate dest
588 .set push
589 .set noat
590 lw $1, TI_CPU(gp)
591 sll $1, $1, LONGLOG
592 PTR_LA \dest, __per_cpu_offset
593 PTR_ADDU $1, $1, \dest
594 lw $1, 0($1)
595 PTR_LA \dest, cps_cpu_state
596 PTR_ADDU \dest, \dest, $1
597 .set pop
598 .endm
599
600LEAF(mips_cps_pm_save)
601 /* Save CPU state */
602 SUSPEND_SAVE_REGS
603 psstate t1
604 SUSPEND_SAVE_STATIC
605 jr v0
606 nop
607 END(mips_cps_pm_save)
608
609LEAF(mips_cps_pm_restore)
610 /* Restore CPU state */
611 psstate t1
612 RESUME_RESTORE_STATIC
613 RESUME_RESTORE_REGS_RETURN
614 END(mips_cps_pm_restore)
615
616#endif /* CONFIG_MIPS_CPS_PM && CONFIG_CPU_PM */