projects/linux·blob master · refs

466 lines · 10.8 KiB

1/* SPDX-License-Identifier: GPL-2.0-or-later */
2/*
3 * PowerPC version
4 * Copyright (C) 1995-1996 Gary Thomas (gdt@linuxppc.org)
5 * Rewritten by Cort Dougan (cort@fsmlabs.com) for PReP
6 * Copyright (C) 1996 Cort Dougan <cort@fsmlabs.com>
7 * Adapted for Power Macintosh by Paul Mackerras.
8 * Low-level exception handlers and MMU support
9 * rewritten by Paul Mackerras.
10 * Copyright (C) 1996 Paul Mackerras.
11 * MPC8xx modifications Copyright (C) 1997 Dan Malek (dmalek@jlc.net).
12 *
13 * This file contains the system call entry code, context switch
14 * code, and exception/interrupt return code for PowerPC.
15 */
16
17#include <linux/errno.h>
18#include <linux/err.h>
19#include <linux/sys.h>
20#include <linux/threads.h>
21#include <linux/linkage.h>
22
23#include <asm/reg.h>
24#include <asm/page.h>
25#include <asm/mmu.h>
26#include <asm/cputable.h>
27#include <asm/thread_info.h>
28#include <asm/ppc_asm.h>
29#include <asm/asm-offsets.h>
30#include <asm/unistd.h>
31#include <asm/ptrace.h>
32#include <asm/feature-fixups.h>
33#include <asm/barrier.h>
34#include <asm/kup.h>
35#include <asm/bug.h>
36#include <asm/interrupt.h>
37
38#include "head_32.h"
39
40/*
41 * powerpc relies on return from interrupt/syscall being context synchronising
42 * (which rfi is) to support ARCH_HAS_MEMBARRIER_SYNC_CORE without additional
43 * synchronisation instructions.
44 */
45
46/*
47 * Align to 4k in order to ensure that all functions modyfing srr0/srr1
48 * fit into one page in order to not encounter a TLB miss between the
49 * modification of srr0/srr1 and the associated rfi.
50 */
51 .align 12
52
53#if defined(CONFIG_PPC_BOOK3S_32) || defined(CONFIG_PPC_E500)
54 .globl prepare_transfer_to_handler
55prepare_transfer_to_handler:
56 /* if from kernel, check interrupted DOZE/NAP mode */
57 lwz r12,TI_LOCAL_FLAGS(r2)
58 mtcrf 0x01,r12
59 bt- 31-TLF_NAPPING,4f
60 bt- 31-TLF_SLEEPING,7f
61 blr
62
634: rlwinm r12,r12,0,~_TLF_NAPPING
64 stw r12,TI_LOCAL_FLAGS(r2)
65 b power_save_ppc32_restore
66
677: rlwinm r12,r12,0,~_TLF_SLEEPING
68 stw r12,TI_LOCAL_FLAGS(r2)
69 lwz r9,_MSR(r11) /* if sleeping, clear MSR.EE */
70 rlwinm r9,r9,0,~MSR_EE
71 lwz r12,_LINK(r11) /* and return to address in LR */
72 REST_GPR(2, r11)
73 b fast_exception_return
74_ASM_NOKPROBE_SYMBOL(prepare_transfer_to_handler)
75#endif /* CONFIG_PPC_BOOK3S_32 || CONFIG_PPC_E500 */
76
77#if defined(CONFIG_PPC_KUEP) && defined(CONFIG_PPC_BOOK3S_32)
78SYM_FUNC_START(__kuep_lock)
79 lwz r9, THREAD+THSR0(r2)
80 update_user_segments_by_4 r9, r10, r11, r12
81 blr
82SYM_FUNC_END(__kuep_lock)
83
84SYM_FUNC_START_LOCAL(__kuep_unlock)
85 lwz r9, THREAD+THSR0(r2)
86 rlwinm r9,r9,0,~SR_NX
87 update_user_segments_by_4 r9, r10, r11, r12
88 blr
89SYM_FUNC_END(__kuep_unlock)
90
91.macro kuep_lock
92 bl __kuep_lock
93.endm
94.macro kuep_unlock
95 bl __kuep_unlock
96.endm
97#else
98.macro kuep_lock
99.endm
100.macro kuep_unlock
101.endm
102#endif
103
104 .globl transfer_to_syscall
105transfer_to_syscall:
106 stw r3, ORIG_GPR3(r1)
107 stw r11, GPR1(r1)
108 stw r11, 0(r1)
109 mflr r12
110 stw r12, _LINK(r1)
111#ifdef CONFIG_BOOKE
112 rlwinm r9,r9,0,14,12 /* clear MSR_WE (necessary?) */
113#endif
114 lis r12,STACK_FRAME_REGS_MARKER@ha /* exception frame marker */
115 SAVE_GPR(2, r1)
116 addi r12,r12,STACK_FRAME_REGS_MARKER@l
117 stw r9,_MSR(r1)
118 li r2, INTERRUPT_SYSCALL
119 stw r12,STACK_INT_FRAME_MARKER(r1)
120 stw r2,_TRAP(r1)
121 SAVE_GPR(0, r1)
122 SAVE_GPRS(3, 8, r1)
123 addi r2,r10,-THREAD
124 SAVE_NVGPRS(r1)
125 kuep_lock
126
127 /* Calling convention has r3 = regs, r4 = orig r0 */
128 addi r3,r1,STACK_INT_FRAME_REGS
129 mr r4,r0
130 bl system_call_exception
131
132ret_from_syscall:
133 addi r4,r1,STACK_INT_FRAME_REGS
134 li r5,0
135 bl syscall_exit_prepare
136#ifdef CONFIG_PPC_47x
137 lis r4,icache_44x_need_flush@ha
138 lwz r5,icache_44x_need_flush@l(r4)
139 cmplwi cr0,r5,0
140 bne- .L44x_icache_flush
141#endif /* CONFIG_PPC_47x */
142.L44x_icache_flush_return:
143 kuep_unlock
144 lwz r4,_LINK(r1)
145 lwz r5,_CCR(r1)
146 mtlr r4
147 lwz r7,_NIP(r1)
148 lwz r8,_MSR(r1)
149 cmpwi r3,0
150 REST_GPR(3, r1)
151syscall_exit_finish:
152 mtspr SPRN_SRR0,r7
153 mtspr SPRN_SRR1,r8
154
155 bne 3f
156 mtcr r5
157
1581: REST_GPR(2, r1)
159 REST_GPR(1, r1)
160 rfi
161
1623: mtcr r5
163 lwz r4,_CTR(r1)
164 lwz r5,_XER(r1)
165 REST_NVGPRS(r1)
166 mtctr r4
167 mtxer r5
168 REST_GPR(0, r1)
169 REST_GPRS(3, 12, r1)
170 b 1b
171_ASM_NOKPROBE_SYMBOL(syscall_exit_finish)
172
173#ifdef CONFIG_44x
174.L44x_icache_flush:
175 li r7,0
176 iccci r0,r0
177 stw r7,icache_44x_need_flush@l(r4)
178 b .L44x_icache_flush_return
179#endif /* CONFIG_44x */
180
181 .globl ret_from_fork
182ret_from_fork:
183 REST_NVGPRS(r1)
184 bl schedule_tail
185 li r3,0 /* fork() return value */
186 b ret_from_syscall
187
188 .globl ret_from_kernel_user_thread
189ret_from_kernel_user_thread:
190 bl schedule_tail
191 mtctr r14
192 mr r3,r15
193 PPC440EP_ERR42
194 bctrl
195 li r3,0
196 b ret_from_syscall
197
198 .globl start_kernel_thread
199start_kernel_thread:
200 bl schedule_tail
201 mtctr r14
202 mr r3,r15
203 PPC440EP_ERR42
204 bctrl
205 /*
206 * This must not return. We actually want to BUG here, not WARN,
207 * because BUG will exit the process which is what the kernel thread
208 * should have done, which may give some hope of continuing.
209 */
210100: trap
211 EMIT_BUG_ENTRY 100b,__FILE__,__LINE__,0
212
213 .globl fast_exception_return
214fast_exception_return:
215#ifndef CONFIG_BOOKE
216 andi. r10,r9,MSR_RI /* check for recoverable interrupt */
217 beq 3f /* if not, we've got problems */
218#endif
219
220 lwz r10,_CCR(r11)
221 REST_GPRS(1, 6, r11)
222 mtcr r10
223 lwz r10,_LINK(r11)
224 mtlr r10
225 /* Clear the exception marker on the stack to avoid confusing stacktrace */
226 li r10, 0
227 stw r10, 8(r11)
228 mtspr SPRN_SRR1,r9
229 mtspr SPRN_SRR0,r12
230 REST_GPR(9, r11)
231 REST_GPR(10, r11)
232 REST_GPR(12, r11)
233 REST_GPR(11, r11)
234 rfi
235_ASM_NOKPROBE_SYMBOL(fast_exception_return)
236
237/* aargh, a nonrecoverable interrupt, panic */
238/* aargh, we don't know which trap this is */
2393:
240 li r10,-1
241 stw r10,_TRAP(r11)
242 prepare_transfer_to_handler
243 bl unrecoverable_exception
244 trap /* should not get here */
245
246 .globl interrupt_return
247interrupt_return:
248 lwz r4,_MSR(r1)
249 addi r3,r1,STACK_INT_FRAME_REGS
250 andi. r0,r4,MSR_PR
251 beq .Lkernel_interrupt_return
252 bl interrupt_exit_user_prepare
253 cmpwi r3,0
254 kuep_unlock
255 bne- .Lrestore_nvgprs
256
257.Lfast_user_interrupt_return:
258 lwz r11,_NIP(r1)
259 lwz r12,_MSR(r1)
260 mtspr SPRN_SRR0,r11
261 mtspr SPRN_SRR1,r12
262
263BEGIN_FTR_SECTION
264 lwarx r0,0,r1
265END_FTR_SECTION_IFSET(CPU_FTR_NEED_PAIRED_STWCX)
266 stwcx. r0,0,r1 /* to clear the reservation */
267
268 lwz r3,_CCR(r1)
269 lwz r4,_LINK(r1)
270 lwz r5,_CTR(r1)
271 lwz r6,_XER(r1)
272 li r0,0
273
274 /*
275 * Leaving a stale exception marker on the stack can confuse
276 * the reliable stack unwinder later on. Clear it.
277 */
278 stw r0,8(r1)
279 REST_GPRS(7, 12, r1)
280
281 mtcr r3
282 mtlr r4
283 mtctr r5
284 mtspr SPRN_XER,r6
285
286 REST_GPRS(2, 6, r1)
287 REST_GPR(0, r1)
288 REST_GPR(1, r1)
289 rfi
290
291.Lrestore_nvgprs:
292 REST_NVGPRS(r1)
293 b .Lfast_user_interrupt_return
294
295.Lkernel_interrupt_return:
296 bl interrupt_exit_kernel_prepare
297
298.Lfast_kernel_interrupt_return:
299 cmpwi cr1,r3,0
300 lwz r11,_NIP(r1)
301 lwz r12,_MSR(r1)
302 mtspr SPRN_SRR0,r11
303 mtspr SPRN_SRR1,r12
304
305BEGIN_FTR_SECTION
306 lwarx r0,0,r1
307END_FTR_SECTION_IFSET(CPU_FTR_NEED_PAIRED_STWCX)
308 stwcx. r0,0,r1 /* to clear the reservation */
309
310 lwz r3,_LINK(r1)
311 lwz r4,_CTR(r1)
312 lwz r5,_XER(r1)
313 lwz r6,_CCR(r1)
314 li r0,0
315
316 REST_GPRS(7, 12, r1)
317
318 mtlr r3
319 mtctr r4
320 mtspr SPRN_XER,r5
321
322 /*
323 * Leaving a stale exception marker on the stack can confuse
324 * the reliable stack unwinder later on. Clear it.
325 */
326 stw r0,8(r1)
327
328 REST_GPRS(2, 5, r1)
329
330 bne- cr1,1f /* emulate stack store */
331 mtcr r6
332 REST_GPR(6, r1)
333 REST_GPR(0, r1)
334 REST_GPR(1, r1)
335 rfi
336
3371: /*
338 * Emulate stack store with update. New r1 value was already calculated
339 * and updated in our interrupt regs by emulate_loadstore, but we can't
340 * store the previous value of r1 to the stack before re-loading our
341 * registers from it, otherwise they could be clobbered. Use
342 * SPRG Scratch0 as temporary storage to hold the store
343 * data, as interrupts are disabled here so it won't be clobbered.
344 */
345 mtcr r6
346#ifdef CONFIG_BOOKE
347 mtspr SPRN_SPRG_WSCRATCH0, r9
348#else
349 mtspr SPRN_SPRG_SCRATCH0, r9
350#endif
351 addi r9,r1,INT_FRAME_SIZE /* get original r1 */
352 REST_GPR(6, r1)
353 REST_GPR(0, r1)
354 REST_GPR(1, r1)
355 stw r9,0(r1) /* perform store component of stwu */
356#ifdef CONFIG_BOOKE
357 mfspr r9, SPRN_SPRG_RSCRATCH0
358#else
359 mfspr r9, SPRN_SPRG_SCRATCH0
360#endif
361 rfi
362_ASM_NOKPROBE_SYMBOL(interrupt_return)
363
364#ifdef CONFIG_BOOKE
365
366/*
367 * Returning from a critical interrupt in user mode doesn't need
368 * to be any different from a normal exception. For a critical
369 * interrupt in the kernel, we just return (without checking for
370 * preemption) since the interrupt may have happened at some crucial
371 * place (e.g. inside the TLB miss handler), and because we will be
372 * running with r1 pointing into critical_stack, not the current
373 * process's kernel stack (and therefore current_thread_info() will
374 * give the wrong answer).
375 * We have to restore various SPRs that may have been in use at the
376 * time of the critical interrupt.
377 *
378 */
379
380#define RET_FROM_EXC_LEVEL(exc_lvl_srr0, exc_lvl_srr1, exc_lvl_rfi) \
381 REST_NVGPRS(r1); \
382 lwz r3,_MSR(r1); \
383 andi. r3,r3,MSR_PR; \
384 bne interrupt_return; \
385 REST_GPR(0, r1); \
386 REST_GPRS(2, 8, r1); \
387 lwz r10,_XER(r1); \
388 lwz r11,_CTR(r1); \
389 mtspr SPRN_XER,r10; \
390 mtctr r11; \
391 stwcx. r0,0,r1; /* to clear the reservation */ \
392 lwz r11,_LINK(r1); \
393 mtlr r11; \
394 lwz r10,_CCR(r1); \
395 mtcrf 0xff,r10; \
396 lwz r9,_DEAR(r1); \
397 lwz r10,_ESR(r1); \
398 mtspr SPRN_DEAR,r9; \
399 mtspr SPRN_ESR,r10; \
400 lwz r11,_NIP(r1); \
401 lwz r12,_MSR(r1); \
402 mtspr exc_lvl_srr0,r11; \
403 mtspr exc_lvl_srr1,r12; \
404 REST_GPRS(9, 12, r1); \
405 REST_GPR(1, r1); \
406 exc_lvl_rfi; \
407 b .; /* prevent prefetch past exc_lvl_rfi */
408
409#define RESTORE_xSRR(exc_lvl_srr0, exc_lvl_srr1) \
410 lwz r9,_##exc_lvl_srr0(r1); \
411 lwz r10,_##exc_lvl_srr1(r1); \
412 mtspr SPRN_##exc_lvl_srr0,r9; \
413 mtspr SPRN_##exc_lvl_srr1,r10;
414
415#if defined(CONFIG_PPC_E500)
416#ifdef CONFIG_PHYS_64BIT
417#define RESTORE_MAS7 \
418 lwz r11,MAS7(r1); \
419 mtspr SPRN_MAS7,r11;
420#else
421#define RESTORE_MAS7
422#endif /* CONFIG_PHYS_64BIT */
423#define RESTORE_MMU_REGS \
424 lwz r9,MAS0(r1); \
425 lwz r10,MAS1(r1); \
426 lwz r11,MAS2(r1); \
427 mtspr SPRN_MAS0,r9; \
428 lwz r9,MAS3(r1); \
429 mtspr SPRN_MAS1,r10; \
430 lwz r10,MAS6(r1); \
431 mtspr SPRN_MAS2,r11; \
432 mtspr SPRN_MAS3,r9; \
433 mtspr SPRN_MAS6,r10; \
434 RESTORE_MAS7;
435#elif defined(CONFIG_44x)
436#define RESTORE_MMU_REGS \
437 lwz r9,MMUCR(r1); \
438 mtspr SPRN_MMUCR,r9;
439#else
440#define RESTORE_MMU_REGS
441#endif
442
443 .globl ret_from_crit_exc
444ret_from_crit_exc:
445 RESTORE_xSRR(SRR0,SRR1);
446 RESTORE_MMU_REGS;
447 RET_FROM_EXC_LEVEL(SPRN_CSRR0, SPRN_CSRR1, PPC_RFCI)
448_ASM_NOKPROBE_SYMBOL(ret_from_crit_exc)
449
450 .globl ret_from_debug_exc
451ret_from_debug_exc:
452 RESTORE_xSRR(SRR0,SRR1);
453 RESTORE_xSRR(CSRR0,CSRR1);
454 RESTORE_MMU_REGS;
455 RET_FROM_EXC_LEVEL(SPRN_DSRR0, SPRN_DSRR1, PPC_RFDI)
456_ASM_NOKPROBE_SYMBOL(ret_from_debug_exc)
457
458 .globl ret_from_mcheck_exc
459ret_from_mcheck_exc:
460 RESTORE_xSRR(SRR0,SRR1);
461 RESTORE_xSRR(CSRR0,CSRR1);
462 RESTORE_xSRR(DSRR0,DSRR1);
463 RESTORE_MMU_REGS;
464 RET_FROM_EXC_LEVEL(SPRN_MCSRR0, SPRN_MCSRR1, PPC_RFMCI)
465_ASM_NOKPROBE_SYMBOL(ret_from_mcheck_exc)
466#endif /* CONFIG_BOOKE */