projects/linux·blob master · refs

514 lines · 13.5 KiB

1/* SPDX-License-Identifier: GPL-2.0-only */
2/*
3 * Copyright (C) 2012 Regents of the University of California
4 * Copyright (C) 2017 SiFive
5 */
6
7#include <linux/init.h>
8#include <linux/linkage.h>
9
10#include <asm/alternative-macros.h>
11#include <asm/asm.h>
12#include <asm/csr.h>
13#include <asm/scs.h>
14#include <asm/unistd.h>
15#include <asm/page.h>
16#include <asm/thread_info.h>
17#include <asm/asm-offsets.h>
18#include <asm/errata_list.h>
19#include <linux/sizes.h>
20
21 .section .irqentry.text, "ax"
22
23.macro new_valid_map_cpus_check
24 REG_S a0, TASK_TI_A0(tp)
25 csrr a0, CSR_CAUSE
26 /* Exclude IRQs */
27 blt a0, zero, .Lnew_valid_map_cpus_restore_context_a0
28
29 REG_S a1, TASK_TI_A1(tp)
30 /* Only check new_valid_map_cpus if we are in page/protection fault */
31 li a1, EXC_LOAD_PAGE_FAULT
32 beq a0, a1, .Lnew_valid_map_cpus_kernel_address
33 li a1, EXC_STORE_PAGE_FAULT
34 beq a0, a1, .Lnew_valid_map_cpus_kernel_address
35 li a1, EXC_INST_PAGE_FAULT
36 bne a0, a1, .Lnew_valid_map_cpus_restore_context_a1
37
38.Lnew_valid_map_cpus_kernel_address:
39 /* Is it a kernel address? */
40 csrr a0, CSR_TVAL
41 bge a0, zero, .Lnew_valid_map_cpus_restore_context_a1
42
43 /* Check if a new vmalloc mapping appeared that could explain the trap */
44 REG_S a2, TASK_TI_A2(tp)
45 /*
46 * Computes:
47 * a0 = &new_valid_map_cpus[BIT_WORD(cpu)]
48 * a1 = BIT_MASK(cpu)
49 */
50 lw a2, TASK_TI_CPU(tp)
51 /*
52 * Compute the new_valid_map_cpus element position:
53 * (cpu / 64) * 8 = (cpu >> 6) << 3
54 */
55 srli a1, a2, 6
56 slli a1, a1, 3
57 la a0, new_valid_map_cpus
58 add a0, a0, a1
59 /*
60 * Compute the bit position in the new_valid_map_cpus element:
61 * bit_pos = cpu % 64 = cpu - (cpu / 64) * 64 = cpu - (cpu >> 6) << 6
62 * = cpu - ((cpu >> 6) << 3) << 3
63 */
64 slli a1, a1, 3
65 sub a1, a2, a1
66 /* Compute the "get mask": 1 << bit_pos */
67 li a2, 1
68 sll a1, a2, a1
69
70 /* Check the value of new_valid_map_cpus for this cpu */
71 REG_L a2, 0(a0)
72 and a2, a2, a1
73 beq a2, zero, .Lnew_valid_map_cpus_restore_context
74
75 /* Atomically reset the current cpu bit in new_valid_map_cpus */
76 amoxor.d a0, a1, (a0)
77
78 /*
79 * A sfence.vma is required here. Even if we had Svvptc, there's no
80 * guarantee that after returning we wouldn't just fault again.
81 */
82 sfence.vma
83
84 REG_L a0, TASK_TI_A0(tp)
85 REG_L a1, TASK_TI_A1(tp)
86 REG_L a2, TASK_TI_A2(tp)
87 csrw CSR_SCRATCH, x0
88 sret
89
90.Lnew_valid_map_cpus_restore_context:
91 REG_L a2, TASK_TI_A2(tp)
92.Lnew_valid_map_cpus_restore_context_a1:
93 REG_L a1, TASK_TI_A1(tp)
94.Lnew_valid_map_cpus_restore_context_a0:
95 REG_L a0, TASK_TI_A0(tp)
96.endm
97
98/*
99 * If previous mode was U, capture shadow stack pointer and save it away
100 * Zero CSR_SSP at the same time for sanitization.
101 */
102.macro save_userssp tmp, status
103 ALTERNATIVE("nops(4)",
104 __stringify( \
105 andi \tmp, \status, SR_SPP; \
106 bnez \tmp, skip_ssp_save; \
107 csrrw \tmp, CSR_SSP, x0; \
108 REG_S \tmp, TASK_TI_USER_SSP(tp); \
109 skip_ssp_save:),
110 0,
111 RISCV_ISA_EXT_ZICFISS,
112 CONFIG_RISCV_USER_CFI)
113.endm
114
115.macro restore_userssp tmp, status
116 ALTERNATIVE("nops(4)",
117 __stringify( \
118 andi \tmp, \status, SR_SPP; \
119 bnez \tmp, skip_ssp_restore; \
120 REG_L \tmp, TASK_TI_USER_SSP(tp); \
121 csrw CSR_SSP, \tmp; \
122 skip_ssp_restore:),
123 0,
124 RISCV_ISA_EXT_ZICFISS,
125 CONFIG_RISCV_USER_CFI)
126.endm
127
128SYM_CODE_START(handle_exception)
129 /*
130 * If coming from userspace, preserve the user thread pointer and load
131 * the kernel thread pointer. If we came from the kernel, the scratch
132 * register will contain 0, and we should continue on the current TP.
133 */
134 csrrw tp, CSR_SCRATCH, tp
135 bnez tp, .Lsave_context
136
137.Lrestore_kernel_tpsp:
138 csrr tp, CSR_SCRATCH
139
140#if defined(CONFIG_64BIT) && defined(CONFIG_MMU)
141 /*
142 * The RISC-V kernel does not flush TLBs on all CPUS after each new
143 * vmalloc mapping or kfence_unprotect(), which may result in
144 * exceptions:
145 *
146 * - if the uarch caches invalid entries, the new mapping would not be
147 * observed by the page table walker and an invalidation is needed.
148 * - if the uarch does not cache invalid entries, a reordered access
149 * could "miss" the new mapping and traps: in that case, we only need
150 * to retry the access, no sfence.vma is required.
151 */
152 new_valid_map_cpus_check
153#endif
154
155 REG_S sp, TASK_TI_KERNEL_SP(tp)
156
157#ifdef CONFIG_VMAP_STACK
158 addi sp, sp, -(PT_SIZE_ON_STACK)
159 srli sp, sp, THREAD_SHIFT
160 andi sp, sp, 0x1
161 bnez sp, handle_kernel_stack_overflow
162 REG_L sp, TASK_TI_KERNEL_SP(tp)
163#endif
164
165.Lsave_context:
166 REG_S sp, TASK_TI_USER_SP(tp)
167 REG_L sp, TASK_TI_KERNEL_SP(tp)
168 addi sp, sp, -(PT_SIZE_ON_STACK)
169 REG_S x1, PT_RA(sp)
170 REG_S x3, PT_GP(sp)
171 REG_S x5, PT_T0(sp)
172 save_from_x6_to_x31
173
174 /*
175 * Disable user-mode memory access as it should only be set in the
176 * actual user copy routines.
177 *
178 * Disable the FPU/Vector to detect illegal usage of floating point
179 * or vector in kernel space.
180 */
181 li t0, SR_SUM | SR_FS_VS
182#ifdef CONFIG_64BIT
183 li t1, SR_ELP
184 or t0, t0, t1
185#endif
186
187 REG_L s0, TASK_TI_USER_SP(tp)
188 csrrc s1, CSR_STATUS, t0
189 save_userssp s2, s1
190 csrr s2, CSR_EPC
191 csrr s3, CSR_TVAL
192 csrr s4, CSR_CAUSE
193 csrr s5, CSR_SCRATCH
194 REG_S s0, PT_SP(sp)
195 REG_S s1, PT_STATUS(sp)
196 REG_S s2, PT_EPC(sp)
197 REG_S s3, PT_BADADDR(sp)
198 REG_S s4, PT_CAUSE(sp)
199 REG_S s5, PT_TP(sp)
200
201 /*
202 * Set the scratch register to 0, so that if a recursive exception
203 * occurs, the exception vector knows it came from the kernel
204 */
205 csrw CSR_SCRATCH, x0
206
207 /* Load the global pointer */
208 load_global_pointer
209
210 /* Load the kernel shadow call stack pointer if coming from userspace */
211 scs_load_current_if_from_user s1, s5
212
213#ifdef CONFIG_RISCV_ISA_V_PREEMPTIVE
214 move a0, sp
215 call riscv_v_context_nesting_start
216#endif
217 move a0, sp /* pt_regs */
218
219 /*
220 * MSB of cause differentiates between
221 * interrupts and exceptions
222 */
223 bge s4, zero, 1f
224
225 /* Handle interrupts */
226 call do_irq
227 j ret_from_exception
2281:
229 /* Handle other exceptions */
230 slli t0, s4, RISCV_LGPTR
231 la t1, excp_vect_table
232 la t2, excp_vect_table_end
233 add t0, t1, t0
234 /* Check if exception code lies within bounds */
235 bgeu t0, t2, 3f
236 REG_L t1, 0(t0)
2372: jalr t1
238 j ret_from_exception
2393:
240
241 la t1, do_trap_unknown
242 j 2b
243SYM_CODE_END(handle_exception)
244ASM_NOKPROBE(handle_exception)
245
246/*
247 * The ret_from_exception must be called with interrupt disabled. Here is the
248 * caller list:
249 * - handle_exception
250 * - ret_from_fork
251 */
252SYM_CODE_START_NOALIGN(ret_from_exception)
253 REG_L s0, PT_STATUS(sp)
254#ifdef CONFIG_RISCV_M_MODE
255 /* the MPP value is too large to be used as an immediate arg for addi */
256 li t0, SR_MPP
257 and s0, s0, t0
258#else
259 andi s0, s0, SR_SPP
260#endif
261 bnez s0, 1f
262
263#ifdef CONFIG_KSTACK_ERASE
264 call stackleak_erase_on_task_stack
265#endif
266
267 /* Save unwound kernel stack pointer in thread_info */
268 addi s0, sp, PT_SIZE_ON_STACK
269 REG_S s0, TASK_TI_KERNEL_SP(tp)
270
271 /* Save the kernel shadow call stack pointer */
272 scs_save_current
273
274 /*
275 * Save TP into the scratch register , so we can find the kernel data
276 * structures again.
277 */
278 csrw CSR_SCRATCH, tp
2791:
280#ifdef CONFIG_RISCV_ISA_V_PREEMPTIVE
281 move a0, sp
282 call riscv_v_context_nesting_end
283#endif
284 REG_L a0, PT_STATUS(sp)
285 restore_userssp s3, a0
286 /*
287 * The current load reservation is effectively part of the processor's
288 * state, in the sense that load reservations cannot be shared between
289 * different hart contexts. We can't actually save and restore a load
290 * reservation, so instead here we clear any existing reservation --
291 * it's always legal for implementations to clear load reservations at
292 * any point (as long as the forward progress guarantee is kept, but
293 * we'll ignore that here).
294 *
295 * Dangling load reservations can be the result of taking a trap in the
296 * middle of an LR/SC sequence, but can also be the result of a taken
297 * forward branch around an SC -- which is how we implement CAS. As a
298 * result we need to clear reservations between the last CAS and the
299 * jump back to the new context. While it is unlikely the store
300 * completes, implementations are allowed to expand reservations to be
301 * arbitrarily large.
302 */
303 REG_L a2, PT_EPC(sp)
304 REG_SC x0, a2, PT_EPC(sp)
305
306 csrw CSR_STATUS, a0
307 csrw CSR_EPC, a2
308
309 REG_L x1, PT_RA(sp)
310 REG_L x3, PT_GP(sp)
311 REG_L x4, PT_TP(sp)
312 REG_L x5, PT_T0(sp)
313 restore_from_x6_to_x31
314
315 REG_L x2, PT_SP(sp)
316
317#ifdef CONFIG_RISCV_M_MODE
318 mret
319#else
320 sret
321#endif
322SYM_INNER_LABEL(ret_from_exception_end, SYM_L_GLOBAL)
323SYM_CODE_END(ret_from_exception)
324ASM_NOKPROBE(ret_from_exception)
325
326#ifdef CONFIG_VMAP_STACK
327SYM_CODE_START_LOCAL(handle_kernel_stack_overflow)
328 /* we reach here from kernel context, sscratch must be 0 */
329 csrrw x31, CSR_SCRATCH, x31
330 asm_per_cpu sp, overflow_stack, x31
331 li x31, OVERFLOW_STACK_SIZE
332 add sp, sp, x31
333 /* zero out x31 again and restore x31 */
334 xor x31, x31, x31
335 csrrw x31, CSR_SCRATCH, x31
336
337 addi sp, sp, -(PT_SIZE_ON_STACK)
338
339 //save context to overflow stack
340 REG_S x1, PT_RA(sp)
341 REG_S x3, PT_GP(sp)
342 REG_S x5, PT_T0(sp)
343 save_from_x6_to_x31
344
345 REG_L s0, TASK_TI_KERNEL_SP(tp)
346 csrr s1, CSR_STATUS
347 csrr s2, CSR_EPC
348 csrr s3, CSR_TVAL
349 csrr s4, CSR_CAUSE
350 csrr s5, CSR_SCRATCH
351 REG_S s0, PT_SP(sp)
352 REG_S s1, PT_STATUS(sp)
353 REG_S s2, PT_EPC(sp)
354 REG_S s3, PT_BADADDR(sp)
355 REG_S s4, PT_CAUSE(sp)
356 REG_S s5, PT_TP(sp)
357 move a0, sp
358 tail handle_bad_stack
359SYM_CODE_END(handle_kernel_stack_overflow)
360ASM_NOKPROBE(handle_kernel_stack_overflow)
361#endif
362
363SYM_CODE_START(ret_from_fork_kernel_asm)
364 call schedule_tail
365 move a0, s1 /* fn_arg */
366 move a1, s0 /* fn */
367 move a2, sp /* pt_regs */
368 call ret_from_fork_kernel
369 j ret_from_exception
370SYM_CODE_END(ret_from_fork_kernel_asm)
371
372SYM_CODE_START(ret_from_fork_user_asm)
373 call schedule_tail
374 move a0, sp /* pt_regs */
375 call ret_from_fork_user
376 j ret_from_exception
377SYM_CODE_END(ret_from_fork_user_asm)
378
379#ifdef CONFIG_IRQ_STACKS
380/*
381 * void call_on_irq_stack(struct pt_regs *regs,
382 * void (*func)(struct pt_regs *));
383 *
384 * Calls func(regs) using the per-CPU IRQ stack.
385 */
386SYM_FUNC_START(call_on_irq_stack)
387 /* Create a frame record to save ra and s0 (fp) */
388 addi sp, sp, -STACKFRAME_SIZE_ON_STACK
389 REG_S ra, (STACKFRAME_SIZE_ON_STACK + STACKFRAME_RA)(sp)
390 REG_S s0, (STACKFRAME_SIZE_ON_STACK + STACKFRAME_FP)(sp)
391 addi s0, sp, STACKFRAME_SIZE_ON_STACK
392
393 /* Switch to the per-CPU shadow call stack */
394 scs_save_current
395 scs_load_irq_stack t0
396
397 /* Switch to the per-CPU IRQ stack and call the handler */
398 load_per_cpu t0, irq_stack_ptr, t1
399 li t1, IRQ_STACK_SIZE
400 add sp, t0, t1
401 jalr a1
402
403 /* Switch back to the thread shadow call stack */
404 scs_load_current
405
406 /* Switch back to the thread stack and restore ra and s0 */
407 addi sp, s0, -STACKFRAME_SIZE_ON_STACK
408 REG_L ra, (STACKFRAME_SIZE_ON_STACK + STACKFRAME_RA)(sp)
409 REG_L s0, (STACKFRAME_SIZE_ON_STACK + STACKFRAME_FP)(sp)
410 addi sp, sp, STACKFRAME_SIZE_ON_STACK
411
412 ret
413SYM_FUNC_END(call_on_irq_stack)
414#endif /* CONFIG_IRQ_STACKS */
415
416/*
417 * Integer register context switch
418 * The callee-saved registers must be saved and restored.
419 *
420 * a0: previous task_struct (must be preserved across the switch)
421 * a1: next task_struct
422 *
423 * The value of a0 and a1 must be preserved by this function, as that's how
424 * arguments are passed to schedule_tail.
425 */
426SYM_FUNC_START(__switch_to)
427 /* Save context into prev->thread */
428 li a4, TASK_THREAD_RA
429 add a3, a0, a4
430 add a4, a1, a4
431 REG_S ra, TASK_THREAD_RA_RA(a3)
432 REG_S sp, TASK_THREAD_SP_RA(a3)
433 REG_S s0, TASK_THREAD_S0_RA(a3)
434 REG_S s1, TASK_THREAD_S1_RA(a3)
435 REG_S s2, TASK_THREAD_S2_RA(a3)
436 REG_S s3, TASK_THREAD_S3_RA(a3)
437 REG_S s4, TASK_THREAD_S4_RA(a3)
438 REG_S s5, TASK_THREAD_S5_RA(a3)
439 REG_S s6, TASK_THREAD_S6_RA(a3)
440 REG_S s7, TASK_THREAD_S7_RA(a3)
441 REG_S s8, TASK_THREAD_S8_RA(a3)
442 REG_S s9, TASK_THREAD_S9_RA(a3)
443 REG_S s10, TASK_THREAD_S10_RA(a3)
444 REG_S s11, TASK_THREAD_S11_RA(a3)
445
446 /* save the user space access flag */
447 csrr s0, CSR_STATUS
448 REG_S s0, TASK_THREAD_SUM_RA(a3)
449
450 /* Save the kernel shadow call stack pointer */
451 scs_save_current
452 /* Restore context from next->thread */
453 REG_L s0, TASK_THREAD_SUM_RA(a4)
454 li s1, SR_SUM
455 and s0, s0, s1
456 csrs CSR_STATUS, s0
457 REG_L ra, TASK_THREAD_RA_RA(a4)
458 REG_L sp, TASK_THREAD_SP_RA(a4)
459 REG_L s0, TASK_THREAD_S0_RA(a4)
460 REG_L s1, TASK_THREAD_S1_RA(a4)
461 REG_L s2, TASK_THREAD_S2_RA(a4)
462 REG_L s3, TASK_THREAD_S3_RA(a4)
463 REG_L s4, TASK_THREAD_S4_RA(a4)
464 REG_L s5, TASK_THREAD_S5_RA(a4)
465 REG_L s6, TASK_THREAD_S6_RA(a4)
466 REG_L s7, TASK_THREAD_S7_RA(a4)
467 REG_L s8, TASK_THREAD_S8_RA(a4)
468 REG_L s9, TASK_THREAD_S9_RA(a4)
469 REG_L s10, TASK_THREAD_S10_RA(a4)
470 REG_L s11, TASK_THREAD_S11_RA(a4)
471 /* The offset of thread_info in task_struct is zero. */
472 move tp, a1
473 /* Switch to the next shadow call stack */
474 scs_load_current
475 ret
476SYM_FUNC_END(__switch_to)
477
478#ifndef CONFIG_MMU
479#define do_page_fault do_trap_unknown
480#endif
481
482 .section ".rodata"
483 .align LGREG
484 /* Exception vector table */
485SYM_DATA_START_LOCAL(excp_vect_table)
486 RISCV_PTR do_trap_insn_misaligned
487 ALT_INSN_FAULT(RISCV_PTR do_trap_insn_fault)
488 RISCV_PTR do_trap_insn_illegal
489 RISCV_PTR do_trap_break
490 RISCV_PTR do_trap_load_misaligned
491 RISCV_PTR do_trap_load_fault
492 RISCV_PTR do_trap_store_misaligned
493 RISCV_PTR do_trap_store_fault
494 RISCV_PTR do_trap_ecall_u /* system call */
495 RISCV_PTR do_trap_ecall_s
496 RISCV_PTR do_trap_unknown
497 RISCV_PTR do_trap_ecall_m
498 /* instruction page fault */
499 ALT_PAGE_FAULT(RISCV_PTR do_page_fault)
500 RISCV_PTR do_page_fault /* load page fault */
501 RISCV_PTR do_trap_unknown
502 RISCV_PTR do_page_fault /* store page fault */
503 RISCV_PTR do_trap_unknown /* cause=16 */
504 RISCV_PTR do_trap_unknown /* cause=17 */
505 RISCV_PTR do_trap_software_check /* cause=18 is sw check exception */
506 RISCV_PTR do_trap_hardware_error /* hardware error (19) */
507SYM_DATA_END_LABEL(excp_vect_table, SYM_L_LOCAL, excp_vect_table_end)
508
509#ifndef CONFIG_MMU
510SYM_DATA_START(__user_rt_sigreturn)
511 li a7, __NR_rt_sigreturn
512 ecall
513SYM_DATA_END(__user_rt_sigreturn)
514#endif