projects/linux·blob master · refs

1074 lines · 26.7 KiB

1// SPDX-License-Identifier: GPL-2.0
2/*
3 * Exception handling code
4 *
5 * Copyright (C) 2019 ARM Ltd.
6 */
7
8#include <linux/context_tracking.h>
9#include <linux/irq-entry-common.h>
10#include <linux/kasan.h>
11#include <linux/linkage.h>
12#include <linux/livepatch.h>
13#include <linux/lockdep.h>
14#include <linux/ptrace.h>
15#include <linux/resume_user_mode.h>
16#include <linux/sched.h>
17#include <linux/sched/debug.h>
18#include <linux/thread_info.h>
19
20#include <asm/cpufeature.h>
21#include <asm/daifflags.h>
22#include <asm/esr.h>
23#include <asm/exception.h>
24#include <asm/fpsimd.h>
25#include <asm/irq_regs.h>
26#include <asm/kprobes.h>
27#include <asm/mmu.h>
28#include <asm/percpu.h>
29#include <asm/processor.h>
30#include <asm/sdei.h>
31#include <asm/stacktrace.h>
32#include <asm/sysreg.h>
33#include <asm/system_misc.h>
34
35/*
36 * Where the context being returned to had an active percpu GPR critical
37 * section, ensure that the offset and address GPRs are updated to match the
38 * current CPU.
39 *
40 * For simplicity we always update the GPRs when a critical section is active
41 * and preemption was *possible*, regardless of whether preemption actually
42 * occurred. Where preemption did not occur, the updates are redundant but not
43 * harmful.
44 */
45static __always_inline void irqentry_exit_pcpu_adjust(struct pt_regs *regs)
46{
47 int reg_pcp, reg_off, reg_addr;
48 unsigned long pcp, off, addr;
49 u16 gprs = regs->pcpu_gprs;
50
51 /*
52 * Zero means no active PCPU GPRs. As the PCPU GPRs must be distinct,
53 * a PCPU critical section cannot possibly use {x0,x0,x0}.
54 */
55 if (likely(!gprs))
56 return;
57
58 reg_pcp = FIELD_GET(PCPU_GPR_PCP, gprs);
59 reg_off = FIELD_GET(PCPU_GPR_OFF, gprs);
60 reg_addr = FIELD_GET(PCPU_GPR_ADDR, gprs);
61
62 pcp = pt_regs_read_reg(regs, reg_pcp);
63
64 off = __kern_my_cpu_offset();
65 pt_regs_write_reg(regs, reg_off, off);
66
67 addr = pcp + off;
68 pt_regs_write_reg(regs, reg_addr, addr);
69}
70
71/*
72 * Handle IRQ/context state management when entering from kernel mode.
73 * Before this function is called it is not safe to call regular kernel code,
74 * instrumentable code, or any code which may trigger an exception.
75 */
76static noinstr irqentry_state_t arm64_enter_from_kernel_mode(struct pt_regs *regs)
77{
78 irqentry_state_t state;
79
80 state = irqentry_enter_from_kernel_mode(regs);
81 mte_check_tfsr_entry();
82 mte_disable_tco_entry(current);
83
84 return state;
85}
86
87/*
88 * Handle IRQ/context state management when exiting to kernel mode.
89 * After this function returns it is not safe to call regular kernel code,
90 * instrumentable code, or any code which may trigger an exception.
91 */
92static void noinstr __arm64_exit_to_kernel_mode(struct pt_regs *regs,
93 irqentry_state_t state)
94{
95 local_daif_mask();
96 irqentry_exit_pcpu_adjust(regs);
97 mte_check_tfsr_exit();
98 irqentry_exit_to_kernel_mode_after_preempt(regs, state);
99}
100
101/*
102 * We are returning from the context which allows involuntary kernel preemption
103 */
104static void noinstr arm64_exit_to_kernel_mode_preempt(struct pt_regs *regs,
105 irqentry_state_t state)
106{
107 irqentry_exit_to_kernel_mode_preempt(regs, state);
108 __arm64_exit_to_kernel_mode(regs, state);
109}
110
111static void noinstr arm64_exit_to_kernel_mode(struct pt_regs *regs,
112 irqentry_state_t state)
113{
114 if (!regs_irqs_disabled(regs)) {
115 local_irq_disable();
116 arm64_exit_to_kernel_mode_preempt(regs, state);
117 return;
118 }
119
120 __arm64_exit_to_kernel_mode(regs, state);
121}
122
123static __always_inline void arm64_syscall_enter_from_user_mode(struct pt_regs *regs)
124{
125 enter_from_user_mode(regs);
126 mte_disable_tco_entry(current);
127 sme_enter_from_user_mode();
128}
129
130/*
131 * Handle IRQ/context state management when entering from user mode.
132 * Before this function is called it is not safe to call regular kernel code,
133 * instrumentable code, or any code which may trigger an exception.
134 */
135static __always_inline void arm64_enter_from_user_mode(struct pt_regs *regs)
136{
137 enter_from_user_mode(regs);
138 rseq_note_user_irq_entry();
139 mte_disable_tco_entry(current);
140 sme_enter_from_user_mode();
141}
142
143static __always_inline void arm64_syscall_exit_to_user_mode(struct pt_regs *regs)
144{
145 local_irq_disable();
146 syscall_exit_to_user_mode_prepare(regs);
147 local_daif_mask();
148 sme_exit_to_user_mode();
149 mte_check_tfsr_exit();
150 exit_to_user_mode();
151}
152
153/*
154 * Handle IRQ/context state management when exiting to user mode.
155 * After this function returns it is not safe to call regular kernel code,
156 * instrumentable code, or any code which may trigger an exception.
157 */
158static __always_inline void arm64_exit_to_user_mode(struct pt_regs *regs)
159{
160 local_irq_disable();
161 irqentry_exit_to_user_mode_prepare(regs);
162 local_daif_mask();
163 sme_exit_to_user_mode();
164 mte_check_tfsr_exit();
165 exit_to_user_mode();
166}
167
168asmlinkage void noinstr asm_exit_to_user_mode(struct pt_regs *regs)
169{
170 arm64_syscall_exit_to_user_mode(regs);
171}
172
173/*
174 * Handle IRQ/context state management when entering a debug exception from
175 * kernel mode. Before this function is called it is not safe to call regular
176 * kernel code, instrumentable code, or any code which may trigger an exception.
177 */
178static noinstr irqentry_state_t arm64_enter_el1_dbg(struct pt_regs *regs)
179{
180 irqentry_state_t state;
181
182 state.lockdep = lockdep_hardirqs_enabled();
183
184 lockdep_hardirqs_off(CALLER_ADDR0);
185 ct_nmi_enter();
186
187 trace_hardirqs_off_finish();
188
189 return state;
190}
191
192/*
193 * Handle IRQ/context state management when exiting a debug exception from
194 * kernel mode. After this function returns it is not safe to call regular
195 * kernel code, instrumentable code, or any code which may trigger an exception.
196 */
197static void noinstr arm64_exit_el1_dbg(struct pt_regs *regs,
198 irqentry_state_t state)
199{
200 if (state.lockdep) {
201 trace_hardirqs_on_prepare();
202 lockdep_hardirqs_on_prepare();
203 }
204
205 ct_nmi_exit();
206 if (state.lockdep)
207 lockdep_hardirqs_on(CALLER_ADDR0);
208}
209
210static void do_interrupt_handler(struct pt_regs *regs,
211 void (*handler)(struct pt_regs *))
212{
213 struct pt_regs *old_regs = set_irq_regs(regs);
214
215 if (on_thread_stack())
216 call_on_irq_stack(regs, handler);
217 else
218 handler(regs);
219
220 set_irq_regs(old_regs);
221}
222
223extern void (*handle_arch_irq)(struct pt_regs *);
224extern void (*handle_arch_fiq)(struct pt_regs *);
225
226static void noinstr __panic_unhandled(struct pt_regs *regs, const char *vector,
227 unsigned long esr)
228{
229 irqentry_nmi_enter(regs);
230
231 console_verbose();
232
233 pr_crit("Unhandled %s exception on CPU%d, ESR 0x%016lx -- %s\n",
234 vector, smp_processor_id(), esr,
235 esr_get_class_string(esr));
236
237 __show_regs(regs);
238 panic("Unhandled exception");
239}
240
241#define UNHANDLED(el, regsize, vector) \
242asmlinkage void noinstr el##_##regsize##_##vector##_handler(struct pt_regs *regs) \
243{ \
244 const char *desc = #regsize "-bit " #el " " #vector; \
245 __panic_unhandled(regs, desc, read_sysreg(esr_el1)); \
246}
247
248#ifdef CONFIG_ARM64_ERRATUM_1463225
249static DEFINE_PER_CPU(int, __in_cortex_a76_erratum_1463225_wa);
250
251static void cortex_a76_erratum_1463225_svc_handler(void)
252{
253 u64 reg, val;
254
255 if (!unlikely(test_thread_flag(TIF_SINGLESTEP)))
256 return;
257
258 if (!unlikely(this_cpu_has_cap(ARM64_WORKAROUND_1463225)))
259 return;
260
261 __this_cpu_write(__in_cortex_a76_erratum_1463225_wa, 1);
262 reg = read_sysreg(mdscr_el1);
263 val = reg | MDSCR_EL1_SS | MDSCR_EL1_KDE;
264 write_sysreg(val, mdscr_el1);
265 asm volatile("msr daifclr, #8");
266 isb();
267
268 /* We will have taken a single-step exception by this point */
269
270 write_sysreg(reg, mdscr_el1);
271 __this_cpu_write(__in_cortex_a76_erratum_1463225_wa, 0);
272}
273
274static __always_inline bool
275cortex_a76_erratum_1463225_debug_handler(struct pt_regs *regs)
276{
277 if (!__this_cpu_read(__in_cortex_a76_erratum_1463225_wa))
278 return false;
279
280 /*
281 * We've taken a dummy step exception from the kernel to ensure
282 * that interrupts are re-enabled on the syscall path. Return back
283 * to cortex_a76_erratum_1463225_svc_handler() with debug exceptions
284 * masked so that we can safely restore the mdscr and get on with
285 * handling the syscall.
286 */
287 regs->pstate |= PSR_D_BIT;
288 return true;
289}
290#else /* CONFIG_ARM64_ERRATUM_1463225 */
291static void cortex_a76_erratum_1463225_svc_handler(void) { }
292static bool cortex_a76_erratum_1463225_debug_handler(struct pt_regs *regs)
293{
294 return false;
295}
296#endif /* CONFIG_ARM64_ERRATUM_1463225 */
297
298/*
299 * As per the ABI exit SME streaming mode and clear the SVE state not
300 * shared with FPSIMD on syscall entry.
301 */
302static inline void fpsimd_syscall_enter(void)
303{
304 /* Ensure PSTATE.SM is clear, but leave PSTATE.ZA as-is. */
305 if (system_supports_sme())
306 sme_smstop_sm();
307
308 /*
309 * The CPU is not in streaming mode. If non-streaming SVE is not
310 * supported, there is no SVE state that needs to be discarded.
311 */
312 if (!system_supports_sve())
313 return;
314
315 if (test_thread_flag(TIF_SVE))
316 sve_flush_live();
317
318 /*
319 * Any live non-FPSIMD SVE state has been zeroed. Allow
320 * fpsimd_save_user_state() to lazily discard SVE state until either
321 * the live state is unbound or fpsimd_syscall_exit() is called.
322 */
323 __this_cpu_write(fpsimd_last_state.to_save, FP_STATE_FPSIMD);
324}
325
326static __always_inline void fpsimd_syscall_exit(void)
327{
328 if (!system_supports_sve())
329 return;
330
331 /*
332 * The current task's user FPSIMD/SVE/SME state is now bound to this
333 * CPU. The fpsimd_last_state.to_save value is either:
334 *
335 * - FP_STATE_FPSIMD, if the state has not been reloaded on this CPU
336 * since fpsimd_syscall_enter().
337 *
338 * - FP_STATE_CURRENT, if the state has been reloaded on this CPU at
339 * any point.
340 *
341 * Reset this to FP_STATE_CURRENT to stop lazy discarding.
342 */
343 __this_cpu_write(fpsimd_last_state.to_save, FP_STATE_CURRENT);
344}
345
346/*
347 * In debug exception context, we explicitly disable preemption despite
348 * having interrupts disabled.
349 * This serves two purposes: it makes it much less likely that we would
350 * accidentally schedule in exception context and it will force a warning
351 * if we somehow manage to schedule by accident.
352 */
353static void debug_exception_enter(struct pt_regs *regs)
354{
355 preempt_disable();
356
357 /* This code is a bit fragile. Test it. */
358 RCU_LOCKDEP_WARN(!rcu_is_watching(), "exception_enter didn't work");
359}
360NOKPROBE_SYMBOL(debug_exception_enter);
361
362static void debug_exception_exit(struct pt_regs *regs)
363{
364 preempt_enable_no_resched();
365}
366NOKPROBE_SYMBOL(debug_exception_exit);
367
368UNHANDLED(el1t, 64, sync)
369UNHANDLED(el1t, 64, irq)
370UNHANDLED(el1t, 64, fiq)
371UNHANDLED(el1t, 64, error)
372
373static void noinstr el1_abort(struct pt_regs *regs, unsigned long esr)
374{
375 unsigned long far = read_sysreg(far_el1);
376 irqentry_state_t state;
377
378 state = arm64_enter_from_kernel_mode(regs);
379 local_daif_inherit(regs);
380 do_mem_abort(far, esr, regs);
381 arm64_exit_to_kernel_mode(regs, state);
382}
383
384static void noinstr el1_pc(struct pt_regs *regs, unsigned long esr)
385{
386 unsigned long far = read_sysreg(far_el1);
387 irqentry_state_t state;
388
389 state = arm64_enter_from_kernel_mode(regs);
390 local_daif_inherit(regs);
391 do_sp_pc_abort(far, esr, regs);
392 arm64_exit_to_kernel_mode(regs, state);
393}
394
395static void noinstr el1_undef(struct pt_regs *regs, unsigned long esr)
396{
397 irqentry_state_t state;
398
399 state = arm64_enter_from_kernel_mode(regs);
400 local_daif_inherit(regs);
401 do_el1_undef(regs, esr);
402 arm64_exit_to_kernel_mode(regs, state);
403}
404
405static void noinstr el1_bti(struct pt_regs *regs, unsigned long esr)
406{
407 irqentry_state_t state;
408
409 state = arm64_enter_from_kernel_mode(regs);
410 local_daif_inherit(regs);
411 do_el1_bti(regs, esr);
412 arm64_exit_to_kernel_mode(regs, state);
413}
414
415static void noinstr el1_gcs(struct pt_regs *regs, unsigned long esr)
416{
417 irqentry_state_t state;
418
419 state = arm64_enter_from_kernel_mode(regs);
420 local_daif_inherit(regs);
421 do_el1_gcs(regs, esr);
422 arm64_exit_to_kernel_mode(regs, state);
423}
424
425static void noinstr el1_mops(struct pt_regs *regs, unsigned long esr)
426{
427 irqentry_state_t state;
428
429 state = arm64_enter_from_kernel_mode(regs);
430 local_daif_inherit(regs);
431 do_el1_mops(regs, esr);
432 arm64_exit_to_kernel_mode(regs, state);
433}
434
435static void noinstr el1_breakpt(struct pt_regs *regs, unsigned long esr)
436{
437 irqentry_state_t state;
438
439 state = arm64_enter_el1_dbg(regs);
440 debug_exception_enter(regs);
441 do_breakpoint(esr, regs);
442 debug_exception_exit(regs);
443 arm64_exit_el1_dbg(regs, state);
444}
445
446static void noinstr el1_softstp(struct pt_regs *regs, unsigned long esr)
447{
448 irqentry_state_t state;
449
450 state = arm64_enter_el1_dbg(regs);
451 if (!cortex_a76_erratum_1463225_debug_handler(regs)) {
452 debug_exception_enter(regs);
453 /*
454 * After handling a breakpoint, we suspend the breakpoint
455 * and use single-step to move to the next instruction.
456 * If we are stepping a suspended breakpoint there's nothing more to do:
457 * the single-step is complete.
458 */
459 if (!try_step_suspended_breakpoints(regs))
460 do_el1_softstep(esr, regs);
461 debug_exception_exit(regs);
462 }
463 arm64_exit_el1_dbg(regs, state);
464}
465
466static void noinstr el1_watchpt(struct pt_regs *regs, unsigned long esr)
467{
468 /* Watchpoints are the only debug exception to write FAR_EL1 */
469 unsigned long far = read_sysreg(far_el1);
470 irqentry_state_t state;
471
472 state = arm64_enter_el1_dbg(regs);
473 debug_exception_enter(regs);
474 do_watchpoint(far, esr, regs);
475 debug_exception_exit(regs);
476 arm64_exit_el1_dbg(regs, state);
477}
478
479static void noinstr el1_brk64(struct pt_regs *regs, unsigned long esr)
480{
481 irqentry_state_t state;
482
483 state = arm64_enter_el1_dbg(regs);
484 debug_exception_enter(regs);
485 do_el1_brk64(esr, regs);
486 debug_exception_exit(regs);
487 arm64_exit_el1_dbg(regs, state);
488}
489
490static void noinstr el1_fpac(struct pt_regs *regs, unsigned long esr)
491{
492 irqentry_state_t state;
493
494 state = arm64_enter_from_kernel_mode(regs);
495 local_daif_inherit(regs);
496 do_el1_fpac(regs, esr);
497 arm64_exit_to_kernel_mode(regs, state);
498}
499
500asmlinkage void noinstr el1h_64_sync_handler(struct pt_regs *regs)
501{
502 unsigned long esr = read_sysreg(esr_el1);
503
504 switch (ESR_ELx_EC(esr)) {
505 case ESR_ELx_EC_DABT_CUR:
506 case ESR_ELx_EC_IABT_CUR:
507 el1_abort(regs, esr);
508 break;
509 /*
510 * We don't handle ESR_ELx_EC_SP_ALIGN, since we will have hit a
511 * recursive exception when trying to push the initial pt_regs.
512 */
513 case ESR_ELx_EC_PC_ALIGN:
514 el1_pc(regs, esr);
515 break;
516 case ESR_ELx_EC_SYS64:
517 case ESR_ELx_EC_UNKNOWN:
518 el1_undef(regs, esr);
519 break;
520 case ESR_ELx_EC_BTI:
521 el1_bti(regs, esr);
522 break;
523 case ESR_ELx_EC_GCS:
524 el1_gcs(regs, esr);
525 break;
526 case ESR_ELx_EC_MOPS:
527 el1_mops(regs, esr);
528 break;
529 case ESR_ELx_EC_BREAKPT_CUR:
530 el1_breakpt(regs, esr);
531 break;
532 case ESR_ELx_EC_SOFTSTP_CUR:
533 el1_softstp(regs, esr);
534 break;
535 case ESR_ELx_EC_WATCHPT_CUR:
536 el1_watchpt(regs, esr);
537 break;
538 case ESR_ELx_EC_BRK64:
539 el1_brk64(regs, esr);
540 break;
541 case ESR_ELx_EC_FPAC:
542 el1_fpac(regs, esr);
543 break;
544 default:
545 __panic_unhandled(regs, "64-bit el1h sync", esr);
546 }
547}
548
549static __always_inline void __el1_pnmi(struct pt_regs *regs,
550 void (*handler)(struct pt_regs *))
551{
552 irqentry_state_t state;
553
554 state = irqentry_nmi_enter(regs);
555 do_interrupt_handler(regs, handler);
556 local_daif_mask();
557 irqentry_nmi_exit(regs, state);
558}
559
560static __always_inline void __el1_irq(struct pt_regs *regs,
561 void (*handler)(struct pt_regs *))
562{
563 irqentry_state_t state;
564
565 state = arm64_enter_from_kernel_mode(regs);
566
567 irq_enter_rcu();
568 do_interrupt_handler(regs, handler);
569 irq_exit_rcu();
570
571 arm64_exit_to_kernel_mode_preempt(regs, state);
572}
573static void noinstr el1_interrupt(struct pt_regs *regs,
574 void (*handler)(struct pt_regs *))
575{
576 write_sysreg(DAIF_PROCCTX_NOIRQ, daif);
577
578 if (IS_ENABLED(CONFIG_ARM64_PSEUDO_NMI) && regs_irqs_disabled(regs))
579 __el1_pnmi(regs, handler);
580 else
581 __el1_irq(regs, handler);
582}
583
584asmlinkage void noinstr el1h_64_irq_handler(struct pt_regs *regs)
585{
586 el1_interrupt(regs, handle_arch_irq);
587}
588
589asmlinkage void noinstr el1h_64_fiq_handler(struct pt_regs *regs)
590{
591 el1_interrupt(regs, handle_arch_fiq);
592}
593
594asmlinkage void noinstr el1h_64_error_handler(struct pt_regs *regs)
595{
596 unsigned long esr = read_sysreg(esr_el1);
597 irqentry_state_t state;
598
599 local_daif_restore(DAIF_ERRCTX);
600 state = irqentry_nmi_enter(regs);
601 do_serror(regs, esr);
602 local_daif_mask();
603 irqentry_nmi_exit(regs, state);
604}
605
606static void noinstr el0_da(struct pt_regs *regs, unsigned long esr)
607{
608 unsigned long far = read_sysreg(far_el1);
609
610 arm64_enter_from_user_mode(regs);
611 local_daif_restore(DAIF_PROCCTX);
612 do_mem_abort(far, esr, regs);
613 arm64_exit_to_user_mode(regs);
614}
615
616static void noinstr el0_ia(struct pt_regs *regs, unsigned long esr)
617{
618 unsigned long far = read_sysreg(far_el1);
619
620 /*
621 * We've taken an instruction abort from userspace and not yet
622 * re-enabled IRQs. If the address is a kernel address, apply
623 * BP hardening prior to enabling IRQs and pre-emption.
624 */
625 if (!is_ttbr0_addr(far))
626 arm64_apply_bp_hardening();
627
628 arm64_enter_from_user_mode(regs);
629 local_daif_restore(DAIF_PROCCTX);
630 do_mem_abort(far, esr, regs);
631 arm64_exit_to_user_mode(regs);
632}
633
634static void noinstr el0_fpsimd_acc(struct pt_regs *regs, unsigned long esr)
635{
636 arm64_enter_from_user_mode(regs);
637 local_daif_restore(DAIF_PROCCTX);
638 do_fpsimd_acc(esr, regs);
639 arm64_exit_to_user_mode(regs);
640}
641
642static void noinstr el0_sve_acc(struct pt_regs *regs, unsigned long esr)
643{
644 arm64_enter_from_user_mode(regs);
645 local_daif_restore(DAIF_PROCCTX);
646 do_sve_acc(esr, regs);
647 arm64_exit_to_user_mode(regs);
648}
649
650static void noinstr el0_sme_acc(struct pt_regs *regs, unsigned long esr)
651{
652 arm64_enter_from_user_mode(regs);
653 local_daif_restore(DAIF_PROCCTX);
654 do_sme_acc(esr, regs);
655 arm64_exit_to_user_mode(regs);
656}
657
658static void noinstr el0_fpsimd_exc(struct pt_regs *regs, unsigned long esr)
659{
660 arm64_enter_from_user_mode(regs);
661 local_daif_restore(DAIF_PROCCTX);
662 do_fpsimd_exc(esr, regs);
663 arm64_exit_to_user_mode(regs);
664}
665
666static void noinstr el0_sys(struct pt_regs *regs, unsigned long esr)
667{
668 arm64_enter_from_user_mode(regs);
669 local_daif_restore(DAIF_PROCCTX);
670 do_el0_sys(esr, regs);
671 arm64_exit_to_user_mode(regs);
672}
673
674static void noinstr el0_pc(struct pt_regs *regs, unsigned long esr)
675{
676 unsigned long far = read_sysreg(far_el1);
677
678 if (!is_ttbr0_addr(instruction_pointer(regs)))
679 arm64_apply_bp_hardening();
680
681 arm64_enter_from_user_mode(regs);
682 local_daif_restore(DAIF_PROCCTX);
683 do_sp_pc_abort(far, esr, regs);
684 arm64_exit_to_user_mode(regs);
685}
686
687static void noinstr el0_sp(struct pt_regs *regs, unsigned long esr)
688{
689 arm64_enter_from_user_mode(regs);
690 local_daif_restore(DAIF_PROCCTX);
691 do_sp_pc_abort(regs->sp, esr, regs);
692 arm64_exit_to_user_mode(regs);
693}
694
695static void noinstr el0_undef(struct pt_regs *regs, unsigned long esr)
696{
697 arm64_enter_from_user_mode(regs);
698 local_daif_restore(DAIF_PROCCTX);
699 do_el0_undef(regs, esr);
700 arm64_exit_to_user_mode(regs);
701}
702
703static void noinstr el0_bti(struct pt_regs *regs)
704{
705 arm64_enter_from_user_mode(regs);
706 local_daif_restore(DAIF_PROCCTX);
707 do_el0_bti(regs);
708 arm64_exit_to_user_mode(regs);
709}
710
711static void noinstr el0_mops(struct pt_regs *regs, unsigned long esr)
712{
713 arm64_enter_from_user_mode(regs);
714 local_daif_restore(DAIF_PROCCTX);
715 do_el0_mops(regs, esr);
716 arm64_exit_to_user_mode(regs);
717}
718
719static void noinstr el0_gcs(struct pt_regs *regs, unsigned long esr)
720{
721 arm64_enter_from_user_mode(regs);
722 local_daif_restore(DAIF_PROCCTX);
723 do_el0_gcs(regs, esr);
724 arm64_exit_to_user_mode(regs);
725}
726
727static void noinstr el0_inv(struct pt_regs *regs, unsigned long esr)
728{
729 arm64_enter_from_user_mode(regs);
730 local_daif_restore(DAIF_PROCCTX);
731 bad_el0_sync(regs, 0, esr);
732 arm64_exit_to_user_mode(regs);
733}
734
735static void noinstr el0_breakpt(struct pt_regs *regs, unsigned long esr)
736{
737 if (!is_ttbr0_addr(regs->pc))
738 arm64_apply_bp_hardening();
739
740 arm64_enter_from_user_mode(regs);
741 debug_exception_enter(regs);
742 do_breakpoint(esr, regs);
743 debug_exception_exit(regs);
744 local_daif_restore(DAIF_PROCCTX);
745 arm64_exit_to_user_mode(regs);
746}
747
748static void noinstr el0_softstp(struct pt_regs *regs, unsigned long esr)
749{
750 bool step_done;
751
752 if (!is_ttbr0_addr(regs->pc))
753 arm64_apply_bp_hardening();
754
755 arm64_enter_from_user_mode(regs);
756 /*
757 * After handling a breakpoint, we suspend the breakpoint
758 * and use single-step to move to the next instruction.
759 * If we are stepping a suspended breakpoint there's nothing more to do:
760 * the single-step is complete.
761 */
762 step_done = try_step_suspended_breakpoints(regs);
763 local_daif_restore(DAIF_PROCCTX);
764 if (!step_done)
765 do_el0_softstep(esr, regs);
766 arm64_exit_to_user_mode(regs);
767}
768
769static void noinstr el0_watchpt(struct pt_regs *regs, unsigned long esr)
770{
771 /* Watchpoints are the only debug exception to write FAR_EL1 */
772 unsigned long far = read_sysreg(far_el1);
773
774 arm64_enter_from_user_mode(regs);
775 debug_exception_enter(regs);
776 do_watchpoint(far, esr, regs);
777 debug_exception_exit(regs);
778 local_daif_restore(DAIF_PROCCTX);
779 arm64_exit_to_user_mode(regs);
780}
781
782static void noinstr el0_brk64(struct pt_regs *regs, unsigned long esr)
783{
784 arm64_enter_from_user_mode(regs);
785 local_daif_restore(DAIF_PROCCTX);
786 do_el0_brk64(esr, regs);
787 arm64_exit_to_user_mode(regs);
788}
789
790static void noinstr el0_svc(struct pt_regs *regs)
791{
792 arm64_syscall_enter_from_user_mode(regs);
793 cortex_a76_erratum_1463225_svc_handler();
794 fpsimd_syscall_enter();
795 local_daif_restore(DAIF_PROCCTX);
796 do_el0_svc(regs);
797 arm64_syscall_exit_to_user_mode(regs);
798 fpsimd_syscall_exit();
799}
800
801static void noinstr el0_fpac(struct pt_regs *regs, unsigned long esr)
802{
803 arm64_enter_from_user_mode(regs);
804 local_daif_restore(DAIF_PROCCTX);
805 do_el0_fpac(regs, esr);
806 arm64_exit_to_user_mode(regs);
807}
808
809asmlinkage void noinstr el0t_64_sync_handler(struct pt_regs *regs)
810{
811 unsigned long esr = read_sysreg(esr_el1);
812
813 switch (ESR_ELx_EC(esr)) {
814 case ESR_ELx_EC_SVC64:
815 el0_svc(regs);
816 break;
817 case ESR_ELx_EC_DABT_LOW:
818 el0_da(regs, esr);
819 break;
820 case ESR_ELx_EC_IABT_LOW:
821 el0_ia(regs, esr);
822 break;
823 case ESR_ELx_EC_FP_ASIMD:
824 el0_fpsimd_acc(regs, esr);
825 break;
826 case ESR_ELx_EC_SVE:
827 el0_sve_acc(regs, esr);
828 break;
829 case ESR_ELx_EC_SME:
830 el0_sme_acc(regs, esr);
831 break;
832 case ESR_ELx_EC_FP_EXC64:
833 el0_fpsimd_exc(regs, esr);
834 break;
835 case ESR_ELx_EC_SYS64:
836 case ESR_ELx_EC_WFx:
837 el0_sys(regs, esr);
838 break;
839 case ESR_ELx_EC_SP_ALIGN:
840 el0_sp(regs, esr);
841 break;
842 case ESR_ELx_EC_PC_ALIGN:
843 el0_pc(regs, esr);
844 break;
845 case ESR_ELx_EC_UNKNOWN:
846 el0_undef(regs, esr);
847 break;
848 case ESR_ELx_EC_BTI:
849 el0_bti(regs);
850 break;
851 case ESR_ELx_EC_MOPS:
852 el0_mops(regs, esr);
853 break;
854 case ESR_ELx_EC_GCS:
855 el0_gcs(regs, esr);
856 break;
857 case ESR_ELx_EC_BREAKPT_LOW:
858 el0_breakpt(regs, esr);
859 break;
860 case ESR_ELx_EC_SOFTSTP_LOW:
861 el0_softstp(regs, esr);
862 break;
863 case ESR_ELx_EC_WATCHPT_LOW:
864 el0_watchpt(regs, esr);
865 break;
866 case ESR_ELx_EC_BRK64:
867 el0_brk64(regs, esr);
868 break;
869 case ESR_ELx_EC_FPAC:
870 el0_fpac(regs, esr);
871 break;
872 default:
873 el0_inv(regs, esr);
874 }
875}
876
877static void noinstr el0_interrupt(struct pt_regs *regs,
878 void (*handler)(struct pt_regs *))
879{
880 arm64_enter_from_user_mode(regs);
881
882 write_sysreg(DAIF_PROCCTX_NOIRQ, daif);
883
884 if (regs->pc & BIT(55))
885 arm64_apply_bp_hardening();
886
887 irq_enter_rcu();
888 do_interrupt_handler(regs, handler);
889 irq_exit_rcu();
890
891 arm64_exit_to_user_mode(regs);
892}
893
894static void noinstr __el0_irq_handler_common(struct pt_regs *regs)
895{
896 el0_interrupt(regs, handle_arch_irq);
897}
898
899asmlinkage void noinstr el0t_64_irq_handler(struct pt_regs *regs)
900{
901 __el0_irq_handler_common(regs);
902}
903
904static void noinstr __el0_fiq_handler_common(struct pt_regs *regs)
905{
906 el0_interrupt(regs, handle_arch_fiq);
907}
908
909asmlinkage void noinstr el0t_64_fiq_handler(struct pt_regs *regs)
910{
911 __el0_fiq_handler_common(regs);
912}
913
914static void noinstr __el0_error_handler_common(struct pt_regs *regs)
915{
916 unsigned long esr = read_sysreg(esr_el1);
917 irqentry_state_t state;
918
919 arm64_enter_from_user_mode(regs);
920 local_daif_restore(DAIF_ERRCTX);
921 state = irqentry_nmi_enter(regs);
922 do_serror(regs, esr);
923 irqentry_nmi_exit(regs, state);
924 local_daif_restore(DAIF_PROCCTX);
925 arm64_exit_to_user_mode(regs);
926}
927
928asmlinkage void noinstr el0t_64_error_handler(struct pt_regs *regs)
929{
930 __el0_error_handler_common(regs);
931}
932
933#ifdef CONFIG_COMPAT
934static void noinstr el0_cp15(struct pt_regs *regs, unsigned long esr)
935{
936 arm64_enter_from_user_mode(regs);
937 local_daif_restore(DAIF_PROCCTX);
938 do_el0_cp15(esr, regs);
939 arm64_exit_to_user_mode(regs);
940}
941
942static void noinstr el0_svc_compat(struct pt_regs *regs)
943{
944 arm64_syscall_enter_from_user_mode(regs);
945 cortex_a76_erratum_1463225_svc_handler();
946 local_daif_restore(DAIF_PROCCTX);
947 do_el0_svc_compat(regs);
948 arm64_syscall_exit_to_user_mode(regs);
949}
950
951static void noinstr el0_bkpt32(struct pt_regs *regs, unsigned long esr)
952{
953 arm64_enter_from_user_mode(regs);
954 local_daif_restore(DAIF_PROCCTX);
955 do_bkpt32(esr, regs);
956 arm64_exit_to_user_mode(regs);
957}
958
959asmlinkage void noinstr el0t_32_sync_handler(struct pt_regs *regs)
960{
961 unsigned long esr = read_sysreg(esr_el1);
962
963 switch (ESR_ELx_EC(esr)) {
964 case ESR_ELx_EC_SVC32:
965 el0_svc_compat(regs);
966 break;
967 case ESR_ELx_EC_DABT_LOW:
968 el0_da(regs, esr);
969 break;
970 case ESR_ELx_EC_IABT_LOW:
971 el0_ia(regs, esr);
972 break;
973 case ESR_ELx_EC_FP_ASIMD:
974 el0_fpsimd_acc(regs, esr);
975 break;
976 case ESR_ELx_EC_FP_EXC32:
977 el0_fpsimd_exc(regs, esr);
978 break;
979 case ESR_ELx_EC_PC_ALIGN:
980 el0_pc(regs, esr);
981 break;
982 case ESR_ELx_EC_UNKNOWN:
983 case ESR_ELx_EC_CP14_MR:
984 case ESR_ELx_EC_CP14_LS:
985 case ESR_ELx_EC_CP14_64:
986 el0_undef(regs, esr);
987 break;
988 case ESR_ELx_EC_CP15_32:
989 case ESR_ELx_EC_CP15_64:
990 el0_cp15(regs, esr);
991 break;
992 case ESR_ELx_EC_BREAKPT_LOW:
993 el0_breakpt(regs, esr);
994 break;
995 case ESR_ELx_EC_SOFTSTP_LOW:
996 el0_softstp(regs, esr);
997 break;
998 case ESR_ELx_EC_WATCHPT_LOW:
999 el0_watchpt(regs, esr);
1000 break;
1001 case ESR_ELx_EC_BKPT32:
1002 el0_bkpt32(regs, esr);
1003 break;
1004 default:
1005 el0_inv(regs, esr);
1006 }
1007}
1008
1009asmlinkage void noinstr el0t_32_irq_handler(struct pt_regs *regs)
1010{
1011 __el0_irq_handler_common(regs);
1012}
1013
1014asmlinkage void noinstr el0t_32_fiq_handler(struct pt_regs *regs)
1015{
1016 __el0_fiq_handler_common(regs);
1017}
1018
1019asmlinkage void noinstr el0t_32_error_handler(struct pt_regs *regs)
1020{
1021 __el0_error_handler_common(regs);
1022}
1023#else /* CONFIG_COMPAT */
1024UNHANDLED(el0t, 32, sync)
1025UNHANDLED(el0t, 32, irq)
1026UNHANDLED(el0t, 32, fiq)
1027UNHANDLED(el0t, 32, error)
1028#endif /* CONFIG_COMPAT */
1029
1030asmlinkage void noinstr __noreturn handle_bad_stack(struct pt_regs *regs)
1031{
1032 unsigned long esr = read_sysreg(esr_el1);
1033 unsigned long far = read_sysreg(far_el1);
1034
1035 irqentry_nmi_enter(regs);
1036 panic_bad_stack(regs, esr, far);
1037}
1038
1039#ifdef CONFIG_ARM_SDE_INTERFACE
1040asmlinkage noinstr unsigned long
1041__sdei_handler(struct pt_regs *regs, struct sdei_registered_event *arg)
1042{
1043 irqentry_state_t state;
1044 unsigned long ret;
1045
1046 /*
1047 * We didn't take an exception to get here, so the HW hasn't
1048 * set/cleared bits in PSTATE that we may rely on.
1049 *
1050 * The original SDEI spec (ARM DEN 0054A) can be read ambiguously as to
1051 * whether PSTATE bits are inherited unchanged or generated from
1052 * scratch, and the TF-A implementation always clears PAN and always
1053 * clears UAO. There are no other known implementations.
1054 *
1055 * Subsequent revisions (ARM DEN 0054B) follow the usual rules for how
1056 * PSTATE is modified upon architectural exceptions, and so PAN is
1057 * either inherited or set per SCTLR_ELx.SPAN, and UAO is always
1058 * cleared.
1059 *
1060 * We must explicitly reset PAN to the expected state, including
1061 * clearing it when the host isn't using it, in case a VM had it set.
1062 */
1063 if (system_uses_hw_pan())
1064 set_pstate_pan(1);
1065 else if (cpu_has_pan())
1066 set_pstate_pan(0);
1067
1068 state = irqentry_nmi_enter(regs);
1069 ret = do_sdei_event(regs, arg);
1070 irqentry_nmi_exit(regs, state);
1071
1072 return ret;
1073}
1074#endif /* CONFIG_ARM_SDE_INTERFACE */