projects/linux·blob master · refs

537 lines · 14.9 KiB

1/* SPDX-License-Identifier: GPL-2.0 */
2
3#ifndef _ASM_PPC_ENTRY_COMMON_H
4#define _ASM_PPC_ENTRY_COMMON_H
5
6#include <asm/cputime.h>
7#include <asm/interrupt.h>
8#include <asm/runlatch.h>
9#include <asm/stacktrace.h>
10#include <asm/switch_to.h>
11#include <asm/tm.h>
12
13#ifdef CONFIG_PPC_IRQ_SOFT_MASK_DEBUG
14/*
15 * WARN/BUG is handled with a program interrupt so minimise checks here to
16 * avoid recursion and maximise the chance of getting the first oops handled.
17 */
18#define INT_SOFT_MASK_BUG_ON(regs, cond) \
19do { \
20 if ((user_mode(regs) || (TRAP(regs) != INTERRUPT_PROGRAM))) \
21 BUG_ON(cond); \
22} while (0)
23#else
24#define INT_SOFT_MASK_BUG_ON(regs, cond)
25#endif
26
27#ifdef CONFIG_PPC_BOOK3S_64
28extern char __end_soft_masked[];
29bool search_kernel_soft_mask_table(unsigned long addr);
30unsigned long search_kernel_restart_table(unsigned long addr);
31
32DECLARE_STATIC_KEY_FALSE(interrupt_exit_not_reentrant);
33
34static inline bool is_implicit_soft_masked(struct pt_regs *regs)
35{
36 if (user_mode(regs))
37 return false;
38
39 if (regs->nip >= (unsigned long)__end_soft_masked)
40 return false;
41
42 return search_kernel_soft_mask_table(regs->nip);
43}
44
45static inline void srr_regs_clobbered(void)
46{
47 local_paca->srr_valid = 0;
48 local_paca->hsrr_valid = 0;
49}
50#else
51static inline unsigned long search_kernel_restart_table(unsigned long addr)
52{
53 return 0;
54}
55
56static inline bool is_implicit_soft_masked(struct pt_regs *regs)
57{
58 return false;
59}
60
61static inline void srr_regs_clobbered(void)
62{
63}
64#endif
65
66static inline void nap_adjust_return(struct pt_regs *regs)
67{
68#ifdef CONFIG_PPC_970_NAP
69 /*
70 * Adjust the nap return address before irq_exit_rcu(). irq_exit_rcu()
71 * may invoke softirqs with interrupts re-enabled, allowing a nested
72 * async interrupt to arrive. If _TLF_NAPPING is still set at that
73 * point, the nested interrupt would erroneously redirect its own
74 * return address to power4_idle_nap_return, corrupting the stack.
75 */
76 if (unlikely(test_thread_local_flags(_TLF_NAPPING))) {
77 /* Can avoid a test-and-clear because NMIs do not call this */
78 clear_thread_local_flags(_TLF_NAPPING);
79 regs_set_return_ip(regs, (unsigned long)power4_idle_nap_return);
80 }
81#endif
82}
83
84static __always_inline void booke_load_dbcr0(void)
85{
86#ifdef CONFIG_PPC_ADV_DEBUG_REGS
87 unsigned long dbcr0 = current->thread.debug.dbcr0;
88
89 if (likely(!(dbcr0 & DBCR0_IDM)))
90 return;
91
92 /*
93 * Check to see if the dbcr0 register is set up to debug.
94 * Use the internal debug mode bit to do this.
95 */
96 mtmsr(mfmsr() & ~MSR_DE);
97 if (IS_ENABLED(CONFIG_PPC32)) {
98 isync();
99 global_dbcr0[smp_processor_id()] = mfspr(SPRN_DBCR0);
100 }
101 mtspr(SPRN_DBCR0, dbcr0);
102 mtspr(SPRN_DBSR, -1);
103#endif
104}
105
106static inline void booke_restore_dbcr0(void)
107{
108#ifdef CONFIG_PPC_ADV_DEBUG_REGS
109 unsigned long dbcr0 = current->thread.debug.dbcr0;
110
111 if (IS_ENABLED(CONFIG_PPC32) && unlikely(dbcr0 & DBCR0_IDM)) {
112 mtspr(SPRN_DBSR, -1);
113 mtspr(SPRN_DBCR0, global_dbcr0[smp_processor_id()]);
114 }
115#endif
116}
117
118static inline void check_return_regs_valid(struct pt_regs *regs)
119{
120#ifdef CONFIG_PPC_BOOK3S_64
121 unsigned long trap, srr0, srr1;
122 static bool warned;
123 u8 *validp;
124 char *h;
125
126 if (trap_is_scv(regs))
127 return;
128
129 trap = TRAP(regs);
130 // EE in HV mode sets HSRRs like 0xea0
131 if (cpu_has_feature(CPU_FTR_HVMODE) && trap == INTERRUPT_EXTERNAL)
132 trap = 0xea0;
133
134 switch (trap) {
135 case 0x980:
136 case INTERRUPT_H_DATA_STORAGE:
137 case 0xe20:
138 case 0xe40:
139 case INTERRUPT_HMI:
140 case 0xe80:
141 case 0xea0:
142 case INTERRUPT_H_FAC_UNAVAIL:
143 case 0x1200:
144 case 0x1500:
145 case 0x1600:
146 case 0x1800:
147 validp = &local_paca->hsrr_valid;
148 if (!READ_ONCE(*validp))
149 return;
150
151 srr0 = mfspr(SPRN_HSRR0);
152 srr1 = mfspr(SPRN_HSRR1);
153 h = "H";
154
155 break;
156 default:
157 validp = &local_paca->srr_valid;
158 if (!READ_ONCE(*validp))
159 return;
160
161 srr0 = mfspr(SPRN_SRR0);
162 srr1 = mfspr(SPRN_SRR1);
163 h = "";
164 break;
165 }
166
167 if (srr0 == regs->nip && srr1 == regs->msr)
168 return;
169
170 /*
171 * A NMI / soft-NMI interrupt may have come in after we found
172 * srr_valid and before the SRRs are loaded. The interrupt then
173 * comes in and clobbers SRRs and clears srr_valid. Then we load
174 * the SRRs here and test them above and find they don't match.
175 *
176 * Test validity again after that, to catch such false positives.
177 *
178 * This test in general will have some window for false negatives
179 * and may not catch and fix all such cases if an NMI comes in
180 * later and clobbers SRRs without clearing srr_valid, but hopefully
181 * such things will get caught most of the time, statistically
182 * enough to be able to get a warning out.
183 */
184 if (!READ_ONCE(*validp))
185 return;
186
187 if (!data_race(warned)) {
188 data_race(warned = true);
189 pr_warn("%sSRR0 was: %lx should be: %lx\n", h, srr0, regs->nip);
190 pr_warn("%sSRR1 was: %lx should be: %lx\n", h, srr1, regs->msr);
191 show_regs(regs);
192 }
193
194 WRITE_ONCE(*validp, 0); /* fixup */
195#endif
196}
197
198static inline void arch_interrupt_enter_prepare(struct pt_regs *regs)
199{
200#ifdef CONFIG_PPC64
201 irq_soft_mask_set(IRQS_ALL_DISABLED);
202
203 /*
204 * If the interrupt was taken with HARD_DIS clear, then enable MSR[EE].
205 * Asynchronous interrupts get here with HARD_DIS set (see below), so
206 * this enables MSR[EE] for synchronous interrupts. IRQs remain
207 * soft-masked. The interrupt handler may later call
208 * interrupt_cond_local_irq_enable() to achieve a regular process
209 * context.
210 */
211 if (!(local_paca->irq_happened & PACA_IRQ_HARD_DIS)) {
212 INT_SOFT_MASK_BUG_ON(regs, !(regs->msr & MSR_EE));
213 __hard_irq_enable();
214 } else {
215 __hard_RI_enable();
216 }
217 /* Enable MSR[RI] early, to support kernel SLB and hash faults */
218#endif
219
220 if (!regs_irqs_disabled(regs))
221 trace_hardirqs_off();
222
223 if (user_mode(regs)) {
224 kuap_lock();
225 } else {
226 kuap_save_and_lock(regs);
227 /*
228 * CT_WARN_ON comes here via program_check_exception,
229 * so avoid recursion.
230 */
231 if (TRAP(regs) != INTERRUPT_PROGRAM)
232 CT_WARN_ON(ct_state() != CT_STATE_KERNEL &&
233 ct_state() != CT_STATE_IDLE);
234 INT_SOFT_MASK_BUG_ON(regs, is_implicit_soft_masked(regs));
235 INT_SOFT_MASK_BUG_ON(regs, regs_irqs_disabled(regs) &&
236 search_kernel_restart_table(regs->nip));
237 }
238 INT_SOFT_MASK_BUG_ON(regs, !regs_irqs_disabled(regs) &&
239 !(regs->msr & MSR_EE));
240
241 booke_restore_dbcr0();
242}
243
244/*
245 * Care should be taken to note that arch_interrupt_exit_prepare and
246 * arch_interrupt_async_exit_prepare do not necessarily return immediately to
247 * regs context (e.g., if regs is usermode, we don't necessarily return to
248 * user mode). Other interrupts might be taken between here and return,
249 * context switch / preemption may occur in the exit path after this, or a
250 * signal may be delivered, etc.
251 *
252 * The real interrupt exit code is platform specific, e.g.,
253 * interrupt_exit_user_prepare / interrupt_exit_kernel_prepare for 64s.
254 *
255 * However arch_interrupt_nmi_exit_prepare does return directly to regs, because
256 * NMIs do not do "exit work" or replay soft-masked interrupts.
257 */
258static inline void arch_interrupt_exit_prepare(struct pt_regs *regs)
259{
260 if (user_mode(regs)) {
261 BUG_ON(regs_is_unrecoverable(regs));
262 BUG_ON(regs_irqs_disabled(regs));
263 /*
264 * We don't need to restore AMR on the way back to userspace for KUAP.
265 * AMR can only have been unlocked if we interrupted the kernel.
266 */
267 kuap_assert_locked();
268 }
269
270 /* irqentry_exit expects to be called with interrupts disabled */
271 hard_irq_disable();
272}
273
274static inline void arch_interrupt_async_enter_prepare(struct pt_regs *regs)
275{
276#ifdef CONFIG_PPC64
277 /* Ensure arch_interrupt_enter_prepare does not enable MSR[EE] */
278 local_paca->irq_happened |= PACA_IRQ_HARD_DIS;
279#endif
280 arch_interrupt_enter_prepare(regs);
281#ifdef CONFIG_PPC_BOOK3S_64
282 /*
283 * RI=1 is set by arch_interrupt_enter_prepare, so this thread flags access
284 * has to come afterward (it can cause SLB faults).
285 */
286 if (cpu_has_feature(CPU_FTR_CTRL) &&
287 !test_thread_local_flags(_TLF_RUNLATCH))
288 __ppc64_runlatch_on();
289#endif
290}
291
292static inline void arch_interrupt_async_exit_prepare(struct pt_regs *regs)
293{
294 arch_interrupt_exit_prepare(regs);
295}
296
297struct interrupt_nmi_state {
298#ifdef CONFIG_PPC64
299 u8 irq_soft_mask;
300 u8 irq_happened;
301 u8 ftrace_enabled;
302 u64 softe;
303#endif
304};
305
306static inline bool nmi_disables_ftrace(struct pt_regs *regs)
307{
308 /* Allow DEC and PMI to be traced when they are soft-NMI */
309 if (IS_ENABLED(CONFIG_PPC_BOOK3S_64)) {
310 if (TRAP(regs) == INTERRUPT_DECREMENTER)
311 return false;
312 if (TRAP(regs) == INTERRUPT_PERFMON)
313 return false;
314 }
315 if (IS_ENABLED(CONFIG_PPC_BOOK3E_64)) {
316 if (TRAP(regs) == INTERRUPT_PERFMON)
317 return false;
318 }
319
320 return true;
321}
322
323static inline void arch_interrupt_nmi_enter_prepare(struct pt_regs *regs,
324 struct interrupt_nmi_state *state)
325{
326#ifdef CONFIG_PPC64
327 state->irq_soft_mask = local_paca->irq_soft_mask;
328 state->irq_happened = local_paca->irq_happened;
329 state->softe = regs->softe;
330
331 /*
332 * Set IRQS_ALL_DISABLED unconditionally so irqs_disabled() does
333 * the right thing, and set IRQ_HARD_DIS. We do not want to reconcile
334 * because that goes through irq tracing which we don't want in NMI.
335 */
336 local_paca->irq_soft_mask = IRQS_ALL_DISABLED;
337 local_paca->irq_happened |= PACA_IRQ_HARD_DIS;
338
339 if (!(regs->msr & MSR_EE) || is_implicit_soft_masked(regs)) {
340 /*
341 * Adjust regs->softe to be soft-masked if it had not been
342 * reconcied (e.g., interrupt entry with MSR[EE]=0 but softe
343 * not yet set disabled), or if it was in an implicit soft
344 * masked state. This makes regs_irqs_disabled(regs)
345 * behave as expected.
346 */
347 regs->softe = IRQS_ALL_DISABLED;
348 }
349
350 __hard_RI_enable();
351
352 /* Don't do any per-CPU operations until interrupt state is fixed */
353
354 if (nmi_disables_ftrace(regs)) {
355 state->ftrace_enabled = this_cpu_get_ftrace_enabled();
356 this_cpu_set_ftrace_enabled(0);
357 }
358#endif
359}
360
361static inline void arch_interrupt_nmi_exit_prepare(struct pt_regs *regs,
362 struct interrupt_nmi_state *state)
363{
364 /*
365 * nmi does not call nap_adjust_return because nmi should not create
366 * new work to do (must use irq_work for that).
367 */
368
369#ifdef CONFIG_PPC64
370#ifdef CONFIG_PPC_BOOK3S
371 if (regs_irqs_disabled(regs)) {
372 unsigned long rst = search_kernel_restart_table(regs->nip);
373
374 if (rst)
375 regs_set_return_ip(regs, rst);
376 }
377#endif
378
379 if (nmi_disables_ftrace(regs))
380 this_cpu_set_ftrace_enabled(state->ftrace_enabled);
381
382 /* Check we didn't change the pending interrupt mask. */
383 WARN_ON_ONCE((state->irq_happened | PACA_IRQ_HARD_DIS) != local_paca->irq_happened);
384 regs->softe = state->softe;
385 local_paca->irq_happened = state->irq_happened;
386 local_paca->irq_soft_mask = state->irq_soft_mask;
387#endif
388}
389
390static __always_inline void arch_enter_from_user_mode(struct pt_regs *regs)
391{
392 kuap_lock();
393
394 if (IS_ENABLED(CONFIG_PPC_IRQ_SOFT_MASK_DEBUG))
395 BUG_ON(irq_soft_mask_return() != IRQS_ALL_DISABLED);
396
397 BUG_ON(regs_is_unrecoverable(regs));
398 BUG_ON(!user_mode(regs));
399 BUG_ON(regs_irqs_disabled(regs));
400
401#ifdef CONFIG_PPC_PKEY
402 if (mmu_has_feature(MMU_FTR_PKEY) && trap_is_syscall(regs)) {
403 unsigned long amr, iamr;
404 bool flush_needed = false;
405 /*
406 * When entering from userspace we mostly have the AMR/IAMR
407 * different from kernel default values. Hence don't compare.
408 */
409 amr = mfspr(SPRN_AMR);
410 iamr = mfspr(SPRN_IAMR);
411 regs->amr = amr;
412 regs->iamr = iamr;
413 if (mmu_has_feature(MMU_FTR_KUAP)) {
414 mtspr(SPRN_AMR, AMR_KUAP_BLOCKED);
415 flush_needed = true;
416 }
417 if (mmu_has_feature(MMU_FTR_BOOK3S_KUEP)) {
418 mtspr(SPRN_IAMR, AMR_KUEP_BLOCKED);
419 flush_needed = true;
420 }
421 if (flush_needed)
422 isync();
423 }
424#endif
425 kuap_assert_locked();
426 booke_restore_dbcr0();
427 account_cpu_user_entry();
428 account_stolen_time();
429
430 /*
431 * This is not required for the syscall exit path, but makes the
432 * stack frame look nicer. If this was initialised in the first stack
433 * frame, or if the unwinder was taught the first stack frame always
434 * returns to user with IRQS_ENABLED, this store could be avoided!
435 */
436 irq_soft_mask_regs_set_state(regs, IRQS_ENABLED);
437
438 /*
439 * If system call is called with TM active, set _TIF_RESTOREALL to
440 * prevent RFSCV being used to return to userspace, because POWER9
441 * TM implementation has problems with this instruction returning to
442 * transactional state. Final register values are not relevant because
443 * the transaction will be aborted upon return anyway. Or in the case
444 * of unsupported_scv SIGILL fault, the return state does not much
445 * matter because it's an edge case.
446 */
447 if (IS_ENABLED(CONFIG_PPC_TRANSACTIONAL_MEM) &&
448 unlikely(MSR_TM_TRANSACTIONAL(regs->msr)))
449 set_bits(_TIF_RESTOREALL, &current_thread_info()->flags);
450
451 /*
452 * If the system call was made with a transaction active, doom it and
453 * return without performing the system call. Unless it was an
454 * unsupported scv vector, in which case it's treated like an illegal
455 * instruction.
456 */
457#ifdef CONFIG_PPC_TRANSACTIONAL_MEM
458 if (unlikely(MSR_TM_TRANSACTIONAL(regs->msr)) &&
459 !trap_is_unsupported_scv(regs)) {
460 /* Enable TM in the kernel, and disable EE (for scv) */
461 hard_irq_disable();
462 mtmsr(mfmsr() | MSR_TM);
463
464 /* tabort, this dooms the transaction, nothing else */
465 asm volatile(".long 0x7c00071d | ((%0) << 16)"
466 :: "r"(TM_CAUSE_SYSCALL | TM_CAUSE_PERSISTENT));
467
468 /*
469 * Userspace will never see the return value. Execution will
470 * resume after the tbegin. of the aborted transaction with the
471 * checkpointed register state. A context switch could occur
472 * or signal delivered to the process before resuming the
473 * doomed transaction context, but that should all be handled
474 * as expected.
475 */
476 return;
477 }
478#endif /* CONFIG_PPC_TRANSACTIONAL_MEM */
479}
480
481#define arch_enter_from_user_mode arch_enter_from_user_mode
482
483static inline void arch_exit_to_user_mode_prepare(struct pt_regs *regs,
484 unsigned long ti_work)
485{
486 unsigned long mathflags;
487
488 if (IS_ENABLED(CONFIG_PPC_BOOK3S_64) && IS_ENABLED(CONFIG_PPC_FPU)) {
489 if (IS_ENABLED(CONFIG_PPC_TRANSACTIONAL_MEM) &&
490 unlikely((ti_work & _TIF_RESTORE_TM))) {
491 restore_tm_state(regs);
492 } else {
493 mathflags = MSR_FP;
494
495 if (cpu_has_feature(CPU_FTR_VSX))
496 mathflags |= MSR_VEC | MSR_VSX;
497 else if (cpu_has_feature(CPU_FTR_ALTIVEC))
498 mathflags |= MSR_VEC;
499
500 /*
501 * If userspace MSR has all available FP bits set,
502 * then they are live and no need to restore. If not,
503 * it means the regs were given up and restore_math
504 * may decide to restore them (to avoid taking an FP
505 * fault).
506 */
507 if ((regs->msr & mathflags) != mathflags)
508 restore_math(regs);
509 }
510 }
511
512 check_return_regs_valid(regs);
513#ifdef CONFIG_PPC_TRANSACTIONAL_MEM
514 local_paca->tm_scratch = regs->msr;
515#endif
516 /*
517 * Do not restore KUAP here. Generic entry might treat this as the last
518 * arch step before userspace but PowerPC still has kernel work after
519 * irqentry_exit()/syscall_exit_to_user_mode() i.e. in
520 * interrupt_exit_user_prepare() / syscall_exit_prepare() may enable
521 * IRQs and retry. Those functions restore KUAP immediately before rfi,
522 * which is where it should belong.
523 */
524}
525
526#define arch_exit_to_user_mode_prepare arch_exit_to_user_mode_prepare
527
528static __always_inline void arch_exit_to_user_mode(void)
529{
530 booke_load_dbcr0();
531
532 account_cpu_user_exit();
533}
534
535#define arch_exit_to_user_mode arch_exit_to_user_mode
536
537#endif /* _ASM_PPC_ENTRY_COMMON_H */