projects/linux·blob master · refs

353 lines · 8.9 KiB

1// SPDX-License-Identifier: GPL-2.0-or-later
2
3#include <linux/context_tracking.h>
4#include <linux/entry-common.h>
5#include <linux/err.h>
6#include <linux/compat.h>
7#include <linux/rseq.h>
8#include <linux/sched/debug.h> /* for show_regs */
9
10#include <asm/kup.h>
11#include <asm/cputime.h>
12#include <asm/hw_irq.h>
13#include <asm/interrupt.h>
14#include <asm/kprobes.h>
15#include <asm/paca.h>
16#include <asm/ptrace.h>
17#include <asm/reg.h>
18#include <asm/signal.h>
19#include <asm/switch_to.h>
20#include <asm/syscall.h>
21#include <asm/time.h>
22#include <asm/tm.h>
23#include <asm/unistd.h>
24
25#if defined(CONFIG_PPC_ADV_DEBUG_REGS) && defined(CONFIG_PPC32)
26unsigned long global_dbcr0[NR_CPUS];
27#endif
28
29#ifdef CONFIG_PPC_BOOK3S_64
30DEFINE_STATIC_KEY_FALSE(interrupt_exit_not_reentrant);
31static inline bool exit_must_hard_disable(void)
32{
33 return static_branch_unlikely(&interrupt_exit_not_reentrant);
34}
35#else
36static inline bool exit_must_hard_disable(void)
37{
38 return true;
39}
40#endif
41
42/*
43 * local irqs must be disabled. Returns false if the caller must re-enable
44 * them, check for new work, and try again.
45 *
46 * This should be called with local irqs disabled, but if they were previously
47 * enabled when the interrupt handler returns (indicating a process-context /
48 * synchronous interrupt) then irqs_enabled should be true.
49 *
50 * restartable is true then EE/RI can be left on because interrupts are handled
51 * with a restart sequence.
52 */
53static notrace __always_inline bool prep_irq_for_enabled_exit(bool restartable)
54{
55 bool must_hard_disable = (exit_must_hard_disable() || !restartable);
56
57 /* This must be done with RI=1 because tracing may touch vmaps */
58 trace_hardirqs_on();
59
60 if (must_hard_disable)
61 __hard_EE_RI_disable();
62
63#ifdef CONFIG_PPC64
64 /* This pattern matches prep_irq_for_idle */
65 if (unlikely(lazy_irq_pending_nocheck())) {
66 if (must_hard_disable) {
67 local_paca->irq_happened |= PACA_IRQ_HARD_DIS;
68 __hard_RI_enable();
69 }
70 trace_hardirqs_off();
71
72 return false;
73 }
74#endif
75 return true;
76}
77
78/*
79 * This should be called after a syscall returns, with r3 the return value
80 * from the syscall. If this function returns non-zero, the system call
81 * exit assembly should additionally load all GPR registers and CTR and XER
82 * from the interrupt frame.
83 *
84 * The function graph tracer can not trace the return side of this function,
85 * because RI=0 and soft mask state is "unreconciled", so it is marked notrace.
86 */
87notrace unsigned long syscall_exit_prepare(unsigned long r3,
88 struct pt_regs *regs,
89 long scv)
90{
91 unsigned long ti_flags;
92 unsigned long ret = 0;
93 bool is_not_scv = !IS_ENABLED(CONFIG_PPC_BOOK3S_64) || !scv;
94
95 kuap_assert_locked();
96
97 regs->result = r3;
98
99 /* Clear exit_flags so only flags set during this exit are visible */
100 current->thread_info.exit_flags = 0;
101
102 ti_flags = read_thread_flags();
103 if (unlikely(r3 >= (unsigned long)-MAX_ERRNO) && is_not_scv) {
104 if (likely(!(ti_flags & (_TIF_NOERROR | _TIF_RESTOREALL)))) {
105 r3 = -r3;
106 regs->ccr |= 0x10000000; /* Set SO bit in CR */
107 }
108 }
109
110 if (unlikely(ti_flags & _TIF_PERSYSCALL_MASK)) {
111 if (ti_flags & _TIF_RESTOREALL)
112 ret = _TIF_RESTOREALL;
113 else
114 regs->gpr[3] = r3;
115 clear_bits(_TIF_PERSYSCALL_MASK, &current_thread_info()->flags);
116 } else {
117 regs->gpr[3] = r3;
118 }
119
120 if (unlikely(ti_flags & _TIF_SYSCALL_DOTRACE)) {
121 ret |= _TIF_RESTOREALL;
122 }
123
124 syscall_exit_to_user_mode(regs);
125
126again:
127 user_enter_irqoff();
128 if (!prep_irq_for_enabled_exit(true)) {
129 user_exit_irqoff();
130 local_irq_enable();
131 local_irq_disable();
132 goto again;
133 }
134
135 /* Restore user access locks last */
136 kuap_user_restore(regs);
137 ret |= current->thread_info.exit_flags;
138#ifdef CONFIG_PPC64
139 regs->exit_result = ret;
140#endif
141
142 return ret;
143}
144
145#ifdef CONFIG_PPC64
146notrace unsigned long syscall_exit_restart(unsigned long r3, struct pt_regs *regs)
147{
148 unsigned long ret;
149
150 /*
151 * This is called when detecting a soft-pending interrupt as well as
152 * an alternate-return interrupt. So we can't just have the alternate
153 * return path clear SRR1[MSR] and set PACA_IRQ_HARD_DIS (unless
154 * the soft-pending case were to fix things up as well). RI might be
155 * disabled, in which case it gets re-enabled by __hard_irq_disable().
156 */
157 __hard_irq_disable();
158 local_paca->irq_happened |= PACA_IRQ_HARD_DIS;
159
160#ifdef CONFIG_PPC_BOOK3S_64
161 set_kuap(AMR_KUAP_BLOCKED);
162#endif
163
164again:
165 user_enter_irqoff();
166 if (!prep_irq_for_enabled_exit(true)) {
167 user_exit_irqoff();
168 local_irq_enable();
169 local_irq_disable();
170 goto again;
171 }
172
173 kuap_user_restore(regs);
174 ret = current_thread_info()->exit_flags & _TIF_RESTOREALL;
175 current_thread_info()->exit_flags &= ~_TIF_RESTOREALL;
176 regs->exit_result |= ret;
177
178 return regs->exit_result;
179}
180#endif
181
182notrace unsigned long interrupt_exit_user_prepare(struct pt_regs *regs)
183{
184 unsigned long ret;
185
186 BUG_ON(regs_is_unrecoverable(regs));
187 BUG_ON(regs_irqs_disabled(regs));
188
189 /*
190 * We don't need to restore AMR on the way back to userspace for KUAP.
191 * AMR can only have been unlocked if we interrupted the kernel.
192 */
193 kuap_assert_locked();
194
195 /* Clear exit_flags so only flags set during this exit are visible */
196 current_thread_info()->exit_flags = 0;
197
198 local_irq_disable();
199again:
200 check_return_regs_valid(regs);
201 user_enter_irqoff();
202 if (!prep_irq_for_enabled_exit(true)) {
203 user_exit_irqoff();
204 local_irq_enable();
205 local_irq_disable();
206 goto again;
207 }
208
209 /* Restore user access locks last */
210 kuap_user_restore(regs);
211 ret = current_thread_info()->exit_flags & _TIF_RESTOREALL;
212#ifdef CONFIG_PPC64
213 regs->exit_result = ret;
214#endif
215
216 return ret;
217}
218
219void preempt_schedule_irq(void);
220
221notrace unsigned long interrupt_exit_kernel_prepare(struct pt_regs *regs)
222{
223 unsigned long ret = 0;
224 unsigned long kuap;
225 bool stack_store = read_thread_flags() & _TIF_EMULATE_STACK_STORE;
226
227 if (regs_is_unrecoverable(regs))
228 unrecoverable_exception(regs);
229 /*
230 * CT_WARN_ON comes here via program_check_exception, so avoid
231 * recursion.
232 *
233 * Skip the assertion on PMIs on 64e to work around a problem caused
234 * by NMI PMIs incorrectly taking this interrupt return path, it's
235 * possible for this to hit after interrupt exit to user switches
236 * context to user. See also the comment in the performance monitor
237 * handler in exceptions-64e.S
238 */
239 if (!IS_ENABLED(CONFIG_PPC_BOOK3E_64) &&
240 TRAP(regs) != INTERRUPT_PROGRAM &&
241 TRAP(regs) != INTERRUPT_PERFMON)
242 CT_WARN_ON(ct_state() == CT_STATE_USER);
243
244 kuap = kuap_get_and_assert_locked();
245
246 local_irq_disable();
247
248 if (!regs_irqs_disabled(regs)) {
249 /* Returning to a kernel context with local irqs enabled. */
250 WARN_ON_ONCE(!(regs->msr & MSR_EE));
251again:
252
253 check_return_regs_valid(regs);
254
255 /*
256 * Stack store exit can't be restarted because the interrupt
257 * stack frame might have been clobbered.
258 */
259 if (!prep_irq_for_enabled_exit(unlikely(stack_store))) {
260 /*
261 * Replay pending soft-masked interrupts now. Don't
262 * just local_irq_enabe(); local_irq_disable(); because
263 * if we are returning from an asynchronous interrupt
264 * here, another one might hit after irqs are enabled,
265 * and it would exit via this same path allowing
266 * another to fire, and so on unbounded.
267 */
268 hard_irq_disable();
269 replay_soft_interrupts();
270 /* Took an interrupt, may have more exit work to do. */
271 goto again;
272 }
273#ifdef CONFIG_PPC64
274 /*
275 * An interrupt may clear MSR[EE] and set this concurrently,
276 * but it will be marked pending and the exit will be retried.
277 * This leaves a racy window where MSR[EE]=0 and HARD_DIS is
278 * clear, until interrupt_exit_kernel_restart() calls
279 * hard_irq_disable(), which will set HARD_DIS again.
280 */
281 local_paca->irq_happened &= ~PACA_IRQ_HARD_DIS;
282
283 } else {
284 check_return_regs_valid(regs);
285
286 if (unlikely(stack_store))
287 __hard_EE_RI_disable();
288#else
289 } else {
290 __hard_EE_RI_disable();
291#endif /* CONFIG_PPC64 */
292 }
293
294 if (unlikely(stack_store)) {
295 clear_bits(_TIF_EMULATE_STACK_STORE, &current_thread_info()->flags);
296 ret = 1;
297 }
298
299#ifdef CONFIG_PPC_TRANSACTIONAL_MEM
300 local_paca->tm_scratch = regs->msr;
301#endif
302
303 /*
304 * 64s does not want to mfspr(SPRN_AMR) here, because this comes after
305 * mtmsr, which would cause Read-After-Write stalls. Hence, take the
306 * AMR value from the check above.
307 */
308 kuap_kernel_restore(regs, kuap);
309
310 return ret;
311}
312
313#ifdef CONFIG_PPC64
314notrace unsigned long interrupt_exit_user_restart(struct pt_regs *regs)
315{
316 __hard_irq_disable();
317 local_paca->irq_happened |= PACA_IRQ_HARD_DIS;
318
319#ifdef CONFIG_PPC_BOOK3S_64
320 set_kuap(AMR_KUAP_BLOCKED);
321#endif
322
323 trace_hardirqs_off();
324 account_cpu_user_entry();
325
326 BUG_ON(!user_mode(regs));
327
328 regs->exit_result |= interrupt_exit_user_prepare(regs);
329
330 return regs->exit_result;
331}
332
333/*
334 * No real need to return a value here because the stack store case does not
335 * get restarted.
336 */
337notrace unsigned long interrupt_exit_kernel_restart(struct pt_regs *regs)
338{
339 __hard_irq_disable();
340 local_paca->irq_happened |= PACA_IRQ_HARD_DIS;
341
342#ifdef CONFIG_PPC_BOOK3S_64
343 set_kuap(AMR_KUAP_BLOCKED);
344#endif
345
346 if (regs->softe == IRQS_ENABLED)
347 trace_hardirqs_off();
348
349 BUG_ON(user_mode(regs));
350
351 return interrupt_exit_kernel_prepare(regs);
352}
353#endif