projects/linux·blob v7.1 · refs

202 lines · 5.5 KiB

1// SPDX-License-Identifier: GPL-2.0
2
3#include <linux/irq-entry-common.h>
4#include <linux/resume_user_mode.h>
5#include <linux/highmem.h>
6#include <linux/jump_label.h>
7#include <linux/kmsan.h>
8#include <linux/livepatch.h>
9#include <linux/tick.h>
10
11/* Workaround to allow gradual conversion of architecture code */
12void __weak arch_do_signal_or_restart(struct pt_regs *regs) { }
13
14#ifdef CONFIG_HAVE_GENERIC_TIF_BITS
15#define EXIT_TO_USER_MODE_WORK_LOOP (EXIT_TO_USER_MODE_WORK & ~_TIF_RSEQ)
16#else
17#define EXIT_TO_USER_MODE_WORK_LOOP (EXIT_TO_USER_MODE_WORK)
18#endif
19
20/* TIF bits, which prevent a time slice extension. */
21#ifdef CONFIG_PREEMPT_RT
22/*
23 * Since rseq slice ext has a direct correlation to the worst case
24 * scheduling latency (schedule is delayed after all), only have it affect
25 * LAZY reschedules on PREEMPT_RT for now.
26 *
27 * However, since this delay is only applicable to userspace, a value
28 * for rseq_slice_extension_nsec that is strictly less than the worst case
29 * kernel space preempt_disable() region, should mean the scheduling latency
30 * is not affected, even for !LAZY.
31 *
32 * However, since this value depends on the hardware at hand, it cannot be
33 * pre-determined in any sensible way. Hence punt on this problem for now.
34 */
35# define TIF_SLICE_EXT_SCHED (_TIF_NEED_RESCHED_LAZY)
36#else
37# define TIF_SLICE_EXT_SCHED (_TIF_NEED_RESCHED | _TIF_NEED_RESCHED_LAZY)
38#endif
39#define TIF_SLICE_EXT_DENY (EXIT_TO_USER_MODE_WORK & ~TIF_SLICE_EXT_SCHED)
40
41static __always_inline unsigned long __exit_to_user_mode_loop(struct pt_regs *regs,
42 unsigned long ti_work)
43{
44 /*
45 * Before returning to user space ensure that all pending work
46 * items have been completed.
47 */
48 while (ti_work & EXIT_TO_USER_MODE_WORK_LOOP) {
49
50 local_irq_enable();
51
52 if (ti_work & (_TIF_NEED_RESCHED | _TIF_NEED_RESCHED_LAZY)) {
53 if (!rseq_grant_slice_extension(ti_work, TIF_SLICE_EXT_DENY))
54 schedule();
55 }
56
57 if (ti_work & _TIF_UPROBE)
58 uprobe_notify_resume(regs);
59
60 if (ti_work & _TIF_PATCH_PENDING)
61 klp_update_patch_state(current);
62
63 if (ti_work & (_TIF_SIGPENDING | _TIF_NOTIFY_SIGNAL))
64 arch_do_signal_or_restart(regs);
65
66 if (ti_work & _TIF_NOTIFY_RESUME)
67 resume_user_mode_work(regs);
68
69 /* Architecture specific TIF work */
70 arch_exit_to_user_mode_work(regs, ti_work);
71
72 /*
73 * Disable interrupts and reevaluate the work flags as they
74 * might have changed while interrupts and preemption was
75 * enabled above.
76 */
77 local_irq_disable();
78
79 /* Check if any of the above work has queued a deferred wakeup */
80 tick_nohz_user_enter_prepare();
81
82 ti_work = read_thread_flags();
83 }
84
85 /* Return the latest work state for arch_exit_to_user_mode() */
86 return ti_work;
87}
88
89/**
90 * exit_to_user_mode_loop - do any pending work before leaving to user space
91 * @regs: Pointer to pt_regs on entry stack
92 * @ti_work: TIF work flags as read by the caller
93 */
94__always_inline unsigned long exit_to_user_mode_loop(struct pt_regs *regs,
95 unsigned long ti_work)
96{
97 for (;;) {
98 ti_work = __exit_to_user_mode_loop(regs, ti_work);
99
100 if (likely(!rseq_exit_to_user_mode_restart(regs, ti_work)))
101 return ti_work;
102 ti_work = read_thread_flags();
103 }
104}
105
106noinstr irqentry_state_t irqentry_enter(struct pt_regs *regs)
107{
108 if (user_mode(regs)) {
109 irqentry_state_t ret = {
110 .exit_rcu = false,
111 };
112
113 irqentry_enter_from_user_mode(regs);
114 return ret;
115 }
116
117 return irqentry_enter_from_kernel_mode(regs);
118}
119
120/**
121 * arch_irqentry_exit_need_resched - Architecture specific need resched function
122 *
123 * Invoked from raw_irqentry_exit_cond_resched() to check if resched is needed.
124 * Defaults return true.
125 *
126 * The main purpose is to permit arch to avoid preemption of a task from an IRQ.
127 */
128static inline bool arch_irqentry_exit_need_resched(void);
129
130#ifndef arch_irqentry_exit_need_resched
131static inline bool arch_irqentry_exit_need_resched(void) { return true; }
132#endif
133
134void raw_irqentry_exit_cond_resched(void)
135{
136 if (!preempt_count()) {
137 /* Sanity check RCU and thread stack */
138 rcu_irq_exit_check_preempt();
139 if (IS_ENABLED(CONFIG_DEBUG_ENTRY))
140 WARN_ON_ONCE(!on_thread_stack());
141 if (need_resched() && arch_irqentry_exit_need_resched())
142 preempt_schedule_irq();
143 }
144}
145#ifdef CONFIG_PREEMPT_DYNAMIC
146#if defined(CONFIG_HAVE_PREEMPT_DYNAMIC_CALL)
147DEFINE_STATIC_CALL(irqentry_exit_cond_resched, raw_irqentry_exit_cond_resched);
148#elif defined(CONFIG_HAVE_PREEMPT_DYNAMIC_KEY)
149DEFINE_STATIC_KEY_TRUE(sk_dynamic_irqentry_exit_cond_resched);
150void dynamic_irqentry_exit_cond_resched(void)
151{
152 if (!static_branch_unlikely(&sk_dynamic_irqentry_exit_cond_resched))
153 return;
154 raw_irqentry_exit_cond_resched();
155}
156#endif
157#endif
158
159noinstr void irqentry_exit(struct pt_regs *regs, irqentry_state_t state)
160{
161 if (user_mode(regs))
162 irqentry_exit_to_user_mode(regs);
163 else
164 irqentry_exit_to_kernel_mode(regs, state);
165}
166
167irqentry_state_t noinstr irqentry_nmi_enter(struct pt_regs *regs)
168{
169 irqentry_state_t irq_state;
170
171 irq_state.lockdep = lockdep_hardirqs_enabled();
172
173 __nmi_enter();
174 lockdep_hardirqs_off(CALLER_ADDR0);
175 lockdep_hardirq_enter();
176 ct_nmi_enter();
177
178 instrumentation_begin();
179 kmsan_unpoison_entry_regs(regs);
180 trace_hardirqs_off_finish();
181 ftrace_nmi_enter();
182 instrumentation_end();
183
184 return irq_state;
185}
186
187void noinstr irqentry_nmi_exit(struct pt_regs *regs, irqentry_state_t irq_state)
188{
189 instrumentation_begin();
190 ftrace_nmi_exit();
191 if (irq_state.lockdep) {
192 trace_hardirqs_on_prepare();
193 lockdep_hardirqs_on_prepare();
194 }
195 instrumentation_end();
196
197 ct_nmi_exit();
198 lockdep_hardirq_exit();
199 if (irq_state.lockdep)
200 lockdep_hardirqs_on(CALLER_ADDR0);
201 __nmi_exit();
202}