projects/linux·blob master · refs

192 lines · 5.1 KiB

1// SPDX-License-Identifier: GPL-2.0
2
3#include <linux/futex.h>
4#include <linux/highmem.h>
5#include <linux/irq-entry-common.h>
6#include <linux/jump_label.h>
7#include <linux/kmsan.h>
8#include <linux/livepatch.h>
9#include <linux/resume_user_mode.h>
10#include <linux/tick.h>
11
12/* Workaround to allow gradual conversion of architecture code */
13void __weak arch_do_signal_or_restart(struct pt_regs *regs) { }
14
15#ifdef CONFIG_HAVE_GENERIC_TIF_BITS
16#define EXIT_TO_USER_MODE_WORK_LOOP (EXIT_TO_USER_MODE_WORK & ~_TIF_RSEQ)
17#else
18#define EXIT_TO_USER_MODE_WORK_LOOP (EXIT_TO_USER_MODE_WORK)
19#endif
20
21/* TIF bits, which prevent a time slice extension. */
22#ifdef CONFIG_PREEMPT_RT
23/*
24 * Since rseq slice ext has a direct correlation to the worst case
25 * scheduling latency (schedule is delayed after all), only have it affect
26 * LAZY reschedules on PREEMPT_RT for now.
27 *
28 * However, since this delay is only applicable to userspace, a value
29 * for rseq_slice_extension_nsec that is strictly less than the worst case
30 * kernel space preempt_disable() region, should mean the scheduling latency
31 * is not affected, even for !LAZY.
32 *
33 * However, since this value depends on the hardware at hand, it cannot be
34 * pre-determined in any sensible way. Hence punt on this problem for now.
35 */
36# define TIF_SLICE_EXT_SCHED (_TIF_NEED_RESCHED_LAZY)
37#else
38# define TIF_SLICE_EXT_SCHED (_TIF_NEED_RESCHED | _TIF_NEED_RESCHED_LAZY)
39#endif
40#define TIF_SLICE_EXT_DENY (EXIT_TO_USER_MODE_WORK & ~TIF_SLICE_EXT_SCHED)
41
42static __always_inline unsigned long __exit_to_user_mode_loop(struct pt_regs *regs,
43 unsigned long ti_work)
44{
45 /*
46 * Before returning to user space ensure that all pending work
47 * items have been completed.
48 */
49 while (ti_work & EXIT_TO_USER_MODE_WORK_LOOP) {
50
51 local_irq_enable();
52
53 if (ti_work & (_TIF_NEED_RESCHED | _TIF_NEED_RESCHED_LAZY)) {
54 if (!rseq_grant_slice_extension(ti_work, TIF_SLICE_EXT_DENY))
55 schedule();
56 }
57
58 if (ti_work & _TIF_UPROBE)
59 uprobe_notify_resume(regs);
60
61 if (ti_work & _TIF_PATCH_PENDING)
62 klp_update_patch_state(current);
63
64 if (ti_work & (_TIF_SIGPENDING | _TIF_NOTIFY_SIGNAL)) {
65 futex_fixup_robust_unlock(regs);
66 arch_do_signal_or_restart(regs);
67 }
68
69 if (ti_work & _TIF_NOTIFY_RESUME)
70 resume_user_mode_work(regs);
71
72 /* Architecture specific TIF work */
73 arch_exit_to_user_mode_work(regs, ti_work);
74
75 /*
76 * Disable interrupts and reevaluate the work flags as they
77 * might have changed while interrupts and preemption was
78 * enabled above.
79 */
80 local_irq_disable();
81
82 /* Check if any of the above work has queued a deferred wakeup */
83 tick_nohz_user_enter_prepare();
84
85 ti_work = read_thread_flags();
86 }
87
88 /* Return the latest work state for arch_exit_to_user_mode() */
89 return ti_work;
90}
91
92/**
93 * exit_to_user_mode_loop - do any pending work before leaving to user space
94 * @regs: Pointer to pt_regs on entry stack
95 * @ti_work: TIF work flags as read by the caller
96 */
97__always_inline unsigned long exit_to_user_mode_loop(struct pt_regs *regs,
98 unsigned long ti_work)
99{
100 for (;;) {
101 ti_work = __exit_to_user_mode_loop(regs, ti_work);
102
103 if (likely(!rseq_exit_to_user_mode_restart(regs, ti_work)))
104 return ti_work;
105 ti_work = read_thread_flags();
106 }
107}
108
109noinstr irqentry_state_t irqentry_enter(struct pt_regs *regs)
110{
111 if (user_mode(regs)) {
112 irqentry_state_t ret = {
113 .exit_rcu = false,
114 };
115
116 irqentry_enter_from_user_mode(regs);
117 return ret;
118 }
119
120 return irqentry_enter_from_kernel_mode(regs);
121}
122
123/**
124 * arch_irqentry_exit_need_resched - Architecture specific need resched function
125 *
126 * Invoked from irqentry_exit_cond_resched() to check if resched is needed.
127 * Defaults return true.
128 *
129 * The main purpose is to permit arch to avoid preemption of a task from an IRQ.
130 */
131static inline bool arch_irqentry_exit_need_resched(void);
132
133#ifndef arch_irqentry_exit_need_resched
134static inline bool arch_irqentry_exit_need_resched(void) { return true; }
135#endif
136
137void irqentry_exit_cond_resched(void)
138{
139 if (!preempt_count()) {
140 /* Sanity check RCU and thread stack */
141 rcu_irq_exit_check_preempt();
142 if (IS_ENABLED(CONFIG_DEBUG_ENTRY))
143 WARN_ON_ONCE(!on_thread_stack());
144 if (need_resched() && arch_irqentry_exit_need_resched())
145 preempt_schedule_irq();
146 }
147}
148
149noinstr void irqentry_exit(struct pt_regs *regs, irqentry_state_t state)
150{
151 if (user_mode(regs))
152 irqentry_exit_to_user_mode(regs);
153 else
154 irqentry_exit_to_kernel_mode(regs, state);
155}
156
157irqentry_state_t noinstr irqentry_nmi_enter(struct pt_regs *regs)
158{
159 irqentry_state_t irq_state;
160
161 irq_state.lockdep = lockdep_hardirqs_enabled();
162
163 __nmi_enter();
164 lockdep_hardirqs_off(CALLER_ADDR0);
165 lockdep_hardirq_enter();
166 ct_nmi_enter();
167
168 instrumentation_begin();
169 kmsan_unpoison_entry_regs(regs);
170 trace_hardirqs_off_finish();
171 ftrace_nmi_enter();
172 instrumentation_end();
173
174 return irq_state;
175}
176
177void noinstr irqentry_nmi_exit(struct pt_regs *regs, irqentry_state_t irq_state)
178{
179 instrumentation_begin();
180 ftrace_nmi_exit();
181 if (irq_state.lockdep) {
182 trace_hardirqs_on_prepare();
183 lockdep_hardirqs_on_prepare();
184 }
185 instrumentation_end();
186
187 ct_nmi_exit();
188 lockdep_hardirq_exit();
189 if (irq_state.lockdep)
190 lockdep_hardirqs_on(CALLER_ADDR0);
191 __nmi_exit();
192}