projects/linux·blob master · refs

285 lines · 6.9 KiB

1// SPDX-License-Identifier: GPL-2.0-or-later
2/*
3 * Copyright (C) 2012 ARM Ltd.
4 * Author: Catalin Marinas <[email protected]>
5 * Copyright (C) 2017 Linaro Ltd. <[email protected]>
6 * Copyright (C) 2021 SiFive
7 */
8#include <linux/compiler.h>
9#include <linux/irqflags.h>
10#include <linux/percpu.h>
11#include <linux/preempt.h>
12#include <linux/types.h>
13#include <linux/kvm_types.h>
14
15#include <asm/vector.h>
16#include <asm/kvm_vcpu_vector.h>
17#include <asm/switch_to.h>
18#include <asm/simd.h>
19#ifdef CONFIG_RISCV_ISA_V_PREEMPTIVE
20#include <asm/asm-prototypes.h>
21#endif
22
23static void (* __rcu kvm_flush_vector_ctx_callback)(void);
24
25void kvm_riscv_register_vctx_callback(void (*func)(void))
26{
27 if (WARN_ON_ONCE(rcu_access_pointer(kvm_flush_vector_ctx_callback)))
28 return;
29
30 rcu_assign_pointer(kvm_flush_vector_ctx_callback, func);
31}
32EXPORT_SYMBOL_GPL(kvm_riscv_register_vctx_callback);
33
34void kvm_riscv_unregister_vctx_callback(void)
35{
36 rcu_assign_pointer(kvm_flush_vector_ctx_callback, NULL);
37 synchronize_rcu();
38}
39EXPORT_SYMBOL_GPL(kvm_riscv_unregister_vctx_callback);
40
41
42static inline void riscv_v_start(u32 flags)
43{
44 int orig;
45
46 orig = riscv_v_flags();
47 BUG_ON((orig & flags) != 0);
48 riscv_v_flags_set(orig | flags);
49 barrier();
50}
51
52static inline void riscv_v_stop(u32 flags)
53{
54 int orig;
55
56 barrier();
57 orig = riscv_v_flags();
58 BUG_ON((orig & flags) == 0);
59 riscv_v_flags_set(orig & ~flags);
60}
61
62/*
63 * Claim ownership of the CPU vector context for use by the calling context.
64 *
65 * The caller may freely manipulate the vector context metadata until
66 * put_cpu_vector_context() is called.
67 */
68void get_cpu_vector_context(void)
69{
70 /*
71 * disable softirqs so it is impossible for softirqs to nest
72 * get_cpu_vector_context() when kernel is actively using Vector.
73 */
74 if (!IS_ENABLED(CONFIG_PREEMPT_RT)) {
75 if (!irqs_disabled())
76 local_bh_disable();
77 } else {
78 preempt_disable();
79 }
80
81 riscv_v_start(RISCV_KERNEL_MODE_V);
82}
83EXPORT_SYMBOL_FOR_KVM(get_cpu_vector_context);
84
85/*
86 * Release the CPU vector context.
87 *
88 * Must be called from a context in which get_cpu_vector_context() was
89 * previously called, with no call to put_cpu_vector_context() in the
90 * meantime.
91 */
92void put_cpu_vector_context(void)
93{
94 riscv_v_stop(RISCV_KERNEL_MODE_V);
95
96 if (!IS_ENABLED(CONFIG_PREEMPT_RT)) {
97 if (!irqs_disabled())
98 local_bh_enable();
99 } else {
100 preempt_enable();
101 }
102}
103EXPORT_SYMBOL_FOR_KVM(put_cpu_vector_context);
104
105static void __riscv_flush_vector_context(void)
106{
107 void (*vcpu_flush_v_callback)(void);
108
109 if (riscv_v_flags() & RISCV_V_VCPU_CTX) {
110 rcu_read_lock();
111 vcpu_flush_v_callback = rcu_dereference(kvm_flush_vector_ctx_callback);
112 vcpu_flush_v_callback();
113 rcu_read_unlock();
114 return;
115 }
116
117 riscv_v_vstate_save(&current->thread.vstate, task_pt_regs(current));
118 riscv_v_vstate_set_restore(current, task_pt_regs(current));
119}
120
121#ifdef CONFIG_RISCV_ISA_V_PREEMPTIVE
122static __always_inline u32 *riscv_v_flags_ptr(void)
123{
124 return &current->thread.riscv_v_flags;
125}
126
127static inline void riscv_preempt_v_set_dirty(void)
128{
129 *riscv_v_flags_ptr() |= RISCV_PREEMPT_V_DIRTY;
130}
131
132static inline void riscv_preempt_v_reset_flags(void)
133{
134 *riscv_v_flags_ptr() &= ~(RISCV_PREEMPT_V_DIRTY | RISCV_PREEMPT_V_NEED_RESTORE);
135}
136
137static inline void riscv_v_ctx_depth_inc(void)
138{
139 *riscv_v_flags_ptr() += RISCV_V_CTX_UNIT_DEPTH;
140}
141
142static inline void riscv_v_ctx_depth_dec(void)
143{
144 *riscv_v_flags_ptr() -= RISCV_V_CTX_UNIT_DEPTH;
145}
146
147static inline u32 riscv_v_ctx_get_depth(void)
148{
149 return *riscv_v_flags_ptr() & RISCV_V_CTX_DEPTH_MASK;
150}
151
152static int riscv_v_stop_kernel_context(void)
153{
154 if (riscv_v_ctx_get_depth() != 0 || !riscv_preempt_v_started(current))
155 return 1;
156
157 riscv_preempt_v_clear_dirty(current);
158 riscv_v_stop(RISCV_PREEMPT_V);
159 return 0;
160}
161
162static int riscv_v_start_kernel_context(void)
163{
164 struct __riscv_v_ext_state *kvstate;
165
166 kvstate = &current->thread.kernel_vstate;
167 if (!kvstate->datap)
168 return -ENOENT;
169
170 if (riscv_preempt_v_started(current)) {
171 WARN_ON(riscv_v_ctx_get_depth() == 0);
172 get_cpu_vector_context();
173 if (riscv_preempt_v_dirty(current)) {
174 __riscv_v_vstate_save(kvstate, kvstate->datap);
175 riscv_preempt_v_clear_dirty(current);
176 }
177 riscv_preempt_v_set_restore(current);
178 return 0;
179 }
180
181 /* Transfer the ownership of V from user to kernel, then save */
182 get_cpu_vector_context();
183 __riscv_flush_vector_context();
184 put_cpu_vector_context();
185 /*
186 * A voluntary context switch caused by put_cpu_vector_context() can
187 * raise the NEED_RESTORE flag if preempt_v starts too early due to a
188 * failed risv_v_is_on() check.
189 *
190 * This causes the next context_nesting_end pollute the v-reg from
191 * the stale context memory in kernel-mode vector.
192 */
193 riscv_v_start(RISCV_PREEMPT_V);
194 return 0;
195}
196
197/* low-level V context handling code, called with irq disabled */
198asmlinkage void riscv_v_context_nesting_start(struct pt_regs *regs)
199{
200 int depth;
201
202 if (!riscv_preempt_v_started(current))
203 return;
204
205 depth = riscv_v_ctx_get_depth();
206 if (depth == 0 && __riscv_v_vstate_check(regs->status, DIRTY))
207 riscv_preempt_v_set_dirty();
208
209 riscv_v_ctx_depth_inc();
210}
211
212asmlinkage void riscv_v_context_nesting_end(struct pt_regs *regs)
213{
214 struct __riscv_v_ext_state *vstate = &current->thread.kernel_vstate;
215 u32 depth;
216
217 WARN_ON(!irqs_disabled());
218
219 if (!riscv_preempt_v_started(current))
220 return;
221
222 riscv_v_ctx_depth_dec();
223 depth = riscv_v_ctx_get_depth();
224 if (depth == 0) {
225 if (riscv_preempt_v_restore(current)) {
226 __riscv_v_vstate_restore(vstate, vstate->datap);
227 __riscv_v_vstate_clean(regs);
228 riscv_preempt_v_reset_flags();
229 }
230 }
231}
232#else
233#define riscv_v_start_kernel_context() (-ENOENT)
234#define riscv_v_stop_kernel_context() (-ENOENT)
235#endif /* CONFIG_RISCV_ISA_V_PREEMPTIVE */
236
237/*
238 * kernel_vector_begin(): obtain the CPU vector registers for use by the calling
239 * context
240 *
241 * Must not be called unless may_use_simd() returns true.
242 * Task context in the vector registers is saved back to memory as necessary.
243 *
244 * A matching call to kernel_vector_end() must be made before returning from the
245 * calling context.
246 *
247 * The caller may freely use the vector registers until kernel_vector_end() is
248 * called.
249 */
250void kernel_vector_begin(void)
251{
252 if (WARN_ON(!(has_vector() || has_xtheadvector())))
253 return;
254
255 BUG_ON(!may_use_simd());
256
257 if (riscv_v_start_kernel_context()) {
258 get_cpu_vector_context();
259 __riscv_flush_vector_context();
260 }
261
262 riscv_v_enable();
263}
264EXPORT_SYMBOL_GPL(kernel_vector_begin);
265
266/*
267 * kernel_vector_end(): give the CPU vector registers back to the current task
268 *
269 * Must be called from a context in which kernel_vector_begin() was previously
270 * called, with no call to kernel_vector_end() in the meantime.
271 *
272 * The caller must not use the vector registers after this function is called,
273 * unless kernel_vector_begin() is called again in the meantime.
274 */
275void kernel_vector_end(void)
276{
277 if (WARN_ON(!(has_vector() || has_xtheadvector())))
278 return;
279
280 riscv_v_disable();
281
282 if (riscv_v_stop_kernel_context())
283 put_cpu_vector_context();
284}
285EXPORT_SYMBOL_GPL(kernel_vector_end);