projects/linux·blob master · refs

2109 lines · 58.3 KiB

1// SPDX-License-Identifier: GPL-2.0-only
2/*
3 * FP/SIMD context switching and fault handling
4 *
5 * Copyright (C) 2012 ARM Ltd.
6 * Author: Catalin Marinas <[email protected]>
7 */
8
9#include <linux/bitmap.h>
10#include <linux/bitops.h>
11#include <linux/bottom_half.h>
12#include <linux/bug.h>
13#include <linux/cache.h>
14#include <linux/compat.h>
15#include <linux/compiler.h>
16#include <linux/cpu.h>
17#include <linux/cpu_pm.h>
18#include <linux/cpumask.h>
19#include <linux/ctype.h>
20#include <linux/kernel.h>
21#include <linux/linkage.h>
22#include <linux/irqflags.h>
23#include <linux/init.h>
24#include <linux/percpu.h>
25#include <linux/prctl.h>
26#include <linux/preempt.h>
27#include <linux/ptrace.h>
28#include <linux/sched/signal.h>
29#include <linux/sched/task_stack.h>
30#include <linux/signal.h>
31#include <linux/slab.h>
32#include <linux/smp.h>
33#include <linux/stddef.h>
34#include <linux/sysctl.h>
35#include <linux/swab.h>
36
37#include <asm/esr.h>
38#include <asm/exception.h>
39#include <asm/fpsimd.h>
40#include <asm/cpufeature.h>
41#include <asm/cputype.h>
42#include <asm/neon.h>
43#include <asm/processor.h>
44#include <asm/simd.h>
45#include <asm/sigcontext.h>
46#include <asm/sysreg.h>
47#include <asm/traps.h>
48#include <asm/virt.h>
49
50#define FPEXC_IOF (1 << 0)
51#define FPEXC_DZF (1 << 1)
52#define FPEXC_OFF (1 << 2)
53#define FPEXC_UFF (1 << 3)
54#define FPEXC_IXF (1 << 4)
55#define FPEXC_IDF (1 << 7)
56
57/*
58 * (Note: in this discussion, statements about FPSIMD apply equally to SVE.)
59 *
60 * In order to reduce the number of times the FPSIMD state is needlessly saved
61 * and restored, we need to keep track of two things:
62 * (a) for each task, we need to remember which CPU was the last one to have
63 * the task's FPSIMD state loaded into its FPSIMD registers;
64 * (b) for each CPU, we need to remember which task's userland FPSIMD state has
65 * been loaded into its FPSIMD registers most recently, or whether it has
66 * been used to perform kernel mode NEON in the meantime.
67 *
68 * For (a), we add a fpsimd_cpu field to thread_struct, which gets updated to
69 * the id of the current CPU every time the state is loaded onto a CPU. For (b),
70 * we add the per-cpu variable 'fpsimd_last_state' (below), which contains the
71 * address of the userland FPSIMD state of the task that was loaded onto the CPU
72 * the most recently, or NULL if kernel mode NEON has been performed after that.
73 *
74 * With this in place, we no longer have to restore the next FPSIMD state right
75 * when switching between tasks. Instead, we can defer this check to userland
76 * resume, at which time we verify whether the CPU's fpsimd_last_state and the
77 * task's fpsimd_cpu are still mutually in sync. If this is the case, we
78 * can omit the FPSIMD restore.
79 *
80 * As an optimization, we use the thread_info flag TIF_FOREIGN_FPSTATE to
81 * indicate whether or not the userland FPSIMD state of the current task is
82 * present in the registers. The flag is set unless the FPSIMD registers of this
83 * CPU currently contain the most recent userland FPSIMD state of the current
84 * task. If the task is behaving as a VMM, then this is will be managed by
85 * KVM which will clear it to indicate that the vcpu FPSIMD state is currently
86 * loaded on the CPU, allowing the state to be saved if a FPSIMD-aware
87 * softirq kicks in. Upon vcpu_put(), KVM will save the vcpu FP state and
88 * flag the register state as invalid.
89 *
90 * In order to allow softirq handlers to use FPSIMD, kernel_neon_begin() may be
91 * called from softirq context, which will save the task's FPSIMD context back
92 * to task_struct. To prevent this from racing with the manipulation of the
93 * task's FPSIMD state from task context and thereby corrupting the state, it
94 * is necessary to protect any manipulation of a task's fpsimd_state or
95 * TIF_FOREIGN_FPSTATE flag with get_cpu_fpsimd_context(), which will suspend
96 * softirq servicing entirely until put_cpu_fpsimd_context() is called.
97 *
98 * For a certain task, the sequence may look something like this:
99 * - the task gets scheduled in; if both the task's fpsimd_cpu field
100 * contains the id of the current CPU, and the CPU's fpsimd_last_state per-cpu
101 * variable points to the task's fpsimd_state, the TIF_FOREIGN_FPSTATE flag is
102 * cleared, otherwise it is set;
103 *
104 * - the task returns to userland; if TIF_FOREIGN_FPSTATE is set, the task's
105 * userland FPSIMD state is copied from memory to the registers, the task's
106 * fpsimd_cpu field is set to the id of the current CPU, the current
107 * CPU's fpsimd_last_state pointer is set to this task's fpsimd_state and the
108 * TIF_FOREIGN_FPSTATE flag is cleared;
109 *
110 * - the task executes an ordinary syscall; upon return to userland, the
111 * TIF_FOREIGN_FPSTATE flag will still be cleared, so no FPSIMD state is
112 * restored;
113 *
114 * - the task executes a syscall which executes some NEON instructions; this is
115 * preceded by a call to kernel_neon_begin(), which copies the task's FPSIMD
116 * register contents to memory, clears the fpsimd_last_state per-cpu variable
117 * and sets the TIF_FOREIGN_FPSTATE flag;
118 *
119 * - the task gets preempted after kernel_neon_end() is called; as we have not
120 * returned from the 2nd syscall yet, TIF_FOREIGN_FPSTATE is still set so
121 * whatever is in the FPSIMD registers is not saved to memory, but discarded.
122 */
123
124DEFINE_PER_CPU(struct cpu_fp_state, fpsimd_last_state);
125
126__ro_after_init struct vl_info vl_info[ARM64_VEC_MAX] = {
127#ifdef CONFIG_ARM64_SVE
128 [ARM64_VEC_SVE] = {
129 .type = ARM64_VEC_SVE,
130 .name = "SVE",
131 .min_vl = SVE_VL_MIN,
132 .max_vl = SVE_VL_MIN,
133 .max_virtualisable_vl = SVE_VL_MIN,
134 },
135#endif
136#ifdef CONFIG_ARM64_SME
137 [ARM64_VEC_SME] = {
138 .type = ARM64_VEC_SME,
139 .name = "SME",
140 },
141#endif
142};
143
144static unsigned int vec_vl_inherit_flag(enum vec_type type)
145{
146 switch (type) {
147 case ARM64_VEC_SVE:
148 return TIF_SVE_VL_INHERIT;
149 case ARM64_VEC_SME:
150 return TIF_SME_VL_INHERIT;
151 default:
152 WARN_ON_ONCE(1);
153 return 0;
154 }
155}
156
157struct vl_config {
158 int __default_vl; /* Default VL for tasks */
159};
160
161static struct vl_config vl_config[ARM64_VEC_MAX];
162
163static inline int get_default_vl(enum vec_type type)
164{
165 return READ_ONCE(vl_config[type].__default_vl);
166}
167
168#ifdef CONFIG_ARM64_SVE
169
170static inline int get_sve_default_vl(void)
171{
172 return get_default_vl(ARM64_VEC_SVE);
173}
174
175static inline void set_default_vl(enum vec_type type, int val)
176{
177 WRITE_ONCE(vl_config[type].__default_vl, val);
178}
179
180static inline void set_sve_default_vl(int val)
181{
182 set_default_vl(ARM64_VEC_SVE, val);
183}
184
185#endif /* ! CONFIG_ARM64_SVE */
186
187#ifdef CONFIG_ARM64_SME
188
189static int get_sme_default_vl(void)
190{
191 return get_default_vl(ARM64_VEC_SME);
192}
193
194static void set_sme_default_vl(int val)
195{
196 set_default_vl(ARM64_VEC_SME, val);
197}
198
199static void sme_free(struct task_struct *);
200
201#else
202
203static inline void sme_free(struct task_struct *t) { }
204
205#endif
206
207static void fpsimd_bind_task_to_cpu(void);
208
209/*
210 * Claim ownership of the CPU FPSIMD context for use by the calling context.
211 *
212 * The caller may freely manipulate the FPSIMD context metadata until
213 * put_cpu_fpsimd_context() is called.
214 *
215 * On RT kernels local_bh_disable() is not sufficient because it only
216 * serializes soft interrupt related sections via a local lock, but stays
217 * preemptible. Disabling preemption is the right choice here as bottom
218 * half processing is always in thread context on RT kernels so it
219 * implicitly prevents bottom half processing as well.
220 */
221static void get_cpu_fpsimd_context(void)
222{
223 if (!IS_ENABLED(CONFIG_PREEMPT_RT)) {
224 /*
225 * The softirq subsystem lacks a true unmask/mask API, and
226 * re-enabling softirq processing using local_bh_enable() will
227 * not only unmask softirqs, it will also result in immediate
228 * delivery of any pending softirqs.
229 * This is undesirable when running with IRQs disabled, but in
230 * that case, there is no need to mask softirqs in the first
231 * place, so only bother doing so when IRQs are enabled.
232 */
233 if (!irqs_disabled())
234 local_bh_disable();
235 } else {
236 preempt_disable();
237 }
238}
239
240/*
241 * Release the CPU FPSIMD context.
242 *
243 * Must be called from a context in which get_cpu_fpsimd_context() was
244 * previously called, with no call to put_cpu_fpsimd_context() in the
245 * meantime.
246 */
247static void put_cpu_fpsimd_context(void)
248{
249 if (!IS_ENABLED(CONFIG_PREEMPT_RT)) {
250 if (!irqs_disabled())
251 local_bh_enable();
252 } else {
253 preempt_enable();
254 }
255}
256
257unsigned int task_get_vl(const struct task_struct *task, enum vec_type type)
258{
259 return task->thread.vl[type];
260}
261
262void task_set_vl(struct task_struct *task, enum vec_type type,
263 unsigned long vl)
264{
265 task->thread.vl[type] = vl;
266}
267
268unsigned int task_get_vl_onexec(const struct task_struct *task,
269 enum vec_type type)
270{
271 return task->thread.vl_onexec[type];
272}
273
274void task_set_vl_onexec(struct task_struct *task, enum vec_type type,
275 unsigned long vl)
276{
277 task->thread.vl_onexec[type] = vl;
278}
279
280/*
281 * TIF_SME controls whether a task can use SME without trapping while
282 * in userspace, when TIF_SME is set then we must have storage
283 * allocated in sve_state and sme_state to store the contents of both ZA
284 * and the SVE registers for both streaming and non-streaming modes.
285 *
286 * If both SVCR.ZA and SVCR.SM are disabled then at any point we
287 * may disable TIF_SME and reenable traps.
288 */
289
290
291/*
292 * TIF_SVE controls whether a task can use SVE without trapping while
293 * in userspace, and also (together with TIF_SME) the way a task's
294 * FPSIMD/SVE state is stored in thread_struct.
295 *
296 * The kernel uses this flag to track whether a user task is actively
297 * using SVE, and therefore whether full SVE register state needs to
298 * be tracked. If not, the cheaper FPSIMD context handling code can
299 * be used instead of the more costly SVE equivalents.
300 *
301 * * TIF_SVE or SVCR.SM set:
302 *
303 * The task can execute SVE instructions while in userspace without
304 * trapping to the kernel.
305 *
306 * During any syscall, the kernel may optionally clear TIF_SVE and
307 * discard the vector state except for the FPSIMD subset.
308 *
309 * * TIF_SVE clear:
310 *
311 * An attempt by the user task to execute an SVE instruction causes
312 * do_sve_acc() to be called, which does some preparation and then
313 * sets TIF_SVE.
314 *
315 * During any syscall, the kernel may optionally clear TIF_SVE and
316 * discard the vector state except for the FPSIMD subset.
317 *
318 * The data will be stored in one of two formats:
319 *
320 * * FPSIMD only - FP_STATE_FPSIMD:
321 *
322 * When the FPSIMD only state stored task->thread.fp_type is set to
323 * FP_STATE_FPSIMD, the FPSIMD registers V0-V31 are encoded in
324 * task->thread.uw.fpsimd_state; bits [max : 128] for each of Z0-Z31 are
325 * logically zero but not stored anywhere; P0-P15 and FFR are not
326 * stored and have unspecified values from userspace's point of
327 * view. For hygiene purposes, the kernel zeroes them on next use,
328 * but userspace is discouraged from relying on this.
329 *
330 * task->thread.sve_state does not need to be non-NULL, valid or any
331 * particular size: it must not be dereferenced and any data stored
332 * there should be considered stale and not referenced.
333 *
334 * * SVE state - FP_STATE_SVE:
335 *
336 * When the full SVE state is stored task->thread.fp_type is set to
337 * FP_STATE_SVE and Z0-Z31 (incorporating Vn in bits[127:0] or the
338 * corresponding Zn), P0-P15 and FFR are encoded in in
339 * task->thread.sve_state, formatted appropriately for vector
340 * length task->thread.sve_vl or, if SVCR.SM is set,
341 * task->thread.sme_vl. The storage for the vector registers in
342 * task->thread.uw.fpsimd_state should be ignored.
343 *
344 * task->thread.sve_state must point to a valid buffer at least
345 * sve_state_size(task) bytes in size. The data stored in
346 * task->thread.uw.fpsimd_state.vregs should be considered stale
347 * and not referenced.
348 *
349 * * FPSR and FPCR are always stored in task->thread.uw.fpsimd_state
350 * irrespective of whether TIF_SVE is clear or set, since these are
351 * not vector length dependent.
352 */
353
354/*
355 * Update current's FPSIMD/SVE registers from thread_struct.
356 *
357 * This function should be called only when the FPSIMD/SVE state in
358 * thread_struct is known to be up to date, when preparing to enter
359 * userspace.
360 */
361static void task_fpsimd_load(void)
362{
363 bool restore_sve_regs = false;
364 bool restore_ffr;
365
366 WARN_ON(!system_supports_fpsimd());
367 WARN_ON(preemptible());
368 WARN_ON(test_thread_flag(TIF_KERNEL_FPSTATE));
369
370 if (system_supports_sve() || system_supports_sme()) {
371 switch (current->thread.fp_type) {
372 case FP_STATE_FPSIMD:
373 /* Stop tracking SVE for this task until next use. */
374 clear_thread_flag(TIF_SVE);
375 break;
376 case FP_STATE_SVE:
377 if (!thread_sm_enabled(&current->thread))
378 WARN_ON_ONCE(!test_and_set_thread_flag(TIF_SVE));
379
380 if (test_thread_flag(TIF_SVE)) {
381 unsigned long vq = sve_vq_from_vl(task_get_sve_vl(current));
382 sysreg_clear_set_s(SYS_ZCR_EL1, ZCR_ELx_LEN, vq - 1);
383 }
384
385 restore_sve_regs = true;
386 restore_ffr = true;
387 break;
388 default:
389 /*
390 * This indicates either a bug in
391 * fpsimd_save_user_state() or memory corruption, we
392 * should always record an explicit format
393 * when we save. We always at least have the
394 * memory allocated for FPSIMD registers so
395 * try that and hope for the best.
396 */
397 WARN_ON_ONCE(1);
398 clear_thread_flag(TIF_SVE);
399 break;
400 }
401 }
402
403 /* Restore SME, override SVE register configuration if needed */
404 if (system_supports_sme()) {
405 unsigned long sme_vl = task_get_sme_vl(current);
406
407 /* Ensure VL is set up for restoring data */
408 if (test_thread_flag(TIF_SME)) {
409 unsigned long vq = sve_vq_from_vl(sme_vl);
410 sysreg_clear_set_s(SYS_SMCR_EL1, SMCR_ELx_LEN, vq - 1);
411 }
412
413 write_sysreg_s(current->thread.svcr, SYS_SVCR);
414
415 if (thread_za_enabled(&current->thread))
416 sme_load_state(current->thread.sme_state,
417 system_supports_sme2());
418
419 if (thread_sm_enabled(&current->thread))
420 restore_ffr = system_supports_fa64();
421 }
422
423 if (system_supports_fpmr())
424 write_sysreg_s(current->thread.uw.fpmr, SYS_FPMR);
425
426 if (restore_sve_regs) {
427 WARN_ON_ONCE(current->thread.fp_type != FP_STATE_SVE);
428 sve_load_state(current->thread.sve_state, restore_ffr);
429 fpsimd_load_common(&current->thread.uw.fpsimd_state);
430 } else {
431 WARN_ON_ONCE(current->thread.fp_type != FP_STATE_FPSIMD);
432 fpsimd_load_state(&current->thread.uw.fpsimd_state);
433 }
434}
435
436/*
437 * Ensure FPSIMD/SVE storage in memory for the loaded context is up to
438 * date with respect to the CPU registers. Note carefully that the
439 * current context is the context last bound to the CPU stored in
440 * last, if KVM is involved this may be the guest VM context rather
441 * than the host thread for the VM pointed to by current. This means
442 * that we must always reference the state storage via last rather
443 * than via current, if we are saving KVM state then it will have
444 * ensured that the type of registers to save is set in last->to_save.
445 */
446static void fpsimd_save_user_state(void)
447{
448 struct cpu_fp_state const *last =
449 this_cpu_ptr(&fpsimd_last_state);
450 /* set by fpsimd_bind_task_to_cpu() or fpsimd_bind_state_to_cpu() */
451 bool save_sve_regs = false;
452 bool save_ffr;
453 unsigned int vl;
454
455 WARN_ON(!system_supports_fpsimd());
456 WARN_ON(preemptible());
457
458 if (test_thread_flag(TIF_FOREIGN_FPSTATE))
459 return;
460
461 if (system_supports_fpmr())
462 *(last->fpmr) = read_sysreg_s(SYS_FPMR);
463
464 /*
465 * Save SVE state if it is live.
466 *
467 * The syscall ABI discards live SVE state at syscall entry. When
468 * entering a syscall, fpsimd_syscall_enter() sets to_save to
469 * FP_STATE_FPSIMD to allow the SVE state to be lazily discarded until
470 * either new SVE state is loaded+bound or fpsimd_syscall_exit() is
471 * called prior to a return to userspace.
472 */
473 if ((last->to_save == FP_STATE_CURRENT && test_thread_flag(TIF_SVE)) ||
474 last->to_save == FP_STATE_SVE) {
475 save_sve_regs = true;
476 save_ffr = true;
477 vl = last->sve_vl;
478 }
479
480 if (system_supports_sme()) {
481 u64 *svcr = last->svcr;
482
483 *svcr = read_sysreg_s(SYS_SVCR);
484
485 if (*svcr & SVCR_ZA_MASK)
486 sme_save_state(last->sme_state,
487 system_supports_sme2());
488
489 /* If we are in streaming mode override regular SVE. */
490 if (*svcr & SVCR_SM_MASK) {
491 save_sve_regs = true;
492 save_ffr = system_supports_fa64();
493 vl = last->sme_vl;
494 }
495 }
496
497 if (IS_ENABLED(CONFIG_ARM64_SVE) && save_sve_regs) {
498 /* Get the configured VL from RDVL, will account for SM */
499 if (WARN_ON(sve_get_vl() != vl)) {
500 /*
501 * Can't save the user regs, so current would
502 * re-enter user with corrupt state.
503 * There's no way to recover, so kill it:
504 */
505 force_signal_inject(SIGKILL, SI_KERNEL, 0, 0);
506 return;
507 }
508
509 sve_save_state(last->sve_state, save_ffr);
510 fpsimd_save_common(last->st);
511 *last->fp_type = FP_STATE_SVE;
512 } else {
513 fpsimd_save_state(last->st);
514 *last->fp_type = FP_STATE_FPSIMD;
515 }
516}
517
518/*
519 * All vector length selection from userspace comes through here.
520 * We're on a slow path, so some sanity-checks are included.
521 * If things go wrong there's a bug somewhere, but try to fall back to a
522 * safe choice.
523 */
524static unsigned int find_supported_vector_length(enum vec_type type,
525 unsigned int vl)
526{
527 struct vl_info *info = &vl_info[type];
528 int bit;
529 int max_vl = info->max_vl;
530
531 if (WARN_ON(!sve_vl_valid(vl)))
532 vl = info->min_vl;
533
534 if (WARN_ON(!sve_vl_valid(max_vl)))
535 max_vl = info->min_vl;
536
537 if (vl > max_vl)
538 vl = max_vl;
539 if (vl < info->min_vl)
540 vl = info->min_vl;
541
542 bit = find_next_bit(info->vq_map, SVE_VQ_MAX,
543 __vq_to_bit(sve_vq_from_vl(vl)));
544 return sve_vl_from_vq(__bit_to_vq(bit));
545}
546
547#if defined(CONFIG_ARM64_SVE) && defined(CONFIG_SYSCTL)
548
549static int vec_proc_do_default_vl(const struct ctl_table *table, int write,
550 void *buffer, size_t *lenp, loff_t *ppos)
551{
552 struct vl_info *info = table->extra1;
553 enum vec_type type = info->type;
554 int ret;
555 int vl = get_default_vl(type);
556 struct ctl_table tmp_table = {
557 .data = &vl,
558 .maxlen = sizeof(vl),
559 };
560
561 ret = proc_dointvec(&tmp_table, write, buffer, lenp, ppos);
562 if (ret || !write)
563 return ret;
564
565 /* Writing -1 has the special meaning "set to max": */
566 if (vl == -1)
567 vl = info->max_vl;
568
569 if (!sve_vl_valid(vl))
570 return -EINVAL;
571
572 set_default_vl(type, find_supported_vector_length(type, vl));
573 return 0;
574}
575
576static const struct ctl_table sve_default_vl_table[] = {
577 {
578 .procname = "sve_default_vector_length",
579 .mode = 0644,
580 .proc_handler = vec_proc_do_default_vl,
581 .extra1 = &vl_info[ARM64_VEC_SVE],
582 },
583};
584
585static int __init sve_sysctl_init(void)
586{
587 if (system_supports_sve())
588 if (!register_sysctl("abi", sve_default_vl_table))
589 return -EINVAL;
590
591 return 0;
592}
593
594#else /* ! (CONFIG_ARM64_SVE && CONFIG_SYSCTL) */
595static int __init sve_sysctl_init(void) { return 0; }
596#endif /* ! (CONFIG_ARM64_SVE && CONFIG_SYSCTL) */
597
598#if defined(CONFIG_ARM64_SME) && defined(CONFIG_SYSCTL)
599static const struct ctl_table sme_default_vl_table[] = {
600 {
601 .procname = "sme_default_vector_length",
602 .mode = 0644,
603 .proc_handler = vec_proc_do_default_vl,
604 .extra1 = &vl_info[ARM64_VEC_SME],
605 },
606};
607
608static int __init sme_sysctl_init(void)
609{
610 if (system_supports_sme())
611 if (!register_sysctl("abi", sme_default_vl_table))
612 return -EINVAL;
613
614 return 0;
615}
616
617#else /* ! (CONFIG_ARM64_SME && CONFIG_SYSCTL) */
618static int __init sme_sysctl_init(void) { return 0; }
619#endif /* ! (CONFIG_ARM64_SME && CONFIG_SYSCTL) */
620
621#define ZREG(sve_state, vq, n) ((char *)(sve_state) + \
622 (SVE_SIG_ZREG_OFFSET(vq, n) - SVE_SIG_REGS_OFFSET))
623
624static void __fpsimd_to_sve(struct arm64_sve_state *sst,
625 struct user_fpsimd_state const *fst,
626 unsigned int vq)
627{
628 unsigned int i;
629 __uint128_t *p;
630
631 for (i = 0; i < SVE_NUM_ZREGS; ++i) {
632 p = (__uint128_t *)ZREG(sst, vq, i);
633 *p = fst->vregs[i];
634 }
635}
636
637/*
638 * Transfer the FPSIMD state in task->thread.uw.fpsimd_state to
639 * task->thread.sve_state.
640 *
641 * Task can be a non-runnable task, or current. In the latter case,
642 * the caller must have ownership of the cpu FPSIMD context before calling
643 * this function.
644 * task->thread.sve_state must point to at least sve_state_size(task)
645 * bytes of allocated kernel memory.
646 * task->thread.uw.fpsimd_state must be up to date before calling this
647 * function.
648 */
649static inline void fpsimd_to_sve(struct task_struct *task)
650{
651 unsigned int vq;
652 struct arm64_sve_state *sst = task->thread.sve_state;
653 struct user_fpsimd_state const *fst = &task->thread.uw.fpsimd_state;
654
655 if (!system_supports_sve() && !system_supports_sme())
656 return;
657
658 vq = sve_vq_from_vl(thread_get_cur_vl(&task->thread));
659 __fpsimd_to_sve(sst, fst, vq);
660}
661
662/*
663 * Transfer the SVE state in task->thread.sve_state to
664 * task->thread.uw.fpsimd_state.
665 *
666 * Task can be a non-runnable task, or current. In the latter case,
667 * the caller must have ownership of the cpu FPSIMD context before calling
668 * this function.
669 * task->thread.sve_state must point to at least sve_state_size(task)
670 * bytes of allocated kernel memory.
671 * task->thread.sve_state must be up to date before calling this function.
672 */
673static inline void sve_to_fpsimd(struct task_struct *task)
674{
675 unsigned int vq, vl;
676 const struct arm64_sve_state *sst = task->thread.sve_state;
677 struct user_fpsimd_state *fst = &task->thread.uw.fpsimd_state;
678 unsigned int i;
679 __uint128_t const *p;
680
681 if (!system_supports_sve() && !system_supports_sme())
682 return;
683
684 vl = thread_get_cur_vl(&task->thread);
685 vq = sve_vq_from_vl(vl);
686 for (i = 0; i < SVE_NUM_ZREGS; ++i) {
687 p = (__uint128_t const *)ZREG(sst, vq, i);
688 fst->vregs[i] = *p;
689 }
690}
691
692static inline void __fpsimd_zero_vregs(struct user_fpsimd_state *fpsimd)
693{
694 memset(&fpsimd->vregs, 0, sizeof(fpsimd->vregs));
695}
696
697/*
698 * Simulate the effects of an SMSTOP SM instruction.
699 */
700void task_smstop_sm(struct task_struct *task)
701{
702 if (!thread_sm_enabled(&task->thread))
703 return;
704
705 __fpsimd_zero_vregs(&task->thread.uw.fpsimd_state);
706 task->thread.uw.fpsimd_state.fpsr = 0x0800009f;
707 if (system_supports_fpmr())
708 task->thread.uw.fpmr = 0;
709
710 task->thread.svcr &= ~SVCR_SM_MASK;
711 task->thread.fp_type = FP_STATE_FPSIMD;
712}
713
714void cpu_enable_fpmr(const struct arm64_cpu_capabilities *__always_unused p)
715{
716 write_sysreg_s(read_sysreg_s(SYS_SCTLR_EL1) | SCTLR_EL1_EnFPM_MASK,
717 SYS_SCTLR_EL1);
718}
719
720#ifdef CONFIG_ARM64_SVE
721static void sve_free(struct task_struct *task)
722{
723 kfree(task->thread.sve_state);
724 task->thread.sve_state = NULL;
725}
726
727/*
728 * Ensure that task->thread.sve_state is allocated and sufficiently large.
729 *
730 * This function should be used only in preparation for replacing
731 * task->thread.sve_state with new data. The memory is always zeroed
732 * here to prevent stale data from showing through: this is done in
733 * the interest of testability and predictability: except in the
734 * do_sve_acc() case, there is no ABI requirement to hide stale data
735 * written previously be task.
736 */
737void sve_alloc(struct task_struct *task, bool flush)
738{
739 if (task->thread.sve_state) {
740 if (flush)
741 memset(task->thread.sve_state, 0,
742 sve_state_size(task));
743 return;
744 }
745
746 /* This is a small allocation (maximum ~8KB) and Should Not Fail. */
747 task->thread.sve_state =
748 kzalloc(sve_state_size(task), GFP_KERNEL);
749}
750
751/*
752 * Ensure that task->thread.uw.fpsimd_state is up to date with respect to the
753 * task's currently effective FPSIMD/SVE state.
754 *
755 * The task's FPSIMD/SVE/SME state must not be subject to concurrent
756 * manipulation.
757 */
758void fpsimd_sync_from_effective_state(struct task_struct *task)
759{
760 if (task->thread.fp_type == FP_STATE_SVE)
761 sve_to_fpsimd(task);
762}
763
764/*
765 * Ensure that the task's currently effective FPSIMD/SVE state is up to date
766 * with respect to task->thread.uw.fpsimd_state, zeroing any effective
767 * non-FPSIMD (S)SVE state.
768 *
769 * The task's FPSIMD/SVE/SME state must not be subject to concurrent
770 * manipulation.
771 */
772void fpsimd_sync_to_effective_state_zeropad(struct task_struct *task)
773{
774 unsigned int vq;
775 struct arm64_sve_state *sst = task->thread.sve_state;
776 struct user_fpsimd_state const *fst = &task->thread.uw.fpsimd_state;
777
778 if (task->thread.fp_type != FP_STATE_SVE)
779 return;
780
781 vq = sve_vq_from_vl(thread_get_cur_vl(&task->thread));
782
783 memset(sst, 0, SVE_SIG_REGS_SIZE(vq));
784 __fpsimd_to_sve(sst, fst, vq);
785}
786
787static int change_live_vector_length(struct task_struct *task,
788 enum vec_type type,
789 unsigned long vl)
790{
791 unsigned int sve_vl = task_get_sve_vl(task);
792 unsigned int sme_vl = task_get_sme_vl(task);
793 struct arm64_sve_state *sve_state = NULL;
794 struct arm64_sme_state *sme_state = NULL;
795
796 if (type == ARM64_VEC_SME)
797 sme_vl = vl;
798 else
799 sve_vl = vl;
800
801 /*
802 * Allocate the new sve_state and sme_state before freeing the old
803 * copies so that allocation failure can be handled without needing to
804 * mutate the task's state in any way.
805 *
806 * Changes to the SVE vector length must not discard live ZA state or
807 * clear PSTATE.ZA, as userspace code which is unaware of the AAPCS64
808 * ZA lazy saving scheme may attempt to change the SVE vector length
809 * while unsaved/dormant ZA state exists.
810 */
811 sve_state = kzalloc(__sve_state_size(sve_vl, sme_vl), GFP_KERNEL);
812 if (!sve_state)
813 goto out_mem;
814
815 if (type == ARM64_VEC_SME) {
816 sme_state = kzalloc(__sme_state_size(sme_vl), GFP_KERNEL);
817 if (!sme_state)
818 goto out_mem;
819 }
820
821 if (task == current)
822 fpsimd_save_and_flush_current_state();
823 else
824 fpsimd_flush_task_state(task);
825
826 /*
827 * Always preserve PSTATE.SM and the effective FPSIMD state, zeroing
828 * other SVE state.
829 */
830 fpsimd_sync_from_effective_state(task);
831 task_set_vl(task, type, vl);
832 kfree(task->thread.sve_state);
833 task->thread.sve_state = sve_state;
834 fpsimd_sync_to_effective_state_zeropad(task);
835
836 if (type == ARM64_VEC_SME) {
837 task->thread.svcr &= ~SVCR_ZA_MASK;
838 kfree(task->thread.sme_state);
839 task->thread.sme_state = sme_state;
840 }
841
842 return 0;
843
844out_mem:
845 kfree(sve_state);
846 kfree(sme_state);
847 return -ENOMEM;
848}
849
850int vec_set_vector_length(struct task_struct *task, enum vec_type type,
851 unsigned long vl, unsigned long flags)
852{
853 bool onexec = flags & PR_SVE_SET_VL_ONEXEC;
854 bool inherit = flags & PR_SVE_VL_INHERIT;
855
856 if (flags & ~(unsigned long)(PR_SVE_VL_INHERIT |
857 PR_SVE_SET_VL_ONEXEC))
858 return -EINVAL;
859
860 if (!sve_vl_valid(vl))
861 return -EINVAL;
862
863 /*
864 * Clamp to the maximum vector length that VL-agnostic code
865 * can work with. A flag may be assigned in the future to
866 * allow setting of larger vector lengths without confusing
867 * older software.
868 */
869 if (vl > VL_ARCH_MAX)
870 vl = VL_ARCH_MAX;
871
872 vl = find_supported_vector_length(type, vl);
873
874 if (!onexec && vl != task_get_vl(task, type)) {
875 if (change_live_vector_length(task, type, vl))
876 return -ENOMEM;
877 }
878
879 if (onexec || inherit)
880 task_set_vl_onexec(task, type, vl);
881 else
882 /* Reset VL to system default on next exec: */
883 task_set_vl_onexec(task, type, 0);
884
885 update_tsk_thread_flag(task, vec_vl_inherit_flag(type),
886 flags & PR_SVE_VL_INHERIT);
887
888 return 0;
889}
890
891/*
892 * Encode the current vector length and flags for return.
893 * This is only required for prctl(): ptrace has separate fields.
894 * SVE and SME use the same bits for _ONEXEC and _INHERIT.
895 *
896 * flags are as for vec_set_vector_length().
897 */
898static int vec_prctl_status(enum vec_type type, unsigned long flags)
899{
900 int ret;
901
902 if (flags & PR_SVE_SET_VL_ONEXEC)
903 ret = task_get_vl_onexec(current, type);
904 else
905 ret = task_get_vl(current, type);
906
907 if (test_thread_flag(vec_vl_inherit_flag(type)))
908 ret |= PR_SVE_VL_INHERIT;
909
910 return ret;
911}
912
913/* PR_SVE_SET_VL */
914int sve_set_current_vl(unsigned long arg)
915{
916 unsigned long vl, flags;
917 int ret;
918
919 vl = arg & PR_SVE_VL_LEN_MASK;
920 flags = arg & ~vl;
921
922 if (!system_supports_sve() || is_compat_task())
923 return -EINVAL;
924
925 ret = vec_set_vector_length(current, ARM64_VEC_SVE, vl, flags);
926 if (ret)
927 return ret;
928
929 return vec_prctl_status(ARM64_VEC_SVE, flags);
930}
931
932/* PR_SVE_GET_VL */
933int sve_get_current_vl(void)
934{
935 if (!system_supports_sve() || is_compat_task())
936 return -EINVAL;
937
938 return vec_prctl_status(ARM64_VEC_SVE, 0);
939}
940
941#ifdef CONFIG_ARM64_SME
942/* PR_SME_SET_VL */
943int sme_set_current_vl(unsigned long arg)
944{
945 unsigned long vl, flags;
946 int ret;
947
948 vl = arg & PR_SME_VL_LEN_MASK;
949 flags = arg & ~vl;
950
951 if (!system_supports_sme() || is_compat_task())
952 return -EINVAL;
953
954 ret = vec_set_vector_length(current, ARM64_VEC_SME, vl, flags);
955 if (ret)
956 return ret;
957
958 return vec_prctl_status(ARM64_VEC_SME, flags);
959}
960
961/* PR_SME_GET_VL */
962int sme_get_current_vl(void)
963{
964 if (!system_supports_sme() || is_compat_task())
965 return -EINVAL;
966
967 return vec_prctl_status(ARM64_VEC_SME, 0);
968}
969#endif /* CONFIG_ARM64_SME */
970
971static void vec_probe_vqs(struct vl_info *info,
972 DECLARE_BITMAP(map, SVE_VQ_MAX))
973{
974 unsigned int vq, vl;
975
976 bitmap_zero(map, SVE_VQ_MAX);
977
978 for (vq = SVE_VQ_MAX; vq >= SVE_VQ_MIN; --vq) {
979 write_vl(info->type, vq - 1); /* self-syncing */
980
981 switch (info->type) {
982 case ARM64_VEC_SVE:
983 vl = sve_get_vl();
984 break;
985 case ARM64_VEC_SME:
986 vl = sme_get_vl();
987 break;
988 default:
989 vl = 0;
990 break;
991 }
992
993 /* Minimum VL identified? */
994 if (sve_vq_from_vl(vl) > vq)
995 break;
996
997 vq = sve_vq_from_vl(vl); /* skip intervening lengths */
998 set_bit(__vq_to_bit(vq), map);
999 }
1000}
1001
1002/*
1003 * Initialise the set of known supported VQs for the boot CPU.
1004 * This is called during kernel boot, before secondary CPUs are brought up.
1005 */
1006void __init vec_init_vq_map(enum vec_type type)
1007{
1008 struct vl_info *info = &vl_info[type];
1009 vec_probe_vqs(info, info->vq_map);
1010 bitmap_copy(info->vq_partial_map, info->vq_map, SVE_VQ_MAX);
1011}
1012
1013/*
1014 * If we haven't committed to the set of supported VQs yet, filter out
1015 * those not supported by the current CPU.
1016 * This function is called during the bring-up of early secondary CPUs only.
1017 */
1018void vec_update_vq_map(enum vec_type type)
1019{
1020 struct vl_info *info = &vl_info[type];
1021 DECLARE_BITMAP(tmp_map, SVE_VQ_MAX);
1022
1023 vec_probe_vqs(info, tmp_map);
1024 bitmap_and(info->vq_map, info->vq_map, tmp_map, SVE_VQ_MAX);
1025 bitmap_or(info->vq_partial_map, info->vq_partial_map, tmp_map,
1026 SVE_VQ_MAX);
1027}
1028
1029/*
1030 * Check whether the current CPU supports all VQs in the committed set.
1031 * This function is called during the bring-up of late secondary CPUs only.
1032 */
1033int vec_verify_vq_map(enum vec_type type)
1034{
1035 struct vl_info *info = &vl_info[type];
1036 DECLARE_BITMAP(tmp_map, SVE_VQ_MAX);
1037 unsigned long b;
1038
1039 vec_probe_vqs(info, tmp_map);
1040
1041 bitmap_complement(tmp_map, tmp_map, SVE_VQ_MAX);
1042 if (bitmap_intersects(tmp_map, info->vq_map, SVE_VQ_MAX)) {
1043 pr_warn("%s: cpu%d: Required vector length(s) missing\n",
1044 info->name, smp_processor_id());
1045 return -EINVAL;
1046 }
1047
1048 if (!IS_ENABLED(CONFIG_KVM) || !is_hyp_mode_available())
1049 return 0;
1050
1051 /*
1052 * For KVM, it is necessary to ensure that this CPU doesn't
1053 * support any vector length that guests may have probed as
1054 * unsupported.
1055 */
1056
1057 /* Recover the set of supported VQs: */
1058 bitmap_complement(tmp_map, tmp_map, SVE_VQ_MAX);
1059 /* Find VQs supported that are not globally supported: */
1060 bitmap_andnot(tmp_map, tmp_map, info->vq_map, SVE_VQ_MAX);
1061
1062 /* Find the lowest such VQ, if any: */
1063 b = find_last_bit(tmp_map, SVE_VQ_MAX);
1064 if (b >= SVE_VQ_MAX)
1065 return 0; /* no mismatches */
1066
1067 /*
1068 * Mismatches above sve_max_virtualisable_vl are fine, since
1069 * no guest is allowed to configure ZCR_EL2.LEN to exceed this:
1070 */
1071 if (sve_vl_from_vq(__bit_to_vq(b)) <= info->max_virtualisable_vl) {
1072 pr_warn("%s: cpu%d: Unsupported vector length(s) present\n",
1073 info->name, smp_processor_id());
1074 return -EINVAL;
1075 }
1076
1077 return 0;
1078}
1079
1080void cpu_enable_sve(const struct arm64_cpu_capabilities *__always_unused p)
1081{
1082 write_sysreg(read_sysreg(CPACR_EL1) | CPACR_EL1_ZEN_EL1EN, CPACR_EL1);
1083 isb();
1084
1085 write_sysreg_s(0, SYS_ZCR_EL1);
1086}
1087
1088void __init sve_setup(void)
1089{
1090 struct vl_info *info = &vl_info[ARM64_VEC_SVE];
1091 DECLARE_BITMAP(tmp_map, SVE_VQ_MAX);
1092 unsigned long b;
1093 int max_bit;
1094
1095 if (!system_supports_sve())
1096 return;
1097
1098 /*
1099 * The SVE architecture mandates support for 128-bit vectors,
1100 * so sve_vq_map must have at least SVE_VQ_MIN set.
1101 * If something went wrong, at least try to patch it up:
1102 */
1103 if (WARN_ON(!test_bit(__vq_to_bit(SVE_VQ_MIN), info->vq_map)))
1104 set_bit(__vq_to_bit(SVE_VQ_MIN), info->vq_map);
1105
1106 max_bit = find_first_bit(info->vq_map, SVE_VQ_MAX);
1107 info->max_vl = sve_vl_from_vq(__bit_to_vq(max_bit));
1108
1109 /*
1110 * For the default VL, pick the maximum supported value <= 64.
1111 * VL == 64 is guaranteed not to grow the signal frame.
1112 */
1113 set_sve_default_vl(find_supported_vector_length(ARM64_VEC_SVE, 64));
1114
1115 bitmap_andnot(tmp_map, info->vq_partial_map, info->vq_map,
1116 SVE_VQ_MAX);
1117
1118 b = find_last_bit(tmp_map, SVE_VQ_MAX);
1119 if (b >= SVE_VQ_MAX)
1120 /* No non-virtualisable VLs found */
1121 info->max_virtualisable_vl = SVE_VQ_MAX;
1122 else if (WARN_ON(b == SVE_VQ_MAX - 1))
1123 /* No virtualisable VLs? This is architecturally forbidden. */
1124 info->max_virtualisable_vl = SVE_VQ_MIN;
1125 else /* b + 1 < SVE_VQ_MAX */
1126 info->max_virtualisable_vl = sve_vl_from_vq(__bit_to_vq(b + 1));
1127
1128 if (info->max_virtualisable_vl > info->max_vl)
1129 info->max_virtualisable_vl = info->max_vl;
1130
1131 pr_info("%s: maximum available vector length %u bytes per vector\n",
1132 info->name, info->max_vl);
1133 pr_info("%s: default vector length %u bytes per vector\n",
1134 info->name, get_sve_default_vl());
1135
1136 /* KVM decides whether to support mismatched systems. Just warn here: */
1137 if (sve_max_virtualisable_vl() < sve_max_vl())
1138 pr_warn("%s: unvirtualisable vector lengths present\n",
1139 info->name);
1140}
1141
1142/*
1143 * Called from the put_task_struct() path, which cannot get here
1144 * unless dead_task is really dead and not schedulable.
1145 */
1146void fpsimd_release_task(struct task_struct *dead_task)
1147{
1148 sve_free(dead_task);
1149 sme_free(dead_task);
1150}
1151
1152#endif /* CONFIG_ARM64_SVE */
1153
1154#ifdef CONFIG_ARM64_SME
1155
1156/*
1157 * Ensure that task->thread.sme_state is allocated and sufficiently large.
1158 *
1159 * This function should be used only in preparation for replacing
1160 * task->thread.sme_state with new data. The memory is always zeroed
1161 * here to prevent stale data from showing through: this is done in
1162 * the interest of testability and predictability, the architecture
1163 * guarantees that when ZA is enabled it will be zeroed.
1164 */
1165void sme_alloc(struct task_struct *task, bool flush)
1166{
1167 if (task->thread.sme_state) {
1168 if (flush)
1169 memset(task->thread.sme_state, 0,
1170 sme_state_size(task));
1171 return;
1172 }
1173
1174 /* This could potentially be up to 64K. */
1175 task->thread.sme_state =
1176 kzalloc(sme_state_size(task), GFP_KERNEL);
1177}
1178
1179static void sme_free(struct task_struct *task)
1180{
1181 kfree(task->thread.sme_state);
1182 task->thread.sme_state = NULL;
1183}
1184
1185void cpu_enable_sme(const struct arm64_cpu_capabilities *__always_unused p)
1186{
1187 /* Set priority for all PEs to architecturally defined minimum */
1188 write_sysreg_s(read_sysreg_s(SYS_SMPRI_EL1) & ~SMPRI_EL1_PRIORITY_MASK,
1189 SYS_SMPRI_EL1);
1190
1191 /* Allow SME in kernel */
1192 write_sysreg(read_sysreg(CPACR_EL1) | CPACR_EL1_SMEN_EL1EN, CPACR_EL1);
1193 isb();
1194
1195 /* Ensure all bits in SMCR are set to known values */
1196 write_sysreg_s(0, SYS_SMCR_EL1);
1197
1198 /* Allow EL0 to access TPIDR2 */
1199 write_sysreg(read_sysreg(SCTLR_EL1) | SCTLR_ELx_ENTP2, SCTLR_EL1);
1200 isb();
1201}
1202
1203void cpu_enable_sme2(const struct arm64_cpu_capabilities *__always_unused p)
1204{
1205 /* This must be enabled after SME */
1206 BUILD_BUG_ON(ARM64_SME2 <= ARM64_SME);
1207
1208 /* Allow use of ZT0 */
1209 write_sysreg_s(read_sysreg_s(SYS_SMCR_EL1) | SMCR_ELx_EZT0_MASK,
1210 SYS_SMCR_EL1);
1211}
1212
1213void cpu_enable_fa64(const struct arm64_cpu_capabilities *__always_unused p)
1214{
1215 /* This must be enabled after SME */
1216 BUILD_BUG_ON(ARM64_SME_FA64 <= ARM64_SME);
1217
1218 /* Allow use of FA64 */
1219 write_sysreg_s(read_sysreg_s(SYS_SMCR_EL1) | SMCR_ELx_FA64_MASK,
1220 SYS_SMCR_EL1);
1221}
1222
1223void __init sme_setup(void)
1224{
1225 struct vl_info *info = &vl_info[ARM64_VEC_SME];
1226 int min_bit, max_bit;
1227
1228 if (!system_supports_sme())
1229 return;
1230
1231 min_bit = find_last_bit(info->vq_map, SVE_VQ_MAX);
1232
1233 /*
1234 * SME doesn't require any particular vector length be
1235 * supported but it does require at least one. We should have
1236 * disabled the feature entirely while bringing up CPUs but
1237 * let's double check here. The bitmap is SVE_VQ_MAP sized for
1238 * sharing with SVE.
1239 */
1240 WARN_ON(min_bit >= SVE_VQ_MAX);
1241
1242 info->min_vl = sve_vl_from_vq(__bit_to_vq(min_bit));
1243
1244 max_bit = find_first_bit(info->vq_map, SVE_VQ_MAX);
1245 info->max_vl = sve_vl_from_vq(__bit_to_vq(max_bit));
1246
1247 WARN_ON(info->min_vl > info->max_vl);
1248
1249 /*
1250 * For the default VL, pick the maximum supported value <= 32
1251 * (256 bits) if there is one since this is guaranteed not to
1252 * grow the signal frame when in streaming mode, otherwise the
1253 * minimum available VL will be used.
1254 */
1255 set_sme_default_vl(find_supported_vector_length(ARM64_VEC_SME, 32));
1256
1257 pr_info("SME: minimum available vector length %u bytes per vector\n",
1258 info->min_vl);
1259 pr_info("SME: maximum available vector length %u bytes per vector\n",
1260 info->max_vl);
1261 pr_info("SME: default vector length %u bytes per vector\n",
1262 get_sme_default_vl());
1263}
1264
1265void sme_suspend_exit(void)
1266{
1267 u64 smcr = 0;
1268
1269 if (!system_supports_sme())
1270 return;
1271
1272 if (system_supports_fa64())
1273 smcr |= SMCR_ELx_FA64;
1274 if (system_supports_sme2())
1275 smcr |= SMCR_ELx_EZT0;
1276
1277 write_sysreg_s(smcr, SYS_SMCR_EL1);
1278 write_sysreg_s(0, SYS_SMPRI_EL1);
1279}
1280
1281#endif /* CONFIG_ARM64_SME */
1282
1283/*
1284 * Trapped SVE access
1285 *
1286 * Storage is allocated for the full SVE state, the current FPSIMD
1287 * register contents are migrated across, and the access trap is
1288 * disabled.
1289 *
1290 * TIF_SVE should be clear on entry: otherwise, fpsimd_restore_current_state()
1291 * would have disabled the SVE access trap for userspace during
1292 * ret_to_user, making an SVE access trap impossible in that case.
1293 */
1294void do_sve_acc(unsigned long esr, struct pt_regs *regs)
1295{
1296 /* Even if we chose not to use SVE, the hardware could still trap: */
1297 if (unlikely(!system_supports_sve()) || WARN_ON(is_compat_task())) {
1298 force_signal_inject(SIGILL, ILL_ILLOPC, regs->pc, 0);
1299 return;
1300 }
1301
1302 sve_alloc(current, false);
1303 if (!current->thread.sve_state) {
1304 force_sig(SIGKILL);
1305 return;
1306 }
1307
1308 get_cpu_fpsimd_context();
1309
1310 if (test_and_set_thread_flag(TIF_SVE))
1311 WARN_ON(1); /* SVE access shouldn't have trapped */
1312
1313 /*
1314 * Convert the FPSIMD state to SVE. Stale SVE state can be present in
1315 * registers or memory, so we must zero all state that is not shared
1316 * with FPSIMD.
1317 *
1318 * SVE traps cannot be taken from streaming mode, so there cannot be
1319 * any effective streaming mode SVE state.
1320 */
1321 if (!test_thread_flag(TIF_FOREIGN_FPSTATE)) {
1322 unsigned long vq = sve_vq_from_vl(task_get_sve_vl(current));
1323 sysreg_clear_set_s(SYS_ZCR_EL1, ZCR_ELx_LEN, vq - 1);
1324 sve_flush_live();
1325 fpsimd_bind_task_to_cpu();
1326 } else {
1327 memset(current->thread.sve_state, 0, sve_state_size(current));
1328 fpsimd_to_sve(current);
1329 current->thread.fp_type = FP_STATE_SVE;
1330 fpsimd_flush_task_state(current);
1331 }
1332
1333 put_cpu_fpsimd_context();
1334}
1335
1336#ifdef CONFIG_ARM64_ERRATUM_4193714
1337
1338/*
1339 * SME/CME erratum handling.
1340 */
1341static cpumask_t sme_dvmsync_cpus;
1342cpumask_t sme_active_cpus;
1343
1344/*
1345 * These helpers are only called from non-preemptible contexts, so
1346 * smp_processor_id() is safe here.
1347 */
1348void sme_set_active(void)
1349{
1350 unsigned int cpu = smp_processor_id();
1351
1352 if (!cpumask_test_cpu(cpu, &sme_dvmsync_cpus))
1353 return;
1354
1355 cpumask_set_cpu(cpu, mm_cpumask(current->mm));
1356 cpumask_set_cpu(cpu, &sme_active_cpus);
1357
1358 /*
1359 * A subsequent (post ERET) SME access may use a stale address
1360 * translation. On C1-Pro, a TLBI+DSB on a different CPU will wait for
1361 * the completion of the cpumask_set_cpu() operations above as they
1362 * appear in program order before the SME access. The post-TLBI+DSB
1363 * read of mm_cpumask() or sme_active_cpus will lead to the IPI being
1364 * issued.
1365 *
1366 * https://lore.kernel.org/r/ablEXwhfKyJW1i7l@J2N7QTR9R3
1367 */
1368}
1369
1370void sme_clear_active(void)
1371{
1372 unsigned int cpu = smp_processor_id();
1373
1374 if (!cpumask_test_cpu(cpu, &sme_dvmsync_cpus))
1375 return;
1376
1377 /*
1378 * With SCTLR_EL1.IESB enabled, the SME memory transactions are
1379 * completed on entering EL1.
1380 */
1381 cpumask_clear_cpu(cpu, mm_cpumask(current->mm));
1382 cpumask_clear_cpu(cpu, &sme_active_cpus);
1383}
1384
1385static void sme_dvmsync_ipi(void *unused)
1386{
1387 /*
1388 * With SCTLR_EL1.IESB on, taking an exception is sufficient to ensure
1389 * the completion of the SME memory accesses, so no need for an
1390 * explicit DSB.
1391 */
1392}
1393
1394void sme_do_dvmsync(const struct cpumask *mask)
1395{
1396 /*
1397 * This is called from the TLB maintenance functions after the DSB ISH
1398 * to send the hardware DVMSync message. If this CPU sees the mask as
1399 * empty, the remote CPU executing sme_set_active() would have seen
1400 * the DVMSync and no IPI required.
1401 */
1402 if (cpumask_empty(mask))
1403 return;
1404
1405 preempt_disable();
1406 smp_call_function_many(mask, sme_dvmsync_ipi, NULL, true);
1407 preempt_enable();
1408}
1409
1410void sme_enable_dvmsync(void)
1411{
1412 cpumask_set_cpu(smp_processor_id(), &sme_dvmsync_cpus);
1413}
1414
1415#endif /* CONFIG_ARM64_ERRATUM_4193714 */
1416
1417/*
1418 * Trapped SME access
1419 *
1420 * Storage is allocated for the full SVE and SME state, the current
1421 * FPSIMD register contents are migrated to SVE if SVE is not already
1422 * active, and the access trap is disabled.
1423 *
1424 * TIF_SME should be clear on entry: otherwise, fpsimd_restore_current_state()
1425 * would have disabled the SME access trap for userspace during
1426 * ret_to_user, making an SME access trap impossible in that case.
1427 */
1428void do_sme_acc(unsigned long esr, struct pt_regs *regs)
1429{
1430 /* Even if we chose not to use SME, the hardware could still trap: */
1431 if (unlikely(!system_supports_sme()) || WARN_ON(is_compat_task())) {
1432 force_signal_inject(SIGILL, ILL_ILLOPC, regs->pc, 0);
1433 return;
1434 }
1435
1436 /*
1437 * If this not a trap due to SME being disabled then something
1438 * is being used in the wrong mode, report as SIGILL.
1439 */
1440 if (ESR_ELx_SME_ISS_SMTC(esr) != ESR_ELx_SME_ISS_SMTC_SME_DISABLED) {
1441 force_signal_inject(SIGILL, ILL_ILLOPC, regs->pc, 0);
1442 return;
1443 }
1444
1445 sve_alloc(current, false);
1446 sme_alloc(current, true);
1447 if (!current->thread.sve_state || !current->thread.sme_state) {
1448 force_sig(SIGKILL);
1449 return;
1450 }
1451
1452 get_cpu_fpsimd_context();
1453
1454 /* With TIF_SME userspace shouldn't generate any traps */
1455 if (test_and_set_thread_flag(TIF_SME))
1456 WARN_ON(1);
1457
1458 if (!test_thread_flag(TIF_FOREIGN_FPSTATE)) {
1459 unsigned long vq = sve_vq_from_vl(task_get_sme_vl(current));
1460 sysreg_clear_set_s(SYS_SMCR_EL1, SMCR_ELx_LEN, vq - 1);
1461
1462 fpsimd_bind_task_to_cpu();
1463 } else {
1464 fpsimd_flush_task_state(current);
1465 }
1466
1467 put_cpu_fpsimd_context();
1468}
1469
1470/*
1471 * Trapped FP/ASIMD access.
1472 */
1473void do_fpsimd_acc(unsigned long esr, struct pt_regs *regs)
1474{
1475 /* Even if we chose not to use FPSIMD, the hardware could still trap: */
1476 if (!system_supports_fpsimd()) {
1477 force_signal_inject(SIGILL, ILL_ILLOPC, regs->pc, 0);
1478 return;
1479 }
1480
1481 /*
1482 * When FPSIMD is enabled, we should never take a trap unless something
1483 * has gone very wrong.
1484 */
1485 BUG();
1486}
1487
1488/*
1489 * Raise a SIGFPE for the current process.
1490 */
1491void do_fpsimd_exc(unsigned long esr, struct pt_regs *regs)
1492{
1493 unsigned int si_code = FPE_FLTUNK;
1494
1495 if (esr & ESR_ELx_FP_EXC_TFV) {
1496 if (esr & FPEXC_IOF)
1497 si_code = FPE_FLTINV;
1498 else if (esr & FPEXC_DZF)
1499 si_code = FPE_FLTDIV;
1500 else if (esr & FPEXC_OFF)
1501 si_code = FPE_FLTOVF;
1502 else if (esr & FPEXC_UFF)
1503 si_code = FPE_FLTUND;
1504 else if (esr & FPEXC_IXF)
1505 si_code = FPE_FLTRES;
1506 }
1507
1508 send_sig_fault(SIGFPE, si_code,
1509 (void __user *)instruction_pointer(regs),
1510 current);
1511}
1512
1513static void fpsimd_load_kernel_state(struct task_struct *task)
1514{
1515 struct cpu_fp_state *last = this_cpu_ptr(&fpsimd_last_state);
1516
1517 /*
1518 * Elide the load if this CPU holds the most recent kernel mode
1519 * FPSIMD context of the current task.
1520 */
1521 if (last->st == task->thread.kernel_fpsimd_state &&
1522 task->thread.kernel_fpsimd_cpu == smp_processor_id())
1523 return;
1524
1525 fpsimd_load_state(task->thread.kernel_fpsimd_state);
1526}
1527
1528static void fpsimd_save_kernel_state(struct task_struct *task)
1529{
1530 struct cpu_fp_state cpu_fp_state = {
1531 .st = task->thread.kernel_fpsimd_state,
1532 .to_save = FP_STATE_FPSIMD,
1533 };
1534
1535 BUG_ON(!cpu_fp_state.st);
1536
1537 fpsimd_save_state(task->thread.kernel_fpsimd_state);
1538 fpsimd_bind_state_to_cpu(&cpu_fp_state);
1539
1540 task->thread.kernel_fpsimd_cpu = smp_processor_id();
1541}
1542
1543/*
1544 * Invalidate any task's FPSIMD state that is present on this cpu.
1545 * The FPSIMD context should be acquired with get_cpu_fpsimd_context()
1546 * before calling this function.
1547 */
1548static void fpsimd_flush_cpu_state(void)
1549{
1550 WARN_ON(!system_supports_fpsimd());
1551 __this_cpu_write(fpsimd_last_state.st, NULL);
1552
1553 /*
1554 * Leaving streaming mode enabled will cause issues for any kernel
1555 * NEON and leaving streaming mode or ZA enabled may increase power
1556 * consumption.
1557 */
1558 if (system_supports_sme())
1559 sme_smstop();
1560
1561 set_thread_flag(TIF_FOREIGN_FPSTATE);
1562}
1563
1564void fpsimd_thread_switch(struct task_struct *next)
1565{
1566 bool wrong_task, wrong_cpu;
1567
1568 if (!system_supports_fpsimd())
1569 return;
1570
1571 WARN_ON_ONCE(!irqs_disabled());
1572
1573 /* Save unsaved fpsimd state, if any: */
1574 if (test_thread_flag(TIF_KERNEL_FPSTATE))
1575 fpsimd_save_kernel_state(current);
1576 else
1577 fpsimd_save_user_state();
1578
1579 if (test_tsk_thread_flag(next, TIF_KERNEL_FPSTATE)) {
1580 fpsimd_flush_cpu_state();
1581 fpsimd_load_kernel_state(next);
1582 } else {
1583 /*
1584 * Fix up TIF_FOREIGN_FPSTATE to correctly describe next's
1585 * state. For kernel threads, FPSIMD registers are never
1586 * loaded with user mode FPSIMD state and so wrong_task and
1587 * wrong_cpu will always be true.
1588 */
1589 wrong_task = __this_cpu_read(fpsimd_last_state.st) !=
1590 &next->thread.uw.fpsimd_state;
1591 wrong_cpu = next->thread.fpsimd_cpu != smp_processor_id();
1592
1593 update_tsk_thread_flag(next, TIF_FOREIGN_FPSTATE,
1594 wrong_task || wrong_cpu);
1595 }
1596}
1597
1598static void fpsimd_flush_thread_vl(enum vec_type type)
1599{
1600 int vl, supported_vl;
1601
1602 /*
1603 * Reset the task vector length as required. This is where we
1604 * ensure that all user tasks have a valid vector length
1605 * configured: no kernel task can become a user task without
1606 * an exec and hence a call to this function. By the time the
1607 * first call to this function is made, all early hardware
1608 * probing is complete, so __sve_default_vl should be valid.
1609 * If a bug causes this to go wrong, we make some noise and
1610 * try to fudge thread.sve_vl to a safe value here.
1611 */
1612 vl = task_get_vl_onexec(current, type);
1613 if (!vl)
1614 vl = get_default_vl(type);
1615
1616 if (WARN_ON(!sve_vl_valid(vl)))
1617 vl = vl_info[type].min_vl;
1618
1619 supported_vl = find_supported_vector_length(type, vl);
1620 if (WARN_ON(supported_vl != vl))
1621 vl = supported_vl;
1622
1623 task_set_vl(current, type, vl);
1624
1625 /*
1626 * If the task is not set to inherit, ensure that the vector
1627 * length will be reset by a subsequent exec:
1628 */
1629 if (!test_thread_flag(vec_vl_inherit_flag(type)))
1630 task_set_vl_onexec(current, type, 0);
1631}
1632
1633void fpsimd_flush_thread(void)
1634{
1635 struct arm64_sve_state *sve_state = NULL;
1636 struct arm64_sme_state *sme_state = NULL;
1637
1638 if (!system_supports_fpsimd())
1639 return;
1640
1641 get_cpu_fpsimd_context();
1642
1643 fpsimd_flush_task_state(current);
1644 memset(&current->thread.uw.fpsimd_state, 0,
1645 sizeof(current->thread.uw.fpsimd_state));
1646
1647 if (system_supports_sve()) {
1648 clear_thread_flag(TIF_SVE);
1649
1650 /* Defer kfree() while in atomic context */
1651 sve_state = current->thread.sve_state;
1652 current->thread.sve_state = NULL;
1653
1654 fpsimd_flush_thread_vl(ARM64_VEC_SVE);
1655 }
1656
1657 if (system_supports_sme()) {
1658 clear_thread_flag(TIF_SME);
1659
1660 /* Defer kfree() while in atomic context */
1661 sme_state = current->thread.sme_state;
1662 current->thread.sme_state = NULL;
1663
1664 fpsimd_flush_thread_vl(ARM64_VEC_SME);
1665 current->thread.svcr = 0;
1666 }
1667
1668 if (system_supports_fpmr())
1669 current->thread.uw.fpmr = 0;
1670
1671 current->thread.fp_type = FP_STATE_FPSIMD;
1672
1673 put_cpu_fpsimd_context();
1674 kfree(sve_state);
1675 kfree(sme_state);
1676}
1677
1678/*
1679 * Save the userland FPSIMD state of 'current' to memory, but only if the state
1680 * currently held in the registers does in fact belong to 'current'
1681 */
1682void fpsimd_preserve_current_state(void)
1683{
1684 if (!system_supports_fpsimd())
1685 return;
1686
1687 get_cpu_fpsimd_context();
1688 fpsimd_save_user_state();
1689 put_cpu_fpsimd_context();
1690}
1691
1692/*
1693 * Associate current's FPSIMD context with this cpu
1694 * The caller must have ownership of the cpu FPSIMD context before calling
1695 * this function.
1696 */
1697static void fpsimd_bind_task_to_cpu(void)
1698{
1699 struct cpu_fp_state *last = this_cpu_ptr(&fpsimd_last_state);
1700
1701 WARN_ON(!system_supports_fpsimd());
1702 last->st = &current->thread.uw.fpsimd_state;
1703 last->sve_state = current->thread.sve_state;
1704 last->sme_state = current->thread.sme_state;
1705 last->sve_vl = task_get_sve_vl(current);
1706 last->sme_vl = task_get_sme_vl(current);
1707 last->svcr = &current->thread.svcr;
1708 last->fpmr = &current->thread.uw.fpmr;
1709 last->fp_type = &current->thread.fp_type;
1710 last->to_save = FP_STATE_CURRENT;
1711 current->thread.fpsimd_cpu = smp_processor_id();
1712
1713 /*
1714 * Toggle SVE and SME trapping for userspace if needed, these
1715 * are serialsied by ret_to_user().
1716 */
1717 if (system_supports_sme()) {
1718 if (test_thread_flag(TIF_SME))
1719 sme_user_enable();
1720 else
1721 sme_user_disable();
1722 }
1723
1724 if (system_supports_sve()) {
1725 if (test_thread_flag(TIF_SVE))
1726 sve_user_enable();
1727 else
1728 sve_user_disable();
1729 }
1730}
1731
1732void fpsimd_bind_state_to_cpu(struct cpu_fp_state *state)
1733{
1734 struct cpu_fp_state *last = this_cpu_ptr(&fpsimd_last_state);
1735
1736 WARN_ON(!system_supports_fpsimd());
1737 WARN_ON(!in_softirq() && !irqs_disabled());
1738
1739 *last = *state;
1740}
1741
1742/*
1743 * Load the userland FPSIMD state of 'current' from memory, but only if the
1744 * FPSIMD state already held in the registers is /not/ the most recent FPSIMD
1745 * state of 'current'. This is called when we are preparing to return to
1746 * userspace to ensure that userspace sees a good register state.
1747 */
1748void fpsimd_restore_current_state(void)
1749{
1750 /*
1751 * TIF_FOREIGN_FPSTATE is set on the init task and copied by
1752 * arch_dup_task_struct() regardless of whether FP/SIMD is detected.
1753 * Thus user threads can have this set even when FP/SIMD hasn't been
1754 * detected.
1755 *
1756 * When FP/SIMD is detected, begin_new_exec() will set
1757 * TIF_FOREIGN_FPSTATE via flush_thread() -> fpsimd_flush_thread(),
1758 * and fpsimd_thread_switch() will set TIF_FOREIGN_FPSTATE when
1759 * switching tasks. We detect FP/SIMD before we exec the first user
1760 * process, ensuring this has TIF_FOREIGN_FPSTATE set and
1761 * do_notify_resume() will call fpsimd_restore_current_state() to
1762 * install the user FP/SIMD context.
1763 *
1764 * When FP/SIMD is not detected, nothing else will clear or set
1765 * TIF_FOREIGN_FPSTATE prior to the first return to userspace, and
1766 * we must clear TIF_FOREIGN_FPSTATE to avoid do_notify_resume()
1767 * looping forever calling fpsimd_restore_current_state().
1768 */
1769 if (!system_supports_fpsimd()) {
1770 clear_thread_flag(TIF_FOREIGN_FPSTATE);
1771 return;
1772 }
1773
1774 get_cpu_fpsimd_context();
1775
1776 if (test_and_clear_thread_flag(TIF_FOREIGN_FPSTATE)) {
1777 task_fpsimd_load();
1778 fpsimd_bind_task_to_cpu();
1779 }
1780
1781 put_cpu_fpsimd_context();
1782}
1783
1784void fpsimd_update_current_state(struct user_fpsimd_state const *state)
1785{
1786 if (WARN_ON(!system_supports_fpsimd()))
1787 return;
1788
1789 current->thread.uw.fpsimd_state = *state;
1790 if (current->thread.fp_type == FP_STATE_SVE)
1791 fpsimd_to_sve(current);
1792}
1793
1794/*
1795 * Invalidate live CPU copies of task t's FPSIMD state
1796 *
1797 * This function may be called with preemption enabled. The barrier()
1798 * ensures that the assignment to fpsimd_cpu is visible to any
1799 * preemption/softirq that could race with set_tsk_thread_flag(), so
1800 * that TIF_FOREIGN_FPSTATE cannot be spuriously re-cleared.
1801 *
1802 * The final barrier ensures that TIF_FOREIGN_FPSTATE is seen set by any
1803 * subsequent code.
1804 */
1805void fpsimd_flush_task_state(struct task_struct *t)
1806{
1807 t->thread.fpsimd_cpu = NR_CPUS;
1808 t->thread.kernel_fpsimd_state = NULL;
1809 /*
1810 * If we don't support fpsimd, bail out after we have
1811 * reset the fpsimd_cpu for this task and clear the
1812 * FPSTATE.
1813 */
1814 if (!system_supports_fpsimd())
1815 return;
1816 barrier();
1817 set_tsk_thread_flag(t, TIF_FOREIGN_FPSTATE);
1818
1819 barrier();
1820}
1821
1822void fpsimd_save_and_flush_current_state(void)
1823{
1824 if (!system_supports_fpsimd())
1825 return;
1826
1827 get_cpu_fpsimd_context();
1828 fpsimd_save_user_state();
1829 fpsimd_flush_task_state(current);
1830 put_cpu_fpsimd_context();
1831}
1832
1833/*
1834 * Save the FPSIMD state to memory and invalidate cpu view.
1835 * This function must be called with preemption disabled.
1836 */
1837void fpsimd_save_and_flush_cpu_state(void)
1838{
1839 unsigned long flags;
1840
1841 if (!system_supports_fpsimd())
1842 return;
1843 WARN_ON(preemptible());
1844 local_irq_save(flags);
1845 fpsimd_save_user_state();
1846 fpsimd_flush_cpu_state();
1847 local_irq_restore(flags);
1848}
1849
1850#ifdef CONFIG_KERNEL_MODE_NEON
1851
1852/*
1853 * Kernel-side NEON support functions
1854 */
1855
1856/*
1857 * kernel_neon_begin(): obtain the CPU FPSIMD registers for use by the calling
1858 * context
1859 *
1860 * Must not be called unless may_use_simd() returns true.
1861 * Task context in the FPSIMD registers is saved back to memory as necessary.
1862 *
1863 * A matching call to kernel_neon_end() must be made before returning from the
1864 * calling context.
1865 *
1866 * The caller may freely use the FPSIMD registers until kernel_neon_end() is
1867 * called.
1868 *
1869 * Unless called from non-preemptible task context, @state must point to a
1870 * caller provided buffer that will be used to preserve the task's kernel mode
1871 * FPSIMD context when it is scheduled out, or if it is interrupted by kernel
1872 * mode FPSIMD occurring in softirq context. May be %NULL otherwise.
1873 */
1874void kernel_neon_begin(struct user_fpsimd_state *state)
1875{
1876 if (WARN_ON(!system_supports_fpsimd()))
1877 return;
1878
1879 WARN_ON((preemptible() || in_serving_softirq()) && !state);
1880
1881 BUG_ON(!may_use_simd());
1882
1883 get_cpu_fpsimd_context();
1884
1885 /* Save unsaved fpsimd state, if any: */
1886 if (test_thread_flag(TIF_KERNEL_FPSTATE)) {
1887 BUG_ON(IS_ENABLED(CONFIG_PREEMPT_RT) || !in_serving_softirq());
1888 fpsimd_save_state(state);
1889 } else {
1890 fpsimd_save_user_state();
1891
1892 /*
1893 * Set the thread flag so that the kernel mode FPSIMD state
1894 * will be context switched along with the rest of the task
1895 * state.
1896 *
1897 * On non-PREEMPT_RT, softirqs may interrupt task level kernel
1898 * mode FPSIMD, but the task will not be preemptible so setting
1899 * TIF_KERNEL_FPSTATE for those would be both wrong (as it
1900 * would mark the task context FPSIMD state as requiring a
1901 * context switch) and unnecessary.
1902 *
1903 * On PREEMPT_RT, softirqs are serviced from a separate thread,
1904 * which is scheduled as usual, and this guarantees that these
1905 * softirqs are not interrupting use of the FPSIMD in kernel
1906 * mode in task context. So in this case, setting the flag here
1907 * is always appropriate.
1908 */
1909 if (IS_ENABLED(CONFIG_PREEMPT_RT) || !in_serving_softirq()) {
1910 /*
1911 * Record the caller provided buffer as the kernel mode
1912 * FP/SIMD buffer for this task, so that the state can
1913 * be preserved and restored on a context switch.
1914 */
1915 WARN_ON(current->thread.kernel_fpsimd_state != NULL);
1916 current->thread.kernel_fpsimd_state = state;
1917 set_thread_flag(TIF_KERNEL_FPSTATE);
1918 }
1919 }
1920
1921 /* Invalidate any task state remaining in the fpsimd regs: */
1922 fpsimd_flush_cpu_state();
1923
1924 put_cpu_fpsimd_context();
1925}
1926EXPORT_SYMBOL_GPL(kernel_neon_begin);
1927
1928/*
1929 * kernel_neon_end(): give the CPU FPSIMD registers back to the current task
1930 *
1931 * Must be called from a context in which kernel_neon_begin() was previously
1932 * called, with no call to kernel_neon_end() in the meantime.
1933 *
1934 * The caller must not use the FPSIMD registers after this function is called,
1935 * unless kernel_neon_begin() is called again in the meantime.
1936 *
1937 * The value of @state must match the value passed to the preceding call to
1938 * kernel_neon_begin().
1939 */
1940void kernel_neon_end(struct user_fpsimd_state *state)
1941{
1942 if (!system_supports_fpsimd())
1943 return;
1944
1945 if (!test_thread_flag(TIF_KERNEL_FPSTATE))
1946 return;
1947
1948 /*
1949 * If we are returning from a nested use of kernel mode FPSIMD, restore
1950 * the task context kernel mode FPSIMD state. This can only happen when
1951 * running in softirq context on non-PREEMPT_RT.
1952 */
1953 if (!IS_ENABLED(CONFIG_PREEMPT_RT) && in_serving_softirq()) {
1954 fpsimd_load_state(state);
1955 } else {
1956 clear_thread_flag(TIF_KERNEL_FPSTATE);
1957 WARN_ON(current->thread.kernel_fpsimd_state != state);
1958 current->thread.kernel_fpsimd_state = NULL;
1959 }
1960}
1961EXPORT_SYMBOL_GPL(kernel_neon_end);
1962
1963#ifdef CONFIG_EFI
1964
1965static struct user_fpsimd_state efi_fpsimd_state;
1966
1967/*
1968 * EFI runtime services support functions
1969 *
1970 * The ABI for EFI runtime services allows EFI to use FPSIMD during the call.
1971 * This means that for EFI (and only for EFI), we have to assume that FPSIMD
1972 * is always used rather than being an optional accelerator.
1973 *
1974 * These functions provide the necessary support for ensuring FPSIMD
1975 * save/restore in the contexts from which EFI is used.
1976 *
1977 * Do not use them for any other purpose -- if tempted to do so, you are
1978 * either doing something wrong or you need to propose some refactoring.
1979 */
1980
1981/*
1982 * __efi_fpsimd_begin(): prepare FPSIMD for making an EFI runtime services call
1983 */
1984void __efi_fpsimd_begin(void)
1985{
1986 if (!system_supports_fpsimd())
1987 return;
1988
1989 if (may_use_simd()) {
1990 kernel_neon_begin(&efi_fpsimd_state);
1991 } else {
1992 /*
1993 * We are running in hardirq or NMI context, and the only
1994 * legitimate case where this might happen is when EFI pstore
1995 * is attempting to record the system's dying gasps into EFI
1996 * variables. This could be due to an oops, a panic or a call
1997 * to emergency_restart(), and in none of those cases, we can
1998 * expect the current task to ever return to user space again,
1999 * or for the kernel to resume any normal execution, for that
2000 * matter (an oops in hardirq context triggers a panic too).
2001 *
2002 * Therefore, there is no point in attempting to preserve any
2003 * SVE/SME state here. On the off chance that we might have
2004 * ended up here for a different reason inadvertently, kill the
2005 * task and preserve/restore the base FP/SIMD state, which
2006 * might belong to kernel mode FP/SIMD.
2007 */
2008 pr_warn_ratelimited("Calling EFI runtime from %s context\n",
2009 in_nmi() ? "NMI" : "hardirq");
2010 force_signal_inject(SIGKILL, SI_KERNEL, 0, 0);
2011 fpsimd_save_state(&efi_fpsimd_state);
2012 }
2013}
2014
2015/*
2016 * __efi_fpsimd_end(): clean up FPSIMD after an EFI runtime services call
2017 */
2018void __efi_fpsimd_end(void)
2019{
2020 if (!system_supports_fpsimd())
2021 return;
2022
2023 if (may_use_simd()) {
2024 kernel_neon_end(&efi_fpsimd_state);
2025 } else {
2026 fpsimd_load_state(&efi_fpsimd_state);
2027 }
2028}
2029
2030#endif /* CONFIG_EFI */
2031
2032#endif /* CONFIG_KERNEL_MODE_NEON */
2033
2034#ifdef CONFIG_CPU_PM
2035static int fpsimd_cpu_pm_notifier(struct notifier_block *self,
2036 unsigned long cmd, void *v)
2037{
2038 switch (cmd) {
2039 case CPU_PM_ENTER:
2040 fpsimd_save_and_flush_cpu_state();
2041 break;
2042 case CPU_PM_EXIT:
2043 break;
2044 case CPU_PM_ENTER_FAILED:
2045 default:
2046 return NOTIFY_DONE;
2047 }
2048 return NOTIFY_OK;
2049}
2050
2051static struct notifier_block fpsimd_cpu_pm_notifier_block = {
2052 .notifier_call = fpsimd_cpu_pm_notifier,
2053};
2054
2055static void __init fpsimd_pm_init(void)
2056{
2057 cpu_pm_register_notifier(&fpsimd_cpu_pm_notifier_block);
2058}
2059
2060#else
2061static inline void fpsimd_pm_init(void) { }
2062#endif /* CONFIG_CPU_PM */
2063
2064#ifdef CONFIG_HOTPLUG_CPU
2065static int fpsimd_cpu_dead(unsigned int cpu)
2066{
2067 per_cpu(fpsimd_last_state.st, cpu) = NULL;
2068 return 0;
2069}
2070
2071static inline void fpsimd_hotplug_init(void)
2072{
2073 cpuhp_setup_state_nocalls(CPUHP_ARM64_FPSIMD_DEAD, "arm64/fpsimd:dead",
2074 NULL, fpsimd_cpu_dead);
2075}
2076
2077#else
2078static inline void fpsimd_hotplug_init(void) { }
2079#endif
2080
2081void cpu_enable_fpsimd(const struct arm64_cpu_capabilities *__always_unused p)
2082{
2083 unsigned long enable = CPACR_EL1_FPEN_EL1EN | CPACR_EL1_FPEN_EL0EN;
2084 write_sysreg(read_sysreg(CPACR_EL1) | enable, CPACR_EL1);
2085 isb();
2086}
2087
2088/*
2089 * FP/SIMD support code initialisation.
2090 */
2091static int __init fpsimd_init(void)
2092{
2093 if (cpu_have_named_feature(FP)) {
2094 fpsimd_pm_init();
2095 fpsimd_hotplug_init();
2096 } else {
2097 pr_notice("Floating-point is not implemented\n");
2098 }
2099
2100 if (!cpu_have_named_feature(ASIMD))
2101 pr_notice("Advanced SIMD is not implemented\n");
2102
2103
2104 sve_sysctl_init();
2105 sme_sysctl_init();
2106
2107 return 0;
2108}
2109core_initcall(fpsimd_init);