projects/linux·blob master · refs

1213 lines · 31.8 KiB

1// SPDX-License-Identifier: GPL-2.0-only
2/*
3 * Handle detection, reporting and mitigation of Spectre v1, v2, v3a and v4, as
4 * detailed at:
5 *
6 * https://developer.arm.com/support/arm-security-updates/speculative-processor-vulnerability
7 *
8 * This code was originally written hastily under an awful lot of stress and so
9 * aspects of it are somewhat hacky. Unfortunately, changing anything in here
10 * instantly makes me feel ill. Thanks, Jann. Thann.
11 *
12 * Copyright (C) 2018 ARM Ltd, All Rights Reserved.
13 * Copyright (C) 2020 Google LLC
14 *
15 * "If there's something strange in your neighbourhood, who you gonna call?"
16 *
17 * Authors: Will Deacon <[email protected]> and Marc Zyngier <[email protected]>
18 */
19
20#include <linux/arm-smccc.h>
21#include <linux/bpf.h>
22#include <linux/cpu.h>
23#include <linux/device.h>
24#include <linux/nospec.h>
25#include <linux/prctl.h>
26#include <linux/sched/task_stack.h>
27#include <linux/sysfs.h>
28
29#include <asm/debug-monitors.h>
30#include <asm/insn.h>
31#include <asm/spectre.h>
32#include <asm/traps.h>
33#include <asm/vectors.h>
34#include <asm/virt.h>
35
36/*
37 * We try to ensure that the mitigation state can never change as the result of
38 * onlining a late CPU.
39 */
40static void update_mitigation_state(enum mitigation_state *oldp,
41 enum mitigation_state new)
42{
43 enum mitigation_state state;
44
45 do {
46 state = READ_ONCE(*oldp);
47 if (new <= state)
48 break;
49
50 /* Userspace almost certainly can't deal with this. */
51 if (WARN_ON(system_capabilities_finalized()))
52 break;
53 } while (cmpxchg_relaxed(oldp, state, new) != state);
54}
55
56/*
57 * Spectre v1.
58 *
59 * The kernel can't protect userspace for this one: it's each person for
60 * themselves. Advertise what we're doing and be done with it.
61 */
62ssize_t cpu_show_spectre_v1(struct device *dev, struct device_attribute *attr,
63 char *buf)
64{
65 return sysfs_emit(buf, "Mitigation: __user pointer sanitization\n");
66}
67
68/*
69 * Spectre v2.
70 *
71 * This one sucks. A CPU is either:
72 *
73 * - Mitigated in hardware and advertised by ID_AA64PFR0_EL1.CSV2.
74 * - Mitigated in hardware and listed in our "safe list".
75 * - Mitigated in software by firmware.
76 * - Mitigated in software by a CPU-specific dance in the kernel and a
77 * firmware call at EL2.
78 * - Vulnerable.
79 *
80 * It's not unlikely for different CPUs in a big.LITTLE system to fall into
81 * different camps.
82 */
83static enum mitigation_state spectre_v2_state;
84
85static bool __read_mostly __nospectre_v2;
86static int __init parse_spectre_v2_param(char *str)
87{
88 __nospectre_v2 = true;
89 return 0;
90}
91early_param("nospectre_v2", parse_spectre_v2_param);
92
93static bool spectre_v2_mitigations_off(void)
94{
95 return __nospectre_v2 || cpu_mitigations_off();
96}
97
98static const char *get_bhb_affected_string(enum mitigation_state bhb_state)
99{
100 switch (bhb_state) {
101 case SPECTRE_UNAFFECTED:
102 return "";
103 default:
104 case SPECTRE_VULNERABLE:
105 return ", but not BHB";
106 case SPECTRE_MITIGATED:
107 return ", BHB";
108 }
109}
110
111static bool _unprivileged_ebpf_enabled(void)
112{
113#ifdef CONFIG_BPF_SYSCALL
114 return !sysctl_unprivileged_bpf_disabled;
115#else
116 return false;
117#endif
118}
119
120ssize_t cpu_show_spectre_v2(struct device *dev, struct device_attribute *attr,
121 char *buf)
122{
123 enum mitigation_state bhb_state = arm64_get_spectre_bhb_state();
124 const char *bhb_str = get_bhb_affected_string(bhb_state);
125 const char *v2_str = "Branch predictor hardening";
126
127 switch (spectre_v2_state) {
128 case SPECTRE_UNAFFECTED:
129 if (bhb_state == SPECTRE_UNAFFECTED)
130 return sysfs_emit(buf, "Not affected\n");
131
132 /*
133 * Platforms affected by Spectre-BHB can't report
134 * "Not affected" for Spectre-v2.
135 */
136 v2_str = "CSV2";
137 fallthrough;
138 case SPECTRE_MITIGATED:
139 if (bhb_state == SPECTRE_MITIGATED && _unprivileged_ebpf_enabled())
140 return sysfs_emit(buf, "Vulnerable: Unprivileged eBPF enabled\n");
141
142 return sysfs_emit(buf, "Mitigation: %s%s\n", v2_str, bhb_str);
143 case SPECTRE_VULNERABLE:
144 fallthrough;
145 default:
146 return sysfs_emit(buf, "Vulnerable\n");
147 }
148}
149
150static enum mitigation_state spectre_v2_get_cpu_hw_mitigation_state(void)
151{
152 u64 pfr0;
153 static const struct midr_range spectre_v2_safe_list[] = {
154 MIDR_ALL_VERSIONS(MIDR_CORTEX_A35),
155 MIDR_ALL_VERSIONS(MIDR_CORTEX_A53),
156 MIDR_ALL_VERSIONS(MIDR_CORTEX_A55),
157 MIDR_ALL_VERSIONS(MIDR_BRAHMA_B53),
158 MIDR_ALL_VERSIONS(MIDR_HISI_TSV110),
159 MIDR_ALL_VERSIONS(MIDR_QCOM_KRYO_2XX_SILVER),
160 MIDR_ALL_VERSIONS(MIDR_QCOM_KRYO_3XX_SILVER),
161 MIDR_ALL_VERSIONS(MIDR_QCOM_KRYO_4XX_SILVER),
162 { /* sentinel */ }
163 };
164
165 /* If the CPU has CSV2 set, we're safe */
166 pfr0 = read_cpuid(ID_AA64PFR0_EL1);
167 if (cpuid_feature_extract_unsigned_field(pfr0, ID_AA64PFR0_EL1_CSV2_SHIFT))
168 return SPECTRE_UNAFFECTED;
169
170 /* Alternatively, we have a list of unaffected CPUs */
171 if (is_midr_in_range_list(spectre_v2_safe_list))
172 return SPECTRE_UNAFFECTED;
173
174 return SPECTRE_VULNERABLE;
175}
176
177static enum mitigation_state spectre_v2_get_cpu_fw_mitigation_state(void)
178{
179 int ret;
180 struct arm_smccc_res res;
181
182 arm_smccc_1_1_invoke(ARM_SMCCC_ARCH_FEATURES_FUNC_ID,
183 ARM_SMCCC_ARCH_WORKAROUND_1, &res);
184
185 ret = res.a0;
186 switch (ret) {
187 case SMCCC_RET_SUCCESS:
188 return SPECTRE_MITIGATED;
189 case SMCCC_ARCH_WORKAROUND_RET_UNAFFECTED:
190 return SPECTRE_UNAFFECTED;
191 default:
192 fallthrough;
193 case SMCCC_RET_NOT_SUPPORTED:
194 return SPECTRE_VULNERABLE;
195 }
196}
197
198bool has_spectre_v2(const struct arm64_cpu_capabilities *entry, int scope)
199{
200 WARN_ON(scope != SCOPE_LOCAL_CPU || preemptible());
201
202 if (spectre_v2_get_cpu_hw_mitigation_state() == SPECTRE_UNAFFECTED)
203 return false;
204
205 if (spectre_v2_get_cpu_fw_mitigation_state() == SPECTRE_UNAFFECTED)
206 return false;
207
208 return true;
209}
210
211enum mitigation_state arm64_get_spectre_v2_state(void)
212{
213 return spectre_v2_state;
214}
215
216DEFINE_PER_CPU_READ_MOSTLY(struct bp_hardening_data, bp_hardening_data);
217
218static void install_bp_hardening_cb(bp_hardening_cb_t fn)
219{
220 __this_cpu_write(bp_hardening_data.fn, fn);
221
222 /*
223 * Vinz Clortho takes the hyp_vecs start/end "keys" at
224 * the door when we're a guest. Skip the hyp-vectors work.
225 */
226 if (!is_hyp_mode_available())
227 return;
228
229 __this_cpu_write(bp_hardening_data.slot, HYP_VECTOR_SPECTRE_DIRECT);
230}
231
232/* Called during entry so must be noinstr */
233static noinstr void call_smc_arch_workaround_1(void)
234{
235 arm_smccc_1_1_smc(ARM_SMCCC_ARCH_WORKAROUND_1, NULL);
236}
237
238/* Called during entry so must be noinstr */
239static noinstr void call_hvc_arch_workaround_1(void)
240{
241 arm_smccc_1_1_hvc(ARM_SMCCC_ARCH_WORKAROUND_1, NULL);
242}
243
244/* Called during entry so must be noinstr */
245static noinstr void qcom_link_stack_sanitisation(void)
246{
247 u64 tmp;
248
249 asm volatile("mov %0, x30 \n"
250 ".rept 16 \n"
251 "bl . + 4 \n"
252 ".endr \n"
253 "mov x30, %0 \n"
254 : "=&r" (tmp));
255}
256
257static bp_hardening_cb_t spectre_v2_get_sw_mitigation_cb(void)
258{
259 u32 midr = read_cpuid_id();
260 if (((midr & MIDR_CPU_MODEL_MASK) != MIDR_QCOM_FALKOR) &&
261 ((midr & MIDR_CPU_MODEL_MASK) != MIDR_QCOM_FALKOR_V1))
262 return NULL;
263
264 return qcom_link_stack_sanitisation;
265}
266
267static enum mitigation_state spectre_v2_enable_fw_mitigation(void)
268{
269 bp_hardening_cb_t cb;
270 enum mitigation_state state;
271
272 state = spectre_v2_get_cpu_fw_mitigation_state();
273 if (state != SPECTRE_MITIGATED)
274 return state;
275
276 if (spectre_v2_mitigations_off())
277 return SPECTRE_VULNERABLE;
278
279 switch (arm_smccc_1_1_get_conduit()) {
280 case SMCCC_CONDUIT_HVC:
281 cb = call_hvc_arch_workaround_1;
282 break;
283
284 case SMCCC_CONDUIT_SMC:
285 cb = call_smc_arch_workaround_1;
286 break;
287
288 default:
289 return SPECTRE_VULNERABLE;
290 }
291
292 /*
293 * Prefer a CPU-specific workaround if it exists. Note that we
294 * still rely on firmware for the mitigation at EL2.
295 */
296 cb = spectre_v2_get_sw_mitigation_cb() ?: cb;
297 install_bp_hardening_cb(cb);
298 return SPECTRE_MITIGATED;
299}
300
301void spectre_v2_enable_mitigation(const struct arm64_cpu_capabilities *__unused)
302{
303 enum mitigation_state state;
304
305 WARN_ON(preemptible());
306
307 state = spectre_v2_get_cpu_hw_mitigation_state();
308 if (state == SPECTRE_VULNERABLE)
309 state = spectre_v2_enable_fw_mitigation();
310
311 update_mitigation_state(&spectre_v2_state, state);
312}
313
314/*
315 * Spectre-v3a.
316 *
317 * Phew, there's not an awful lot to do here! We just instruct EL2 to use
318 * an indirect trampoline for the hyp vectors so that guests can't read
319 * VBAR_EL2 to defeat randomisation of the hypervisor VA layout.
320 */
321bool has_spectre_v3a(const struct arm64_cpu_capabilities *entry, int scope)
322{
323 static const struct midr_range spectre_v3a_unsafe_list[] = {
324 MIDR_ALL_VERSIONS(MIDR_CORTEX_A57),
325 MIDR_ALL_VERSIONS(MIDR_CORTEX_A72),
326 {},
327 };
328
329 WARN_ON(scope != SCOPE_LOCAL_CPU || preemptible());
330 return is_midr_in_range_list(spectre_v3a_unsafe_list);
331}
332
333void spectre_v3a_enable_mitigation(const struct arm64_cpu_capabilities *__unused)
334{
335 struct bp_hardening_data *data = this_cpu_ptr(&bp_hardening_data);
336
337 if (this_cpu_has_cap(ARM64_SPECTRE_V3A))
338 data->slot += HYP_VECTOR_INDIRECT;
339}
340
341/*
342 * Spectre v4.
343 *
344 * If you thought Spectre v2 was nasty, wait until you see this mess. A CPU is
345 * either:
346 *
347 * - Mitigated in hardware and listed in our "safe list".
348 * - Mitigated in hardware via PSTATE.SSBS.
349 * - Mitigated in software by firmware (sometimes referred to as SSBD).
350 *
351 * Wait, that doesn't sound so bad, does it? Keep reading...
352 *
353 * A major source of headaches is that the software mitigation is enabled both
354 * on a per-task basis, but can also be forced on for the kernel, necessitating
355 * both context-switch *and* entry/exit hooks. To make it even worse, some CPUs
356 * allow EL0 to toggle SSBS directly, which can end up with the prctl() state
357 * being stale when re-entering the kernel. The usual big.LITTLE caveats apply,
358 * so you can have systems that have both firmware and SSBS mitigations. This
359 * means we actually have to reject late onlining of CPUs with mitigations if
360 * all of the currently onlined CPUs are safelisted, as the mitigation tends to
361 * be opt-in for userspace. Yes, really, the cure is worse than the disease.
362 *
363 * The only good part is that if the firmware mitigation is present, then it is
364 * present for all CPUs, meaning we don't have to worry about late onlining of a
365 * vulnerable CPU if one of the boot CPUs is using the firmware mitigation.
366 *
367 * Give me a VAX-11/780 any day of the week...
368 */
369static enum mitigation_state spectre_v4_state;
370
371/* This is the per-cpu state tracking whether we need to talk to firmware */
372DEFINE_PER_CPU_READ_MOSTLY(u64, arm64_ssbd_callback_required);
373
374enum spectre_v4_policy {
375 SPECTRE_V4_POLICY_MITIGATION_DYNAMIC,
376 SPECTRE_V4_POLICY_MITIGATION_ENABLED,
377 SPECTRE_V4_POLICY_MITIGATION_DISABLED,
378};
379
380static enum spectre_v4_policy __read_mostly __spectre_v4_policy;
381
382static const struct spectre_v4_param {
383 const char *str;
384 enum spectre_v4_policy policy;
385} spectre_v4_params[] = {
386 { "force-on", SPECTRE_V4_POLICY_MITIGATION_ENABLED, },
387 { "force-off", SPECTRE_V4_POLICY_MITIGATION_DISABLED, },
388 { "kernel", SPECTRE_V4_POLICY_MITIGATION_DYNAMIC, },
389};
390static int __init parse_spectre_v4_param(char *str)
391{
392 int i;
393
394 if (!str || !str[0])
395 return -EINVAL;
396
397 for (i = 0; i < ARRAY_SIZE(spectre_v4_params); i++) {
398 const struct spectre_v4_param *param = &spectre_v4_params[i];
399
400 if (strncmp(str, param->str, strlen(param->str)))
401 continue;
402
403 __spectre_v4_policy = param->policy;
404 return 0;
405 }
406
407 return -EINVAL;
408}
409early_param("ssbd", parse_spectre_v4_param);
410
411/*
412 * Because this was all written in a rush by people working in different silos,
413 * we've ended up with multiple command line options to control the same thing.
414 * Wrap these up in some helpers, which prefer disabling the mitigation if faced
415 * with contradictory parameters. The mitigation is always either "off",
416 * "dynamic" or "on".
417 */
418static bool spectre_v4_mitigations_off(void)
419{
420 return cpu_mitigations_off() ||
421 __spectre_v4_policy == SPECTRE_V4_POLICY_MITIGATION_DISABLED;
422}
423
424/* Do we need to toggle the mitigation state on entry to/exit from the kernel? */
425static bool spectre_v4_mitigations_dynamic(void)
426{
427 return !spectre_v4_mitigations_off() &&
428 __spectre_v4_policy == SPECTRE_V4_POLICY_MITIGATION_DYNAMIC;
429}
430
431static bool spectre_v4_mitigations_on(void)
432{
433 return !spectre_v4_mitigations_off() &&
434 __spectre_v4_policy == SPECTRE_V4_POLICY_MITIGATION_ENABLED;
435}
436
437ssize_t cpu_show_spec_store_bypass(struct device *dev,
438 struct device_attribute *attr, char *buf)
439{
440 switch (spectre_v4_state) {
441 case SPECTRE_UNAFFECTED:
442 return sysfs_emit(buf, "Not affected\n");
443 case SPECTRE_MITIGATED:
444 return sysfs_emit(buf, "Mitigation: Speculative Store Bypass disabled via prctl\n");
445 case SPECTRE_VULNERABLE:
446 fallthrough;
447 default:
448 return sysfs_emit(buf, "Vulnerable\n");
449 }
450}
451
452enum mitigation_state arm64_get_spectre_v4_state(void)
453{
454 return spectre_v4_state;
455}
456
457static enum mitigation_state spectre_v4_get_cpu_hw_mitigation_state(void)
458{
459 static const struct midr_range spectre_v4_safe_list[] = {
460 MIDR_ALL_VERSIONS(MIDR_CORTEX_A35),
461 MIDR_ALL_VERSIONS(MIDR_CORTEX_A53),
462 MIDR_ALL_VERSIONS(MIDR_CORTEX_A55),
463 MIDR_ALL_VERSIONS(MIDR_BRAHMA_B53),
464 MIDR_ALL_VERSIONS(MIDR_QCOM_KRYO_3XX_SILVER),
465 MIDR_ALL_VERSIONS(MIDR_QCOM_KRYO_4XX_SILVER),
466 { /* sentinel */ },
467 };
468
469 if (is_midr_in_range_list(spectre_v4_safe_list))
470 return SPECTRE_UNAFFECTED;
471
472 /* CPU features are detected first */
473 if (this_cpu_has_cap(ARM64_SSBS))
474 return SPECTRE_MITIGATED;
475
476 return SPECTRE_VULNERABLE;
477}
478
479static enum mitigation_state spectre_v4_get_cpu_fw_mitigation_state(void)
480{
481 int ret;
482 struct arm_smccc_res res;
483
484 arm_smccc_1_1_invoke(ARM_SMCCC_ARCH_FEATURES_FUNC_ID,
485 ARM_SMCCC_ARCH_WORKAROUND_2, &res);
486
487 ret = res.a0;
488 switch (ret) {
489 case SMCCC_RET_SUCCESS:
490 return SPECTRE_MITIGATED;
491 case SMCCC_ARCH_WORKAROUND_RET_UNAFFECTED:
492 fallthrough;
493 case SMCCC_RET_NOT_REQUIRED:
494 return SPECTRE_UNAFFECTED;
495 default:
496 fallthrough;
497 case SMCCC_RET_NOT_SUPPORTED:
498 return SPECTRE_VULNERABLE;
499 }
500}
501
502bool has_spectre_v4(const struct arm64_cpu_capabilities *cap, int scope)
503{
504 enum mitigation_state state;
505
506 WARN_ON(scope != SCOPE_LOCAL_CPU || preemptible());
507
508 state = spectre_v4_get_cpu_hw_mitigation_state();
509 if (state == SPECTRE_VULNERABLE)
510 state = spectre_v4_get_cpu_fw_mitigation_state();
511
512 return state != SPECTRE_UNAFFECTED;
513}
514
515bool try_emulate_el1_ssbs(struct pt_regs *regs, u32 instr)
516{
517 const u32 instr_mask = ~(1U << PSTATE_Imm_shift);
518 const u32 instr_val = 0xd500401f | PSTATE_SSBS;
519
520 if ((instr & instr_mask) != instr_val)
521 return false;
522
523 if (instr & BIT(PSTATE_Imm_shift))
524 regs->pstate |= PSR_SSBS_BIT;
525 else
526 regs->pstate &= ~PSR_SSBS_BIT;
527
528 arm64_skip_faulting_instruction(regs, 4);
529 return true;
530}
531
532static enum mitigation_state spectre_v4_enable_hw_mitigation(void)
533{
534 enum mitigation_state state;
535
536 /*
537 * If the system is mitigated but this CPU doesn't have SSBS, then
538 * we must be on the safelist and there's nothing more to do.
539 */
540 state = spectre_v4_get_cpu_hw_mitigation_state();
541 if (state != SPECTRE_MITIGATED || !this_cpu_has_cap(ARM64_SSBS))
542 return state;
543
544 if (spectre_v4_mitigations_off()) {
545 sysreg_clear_set(sctlr_el1, 0, SCTLR_ELx_DSSBS);
546 set_pstate_ssbs(1);
547 return SPECTRE_VULNERABLE;
548 }
549
550 /* SCTLR_EL1.DSSBS was initialised to 0 during boot */
551 set_pstate_ssbs(0);
552
553 /*
554 * SSBS is self-synchronizing and is intended to affect subsequent
555 * speculative instructions, but some CPUs can speculate with a stale
556 * value of SSBS.
557 *
558 * Mitigate this with an unconditional speculation barrier, as CPUs
559 * could mis-speculate branches and bypass a conditional barrier.
560 */
561 if (IS_ENABLED(CONFIG_ARM64_ERRATUM_3194386))
562 spec_bar();
563
564 return SPECTRE_MITIGATED;
565}
566
567/*
568 * Patch a branch over the Spectre-v4 mitigation code with a NOP so that
569 * we fallthrough and check whether firmware needs to be called on this CPU.
570 */
571void __init spectre_v4_patch_fw_mitigation_enable(struct alt_instr *alt,
572 __le32 *origptr,
573 __le32 *updptr, int nr_inst)
574{
575 BUG_ON(nr_inst != 1); /* Branch -> NOP */
576
577 if (spectre_v4_mitigations_off())
578 return;
579
580 if (cpus_have_cap(ARM64_SSBS))
581 return;
582
583 if (spectre_v4_mitigations_dynamic())
584 *updptr = cpu_to_le32(aarch64_insn_gen_nop());
585}
586
587/*
588 * Patch a NOP in the Spectre-v4 mitigation code with an SMC/HVC instruction
589 * to call into firmware to adjust the mitigation state.
590 */
591void __init smccc_patch_fw_mitigation_conduit(struct alt_instr *alt,
592 __le32 *origptr,
593 __le32 *updptr, int nr_inst)
594{
595 u32 insn;
596
597 BUG_ON(nr_inst != 1); /* NOP -> HVC/SMC */
598
599 switch (arm_smccc_1_1_get_conduit()) {
600 case SMCCC_CONDUIT_HVC:
601 insn = aarch64_insn_get_hvc_value();
602 break;
603 case SMCCC_CONDUIT_SMC:
604 insn = aarch64_insn_get_smc_value();
605 break;
606 default:
607 return;
608 }
609
610 *updptr = cpu_to_le32(insn);
611}
612
613static enum mitigation_state spectre_v4_enable_fw_mitigation(void)
614{
615 enum mitigation_state state;
616
617 state = spectre_v4_get_cpu_fw_mitigation_state();
618 if (state != SPECTRE_MITIGATED)
619 return state;
620
621 if (spectre_v4_mitigations_off()) {
622 arm_smccc_1_1_invoke(ARM_SMCCC_ARCH_WORKAROUND_2, false, NULL);
623 return SPECTRE_VULNERABLE;
624 }
625
626 arm_smccc_1_1_invoke(ARM_SMCCC_ARCH_WORKAROUND_2, true, NULL);
627
628 if (spectre_v4_mitigations_dynamic())
629 __this_cpu_write(arm64_ssbd_callback_required, 1);
630
631 return SPECTRE_MITIGATED;
632}
633
634void spectre_v4_enable_mitigation(const struct arm64_cpu_capabilities *__unused)
635{
636 enum mitigation_state state;
637
638 WARN_ON(preemptible());
639
640 state = spectre_v4_enable_hw_mitigation();
641 if (state == SPECTRE_VULNERABLE)
642 state = spectre_v4_enable_fw_mitigation();
643
644 update_mitigation_state(&spectre_v4_state, state);
645}
646
647static void __update_pstate_ssbs(struct pt_regs *regs, bool state)
648{
649 u64 bit = compat_user_mode(regs) ? PSR_AA32_SSBS_BIT : PSR_SSBS_BIT;
650
651 if (state)
652 regs->pstate |= bit;
653 else
654 regs->pstate &= ~bit;
655}
656
657void spectre_v4_enable_task_mitigation(struct task_struct *tsk)
658{
659 struct pt_regs *regs = task_pt_regs(tsk);
660 bool ssbs = false, kthread = tsk->flags & PF_KTHREAD;
661
662 if (spectre_v4_mitigations_off())
663 ssbs = true;
664 else if (spectre_v4_mitigations_dynamic() && !kthread)
665 ssbs = !test_tsk_thread_flag(tsk, TIF_SSBD);
666
667 __update_pstate_ssbs(regs, ssbs);
668}
669
670/*
671 * The Spectre-v4 mitigation can be controlled via a prctl() from userspace.
672 * This is interesting because the "speculation disabled" behaviour can be
673 * configured so that it is preserved across exec(), which means that the
674 * prctl() may be necessary even when PSTATE.SSBS can be toggled directly
675 * from userspace.
676 */
677static void ssbd_prctl_enable_mitigation(struct task_struct *task)
678{
679 task_clear_spec_ssb_noexec(task);
680 task_set_spec_ssb_disable(task);
681 set_tsk_thread_flag(task, TIF_SSBD);
682}
683
684static void ssbd_prctl_disable_mitigation(struct task_struct *task)
685{
686 task_clear_spec_ssb_noexec(task);
687 task_clear_spec_ssb_disable(task);
688 clear_tsk_thread_flag(task, TIF_SSBD);
689}
690
691static int ssbd_prctl_set(struct task_struct *task, unsigned long ctrl)
692{
693 switch (ctrl) {
694 case PR_SPEC_ENABLE:
695 /* Enable speculation: disable mitigation */
696 /*
697 * Force disabled speculation prevents it from being
698 * re-enabled.
699 */
700 if (task_spec_ssb_force_disable(task))
701 return -EPERM;
702
703 /*
704 * If the mitigation is forced on, then speculation is forced
705 * off and we again prevent it from being re-enabled.
706 */
707 if (spectre_v4_mitigations_on())
708 return -EPERM;
709
710 ssbd_prctl_disable_mitigation(task);
711 break;
712 case PR_SPEC_FORCE_DISABLE:
713 /* Force disable speculation: force enable mitigation */
714 /*
715 * If the mitigation is forced off, then speculation is forced
716 * on and we prevent it from being disabled.
717 */
718 if (spectre_v4_mitigations_off())
719 return -EPERM;
720
721 task_set_spec_ssb_force_disable(task);
722 fallthrough;
723 case PR_SPEC_DISABLE:
724 /* Disable speculation: enable mitigation */
725 /* Same as PR_SPEC_FORCE_DISABLE */
726 if (spectre_v4_mitigations_off())
727 return -EPERM;
728
729 ssbd_prctl_enable_mitigation(task);
730 break;
731 case PR_SPEC_DISABLE_NOEXEC:
732 /* Disable speculation until execve(): enable mitigation */
733 /*
734 * If the mitigation state is forced one way or the other, then
735 * we must fail now before we try to toggle it on execve().
736 */
737 if (task_spec_ssb_force_disable(task) ||
738 spectre_v4_mitigations_off() ||
739 spectre_v4_mitigations_on()) {
740 return -EPERM;
741 }
742
743 ssbd_prctl_enable_mitigation(task);
744 task_set_spec_ssb_noexec(task);
745 break;
746 default:
747 return -ERANGE;
748 }
749
750 spectre_v4_enable_task_mitigation(task);
751 return 0;
752}
753
754int arch_prctl_spec_ctrl_set(struct task_struct *task, unsigned long which,
755 unsigned long ctrl)
756{
757 switch (which) {
758 case PR_SPEC_STORE_BYPASS:
759 return ssbd_prctl_set(task, ctrl);
760 default:
761 return -ENODEV;
762 }
763}
764
765static int ssbd_prctl_get(struct task_struct *task)
766{
767 switch (spectre_v4_state) {
768 case SPECTRE_UNAFFECTED:
769 return PR_SPEC_NOT_AFFECTED;
770 case SPECTRE_MITIGATED:
771 if (spectre_v4_mitigations_on())
772 return PR_SPEC_NOT_AFFECTED;
773
774 if (spectre_v4_mitigations_dynamic())
775 break;
776
777 /* Mitigations are disabled, so we're vulnerable. */
778 fallthrough;
779 case SPECTRE_VULNERABLE:
780 fallthrough;
781 default:
782 return PR_SPEC_ENABLE;
783 }
784
785 /* Check the mitigation state for this task */
786 if (task_spec_ssb_force_disable(task))
787 return PR_SPEC_PRCTL | PR_SPEC_FORCE_DISABLE;
788
789 if (task_spec_ssb_noexec(task))
790 return PR_SPEC_PRCTL | PR_SPEC_DISABLE_NOEXEC;
791
792 if (task_spec_ssb_disable(task))
793 return PR_SPEC_PRCTL | PR_SPEC_DISABLE;
794
795 return PR_SPEC_PRCTL | PR_SPEC_ENABLE;
796}
797
798int arch_prctl_spec_ctrl_get(struct task_struct *task, unsigned long which)
799{
800 switch (which) {
801 case PR_SPEC_STORE_BYPASS:
802 return ssbd_prctl_get(task);
803 default:
804 return -ENODEV;
805 }
806}
807
808/*
809 * Spectre BHB.
810 *
811 * A CPU is either:
812 * - Mitigated by a branchy loop a CPU specific number of times, and listed
813 * in our "loop mitigated list".
814 * - Mitigated in software by the firmware Spectre v2 call.
815 * - Has the ClearBHB instruction to perform the mitigation.
816 * - Has the 'Exception Clears Branch History Buffer' (ECBHB) feature, so no
817 * software mitigation in the vectors is needed.
818 * - Has CSV2.3, so is unaffected.
819 */
820static enum mitigation_state spectre_bhb_state;
821
822enum mitigation_state arm64_get_spectre_bhb_state(void)
823{
824 return spectre_bhb_state;
825}
826
827enum bhb_mitigation_bits {
828 BHB_LOOP,
829 BHB_FW,
830 BHB_HW,
831 BHB_INSN,
832};
833static unsigned long system_bhb_mitigations;
834
835/*
836 * This must be called with SCOPE_LOCAL_CPU for each type of CPU, before any
837 * SCOPE_SYSTEM call will give the right answer.
838 */
839static bool is_spectre_bhb_safe(int scope)
840{
841 static const struct midr_range spectre_bhb_safe_list[] = {
842 MIDR_ALL_VERSIONS(MIDR_CORTEX_A35),
843 MIDR_ALL_VERSIONS(MIDR_CORTEX_A53),
844 MIDR_ALL_VERSIONS(MIDR_CORTEX_A55),
845 MIDR_ALL_VERSIONS(MIDR_CORTEX_A510),
846 MIDR_ALL_VERSIONS(MIDR_CORTEX_A520),
847 MIDR_ALL_VERSIONS(MIDR_BRAHMA_B53),
848 MIDR_ALL_VERSIONS(MIDR_QCOM_KRYO_2XX_SILVER),
849 MIDR_ALL_VERSIONS(MIDR_QCOM_KRYO_3XX_SILVER),
850 MIDR_ALL_VERSIONS(MIDR_QCOM_KRYO_4XX_SILVER),
851 {},
852 };
853 static bool all_safe = true;
854
855 if (scope != SCOPE_LOCAL_CPU)
856 return all_safe;
857
858 if (is_midr_in_range_list(spectre_bhb_safe_list))
859 return true;
860
861 all_safe = false;
862
863 return false;
864}
865
866static u8 spectre_bhb_loop_affected(void)
867{
868 u8 k = 0;
869
870 static const struct midr_range spectre_bhb_k132_list[] = {
871 MIDR_ALL_VERSIONS(MIDR_CORTEX_X3),
872 MIDR_ALL_VERSIONS(MIDR_NEOVERSE_V2),
873 {},
874 };
875 static const struct midr_range spectre_bhb_k38_list[] = {
876 MIDR_ALL_VERSIONS(MIDR_CORTEX_A715),
877 MIDR_ALL_VERSIONS(MIDR_CORTEX_A720),
878 MIDR_ALL_VERSIONS(MIDR_CORTEX_A720AE),
879 {},
880 };
881 static const struct midr_range spectre_bhb_k32_list[] = {
882 MIDR_ALL_VERSIONS(MIDR_CORTEX_A78),
883 MIDR_ALL_VERSIONS(MIDR_CORTEX_A78AE),
884 MIDR_ALL_VERSIONS(MIDR_CORTEX_A78C),
885 MIDR_ALL_VERSIONS(MIDR_CORTEX_X1),
886 MIDR_ALL_VERSIONS(MIDR_CORTEX_X1C),
887 MIDR_ALL_VERSIONS(MIDR_CORTEX_A710),
888 MIDR_ALL_VERSIONS(MIDR_CORTEX_X2),
889 MIDR_ALL_VERSIONS(MIDR_NEOVERSE_N2),
890 MIDR_ALL_VERSIONS(MIDR_NEOVERSE_V1),
891 MIDR_ALL_VERSIONS(MIDR_HISI_TSV110),
892 {},
893 };
894 static const struct midr_range spectre_bhb_k24_list[] = {
895 MIDR_ALL_VERSIONS(MIDR_CORTEX_A76),
896 MIDR_ALL_VERSIONS(MIDR_CORTEX_A76AE),
897 MIDR_ALL_VERSIONS(MIDR_CORTEX_A77),
898 MIDR_ALL_VERSIONS(MIDR_NEOVERSE_N1),
899 MIDR_ALL_VERSIONS(MIDR_QCOM_KRYO_4XX_GOLD),
900 MIDR_ALL_VERSIONS(MIDR_HISI_HIP09),
901 {},
902 };
903 static const struct midr_range spectre_bhb_k11_list[] = {
904 MIDR_ALL_VERSIONS(MIDR_AMPERE1),
905 {},
906 };
907 static const struct midr_range spectre_bhb_k8_list[] = {
908 MIDR_ALL_VERSIONS(MIDR_CORTEX_A72),
909 MIDR_ALL_VERSIONS(MIDR_CORTEX_A57),
910 {},
911 };
912
913 if (is_midr_in_range_list(spectre_bhb_k132_list))
914 k = 132;
915 else if (is_midr_in_range_list(spectre_bhb_k38_list))
916 k = 38;
917 else if (is_midr_in_range_list(spectre_bhb_k32_list))
918 k = 32;
919 else if (is_midr_in_range_list(spectre_bhb_k24_list))
920 k = 24;
921 else if (is_midr_in_range_list(spectre_bhb_k11_list))
922 k = 11;
923 else if (is_midr_in_range_list(spectre_bhb_k8_list))
924 k = 8;
925
926 return k;
927}
928
929static enum mitigation_state spectre_bhb_get_cpu_fw_mitigation_state(void)
930{
931 int ret;
932 struct arm_smccc_res res;
933
934 arm_smccc_1_1_invoke(ARM_SMCCC_ARCH_FEATURES_FUNC_ID,
935 ARM_SMCCC_ARCH_WORKAROUND_3, &res);
936
937 ret = res.a0;
938 switch (ret) {
939 case SMCCC_RET_SUCCESS:
940 return SPECTRE_MITIGATED;
941 case SMCCC_ARCH_WORKAROUND_RET_UNAFFECTED:
942 return SPECTRE_UNAFFECTED;
943 default:
944 fallthrough;
945 case SMCCC_RET_NOT_SUPPORTED:
946 return SPECTRE_VULNERABLE;
947 }
948}
949
950static bool has_spectre_bhb_fw_mitigation(void)
951{
952 enum mitigation_state fw_state;
953 bool has_smccc = arm_smccc_1_1_get_conduit() != SMCCC_CONDUIT_NONE;
954
955 fw_state = spectre_bhb_get_cpu_fw_mitigation_state();
956 return has_smccc && fw_state == SPECTRE_MITIGATED;
957}
958
959static bool supports_ecbhb(int scope)
960{
961 u64 mmfr1;
962
963 if (scope == SCOPE_LOCAL_CPU)
964 mmfr1 = read_sysreg_s(SYS_ID_AA64MMFR1_EL1);
965 else
966 mmfr1 = read_sanitised_ftr_reg(SYS_ID_AA64MMFR1_EL1);
967
968 return cpuid_feature_extract_unsigned_field(mmfr1,
969 ID_AA64MMFR1_EL1_ECBHB_SHIFT);
970}
971
972static u8 max_bhb_k;
973
974bool is_spectre_bhb_affected(const struct arm64_cpu_capabilities *entry,
975 int scope)
976{
977 WARN_ON(scope != SCOPE_LOCAL_CPU || preemptible());
978
979 if (supports_csv2p3(scope))
980 return false;
981
982 if (is_spectre_bhb_safe(scope))
983 return false;
984
985 /*
986 * At this point the core isn't known to be "safe" so we're going to
987 * assume it's vulnerable. We still need to update `max_bhb_k` though,
988 * but only if we aren't mitigating with clearbhb though.
989 */
990 if (scope == SCOPE_LOCAL_CPU && !supports_clearbhb(SCOPE_LOCAL_CPU))
991 max_bhb_k = max(max_bhb_k, spectre_bhb_loop_affected());
992
993 return true;
994}
995
996u8 get_spectre_bhb_loop_value(void)
997{
998 return max_bhb_k;
999}
1000
1001static void this_cpu_set_vectors(enum arm64_bp_harden_el1_vectors slot)
1002{
1003 const char *v = arm64_get_bp_hardening_vector(slot);
1004
1005 __this_cpu_write(this_cpu_vector, v);
1006
1007 /*
1008 * When KPTI is in use, the vectors are switched when exiting to
1009 * user-space.
1010 */
1011 if (cpus_have_cap(ARM64_UNMAP_KERNEL_AT_EL0))
1012 return;
1013
1014 write_sysreg(v, vbar_el1);
1015 isb();
1016}
1017
1018bool __read_mostly __nospectre_bhb;
1019static int __init parse_spectre_bhb_param(char *str)
1020{
1021 __nospectre_bhb = true;
1022 return 0;
1023}
1024early_param("nospectre_bhb", parse_spectre_bhb_param);
1025
1026static bool spectre_bhb_mitigations_off(void)
1027{
1028 return __nospectre_bhb || cpu_mitigations_off();
1029}
1030
1031void spectre_bhb_enable_mitigation(const struct arm64_cpu_capabilities *entry)
1032{
1033 bp_hardening_cb_t cpu_cb;
1034 enum mitigation_state state = SPECTRE_VULNERABLE;
1035 struct bp_hardening_data *data = this_cpu_ptr(&bp_hardening_data);
1036
1037 if (!is_spectre_bhb_affected(entry, SCOPE_LOCAL_CPU))
1038 return;
1039
1040 if (arm64_get_spectre_v2_state() == SPECTRE_VULNERABLE) {
1041 /* No point mitigating Spectre-BHB alone. */
1042 } else if (!IS_ENABLED(CONFIG_MITIGATE_SPECTRE_BRANCH_HISTORY)) {
1043 /* Do nothing */
1044 } else if (spectre_bhb_mitigations_off()) {
1045 /* Mitigation disabled on the command line */
1046 } else if (supports_ecbhb(SCOPE_LOCAL_CPU)) {
1047 state = SPECTRE_MITIGATED;
1048 set_bit(BHB_HW, &system_bhb_mitigations);
1049 } else if (supports_clearbhb(SCOPE_LOCAL_CPU)) {
1050 /*
1051 * Ensure KVM uses the indirect vector which will have ClearBHB
1052 * added.
1053 */
1054 if (!data->slot)
1055 data->slot = HYP_VECTOR_INDIRECT;
1056
1057 this_cpu_set_vectors(EL1_VECTOR_BHB_CLEAR_INSN);
1058 state = SPECTRE_MITIGATED;
1059 set_bit(BHB_INSN, &system_bhb_mitigations);
1060 } else if (spectre_bhb_loop_affected()) {
1061 /*
1062 * Ensure KVM uses the indirect vector which will have the
1063 * branchy-loop added. A57/A72-r0 will already have selected
1064 * the spectre-indirect vector, which is sufficient for BHB
1065 * too.
1066 */
1067 if (!data->slot)
1068 data->slot = HYP_VECTOR_INDIRECT;
1069
1070 this_cpu_set_vectors(EL1_VECTOR_BHB_LOOP);
1071 state = SPECTRE_MITIGATED;
1072 set_bit(BHB_LOOP, &system_bhb_mitigations);
1073 } else if (has_spectre_bhb_fw_mitigation()) {
1074 /*
1075 * Ensure KVM uses one of the spectre bp_hardening
1076 * vectors. The indirect vector doesn't include the EL3
1077 * call, so needs upgrading to
1078 * HYP_VECTOR_SPECTRE_INDIRECT.
1079 */
1080 if (!data->slot || data->slot == HYP_VECTOR_INDIRECT)
1081 data->slot += 1;
1082
1083 this_cpu_set_vectors(EL1_VECTOR_BHB_FW);
1084
1085 /*
1086 * The WA3 call in the vectors supersedes the WA1 call
1087 * made during context-switch. Uninstall any firmware
1088 * bp_hardening callback.
1089 */
1090 cpu_cb = spectre_v2_get_sw_mitigation_cb();
1091 if (__this_cpu_read(bp_hardening_data.fn) != cpu_cb)
1092 __this_cpu_write(bp_hardening_data.fn, NULL);
1093
1094 state = SPECTRE_MITIGATED;
1095 set_bit(BHB_FW, &system_bhb_mitigations);
1096 }
1097
1098 update_mitigation_state(&spectre_bhb_state, state);
1099}
1100
1101bool is_spectre_bhb_fw_mitigated(void)
1102{
1103 return test_bit(BHB_FW, &system_bhb_mitigations);
1104}
1105
1106/* Patched to NOP when enabled */
1107void noinstr spectre_bhb_patch_loop_mitigation_enable(struct alt_instr *alt,
1108 __le32 *origptr,
1109 __le32 *updptr, int nr_inst)
1110{
1111 BUG_ON(nr_inst != 1);
1112
1113 if (test_bit(BHB_LOOP, &system_bhb_mitigations))
1114 *updptr++ = cpu_to_le32(aarch64_insn_gen_nop());
1115}
1116
1117/* Patched to NOP when enabled */
1118void noinstr spectre_bhb_patch_fw_mitigation_enabled(struct alt_instr *alt,
1119 __le32 *origptr,
1120 __le32 *updptr, int nr_inst)
1121{
1122 BUG_ON(nr_inst != 1);
1123
1124 if (test_bit(BHB_FW, &system_bhb_mitigations))
1125 *updptr++ = cpu_to_le32(aarch64_insn_gen_nop());
1126}
1127
1128/* Patched to correct the immediate */
1129void noinstr spectre_bhb_patch_loop_iter(struct alt_instr *alt,
1130 __le32 *origptr, __le32 *updptr, int nr_inst)
1131{
1132 u8 rd;
1133 u32 insn;
1134
1135 BUG_ON(nr_inst != 1); /* MOV -> MOV */
1136
1137 if (!IS_ENABLED(CONFIG_MITIGATE_SPECTRE_BRANCH_HISTORY))
1138 return;
1139
1140 insn = le32_to_cpu(*origptr);
1141 rd = aarch64_insn_decode_register(AARCH64_INSN_REGTYPE_RD, insn);
1142 insn = aarch64_insn_gen_movewide(rd, max_bhb_k, 0,
1143 AARCH64_INSN_VARIANT_64BIT,
1144 AARCH64_INSN_MOVEWIDE_ZERO);
1145 *updptr++ = cpu_to_le32(insn);
1146}
1147
1148/* Patched to mov WA3 when supported */
1149void noinstr spectre_bhb_patch_wa3(struct alt_instr *alt,
1150 __le32 *origptr, __le32 *updptr, int nr_inst)
1151{
1152 u8 rd;
1153 u32 insn;
1154
1155 BUG_ON(nr_inst != 1); /* MOV -> MOV */
1156
1157 if (!IS_ENABLED(CONFIG_MITIGATE_SPECTRE_BRANCH_HISTORY) ||
1158 !test_bit(BHB_FW, &system_bhb_mitigations))
1159 return;
1160
1161 insn = le32_to_cpu(*origptr);
1162 rd = aarch64_insn_decode_register(AARCH64_INSN_REGTYPE_RD, insn);
1163
1164 insn = aarch64_insn_gen_logical_immediate(AARCH64_INSN_LOGIC_ORR,
1165 AARCH64_INSN_VARIANT_32BIT,
1166 AARCH64_INSN_REG_ZR, rd,
1167 ARM_SMCCC_ARCH_WORKAROUND_3);
1168 if (WARN_ON_ONCE(insn == AARCH64_BREAK_FAULT))
1169 return;
1170
1171 *updptr++ = cpu_to_le32(insn);
1172}
1173
1174/* Patched to NOP when not supported */
1175void __init spectre_bhb_patch_clearbhb(struct alt_instr *alt,
1176 __le32 *origptr, __le32 *updptr, int nr_inst)
1177{
1178 BUG_ON(nr_inst != 2);
1179
1180 if (test_bit(BHB_INSN, &system_bhb_mitigations))
1181 return;
1182
1183 *updptr++ = cpu_to_le32(aarch64_insn_gen_nop());
1184 *updptr++ = cpu_to_le32(aarch64_insn_gen_nop());
1185}
1186
1187#ifdef CONFIG_BPF_SYSCALL
1188#define EBPF_WARN "Unprivileged eBPF is enabled, data leaks possible via Spectre v2 BHB attacks!\n"
1189void unpriv_ebpf_notify(int new_state)
1190{
1191 if (spectre_v2_state == SPECTRE_VULNERABLE ||
1192 spectre_bhb_state != SPECTRE_MITIGATED)
1193 return;
1194
1195 if (!new_state)
1196 pr_err("WARNING: %s", EBPF_WARN);
1197}
1198#endif
1199
1200void spectre_print_disabled_mitigations(void)
1201{
1202 /* Keep a single copy of the common message suffix to avoid duplication. */
1203 const char *spectre_disabled_suffix = "mitigation disabled by command-line option\n";
1204
1205 if (spectre_v2_mitigations_off())
1206 pr_info("spectre-v2 %s", spectre_disabled_suffix);
1207
1208 if (spectre_v4_mitigations_off())
1209 pr_info("spectre-v4 %s", spectre_disabled_suffix);
1210
1211 if (spectre_bhb_mitigations_off())
1212 pr_info("spectre-bhb %s", spectre_disabled_suffix);
1213}