projects/linux·blob master · refs

867 lines · 21.7 KiB

1// SPDX-License-Identifier: GPL-2.0+
2//
3// Security related flags and so on.
4//
5// Copyright 2018, Michael Ellerman, IBM Corporation.
6
7#include <linux/cpu.h>
8#include <linux/kernel.h>
9#include <linux/device.h>
10#include <linux/memblock.h>
11#include <linux/nospec.h>
12#include <linux/prctl.h>
13#include <linux/seq_buf.h>
14#include <linux/sysfs.h>
15#include <linux/debugfs.h>
16
17#include <asm/asm-prototypes.h>
18#include <asm/text-patching.h>
19#include <asm/security_features.h>
20#include <asm/sections.h>
21#include <asm/setup.h>
22#include <asm/inst.h>
23
24#include "setup.h"
25
26u64 powerpc_security_features __read_mostly = SEC_FTR_DEFAULT;
27
28enum branch_cache_flush_type {
29 BRANCH_CACHE_FLUSH_NONE = 0x1,
30 BRANCH_CACHE_FLUSH_SW = 0x2,
31 BRANCH_CACHE_FLUSH_HW = 0x4,
32};
33static enum branch_cache_flush_type count_cache_flush_type = BRANCH_CACHE_FLUSH_NONE;
34static enum branch_cache_flush_type link_stack_flush_type = BRANCH_CACHE_FLUSH_NONE;
35
36bool barrier_nospec_enabled;
37static bool no_nospec;
38static bool btb_flush_enabled;
39#if defined(CONFIG_PPC_E500) || defined(CONFIG_PPC_BOOK3S_64)
40static bool no_spectrev2;
41#endif
42
43static void enable_barrier_nospec(bool enable)
44{
45 barrier_nospec_enabled = enable;
46 do_barrier_nospec_fixups(enable);
47}
48
49void __init setup_barrier_nospec(void)
50{
51 bool enable;
52
53 /*
54 * It would make sense to check SEC_FTR_SPEC_BAR_ORI31 below as well.
55 * But there's a good reason not to. The two flags we check below are
56 * both are enabled by default in the kernel, so if the hcall is not
57 * functional they will be enabled.
58 * On a system where the host firmware has been updated (so the ori
59 * functions as a barrier), but on which the hypervisor (KVM/Qemu) has
60 * not been updated, we would like to enable the barrier. Dropping the
61 * check for SEC_FTR_SPEC_BAR_ORI31 achieves that. The only downside is
62 * we potentially enable the barrier on systems where the host firmware
63 * is not updated, but that's harmless as it's a no-op.
64 */
65 enable = security_ftr_enabled(SEC_FTR_FAVOUR_SECURITY) &&
66 security_ftr_enabled(SEC_FTR_BNDS_CHK_SPEC_BAR);
67
68 if (!no_nospec && !cpu_mitigations_off())
69 enable_barrier_nospec(enable);
70}
71
72static int __init handle_nospectre_v1(char *p)
73{
74 no_nospec = true;
75
76 return 0;
77}
78early_param("nospectre_v1", handle_nospectre_v1);
79
80#ifdef CONFIG_DEBUG_FS
81static int barrier_nospec_set(void *data, u64 val)
82{
83 switch (val) {
84 case 0:
85 case 1:
86 break;
87 default:
88 return -EINVAL;
89 }
90
91 if (!!val == !!barrier_nospec_enabled)
92 return 0;
93
94 enable_barrier_nospec(!!val);
95
96 return 0;
97}
98
99static int barrier_nospec_get(void *data, u64 *val)
100{
101 *val = barrier_nospec_enabled ? 1 : 0;
102 return 0;
103}
104
105DEFINE_DEBUGFS_ATTRIBUTE(fops_barrier_nospec, barrier_nospec_get,
106 barrier_nospec_set, "%llu\n");
107
108static __init int barrier_nospec_debugfs_init(void)
109{
110 debugfs_create_file_unsafe("barrier_nospec", 0600,
111 arch_debugfs_dir, NULL,
112 &fops_barrier_nospec);
113 return 0;
114}
115device_initcall(barrier_nospec_debugfs_init);
116
117static __init int security_feature_debugfs_init(void)
118{
119 debugfs_create_x64("security_features", 0400, arch_debugfs_dir,
120 &powerpc_security_features);
121 return 0;
122}
123device_initcall(security_feature_debugfs_init);
124#endif /* CONFIG_DEBUG_FS */
125
126#if defined(CONFIG_PPC_E500) || defined(CONFIG_PPC_BOOK3S_64)
127static int __init handle_nospectre_v2(char *p)
128{
129 no_spectrev2 = true;
130
131 return 0;
132}
133early_param("nospectre_v2", handle_nospectre_v2);
134#endif /* CONFIG_PPC_E500 || CONFIG_PPC_BOOK3S_64 */
135
136#ifdef CONFIG_PPC_E500
137void __init setup_spectre_v2(void)
138{
139 if (no_spectrev2 || cpu_mitigations_off())
140 do_btb_flush_fixups();
141 else
142 btb_flush_enabled = true;
143}
144#endif /* CONFIG_PPC_E500 */
145
146#ifdef CONFIG_PPC_BOOK3S_64
147ssize_t cpu_show_meltdown(struct device *dev, struct device_attribute *attr, char *buf)
148{
149 bool thread_priv;
150
151 thread_priv = security_ftr_enabled(SEC_FTR_L1D_THREAD_PRIV);
152
153 if (rfi_flush) {
154 struct seq_buf s;
155 seq_buf_init(&s, buf, PAGE_SIZE - 1);
156
157 seq_buf_printf(&s, "Mitigation: RFI Flush");
158 if (thread_priv)
159 seq_buf_printf(&s, ", L1D private per thread");
160
161 seq_buf_printf(&s, "\n");
162
163 return s.len;
164 }
165
166 if (thread_priv)
167 return sysfs_emit(buf, "Vulnerable: L1D private per thread\n");
168
169 if (!security_ftr_enabled(SEC_FTR_L1D_FLUSH_HV) &&
170 !security_ftr_enabled(SEC_FTR_L1D_FLUSH_PR))
171 return sysfs_emit(buf, "Not affected\n");
172
173 return sysfs_emit(buf, "Vulnerable\n");
174}
175
176ssize_t cpu_show_l1tf(struct device *dev, struct device_attribute *attr, char *buf)
177{
178 return cpu_show_meltdown(dev, attr, buf);
179}
180#endif
181
182ssize_t cpu_show_spectre_v1(struct device *dev, struct device_attribute *attr, char *buf)
183{
184 struct seq_buf s;
185
186 seq_buf_init(&s, buf, PAGE_SIZE - 1);
187
188 if (security_ftr_enabled(SEC_FTR_BNDS_CHK_SPEC_BAR)) {
189 if (barrier_nospec_enabled)
190 seq_buf_printf(&s, "Mitigation: __user pointer sanitization");
191 else
192 seq_buf_printf(&s, "Vulnerable");
193
194 if (security_ftr_enabled(SEC_FTR_SPEC_BAR_ORI31))
195 seq_buf_printf(&s, ", ori31 speculation barrier enabled");
196
197 seq_buf_printf(&s, "\n");
198 } else
199 seq_buf_printf(&s, "Not affected\n");
200
201 return s.len;
202}
203
204ssize_t cpu_show_spectre_v2(struct device *dev, struct device_attribute *attr, char *buf)
205{
206 struct seq_buf s;
207 bool bcs, ccd;
208
209 seq_buf_init(&s, buf, PAGE_SIZE - 1);
210
211 bcs = security_ftr_enabled(SEC_FTR_BCCTRL_SERIALISED);
212 ccd = security_ftr_enabled(SEC_FTR_COUNT_CACHE_DISABLED);
213
214 if (bcs || ccd) {
215 seq_buf_printf(&s, "Mitigation: ");
216
217 if (bcs)
218 seq_buf_printf(&s, "Indirect branch serialisation (kernel only)");
219
220 if (bcs && ccd)
221 seq_buf_printf(&s, ", ");
222
223 if (ccd)
224 seq_buf_printf(&s, "Indirect branch cache disabled");
225
226 } else if (count_cache_flush_type != BRANCH_CACHE_FLUSH_NONE) {
227 seq_buf_printf(&s, "Mitigation: Software count cache flush");
228
229 if (count_cache_flush_type == BRANCH_CACHE_FLUSH_HW)
230 seq_buf_printf(&s, " (hardware accelerated)");
231
232 } else if (btb_flush_enabled) {
233 seq_buf_printf(&s, "Mitigation: Branch predictor state flush");
234 } else {
235 seq_buf_printf(&s, "Vulnerable");
236 }
237
238 if (bcs || ccd || count_cache_flush_type != BRANCH_CACHE_FLUSH_NONE) {
239 if (link_stack_flush_type != BRANCH_CACHE_FLUSH_NONE)
240 seq_buf_printf(&s, ", Software link stack flush");
241 if (link_stack_flush_type == BRANCH_CACHE_FLUSH_HW)
242 seq_buf_printf(&s, " (hardware accelerated)");
243 }
244
245 seq_buf_printf(&s, "\n");
246
247 return s.len;
248}
249
250#ifdef CONFIG_PPC_BOOK3S_64
251/*
252 * Store-forwarding barrier support.
253 */
254
255static enum stf_barrier_type stf_enabled_flush_types;
256static bool no_stf_barrier;
257static bool stf_barrier;
258
259static int __init handle_no_stf_barrier(char *p)
260{
261 pr_info("stf-barrier: disabled on command line.");
262 no_stf_barrier = true;
263 return 0;
264}
265
266early_param("no_stf_barrier", handle_no_stf_barrier);
267
268enum stf_barrier_type stf_barrier_type_get(void)
269{
270 return stf_enabled_flush_types;
271}
272
273/* This is the generic flag used by other architectures */
274static int __init handle_ssbd(char *p)
275{
276 if (!p || strncmp(p, "auto", 5) == 0 || strncmp(p, "on", 2) == 0 ) {
277 /* Until firmware tells us, we have the barrier with auto */
278 return 0;
279 } else if (strncmp(p, "off", 3) == 0) {
280 handle_no_stf_barrier(NULL);
281 return 0;
282 } else
283 return 1;
284
285 return 0;
286}
287early_param("spec_store_bypass_disable", handle_ssbd);
288
289/* This is the generic flag used by other architectures */
290static int __init handle_no_ssbd(char *p)
291{
292 handle_no_stf_barrier(NULL);
293 return 0;
294}
295early_param("nospec_store_bypass_disable", handle_no_ssbd);
296
297static void stf_barrier_enable(bool enable)
298{
299 if (enable)
300 do_stf_barrier_fixups(stf_enabled_flush_types);
301 else
302 do_stf_barrier_fixups(STF_BARRIER_NONE);
303
304 stf_barrier = enable;
305}
306
307void setup_stf_barrier(void)
308{
309 enum stf_barrier_type type;
310 bool enable;
311
312 /* Default to fallback in case fw-features are not available */
313 if (cpu_has_feature(CPU_FTR_ARCH_300))
314 type = STF_BARRIER_EIEIO;
315 else if (cpu_has_feature(CPU_FTR_ARCH_207S))
316 type = STF_BARRIER_SYNC_ORI;
317 else if (cpu_has_feature(CPU_FTR_ARCH_206))
318 type = STF_BARRIER_FALLBACK;
319 else
320 type = STF_BARRIER_NONE;
321
322 enable = security_ftr_enabled(SEC_FTR_FAVOUR_SECURITY) &&
323 security_ftr_enabled(SEC_FTR_STF_BARRIER);
324
325 if (type == STF_BARRIER_FALLBACK) {
326 pr_info("stf-barrier: fallback barrier available\n");
327 } else if (type == STF_BARRIER_SYNC_ORI) {
328 pr_info("stf-barrier: hwsync barrier available\n");
329 } else if (type == STF_BARRIER_EIEIO) {
330 pr_info("stf-barrier: eieio barrier available\n");
331 }
332
333 stf_enabled_flush_types = type;
334
335 if (!no_stf_barrier && !cpu_mitigations_off())
336 stf_barrier_enable(enable);
337}
338
339ssize_t cpu_show_spec_store_bypass(struct device *dev, struct device_attribute *attr, char *buf)
340{
341 if (stf_barrier && stf_enabled_flush_types != STF_BARRIER_NONE) {
342 const char *type;
343 switch (stf_enabled_flush_types) {
344 case STF_BARRIER_EIEIO:
345 type = "eieio";
346 break;
347 case STF_BARRIER_SYNC_ORI:
348 type = "hwsync";
349 break;
350 case STF_BARRIER_FALLBACK:
351 type = "fallback";
352 break;
353 default:
354 type = "unknown";
355 }
356 return sysfs_emit(buf, "Mitigation: Kernel entry/exit barrier (%s)\n", type);
357 }
358
359 if (!security_ftr_enabled(SEC_FTR_L1D_FLUSH_HV) &&
360 !security_ftr_enabled(SEC_FTR_L1D_FLUSH_PR))
361 return sysfs_emit(buf, "Not affected\n");
362
363 return sysfs_emit(buf, "Vulnerable\n");
364}
365
366static int ssb_prctl_get(struct task_struct *task)
367{
368 /*
369 * The STF_BARRIER feature is on by default, so if it's off that means
370 * firmware has explicitly said the CPU is not vulnerable via either
371 * the hypercall or device tree.
372 */
373 if (!security_ftr_enabled(SEC_FTR_STF_BARRIER))
374 return PR_SPEC_NOT_AFFECTED;
375
376 /*
377 * If the system's CPU has no known barrier (see setup_stf_barrier())
378 * then assume that the CPU is not vulnerable.
379 */
380 if (stf_enabled_flush_types == STF_BARRIER_NONE)
381 return PR_SPEC_NOT_AFFECTED;
382
383 /*
384 * Otherwise the CPU is vulnerable. The barrier is not a global or
385 * per-process mitigation, so the only value that can be reported here
386 * is PR_SPEC_ENABLE, which appears as "vulnerable" in /proc.
387 */
388 return PR_SPEC_ENABLE;
389}
390
391int arch_prctl_spec_ctrl_get(struct task_struct *task, unsigned long which)
392{
393 switch (which) {
394 case PR_SPEC_STORE_BYPASS:
395 return ssb_prctl_get(task);
396 default:
397 return -ENODEV;
398 }
399}
400
401#ifdef CONFIG_DEBUG_FS
402static int stf_barrier_set(void *data, u64 val)
403{
404 bool enable;
405
406 if (val == 1)
407 enable = true;
408 else if (val == 0)
409 enable = false;
410 else
411 return -EINVAL;
412
413 /* Only do anything if we're changing state */
414 if (enable != stf_barrier)
415 stf_barrier_enable(enable);
416
417 return 0;
418}
419
420static int stf_barrier_get(void *data, u64 *val)
421{
422 *val = stf_barrier ? 1 : 0;
423 return 0;
424}
425
426DEFINE_DEBUGFS_ATTRIBUTE(fops_stf_barrier, stf_barrier_get, stf_barrier_set,
427 "%llu\n");
428
429static __init int stf_barrier_debugfs_init(void)
430{
431 debugfs_create_file_unsafe("stf_barrier", 0600, arch_debugfs_dir,
432 NULL, &fops_stf_barrier);
433 return 0;
434}
435device_initcall(stf_barrier_debugfs_init);
436#endif /* CONFIG_DEBUG_FS */
437
438static void update_branch_cache_flush(void)
439{
440 u32 *site, __maybe_unused *site2;
441
442#ifdef CONFIG_KVM_BOOK3S_HV_POSSIBLE
443 site = &patch__call_kvm_flush_link_stack;
444 site2 = &patch__call_kvm_flush_link_stack_p9;
445 // This controls the branch from guest_exit_cont to kvm_flush_link_stack
446 if (link_stack_flush_type == BRANCH_CACHE_FLUSH_NONE) {
447 patch_instruction_site(site, ppc_inst(PPC_RAW_NOP()));
448 patch_instruction_site(site2, ppc_inst(PPC_RAW_NOP()));
449 } else {
450 // Could use HW flush, but that could also flush count cache
451 patch_branch_site(site, (u64)&kvm_flush_link_stack, BRANCH_SET_LINK);
452 patch_branch_site(site2, (u64)&kvm_flush_link_stack, BRANCH_SET_LINK);
453 }
454#endif
455
456 // Patch out the bcctr first, then nop the rest
457 site = &patch__call_flush_branch_caches3;
458 patch_instruction_site(site, ppc_inst(PPC_RAW_NOP()));
459 site = &patch__call_flush_branch_caches2;
460 patch_instruction_site(site, ppc_inst(PPC_RAW_NOP()));
461 site = &patch__call_flush_branch_caches1;
462 patch_instruction_site(site, ppc_inst(PPC_RAW_NOP()));
463
464 // This controls the branch from _switch to flush_branch_caches
465 if (count_cache_flush_type == BRANCH_CACHE_FLUSH_NONE &&
466 link_stack_flush_type == BRANCH_CACHE_FLUSH_NONE) {
467 // Nothing to be done
468
469 } else if (count_cache_flush_type == BRANCH_CACHE_FLUSH_HW &&
470 link_stack_flush_type == BRANCH_CACHE_FLUSH_HW) {
471 // Patch in the bcctr last
472 site = &patch__call_flush_branch_caches1;
473 patch_instruction_site(site, ppc_inst(0x39207fff)); // li r9,0x7fff
474 site = &patch__call_flush_branch_caches2;
475 patch_instruction_site(site, ppc_inst(0x7d2903a6)); // mtctr r9
476 site = &patch__call_flush_branch_caches3;
477 patch_instruction_site(site, ppc_inst(PPC_INST_BCCTR_FLUSH));
478
479 } else {
480 patch_branch_site(site, (u64)&flush_branch_caches, BRANCH_SET_LINK);
481
482 // If we just need to flush the link stack, early return
483 if (count_cache_flush_type == BRANCH_CACHE_FLUSH_NONE) {
484 patch_instruction_site(&patch__flush_link_stack_return,
485 ppc_inst(PPC_RAW_BLR()));
486
487 // If we have flush instruction, early return
488 } else if (count_cache_flush_type == BRANCH_CACHE_FLUSH_HW) {
489 patch_instruction_site(&patch__flush_count_cache_return,
490 ppc_inst(PPC_RAW_BLR()));
491 }
492 }
493}
494
495static void toggle_branch_cache_flush(bool enable)
496{
497 if (!enable || !security_ftr_enabled(SEC_FTR_FLUSH_COUNT_CACHE)) {
498 if (count_cache_flush_type != BRANCH_CACHE_FLUSH_NONE)
499 count_cache_flush_type = BRANCH_CACHE_FLUSH_NONE;
500
501 pr_info("count-cache-flush: flush disabled.\n");
502 } else {
503 if (security_ftr_enabled(SEC_FTR_BCCTR_FLUSH_ASSIST)) {
504 count_cache_flush_type = BRANCH_CACHE_FLUSH_HW;
505 pr_info("count-cache-flush: hardware flush enabled.\n");
506 } else {
507 count_cache_flush_type = BRANCH_CACHE_FLUSH_SW;
508 pr_info("count-cache-flush: software flush enabled.\n");
509 }
510 }
511
512 if (!enable || !security_ftr_enabled(SEC_FTR_FLUSH_LINK_STACK)) {
513 if (link_stack_flush_type != BRANCH_CACHE_FLUSH_NONE)
514 link_stack_flush_type = BRANCH_CACHE_FLUSH_NONE;
515
516 pr_info("link-stack-flush: flush disabled.\n");
517 } else {
518 if (security_ftr_enabled(SEC_FTR_BCCTR_LINK_FLUSH_ASSIST)) {
519 link_stack_flush_type = BRANCH_CACHE_FLUSH_HW;
520 pr_info("link-stack-flush: hardware flush enabled.\n");
521 } else {
522 link_stack_flush_type = BRANCH_CACHE_FLUSH_SW;
523 pr_info("link-stack-flush: software flush enabled.\n");
524 }
525 }
526
527 update_branch_cache_flush();
528}
529
530void setup_count_cache_flush(void)
531{
532 bool enable = true;
533
534 if (no_spectrev2 || cpu_mitigations_off()) {
535 if (security_ftr_enabled(SEC_FTR_BCCTRL_SERIALISED) ||
536 security_ftr_enabled(SEC_FTR_COUNT_CACHE_DISABLED))
537 pr_warn("Spectre v2 mitigations not fully under software control, can't disable\n");
538
539 enable = false;
540 }
541
542 /*
543 * There's no firmware feature flag/hypervisor bit to tell us we need to
544 * flush the link stack on context switch. So we set it here if we see
545 * either of the Spectre v2 mitigations that aim to protect userspace.
546 */
547 if (security_ftr_enabled(SEC_FTR_COUNT_CACHE_DISABLED) ||
548 security_ftr_enabled(SEC_FTR_FLUSH_COUNT_CACHE))
549 security_ftr_set(SEC_FTR_FLUSH_LINK_STACK);
550
551 toggle_branch_cache_flush(enable);
552}
553
554static enum l1d_flush_type enabled_flush_types;
555static void *l1d_flush_fallback_area;
556static bool no_rfi_flush;
557static bool no_entry_flush;
558static bool no_uaccess_flush;
559bool rfi_flush;
560static bool entry_flush;
561static bool uaccess_flush;
562DEFINE_STATIC_KEY_FALSE(uaccess_flush_key);
563EXPORT_SYMBOL(uaccess_flush_key);
564
565static int __init handle_no_rfi_flush(char *p)
566{
567 pr_info("rfi-flush: disabled on command line.");
568 no_rfi_flush = true;
569 return 0;
570}
571early_param("no_rfi_flush", handle_no_rfi_flush);
572
573static int __init handle_no_entry_flush(char *p)
574{
575 pr_info("entry-flush: disabled on command line.");
576 no_entry_flush = true;
577 return 0;
578}
579early_param("no_entry_flush", handle_no_entry_flush);
580
581static int __init handle_no_uaccess_flush(char *p)
582{
583 pr_info("uaccess-flush: disabled on command line.");
584 no_uaccess_flush = true;
585 return 0;
586}
587early_param("no_uaccess_flush", handle_no_uaccess_flush);
588
589/*
590 * The RFI flush is not KPTI, but because users will see doco that says to use
591 * nopti we hijack that option here to also disable the RFI flush.
592 */
593static int __init handle_no_pti(char *p)
594{
595 pr_info("rfi-flush: disabling due to 'nopti' on command line.\n");
596 handle_no_rfi_flush(NULL);
597 return 0;
598}
599early_param("nopti", handle_no_pti);
600
601static void do_nothing(void *unused)
602{
603 /*
604 * We don't need to do the flush explicitly, just enter+exit kernel is
605 * sufficient, the RFI exit handlers will do the right thing.
606 */
607}
608
609void rfi_flush_enable(bool enable)
610{
611 if (enable) {
612 do_rfi_flush_fixups(enabled_flush_types);
613 on_each_cpu(do_nothing, NULL, 1);
614 } else
615 do_rfi_flush_fixups(L1D_FLUSH_NONE);
616
617 rfi_flush = enable;
618}
619
620static void entry_flush_enable(bool enable)
621{
622 if (enable) {
623 do_entry_flush_fixups(enabled_flush_types);
624 on_each_cpu(do_nothing, NULL, 1);
625 } else {
626 do_entry_flush_fixups(L1D_FLUSH_NONE);
627 }
628
629 entry_flush = enable;
630}
631
632static void uaccess_flush_enable(bool enable)
633{
634 if (enable) {
635 do_uaccess_flush_fixups(enabled_flush_types);
636 static_branch_enable(&uaccess_flush_key);
637 on_each_cpu(do_nothing, NULL, 1);
638 } else {
639 static_branch_disable(&uaccess_flush_key);
640 do_uaccess_flush_fixups(L1D_FLUSH_NONE);
641 }
642
643 uaccess_flush = enable;
644}
645
646static void __ref init_fallback_flush(void)
647{
648 u64 l1d_size, limit;
649 int cpu;
650
651 /* Only allocate the fallback flush area once (at boot time). */
652 if (l1d_flush_fallback_area)
653 return;
654
655 l1d_size = ppc64_caches.l1d.size;
656
657 /*
658 * If there is no d-cache-size property in the device tree, l1d_size
659 * could be zero. That leads to the loop in the asm wrapping around to
660 * 2^64-1, and then walking off the end of the fallback area and
661 * eventually causing a page fault which is fatal. Just default to
662 * something vaguely sane.
663 */
664 if (!l1d_size)
665 l1d_size = (64 * 1024);
666
667 limit = min(ppc64_bolted_size(), ppc64_rma_size);
668
669 /*
670 * Align to L1d size, and size it at 2x L1d size, to catch possible
671 * hardware prefetch runoff. We don't have a recipe for load patterns to
672 * reliably avoid the prefetcher.
673 */
674 l1d_flush_fallback_area = memblock_alloc_try_nid(l1d_size * 2,
675 l1d_size, MEMBLOCK_LOW_LIMIT,
676 limit, NUMA_NO_NODE);
677 if (!l1d_flush_fallback_area)
678 panic("%s: Failed to allocate %llu bytes align=0x%llx max_addr=%pa\n",
679 __func__, l1d_size * 2, l1d_size, &limit);
680
681
682 for_each_possible_cpu(cpu) {
683 struct paca_struct *paca = paca_ptrs[cpu];
684 paca->rfi_flush_fallback_area = l1d_flush_fallback_area;
685 paca->l1d_flush_size = l1d_size;
686 }
687}
688
689void setup_rfi_flush(enum l1d_flush_type types, bool enable)
690{
691 if (types & L1D_FLUSH_FALLBACK) {
692 pr_info("rfi-flush: fallback displacement flush available\n");
693 init_fallback_flush();
694 }
695
696 if (types & L1D_FLUSH_ORI)
697 pr_info("rfi-flush: ori type flush available\n");
698
699 if (types & L1D_FLUSH_MTTRIG)
700 pr_info("rfi-flush: mttrig type flush available\n");
701
702 enabled_flush_types = types;
703
704 if (!cpu_mitigations_off() && !no_rfi_flush)
705 rfi_flush_enable(enable);
706}
707
708void setup_entry_flush(bool enable)
709{
710 if (cpu_mitigations_off())
711 return;
712
713 if (!no_entry_flush)
714 entry_flush_enable(enable);
715}
716
717void setup_uaccess_flush(bool enable)
718{
719 if (cpu_mitigations_off())
720 return;
721
722 if (!no_uaccess_flush)
723 uaccess_flush_enable(enable);
724}
725
726#ifdef CONFIG_DEBUG_FS
727static int count_cache_flush_set(void *data, u64 val)
728{
729 bool enable;
730
731 if (val == 1)
732 enable = true;
733 else if (val == 0)
734 enable = false;
735 else
736 return -EINVAL;
737
738 toggle_branch_cache_flush(enable);
739
740 return 0;
741}
742
743static int count_cache_flush_get(void *data, u64 *val)
744{
745 if (count_cache_flush_type == BRANCH_CACHE_FLUSH_NONE)
746 *val = 0;
747 else
748 *val = 1;
749
750 return 0;
751}
752
753static int link_stack_flush_get(void *data, u64 *val)
754{
755 if (link_stack_flush_type == BRANCH_CACHE_FLUSH_NONE)
756 *val = 0;
757 else
758 *val = 1;
759
760 return 0;
761}
762
763DEFINE_DEBUGFS_ATTRIBUTE(fops_count_cache_flush, count_cache_flush_get,
764 count_cache_flush_set, "%llu\n");
765DEFINE_DEBUGFS_ATTRIBUTE(fops_link_stack_flush, link_stack_flush_get,
766 count_cache_flush_set, "%llu\n");
767
768static __init int count_cache_flush_debugfs_init(void)
769{
770 debugfs_create_file_unsafe("count_cache_flush", 0600,
771 arch_debugfs_dir, NULL,
772 &fops_count_cache_flush);
773 debugfs_create_file_unsafe("link_stack_flush", 0600,
774 arch_debugfs_dir, NULL,
775 &fops_link_stack_flush);
776 return 0;
777}
778device_initcall(count_cache_flush_debugfs_init);
779
780static int rfi_flush_set(void *data, u64 val)
781{
782 bool enable;
783
784 if (val == 1)
785 enable = true;
786 else if (val == 0)
787 enable = false;
788 else
789 return -EINVAL;
790
791 /* Only do anything if we're changing state */
792 if (enable != rfi_flush)
793 rfi_flush_enable(enable);
794
795 return 0;
796}
797
798static int rfi_flush_get(void *data, u64 *val)
799{
800 *val = rfi_flush ? 1 : 0;
801 return 0;
802}
803
804DEFINE_SIMPLE_ATTRIBUTE(fops_rfi_flush, rfi_flush_get, rfi_flush_set, "%llu\n");
805
806static int entry_flush_set(void *data, u64 val)
807{
808 bool enable;
809
810 if (val == 1)
811 enable = true;
812 else if (val == 0)
813 enable = false;
814 else
815 return -EINVAL;
816
817 /* Only do anything if we're changing state */
818 if (enable != entry_flush)
819 entry_flush_enable(enable);
820
821 return 0;
822}
823
824static int entry_flush_get(void *data, u64 *val)
825{
826 *val = entry_flush ? 1 : 0;
827 return 0;
828}
829
830DEFINE_SIMPLE_ATTRIBUTE(fops_entry_flush, entry_flush_get, entry_flush_set, "%llu\n");
831
832static int uaccess_flush_set(void *data, u64 val)
833{
834 bool enable;
835
836 if (val == 1)
837 enable = true;
838 else if (val == 0)
839 enable = false;
840 else
841 return -EINVAL;
842
843 /* Only do anything if we're changing state */
844 if (enable != uaccess_flush)
845 uaccess_flush_enable(enable);
846
847 return 0;
848}
849
850static int uaccess_flush_get(void *data, u64 *val)
851{
852 *val = uaccess_flush ? 1 : 0;
853 return 0;
854}
855
856DEFINE_SIMPLE_ATTRIBUTE(fops_uaccess_flush, uaccess_flush_get, uaccess_flush_set, "%llu\n");
857
858static __init int rfi_flush_debugfs_init(void)
859{
860 debugfs_create_file("rfi_flush", 0600, arch_debugfs_dir, NULL, &fops_rfi_flush);
861 debugfs_create_file("entry_flush", 0600, arch_debugfs_dir, NULL, &fops_entry_flush);
862 debugfs_create_file("uaccess_flush", 0600, arch_debugfs_dir, NULL, &fops_uaccess_flush);
863 return 0;
864}
865device_initcall(rfi_flush_debugfs_init);
866#endif /* CONFIG_DEBUG_FS */
867#endif /* CONFIG_PPC_BOOK3S_64 */