| 1 | |
| 2 |
|
| 3 | #define USE_EARLY_PGTABLE_L5
|
| 4 |
|
| 5 | #include <linux/memblock.h>
|
| 6 | #include <linux/linkage.h>
|
| 7 | #include <linux/bitops.h>
|
| 8 | #include <linux/kernel.h>
|
| 9 | #include <linux/export.h>
|
| 10 | #include <linux/kvm_types.h>
|
| 11 | #include <linux/percpu.h>
|
| 12 | #include <linux/string.h>
|
| 13 | #include <linux/ctype.h>
|
| 14 | #include <linux/delay.h>
|
| 15 | #include <linux/sched/mm.h>
|
| 16 | #include <linux/sched/clock.h>
|
| 17 | #include <linux/sched/task.h>
|
| 18 | #include <linux/sched/smt.h>
|
| 19 | #include <linux/init.h>
|
| 20 | #include <linux/kprobes.h>
|
| 21 | #include <linux/kgdb.h>
|
| 22 | #include <linux/mem_encrypt.h>
|
| 23 | #include <linux/smp.h>
|
| 24 | #include <linux/cpu.h>
|
| 25 | #include <linux/io.h>
|
| 26 | #include <linux/syscore_ops.h>
|
| 27 | #include <linux/pgtable.h>
|
| 28 | #include <linux/stackprotector.h>
|
| 29 | #include <linux/utsname.h>
|
| 30 | #include <linux/efi.h>
|
| 31 |
|
| 32 | #include <asm/alternative.h>
|
| 33 | #include <asm/cmdline.h>
|
| 34 | #include <asm/cpuid/api.h>
|
| 35 | #include <asm/perf_event.h>
|
| 36 | #include <asm/mmu_context.h>
|
| 37 | #include <asm/doublefault.h>
|
| 38 | #include <asm/archrandom.h>
|
| 39 | #include <asm/hypervisor.h>
|
| 40 | #include <asm/processor.h>
|
| 41 | #include <asm/tlbflush.h>
|
| 42 | #include <asm/debugreg.h>
|
| 43 | #include <asm/sections.h>
|
| 44 | #include <asm/vsyscall.h>
|
| 45 | #include <linux/topology.h>
|
| 46 | #include <linux/cpumask.h>
|
| 47 | #include <linux/atomic.h>
|
| 48 | #include <asm/proto.h>
|
| 49 | #include <asm/setup.h>
|
| 50 | #include <asm/apic.h>
|
| 51 | #include <asm/desc.h>
|
| 52 | #include <asm/fpu/api.h>
|
| 53 | #include <asm/mtrr.h>
|
| 54 | #include <asm/hwcap2.h>
|
| 55 | #include <linux/numa.h>
|
| 56 | #include <asm/numa.h>
|
| 57 | #include <asm/asm.h>
|
| 58 | #include <asm/bugs.h>
|
| 59 | #include <asm/cpu.h>
|
| 60 | #include <asm/mce.h>
|
| 61 | #include <asm/msr.h>
|
| 62 | #include <asm/cacheinfo.h>
|
| 63 | #include <asm/memtype.h>
|
| 64 | #include <asm/microcode.h>
|
| 65 | #include <asm/intel-family.h>
|
| 66 | #include <asm/cpu_device_id.h>
|
| 67 | #include <asm/fred.h>
|
| 68 | #include <asm/uv/uv.h>
|
| 69 | #include <asm/ia32.h>
|
| 70 | #include <asm/set_memory.h>
|
| 71 | #include <asm/traps.h>
|
| 72 | #include <asm/sev.h>
|
| 73 | #include <asm/tdx.h>
|
| 74 | #include <asm/virt.h>
|
| 75 | #include <asm/posted_intr.h>
|
| 76 | #include <asm/runtime-const.h>
|
| 77 |
|
| 78 | #include "cpu.h"
|
| 79 |
|
| 80 | DEFINE_PER_CPU_READ_MOSTLY(struct cpuinfo_x86, cpu_info);
|
| 81 | EXPORT_PER_CPU_SYMBOL(cpu_info);
|
| 82 |
|
| 83 |
|
| 84 | unsigned long USER_PTR_MAX;
|
| 85 | EXPORT_SYMBOL(USER_PTR_MAX);
|
| 86 |
|
| 87 | u32 elf_hwcap2 __read_mostly;
|
| 88 |
|
| 89 |
|
| 90 | unsigned int __max_threads_per_core __ro_after_init = 1;
|
| 91 | EXPORT_SYMBOL(__max_threads_per_core);
|
| 92 |
|
| 93 | unsigned int __max_dies_per_package __ro_after_init = 1;
|
| 94 | EXPORT_SYMBOL(__max_dies_per_package);
|
| 95 |
|
| 96 | unsigned int __max_logical_packages __ro_after_init = 1;
|
| 97 | EXPORT_SYMBOL(__max_logical_packages);
|
| 98 |
|
| 99 | unsigned int __num_nodes_per_package __ro_after_init = 1;
|
| 100 | EXPORT_SYMBOL(__num_nodes_per_package);
|
| 101 |
|
| 102 | unsigned int __num_cores_per_package __ro_after_init = 1;
|
| 103 | EXPORT_SYMBOL(__num_cores_per_package);
|
| 104 |
|
| 105 | unsigned int __num_threads_per_package __ro_after_init = 1;
|
| 106 | EXPORT_SYMBOL(__num_threads_per_package);
|
| 107 |
|
| 108 | static struct ppin_info {
|
| 109 | int feature;
|
| 110 | int msr_ppin_ctl;
|
| 111 | int msr_ppin;
|
| 112 | } ppin_info[] = {
|
| 113 | [X86_VENDOR_INTEL] = {
|
| 114 | .feature = X86_FEATURE_INTEL_PPIN,
|
| 115 | .msr_ppin_ctl = MSR_PPIN_CTL,
|
| 116 | .msr_ppin = MSR_PPIN
|
| 117 | },
|
| 118 | [X86_VENDOR_AMD] = {
|
| 119 | .feature = X86_FEATURE_AMD_PPIN,
|
| 120 | .msr_ppin_ctl = MSR_AMD_PPIN_CTL,
|
| 121 | .msr_ppin = MSR_AMD_PPIN
|
| 122 | },
|
| 123 | };
|
| 124 |
|
| 125 | static const struct x86_cpu_id ppin_cpuids[] = {
|
| 126 | X86_MATCH_FEATURE(X86_FEATURE_AMD_PPIN, &ppin_info[X86_VENDOR_AMD]),
|
| 127 | X86_MATCH_FEATURE(X86_FEATURE_INTEL_PPIN, &ppin_info[X86_VENDOR_INTEL]),
|
| 128 |
|
| 129 |
|
| 130 | X86_MATCH_VFM(INTEL_IVYBRIDGE_X, &ppin_info[X86_VENDOR_INTEL]),
|
| 131 | X86_MATCH_VFM(INTEL_HASWELL_X, &ppin_info[X86_VENDOR_INTEL]),
|
| 132 | X86_MATCH_VFM(INTEL_BROADWELL_D, &ppin_info[X86_VENDOR_INTEL]),
|
| 133 | X86_MATCH_VFM(INTEL_BROADWELL_X, &ppin_info[X86_VENDOR_INTEL]),
|
| 134 | X86_MATCH_VFM(INTEL_SKYLAKE_X, &ppin_info[X86_VENDOR_INTEL]),
|
| 135 | X86_MATCH_VFM(INTEL_ICELAKE_X, &ppin_info[X86_VENDOR_INTEL]),
|
| 136 | X86_MATCH_VFM(INTEL_ICELAKE_D, &ppin_info[X86_VENDOR_INTEL]),
|
| 137 | X86_MATCH_VFM(INTEL_SAPPHIRERAPIDS_X, &ppin_info[X86_VENDOR_INTEL]),
|
| 138 | X86_MATCH_VFM(INTEL_EMERALDRAPIDS_X, &ppin_info[X86_VENDOR_INTEL]),
|
| 139 | X86_MATCH_VFM(INTEL_XEON_PHI_KNL, &ppin_info[X86_VENDOR_INTEL]),
|
| 140 | X86_MATCH_VFM(INTEL_XEON_PHI_KNM, &ppin_info[X86_VENDOR_INTEL]),
|
| 141 |
|
| 142 | {}
|
| 143 | };
|
| 144 |
|
| 145 | static void ppin_init(struct cpuinfo_x86 *c)
|
| 146 | {
|
| 147 | const struct x86_cpu_id *id;
|
| 148 | unsigned long long val;
|
| 149 | struct ppin_info *info;
|
| 150 |
|
| 151 | id = x86_match_cpu(ppin_cpuids);
|
| 152 | if (!id)
|
| 153 | return;
|
| 154 |
|
| 155 | |
| 156 | * Testing the presence of the MSR is not enough. Need to check
|
| 157 | * that the PPIN_CTL allows reading of the PPIN.
|
| 158 |
|
| 159 | info = (struct ppin_info *)id->driver_data;
|
| 160 |
|
| 161 | if (rdmsrq_safe(info->msr_ppin_ctl, &val))
|
| 162 | goto clear_ppin;
|
| 163 |
|
| 164 | if ((val & 3UL) == 1UL) {
|
| 165 |
|
| 166 | goto clear_ppin;
|
| 167 | }
|
| 168 |
|
| 169 |
|
| 170 | if (!(val & 2UL)) {
|
| 171 | wrmsrq_safe(info->msr_ppin_ctl, val | 2UL);
|
| 172 | rdmsrq_safe(info->msr_ppin_ctl, &val);
|
| 173 | }
|
| 174 |
|
| 175 |
|
| 176 | if (val & 2UL) {
|
| 177 | c->ppin = native_rdmsrq(info->msr_ppin);
|
| 178 | set_cpu_cap(c, info->feature);
|
| 179 | return;
|
| 180 | }
|
| 181 |
|
| 182 | clear_ppin:
|
| 183 | setup_clear_cpu_cap(info->feature);
|
| 184 | }
|
| 185 |
|
| 186 | static void default_init(struct cpuinfo_x86 *c)
|
| 187 | {
|
| 188 | #ifdef CONFIG_X86_64
|
| 189 | cpu_detect_cache_sizes(c);
|
| 190 | #else
|
| 191 |
|
| 192 |
|
| 193 | if (c->cpuid_level == -1) {
|
| 194 |
|
| 195 | if (c->x86 == 4)
|
| 196 | strcpy(c->x86_model_id, "486");
|
| 197 | else if (c->x86 == 3)
|
| 198 | strcpy(c->x86_model_id, "386");
|
| 199 | }
|
| 200 | #endif
|
| 201 | }
|
| 202 |
|
| 203 | static const struct cpu_dev default_cpu = {
|
| 204 | .c_init = default_init,
|
| 205 | .c_vendor = "Unknown",
|
| 206 | .c_x86_vendor = X86_VENDOR_UNKNOWN,
|
| 207 | };
|
| 208 |
|
| 209 | static const struct cpu_dev *this_cpu = &default_cpu;
|
| 210 |
|
| 211 | DEFINE_PER_CPU_PAGE_ALIGNED(struct gdt_page, gdt_page) = { .gdt = {
|
| 212 | #ifdef CONFIG_X86_64
|
| 213 | |
| 214 | * We need valid kernel segments for data and code in long mode too
|
| 215 | * IRET will check the segment types kkeil 2000/10/28
|
| 216 | * Also sysret mandates a special GDT layout
|
| 217 | *
|
| 218 | * TLS descriptors are currently at a different place compared to i386.
|
| 219 | * Hopefully nobody expects them at a fixed place (Wine?)
|
| 220 |
|
| 221 | [GDT_ENTRY_KERNEL32_CS] = GDT_ENTRY_INIT(DESC_CODE32, 0, 0xfffff),
|
| 222 | [GDT_ENTRY_KERNEL_CS] = GDT_ENTRY_INIT(DESC_CODE64, 0, 0xfffff),
|
| 223 | [GDT_ENTRY_KERNEL_DS] = GDT_ENTRY_INIT(DESC_DATA64, 0, 0xfffff),
|
| 224 | [GDT_ENTRY_DEFAULT_USER32_CS] = GDT_ENTRY_INIT(DESC_CODE32 | DESC_USER, 0, 0xfffff),
|
| 225 | [GDT_ENTRY_DEFAULT_USER_DS] = GDT_ENTRY_INIT(DESC_DATA64 | DESC_USER, 0, 0xfffff),
|
| 226 | [GDT_ENTRY_DEFAULT_USER_CS] = GDT_ENTRY_INIT(DESC_CODE64 | DESC_USER, 0, 0xfffff),
|
| 227 | #else
|
| 228 | [GDT_ENTRY_KERNEL_CS] = GDT_ENTRY_INIT(DESC_CODE32, 0, 0xfffff),
|
| 229 | [GDT_ENTRY_KERNEL_DS] = GDT_ENTRY_INIT(DESC_DATA32, 0, 0xfffff),
|
| 230 | [GDT_ENTRY_DEFAULT_USER_CS] = GDT_ENTRY_INIT(DESC_CODE32 | DESC_USER, 0, 0xfffff),
|
| 231 | [GDT_ENTRY_DEFAULT_USER_DS] = GDT_ENTRY_INIT(DESC_DATA32 | DESC_USER, 0, 0xfffff),
|
| 232 | |
| 233 | * Segments used for calling PnP BIOS have byte granularity.
|
| 234 | * They code segments and data segments have fixed 64k limits,
|
| 235 | * the transfer segment sizes are set at run time.
|
| 236 |
|
| 237 | [GDT_ENTRY_PNPBIOS_CS32] = GDT_ENTRY_INIT(DESC_CODE32_BIOS, 0, 0xffff),
|
| 238 | [GDT_ENTRY_PNPBIOS_CS16] = GDT_ENTRY_INIT(DESC_CODE16, 0, 0xffff),
|
| 239 | [GDT_ENTRY_PNPBIOS_DS] = GDT_ENTRY_INIT(DESC_DATA16, 0, 0xffff),
|
| 240 | [GDT_ENTRY_PNPBIOS_TS1] = GDT_ENTRY_INIT(DESC_DATA16, 0, 0),
|
| 241 | [GDT_ENTRY_PNPBIOS_TS2] = GDT_ENTRY_INIT(DESC_DATA16, 0, 0),
|
| 242 | |
| 243 | * The APM segments have byte granularity and their bases
|
| 244 | * are set at run time. All have 64k limits.
|
| 245 |
|
| 246 | [GDT_ENTRY_APMBIOS_BASE] = GDT_ENTRY_INIT(DESC_CODE32_BIOS, 0, 0xffff),
|
| 247 | [GDT_ENTRY_APMBIOS_BASE+1] = GDT_ENTRY_INIT(DESC_CODE16, 0, 0xffff),
|
| 248 | [GDT_ENTRY_APMBIOS_BASE+2] = GDT_ENTRY_INIT(DESC_DATA32_BIOS, 0, 0xffff),
|
| 249 |
|
| 250 | [GDT_ENTRY_ESPFIX_SS] = GDT_ENTRY_INIT(DESC_DATA32, 0, 0xfffff),
|
| 251 | [GDT_ENTRY_PERCPU] = GDT_ENTRY_INIT(DESC_DATA32, 0, 0xfffff),
|
| 252 | #endif
|
| 253 | } };
|
| 254 | EXPORT_PER_CPU_SYMBOL_GPL(gdt_page);
|
| 255 | SYM_PIC_ALIAS(gdt_page);
|
| 256 |
|
| 257 | #ifdef CONFIG_X86_64
|
| 258 | static int __init x86_nopcid_setup(char *s)
|
| 259 | {
|
| 260 |
|
| 261 | if (s)
|
| 262 | return -EINVAL;
|
| 263 |
|
| 264 |
|
| 265 | if (!boot_cpu_has(X86_FEATURE_PCID))
|
| 266 | return 0;
|
| 267 |
|
| 268 | setup_clear_cpu_cap(X86_FEATURE_PCID);
|
| 269 | pr_info("nopcid: PCID feature disabled\n");
|
| 270 | return 0;
|
| 271 | }
|
| 272 | early_param("nopcid", x86_nopcid_setup);
|
| 273 | #endif
|
| 274 |
|
| 275 | static int __init x86_noinvpcid_setup(char *s)
|
| 276 | {
|
| 277 |
|
| 278 | if (s)
|
| 279 | return -EINVAL;
|
| 280 |
|
| 281 |
|
| 282 | if (!boot_cpu_has(X86_FEATURE_INVPCID))
|
| 283 | return 0;
|
| 284 |
|
| 285 | setup_clear_cpu_cap(X86_FEATURE_INVPCID);
|
| 286 | pr_info("noinvpcid: INVPCID feature disabled\n");
|
| 287 | return 0;
|
| 288 | }
|
| 289 | early_param("noinvpcid", x86_noinvpcid_setup);
|
| 290 |
|
| 291 |
|
| 292 | static inline bool flag_is_changeable_p(unsigned long flag)
|
| 293 | {
|
| 294 | unsigned long f1, f2;
|
| 295 |
|
| 296 | if (!IS_ENABLED(CONFIG_X86_32))
|
| 297 | return true;
|
| 298 |
|
| 299 | |
| 300 | * Cyrix and IDT cpus allow disabling of CPUID
|
| 301 | * so the code below may return different results
|
| 302 | * when it is executed before and after enabling
|
| 303 | * the CPUID. Add "volatile" to not allow gcc to
|
| 304 | * optimize the subsequent calls to this function.
|
| 305 |
|
| 306 | asm volatile ("pushfl \n\t"
|
| 307 | "pushfl \n\t"
|
| 308 | "popl %0 \n\t"
|
| 309 | "movl %0, %1 \n\t"
|
| 310 | "xorl %2, %0 \n\t"
|
| 311 | "pushl %0 \n\t"
|
| 312 | "popfl \n\t"
|
| 313 | "pushfl \n\t"
|
| 314 | "popl %0 \n\t"
|
| 315 | "popfl \n\t"
|
| 316 |
|
| 317 | : "=&r" (f1), "=&r" (f2)
|
| 318 | : "ir" (flag));
|
| 319 |
|
| 320 | return (f1 ^ f2) & flag;
|
| 321 | }
|
| 322 |
|
| 323 | #ifdef CONFIG_X86_32
|
| 324 | static int cachesize_override = -1;
|
| 325 | static int disable_x86_serial_nr = 1;
|
| 326 |
|
| 327 | static int __init cachesize_setup(char *str)
|
| 328 | {
|
| 329 | get_option(&str, &cachesize_override);
|
| 330 | return 1;
|
| 331 | }
|
| 332 | __setup("cachesize=", cachesize_setup);
|
| 333 |
|
| 334 |
|
| 335 | bool cpuid_feature(void)
|
| 336 | {
|
| 337 | return flag_is_changeable_p(X86_EFLAGS_ID);
|
| 338 | }
|
| 339 |
|
| 340 | static void squash_the_stupid_serial_number(struct cpuinfo_x86 *c)
|
| 341 | {
|
| 342 | struct msr val;
|
| 343 |
|
| 344 | if (!cpu_has(c, X86_FEATURE_PN) || !disable_x86_serial_nr)
|
| 345 | return;
|
| 346 |
|
| 347 |
|
| 348 |
|
| 349 | rdmsrq(MSR_IA32_BBL_CR_CTL, val.q);
|
| 350 | val.l |= 0x200000;
|
| 351 | wrmsrq(MSR_IA32_BBL_CR_CTL, val.q);
|
| 352 |
|
| 353 | pr_notice("CPU serial number disabled.\n");
|
| 354 | clear_cpu_cap(c, X86_FEATURE_PN);
|
| 355 |
|
| 356 |
|
| 357 | c->cpuid_level = cpuid_eax(0);
|
| 358 | }
|
| 359 |
|
| 360 | static int __init x86_serial_nr_setup(char *s)
|
| 361 | {
|
| 362 | disable_x86_serial_nr = 0;
|
| 363 | return 1;
|
| 364 | }
|
| 365 | __setup("serialnumber", x86_serial_nr_setup);
|
| 366 | #else
|
| 367 | static inline void squash_the_stupid_serial_number(struct cpuinfo_x86 *c)
|
| 368 | {
|
| 369 | }
|
| 370 | #endif
|
| 371 |
|
| 372 | static __always_inline void setup_smep(struct cpuinfo_x86 *c)
|
| 373 | {
|
| 374 | if (cpu_has(c, X86_FEATURE_SMEP))
|
| 375 | cr4_set_bits(X86_CR4_SMEP);
|
| 376 | }
|
| 377 |
|
| 378 | static __always_inline void setup_smap(struct cpuinfo_x86 *c)
|
| 379 | {
|
| 380 | unsigned long eflags = native_save_fl();
|
| 381 |
|
| 382 |
|
| 383 | BUG_ON(eflags & X86_EFLAGS_AC);
|
| 384 |
|
| 385 | if (cpu_has(c, X86_FEATURE_SMAP))
|
| 386 | cr4_set_bits(X86_CR4_SMAP);
|
| 387 | }
|
| 388 |
|
| 389 | static __always_inline void setup_umip(struct cpuinfo_x86 *c)
|
| 390 | {
|
| 391 |
|
| 392 | if (!cpu_feature_enabled(X86_FEATURE_UMIP))
|
| 393 | goto out;
|
| 394 |
|
| 395 |
|
| 396 | if (!cpu_has(c, X86_FEATURE_UMIP))
|
| 397 | goto out;
|
| 398 |
|
| 399 | cr4_set_bits(X86_CR4_UMIP);
|
| 400 |
|
| 401 | pr_info_once("x86/cpu: User Mode Instruction Prevention (UMIP) activated\n");
|
| 402 |
|
| 403 | return;
|
| 404 |
|
| 405 | out:
|
| 406 | |
| 407 | * Make sure UMIP is disabled in case it was enabled in a
|
| 408 | * previous boot (e.g., via kexec).
|
| 409 |
|
| 410 | cr4_clear_bits(X86_CR4_UMIP);
|
| 411 | }
|
| 412 |
|
| 413 | static int enable_lass(unsigned int cpu)
|
| 414 | {
|
| 415 | cr4_set_bits(X86_CR4_LASS);
|
| 416 |
|
| 417 | return 0;
|
| 418 | }
|
| 419 |
|
| 420 | |
| 421 | * Finalize features that need to be enabled just before entering
|
| 422 | * userspace. Note that this only runs on a single CPU. Use appropriate
|
| 423 | * callbacks if all the CPUs need to reflect the same change.
|
| 424 |
|
| 425 | static int cpu_finalize_pre_userspace(void)
|
| 426 | {
|
| 427 | if (!cpu_feature_enabled(X86_FEATURE_LASS))
|
| 428 | return 0;
|
| 429 |
|
| 430 |
|
| 431 | cpuhp_setup_state(CPUHP_AP_ONLINE_DYN, "x86/lass:enable", enable_lass, NULL);
|
| 432 |
|
| 433 | return 0;
|
| 434 | }
|
| 435 | late_initcall(cpu_finalize_pre_userspace);
|
| 436 |
|
| 437 |
|
| 438 | static const unsigned long cr4_pinned_mask = X86_CR4_SMEP | X86_CR4_SMAP | X86_CR4_UMIP |
|
| 439 | X86_CR4_FSGSBASE | X86_CR4_CET;
|
| 440 |
|
| 441 | |
| 442 | * The CR pinning protects against ROP on the 'mov %reg, %CRn' instruction(s).
|
| 443 | * Since you can ROP directly to these instructions (barring shadow stack),
|
| 444 | * any protection must follow immediately and unconditionally after that.
|
| 445 | *
|
| 446 | * Specifically, the CR[04] write functions below will have the value
|
| 447 | * validation controlled by the @cr_pinning static_branch which is
|
| 448 | * __ro_after_init, just like the cr4_pinned_bits value.
|
| 449 | *
|
| 450 | * Once set, an attacker will have to defeat page-tables to get around these
|
| 451 | * restrictions. Which is a much bigger ask than 'simple' ROP.
|
| 452 |
|
| 453 | static DEFINE_STATIC_KEY_FALSE_RO(cr_pinning);
|
| 454 | static unsigned long cr4_pinned_bits __ro_after_init;
|
| 455 |
|
| 456 | void native_write_cr0(unsigned long val)
|
| 457 | {
|
| 458 | unsigned long bits_missing = 0;
|
| 459 |
|
| 460 | set_register:
|
| 461 | asm volatile("mov %0,%%cr0": "+r" (val) : : "memory");
|
| 462 |
|
| 463 | if (static_branch_likely(&cr_pinning)) {
|
| 464 | if (unlikely((val & X86_CR0_WP) != X86_CR0_WP)) {
|
| 465 | bits_missing = X86_CR0_WP;
|
| 466 | val |= bits_missing;
|
| 467 | goto set_register;
|
| 468 | }
|
| 469 |
|
| 470 | WARN_ONCE(bits_missing, "CR0 WP bit went missing!?\n");
|
| 471 | }
|
| 472 | }
|
| 473 | EXPORT_SYMBOL(native_write_cr0);
|
| 474 |
|
| 475 | void __no_profile native_write_cr4(unsigned long val)
|
| 476 | {
|
| 477 | unsigned long bits_changed = 0;
|
| 478 |
|
| 479 | set_register:
|
| 480 | asm volatile("mov %0,%%cr4": "+r" (val) : : "memory");
|
| 481 |
|
| 482 | if (static_branch_likely(&cr_pinning)) {
|
| 483 | if (unlikely((val & cr4_pinned_mask) != cr4_pinned_bits)) {
|
| 484 | bits_changed = (val & cr4_pinned_mask) ^ cr4_pinned_bits;
|
| 485 | val = (val & ~cr4_pinned_mask) | cr4_pinned_bits;
|
| 486 | goto set_register;
|
| 487 | }
|
| 488 |
|
| 489 | WARN_ONCE(bits_changed, "pinned CR4 bits changed: 0x%lx!?\n",
|
| 490 | bits_changed);
|
| 491 | }
|
| 492 | }
|
| 493 | #if IS_MODULE(CONFIG_LKDTM)
|
| 494 | EXPORT_SYMBOL_GPL(native_write_cr4);
|
| 495 | #endif
|
| 496 |
|
| 497 | void cr4_update_irqsoff(unsigned long set, unsigned long clear)
|
| 498 | {
|
| 499 | unsigned long newval, cr4 = this_cpu_read(cpu_tlbstate.cr4);
|
| 500 |
|
| 501 | lockdep_assert_irqs_disabled();
|
| 502 |
|
| 503 | newval = (cr4 & ~clear) | set;
|
| 504 | if (newval != cr4) {
|
| 505 | this_cpu_write(cpu_tlbstate.cr4, newval);
|
| 506 | __write_cr4(newval);
|
| 507 | }
|
| 508 | }
|
| 509 | EXPORT_SYMBOL_FOR_KVM(cr4_update_irqsoff);
|
| 510 |
|
| 511 |
|
| 512 | unsigned long cr4_read_shadow(void)
|
| 513 | {
|
| 514 | return this_cpu_read(cpu_tlbstate.cr4);
|
| 515 | }
|
| 516 | EXPORT_SYMBOL_FOR_KVM(cr4_read_shadow);
|
| 517 |
|
| 518 | void cr4_init(void)
|
| 519 | {
|
| 520 | unsigned long cr4 = __read_cr4();
|
| 521 |
|
| 522 | if (boot_cpu_has(X86_FEATURE_PCID))
|
| 523 | cr4 |= X86_CR4_PCIDE;
|
| 524 | if (static_branch_likely(&cr_pinning))
|
| 525 | cr4 = (cr4 & ~cr4_pinned_mask) | cr4_pinned_bits;
|
| 526 |
|
| 527 | __write_cr4(cr4);
|
| 528 |
|
| 529 |
|
| 530 | this_cpu_write(cpu_tlbstate.cr4, cr4);
|
| 531 | }
|
| 532 |
|
| 533 | |
| 534 | * Once CPU feature detection is finished (and boot params have been
|
| 535 | * parsed), record any of the sensitive CR bits that are set, and
|
| 536 | * enable CR pinning.
|
| 537 |
|
| 538 | static void __init setup_cr_pinning(void)
|
| 539 | {
|
| 540 | cr4_pinned_bits = this_cpu_read(cpu_tlbstate.cr4) & cr4_pinned_mask;
|
| 541 | static_key_enable(&cr_pinning.key);
|
| 542 | }
|
| 543 |
|
| 544 | static __init int x86_nofsgsbase_setup(char *arg)
|
| 545 | {
|
| 546 |
|
| 547 | if (strlen(arg))
|
| 548 | return 0;
|
| 549 |
|
| 550 |
|
| 551 | if (!boot_cpu_has(X86_FEATURE_FSGSBASE))
|
| 552 | return 1;
|
| 553 |
|
| 554 | setup_clear_cpu_cap(X86_FEATURE_FSGSBASE);
|
| 555 | pr_info("FSGSBASE disabled via kernel command line\n");
|
| 556 | return 1;
|
| 557 | }
|
| 558 | __setup("nofsgsbase", x86_nofsgsbase_setup);
|
| 559 |
|
| 560 | |
| 561 | * Protection Keys are not available in 32-bit mode.
|
| 562 |
|
| 563 | static bool pku_disabled;
|
| 564 |
|
| 565 | static __always_inline void setup_pku(struct cpuinfo_x86 *c)
|
| 566 | {
|
| 567 | if (c == &boot_cpu_data) {
|
| 568 | if (pku_disabled || !cpu_feature_enabled(X86_FEATURE_PKU))
|
| 569 | return;
|
| 570 | |
| 571 | * Setting CR4.PKE will cause the X86_FEATURE_OSPKE cpuid
|
| 572 | * bit to be set. Enforce it.
|
| 573 |
|
| 574 | setup_force_cpu_cap(X86_FEATURE_OSPKE);
|
| 575 |
|
| 576 | } else if (!cpu_feature_enabled(X86_FEATURE_OSPKE)) {
|
| 577 | return;
|
| 578 | }
|
| 579 |
|
| 580 | cr4_set_bits(X86_CR4_PKE);
|
| 581 |
|
| 582 | pkru_write_default();
|
| 583 | }
|
| 584 |
|
| 585 | #ifdef CONFIG_X86_INTEL_MEMORY_PROTECTION_KEYS
|
| 586 | static __init int setup_disable_pku(char *arg)
|
| 587 | {
|
| 588 | |
| 589 | * Do not clear the X86_FEATURE_PKU bit. All of the
|
| 590 | * runtime checks are against OSPKE so clearing the
|
| 591 | * bit does nothing.
|
| 592 | *
|
| 593 | * This way, we will see "pku" in cpuinfo, but not
|
| 594 | * "ospke", which is exactly what we want. It shows
|
| 595 | * that the CPU has PKU, but the OS has not enabled it.
|
| 596 | * This happens to be exactly how a system would look
|
| 597 | * if we disabled the config option.
|
| 598 |
|
| 599 | pr_info("x86: 'nopku' specified, disabling Memory Protection Keys\n");
|
| 600 | pku_disabled = true;
|
| 601 | return 1;
|
| 602 | }
|
| 603 | __setup("nopku", setup_disable_pku);
|
| 604 | #endif
|
| 605 |
|
| 606 | #ifdef CONFIG_X86_KERNEL_IBT
|
| 607 |
|
| 608 | __noendbr u64 ibt_save(bool disable)
|
| 609 | {
|
| 610 | u64 msr = 0;
|
| 611 |
|
| 612 | if (cpu_feature_enabled(X86_FEATURE_IBT)) {
|
| 613 | rdmsrq(MSR_IA32_S_CET, msr);
|
| 614 | if (disable)
|
| 615 | wrmsrq(MSR_IA32_S_CET, msr & ~CET_ENDBR_EN);
|
| 616 | }
|
| 617 |
|
| 618 | return msr;
|
| 619 | }
|
| 620 |
|
| 621 | __noendbr void ibt_restore(u64 save)
|
| 622 | {
|
| 623 | u64 msr;
|
| 624 |
|
| 625 | if (cpu_feature_enabled(X86_FEATURE_IBT)) {
|
| 626 | rdmsrq(MSR_IA32_S_CET, msr);
|
| 627 | msr &= ~CET_ENDBR_EN;
|
| 628 | msr |= (save & CET_ENDBR_EN);
|
| 629 | wrmsrq(MSR_IA32_S_CET, msr);
|
| 630 | }
|
| 631 | }
|
| 632 |
|
| 633 | #endif
|
| 634 |
|
| 635 | static __always_inline void setup_cet(struct cpuinfo_x86 *c)
|
| 636 | {
|
| 637 | bool user_shstk, kernel_ibt;
|
| 638 |
|
| 639 | if (!IS_ENABLED(CONFIG_X86_CET))
|
| 640 | return;
|
| 641 |
|
| 642 | kernel_ibt = HAS_KERNEL_IBT && cpu_feature_enabled(X86_FEATURE_IBT);
|
| 643 | user_shstk = cpu_feature_enabled(X86_FEATURE_SHSTK) &&
|
| 644 | IS_ENABLED(CONFIG_X86_USER_SHADOW_STACK);
|
| 645 |
|
| 646 | if (!kernel_ibt && !user_shstk)
|
| 647 | return;
|
| 648 |
|
| 649 | if (user_shstk)
|
| 650 | set_cpu_cap(c, X86_FEATURE_USER_SHSTK);
|
| 651 |
|
| 652 | if (kernel_ibt)
|
| 653 | wrmsrq(MSR_IA32_S_CET, CET_ENDBR_EN);
|
| 654 | else
|
| 655 | wrmsrq(MSR_IA32_S_CET, 0);
|
| 656 |
|
| 657 | cr4_set_bits(X86_CR4_CET);
|
| 658 |
|
| 659 | if (kernel_ibt && ibt_selftest()) {
|
| 660 | pr_err("IBT selftest: Failed!\n");
|
| 661 | wrmsrq(MSR_IA32_S_CET, 0);
|
| 662 | setup_clear_cpu_cap(X86_FEATURE_IBT);
|
| 663 | }
|
| 664 | }
|
| 665 |
|
| 666 | __noendbr void cet_disable(void)
|
| 667 | {
|
| 668 | if (!(cpu_feature_enabled(X86_FEATURE_IBT) ||
|
| 669 | cpu_feature_enabled(X86_FEATURE_SHSTK)))
|
| 670 | return;
|
| 671 |
|
| 672 | wrmsrq(MSR_IA32_S_CET, 0);
|
| 673 | wrmsrq(MSR_IA32_U_CET, 0);
|
| 674 | }
|
| 675 |
|
| 676 | |
| 677 | * Some CPU features depend on higher CPUID levels, which may not always
|
| 678 | * be available due to CPUID level capping or broken virtualization
|
| 679 | * software. Add those features to this table to auto-disable them.
|
| 680 |
|
| 681 | struct cpuid_dependent_feature {
|
| 682 | u32 feature;
|
| 683 | u32 level;
|
| 684 | };
|
| 685 |
|
| 686 | static const struct cpuid_dependent_feature
|
| 687 | cpuid_dependent_features[] = {
|
| 688 | { X86_FEATURE_MWAIT, CPUID_LEAF_MWAIT },
|
| 689 | { X86_FEATURE_DCA, CPUID_LEAF_DCA },
|
| 690 | { X86_FEATURE_XSAVE, CPUID_LEAF_XSTATE },
|
| 691 | { 0, 0 }
|
| 692 | };
|
| 693 |
|
| 694 | static void filter_cpuid_features(struct cpuinfo_x86 *c, bool warn)
|
| 695 | {
|
| 696 | const struct cpuid_dependent_feature *df;
|
| 697 |
|
| 698 | for (df = cpuid_dependent_features; df->feature; df++) {
|
| 699 |
|
| 700 | if (!cpu_has(c, df->feature))
|
| 701 | continue;
|
| 702 | |
| 703 | * Note: cpuid_level is set to -1 if unavailable, but
|
| 704 | * extended_extended_level is set to 0 if unavailable
|
| 705 | * and the legitimate extended levels are all negative
|
| 706 | * when signed; hence the weird messing around with
|
| 707 | * signs here...
|
| 708 |
|
| 709 | if (!((s32)df->level < 0 ?
|
| 710 | (u32)df->level > (u32)c->extended_cpuid_level :
|
| 711 | (s32)df->level > (s32)c->cpuid_level))
|
| 712 | continue;
|
| 713 |
|
| 714 | clear_cpu_cap(c, df->feature);
|
| 715 | if (!warn)
|
| 716 | continue;
|
| 717 |
|
| 718 | pr_warn("CPU: CPU feature %s disabled, no CPUID level 0x%x\n",
|
| 719 | x86_cap_flags[df->feature], df->level);
|
| 720 | }
|
| 721 | }
|
| 722 |
|
| 723 | |
| 724 | * Naming convention should be: <Name> [(<Codename>)]
|
| 725 | * This table only is used unless init_<vendor>() below doesn't set it;
|
| 726 | * in particular, if CPUID levels 0x80000002..4 are supported, this
|
| 727 | * isn't used
|
| 728 |
|
| 729 |
|
| 730 |
|
| 731 | static const char *table_lookup_model(struct cpuinfo_x86 *c)
|
| 732 | {
|
| 733 | #ifdef CONFIG_X86_32
|
| 734 | const struct legacy_cpu_model_info *info;
|
| 735 |
|
| 736 | if (c->x86_model >= 16)
|
| 737 | return NULL;
|
| 738 |
|
| 739 | if (!this_cpu)
|
| 740 | return NULL;
|
| 741 |
|
| 742 | info = this_cpu->legacy_models;
|
| 743 |
|
| 744 | while (info->family) {
|
| 745 | if (info->family == c->x86)
|
| 746 | return info->model_names[c->x86_model];
|
| 747 | info++;
|
| 748 | }
|
| 749 | #endif
|
| 750 | return NULL;
|
| 751 | }
|
| 752 |
|
| 753 |
|
| 754 | __u32 cpu_caps_cleared[NCAPINTS + NBUGINTS] __aligned(sizeof(unsigned long));
|
| 755 | __u32 cpu_caps_set[NCAPINTS + NBUGINTS] __aligned(sizeof(unsigned long));
|
| 756 |
|
| 757 | #ifdef CONFIG_X86_32
|
| 758 |
|
| 759 | DEFINE_PER_CPU(struct cpu_entry_area *, cpu_entry_area);
|
| 760 | #endif
|
| 761 |
|
| 762 |
|
| 763 | void load_direct_gdt(int cpu)
|
| 764 | {
|
| 765 | struct desc_ptr gdt_descr;
|
| 766 |
|
| 767 | gdt_descr.address = (long)get_cpu_gdt_rw(cpu);
|
| 768 | gdt_descr.size = GDT_SIZE - 1;
|
| 769 | load_gdt(&gdt_descr);
|
| 770 | }
|
| 771 | EXPORT_SYMBOL_FOR_KVM(load_direct_gdt);
|
| 772 |
|
| 773 |
|
| 774 | void load_fixmap_gdt(int cpu)
|
| 775 | {
|
| 776 | struct desc_ptr gdt_descr;
|
| 777 |
|
| 778 | gdt_descr.address = (long)get_cpu_gdt_ro(cpu);
|
| 779 | gdt_descr.size = GDT_SIZE - 1;
|
| 780 | load_gdt(&gdt_descr);
|
| 781 | }
|
| 782 | EXPORT_SYMBOL_GPL(load_fixmap_gdt);
|
| 783 |
|
| 784 | |
| 785 | * switch_gdt_and_percpu_base - Switch to direct GDT and runtime per CPU base
|
| 786 | * @cpu: The CPU number for which this is invoked
|
| 787 | *
|
| 788 | * Invoked during early boot to switch from early GDT and early per CPU to
|
| 789 | * the direct GDT and the runtime per CPU area. On 32-bit the percpu base
|
| 790 | * switch is implicit by loading the direct GDT. On 64bit this requires
|
| 791 | * to update GSBASE.
|
| 792 |
|
| 793 | void __init switch_gdt_and_percpu_base(int cpu)
|
| 794 | {
|
| 795 | load_direct_gdt(cpu);
|
| 796 |
|
| 797 | #ifdef CONFIG_X86_64
|
| 798 | |
| 799 | * No need to load %gs. It is already correct.
|
| 800 | *
|
| 801 | * Writing %gs on 64bit would zero GSBASE which would make any per
|
| 802 | * CPU operation up to the point of the wrmsrq() fault.
|
| 803 | *
|
| 804 | * Set GSBASE to the new offset. Until the wrmsrq() happens the
|
| 805 | * early mapping is still valid. That means the GSBASE update will
|
| 806 | * lose any prior per CPU data which was not copied over in
|
| 807 | * setup_per_cpu_areas().
|
| 808 | *
|
| 809 | * This works even with stackprotector enabled because the
|
| 810 | * per CPU stack canary is 0 in both per CPU areas.
|
| 811 |
|
| 812 | wrmsrq(MSR_GS_BASE, cpu_kernelmode_gs_base(cpu));
|
| 813 | #else
|
| 814 | |
| 815 | * %fs is already set to __KERNEL_PERCPU, but after switching GDT
|
| 816 | * it is required to load FS again so that the 'hidden' part is
|
| 817 | * updated from the new GDT. Up to this point the early per CPU
|
| 818 | * translation is active. Any content of the early per CPU data
|
| 819 | * which was not copied over in setup_per_cpu_areas() is lost.
|
| 820 |
|
| 821 | loadsegment(fs, __KERNEL_PERCPU);
|
| 822 | #endif
|
| 823 | }
|
| 824 |
|
| 825 | static const struct cpu_dev *cpu_devs[X86_VENDOR_NUM] = {};
|
| 826 |
|
| 827 | static void get_model_name(struct cpuinfo_x86 *c)
|
| 828 | {
|
| 829 | unsigned int *v;
|
| 830 | char *p, *q, *s;
|
| 831 |
|
| 832 | if (c->extended_cpuid_level < 0x80000004)
|
| 833 | return;
|
| 834 |
|
| 835 | v = (unsigned int *)c->x86_model_id;
|
| 836 | cpuid(0x80000002, &v[0], &v[1], &v[2], &v[3]);
|
| 837 | cpuid(0x80000003, &v[4], &v[5], &v[6], &v[7]);
|
| 838 | cpuid(0x80000004, &v[8], &v[9], &v[10], &v[11]);
|
| 839 | c->x86_model_id[48] = 0;
|
| 840 |
|
| 841 |
|
| 842 | p = q = s = &c->x86_model_id[0];
|
| 843 |
|
| 844 | while (*p == ' ')
|
| 845 | p++;
|
| 846 |
|
| 847 | while (*p) {
|
| 848 |
|
| 849 | if (!isspace(*p))
|
| 850 | s = q;
|
| 851 |
|
| 852 | *q++ = *p++;
|
| 853 | }
|
| 854 |
|
| 855 | *(s + 1) = '\0';
|
| 856 | }
|
| 857 |
|
| 858 | void cpu_detect_cache_sizes(struct cpuinfo_x86 *c)
|
| 859 | {
|
| 860 | unsigned int n, dummy, ebx, ecx, edx, l2size, shift __maybe_unused;
|
| 861 |
|
| 862 | n = c->extended_cpuid_level;
|
| 863 |
|
| 864 | if (n >= 0x80000005) {
|
| 865 | cpuid(0x80000005, &dummy, &ebx, &ecx, &edx);
|
| 866 | c->x86_cache_size = (ecx>>24) + (edx>>24);
|
| 867 | #ifdef CONFIG_X86_64
|
| 868 |
|
| 869 | c->x86_tlbsize = 0;
|
| 870 | #endif
|
| 871 | }
|
| 872 |
|
| 873 | if (n < 0x80000006)
|
| 874 | return;
|
| 875 |
|
| 876 | cpuid(0x80000006, &dummy, &ebx, &ecx, &edx);
|
| 877 | l2size = ecx >> 16;
|
| 878 |
|
| 879 | #ifdef CONFIG_X86_64
|
| 880 | shift = !!cpu_has(c, X86_FEATURE_L2_TLB_SIZE_X32) * 5;
|
| 881 | c->x86_tlbsize += ((ebx >> 16) & 0xfff) + (ebx & 0xfff);
|
| 882 | c->x86_tlbsize <<= shift;
|
| 883 | #else
|
| 884 |
|
| 885 | if (this_cpu->legacy_cache_size)
|
| 886 | l2size = this_cpu->legacy_cache_size(c, l2size);
|
| 887 |
|
| 888 |
|
| 889 | if (cachesize_override != -1)
|
| 890 | l2size = cachesize_override;
|
| 891 |
|
| 892 | if (l2size == 0)
|
| 893 | return;
|
| 894 | #endif
|
| 895 |
|
| 896 | c->x86_cache_size = l2size;
|
| 897 | }
|
| 898 |
|
| 899 | u16 __read_mostly tlb_lli_4k;
|
| 900 | u16 __read_mostly tlb_lli_2m;
|
| 901 | u16 __read_mostly tlb_lli_4m;
|
| 902 | u16 __read_mostly tlb_lld_4k;
|
| 903 | u16 __read_mostly tlb_lld_2m;
|
| 904 | u16 __read_mostly tlb_lld_4m;
|
| 905 | u16 __read_mostly tlb_lld_1g;
|
| 906 |
|
| 907 | static void cpu_detect_tlb(struct cpuinfo_x86 *c)
|
| 908 | {
|
| 909 | if (this_cpu->c_detect_tlb)
|
| 910 | this_cpu->c_detect_tlb(c);
|
| 911 |
|
| 912 | pr_info("Last level iTLB entries: 4KB %d, 2MB %d, 4MB %d\n",
|
| 913 | tlb_lli_4k, tlb_lli_2m, tlb_lli_4m);
|
| 914 |
|
| 915 | pr_info("Last level dTLB entries: 4KB %d, 2MB %d, 4MB %d, 1GB %d\n",
|
| 916 | tlb_lld_4k, tlb_lld_2m, tlb_lld_4m, tlb_lld_1g);
|
| 917 | }
|
| 918 |
|
| 919 | void get_cpu_vendor(struct cpuinfo_x86 *c)
|
| 920 | {
|
| 921 | char *v = c->x86_vendor_id;
|
| 922 | int i;
|
| 923 |
|
| 924 | for (i = 0; i < X86_VENDOR_NUM; i++) {
|
| 925 | if (!cpu_devs[i])
|
| 926 | break;
|
| 927 |
|
| 928 | if (!strcmp(v, cpu_devs[i]->c_ident[0]) ||
|
| 929 | (cpu_devs[i]->c_ident[1] &&
|
| 930 | !strcmp(v, cpu_devs[i]->c_ident[1]))) {
|
| 931 |
|
| 932 | this_cpu = cpu_devs[i];
|
| 933 | c->x86_vendor = this_cpu->c_x86_vendor;
|
| 934 | return;
|
| 935 | }
|
| 936 | }
|
| 937 |
|
| 938 | pr_err_once("CPU: vendor_id '%s' unknown, using generic init.\n" \
|
| 939 | "CPU: Your system may be unstable.\n", v);
|
| 940 |
|
| 941 | c->x86_vendor = X86_VENDOR_UNKNOWN;
|
| 942 | this_cpu = &default_cpu;
|
| 943 | }
|
| 944 |
|
| 945 | void cpu_detect(struct cpuinfo_x86 *c)
|
| 946 | {
|
| 947 | const struct leaf_0x0_0 *l0 = cpuid_leaf(c, 0x0);
|
| 948 | const struct leaf_0x1_0 *l1;
|
| 949 |
|
| 950 | c->cpuid_level = l0->max_std_leaf;
|
| 951 | *(u32 *)&c->x86_vendor_id[0] = l0->cpu_vendorid_0;
|
| 952 | *(u32 *)&c->x86_vendor_id[4] = l0->cpu_vendorid_1;
|
| 953 | *(u32 *)&c->x86_vendor_id[8] = l0->cpu_vendorid_2;
|
| 954 |
|
| 955 | c->x86 = 4;
|
| 956 |
|
| 957 | l1 = cpuid_leaf(c, 0x1);
|
| 958 | if (!l1)
|
| 959 | return;
|
| 960 |
|
| 961 | c->x86 = cpuid_family(l1);
|
| 962 | c->x86_model = cpuid_model(l1);
|
| 963 | c->x86_stepping = l1->stepping;
|
| 964 |
|
| 965 | if (l1->clflush) {
|
| 966 | c->x86_clflush_size = l1->clflush_size * 8;
|
| 967 | c->x86_cache_alignment = c->x86_clflush_size;
|
| 968 | }
|
| 969 | }
|
| 970 |
|
| 971 | static void apply_forced_caps(struct cpuinfo_x86 *c)
|
| 972 | {
|
| 973 | int i;
|
| 974 |
|
| 975 | for (i = 0; i < NCAPINTS + NBUGINTS; i++) {
|
| 976 | c->x86_capability[i] &= ~cpu_caps_cleared[i];
|
| 977 | c->x86_capability[i] |= cpu_caps_set[i];
|
| 978 | }
|
| 979 | }
|
| 980 |
|
| 981 | static void init_speculation_control(struct cpuinfo_x86 *c)
|
| 982 | {
|
| 983 | |
| 984 | * The Intel SPEC_CTRL CPUID bit implies IBRS and IBPB support,
|
| 985 | * and they also have a different bit for STIBP support. Also,
|
| 986 | * a hypervisor might have set the individual AMD bits even on
|
| 987 | * Intel CPUs, for finer-grained selection of what's available.
|
| 988 |
|
| 989 | if (cpu_has(c, X86_FEATURE_SPEC_CTRL)) {
|
| 990 | set_cpu_cap(c, X86_FEATURE_IBRS);
|
| 991 | set_cpu_cap(c, X86_FEATURE_IBPB);
|
| 992 | set_cpu_cap(c, X86_FEATURE_MSR_SPEC_CTRL);
|
| 993 | }
|
| 994 |
|
| 995 | if (cpu_has(c, X86_FEATURE_INTEL_STIBP))
|
| 996 | set_cpu_cap(c, X86_FEATURE_STIBP);
|
| 997 |
|
| 998 | if (cpu_has(c, X86_FEATURE_SPEC_CTRL_SSBD) ||
|
| 999 | cpu_has(c, X86_FEATURE_VIRT_SSBD))
|
| 1000 | set_cpu_cap(c, X86_FEATURE_SSBD);
|
| 1001 |
|
| 1002 | if (cpu_has(c, X86_FEATURE_AMD_IBRS)) {
|
| 1003 | set_cpu_cap(c, X86_FEATURE_IBRS);
|
| 1004 | set_cpu_cap(c, X86_FEATURE_MSR_SPEC_CTRL);
|
| 1005 | }
|
| 1006 |
|
| 1007 | if (cpu_has(c, X86_FEATURE_AMD_IBPB))
|
| 1008 | set_cpu_cap(c, X86_FEATURE_IBPB);
|
| 1009 |
|
| 1010 | if (cpu_has(c, X86_FEATURE_AMD_STIBP)) {
|
| 1011 | set_cpu_cap(c, X86_FEATURE_STIBP);
|
| 1012 | set_cpu_cap(c, X86_FEATURE_MSR_SPEC_CTRL);
|
| 1013 | }
|
| 1014 |
|
| 1015 | if (cpu_has(c, X86_FEATURE_AMD_SSBD)) {
|
| 1016 | set_cpu_cap(c, X86_FEATURE_SSBD);
|
| 1017 | set_cpu_cap(c, X86_FEATURE_MSR_SPEC_CTRL);
|
| 1018 | clear_cpu_cap(c, X86_FEATURE_VIRT_SSBD);
|
| 1019 | }
|
| 1020 | }
|
| 1021 |
|
| 1022 | void get_cpu_cap(struct cpuinfo_x86 *c)
|
| 1023 | {
|
| 1024 | u32 eax, ebx, ecx, edx;
|
| 1025 |
|
| 1026 |
|
| 1027 | if (c->cpuid_level >= 0x00000001) {
|
| 1028 | cpuid(0x00000001, &eax, &ebx, &ecx, &edx);
|
| 1029 |
|
| 1030 | c->x86_capability[CPUID_1_ECX] = ecx;
|
| 1031 | c->x86_capability[CPUID_1_EDX] = edx;
|
| 1032 | }
|
| 1033 |
|
| 1034 |
|
| 1035 | if (c->cpuid_level >= 0x00000006)
|
| 1036 | c->x86_capability[CPUID_6_EAX] = cpuid_eax(0x00000006);
|
| 1037 |
|
| 1038 |
|
| 1039 | if (c->cpuid_level >= 0x00000007) {
|
| 1040 | cpuid_count(0x00000007, 0, &eax, &ebx, &ecx, &edx);
|
| 1041 | c->x86_capability[CPUID_7_0_EBX] = ebx;
|
| 1042 | c->x86_capability[CPUID_7_ECX] = ecx;
|
| 1043 | c->x86_capability[CPUID_7_EDX] = edx;
|
| 1044 |
|
| 1045 |
|
| 1046 | if (eax >= 1) {
|
| 1047 | cpuid_count(0x00000007, 1, &eax, &ebx, &ecx, &edx);
|
| 1048 | c->x86_capability[CPUID_7_1_EAX] = eax;
|
| 1049 | }
|
| 1050 | }
|
| 1051 |
|
| 1052 |
|
| 1053 | if (c->cpuid_level >= 0x0000000d) {
|
| 1054 | cpuid_count(0x0000000d, 1, &eax, &ebx, &ecx, &edx);
|
| 1055 |
|
| 1056 | c->x86_capability[CPUID_D_1_EAX] = eax;
|
| 1057 | }
|
| 1058 |
|
| 1059 | |
| 1060 | * Check if extended CPUID leaves are implemented: Max extended
|
| 1061 | * CPUID leaf must be in the 0x80000001-0x8000ffff range.
|
| 1062 |
|
| 1063 | eax = cpuid_eax(0x80000000);
|
| 1064 | c->extended_cpuid_level = ((eax & 0xffff0000) == 0x80000000) ? eax : 0;
|
| 1065 |
|
| 1066 | if (c->extended_cpuid_level >= 0x80000001) {
|
| 1067 | cpuid(0x80000001, &eax, &ebx, &ecx, &edx);
|
| 1068 |
|
| 1069 | c->x86_capability[CPUID_8000_0001_ECX] = ecx;
|
| 1070 | c->x86_capability[CPUID_8000_0001_EDX] = edx;
|
| 1071 | }
|
| 1072 |
|
| 1073 | if (c->extended_cpuid_level >= 0x80000007)
|
| 1074 | c->x86_power = cpuid_edx(0x80000007);
|
| 1075 |
|
| 1076 | if (c->extended_cpuid_level >= 0x80000008) {
|
| 1077 | cpuid(0x80000008, &eax, &ebx, &ecx, &edx);
|
| 1078 | c->x86_capability[CPUID_8000_0008_EBX] = ebx;
|
| 1079 | }
|
| 1080 |
|
| 1081 | if (c->extended_cpuid_level >= 0x8000000a)
|
| 1082 | c->x86_capability[CPUID_8000_000A_EDX] = cpuid_edx(0x8000000a);
|
| 1083 |
|
| 1084 | if (c->extended_cpuid_level >= 0x8000001f)
|
| 1085 | c->x86_capability[CPUID_8000_001F_EAX] = cpuid_eax(0x8000001f);
|
| 1086 |
|
| 1087 | if (c->extended_cpuid_level >= 0x80000021)
|
| 1088 | c->x86_capability[CPUID_8000_0021_EAX] = cpuid_eax(0x80000021);
|
| 1089 |
|
| 1090 | init_scattered_cpuid_features(c);
|
| 1091 | init_speculation_control(c);
|
| 1092 |
|
| 1093 | if (IS_ENABLED(CONFIG_X86_64) || cpu_has(c, X86_FEATURE_SEP))
|
| 1094 | set_cpu_cap(c, X86_FEATURE_SYSFAST32);
|
| 1095 |
|
| 1096 | |
| 1097 | * Clear/Set all flags overridden by options, after probe.
|
| 1098 | * This needs to happen each time we re-probe, which may happen
|
| 1099 | * several times during CPU initialization.
|
| 1100 |
|
| 1101 | apply_forced_caps(c);
|
| 1102 | }
|
| 1103 |
|
| 1104 | void get_cpu_address_sizes(struct cpuinfo_x86 *c)
|
| 1105 | {
|
| 1106 | u32 eax, ebx, ecx, edx;
|
| 1107 |
|
| 1108 | if (!cpu_has(c, X86_FEATURE_CPUID) ||
|
| 1109 | (c->extended_cpuid_level < 0x80000008)) {
|
| 1110 | if (IS_ENABLED(CONFIG_X86_64)) {
|
| 1111 | c->x86_clflush_size = 64;
|
| 1112 | c->x86_phys_bits = 36;
|
| 1113 | c->x86_virt_bits = 48;
|
| 1114 | } else {
|
| 1115 | c->x86_clflush_size = 32;
|
| 1116 | c->x86_virt_bits = 32;
|
| 1117 | c->x86_phys_bits = 32;
|
| 1118 |
|
| 1119 | if (cpu_has(c, X86_FEATURE_PAE) ||
|
| 1120 | cpu_has(c, X86_FEATURE_PSE36))
|
| 1121 | c->x86_phys_bits = 36;
|
| 1122 | }
|
| 1123 | } else {
|
| 1124 | cpuid(0x80000008, &eax, &ebx, &ecx, &edx);
|
| 1125 |
|
| 1126 | c->x86_virt_bits = (eax >> 8) & 0xff;
|
| 1127 | c->x86_phys_bits = eax & 0xff;
|
| 1128 |
|
| 1129 |
|
| 1130 | if (!c->x86_clflush_size)
|
| 1131 | c->x86_clflush_size = 32;
|
| 1132 | }
|
| 1133 |
|
| 1134 | c->x86_cache_bits = c->x86_phys_bits;
|
| 1135 | c->x86_cache_alignment = c->x86_clflush_size;
|
| 1136 | }
|
| 1137 |
|
| 1138 | static void identify_cpu_without_cpuid(struct cpuinfo_x86 *c)
|
| 1139 | {
|
| 1140 | int i;
|
| 1141 |
|
| 1142 | |
| 1143 | * First of all, decide if this is a 486 or higher
|
| 1144 | * It's a 486 if we can modify the AC flag
|
| 1145 |
|
| 1146 | if (flag_is_changeable_p(X86_EFLAGS_AC))
|
| 1147 | c->x86 = 4;
|
| 1148 | else
|
| 1149 | c->x86 = 3;
|
| 1150 |
|
| 1151 | for (i = 0; i < X86_VENDOR_NUM; i++)
|
| 1152 | if (cpu_devs[i] && cpu_devs[i]->c_identify) {
|
| 1153 | c->x86_vendor_id[0] = 0;
|
| 1154 | cpu_devs[i]->c_identify(c);
|
| 1155 | if (c->x86_vendor_id[0]) {
|
| 1156 | get_cpu_vendor(c);
|
| 1157 | break;
|
| 1158 | }
|
| 1159 | }
|
| 1160 | }
|
| 1161 |
|
| 1162 | #define NO_SPECULATION BIT(0)
|
| 1163 | #define NO_MELTDOWN BIT(1)
|
| 1164 | #define NO_SSB BIT(2)
|
| 1165 | #define NO_L1TF BIT(3)
|
| 1166 | #define NO_MDS BIT(4)
|
| 1167 | #define MSBDS_ONLY BIT(5)
|
| 1168 | #define NO_SWAPGS BIT(6)
|
| 1169 | #define NO_ITLB_MULTIHIT BIT(7)
|
| 1170 | #define NO_SPECTRE_V2 BIT(8)
|
| 1171 | #define NO_MMIO BIT(9)
|
| 1172 | #define NO_EIBRS_PBRSB BIT(10)
|
| 1173 | #define NO_BHI BIT(11)
|
| 1174 |
|
| 1175 | #define VULNWL(vendor, family, model, whitelist) \
|
| 1176 | X86_MATCH_VENDOR_FAM_MODEL(vendor, family, model, whitelist)
|
| 1177 |
|
| 1178 | #define VULNWL_INTEL(vfm, whitelist) \
|
| 1179 | X86_MATCH_VFM(vfm, whitelist)
|
| 1180 |
|
| 1181 | #define VULNWL_AMD(family, whitelist) \
|
| 1182 | VULNWL(AMD, family, X86_MODEL_ANY, whitelist)
|
| 1183 |
|
| 1184 | #define VULNWL_HYGON(family, whitelist) \
|
| 1185 | VULNWL(HYGON, family, X86_MODEL_ANY, whitelist)
|
| 1186 |
|
| 1187 | static const __initconst struct x86_cpu_id cpu_vuln_whitelist[] = {
|
| 1188 | VULNWL(ANY, 4, X86_MODEL_ANY, NO_SPECULATION),
|
| 1189 | VULNWL(CENTAUR, 5, X86_MODEL_ANY, NO_SPECULATION),
|
| 1190 | VULNWL(INTEL, 5, X86_MODEL_ANY, NO_SPECULATION),
|
| 1191 | VULNWL(NSC, 5, X86_MODEL_ANY, NO_SPECULATION),
|
| 1192 | VULNWL(VORTEX, 5, X86_MODEL_ANY, NO_SPECULATION),
|
| 1193 | VULNWL(VORTEX, 6, X86_MODEL_ANY, NO_SPECULATION),
|
| 1194 |
|
| 1195 |
|
| 1196 | VULNWL_INTEL(INTEL_TIGERLAKE, NO_MMIO),
|
| 1197 | VULNWL_INTEL(INTEL_TIGERLAKE_L, NO_MMIO),
|
| 1198 | VULNWL_INTEL(INTEL_ALDERLAKE, NO_MMIO),
|
| 1199 | VULNWL_INTEL(INTEL_ALDERLAKE_L, NO_MMIO),
|
| 1200 |
|
| 1201 | VULNWL_INTEL(INTEL_ATOM_SALTWELL, NO_SPECULATION | NO_ITLB_MULTIHIT),
|
| 1202 | VULNWL_INTEL(INTEL_ATOM_SALTWELL_TABLET, NO_SPECULATION | NO_ITLB_MULTIHIT),
|
| 1203 | VULNWL_INTEL(INTEL_ATOM_SALTWELL_MID, NO_SPECULATION | NO_ITLB_MULTIHIT),
|
| 1204 | VULNWL_INTEL(INTEL_ATOM_BONNELL, NO_SPECULATION | NO_ITLB_MULTIHIT),
|
| 1205 | VULNWL_INTEL(INTEL_ATOM_BONNELL_MID, NO_SPECULATION | NO_ITLB_MULTIHIT),
|
| 1206 |
|
| 1207 | VULNWL_INTEL(INTEL_ATOM_SILVERMONT, NO_SSB | NO_L1TF | MSBDS_ONLY | NO_SWAPGS | NO_ITLB_MULTIHIT),
|
| 1208 | VULNWL_INTEL(INTEL_ATOM_SILVERMONT_D, NO_SSB | NO_L1TF | MSBDS_ONLY | NO_SWAPGS | NO_ITLB_MULTIHIT),
|
| 1209 | VULNWL_INTEL(INTEL_ATOM_SILVERMONT_MID, NO_SSB | NO_L1TF | MSBDS_ONLY | NO_SWAPGS | NO_ITLB_MULTIHIT),
|
| 1210 | VULNWL_INTEL(INTEL_ATOM_AIRMONT, NO_SSB | NO_L1TF | MSBDS_ONLY | NO_SWAPGS | NO_ITLB_MULTIHIT),
|
| 1211 | VULNWL_INTEL(INTEL_XEON_PHI_KNL, NO_SSB | NO_L1TF | MSBDS_ONLY | NO_SWAPGS | NO_ITLB_MULTIHIT),
|
| 1212 | VULNWL_INTEL(INTEL_XEON_PHI_KNM, NO_SSB | NO_L1TF | MSBDS_ONLY | NO_SWAPGS | NO_ITLB_MULTIHIT),
|
| 1213 |
|
| 1214 | VULNWL_INTEL(INTEL_CORE_YONAH, NO_SSB),
|
| 1215 |
|
| 1216 | VULNWL_INTEL(INTEL_ATOM_SILVERMONT_MID2,NO_SSB | NO_L1TF | NO_SWAPGS | NO_ITLB_MULTIHIT | MSBDS_ONLY),
|
| 1217 | VULNWL_INTEL(INTEL_ATOM_AIRMONT_NP, NO_SSB | NO_L1TF | NO_SWAPGS | NO_ITLB_MULTIHIT),
|
| 1218 |
|
| 1219 | VULNWL_INTEL(INTEL_ATOM_GOLDMONT, NO_MDS | NO_L1TF | NO_SWAPGS | NO_ITLB_MULTIHIT | NO_MMIO),
|
| 1220 | VULNWL_INTEL(INTEL_ATOM_GOLDMONT_D, NO_MDS | NO_L1TF | NO_SWAPGS | NO_ITLB_MULTIHIT | NO_MMIO),
|
| 1221 | VULNWL_INTEL(INTEL_ATOM_GOLDMONT_PLUS, NO_MDS | NO_L1TF | NO_SWAPGS | NO_ITLB_MULTIHIT | NO_MMIO | NO_EIBRS_PBRSB),
|
| 1222 |
|
| 1223 | |
| 1224 | * Technically, swapgs isn't serializing on AMD (despite it previously
|
| 1225 | * being documented as such in the APM). But according to AMD, %gs is
|
| 1226 | * updated non-speculatively, and the issuing of %gs-relative memory
|
| 1227 | * operands will be blocked until the %gs update completes, which is
|
| 1228 | * good enough for our purposes.
|
| 1229 |
|
| 1230 |
|
| 1231 | VULNWL_INTEL(INTEL_ATOM_TREMONT, NO_EIBRS_PBRSB),
|
| 1232 | VULNWL_INTEL(INTEL_ATOM_TREMONT_L, NO_EIBRS_PBRSB),
|
| 1233 | VULNWL_INTEL(INTEL_ATOM_TREMONT_D, NO_ITLB_MULTIHIT | NO_EIBRS_PBRSB),
|
| 1234 |
|
| 1235 |
|
| 1236 | VULNWL_AMD(0x0f, NO_MELTDOWN | NO_SSB | NO_L1TF | NO_MDS | NO_SWAPGS | NO_ITLB_MULTIHIT | NO_MMIO | NO_BHI),
|
| 1237 | VULNWL_AMD(0x10, NO_MELTDOWN | NO_SSB | NO_L1TF | NO_MDS | NO_SWAPGS | NO_ITLB_MULTIHIT | NO_MMIO | NO_BHI),
|
| 1238 | VULNWL_AMD(0x11, NO_MELTDOWN | NO_SSB | NO_L1TF | NO_MDS | NO_SWAPGS | NO_ITLB_MULTIHIT | NO_MMIO | NO_BHI),
|
| 1239 | VULNWL_AMD(0x12, NO_MELTDOWN | NO_SSB | NO_L1TF | NO_MDS | NO_SWAPGS | NO_ITLB_MULTIHIT | NO_MMIO | NO_BHI),
|
| 1240 |
|
| 1241 |
|
| 1242 | VULNWL_AMD(X86_FAMILY_ANY, NO_MELTDOWN | NO_L1TF | NO_MDS | NO_SWAPGS | NO_ITLB_MULTIHIT | NO_MMIO | NO_EIBRS_PBRSB | NO_BHI),
|
| 1243 | VULNWL_HYGON(X86_FAMILY_ANY, NO_MELTDOWN | NO_L1TF | NO_MDS | NO_SWAPGS | NO_ITLB_MULTIHIT | NO_MMIO | NO_EIBRS_PBRSB | NO_BHI),
|
| 1244 |
|
| 1245 |
|
| 1246 | VULNWL(CENTAUR, 7, X86_MODEL_ANY, NO_SPECTRE_V2 | NO_SWAPGS | NO_MMIO | NO_BHI),
|
| 1247 | VULNWL(ZHAOXIN, 7, X86_MODEL_ANY, NO_SPECTRE_V2 | NO_SWAPGS | NO_MMIO | NO_BHI),
|
| 1248 | {}
|
| 1249 | };
|
| 1250 |
|
| 1251 | #define VULNBL(vendor, family, model, blacklist) \
|
| 1252 | X86_MATCH_VENDOR_FAM_MODEL(vendor, family, model, blacklist)
|
| 1253 |
|
| 1254 | #define VULNBL_INTEL_STEPS(vfm, max_stepping, issues) \
|
| 1255 | X86_MATCH_VFM_STEPS(vfm, X86_STEP_MIN, max_stepping, issues)
|
| 1256 |
|
| 1257 | #define VULNBL_AMD(family, blacklist) \
|
| 1258 | VULNBL(AMD, family, X86_MODEL_ANY, blacklist)
|
| 1259 |
|
| 1260 | #define VULNBL_HYGON(family, blacklist) \
|
| 1261 | VULNBL(HYGON, family, X86_MODEL_ANY, blacklist)
|
| 1262 |
|
| 1263 | #define SRBDS BIT(0)
|
| 1264 |
|
| 1265 | #define MMIO BIT(1)
|
| 1266 |
|
| 1267 | #define MMIO_SBDS BIT(2)
|
| 1268 |
|
| 1269 | #define RETBLEED BIT(3)
|
| 1270 |
|
| 1271 | #define SMT_RSB BIT(4)
|
| 1272 |
|
| 1273 | #define SRSO BIT(5)
|
| 1274 |
|
| 1275 | #define GDS BIT(6)
|
| 1276 |
|
| 1277 | #define RFDS BIT(7)
|
| 1278 |
|
| 1279 | #define ITS BIT(8)
|
| 1280 |
|
| 1281 | #define ITS_NATIVE_ONLY BIT(9)
|
| 1282 |
|
| 1283 | #define TSA BIT(10)
|
| 1284 |
|
| 1285 | #define VMSCAPE BIT(11)
|
| 1286 |
|
| 1287 | static const struct x86_cpu_id cpu_vuln_blacklist[] __initconst = {
|
| 1288 | VULNBL_INTEL_STEPS(INTEL_SANDYBRIDGE_X, X86_STEP_MAX, VMSCAPE),
|
| 1289 | VULNBL_INTEL_STEPS(INTEL_SANDYBRIDGE, X86_STEP_MAX, VMSCAPE),
|
| 1290 | VULNBL_INTEL_STEPS(INTEL_IVYBRIDGE_X, X86_STEP_MAX, VMSCAPE),
|
| 1291 | VULNBL_INTEL_STEPS(INTEL_IVYBRIDGE, X86_STEP_MAX, SRBDS | VMSCAPE),
|
| 1292 | VULNBL_INTEL_STEPS(INTEL_HASWELL, X86_STEP_MAX, SRBDS | VMSCAPE),
|
| 1293 | VULNBL_INTEL_STEPS(INTEL_HASWELL_L, X86_STEP_MAX, SRBDS | VMSCAPE),
|
| 1294 | VULNBL_INTEL_STEPS(INTEL_HASWELL_G, X86_STEP_MAX, SRBDS | VMSCAPE),
|
| 1295 | VULNBL_INTEL_STEPS(INTEL_HASWELL_X, X86_STEP_MAX, MMIO | VMSCAPE),
|
| 1296 | VULNBL_INTEL_STEPS(INTEL_BROADWELL_D, X86_STEP_MAX, MMIO | VMSCAPE),
|
| 1297 | VULNBL_INTEL_STEPS(INTEL_BROADWELL_X, X86_STEP_MAX, MMIO | VMSCAPE),
|
| 1298 | VULNBL_INTEL_STEPS(INTEL_BROADWELL_G, X86_STEP_MAX, SRBDS | VMSCAPE),
|
| 1299 | VULNBL_INTEL_STEPS(INTEL_BROADWELL, X86_STEP_MAX, SRBDS | VMSCAPE),
|
| 1300 | VULNBL_INTEL_STEPS(INTEL_SKYLAKE_X, 0x5, MMIO | RETBLEED | GDS | VMSCAPE),
|
| 1301 | VULNBL_INTEL_STEPS(INTEL_SKYLAKE_X, X86_STEP_MAX, MMIO | RETBLEED | GDS | ITS | VMSCAPE),
|
| 1302 | VULNBL_INTEL_STEPS(INTEL_SKYLAKE_L, X86_STEP_MAX, MMIO | RETBLEED | GDS | SRBDS | VMSCAPE),
|
| 1303 | VULNBL_INTEL_STEPS(INTEL_SKYLAKE, X86_STEP_MAX, MMIO | RETBLEED | GDS | SRBDS | VMSCAPE),
|
| 1304 | VULNBL_INTEL_STEPS(INTEL_KABYLAKE_L, 0xb, MMIO | RETBLEED | GDS | SRBDS | VMSCAPE),
|
| 1305 | VULNBL_INTEL_STEPS(INTEL_KABYLAKE_L, X86_STEP_MAX, MMIO | RETBLEED | GDS | SRBDS | ITS | VMSCAPE),
|
| 1306 | VULNBL_INTEL_STEPS(INTEL_KABYLAKE, 0xc, MMIO | RETBLEED | GDS | SRBDS | VMSCAPE),
|
| 1307 | VULNBL_INTEL_STEPS(INTEL_KABYLAKE, X86_STEP_MAX, MMIO | RETBLEED | GDS | SRBDS | ITS | VMSCAPE),
|
| 1308 | VULNBL_INTEL_STEPS(INTEL_CANNONLAKE_L, X86_STEP_MAX, RETBLEED | VMSCAPE),
|
| 1309 | VULNBL_INTEL_STEPS(INTEL_ICELAKE_L, X86_STEP_MAX, MMIO | MMIO_SBDS | RETBLEED | GDS | ITS | ITS_NATIVE_ONLY),
|
| 1310 | VULNBL_INTEL_STEPS(INTEL_ICELAKE_D, X86_STEP_MAX, MMIO | GDS | ITS | ITS_NATIVE_ONLY),
|
| 1311 | VULNBL_INTEL_STEPS(INTEL_ICELAKE_X, X86_STEP_MAX, MMIO | GDS | ITS | ITS_NATIVE_ONLY),
|
| 1312 | VULNBL_INTEL_STEPS(INTEL_COMETLAKE, X86_STEP_MAX, MMIO | MMIO_SBDS | RETBLEED | GDS | ITS | VMSCAPE),
|
| 1313 | VULNBL_INTEL_STEPS(INTEL_COMETLAKE_L, 0x0, MMIO | RETBLEED | ITS | VMSCAPE),
|
| 1314 | VULNBL_INTEL_STEPS(INTEL_COMETLAKE_L, X86_STEP_MAX, MMIO | MMIO_SBDS | RETBLEED | GDS | ITS | VMSCAPE),
|
| 1315 | VULNBL_INTEL_STEPS(INTEL_TIGERLAKE_L, X86_STEP_MAX, GDS | ITS | ITS_NATIVE_ONLY),
|
| 1316 | VULNBL_INTEL_STEPS(INTEL_TIGERLAKE, X86_STEP_MAX, GDS | ITS | ITS_NATIVE_ONLY),
|
| 1317 | VULNBL_INTEL_STEPS(INTEL_LAKEFIELD, X86_STEP_MAX, MMIO | MMIO_SBDS | RETBLEED),
|
| 1318 | VULNBL_INTEL_STEPS(INTEL_ROCKETLAKE, X86_STEP_MAX, MMIO | RETBLEED | GDS | ITS | ITS_NATIVE_ONLY),
|
| 1319 | VULNBL_INTEL_STEPS(INTEL_ALDERLAKE, X86_STEP_MAX, RFDS | VMSCAPE),
|
| 1320 | VULNBL_INTEL_STEPS(INTEL_ALDERLAKE_L, X86_STEP_MAX, RFDS | VMSCAPE),
|
| 1321 | VULNBL_INTEL_STEPS(INTEL_RAPTORLAKE, X86_STEP_MAX, RFDS | VMSCAPE),
|
| 1322 | VULNBL_INTEL_STEPS(INTEL_RAPTORLAKE_P, X86_STEP_MAX, RFDS | VMSCAPE),
|
| 1323 | VULNBL_INTEL_STEPS(INTEL_RAPTORLAKE_S, X86_STEP_MAX, RFDS | VMSCAPE),
|
| 1324 | VULNBL_INTEL_STEPS(INTEL_METEORLAKE_L, X86_STEP_MAX, VMSCAPE),
|
| 1325 | VULNBL_INTEL_STEPS(INTEL_ARROWLAKE_H, X86_STEP_MAX, VMSCAPE),
|
| 1326 | VULNBL_INTEL_STEPS(INTEL_ARROWLAKE, X86_STEP_MAX, VMSCAPE),
|
| 1327 | VULNBL_INTEL_STEPS(INTEL_ARROWLAKE_U, X86_STEP_MAX, VMSCAPE),
|
| 1328 | VULNBL_INTEL_STEPS(INTEL_LUNARLAKE_M, X86_STEP_MAX, VMSCAPE),
|
| 1329 | VULNBL_INTEL_STEPS(INTEL_SAPPHIRERAPIDS_X, X86_STEP_MAX, VMSCAPE),
|
| 1330 | VULNBL_INTEL_STEPS(INTEL_GRANITERAPIDS_X, X86_STEP_MAX, VMSCAPE),
|
| 1331 | VULNBL_INTEL_STEPS(INTEL_EMERALDRAPIDS_X, X86_STEP_MAX, VMSCAPE),
|
| 1332 | VULNBL_INTEL_STEPS(INTEL_ATOM_GRACEMONT, X86_STEP_MAX, RFDS | VMSCAPE),
|
| 1333 | VULNBL_INTEL_STEPS(INTEL_ATOM_TREMONT, X86_STEP_MAX, MMIO | MMIO_SBDS | RFDS),
|
| 1334 | VULNBL_INTEL_STEPS(INTEL_ATOM_TREMONT_D, X86_STEP_MAX, MMIO | RFDS),
|
| 1335 | VULNBL_INTEL_STEPS(INTEL_ATOM_TREMONT_L, X86_STEP_MAX, MMIO | MMIO_SBDS | RFDS),
|
| 1336 | VULNBL_INTEL_STEPS(INTEL_ATOM_GOLDMONT, X86_STEP_MAX, RFDS),
|
| 1337 | VULNBL_INTEL_STEPS(INTEL_ATOM_GOLDMONT_D, X86_STEP_MAX, RFDS),
|
| 1338 | VULNBL_INTEL_STEPS(INTEL_ATOM_GOLDMONT_PLUS, X86_STEP_MAX, RFDS),
|
| 1339 | VULNBL_INTEL_STEPS(INTEL_ATOM_CRESTMONT_X, X86_STEP_MAX, VMSCAPE),
|
| 1340 |
|
| 1341 | VULNBL_AMD(0x15, RETBLEED),
|
| 1342 | VULNBL_AMD(0x16, RETBLEED),
|
| 1343 | VULNBL_AMD(0x17, RETBLEED | SMT_RSB | SRSO | VMSCAPE),
|
| 1344 | VULNBL_HYGON(0x18, RETBLEED | SMT_RSB | SRSO | VMSCAPE),
|
| 1345 | VULNBL_AMD(0x19, SRSO | TSA | VMSCAPE),
|
| 1346 | VULNBL_AMD(0x1a, SRSO | VMSCAPE),
|
| 1347 | {}
|
| 1348 | };
|
| 1349 |
|
| 1350 | static bool __init cpu_matches(const struct x86_cpu_id *table, unsigned long which)
|
| 1351 | {
|
| 1352 | const struct x86_cpu_id *m = x86_match_cpu(table);
|
| 1353 |
|
| 1354 | return m && !!(m->driver_data & which);
|
| 1355 | }
|
| 1356 |
|
| 1357 | u64 x86_read_arch_cap_msr(void)
|
| 1358 | {
|
| 1359 | u64 x86_arch_cap_msr = 0;
|
| 1360 |
|
| 1361 | if (boot_cpu_has(X86_FEATURE_ARCH_CAPABILITIES))
|
| 1362 | rdmsrq(MSR_IA32_ARCH_CAPABILITIES, x86_arch_cap_msr);
|
| 1363 |
|
| 1364 | return x86_arch_cap_msr;
|
| 1365 | }
|
| 1366 |
|
| 1367 | static bool arch_cap_mmio_immune(u64 x86_arch_cap_msr)
|
| 1368 | {
|
| 1369 | return (x86_arch_cap_msr & ARCH_CAP_FBSDP_NO &&
|
| 1370 | x86_arch_cap_msr & ARCH_CAP_PSDP_NO &&
|
| 1371 | x86_arch_cap_msr & ARCH_CAP_SBDR_SSDP_NO);
|
| 1372 | }
|
| 1373 |
|
| 1374 | static bool __init vulnerable_to_rfds(u64 x86_arch_cap_msr)
|
| 1375 | {
|
| 1376 |
|
| 1377 | if (x86_arch_cap_msr & ARCH_CAP_RFDS_NO)
|
| 1378 | return false;
|
| 1379 |
|
| 1380 | |
| 1381 | * VMMs set ARCH_CAP_RFDS_CLEAR for processors not in the blacklist to
|
| 1382 | * indicate that mitigation is needed because guest is running on a
|
| 1383 | * vulnerable hardware or may migrate to such hardware:
|
| 1384 |
|
| 1385 | if (x86_arch_cap_msr & ARCH_CAP_RFDS_CLEAR)
|
| 1386 | return true;
|
| 1387 |
|
| 1388 |
|
| 1389 | if (!cpu_matches(cpu_vuln_blacklist, RFDS))
|
| 1390 | return false;
|
| 1391 |
|
| 1392 | |
| 1393 | * ADL and RPL are affected only if they have Atom CPUs. Hybrids have
|
| 1394 | * both Core and Atom CPUs. Mark unaffected when Atom CPUs are not
|
| 1395 | * present.
|
| 1396 |
|
| 1397 | if ((boot_cpu_data.x86_model == 0x97 ||
|
| 1398 | boot_cpu_data.x86_model == 0xB7) &&
|
| 1399 | boot_cpu_data.topo.intel_type != INTEL_CPU_TYPE_ATOM &&
|
| 1400 | !boot_cpu_has(X86_FEATURE_HYBRID_CPU))
|
| 1401 | return false;
|
| 1402 |
|
| 1403 | return true;
|
| 1404 | }
|
| 1405 |
|
| 1406 | static bool __init vulnerable_to_its(u64 x86_arch_cap_msr)
|
| 1407 | {
|
| 1408 |
|
| 1409 | if (x86_arch_cap_msr & ARCH_CAP_ITS_NO)
|
| 1410 | return false;
|
| 1411 | if (boot_cpu_data.x86_vendor != X86_VENDOR_INTEL)
|
| 1412 | return false;
|
| 1413 |
|
| 1414 |
|
| 1415 | if (boot_cpu_has(X86_FEATURE_BHI_CTRL))
|
| 1416 | return false;
|
| 1417 |
|
| 1418 | |
| 1419 | * If a VMM did not expose ITS_NO, assume that a guest could
|
| 1420 | * be running on a vulnerable hardware or may migrate to such
|
| 1421 | * hardware.
|
| 1422 |
|
| 1423 | if (boot_cpu_has(X86_FEATURE_HYPERVISOR))
|
| 1424 | return true;
|
| 1425 |
|
| 1426 | if (cpu_matches(cpu_vuln_blacklist, ITS))
|
| 1427 | return true;
|
| 1428 |
|
| 1429 | return false;
|
| 1430 | }
|
| 1431 |
|
| 1432 | static const struct x86_cpu_id cpu_latest_microcode[] __initconst = {
|
| 1433 | #include "microcode/intel-ucode-defs.h"
|
| 1434 | {}
|
| 1435 | };
|
| 1436 |
|
| 1437 | static bool __init cpu_has_old_microcode(void)
|
| 1438 | {
|
| 1439 | const struct x86_cpu_id *m = x86_match_cpu(cpu_latest_microcode);
|
| 1440 |
|
| 1441 |
|
| 1442 | if (!m) {
|
| 1443 |
|
| 1444 | if (boot_cpu_data.x86_vendor == X86_VENDOR_INTEL)
|
| 1445 | pr_info("x86/CPU: Model not found in latest microcode list\n");
|
| 1446 | return false;
|
| 1447 | }
|
| 1448 |
|
| 1449 | |
| 1450 | * Hosts usually lie to guests with a super high microcode
|
| 1451 | * version. Just ignore what hosts tell guests:
|
| 1452 |
|
| 1453 | if (boot_cpu_has(X86_FEATURE_HYPERVISOR))
|
| 1454 | return false;
|
| 1455 |
|
| 1456 |
|
| 1457 | if (boot_cpu_data.microcode & BIT(31))
|
| 1458 | return true;
|
| 1459 |
|
| 1460 |
|
| 1461 | if (boot_cpu_data.microcode >= m->driver_data)
|
| 1462 | return false;
|
| 1463 |
|
| 1464 |
|
| 1465 | return true;
|
| 1466 | }
|
| 1467 |
|
| 1468 | static void __init cpu_set_bug_bits(struct cpuinfo_x86 *c)
|
| 1469 | {
|
| 1470 | u64 x86_arch_cap_msr = x86_read_arch_cap_msr();
|
| 1471 |
|
| 1472 | if (cpu_has_old_microcode()) {
|
| 1473 | pr_warn("x86/CPU: Running old microcode\n");
|
| 1474 | setup_force_cpu_bug(X86_BUG_OLD_MICROCODE);
|
| 1475 | add_taint(TAINT_CPU_OUT_OF_SPEC, LOCKDEP_STILL_OK);
|
| 1476 | }
|
| 1477 |
|
| 1478 |
|
| 1479 | if (!cpu_matches(cpu_vuln_whitelist, NO_ITLB_MULTIHIT) &&
|
| 1480 | !(x86_arch_cap_msr & ARCH_CAP_PSCHANGE_MC_NO))
|
| 1481 | setup_force_cpu_bug(X86_BUG_ITLB_MULTIHIT);
|
| 1482 |
|
| 1483 | if (cpu_matches(cpu_vuln_whitelist, NO_SPECULATION))
|
| 1484 | return;
|
| 1485 |
|
| 1486 | setup_force_cpu_bug(X86_BUG_SPECTRE_V1);
|
| 1487 |
|
| 1488 | if (!cpu_matches(cpu_vuln_whitelist, NO_SPECTRE_V2)) {
|
| 1489 | setup_force_cpu_bug(X86_BUG_SPECTRE_V2);
|
| 1490 | setup_force_cpu_bug(X86_BUG_SPECTRE_V2_USER);
|
| 1491 | }
|
| 1492 |
|
| 1493 | if (!cpu_matches(cpu_vuln_whitelist, NO_SSB) &&
|
| 1494 | !(x86_arch_cap_msr & ARCH_CAP_SSB_NO) &&
|
| 1495 | !cpu_has(c, X86_FEATURE_AMD_SSB_NO))
|
| 1496 | setup_force_cpu_bug(X86_BUG_SPEC_STORE_BYPASS);
|
| 1497 |
|
| 1498 | |
| 1499 | * AMD's AutoIBRS is equivalent to Intel's eIBRS - use the Intel feature
|
| 1500 | * flag and protect from vendor-specific bugs via the whitelist.
|
| 1501 | *
|
| 1502 | * Don't use AutoIBRS when SNP is enabled because it degrades host
|
| 1503 | * userspace indirect branch performance.
|
| 1504 |
|
| 1505 | if ((x86_arch_cap_msr & ARCH_CAP_IBRS_ALL) ||
|
| 1506 | (cpu_has(c, X86_FEATURE_AUTOIBRS) &&
|
| 1507 | !cpu_feature_enabled(X86_FEATURE_SEV_SNP))) {
|
| 1508 | setup_force_cpu_cap(X86_FEATURE_IBRS_ENHANCED);
|
| 1509 | if (!cpu_matches(cpu_vuln_whitelist, NO_EIBRS_PBRSB) &&
|
| 1510 | !(x86_arch_cap_msr & ARCH_CAP_PBRSB_NO))
|
| 1511 | setup_force_cpu_bug(X86_BUG_EIBRS_PBRSB);
|
| 1512 | }
|
| 1513 |
|
| 1514 | if (!cpu_matches(cpu_vuln_whitelist, NO_MDS) &&
|
| 1515 | !(x86_arch_cap_msr & ARCH_CAP_MDS_NO)) {
|
| 1516 | setup_force_cpu_bug(X86_BUG_MDS);
|
| 1517 | if (cpu_matches(cpu_vuln_whitelist, MSBDS_ONLY))
|
| 1518 | setup_force_cpu_bug(X86_BUG_MSBDS_ONLY);
|
| 1519 | }
|
| 1520 |
|
| 1521 | if (!cpu_matches(cpu_vuln_whitelist, NO_SWAPGS))
|
| 1522 | setup_force_cpu_bug(X86_BUG_SWAPGS);
|
| 1523 |
|
| 1524 | |
| 1525 | * When the CPU is not mitigated for TAA (TAA_NO=0) set TAA bug when:
|
| 1526 | * - TSX is supported or
|
| 1527 | * - TSX_CTRL is present
|
| 1528 | *
|
| 1529 | * TSX_CTRL check is needed for cases when TSX could be disabled before
|
| 1530 | * the kernel boot e.g. kexec.
|
| 1531 | * TSX_CTRL check alone is not sufficient for cases when the microcode
|
| 1532 | * update is not present or running as guest that don't get TSX_CTRL.
|
| 1533 |
|
| 1534 | if (!(x86_arch_cap_msr & ARCH_CAP_TAA_NO) &&
|
| 1535 | (cpu_has(c, X86_FEATURE_RTM) ||
|
| 1536 | (x86_arch_cap_msr & ARCH_CAP_TSX_CTRL_MSR)))
|
| 1537 | setup_force_cpu_bug(X86_BUG_TAA);
|
| 1538 |
|
| 1539 | |
| 1540 | * SRBDS affects CPUs which support RDRAND or RDSEED and are listed
|
| 1541 | * in the vulnerability blacklist.
|
| 1542 | *
|
| 1543 | * Some of the implications and mitigation of Shared Buffers Data
|
| 1544 | * Sampling (SBDS) are similar to SRBDS. Give SBDS same treatment as
|
| 1545 | * SRBDS.
|
| 1546 |
|
| 1547 | if ((cpu_has(c, X86_FEATURE_RDRAND) ||
|
| 1548 | cpu_has(c, X86_FEATURE_RDSEED)) &&
|
| 1549 | cpu_matches(cpu_vuln_blacklist, SRBDS | MMIO_SBDS))
|
| 1550 | setup_force_cpu_bug(X86_BUG_SRBDS);
|
| 1551 |
|
| 1552 | |
| 1553 | * Processor MMIO Stale Data bug enumeration
|
| 1554 | *
|
| 1555 | * Affected CPU list is generally enough to enumerate the vulnerability,
|
| 1556 | * but for virtualization case check for ARCH_CAP MSR bits also, VMM may
|
| 1557 | * not want the guest to enumerate the bug.
|
| 1558 |
|
| 1559 | if (!arch_cap_mmio_immune(x86_arch_cap_msr)) {
|
| 1560 | if (cpu_matches(cpu_vuln_blacklist, MMIO))
|
| 1561 | setup_force_cpu_bug(X86_BUG_MMIO_STALE_DATA);
|
| 1562 | }
|
| 1563 |
|
| 1564 | if (!cpu_has(c, X86_FEATURE_BTC_NO)) {
|
| 1565 | if (cpu_matches(cpu_vuln_blacklist, RETBLEED) || (x86_arch_cap_msr & ARCH_CAP_RSBA))
|
| 1566 | setup_force_cpu_bug(X86_BUG_RETBLEED);
|
| 1567 | }
|
| 1568 |
|
| 1569 | if (cpu_matches(cpu_vuln_blacklist, SMT_RSB))
|
| 1570 | setup_force_cpu_bug(X86_BUG_SMT_RSB);
|
| 1571 |
|
| 1572 | if (!cpu_has(c, X86_FEATURE_SRSO_NO)) {
|
| 1573 | if (cpu_matches(cpu_vuln_blacklist, SRSO))
|
| 1574 | setup_force_cpu_bug(X86_BUG_SRSO);
|
| 1575 | }
|
| 1576 |
|
| 1577 | |
| 1578 | * Check if CPU is vulnerable to GDS. If running in a virtual machine on
|
| 1579 | * an affected processor, the VMM may have disabled the use of GATHER by
|
| 1580 | * disabling AVX2. The only way to do this in HW is to clear XCR0[2],
|
| 1581 | * which means that AVX will be disabled.
|
| 1582 |
|
| 1583 | if (cpu_matches(cpu_vuln_blacklist, GDS) && !(x86_arch_cap_msr & ARCH_CAP_GDS_NO) &&
|
| 1584 | boot_cpu_has(X86_FEATURE_AVX))
|
| 1585 | setup_force_cpu_bug(X86_BUG_GDS);
|
| 1586 |
|
| 1587 | if (vulnerable_to_rfds(x86_arch_cap_msr))
|
| 1588 | setup_force_cpu_bug(X86_BUG_RFDS);
|
| 1589 |
|
| 1590 | |
| 1591 | * Intel parts with eIBRS are vulnerable to BHI attacks. Parts with
|
| 1592 | * BHI_NO still need to use the BHI mitigation to prevent Intra-mode
|
| 1593 | * attacks. When virtualized, eIBRS could be hidden, assume vulnerable.
|
| 1594 |
|
| 1595 | if (!cpu_matches(cpu_vuln_whitelist, NO_BHI) &&
|
| 1596 | (boot_cpu_has(X86_FEATURE_IBRS_ENHANCED) ||
|
| 1597 | boot_cpu_has(X86_FEATURE_HYPERVISOR)))
|
| 1598 | setup_force_cpu_bug(X86_BUG_BHI);
|
| 1599 |
|
| 1600 | if (cpu_has(c, X86_FEATURE_AMD_IBPB) && !cpu_has(c, X86_FEATURE_AMD_IBPB_RET))
|
| 1601 | setup_force_cpu_bug(X86_BUG_IBPB_NO_RET);
|
| 1602 |
|
| 1603 | if (vulnerable_to_its(x86_arch_cap_msr)) {
|
| 1604 | setup_force_cpu_bug(X86_BUG_ITS);
|
| 1605 | if (cpu_matches(cpu_vuln_blacklist, ITS_NATIVE_ONLY))
|
| 1606 | setup_force_cpu_bug(X86_BUG_ITS_NATIVE_ONLY);
|
| 1607 | }
|
| 1608 |
|
| 1609 | if (c->x86_vendor == X86_VENDOR_AMD) {
|
| 1610 | if (!cpu_has(c, X86_FEATURE_TSA_SQ_NO) ||
|
| 1611 | !cpu_has(c, X86_FEATURE_TSA_L1_NO)) {
|
| 1612 | if (cpu_matches(cpu_vuln_blacklist, TSA) ||
|
| 1613 |
|
| 1614 | (cpu_has(c, X86_FEATURE_HYPERVISOR) && cpu_has(c, X86_FEATURE_ZEN)))
|
| 1615 | setup_force_cpu_bug(X86_BUG_TSA);
|
| 1616 | }
|
| 1617 | }
|
| 1618 |
|
| 1619 | |
| 1620 | * Set the bug only on bare-metal. A nested hypervisor should already be
|
| 1621 | * deploying IBPB to isolate itself from nested guests.
|
| 1622 |
|
| 1623 | if (cpu_matches(cpu_vuln_blacklist, VMSCAPE) &&
|
| 1624 | !boot_cpu_has(X86_FEATURE_HYPERVISOR))
|
| 1625 | setup_force_cpu_bug(X86_BUG_VMSCAPE);
|
| 1626 |
|
| 1627 | if (cpu_matches(cpu_vuln_whitelist, NO_MELTDOWN))
|
| 1628 | return;
|
| 1629 |
|
| 1630 |
|
| 1631 | if (x86_arch_cap_msr & ARCH_CAP_RDCL_NO)
|
| 1632 | return;
|
| 1633 |
|
| 1634 | setup_force_cpu_bug(X86_BUG_CPU_MELTDOWN);
|
| 1635 |
|
| 1636 | if (cpu_matches(cpu_vuln_whitelist, NO_L1TF))
|
| 1637 | return;
|
| 1638 |
|
| 1639 | setup_force_cpu_bug(X86_BUG_L1TF);
|
| 1640 | }
|
| 1641 |
|
| 1642 | |
| 1643 | * The NOPL instruction is supposed to exist on all CPUs of family >= 6;
|
| 1644 | * unfortunately, that's not true in practice because of early VIA
|
| 1645 | * chips and (more importantly) broken virtualizers that are not easy
|
| 1646 | * to detect. In the latter case it doesn't even *fail* reliably, so
|
| 1647 | * probing for it doesn't even work. Disable it completely on 32-bit
|
| 1648 | * unless we can find a reliable way to detect all the broken cases.
|
| 1649 | * Enable it explicitly on 64-bit for non-constant inputs of cpu_has().
|
| 1650 |
|
| 1651 | static void detect_nopl(void)
|
| 1652 | {
|
| 1653 | #ifdef CONFIG_X86_32
|
| 1654 | setup_clear_cpu_cap(X86_FEATURE_NOPL);
|
| 1655 | #else
|
| 1656 | setup_force_cpu_cap(X86_FEATURE_NOPL);
|
| 1657 | #endif
|
| 1658 | }
|
| 1659 |
|
| 1660 | static inline bool parse_set_clear_cpuid(char *arg, bool set)
|
| 1661 | {
|
| 1662 | char *opt;
|
| 1663 | int taint = 0;
|
| 1664 |
|
| 1665 | while (arg) {
|
| 1666 | bool found = false;
|
| 1667 | unsigned int bit;
|
| 1668 |
|
| 1669 | opt = strsep(&arg, ",");
|
| 1670 |
|
| 1671 | |
| 1672 | * Handle naked numbers first for feature flags which don't
|
| 1673 | * have names. It doesn't make sense for a bug not to have a
|
| 1674 | * name so don't handle bug flags here.
|
| 1675 |
|
| 1676 | if (!kstrtouint(opt, 10, &bit)) {
|
| 1677 | if (bit < NCAPINTS * 32) {
|
| 1678 |
|
| 1679 | if (set) {
|
| 1680 | pr_warn("setcpuid: force-enabling CPU feature flag:");
|
| 1681 | setup_force_cpu_cap(bit);
|
| 1682 | } else {
|
| 1683 | pr_warn("clearcpuid: force-disabling CPU feature flag:");
|
| 1684 | setup_clear_cpu_cap(bit);
|
| 1685 | }
|
| 1686 |
|
| 1687 | if (!x86_cap_flags[bit])
|
| 1688 | pr_cont(" %d:%d\n", bit >> 5, bit & 31);
|
| 1689 | else
|
| 1690 | pr_cont(" %s\n", x86_cap_flags[bit]);
|
| 1691 |
|
| 1692 | taint++;
|
| 1693 | }
|
| 1694 | |
| 1695 | * The assumption is that there are no feature names with only
|
| 1696 | * numbers in the name thus go to the next argument.
|
| 1697 |
|
| 1698 | continue;
|
| 1699 | }
|
| 1700 |
|
| 1701 | for (bit = 0; bit < 32 * (NCAPINTS + NBUGINTS); bit++) {
|
| 1702 | const char *flag;
|
| 1703 | const char *kind;
|
| 1704 |
|
| 1705 | if (bit < 32 * NCAPINTS) {
|
| 1706 | flag = x86_cap_flags[bit];
|
| 1707 | kind = "feature";
|
| 1708 | } else {
|
| 1709 | kind = "bug";
|
| 1710 | flag = x86_bug_flags[bit - (32 * NCAPINTS)];
|
| 1711 | }
|
| 1712 |
|
| 1713 | if (!flag)
|
| 1714 | continue;
|
| 1715 |
|
| 1716 | if (strcmp(flag, opt))
|
| 1717 | continue;
|
| 1718 |
|
| 1719 | if (set) {
|
| 1720 | pr_warn("setcpuid: force-enabling CPU %s flag: %s\n",
|
| 1721 | kind, flag);
|
| 1722 | setup_force_cpu_cap(bit);
|
| 1723 | } else {
|
| 1724 | pr_warn("clearcpuid: force-disabling CPU %s flag: %s\n",
|
| 1725 | kind, flag);
|
| 1726 | setup_clear_cpu_cap(bit);
|
| 1727 | }
|
| 1728 | taint++;
|
| 1729 | found = true;
|
| 1730 | break;
|
| 1731 | }
|
| 1732 |
|
| 1733 | if (!found)
|
| 1734 | pr_warn("%s: unknown CPU flag: %s", set ? "setcpuid" : "clearcpuid", opt);
|
| 1735 | }
|
| 1736 |
|
| 1737 | return taint;
|
| 1738 | }
|
| 1739 |
|
| 1740 |
|
| 1741 | |
| 1742 | * We parse cpu parameters early because fpu__init_system() is executed
|
| 1743 | * before parse_early_param().
|
| 1744 |
|
| 1745 | static void __init cpu_parse_early_param(void)
|
| 1746 | {
|
| 1747 | bool cpuid_taint = false;
|
| 1748 | char arg[128];
|
| 1749 | int arglen;
|
| 1750 |
|
| 1751 | #ifdef CONFIG_X86_32
|
| 1752 | if (cmdline_find_option_bool(boot_command_line, "nofxsr"))
|
| 1753 | setup_clear_cpu_cap(X86_FEATURE_FXSR);
|
| 1754 | #endif
|
| 1755 |
|
| 1756 | if (cmdline_find_option_bool(boot_command_line, "noxsave"))
|
| 1757 | setup_clear_cpu_cap(X86_FEATURE_XSAVE);
|
| 1758 |
|
| 1759 | if (cmdline_find_option_bool(boot_command_line, "noxsaveopt"))
|
| 1760 | setup_clear_cpu_cap(X86_FEATURE_XSAVEOPT);
|
| 1761 |
|
| 1762 | if (cmdline_find_option_bool(boot_command_line, "noxsaves"))
|
| 1763 | setup_clear_cpu_cap(X86_FEATURE_XSAVES);
|
| 1764 |
|
| 1765 | if (cmdline_find_option_bool(boot_command_line, "nousershstk"))
|
| 1766 | setup_clear_cpu_cap(X86_FEATURE_USER_SHSTK);
|
| 1767 |
|
| 1768 |
|
| 1769 | arglen = cmdline_find_option(boot_command_line, "fred", arg, sizeof(arg));
|
| 1770 | if (arglen == 3 && !strncmp(arg, "off", 3))
|
| 1771 | setup_clear_cpu_cap(X86_FEATURE_FRED);
|
| 1772 |
|
| 1773 | arglen = cmdline_find_option(boot_command_line, "clearcpuid", arg, sizeof(arg));
|
| 1774 | if (arglen > 0)
|
| 1775 | cpuid_taint |= parse_set_clear_cpuid(arg, false);
|
| 1776 |
|
| 1777 | arglen = cmdline_find_option(boot_command_line, "setcpuid", arg, sizeof(arg));
|
| 1778 | if (arglen > 0)
|
| 1779 | cpuid_taint |= parse_set_clear_cpuid(arg, true);
|
| 1780 |
|
| 1781 | if (cpuid_taint) {
|
| 1782 | pr_warn("!!! setcpuid=/clearcpuid= in use, this is for TESTING ONLY, may break things horribly. Tainting kernel.\n");
|
| 1783 | add_taint(TAINT_CPU_OUT_OF_SPEC, LOCKDEP_STILL_OK);
|
| 1784 | }
|
| 1785 | }
|
| 1786 |
|
| 1787 | static void init_cpu_info(struct cpuinfo_x86 *c)
|
| 1788 | {
|
| 1789 | c->x86_cache_size = 0;
|
| 1790 | c->x86_vendor = X86_VENDOR_UNKNOWN;
|
| 1791 | c->x86_model = c->x86_stepping = 0;
|
| 1792 | c->x86_vendor_id[0] = '\0';
|
| 1793 | c->x86_model_id[0] = '\0';
|
| 1794 | #ifdef CONFIG_X86_64
|
| 1795 | c->x86_clflush_size = 64;
|
| 1796 | c->x86_phys_bits = 36;
|
| 1797 | c->x86_virt_bits = 48;
|
| 1798 | #else
|
| 1799 | c->cpuid_level = -1;
|
| 1800 | c->x86_clflush_size = 32;
|
| 1801 | c->x86_phys_bits = 32;
|
| 1802 | c->x86_virt_bits = 32;
|
| 1803 | #endif
|
| 1804 | c->x86_cache_alignment = c->x86_clflush_size;
|
| 1805 | memset(&c->x86_capability, 0, sizeof(c->x86_capability));
|
| 1806 | memset(&c->cpuid, 0, sizeof(c->cpuid));
|
| 1807 | #ifdef CONFIG_X86_VMX_FEATURE_NAMES
|
| 1808 | memset(&c->vmx_capability, 0, sizeof(c->vmx_capability));
|
| 1809 | #endif
|
| 1810 | c->extended_cpuid_level = 0;
|
| 1811 | }
|
| 1812 |
|
| 1813 | |
| 1814 | * Do minimum CPU detection early.
|
| 1815 | * Fields really needed: vendor, cpuid_level, family, model, mask,
|
| 1816 | * cache alignment.
|
| 1817 | * The others are reset to their defaults here and only filled in later,
|
| 1818 | * by identify_cpu().
|
| 1819 | *
|
| 1820 | * WARNING: this function is only called on the boot CPU. Don't add code
|
| 1821 | * here that is supposed to run on all CPUs.
|
| 1822 |
|
| 1823 | static void __init early_identify_cpu(struct cpuinfo_x86 *c)
|
| 1824 | {
|
| 1825 | init_cpu_info(c);
|
| 1826 |
|
| 1827 | if (!cpuid_feature())
|
| 1828 | identify_cpu_without_cpuid(c);
|
| 1829 |
|
| 1830 |
|
| 1831 | if (cpuid_feature()) {
|
| 1832 | cpuid_scan_cpu(c);
|
| 1833 | cpu_detect(c);
|
| 1834 | get_cpu_vendor(c);
|
| 1835 | intel_unlock_cpuid_leafs(c);
|
| 1836 | get_cpu_cap(c);
|
| 1837 | setup_force_cpu_cap(X86_FEATURE_CPUID);
|
| 1838 | get_cpu_address_sizes(c);
|
| 1839 | cpu_parse_early_param();
|
| 1840 |
|
| 1841 | cpu_init_topology(c);
|
| 1842 |
|
| 1843 | if (this_cpu->c_early_init)
|
| 1844 | this_cpu->c_early_init(c);
|
| 1845 |
|
| 1846 | c->cpu_index = 0;
|
| 1847 | filter_cpuid_features(c, false);
|
| 1848 | check_cpufeature_deps(c);
|
| 1849 |
|
| 1850 | if (this_cpu->c_bsp_init)
|
| 1851 | this_cpu->c_bsp_init(c);
|
| 1852 | } else {
|
| 1853 | setup_clear_cpu_cap(X86_FEATURE_CPUID);
|
| 1854 | get_cpu_address_sizes(c);
|
| 1855 | cpu_init_topology(c);
|
| 1856 | }
|
| 1857 |
|
| 1858 | setup_force_cpu_cap(X86_FEATURE_ALWAYS);
|
| 1859 |
|
| 1860 | cpu_set_bug_bits(c);
|
| 1861 |
|
| 1862 | sld_setup(c);
|
| 1863 |
|
| 1864 | #ifdef CONFIG_X86_32
|
| 1865 | |
| 1866 | * Regardless of whether PCID is enumerated, the SDM says
|
| 1867 | * that it can't be enabled in 32-bit mode.
|
| 1868 |
|
| 1869 | setup_clear_cpu_cap(X86_FEATURE_PCID);
|
| 1870 |
|
| 1871 | |
| 1872 | * Never use SYSCALL on a 32-bit kernel
|
| 1873 |
|
| 1874 | setup_clear_cpu_cap(X86_FEATURE_SYSCALL32);
|
| 1875 | #endif
|
| 1876 |
|
| 1877 | |
| 1878 | * Later in the boot process pgtable_l5_enabled() relies on
|
| 1879 | * cpu_feature_enabled(X86_FEATURE_LA57). If 5-level paging is not
|
| 1880 | * enabled by this point we need to clear the feature bit to avoid
|
| 1881 | * false-positives at the later stage.
|
| 1882 | *
|
| 1883 | * pgtable_l5_enabled() can be false here for several reasons:
|
| 1884 | * - 5-level paging is disabled compile-time;
|
| 1885 | * - it's 32-bit kernel;
|
| 1886 | * - machine doesn't support 5-level paging;
|
| 1887 | * - user specified 'no5lvl' in kernel command line.
|
| 1888 |
|
| 1889 | if (!pgtable_l5_enabled())
|
| 1890 | setup_clear_cpu_cap(X86_FEATURE_LA57);
|
| 1891 |
|
| 1892 | detect_nopl();
|
| 1893 | mca_bsp_init(c);
|
| 1894 | }
|
| 1895 |
|
| 1896 | void __init init_cpu_devs(void)
|
| 1897 | {
|
| 1898 | const struct cpu_dev *const *cdev;
|
| 1899 | int count = 0;
|
| 1900 |
|
| 1901 | for (cdev = __x86_cpu_dev_start; cdev < __x86_cpu_dev_end; cdev++) {
|
| 1902 | const struct cpu_dev *cpudev = *cdev;
|
| 1903 |
|
| 1904 | if (count >= X86_VENDOR_NUM)
|
| 1905 | break;
|
| 1906 | cpu_devs[count] = cpudev;
|
| 1907 | count++;
|
| 1908 | }
|
| 1909 | }
|
| 1910 |
|
| 1911 | void __init early_cpu_init(void)
|
| 1912 | {
|
| 1913 | #ifdef CONFIG_PROCESSOR_SELECT
|
| 1914 | unsigned int i, j;
|
| 1915 |
|
| 1916 | pr_info("KERNEL supported cpus:\n");
|
| 1917 | #endif
|
| 1918 |
|
| 1919 | init_cpu_devs();
|
| 1920 |
|
| 1921 | #ifdef CONFIG_PROCESSOR_SELECT
|
| 1922 | for (i = 0; i < X86_VENDOR_NUM && cpu_devs[i]; i++) {
|
| 1923 | for (j = 0; j < 2; j++) {
|
| 1924 | if (!cpu_devs[i]->c_ident[j])
|
| 1925 | continue;
|
| 1926 | pr_info(" %s %s\n", cpu_devs[i]->c_vendor,
|
| 1927 | cpu_devs[i]->c_ident[j]);
|
| 1928 | }
|
| 1929 | }
|
| 1930 | #endif
|
| 1931 |
|
| 1932 | early_identify_cpu(&boot_cpu_data);
|
| 1933 | }
|
| 1934 |
|
| 1935 | static bool detect_null_seg_behavior(void)
|
| 1936 | {
|
| 1937 | |
| 1938 | * Empirically, writing zero to a segment selector on AMD does
|
| 1939 | * not clear the base, whereas writing zero to a segment
|
| 1940 | * selector on Intel does clear the base. Intel's behavior
|
| 1941 | * allows slightly faster context switches in the common case
|
| 1942 | * where GS is unused by the prev and next threads.
|
| 1943 | *
|
| 1944 | * Since neither vendor documents this anywhere that I can see,
|
| 1945 | * detect it directly instead of hard-coding the choice by
|
| 1946 | * vendor.
|
| 1947 | *
|
| 1948 | * I've designated AMD's behavior as the "bug" because it's
|
| 1949 | * counterintuitive and less friendly.
|
| 1950 |
|
| 1951 |
|
| 1952 | unsigned long old_base, tmp;
|
| 1953 | rdmsrq(MSR_FS_BASE, old_base);
|
| 1954 | wrmsrq(MSR_FS_BASE, 1);
|
| 1955 | loadsegment(fs, 0);
|
| 1956 | rdmsrq(MSR_FS_BASE, tmp);
|
| 1957 | wrmsrq(MSR_FS_BASE, old_base);
|
| 1958 | return tmp == 0;
|
| 1959 | }
|
| 1960 |
|
| 1961 | void check_null_seg_clears_base(struct cpuinfo_x86 *c)
|
| 1962 | {
|
| 1963 |
|
| 1964 | if (!IS_ENABLED(CONFIG_X86_64))
|
| 1965 | return;
|
| 1966 |
|
| 1967 | if (cpu_has(c, X86_FEATURE_NULL_SEL_CLR_BASE))
|
| 1968 | return;
|
| 1969 |
|
| 1970 | |
| 1971 | * CPUID bit above wasn't set. If this kernel is still running
|
| 1972 | * as a HV guest, then the HV has decided not to advertize
|
| 1973 | * that CPUID bit for whatever reason. For example, one
|
| 1974 | * member of the migration pool might be vulnerable. Which
|
| 1975 | * means, the bug is present: set the BUG flag and return.
|
| 1976 |
|
| 1977 | if (cpu_has(c, X86_FEATURE_HYPERVISOR)) {
|
| 1978 | set_cpu_bug(c, X86_BUG_NULL_SEG);
|
| 1979 | return;
|
| 1980 | }
|
| 1981 |
|
| 1982 | |
| 1983 | * Zen2 CPUs also have this behaviour, but no CPUID bit.
|
| 1984 | * 0x18 is the respective family for Hygon.
|
| 1985 |
|
| 1986 | if ((c->x86 == 0x17 || c->x86 == 0x18) &&
|
| 1987 | detect_null_seg_behavior())
|
| 1988 | return;
|
| 1989 |
|
| 1990 |
|
| 1991 | set_cpu_bug(c, X86_BUG_NULL_SEG);
|
| 1992 | }
|
| 1993 |
|
| 1994 | |
| 1995 | * This does the hard work of actually picking apart the CPU stuff...
|
| 1996 |
|
| 1997 | static void identify_cpu(struct cpuinfo_x86 *c)
|
| 1998 | {
|
| 1999 | int i;
|
| 2000 |
|
| 2001 | c->loops_per_jiffy = loops_per_jiffy;
|
| 2002 |
|
| 2003 | if (!cpuid_feature())
|
| 2004 | identify_cpu_without_cpuid(c);
|
| 2005 |
|
| 2006 |
|
| 2007 | if (!cpuid_feature())
|
| 2008 | goto no_cpuid;
|
| 2009 |
|
| 2010 | cpuid_scan_cpu(c);
|
| 2011 | cpu_detect(c);
|
| 2012 | get_cpu_vendor(c);
|
| 2013 | intel_unlock_cpuid_leafs(c);
|
| 2014 | get_cpu_cap(c);
|
| 2015 |
|
| 2016 | get_cpu_address_sizes(c);
|
| 2017 |
|
| 2018 | get_model_name(c);
|
| 2019 |
|
| 2020 | |
| 2021 | * ESPFIX is a strange bug. All real CPUs have it. Paravirt
|
| 2022 | * systems that run Linux at CPL > 0 may or may not have the
|
| 2023 | * issue, but, even if they have the issue, there's absolutely
|
| 2024 | * nothing we can do about it because we can't use the real IRET
|
| 2025 | * instruction.
|
| 2026 | *
|
| 2027 | * NB: For the time being, only 32-bit kernels support
|
| 2028 | * X86_BUG_ESPFIX as such. 64-bit kernels directly choose
|
| 2029 | * whether to apply espfix using paravirt hooks. If any
|
| 2030 | * non-paravirt system ever shows up that does *not* have the
|
| 2031 | * ESPFIX issue, we can change this.
|
| 2032 |
|
| 2033 | #ifdef CONFIG_X86_32
|
| 2034 | set_cpu_bug(c, X86_BUG_ESPFIX);
|
| 2035 | #endif
|
| 2036 |
|
| 2037 | no_cpuid:
|
| 2038 | cpu_parse_topology(c);
|
| 2039 |
|
| 2040 | if (this_cpu->c_identify)
|
| 2041 | this_cpu->c_identify(c);
|
| 2042 |
|
| 2043 |
|
| 2044 | apply_forced_caps(c);
|
| 2045 |
|
| 2046 | |
| 2047 | * Set default APIC and TSC_DEADLINE MSR fencing flag. AMD and
|
| 2048 | * Hygon will clear it in ->c_init() below.
|
| 2049 |
|
| 2050 | set_cpu_cap(c, X86_FEATURE_APIC_MSRS_FENCE);
|
| 2051 |
|
| 2052 | |
| 2053 | * Vendor-specific initialization. In this section we
|
| 2054 | * canonicalize the feature flags, meaning if there are
|
| 2055 | * features a certain CPU supports which CPUID doesn't
|
| 2056 | * tell us, CPUID claiming incorrect flags, or other bugs,
|
| 2057 | * we handle them here.
|
| 2058 | *
|
| 2059 | * At the end of this section, c->x86_capability better
|
| 2060 | * indicate the features this CPU genuinely supports!
|
| 2061 |
|
| 2062 | if (this_cpu->c_init)
|
| 2063 | this_cpu->c_init(c);
|
| 2064 |
|
| 2065 | bus_lock_init();
|
| 2066 |
|
| 2067 |
|
| 2068 | squash_the_stupid_serial_number(c);
|
| 2069 |
|
| 2070 | setup_smep(c);
|
| 2071 | setup_smap(c);
|
| 2072 | setup_umip(c);
|
| 2073 |
|
| 2074 | |
| 2075 | * The vendor-specific functions might have changed features.
|
| 2076 | * Now we do "generic changes."
|
| 2077 |
|
| 2078 |
|
| 2079 |
|
| 2080 | filter_cpuid_features(c, true);
|
| 2081 |
|
| 2082 |
|
| 2083 | check_cpufeature_deps(c);
|
| 2084 |
|
| 2085 |
|
| 2086 | if (!c->x86_model_id[0]) {
|
| 2087 | const char *p;
|
| 2088 | p = table_lookup_model(c);
|
| 2089 | if (p)
|
| 2090 | strcpy(c->x86_model_id, p);
|
| 2091 | else
|
| 2092 |
|
| 2093 | sprintf(c->x86_model_id, "%02x/%02x",
|
| 2094 | c->x86, c->x86_model);
|
| 2095 | }
|
| 2096 |
|
| 2097 | x86_init_rdrand(c);
|
| 2098 | setup_pku(c);
|
| 2099 | setup_cet(c);
|
| 2100 |
|
| 2101 | |
| 2102 | * Clear/Set all flags overridden by options, need do it
|
| 2103 | * before following smp all cpus cap AND.
|
| 2104 |
|
| 2105 | apply_forced_caps(c);
|
| 2106 |
|
| 2107 | |
| 2108 | * On SMP, boot_cpu_data holds the common feature set between
|
| 2109 | * all CPUs; so make sure that we indicate which features are
|
| 2110 | * common between the CPUs. The first time this routine gets
|
| 2111 | * executed, c == &boot_cpu_data.
|
| 2112 |
|
| 2113 | if (c != &boot_cpu_data) {
|
| 2114 |
|
| 2115 | for (i = 0; i < NCAPINTS; i++)
|
| 2116 | boot_cpu_data.x86_capability[i] &= c->x86_capability[i];
|
| 2117 |
|
| 2118 |
|
| 2119 | for (i = NCAPINTS; i < NCAPINTS + NBUGINTS; i++)
|
| 2120 | c->x86_capability[i] |= boot_cpu_data.x86_capability[i];
|
| 2121 | }
|
| 2122 |
|
| 2123 | ppin_init(c);
|
| 2124 |
|
| 2125 |
|
| 2126 | mcheck_cpu_init(c);
|
| 2127 |
|
| 2128 | numa_add_cpu(smp_processor_id());
|
| 2129 |
|
| 2130 | if (IS_ENABLED(CONFIG_X86_32))
|
| 2131 | enable_sep_cpu();
|
| 2132 | }
|
| 2133 |
|
| 2134 | |
| 2135 | * Set up the CPU state needed to execute SYSENTER/SYSEXIT instructions
|
| 2136 | * on 32-bit kernels:
|
| 2137 |
|
| 2138 | #ifdef CONFIG_X86_32
|
| 2139 | void enable_sep_cpu(void)
|
| 2140 | {
|
| 2141 | struct tss_struct *tss;
|
| 2142 | int cpu;
|
| 2143 |
|
| 2144 | if (!boot_cpu_has(X86_FEATURE_SEP))
|
| 2145 | return;
|
| 2146 |
|
| 2147 | cpu = get_cpu();
|
| 2148 | tss = &per_cpu(cpu_tss_rw, cpu);
|
| 2149 |
|
| 2150 | |
| 2151 | * We cache MSR_IA32_SYSENTER_CS's value in the TSS's ss1 field --
|
| 2152 | * see the big comment in struct x86_hw_tss's definition.
|
| 2153 |
|
| 2154 |
|
| 2155 | tss->x86_tss.ss1 = __KERNEL_CS;
|
| 2156 | wrmsrq(MSR_IA32_SYSENTER_CS, tss->x86_tss.ss1);
|
| 2157 | wrmsrq(MSR_IA32_SYSENTER_ESP, (unsigned long)(cpu_entry_stack(cpu) + 1));
|
| 2158 | wrmsrq(MSR_IA32_SYSENTER_EIP, (unsigned long)entry_SYSENTER_32);
|
| 2159 |
|
| 2160 | put_cpu();
|
| 2161 | }
|
| 2162 | #endif
|
| 2163 |
|
| 2164 | static __init void identify_boot_cpu(void)
|
| 2165 | {
|
| 2166 | identify_cpu(&boot_cpu_data);
|
| 2167 | if (HAS_KERNEL_IBT && cpu_feature_enabled(X86_FEATURE_IBT))
|
| 2168 | pr_info("CET detected: Indirect Branch Tracking enabled\n");
|
| 2169 | cpu_detect_tlb(&boot_cpu_data);
|
| 2170 | setup_cr_pinning();
|
| 2171 |
|
| 2172 | x86_virt_init();
|
| 2173 | tsx_init();
|
| 2174 | tdx_init();
|
| 2175 | lkgs_init();
|
| 2176 | }
|
| 2177 |
|
| 2178 | void identify_secondary_cpu(unsigned int cpu)
|
| 2179 | {
|
| 2180 | struct cpuinfo_x86 *c = &cpu_data(cpu);
|
| 2181 |
|
| 2182 |
|
| 2183 | if (!c->initialized)
|
| 2184 | *c = boot_cpu_data;
|
| 2185 | c->cpu_index = cpu;
|
| 2186 |
|
| 2187 | init_cpu_info(c);
|
| 2188 | identify_cpu(c);
|
| 2189 | x86_spec_ctrl_setup_ap();
|
| 2190 | update_srbds_msr();
|
| 2191 | if (boot_cpu_has_bug(X86_BUG_GDS))
|
| 2192 | update_gds_msr();
|
| 2193 |
|
| 2194 | tsx_ap_init();
|
| 2195 | c->initialized = true;
|
| 2196 | }
|
| 2197 |
|
| 2198 | void print_cpu_info(struct cpuinfo_x86 *c)
|
| 2199 | {
|
| 2200 | const char *vendor = NULL;
|
| 2201 |
|
| 2202 | if (c->x86_vendor < X86_VENDOR_NUM) {
|
| 2203 | vendor = this_cpu->c_vendor;
|
| 2204 | } else {
|
| 2205 | if (c->cpuid_level >= 0)
|
| 2206 | vendor = c->x86_vendor_id;
|
| 2207 | }
|
| 2208 |
|
| 2209 | if (vendor && !strstr(c->x86_model_id, vendor))
|
| 2210 | pr_cont("%s ", vendor);
|
| 2211 |
|
| 2212 | if (c->x86_model_id[0])
|
| 2213 | pr_cont("%s", c->x86_model_id);
|
| 2214 | else
|
| 2215 | pr_cont("%d86", c->x86);
|
| 2216 |
|
| 2217 | pr_cont(" (family: 0x%x, model: 0x%x", c->x86, c->x86_model);
|
| 2218 |
|
| 2219 | if (c->x86_stepping || c->cpuid_level >= 0)
|
| 2220 | pr_cont(", stepping: 0x%x)\n", c->x86_stepping);
|
| 2221 | else
|
| 2222 | pr_cont(")\n");
|
| 2223 | }
|
| 2224 |
|
| 2225 | |
| 2226 | * clearcpuid= and setcpuid= were already parsed in cpu_parse_early_param().
|
| 2227 | * These dummy functions prevent them from becoming an environment variable for
|
| 2228 | * init.
|
| 2229 |
|
| 2230 |
|
| 2231 | static __init int setup_clearcpuid(char *arg)
|
| 2232 | {
|
| 2233 | return 1;
|
| 2234 | }
|
| 2235 | __setup("clearcpuid=", setup_clearcpuid);
|
| 2236 |
|
| 2237 | static __init int setup_setcpuid(char *arg)
|
| 2238 | {
|
| 2239 | return 1;
|
| 2240 | }
|
| 2241 | __setup("setcpuid=", setup_setcpuid);
|
| 2242 |
|
| 2243 | DEFINE_PER_CPU_CACHE_HOT(struct task_struct *, current_task) = &init_task;
|
| 2244 | EXPORT_PER_CPU_SYMBOL(current_task);
|
| 2245 | EXPORT_PER_CPU_SYMBOL(const_current_task);
|
| 2246 |
|
| 2247 | DEFINE_PER_CPU_CACHE_HOT(unsigned long, __preempt_count) = INIT_PREEMPT_COUNT;
|
| 2248 | EXPORT_PER_CPU_SYMBOL(__preempt_count);
|
| 2249 |
|
| 2250 | DEFINE_PER_CPU_CACHE_HOT(unsigned long, cpu_current_top_of_stack) = TOP_OF_INIT_STACK;
|
| 2251 |
|
| 2252 | #ifdef CONFIG_X86_64
|
| 2253 | |
| 2254 | * Note: Do not make this dependant on CONFIG_MITIGATION_CALL_DEPTH_TRACKING
|
| 2255 | * so that this space is reserved in the hot cache section even when the
|
| 2256 | * mitigation is disabled.
|
| 2257 |
|
| 2258 | DEFINE_PER_CPU_CACHE_HOT(u64, __x86_call_depth);
|
| 2259 | EXPORT_PER_CPU_SYMBOL(__x86_call_depth);
|
| 2260 |
|
| 2261 | static void wrmsrq_cstar(unsigned long val)
|
| 2262 | {
|
| 2263 | |
| 2264 | * Intel CPUs do not support 32-bit SYSCALL. Writing to MSR_CSTAR
|
| 2265 | * is so far ignored by the CPU, but raises a #VE trap in a TDX
|
| 2266 | * guest. Avoid the pointless write on all Intel CPUs.
|
| 2267 |
|
| 2268 | if (boot_cpu_data.x86_vendor != X86_VENDOR_INTEL)
|
| 2269 | wrmsrq(MSR_CSTAR, val);
|
| 2270 | }
|
| 2271 |
|
| 2272 | static inline void idt_syscall_init(void)
|
| 2273 | {
|
| 2274 | wrmsrq(MSR_LSTAR, (unsigned long)entry_SYSCALL_64);
|
| 2275 |
|
| 2276 | if (ia32_enabled()) {
|
| 2277 | wrmsrq_cstar((unsigned long)entry_SYSCALL_compat);
|
| 2278 | |
| 2279 | * This only works on Intel CPUs.
|
| 2280 | * On AMD CPUs these MSRs are 32-bit, CPU truncates MSR_IA32_SYSENTER_EIP.
|
| 2281 | * This does not cause SYSENTER to jump to the wrong location, because
|
| 2282 | * AMD doesn't allow SYSENTER in long mode (either 32- or 64-bit).
|
| 2283 |
|
| 2284 | wrmsrq_safe(MSR_IA32_SYSENTER_CS, (u64)__KERNEL_CS);
|
| 2285 | wrmsrq_safe(MSR_IA32_SYSENTER_ESP,
|
| 2286 | (unsigned long)(cpu_entry_stack(smp_processor_id()) + 1));
|
| 2287 | wrmsrq_safe(MSR_IA32_SYSENTER_EIP, (u64)entry_SYSENTER_compat);
|
| 2288 | } else {
|
| 2289 | wrmsrq_cstar((unsigned long)entry_SYSCALL32_ignore);
|
| 2290 | wrmsrq_safe(MSR_IA32_SYSENTER_CS, (u64)GDT_ENTRY_INVALID_SEG);
|
| 2291 | wrmsrq_safe(MSR_IA32_SYSENTER_ESP, 0ULL);
|
| 2292 | wrmsrq_safe(MSR_IA32_SYSENTER_EIP, 0ULL);
|
| 2293 | }
|
| 2294 |
|
| 2295 | |
| 2296 | * Flags to clear on syscall; clear as much as possible
|
| 2297 | * to minimize user space-kernel interference.
|
| 2298 |
|
| 2299 | wrmsrq(MSR_SYSCALL_MASK,
|
| 2300 | X86_EFLAGS_CF|X86_EFLAGS_PF|X86_EFLAGS_AF|
|
| 2301 | X86_EFLAGS_ZF|X86_EFLAGS_SF|X86_EFLAGS_TF|
|
| 2302 | X86_EFLAGS_IF|X86_EFLAGS_DF|X86_EFLAGS_OF|
|
| 2303 | X86_EFLAGS_IOPL|X86_EFLAGS_NT|X86_EFLAGS_RF|
|
| 2304 | X86_EFLAGS_AC|X86_EFLAGS_ID);
|
| 2305 | }
|
| 2306 |
|
| 2307 |
|
| 2308 | void syscall_init(void)
|
| 2309 | {
|
| 2310 | struct msr val = { .h = (__USER32_CS << 16) | __KERNEL_CS };
|
| 2311 |
|
| 2312 |
|
| 2313 | wrmsrq(MSR_STAR, val.q);
|
| 2314 |
|
| 2315 | |
| 2316 | * Except the IA32_STAR MSR, there is NO need to setup SYSCALL and
|
| 2317 | * SYSENTER MSRs for FRED, because FRED uses the ring 3 FRED
|
| 2318 | * entrypoint for SYSCALL and SYSENTER, and ERETU is the only legit
|
| 2319 | * instruction to return to ring 3 (both sysexit and sysret cause
|
| 2320 | * #UD when FRED is enabled).
|
| 2321 |
|
| 2322 | if (!cpu_feature_enabled(X86_FEATURE_FRED))
|
| 2323 | idt_syscall_init();
|
| 2324 | }
|
| 2325 | #endif
|
| 2326 |
|
| 2327 | #ifdef CONFIG_STACKPROTECTOR
|
| 2328 | DEFINE_PER_CPU_CACHE_HOT(unsigned long, __stack_chk_guard);
|
| 2329 | #ifndef CONFIG_SMP
|
| 2330 | EXPORT_PER_CPU_SYMBOL(__stack_chk_guard);
|
| 2331 | #endif
|
| 2332 | #endif
|
| 2333 |
|
| 2334 | static void initialize_debug_regs(void)
|
| 2335 | {
|
| 2336 |
|
| 2337 | set_debugreg(DR7_FIXED_1, 7);
|
| 2338 | set_debugreg(DR6_RESERVED, 6);
|
| 2339 |
|
| 2340 | set_debugreg(0, 3);
|
| 2341 | set_debugreg(0, 2);
|
| 2342 | set_debugreg(0, 1);
|
| 2343 | set_debugreg(0, 0);
|
| 2344 | }
|
| 2345 |
|
| 2346 | #ifdef CONFIG_KGDB
|
| 2347 | |
| 2348 | * Restore debug regs if using kgdbwait and you have a kernel debugger
|
| 2349 | * connection established.
|
| 2350 |
|
| 2351 | static void dbg_restore_debug_regs(void)
|
| 2352 | {
|
| 2353 | if (unlikely(kgdb_connected && arch_kgdb_ops.correct_hw_break))
|
| 2354 | arch_kgdb_ops.correct_hw_break();
|
| 2355 | }
|
| 2356 | #else
|
| 2357 | #define dbg_restore_debug_regs()
|
| 2358 | #endif
|
| 2359 |
|
| 2360 | static inline void setup_getcpu(int cpu)
|
| 2361 | {
|
| 2362 | unsigned long cpudata = vdso_encode_cpunode(cpu, early_cpu_to_node(cpu));
|
| 2363 | struct desc_struct d = { };
|
| 2364 |
|
| 2365 | if (boot_cpu_has(X86_FEATURE_RDTSCP) || boot_cpu_has(X86_FEATURE_RDPID))
|
| 2366 | wrmsrq(MSR_TSC_AUX, cpudata);
|
| 2367 |
|
| 2368 |
|
| 2369 | d.limit0 = cpudata;
|
| 2370 | d.limit1 = cpudata >> 16;
|
| 2371 |
|
| 2372 | d.type = 5;
|
| 2373 | d.dpl = 3;
|
| 2374 | d.s = 1;
|
| 2375 | d.p = 1;
|
| 2376 | d.d = 1;
|
| 2377 |
|
| 2378 | write_gdt_entry(get_cpu_gdt_rw(cpu), GDT_ENTRY_CPUNODE, &d, DESCTYPE_S);
|
| 2379 | }
|
| 2380 |
|
| 2381 | #ifdef CONFIG_X86_64
|
| 2382 | static inline void tss_setup_ist(struct tss_struct *tss)
|
| 2383 | {
|
| 2384 |
|
| 2385 | tss->x86_tss.ist[IST_INDEX_DF] = __this_cpu_ist_top_va(DF);
|
| 2386 | tss->x86_tss.ist[IST_INDEX_NMI] = __this_cpu_ist_top_va(NMI);
|
| 2387 | tss->x86_tss.ist[IST_INDEX_DB] = __this_cpu_ist_top_va(DB);
|
| 2388 | tss->x86_tss.ist[IST_INDEX_MCE] = __this_cpu_ist_top_va(MCE);
|
| 2389 |
|
| 2390 | tss->x86_tss.ist[IST_INDEX_VC] = __this_cpu_ist_top_va(VC);
|
| 2391 | }
|
| 2392 | #else
|
| 2393 | static inline void tss_setup_ist(struct tss_struct *tss) { }
|
| 2394 | #endif
|
| 2395 |
|
| 2396 | static inline void tss_setup_io_bitmap(struct tss_struct *tss)
|
| 2397 | {
|
| 2398 | tss->x86_tss.io_bitmap_base = IO_BITMAP_OFFSET_INVALID;
|
| 2399 |
|
| 2400 | #ifdef CONFIG_X86_IOPL_IOPERM
|
| 2401 | tss->io_bitmap.prev_max = 0;
|
| 2402 | tss->io_bitmap.prev_sequence = 0;
|
| 2403 | memset(tss->io_bitmap.bitmap, 0xff, sizeof(tss->io_bitmap.bitmap));
|
| 2404 | |
| 2405 | * Invalidate the extra array entry past the end of the all
|
| 2406 | * permission bitmap as required by the hardware.
|
| 2407 |
|
| 2408 | tss->io_bitmap.mapall[IO_BITMAP_LONGS] = ~0UL;
|
| 2409 | #endif
|
| 2410 | }
|
| 2411 |
|
| 2412 | |
| 2413 | * Setup everything needed to handle exceptions from the IDT, including the IST
|
| 2414 | * exceptions which use paranoid_entry().
|
| 2415 |
|
| 2416 | void cpu_init_exception_handling(bool boot_cpu)
|
| 2417 | {
|
| 2418 | struct tss_struct *tss = this_cpu_ptr(&cpu_tss_rw);
|
| 2419 | int cpu = raw_smp_processor_id();
|
| 2420 |
|
| 2421 |
|
| 2422 | setup_getcpu(cpu);
|
| 2423 |
|
| 2424 |
|
| 2425 | if (!cpu_feature_enabled(X86_FEATURE_FRED))
|
| 2426 | tss_setup_ist(tss);
|
| 2427 | tss_setup_io_bitmap(tss);
|
| 2428 | set_tss_desc(cpu, &get_cpu_entry_area(cpu)->tss.x86_tss);
|
| 2429 |
|
| 2430 | load_TR_desc();
|
| 2431 |
|
| 2432 |
|
| 2433 | setup_ghcb();
|
| 2434 |
|
| 2435 | |
| 2436 | * On CPUs with FSGSBASE support, paranoid_entry() uses
|
| 2437 | * ALTERNATIVE-patched RDGSBASE/WRGSBASE instructions. Secondary CPUs
|
| 2438 | * boot after alternatives are patched globally, so early exceptions
|
| 2439 | * execute patched code that depends on FSGSBASE. Enable the feature
|
| 2440 | * before any exceptions occur.
|
| 2441 |
|
| 2442 | if (cpu_feature_enabled(X86_FEATURE_FSGSBASE)) {
|
| 2443 | cr4_set_bits(X86_CR4_FSGSBASE);
|
| 2444 | elf_hwcap2 |= HWCAP2_FSGSBASE;
|
| 2445 | }
|
| 2446 |
|
| 2447 | if (cpu_feature_enabled(X86_FEATURE_FRED)) {
|
| 2448 |
|
| 2449 | if (!boot_cpu)
|
| 2450 | cpu_init_fred_exceptions();
|
| 2451 |
|
| 2452 | cpu_init_fred_rsps();
|
| 2453 | } else {
|
| 2454 | load_current_idt();
|
| 2455 | }
|
| 2456 | }
|
| 2457 |
|
| 2458 | void __init cpu_init_replace_early_idt(void)
|
| 2459 | {
|
| 2460 | if (cpu_feature_enabled(X86_FEATURE_FRED))
|
| 2461 | cpu_init_fred_exceptions();
|
| 2462 | else
|
| 2463 | idt_setup_early_pf();
|
| 2464 | }
|
| 2465 |
|
| 2466 | |
| 2467 | * cpu_init() initializes state that is per-CPU. Some data is already
|
| 2468 | * initialized (naturally) in the bootstrap process, such as the GDT. We
|
| 2469 | * reload it nevertheless, this function acts as a 'CPU state barrier',
|
| 2470 | * nothing should get across.
|
| 2471 |
|
| 2472 | void cpu_init(void)
|
| 2473 | {
|
| 2474 | struct task_struct *cur = current;
|
| 2475 | int cpu = raw_smp_processor_id();
|
| 2476 |
|
| 2477 | #ifdef CONFIG_NUMA
|
| 2478 | if (this_cpu_read(numa_node) == 0 &&
|
| 2479 | early_cpu_to_node(cpu) != NUMA_NO_NODE)
|
| 2480 | set_numa_node(early_cpu_to_node(cpu));
|
| 2481 | #endif
|
| 2482 | pr_debug("Initializing CPU#%d\n", cpu);
|
| 2483 |
|
| 2484 | if (IS_ENABLED(CONFIG_X86_64) || cpu_feature_enabled(X86_FEATURE_VME) ||
|
| 2485 | boot_cpu_has(X86_FEATURE_TSC) || boot_cpu_has(X86_FEATURE_DE))
|
| 2486 | cr4_clear_bits(X86_CR4_VME|X86_CR4_PVI|X86_CR4_TSD|X86_CR4_DE);
|
| 2487 |
|
| 2488 | if (IS_ENABLED(CONFIG_X86_64)) {
|
| 2489 | loadsegment(fs, 0);
|
| 2490 | memset(cur->thread.tls_array, 0, GDT_ENTRY_TLS_ENTRIES * 8);
|
| 2491 | syscall_init();
|
| 2492 |
|
| 2493 | wrmsrq(MSR_FS_BASE, 0);
|
| 2494 | wrmsrq(MSR_KERNEL_GS_BASE, 0);
|
| 2495 | barrier();
|
| 2496 |
|
| 2497 | x2apic_setup();
|
| 2498 |
|
| 2499 | intel_posted_msi_init();
|
| 2500 | }
|
| 2501 |
|
| 2502 | mmgrab(&init_mm);
|
| 2503 | cur->active_mm = &init_mm;
|
| 2504 | BUG_ON(cur->mm);
|
| 2505 | initialize_tlbstate_and_flush();
|
| 2506 | enter_lazy_tlb(&init_mm, cur);
|
| 2507 |
|
| 2508 | |
| 2509 | * sp0 points to the entry trampoline stack regardless of what task
|
| 2510 | * is running.
|
| 2511 |
|
| 2512 | load_sp0((unsigned long)(cpu_entry_stack(cpu) + 1));
|
| 2513 |
|
| 2514 | load_mm_ldt(&init_mm);
|
| 2515 |
|
| 2516 | initialize_debug_regs();
|
| 2517 | dbg_restore_debug_regs();
|
| 2518 |
|
| 2519 | doublefault_init_cpu_tss();
|
| 2520 |
|
| 2521 | if (is_uv_system())
|
| 2522 | uv_cpu_init();
|
| 2523 |
|
| 2524 | load_fixmap_gdt(cpu);
|
| 2525 | }
|
| 2526 |
|
| 2527 | #ifdef CONFIG_MICROCODE_LATE_LOADING
|
| 2528 | |
| 2529 | * store_cpu_caps() - Store a snapshot of CPU capabilities
|
| 2530 | * @curr_info: Pointer where to store it
|
| 2531 | *
|
| 2532 | * Returns: None
|
| 2533 |
|
| 2534 | void store_cpu_caps(struct cpuinfo_x86 *curr_info)
|
| 2535 | {
|
| 2536 |
|
| 2537 | curr_info->cpuid_level = cpuid_eax(0);
|
| 2538 |
|
| 2539 |
|
| 2540 | memcpy(&curr_info->x86_capability, &boot_cpu_data.x86_capability,
|
| 2541 | sizeof(curr_info->x86_capability));
|
| 2542 |
|
| 2543 |
|
| 2544 | get_cpu_cap(curr_info);
|
| 2545 | }
|
| 2546 |
|
| 2547 | |
| 2548 | * microcode_check() - Check if any CPU capabilities changed after an update.
|
| 2549 | * @prev_info: CPU capabilities stored before an update.
|
| 2550 | *
|
| 2551 | * The microcode loader calls this upon late microcode load to recheck features,
|
| 2552 | * only when microcode has been updated. Caller holds and CPU hotplug lock.
|
| 2553 | *
|
| 2554 | * Return: None
|
| 2555 |
|
| 2556 | void microcode_check(struct cpuinfo_x86 *prev_info)
|
| 2557 | {
|
| 2558 | struct cpuinfo_x86 curr_info;
|
| 2559 |
|
| 2560 | perf_check_microcode();
|
| 2561 |
|
| 2562 | amd_check_microcode();
|
| 2563 |
|
| 2564 | store_cpu_caps(&curr_info);
|
| 2565 |
|
| 2566 | if (!memcmp(&prev_info->x86_capability, &curr_info.x86_capability,
|
| 2567 | sizeof(prev_info->x86_capability)))
|
| 2568 | return;
|
| 2569 |
|
| 2570 | pr_warn("x86/CPU: CPU features have changed after loading microcode, but might not take effect.\n");
|
| 2571 | pr_warn("x86/CPU: Please consider either early loading through initrd/built-in or a potential BIOS update.\n");
|
| 2572 | }
|
| 2573 | #endif
|
| 2574 |
|
| 2575 | |
| 2576 | * Invoked from core CPU hotplug code after hotplug operations
|
| 2577 |
|
| 2578 | void arch_smt_update(void)
|
| 2579 | {
|
| 2580 |
|
| 2581 | cpu_bugs_smt_update();
|
| 2582 |
|
| 2583 | apic_smt_update();
|
| 2584 | }
|
| 2585 |
|
| 2586 | void __init arch_cpu_finalize_init(void)
|
| 2587 | {
|
| 2588 | struct cpuinfo_x86 *c = this_cpu_ptr(&cpu_info);
|
| 2589 |
|
| 2590 | identify_boot_cpu();
|
| 2591 |
|
| 2592 | select_idle_routine();
|
| 2593 |
|
| 2594 | |
| 2595 | * identify_boot_cpu() initialized SMT support information, let the
|
| 2596 | * core code know.
|
| 2597 |
|
| 2598 | cpu_smt_set_num_threads(__max_threads_per_core, __max_threads_per_core);
|
| 2599 |
|
| 2600 | if (!IS_ENABLED(CONFIG_SMP)) {
|
| 2601 | pr_info("CPU: ");
|
| 2602 | print_cpu_info(&boot_cpu_data);
|
| 2603 | }
|
| 2604 |
|
| 2605 | cpu_select_mitigations();
|
| 2606 |
|
| 2607 | arch_smt_update();
|
| 2608 |
|
| 2609 | if (IS_ENABLED(CONFIG_X86_32)) {
|
| 2610 | |
| 2611 | * Check whether this is a real i386 which is not longer
|
| 2612 | * supported and fixup the utsname.
|
| 2613 |
|
| 2614 | if (boot_cpu_data.x86 < 4)
|
| 2615 | panic("Kernel requires i486+ for 'invlpg' and other features");
|
| 2616 |
|
| 2617 | init_utsname()->machine[1] =
|
| 2618 | '0' + (boot_cpu_data.x86 > 6 ? 6 : boot_cpu_data.x86);
|
| 2619 | }
|
| 2620 |
|
| 2621 | |
| 2622 | * Must be before alternatives because it might set or clear
|
| 2623 | * feature bits.
|
| 2624 |
|
| 2625 | fpu__init_system();
|
| 2626 | fpu__init_cpu();
|
| 2627 |
|
| 2628 | |
| 2629 | * This needs to follow the FPU initializtion, since EFI depends on it.
|
| 2630 |
|
| 2631 | if (efi_enabled(EFI_RUNTIME_SERVICES))
|
| 2632 | efi_enter_virtual_mode();
|
| 2633 |
|
| 2634 | |
| 2635 | * Ensure that access to the per CPU representation has the initial
|
| 2636 | * boot CPU configuration.
|
| 2637 |
|
| 2638 | *c = boot_cpu_data;
|
| 2639 | c->initialized = true;
|
| 2640 |
|
| 2641 | alternative_instructions();
|
| 2642 |
|
| 2643 | if (IS_ENABLED(CONFIG_X86_64)) {
|
| 2644 | USER_PTR_MAX = TASK_SIZE_MAX;
|
| 2645 |
|
| 2646 | |
| 2647 | * Enable this when LAM is gated on LASS support
|
| 2648 | if (cpu_feature_enabled(X86_FEATURE_LAM))
|
| 2649 | USER_PTR_MAX = (1ul << 63) - PAGE_SIZE;
|
| 2650 |
|
| 2651 | runtime_const_init(ptr, USER_PTR_MAX);
|
| 2652 |
|
| 2653 | |
| 2654 | * Make sure the first 2MB area is not mapped by huge pages
|
| 2655 | * There are typically fixed size MTRRs in there and overlapping
|
| 2656 | * MTRRs into large pages causes slow downs.
|
| 2657 | *
|
| 2658 | * Right now we don't do that with gbpages because there seems
|
| 2659 | * very little benefit for that case.
|
| 2660 |
|
| 2661 | if (!direct_gbpages)
|
| 2662 | set_memory_4k((unsigned long)__va(0), 1);
|
| 2663 | } else {
|
| 2664 | fpu__init_check_bugs();
|
| 2665 | }
|
| 2666 |
|
| 2667 | |
| 2668 | * This needs to be called before any devices perform DMA
|
| 2669 | * operations that might use the SWIOTLB bounce buffers. It will
|
| 2670 | * mark the bounce buffers as decrypted so that their usage will
|
| 2671 | * not cause "plain-text" data to be decrypted when accessed. It
|
| 2672 | * must be called after late_time_init() so that Hyper-V x86/x64
|
| 2673 | * hypercalls work when the SWIOTLB bounce buffers are decrypted.
|
| 2674 |
|
| 2675 | mem_encrypt_init();
|
| 2676 | }
|
| 2677 |
|
| 2678 |
|
| 2679 | static int __init tlbi_setup(char *str)
|
| 2680 | {
|
| 2681 | if (!strcmp(str, "ipi"))
|
| 2682 | setup_clear_cpu_cap(X86_FEATURE_INVLPGB);
|
| 2683 |
|
| 2684 | return 1;
|
| 2685 | }
|
| 2686 | __setup("tlbi=", tlbi_setup);
|