| 1 | |
| 2 | |
| 3 | * Machine check handler.
|
| 4 | *
|
| 5 | * K8 parts Copyright 2002,2003 Andi Kleen, SuSE Labs.
|
| 6 | * Rest from unknown author(s).
|
| 7 | * 2004 Andi Kleen. Rewrote most of it.
|
| 8 | * Copyright 2008 Intel Corporation
|
| 9 | * Author: Andi Kleen
|
| 10 |
|
| 11 |
|
| 12 | #include <linux/thread_info.h>
|
| 13 | #include <linux/capability.h>
|
| 14 | #include <linux/miscdevice.h>
|
| 15 | #include <linux/ratelimit.h>
|
| 16 | #include <linux/rcupdate.h>
|
| 17 | #include <linux/kobject.h>
|
| 18 | #include <linux/uaccess.h>
|
| 19 | #include <linux/kdebug.h>
|
| 20 | #include <linux/kernel.h>
|
| 21 | #include <linux/percpu.h>
|
| 22 | #include <linux/string.h>
|
| 23 | #include <linux/device.h>
|
| 24 | #include <linux/syscore_ops.h>
|
| 25 | #include <linux/delay.h>
|
| 26 | #include <linux/ctype.h>
|
| 27 | #include <linux/sched.h>
|
| 28 | #include <linux/sysfs.h>
|
| 29 | #include <linux/types.h>
|
| 30 | #include <linux/slab.h>
|
| 31 | #include <linux/init.h>
|
| 32 | #include <linux/kmod.h>
|
| 33 | #include <linux/poll.h>
|
| 34 | #include <linux/nmi.h>
|
| 35 | #include <linux/cpu.h>
|
| 36 | #include <linux/ras.h>
|
| 37 | #include <linux/smp.h>
|
| 38 | #include <linux/fs.h>
|
| 39 | #include <linux/mm.h>
|
| 40 | #include <linux/debugfs.h>
|
| 41 | #include <linux/irq_work.h>
|
| 42 | #include <linux/export.h>
|
| 43 | #include <linux/set_memory.h>
|
| 44 | #include <linux/sync_core.h>
|
| 45 | #include <linux/task_work.h>
|
| 46 | #include <linux/hardirq.h>
|
| 47 | #include <linux/kexec.h>
|
| 48 | #include <linux/vmcore_info.h>
|
| 49 |
|
| 50 | #include <asm/fred.h>
|
| 51 | #include <asm/cpu_device_id.h>
|
| 52 | #include <asm/cpuid/api.h>
|
| 53 | #include <asm/processor.h>
|
| 54 | #include <asm/traps.h>
|
| 55 | #include <asm/tlbflush.h>
|
| 56 | #include <asm/mce.h>
|
| 57 | #include <asm/msr.h>
|
| 58 | #include <asm/reboot.h>
|
| 59 | #include <asm/tdx.h>
|
| 60 |
|
| 61 | #include "internal.h"
|
| 62 |
|
| 63 |
|
| 64 | static DEFINE_MUTEX(mce_sysfs_mutex);
|
| 65 |
|
| 66 | #define CREATE_TRACE_POINTS
|
| 67 | #include <trace/events/mce.h>
|
| 68 |
|
| 69 | #define SPINUNIT 100
|
| 70 |
|
| 71 | DEFINE_PER_CPU_READ_MOSTLY(unsigned int, mce_num_banks);
|
| 72 |
|
| 73 | DEFINE_PER_CPU_READ_MOSTLY(struct mce_bank[MAX_NR_BANKS], mce_banks_array);
|
| 74 |
|
| 75 | #define ATTR_LEN 16
|
| 76 |
|
| 77 | struct mce_bank_dev {
|
| 78 | struct device_attribute attr;
|
| 79 | char attrname[ATTR_LEN];
|
| 80 | u8 bank;
|
| 81 | };
|
| 82 | static struct mce_bank_dev mce_bank_devs[MAX_NR_BANKS];
|
| 83 |
|
| 84 | struct mce_vendor_flags mce_flags __read_mostly;
|
| 85 |
|
| 86 | struct mca_config mca_cfg __read_mostly = {
|
| 87 | .bootlog = -1,
|
| 88 | .monarch_timeout = -1
|
| 89 | };
|
| 90 |
|
| 91 | static DEFINE_PER_CPU(struct mce_hw_err, hw_errs_seen);
|
| 92 |
|
| 93 | |
| 94 | * MCA banks polled by the period polling timer for corrected events.
|
| 95 | * With Intel CMCI, this only has MCA banks which do not support CMCI (if any).
|
| 96 |
|
| 97 | DEFINE_PER_CPU(mce_banks_t, mce_poll_banks) = {
|
| 98 | [0 ... BITS_TO_LONGS(MAX_NR_BANKS)-1] = ~0UL
|
| 99 | };
|
| 100 |
|
| 101 | |
| 102 | * MCA banks controlled through firmware first for corrected errors.
|
| 103 | * This is a global list of banks for which we won't enable CMCI and we
|
| 104 | * won't poll. Firmware controls these banks and is responsible for
|
| 105 | * reporting corrected errors through GHES. Uncorrected/recoverable
|
| 106 | * errors are still notified through a machine check.
|
| 107 |
|
| 108 | mce_banks_t mce_banks_ce_disabled;
|
| 109 |
|
| 110 | static struct work_struct mce_work;
|
| 111 | static struct irq_work mce_irq_work;
|
| 112 |
|
| 113 | |
| 114 | * CPU/chipset specific EDAC code can register a notifier call here to print
|
| 115 | * MCE errors in a human-readable form.
|
| 116 |
|
| 117 | BLOCKING_NOTIFIER_HEAD(x86_mce_decoder_chain);
|
| 118 |
|
| 119 | void mce_prep_record_common(struct mce *m)
|
| 120 | {
|
| 121 | m->cpuid = cpuid_eax(1);
|
| 122 | m->cpuvendor = boot_cpu_data.x86_vendor;
|
| 123 | m->mcgcap = native_rdmsrq(MSR_IA32_MCG_CAP);
|
| 124 |
|
| 125 | m->time = __ktime_get_real_seconds();
|
| 126 | }
|
| 127 |
|
| 128 | void mce_prep_record_per_cpu(unsigned int cpu, struct mce *m)
|
| 129 | {
|
| 130 | m->cpu = cpu;
|
| 131 | m->extcpu = cpu;
|
| 132 | m->apicid = cpu_data(cpu).topo.initial_apicid;
|
| 133 | m->microcode = cpu_data(cpu).microcode;
|
| 134 | m->ppin = topology_ppin(cpu);
|
| 135 | m->socketid = topology_physical_package_id(cpu);
|
| 136 | }
|
| 137 |
|
| 138 |
|
| 139 | void mce_prep_record(struct mce_hw_err *err)
|
| 140 | {
|
| 141 | struct mce *m = &err->m;
|
| 142 |
|
| 143 | memset(err, 0, sizeof(struct mce_hw_err));
|
| 144 | mce_prep_record_common(m);
|
| 145 | mce_prep_record_per_cpu(smp_processor_id(), m);
|
| 146 | }
|
| 147 |
|
| 148 | DEFINE_PER_CPU(struct mce, injectm);
|
| 149 | EXPORT_PER_CPU_SYMBOL_GPL(injectm);
|
| 150 |
|
| 151 | void mce_log(struct mce_hw_err *err)
|
| 152 | {
|
| 153 | if (mce_gen_pool_add(err)) {
|
| 154 | pr_info(HW_ERR "Machine check events logged\n");
|
| 155 | irq_work_queue(&mce_irq_work);
|
| 156 | }
|
| 157 | }
|
| 158 | EXPORT_SYMBOL_GPL(mce_log);
|
| 159 |
|
| 160 | void mce_register_decode_chain(struct notifier_block *nb)
|
| 161 | {
|
| 162 | if (WARN_ON(nb->priority < MCE_PRIO_LOWEST ||
|
| 163 | nb->priority > MCE_PRIO_HIGHEST))
|
| 164 | return;
|
| 165 |
|
| 166 | blocking_notifier_chain_register(&x86_mce_decoder_chain, nb);
|
| 167 | }
|
| 168 | EXPORT_SYMBOL_GPL(mce_register_decode_chain);
|
| 169 |
|
| 170 | void mce_unregister_decode_chain(struct notifier_block *nb)
|
| 171 | {
|
| 172 | blocking_notifier_chain_unregister(&x86_mce_decoder_chain, nb);
|
| 173 | }
|
| 174 | EXPORT_SYMBOL_GPL(mce_unregister_decode_chain);
|
| 175 |
|
| 176 | static void __print_mce(struct mce_hw_err *err)
|
| 177 | {
|
| 178 | struct mce *m = &err->m;
|
| 179 |
|
| 180 | pr_emerg(HW_ERR "CPU %d: Machine Check%s: %Lx Bank %d: %016Lx\n",
|
| 181 | m->extcpu,
|
| 182 | (m->mcgstatus & MCG_STATUS_MCIP ? " Exception" : ""),
|
| 183 | m->mcgstatus, m->bank, m->status);
|
| 184 |
|
| 185 | if (m->ip) {
|
| 186 | pr_emerg(HW_ERR "RIP%s %02x:<%016Lx> ",
|
| 187 | !(m->mcgstatus & MCG_STATUS_EIPV) ? " !INEXACT!" : "",
|
| 188 | m->cs, m->ip);
|
| 189 |
|
| 190 | if (m->cs == __KERNEL_CS)
|
| 191 | pr_cont("{%pS}", (void *)(unsigned long)m->ip);
|
| 192 | pr_cont("\n");
|
| 193 | }
|
| 194 |
|
| 195 | pr_emerg(HW_ERR "TSC %llx ", m->tsc);
|
| 196 | if (m->addr)
|
| 197 | pr_cont("ADDR %llx ", m->addr);
|
| 198 | if (m->misc)
|
| 199 | pr_cont("MISC %llx ", m->misc);
|
| 200 | if (m->ppin)
|
| 201 | pr_cont("PPIN %llx ", m->ppin);
|
| 202 |
|
| 203 | if (mce_flags.smca) {
|
| 204 | if (m->synd)
|
| 205 | pr_cont("SYND %llx ", m->synd);
|
| 206 | if (err->vendor.amd.synd1)
|
| 207 | pr_cont("SYND1 %llx ", err->vendor.amd.synd1);
|
| 208 | if (err->vendor.amd.synd2)
|
| 209 | pr_cont("SYND2 %llx ", err->vendor.amd.synd2);
|
| 210 | if (m->ipid)
|
| 211 | pr_cont("IPID %llx ", m->ipid);
|
| 212 | }
|
| 213 |
|
| 214 | pr_cont("\n");
|
| 215 |
|
| 216 | |
| 217 | * Note this output is parsed by external tools and old fields
|
| 218 | * should not be changed.
|
| 219 |
|
| 220 | pr_emerg(HW_ERR "PROCESSOR %u:%x TIME %llu SOCKET %u APIC %x microcode %x\n",
|
| 221 | m->cpuvendor, m->cpuid, m->time, m->socketid, m->apicid,
|
| 222 | m->microcode);
|
| 223 | }
|
| 224 |
|
| 225 | static void print_mce(struct mce_hw_err *err)
|
| 226 | {
|
| 227 | struct mce *m = &err->m;
|
| 228 |
|
| 229 | __print_mce(err);
|
| 230 |
|
| 231 | if (m->cpuvendor != X86_VENDOR_AMD && m->cpuvendor != X86_VENDOR_HYGON)
|
| 232 | pr_emerg_ratelimited(HW_ERR "Run the above through 'mcelog --ascii'\n");
|
| 233 | }
|
| 234 |
|
| 235 | #define PANIC_TIMEOUT 5
|
| 236 |
|
| 237 | static atomic_t mce_panicked;
|
| 238 |
|
| 239 | static int fake_panic;
|
| 240 | static atomic_t mce_fake_panicked;
|
| 241 |
|
| 242 |
|
| 243 | static void wait_for_panic(void)
|
| 244 | {
|
| 245 | long timeout = PANIC_TIMEOUT*USEC_PER_SEC;
|
| 246 |
|
| 247 | preempt_disable();
|
| 248 | local_irq_enable();
|
| 249 | while (timeout-- > 0)
|
| 250 | udelay(1);
|
| 251 | if (panic_timeout == 0)
|
| 252 | panic_timeout = mca_cfg.panic_timeout;
|
| 253 | panic("Panicing machine check CPU died");
|
| 254 | }
|
| 255 |
|
| 256 | static const char *mce_dump_aux_info(struct mce *m)
|
| 257 | {
|
| 258 | if (boot_cpu_has_bug(X86_BUG_TDX_PW_MCE))
|
| 259 | return tdx_dump_mce_info(m);
|
| 260 |
|
| 261 | return NULL;
|
| 262 | }
|
| 263 |
|
| 264 | static noinstr void mce_panic(const char *msg, struct mce_hw_err *final, char *exp)
|
| 265 | {
|
| 266 | struct llist_node *pending;
|
| 267 | struct mce_evt_llist *l;
|
| 268 | int apei_err = 0;
|
| 269 | const char *memmsg;
|
| 270 |
|
| 271 | |
| 272 | * Allow instrumentation around external facilities usage. Not that it
|
| 273 | * matters a whole lot since the machine is going to panic anyway.
|
| 274 |
|
| 275 | instrumentation_begin();
|
| 276 |
|
| 277 | if (!fake_panic) {
|
| 278 | |
| 279 | * Make sure only one CPU runs in machine check panic
|
| 280 |
|
| 281 | if (atomic_inc_return(&mce_panicked) > 1)
|
| 282 | wait_for_panic();
|
| 283 | barrier();
|
| 284 |
|
| 285 | bust_spinlocks(1);
|
| 286 | console_verbose();
|
| 287 | } else {
|
| 288 |
|
| 289 | if (atomic_inc_return(&mce_fake_panicked) > 1)
|
| 290 | goto out;
|
| 291 | }
|
| 292 | pending = mce_gen_pool_prepare_records();
|
| 293 |
|
| 294 | llist_for_each_entry(l, pending, llnode) {
|
| 295 | struct mce_hw_err *err = &l->err;
|
| 296 | struct mce *m = &err->m;
|
| 297 | if (!(m->status & MCI_STATUS_UC)) {
|
| 298 | print_mce(err);
|
| 299 | if (!apei_err)
|
| 300 | apei_err = apei_write_mce(m);
|
| 301 | }
|
| 302 | }
|
| 303 |
|
| 304 | llist_for_each_entry(l, pending, llnode) {
|
| 305 | struct mce_hw_err *err = &l->err;
|
| 306 | struct mce *m = &err->m;
|
| 307 | if (!(m->status & MCI_STATUS_UC))
|
| 308 | continue;
|
| 309 | if (!final || mce_cmp(m, &final->m)) {
|
| 310 | print_mce(err);
|
| 311 | if (!apei_err)
|
| 312 | apei_err = apei_write_mce(m);
|
| 313 | }
|
| 314 | }
|
| 315 | if (final) {
|
| 316 | print_mce(final);
|
| 317 | if (!apei_err)
|
| 318 | apei_err = apei_write_mce(&final->m);
|
| 319 | }
|
| 320 | if (exp)
|
| 321 | pr_emerg(HW_ERR "Machine check: %s\n", exp);
|
| 322 |
|
| 323 | memmsg = mce_dump_aux_info(&final->m);
|
| 324 | if (memmsg)
|
| 325 | pr_emerg(HW_ERR "Machine check: %s\n", memmsg);
|
| 326 |
|
| 327 | if (!fake_panic) {
|
| 328 | if (panic_timeout == 0)
|
| 329 | panic_timeout = mca_cfg.panic_timeout;
|
| 330 |
|
| 331 | |
| 332 | * Kdump skips the poisoned page in order to avoid
|
| 333 | * touching the error bits again. Poison the page even
|
| 334 | * if the error is fatal and the machine is about to
|
| 335 | * panic.
|
| 336 |
|
| 337 | if (kexec_crash_loaded()) {
|
| 338 | if (final && (final->m.status & MCI_STATUS_ADDRV)) {
|
| 339 | struct page *p;
|
| 340 | p = pfn_to_online_page(final->m.addr >> PAGE_SHIFT);
|
| 341 | if (p)
|
| 342 | SetPageHWPoison(p);
|
| 343 | }
|
| 344 | }
|
| 345 | panic(msg);
|
| 346 | } else
|
| 347 | pr_emerg(HW_ERR "Fake kernel panic: %s\n", msg);
|
| 348 |
|
| 349 | out:
|
| 350 | instrumentation_end();
|
| 351 | }
|
| 352 |
|
| 353 |
|
| 354 |
|
| 355 | static int msr_to_offset(u32 msr)
|
| 356 | {
|
| 357 | unsigned bank = __this_cpu_read(injectm.bank);
|
| 358 |
|
| 359 | if (msr == mca_cfg.rip_msr)
|
| 360 | return offsetof(struct mce, ip);
|
| 361 | if (msr == mca_msr_reg(bank, MCA_STATUS))
|
| 362 | return offsetof(struct mce, status);
|
| 363 | if (msr == mca_msr_reg(bank, MCA_ADDR))
|
| 364 | return offsetof(struct mce, addr);
|
| 365 | if (msr == mca_msr_reg(bank, MCA_MISC))
|
| 366 | return offsetof(struct mce, misc);
|
| 367 | if (msr == MSR_IA32_MCG_STATUS)
|
| 368 | return offsetof(struct mce, mcgstatus);
|
| 369 | return -1;
|
| 370 | }
|
| 371 |
|
| 372 | void ex_handler_msr_mce(struct pt_regs *regs, bool wrmsr)
|
| 373 | {
|
| 374 | if (wrmsr) {
|
| 375 | pr_emerg("MSR access error: WRMSR to 0x%x (tried to write 0x%08x%08x) at rIP: 0x%lx (%pS)\n",
|
| 376 | (unsigned int)regs->cx, (unsigned int)regs->dx, (unsigned int)regs->ax,
|
| 377 | regs->ip, (void *)regs->ip);
|
| 378 | } else {
|
| 379 | pr_emerg("MSR access error: RDMSR from 0x%x at rIP: 0x%lx (%pS)\n",
|
| 380 | (unsigned int)regs->cx, regs->ip, (void *)regs->ip);
|
| 381 | }
|
| 382 |
|
| 383 | show_stack_regs(regs);
|
| 384 |
|
| 385 | panic("MCA architectural violation!\n");
|
| 386 |
|
| 387 | while (true)
|
| 388 | cpu_relax();
|
| 389 | }
|
| 390 |
|
| 391 |
|
| 392 | noinstr u64 mce_rdmsrq(u32 msr)
|
| 393 | {
|
| 394 | EAX_EDX_DECLARE_ARGS(val, low, high);
|
| 395 |
|
| 396 | if (__this_cpu_read(injectm.finished)) {
|
| 397 | int offset;
|
| 398 | u64 ret;
|
| 399 |
|
| 400 | instrumentation_begin();
|
| 401 |
|
| 402 | offset = msr_to_offset(msr);
|
| 403 | if (offset < 0)
|
| 404 | ret = 0;
|
| 405 | else
|
| 406 | ret = *(u64 *)((char *)this_cpu_ptr(&injectm) + offset);
|
| 407 |
|
| 408 | instrumentation_end();
|
| 409 |
|
| 410 | return ret;
|
| 411 | }
|
| 412 |
|
| 413 | |
| 414 | * RDMSR on MCA MSRs should not fault. If they do, this is very much an
|
| 415 | * architectural violation and needs to be reported to hw vendor. Panic
|
| 416 | * the box to not allow any further progress.
|
| 417 |
|
| 418 | asm volatile("1: rdmsr\n"
|
| 419 | "2:\n"
|
| 420 | _ASM_EXTABLE_TYPE(1b, 2b, EX_TYPE_RDMSR_IN_MCE)
|
| 421 | : EAX_EDX_RET(val, low, high) : "c" (msr));
|
| 422 |
|
| 423 |
|
| 424 | return EAX_EDX_VAL(val, low, high);
|
| 425 | }
|
| 426 |
|
| 427 | noinstr void mce_wrmsrq(u32 msr, u64 v)
|
| 428 | {
|
| 429 | u32 low, high;
|
| 430 |
|
| 431 | if (__this_cpu_read(injectm.finished)) {
|
| 432 | int offset;
|
| 433 |
|
| 434 | instrumentation_begin();
|
| 435 |
|
| 436 | offset = msr_to_offset(msr);
|
| 437 | if (offset >= 0)
|
| 438 | *(u64 *)((char *)this_cpu_ptr(&injectm) + offset) = v;
|
| 439 |
|
| 440 | instrumentation_end();
|
| 441 |
|
| 442 | return;
|
| 443 | }
|
| 444 |
|
| 445 | low = (u32)v;
|
| 446 | high = (u32)(v >> 32);
|
| 447 |
|
| 448 |
|
| 449 | asm volatile("1: wrmsr\n"
|
| 450 | "2:\n"
|
| 451 | _ASM_EXTABLE_TYPE(1b, 2b, EX_TYPE_WRMSR_IN_MCE)
|
| 452 | : : "c" (msr), "a"(low), "d" (high) : "memory");
|
| 453 | }
|
| 454 |
|
| 455 | |
| 456 | * Collect all global (w.r.t. this processor) status about this machine
|
| 457 | * check into our "mce" struct so that we can use it later to assess
|
| 458 | * the severity of the problem as we read per-bank specific details.
|
| 459 |
|
| 460 | static noinstr void mce_gather_info(struct mce_hw_err *err, struct pt_regs *regs)
|
| 461 | {
|
| 462 | struct mce *m;
|
| 463 | |
| 464 | * Enable instrumentation around mce_prep_record() which calls external
|
| 465 | * facilities.
|
| 466 |
|
| 467 | instrumentation_begin();
|
| 468 | mce_prep_record(err);
|
| 469 | instrumentation_end();
|
| 470 |
|
| 471 | m = &err->m;
|
| 472 | m->mcgstatus = mce_rdmsrq(MSR_IA32_MCG_STATUS);
|
| 473 | if (regs) {
|
| 474 | |
| 475 | * Get the address of the instruction at the time of
|
| 476 | * the machine check error.
|
| 477 |
|
| 478 | if (m->mcgstatus & (MCG_STATUS_RIPV|MCG_STATUS_EIPV)) {
|
| 479 | m->ip = regs->ip;
|
| 480 | m->cs = regs->cs;
|
| 481 |
|
| 482 | |
| 483 | * When in VM86 mode make the cs look like ring 3
|
| 484 | * always. This is a lie, but it's better than passing
|
| 485 | * the additional vm86 bit around everywhere.
|
| 486 |
|
| 487 | if (v8086_mode(regs))
|
| 488 | m->cs |= 3;
|
| 489 | }
|
| 490 |
|
| 491 | if (mca_cfg.rip_msr)
|
| 492 | m->ip = mce_rdmsrq(mca_cfg.rip_msr);
|
| 493 | }
|
| 494 | }
|
| 495 |
|
| 496 | bool mce_available(struct cpuinfo_x86 *c)
|
| 497 | {
|
| 498 | if (mca_cfg.disabled)
|
| 499 | return false;
|
| 500 | return cpu_has(c, X86_FEATURE_MCE) && cpu_has(c, X86_FEATURE_MCA);
|
| 501 | }
|
| 502 |
|
| 503 | static void mce_schedule_work(void)
|
| 504 | {
|
| 505 | if (!mce_gen_pool_empty())
|
| 506 | schedule_work(&mce_work);
|
| 507 | }
|
| 508 |
|
| 509 | static void mce_irq_work_cb(struct irq_work *entry)
|
| 510 | {
|
| 511 | mce_schedule_work();
|
| 512 | }
|
| 513 |
|
| 514 | bool mce_usable_address(struct mce *m)
|
| 515 | {
|
| 516 | if (!(m->status & MCI_STATUS_ADDRV))
|
| 517 | return false;
|
| 518 |
|
| 519 | switch (m->cpuvendor) {
|
| 520 | case X86_VENDOR_AMD:
|
| 521 | return amd_mce_usable_address(m);
|
| 522 |
|
| 523 | case X86_VENDOR_INTEL:
|
| 524 | case X86_VENDOR_ZHAOXIN:
|
| 525 | return intel_mce_usable_address(m);
|
| 526 |
|
| 527 | default:
|
| 528 | return true;
|
| 529 | }
|
| 530 | }
|
| 531 | EXPORT_SYMBOL_GPL(mce_usable_address);
|
| 532 |
|
| 533 | bool mce_is_memory_error(struct mce *m)
|
| 534 | {
|
| 535 | switch (m->cpuvendor) {
|
| 536 | case X86_VENDOR_AMD:
|
| 537 | case X86_VENDOR_HYGON:
|
| 538 | return amd_mce_is_memory_error(m);
|
| 539 |
|
| 540 | case X86_VENDOR_INTEL:
|
| 541 | case X86_VENDOR_ZHAOXIN:
|
| 542 | |
| 543 | * Intel SDM Volume 3B - 15.9.2 Compound Error Codes
|
| 544 | *
|
| 545 | * Bit 7 of the MCACOD field of IA32_MCi_STATUS is used for
|
| 546 | * indicating a memory error. Bit 8 is used for indicating a
|
| 547 | * cache hierarchy error. The combination of bit 2 and bit 3
|
| 548 | * is used for indicating a `generic' cache hierarchy error
|
| 549 | * But we can't just blindly check the above bits, because if
|
| 550 | * bit 11 is set, then it is a bus/interconnect error - and
|
| 551 | * either way the above bits just gives more detail on what
|
| 552 | * bus/interconnect error happened. Note that bit 12 can be
|
| 553 | * ignored, as it's the "filter" bit.
|
| 554 |
|
| 555 | return (m->status & 0xef80) == BIT(7) ||
|
| 556 | (m->status & 0xef00) == BIT(8) ||
|
| 557 | (m->status & 0xeffc) == 0xc;
|
| 558 |
|
| 559 | default:
|
| 560 | return false;
|
| 561 | }
|
| 562 | }
|
| 563 | EXPORT_SYMBOL_GPL(mce_is_memory_error);
|
| 564 |
|
| 565 | static bool whole_page(struct mce *m)
|
| 566 | {
|
| 567 | if (!mca_cfg.ser || !(m->status & MCI_STATUS_MISCV))
|
| 568 | return true;
|
| 569 |
|
| 570 | return MCI_MISC_ADDR_LSB(m->misc) >= PAGE_SHIFT;
|
| 571 | }
|
| 572 |
|
| 573 | bool mce_is_correctable(struct mce *m)
|
| 574 | {
|
| 575 | if (m->cpuvendor == X86_VENDOR_AMD && m->status & MCI_STATUS_DEFERRED)
|
| 576 | return false;
|
| 577 |
|
| 578 | if (m->cpuvendor == X86_VENDOR_HYGON && m->status & MCI_STATUS_DEFERRED)
|
| 579 | return false;
|
| 580 |
|
| 581 | if (m->status & MCI_STATUS_UC)
|
| 582 | return false;
|
| 583 |
|
| 584 | return true;
|
| 585 | }
|
| 586 | EXPORT_SYMBOL_GPL(mce_is_correctable);
|
| 587 |
|
| 588 | static int mce_early_notifier(struct notifier_block *nb, unsigned long val,
|
| 589 | void *data)
|
| 590 | {
|
| 591 | struct mce_hw_err *err = to_mce_hw_err(data);
|
| 592 |
|
| 593 | if (!err)
|
| 594 | return NOTIFY_DONE;
|
| 595 |
|
| 596 |
|
| 597 | trace_mce_record(err);
|
| 598 |
|
| 599 | mce_work_trigger();
|
| 600 |
|
| 601 | return NOTIFY_DONE;
|
| 602 | }
|
| 603 |
|
| 604 | static struct notifier_block early_nb = {
|
| 605 | .notifier_call = mce_early_notifier,
|
| 606 | .priority = MCE_PRIO_EARLY,
|
| 607 | };
|
| 608 |
|
| 609 | static int uc_decode_notifier(struct notifier_block *nb, unsigned long val,
|
| 610 | void *data)
|
| 611 | {
|
| 612 | struct mce *mce = (struct mce *)data;
|
| 613 | unsigned long pfn;
|
| 614 |
|
| 615 | if (!mce || !mce_usable_address(mce))
|
| 616 | return NOTIFY_DONE;
|
| 617 |
|
| 618 | if (mce->severity != MCE_AO_SEVERITY &&
|
| 619 | mce->severity != MCE_DEFERRED_SEVERITY)
|
| 620 | return NOTIFY_DONE;
|
| 621 |
|
| 622 | pfn = (mce->addr & MCI_ADDR_PHYSADDR) >> PAGE_SHIFT;
|
| 623 | if (!memory_failure(pfn, 0)) {
|
| 624 | set_mce_nospec(pfn);
|
| 625 | mce->kflags |= MCE_HANDLED_UC;
|
| 626 | }
|
| 627 |
|
| 628 | return NOTIFY_OK;
|
| 629 | }
|
| 630 |
|
| 631 | static struct notifier_block mce_uc_nb = {
|
| 632 | .notifier_call = uc_decode_notifier,
|
| 633 | .priority = MCE_PRIO_UC,
|
| 634 | };
|
| 635 |
|
| 636 | static int mce_default_notifier(struct notifier_block *nb, unsigned long val,
|
| 637 | void *data)
|
| 638 | {
|
| 639 | struct mce_hw_err *err = to_mce_hw_err(data);
|
| 640 |
|
| 641 | if (!err)
|
| 642 | return NOTIFY_DONE;
|
| 643 |
|
| 644 | if (mca_cfg.print_all || !(err->m.kflags))
|
| 645 | __print_mce(err);
|
| 646 |
|
| 647 | return NOTIFY_DONE;
|
| 648 | }
|
| 649 |
|
| 650 | static struct notifier_block mce_default_nb = {
|
| 651 | .notifier_call = mce_default_notifier,
|
| 652 |
|
| 653 | .priority = MCE_PRIO_LOWEST,
|
| 654 | };
|
| 655 |
|
| 656 | |
| 657 | * Read ADDR and MISC registers.
|
| 658 |
|
| 659 | static noinstr void mce_read_aux(struct mce_hw_err *err, int i)
|
| 660 | {
|
| 661 | struct mce *m = &err->m;
|
| 662 |
|
| 663 | if (m->status & MCI_STATUS_MISCV)
|
| 664 | m->misc = mce_rdmsrq(mca_msr_reg(i, MCA_MISC));
|
| 665 |
|
| 666 | if (m->status & MCI_STATUS_ADDRV) {
|
| 667 | if (m->kflags & MCE_CHECK_DFR_REGS)
|
| 668 | m->addr = mce_rdmsrq(MSR_AMD64_SMCA_MCx_DEADDR(i));
|
| 669 | else
|
| 670 | m->addr = mce_rdmsrq(mca_msr_reg(i, MCA_ADDR));
|
| 671 |
|
| 672 | |
| 673 | * Mask the reported address by the reported granularity.
|
| 674 |
|
| 675 | if (mca_cfg.ser && (m->status & MCI_STATUS_MISCV)) {
|
| 676 | u8 shift = MCI_MISC_ADDR_LSB(m->misc);
|
| 677 | m->addr >>= shift;
|
| 678 | m->addr <<= shift;
|
| 679 | }
|
| 680 |
|
| 681 | smca_extract_err_addr(m);
|
| 682 | }
|
| 683 |
|
| 684 | if (mce_flags.smca) {
|
| 685 | m->ipid = mce_rdmsrq(MSR_AMD64_SMCA_MCx_IPID(i));
|
| 686 |
|
| 687 | if (m->status & MCI_STATUS_SYNDV) {
|
| 688 | m->synd = mce_rdmsrq(MSR_AMD64_SMCA_MCx_SYND(i));
|
| 689 | err->vendor.amd.synd1 = mce_rdmsrq(MSR_AMD64_SMCA_MCx_SYND1(i));
|
| 690 | err->vendor.amd.synd2 = mce_rdmsrq(MSR_AMD64_SMCA_MCx_SYND2(i));
|
| 691 | }
|
| 692 | }
|
| 693 | }
|
| 694 |
|
| 695 | |
| 696 | * We have three scenarios for checking for Deferred errors:
|
| 697 | *
|
| 698 | * 1) Non-SMCA systems check MCA_STATUS and log error if found.
|
| 699 | * 2) SMCA systems check MCA_STATUS. If error is found then log it and also
|
| 700 | * clear MCA_DESTAT.
|
| 701 | * 3) SMCA systems check MCA_DESTAT, if error was not found in MCA_STATUS, and
|
| 702 | * log it.
|
| 703 |
|
| 704 | static bool smca_should_log_poll_error(struct mce *m)
|
| 705 | {
|
| 706 | if (m->status & MCI_STATUS_VAL)
|
| 707 | return true;
|
| 708 |
|
| 709 | m->status = mce_rdmsrq(MSR_AMD64_SMCA_MCx_DESTAT(m->bank));
|
| 710 | if ((m->status & MCI_STATUS_VAL) && (m->status & MCI_STATUS_DEFERRED)) {
|
| 711 | m->kflags |= MCE_CHECK_DFR_REGS;
|
| 712 | return true;
|
| 713 | }
|
| 714 |
|
| 715 | return false;
|
| 716 | }
|
| 717 |
|
| 718 | |
| 719 | * Newer Intel systems that support software error
|
| 720 | * recovery need to make additional checks. Other
|
| 721 | * CPUs should skip over uncorrected errors, but log
|
| 722 | * everything else.
|
| 723 |
|
| 724 | static bool ser_should_log_poll_error(struct mce *m)
|
| 725 | {
|
| 726 |
|
| 727 | if (!(m->status & MCI_STATUS_EN))
|
| 728 | return true;
|
| 729 |
|
| 730 | |
| 731 | * Log UCNA (SDM: 15.6.3 "UCR Error Classification")
|
| 732 | * UC == 1 && PCC == 0 && S == 0
|
| 733 |
|
| 734 | if (!(m->status & MCI_STATUS_PCC) && !(m->status & MCI_STATUS_S))
|
| 735 | return true;
|
| 736 |
|
| 737 | return false;
|
| 738 | }
|
| 739 |
|
| 740 | static bool should_log_poll_error(enum mcp_flags flags, struct mce_hw_err *err)
|
| 741 | {
|
| 742 | struct mce *m = &err->m;
|
| 743 |
|
| 744 | if (mce_flags.smca)
|
| 745 | return smca_should_log_poll_error(m);
|
| 746 |
|
| 747 |
|
| 748 | if (!(m->status & MCI_STATUS_VAL))
|
| 749 | return false;
|
| 750 |
|
| 751 | |
| 752 | * If we are logging everything (at CPU online) or this
|
| 753 | * is a corrected error, then we must log it.
|
| 754 |
|
| 755 | if ((flags & MCP_UC) || !(m->status & MCI_STATUS_UC))
|
| 756 | return true;
|
| 757 |
|
| 758 | if (mca_cfg.ser)
|
| 759 | return ser_should_log_poll_error(m);
|
| 760 |
|
| 761 | if (m->status & MCI_STATUS_UC)
|
| 762 | return false;
|
| 763 |
|
| 764 | return true;
|
| 765 | }
|
| 766 |
|
| 767 | static void clear_bank(struct mce *m)
|
| 768 | {
|
| 769 | if (m->cpuvendor == X86_VENDOR_AMD)
|
| 770 | return amd_clear_bank(m);
|
| 771 |
|
| 772 | mce_wrmsrq(mca_msr_reg(m->bank, MCA_STATUS), 0);
|
| 773 | }
|
| 774 |
|
| 775 | |
| 776 | * Poll for corrected events or events that happened before reset.
|
| 777 | * Those are just logged through /dev/mcelog.
|
| 778 | *
|
| 779 | * This is executed in standard interrupt context.
|
| 780 | *
|
| 781 | * Note: spec recommends to panic for fatal unsignalled
|
| 782 | * errors here. However this would be quite problematic --
|
| 783 | * we would need to reimplement the Monarch handling and
|
| 784 | * it would mess up the exclusion between exception handler
|
| 785 | * and poll handler -- * so we skip this for now.
|
| 786 | * These cases should not happen anyways, or only when the CPU
|
| 787 | * is already totally * confused. In this case it's likely it will
|
| 788 | * not fully execute the machine check handler either.
|
| 789 |
|
| 790 | void machine_check_poll(enum mcp_flags flags, mce_banks_t *b)
|
| 791 | {
|
| 792 | struct mce_bank *mce_banks = this_cpu_ptr(mce_banks_array);
|
| 793 | struct mce_hw_err err;
|
| 794 | struct mce *m;
|
| 795 | int i;
|
| 796 |
|
| 797 | inc_irq_stat(MCE_POLL);
|
| 798 |
|
| 799 | mce_gather_info(&err, NULL);
|
| 800 | m = &err.m;
|
| 801 |
|
| 802 | if (flags & MCP_TIMESTAMP)
|
| 803 | m->tsc = rdtsc();
|
| 804 |
|
| 805 | for (i = 0; i < this_cpu_read(mce_num_banks); i++) {
|
| 806 | if (!mce_banks[i].ctl || !test_bit(i, *b))
|
| 807 | continue;
|
| 808 |
|
| 809 | m->misc = 0;
|
| 810 | m->addr = 0;
|
| 811 | m->bank = i;
|
| 812 |
|
| 813 | barrier();
|
| 814 | m->status = mce_rdmsrq(mca_msr_reg(i, MCA_STATUS));
|
| 815 |
|
| 816 | |
| 817 | * Update storm tracking here, before checking for the
|
| 818 | * MCI_STATUS_VAL bit. Valid corrected errors count
|
| 819 | * towards declaring, or maintaining, storm status. No
|
| 820 | * error in a bank counts towards avoiding, or ending,
|
| 821 | * storm status.
|
| 822 |
|
| 823 | if (!mca_cfg.cmci_disabled)
|
| 824 | mce_track_storm(m);
|
| 825 |
|
| 826 |
|
| 827 | if (!should_log_poll_error(flags, &err))
|
| 828 | continue;
|
| 829 |
|
| 830 | mce_read_aux(&err, i);
|
| 831 | m->severity = mce_severity(m, NULL, NULL, false);
|
| 832 | |
| 833 | * Don't get the IP here because it's unlikely to
|
| 834 | * have anything to do with the actual error location.
|
| 835 |
|
| 836 |
|
| 837 | if (mca_cfg.dont_log_ce && !mce_usable_address(m))
|
| 838 | goto clear_it;
|
| 839 |
|
| 840 | if (flags & MCP_QUEUE_LOG)
|
| 841 | mce_gen_pool_add(&err);
|
| 842 | else
|
| 843 | mce_log(&err);
|
| 844 |
|
| 845 | clear_it:
|
| 846 | clear_bank(m);
|
| 847 | }
|
| 848 |
|
| 849 | |
| 850 | * Don't clear MCG_STATUS here because it's only defined for
|
| 851 | * exceptions.
|
| 852 |
|
| 853 |
|
| 854 | sync_core();
|
| 855 | }
|
| 856 | EXPORT_SYMBOL_GPL(machine_check_poll);
|
| 857 |
|
| 858 | |
| 859 | * During IFU recovery Sandy Bridge -EP4S processors set the RIPV and
|
| 860 | * EIPV bits in MCG_STATUS to zero on the affected logical processor (SDM
|
| 861 | * Vol 3B Table 15-20). But this confuses both the code that determines
|
| 862 | * whether the machine check occurred in kernel or user mode, and also
|
| 863 | * the severity assessment code. Pretend that EIPV was set, and take the
|
| 864 | * ip/cs values from the pt_regs that mce_gather_info() ignored earlier.
|
| 865 |
|
| 866 | static __always_inline void
|
| 867 | quirk_sandybridge_ifu(int bank, struct mce *m, struct pt_regs *regs)
|
| 868 | {
|
| 869 | if (bank != 0)
|
| 870 | return;
|
| 871 | if ((m->mcgstatus & (MCG_STATUS_EIPV|MCG_STATUS_RIPV)) != 0)
|
| 872 | return;
|
| 873 | if ((m->status & (MCI_STATUS_OVER|MCI_STATUS_UC|
|
| 874 | MCI_STATUS_EN|MCI_STATUS_MISCV|MCI_STATUS_ADDRV|
|
| 875 | MCI_STATUS_PCC|MCI_STATUS_S|MCI_STATUS_AR|
|
| 876 | MCACOD)) !=
|
| 877 | (MCI_STATUS_UC|MCI_STATUS_EN|
|
| 878 | MCI_STATUS_MISCV|MCI_STATUS_ADDRV|MCI_STATUS_S|
|
| 879 | MCI_STATUS_AR|MCACOD_INSTR))
|
| 880 | return;
|
| 881 |
|
| 882 | m->mcgstatus |= MCG_STATUS_EIPV;
|
| 883 | m->ip = regs->ip;
|
| 884 | m->cs = regs->cs;
|
| 885 | }
|
| 886 |
|
| 887 | |
| 888 | * Disable fast string copy and return from the MCE handler upon the first SRAR
|
| 889 | * MCE on bank 1 due to a CPU erratum on Intel Skylake/Cascade Lake/Cooper Lake
|
| 890 | * CPUs.
|
| 891 | * The fast string copy instructions ("REP; MOVS*") could consume an
|
| 892 | * uncorrectable memory error in the cache line _right after_ the desired region
|
| 893 | * to copy and raise an MCE with RIP pointing to the instruction _after_ the
|
| 894 | * "REP; MOVS*".
|
| 895 | * This mitigation addresses the issue completely with the caveat of performance
|
| 896 | * degradation on the CPU affected. This is still better than the OS crashing on
|
| 897 | * MCEs raised on an irrelevant process due to "REP; MOVS*" accesses from a
|
| 898 | * kernel context (e.g., copy_page).
|
| 899 | *
|
| 900 | * Returns true when fast string copy on CPU has been disabled.
|
| 901 |
|
| 902 | static noinstr bool quirk_skylake_repmov(void)
|
| 903 | {
|
| 904 | u64 mcgstatus = mce_rdmsrq(MSR_IA32_MCG_STATUS);
|
| 905 | u64 misc_enable = mce_rdmsrq(MSR_IA32_MISC_ENABLE);
|
| 906 | u64 mc1_status;
|
| 907 |
|
| 908 | |
| 909 | * Apply the quirk only to local machine checks, i.e., no broadcast
|
| 910 | * sync is needed.
|
| 911 |
|
| 912 | if (!(mcgstatus & MCG_STATUS_LMCES) ||
|
| 913 | !(misc_enable & MSR_IA32_MISC_ENABLE_FAST_STRING))
|
| 914 | return false;
|
| 915 |
|
| 916 | mc1_status = mce_rdmsrq(MSR_IA32_MCx_STATUS(1));
|
| 917 |
|
| 918 |
|
| 919 | if ((mc1_status &
|
| 920 | (MCI_STATUS_VAL | MCI_STATUS_OVER | MCI_STATUS_UC | MCI_STATUS_EN |
|
| 921 | MCI_STATUS_ADDRV | MCI_STATUS_MISCV | MCI_STATUS_PCC |
|
| 922 | MCI_STATUS_AR | MCI_STATUS_S)) ==
|
| 923 | (MCI_STATUS_VAL | MCI_STATUS_UC | MCI_STATUS_EN |
|
| 924 | MCI_STATUS_ADDRV | MCI_STATUS_MISCV |
|
| 925 | MCI_STATUS_AR | MCI_STATUS_S)) {
|
| 926 | misc_enable &= ~MSR_IA32_MISC_ENABLE_FAST_STRING;
|
| 927 | mce_wrmsrq(MSR_IA32_MISC_ENABLE, misc_enable);
|
| 928 | mce_wrmsrq(MSR_IA32_MCx_STATUS(1), 0);
|
| 929 |
|
| 930 | instrumentation_begin();
|
| 931 | pr_err_once("Erratum detected, disable fast string copy instructions.\n");
|
| 932 | instrumentation_end();
|
| 933 |
|
| 934 | return true;
|
| 935 | }
|
| 936 |
|
| 937 | return false;
|
| 938 | }
|
| 939 |
|
| 940 | |
| 941 | * Some Zen-based Instruction Fetch Units set EIPV=RIPV=0 on poison consumption
|
| 942 | * errors. This means mce_gather_info() will not save the "ip" and "cs" registers.
|
| 943 | *
|
| 944 | * However, the context is still valid, so save the "cs" register for later use.
|
| 945 | *
|
| 946 | * The "ip" register is truly unknown, so don't save it or fixup EIPV/RIPV.
|
| 947 | *
|
| 948 | * The Instruction Fetch Unit is at MCA bank 1 for all affected systems.
|
| 949 |
|
| 950 | static __always_inline void quirk_zen_ifu(int bank, struct mce *m, struct pt_regs *regs)
|
| 951 | {
|
| 952 | if (bank != 1)
|
| 953 | return;
|
| 954 | if (!(m->status & MCI_STATUS_POISON))
|
| 955 | return;
|
| 956 |
|
| 957 | m->cs = regs->cs;
|
| 958 | }
|
| 959 |
|
| 960 | |
| 961 | * Do a quick check if any of the events requires a panic.
|
| 962 | * This decides if we keep the events around or clear them.
|
| 963 |
|
| 964 | static __always_inline int mce_no_way_out(struct mce_hw_err *err, char **msg, unsigned long *validp,
|
| 965 | struct pt_regs *regs)
|
| 966 | {
|
| 967 | struct mce *m = &err->m;
|
| 968 | char *tmp = *msg;
|
| 969 | int i;
|
| 970 |
|
| 971 | for (i = 0; i < this_cpu_read(mce_num_banks); i++) {
|
| 972 | m->status = mce_rdmsrq(mca_msr_reg(i, MCA_STATUS));
|
| 973 | if (!(m->status & MCI_STATUS_VAL))
|
| 974 | continue;
|
| 975 |
|
| 976 | arch___set_bit(i, validp);
|
| 977 | if (mce_flags.snb_ifu_quirk)
|
| 978 | quirk_sandybridge_ifu(i, m, regs);
|
| 979 |
|
| 980 | if (mce_flags.zen_ifu_quirk)
|
| 981 | quirk_zen_ifu(i, m, regs);
|
| 982 |
|
| 983 | m->bank = i;
|
| 984 | if (mce_severity(m, regs, &tmp, true) >= MCE_PANIC_SEVERITY) {
|
| 985 | mce_read_aux(err, i);
|
| 986 | *msg = tmp;
|
| 987 | return 1;
|
| 988 | }
|
| 989 | }
|
| 990 | return 0;
|
| 991 | }
|
| 992 |
|
| 993 | |
| 994 | * Variable to establish order between CPUs while scanning.
|
| 995 | * Each CPU spins initially until executing is equal its number.
|
| 996 |
|
| 997 | static atomic_t mce_executing;
|
| 998 |
|
| 999 | |
| 1000 | * Defines order of CPUs on entry. First CPU becomes Monarch.
|
| 1001 |
|
| 1002 | static atomic_t mce_callin;
|
| 1003 |
|
| 1004 | |
| 1005 | * Track which CPUs entered the MCA broadcast synchronization and which not in
|
| 1006 | * order to print holdouts.
|
| 1007 |
|
| 1008 | static cpumask_t mce_missing_cpus = CPU_MASK_ALL;
|
| 1009 |
|
| 1010 | |
| 1011 | * Check if a timeout waiting for other CPUs happened.
|
| 1012 |
|
| 1013 | static noinstr int mce_timed_out(u64 *t, const char *msg)
|
| 1014 | {
|
| 1015 | int ret = 0;
|
| 1016 |
|
| 1017 |
|
| 1018 | instrumentation_begin();
|
| 1019 |
|
| 1020 | |
| 1021 | * The others already did panic for some reason.
|
| 1022 | * Bail out like in a timeout.
|
| 1023 | * rmb() to tell the compiler that system_state
|
| 1024 | * might have been modified by someone else.
|
| 1025 |
|
| 1026 | rmb();
|
| 1027 | if (atomic_read(&mce_panicked))
|
| 1028 | wait_for_panic();
|
| 1029 | if (!mca_cfg.monarch_timeout)
|
| 1030 | goto out;
|
| 1031 | if ((s64)*t < SPINUNIT) {
|
| 1032 | if (cpumask_and(&mce_missing_cpus, cpu_online_mask, &mce_missing_cpus))
|
| 1033 | pr_emerg("CPUs not responding to MCE broadcast (may include false positives): %*pbl\n",
|
| 1034 | cpumask_pr_args(&mce_missing_cpus));
|
| 1035 | mce_panic(msg, NULL, NULL);
|
| 1036 |
|
| 1037 | ret = 1;
|
| 1038 | goto out;
|
| 1039 | }
|
| 1040 | *t -= SPINUNIT;
|
| 1041 |
|
| 1042 | out:
|
| 1043 | touch_nmi_watchdog();
|
| 1044 |
|
| 1045 | instrumentation_end();
|
| 1046 |
|
| 1047 | return ret;
|
| 1048 | }
|
| 1049 |
|
| 1050 | |
| 1051 | * The Monarch's reign. The Monarch is the CPU who entered
|
| 1052 | * the machine check handler first. It waits for the others to
|
| 1053 | * raise the exception too and then grades them. When any
|
| 1054 | * error is fatal panic. Only then let the others continue.
|
| 1055 | *
|
| 1056 | * The other CPUs entering the MCE handler will be controlled by the
|
| 1057 | * Monarch. They are called Subjects.
|
| 1058 | *
|
| 1059 | * This way we prevent any potential data corruption in a unrecoverable case
|
| 1060 | * and also makes sure always all CPU's errors are examined.
|
| 1061 | *
|
| 1062 | * Also this detects the case of a machine check event coming from outer
|
| 1063 | * space (not detected by any CPUs) In this case some external agent wants
|
| 1064 | * us to shut down, so panic too.
|
| 1065 | *
|
| 1066 | * The other CPUs might still decide to panic if the handler happens
|
| 1067 | * in a unrecoverable place, but in this case the system is in a semi-stable
|
| 1068 | * state and won't corrupt anything by itself. It's ok to let the others
|
| 1069 | * continue for a bit first.
|
| 1070 | *
|
| 1071 | * All the spin loops have timeouts; when a timeout happens a CPU
|
| 1072 | * typically elects itself to be Monarch.
|
| 1073 |
|
| 1074 | static void mce_reign(void)
|
| 1075 | {
|
| 1076 | struct mce_hw_err *err = NULL;
|
| 1077 | struct mce *m = NULL;
|
| 1078 | int global_worst = 0;
|
| 1079 | char *msg = NULL;
|
| 1080 | int cpu;
|
| 1081 |
|
| 1082 | |
| 1083 | * This CPU is the Monarch and the other CPUs have run
|
| 1084 | * through their handlers.
|
| 1085 | * Grade the severity of the errors of all the CPUs.
|
| 1086 |
|
| 1087 | for_each_possible_cpu(cpu) {
|
| 1088 | struct mce_hw_err *etmp = &per_cpu(hw_errs_seen, cpu);
|
| 1089 | struct mce *mtmp = &etmp->m;
|
| 1090 |
|
| 1091 | if (mtmp->severity > global_worst) {
|
| 1092 | global_worst = mtmp->severity;
|
| 1093 | err = &per_cpu(hw_errs_seen, cpu);
|
| 1094 | m = &err->m;
|
| 1095 | }
|
| 1096 | }
|
| 1097 |
|
| 1098 | |
| 1099 | * Cannot recover? Panic here then.
|
| 1100 | * This dumps all the mces in the log buffer and stops the
|
| 1101 | * other CPUs.
|
| 1102 |
|
| 1103 | if (m && global_worst >= MCE_PANIC_SEVERITY) {
|
| 1104 |
|
| 1105 | mce_severity(m, NULL, &msg, true);
|
| 1106 | mce_panic("Fatal machine check", err, msg);
|
| 1107 | }
|
| 1108 |
|
| 1109 | |
| 1110 | * For UC somewhere we let the CPU who detects it handle it.
|
| 1111 | * Also must let continue the others, otherwise the handling
|
| 1112 | * CPU could deadlock on a lock.
|
| 1113 |
|
| 1114 |
|
| 1115 | |
| 1116 | * No machine check event found. Must be some external
|
| 1117 | * source or one CPU is hung. Panic.
|
| 1118 |
|
| 1119 | if (global_worst <= MCE_KEEP_SEVERITY)
|
| 1120 | mce_panic("Fatal machine check from unknown source", NULL, NULL);
|
| 1121 |
|
| 1122 | |
| 1123 | * Now clear all the hw_errs_seen so that they don't reappear on
|
| 1124 | * the next mce.
|
| 1125 |
|
| 1126 | for_each_possible_cpu(cpu)
|
| 1127 | memset(&per_cpu(hw_errs_seen, cpu), 0, sizeof(struct mce_hw_err));
|
| 1128 | }
|
| 1129 |
|
| 1130 | static atomic_t global_nwo;
|
| 1131 |
|
| 1132 | |
| 1133 | * Start of Monarch synchronization. This waits until all CPUs have
|
| 1134 | * entered the exception handler and then determines if any of them
|
| 1135 | * saw a fatal event that requires panic. Then it executes them
|
| 1136 | * in the entry order.
|
| 1137 | * TBD double check parallel CPU hotunplug
|
| 1138 |
|
| 1139 | static noinstr int mce_start(int *no_way_out)
|
| 1140 | {
|
| 1141 | u64 timeout = (u64)mca_cfg.monarch_timeout * NSEC_PER_USEC;
|
| 1142 | int order, ret = -1;
|
| 1143 |
|
| 1144 | if (!timeout)
|
| 1145 | return ret;
|
| 1146 |
|
| 1147 | raw_atomic_add(*no_way_out, &global_nwo);
|
| 1148 | |
| 1149 | * Rely on the implied barrier below, such that global_nwo
|
| 1150 | * is updated before mce_callin.
|
| 1151 |
|
| 1152 | order = raw_atomic_inc_return(&mce_callin);
|
| 1153 | arch_cpumask_clear_cpu(smp_processor_id(), &mce_missing_cpus);
|
| 1154 |
|
| 1155 |
|
| 1156 | instrumentation_begin();
|
| 1157 |
|
| 1158 | |
| 1159 | * Wait for everyone.
|
| 1160 |
|
| 1161 | while (raw_atomic_read(&mce_callin) != num_online_cpus()) {
|
| 1162 | if (mce_timed_out(&timeout,
|
| 1163 | "Timeout: Not all CPUs entered broadcast exception handler")) {
|
| 1164 | raw_atomic_set(&global_nwo, 0);
|
| 1165 | goto out;
|
| 1166 | }
|
| 1167 | ndelay(SPINUNIT);
|
| 1168 | }
|
| 1169 |
|
| 1170 | |
| 1171 | * mce_callin should be read before global_nwo
|
| 1172 |
|
| 1173 | smp_rmb();
|
| 1174 |
|
| 1175 | if (order == 1) {
|
| 1176 | |
| 1177 | * Monarch: Starts executing now, the others wait.
|
| 1178 |
|
| 1179 | raw_atomic_set(&mce_executing, 1);
|
| 1180 | } else {
|
| 1181 | |
| 1182 | * Subject: Now start the scanning loop one by one in
|
| 1183 | * the original callin order.
|
| 1184 | * This way when there are any shared banks it will be
|
| 1185 | * only seen by one CPU before cleared, avoiding duplicates.
|
| 1186 |
|
| 1187 | while (raw_atomic_read(&mce_executing) < order) {
|
| 1188 | if (mce_timed_out(&timeout,
|
| 1189 | "Timeout: Subject CPUs unable to finish machine check processing")) {
|
| 1190 | raw_atomic_set(&global_nwo, 0);
|
| 1191 | goto out;
|
| 1192 | }
|
| 1193 | ndelay(SPINUNIT);
|
| 1194 | }
|
| 1195 | }
|
| 1196 |
|
| 1197 | |
| 1198 | * Cache the global no_way_out state.
|
| 1199 |
|
| 1200 | *no_way_out = raw_atomic_read(&global_nwo);
|
| 1201 |
|
| 1202 | ret = order;
|
| 1203 |
|
| 1204 | out:
|
| 1205 | instrumentation_end();
|
| 1206 |
|
| 1207 | return ret;
|
| 1208 | }
|
| 1209 |
|
| 1210 | |
| 1211 | * Synchronize between CPUs after main scanning loop.
|
| 1212 | * This invokes the bulk of the Monarch processing.
|
| 1213 |
|
| 1214 | static noinstr int mce_end(int order)
|
| 1215 | {
|
| 1216 | u64 timeout = (u64)mca_cfg.monarch_timeout * NSEC_PER_USEC;
|
| 1217 | int ret = -1;
|
| 1218 |
|
| 1219 |
|
| 1220 | instrumentation_begin();
|
| 1221 |
|
| 1222 | if (!timeout)
|
| 1223 | goto reset;
|
| 1224 | if (order < 0)
|
| 1225 | goto reset;
|
| 1226 |
|
| 1227 | |
| 1228 | * Allow others to run.
|
| 1229 |
|
| 1230 | atomic_inc(&mce_executing);
|
| 1231 |
|
| 1232 | if (order == 1) {
|
| 1233 | |
| 1234 | * Monarch: Wait for everyone to go through their scanning
|
| 1235 | * loops.
|
| 1236 |
|
| 1237 | while (atomic_read(&mce_executing) <= num_online_cpus()) {
|
| 1238 | if (mce_timed_out(&timeout,
|
| 1239 | "Timeout: Monarch CPU unable to finish machine check processing"))
|
| 1240 | goto reset;
|
| 1241 | ndelay(SPINUNIT);
|
| 1242 | }
|
| 1243 |
|
| 1244 | mce_reign();
|
| 1245 | barrier();
|
| 1246 | ret = 0;
|
| 1247 | } else {
|
| 1248 | |
| 1249 | * Subject: Wait for Monarch to finish.
|
| 1250 |
|
| 1251 | while (atomic_read(&mce_executing) != 0) {
|
| 1252 | if (mce_timed_out(&timeout,
|
| 1253 | "Timeout: Monarch CPU did not finish machine check processing"))
|
| 1254 | goto reset;
|
| 1255 | ndelay(SPINUNIT);
|
| 1256 | }
|
| 1257 |
|
| 1258 | |
| 1259 | * Don't reset anything. That's done by the Monarch.
|
| 1260 |
|
| 1261 | ret = 0;
|
| 1262 | goto out;
|
| 1263 | }
|
| 1264 |
|
| 1265 | |
| 1266 | * Reset all global state.
|
| 1267 |
|
| 1268 | reset:
|
| 1269 | atomic_set(&global_nwo, 0);
|
| 1270 | atomic_set(&mce_callin, 0);
|
| 1271 | cpumask_setall(&mce_missing_cpus);
|
| 1272 | barrier();
|
| 1273 |
|
| 1274 | |
| 1275 | * Let others run again.
|
| 1276 |
|
| 1277 | atomic_set(&mce_executing, 0);
|
| 1278 |
|
| 1279 | out:
|
| 1280 | instrumentation_end();
|
| 1281 |
|
| 1282 | return ret;
|
| 1283 | }
|
| 1284 |
|
| 1285 | static __always_inline void mce_clear_state(unsigned long *toclear)
|
| 1286 | {
|
| 1287 | int i;
|
| 1288 |
|
| 1289 | for (i = 0; i < this_cpu_read(mce_num_banks); i++) {
|
| 1290 | if (arch_test_bit(i, toclear))
|
| 1291 | mce_wrmsrq(mca_msr_reg(i, MCA_STATUS), 0);
|
| 1292 | }
|
| 1293 | }
|
| 1294 |
|
| 1295 | |
| 1296 | * Cases where we avoid rendezvous handler timeout:
|
| 1297 | * 1) If this CPU is offline.
|
| 1298 | *
|
| 1299 | * 2) If crashing_cpu was set, e.g. we're entering kdump and we need to
|
| 1300 | * skip those CPUs which remain looping in the 1st kernel - see
|
| 1301 | * crash_nmi_callback().
|
| 1302 | *
|
| 1303 | * Note: there still is a small window between kexec-ing and the new,
|
| 1304 | * kdump kernel establishing a new #MC handler where a broadcasted MCE
|
| 1305 | * might not get handled properly.
|
| 1306 |
|
| 1307 | static noinstr bool mce_check_crashing_cpu(void)
|
| 1308 | {
|
| 1309 | unsigned int cpu = smp_processor_id();
|
| 1310 |
|
| 1311 | if (arch_cpu_is_offline(cpu) ||
|
| 1312 | (crashing_cpu != -1 && crashing_cpu != cpu)) {
|
| 1313 | u64 mcgstatus;
|
| 1314 |
|
| 1315 | mcgstatus = native_rdmsrq(MSR_IA32_MCG_STATUS);
|
| 1316 |
|
| 1317 | if (boot_cpu_data.x86_vendor == X86_VENDOR_ZHAOXIN) {
|
| 1318 | if (mcgstatus & MCG_STATUS_LMCES)
|
| 1319 | return false;
|
| 1320 | }
|
| 1321 |
|
| 1322 | if (mcgstatus & MCG_STATUS_RIPV) {
|
| 1323 | native_wrmsrq(MSR_IA32_MCG_STATUS, 0);
|
| 1324 | return true;
|
| 1325 | }
|
| 1326 | }
|
| 1327 | return false;
|
| 1328 | }
|
| 1329 |
|
| 1330 | static __always_inline int
|
| 1331 | __mc_scan_banks(struct mce_hw_err *err, struct pt_regs *regs,
|
| 1332 | struct mce_hw_err *final, unsigned long *toclear,
|
| 1333 | unsigned long *valid_banks, int no_way_out, int *worst)
|
| 1334 | {
|
| 1335 | struct mce_bank *mce_banks = this_cpu_ptr(mce_banks_array);
|
| 1336 | struct mca_config *cfg = &mca_cfg;
|
| 1337 | int severity, i, taint = 0;
|
| 1338 | struct mce *m = &err->m;
|
| 1339 |
|
| 1340 | for (i = 0; i < this_cpu_read(mce_num_banks); i++) {
|
| 1341 | arch___clear_bit(i, toclear);
|
| 1342 | if (!arch_test_bit(i, valid_banks))
|
| 1343 | continue;
|
| 1344 |
|
| 1345 | if (!mce_banks[i].ctl)
|
| 1346 | continue;
|
| 1347 |
|
| 1348 | m->misc = 0;
|
| 1349 | m->addr = 0;
|
| 1350 | m->bank = i;
|
| 1351 |
|
| 1352 | m->status = mce_rdmsrq(mca_msr_reg(i, MCA_STATUS));
|
| 1353 | if (!(m->status & MCI_STATUS_VAL))
|
| 1354 | continue;
|
| 1355 |
|
| 1356 | |
| 1357 | * Corrected or non-signaled errors are handled by
|
| 1358 | * machine_check_poll(). Leave them alone, unless this panics.
|
| 1359 |
|
| 1360 | if (!(m->status & (cfg->ser ? MCI_STATUS_S : MCI_STATUS_UC)) &&
|
| 1361 | !no_way_out)
|
| 1362 | continue;
|
| 1363 |
|
| 1364 |
|
| 1365 | taint++;
|
| 1366 |
|
| 1367 | severity = mce_severity(m, regs, NULL, true);
|
| 1368 |
|
| 1369 | |
| 1370 | * When machine check was for corrected/deferred handler don't
|
| 1371 | * touch, unless we're panicking.
|
| 1372 |
|
| 1373 | if ((severity == MCE_KEEP_SEVERITY ||
|
| 1374 | severity == MCE_UCNA_SEVERITY) && !no_way_out)
|
| 1375 | continue;
|
| 1376 |
|
| 1377 | arch___set_bit(i, toclear);
|
| 1378 |
|
| 1379 |
|
| 1380 | if (severity == MCE_NO_SEVERITY)
|
| 1381 | continue;
|
| 1382 |
|
| 1383 | mce_read_aux(err, i);
|
| 1384 |
|
| 1385 |
|
| 1386 | m->severity = severity;
|
| 1387 |
|
| 1388 | |
| 1389 | * Enable instrumentation around the mce_log() call which is
|
| 1390 | * done in #MC context, where instrumentation is disabled.
|
| 1391 |
|
| 1392 | instrumentation_begin();
|
| 1393 | mce_log(err);
|
| 1394 | instrumentation_end();
|
| 1395 |
|
| 1396 | if (severity > *worst) {
|
| 1397 | *final = *err;
|
| 1398 | *worst = severity;
|
| 1399 | }
|
| 1400 | }
|
| 1401 |
|
| 1402 |
|
| 1403 | *err = *final;
|
| 1404 |
|
| 1405 | return taint;
|
| 1406 | }
|
| 1407 |
|
| 1408 | static void kill_me_now(struct callback_head *ch)
|
| 1409 | {
|
| 1410 | struct task_struct *p = container_of(ch, struct task_struct, mce_kill_me);
|
| 1411 |
|
| 1412 | p->mce_count = 0;
|
| 1413 | force_sig(SIGBUS);
|
| 1414 | }
|
| 1415 |
|
| 1416 | static void kill_me_maybe(struct callback_head *cb)
|
| 1417 | {
|
| 1418 | struct task_struct *p = container_of(cb, struct task_struct, mce_kill_me);
|
| 1419 | int flags = MF_ACTION_REQUIRED;
|
| 1420 | unsigned long pfn;
|
| 1421 | int ret;
|
| 1422 |
|
| 1423 | p->mce_count = 0;
|
| 1424 | pr_err("Uncorrected hardware memory error in user-access at %llx", p->mce_addr);
|
| 1425 |
|
| 1426 | if (!p->mce_ripv)
|
| 1427 | flags |= MF_MUST_KILL;
|
| 1428 |
|
| 1429 | pfn = (p->mce_addr & MCI_ADDR_PHYSADDR) >> PAGE_SHIFT;
|
| 1430 | ret = memory_failure(pfn, flags);
|
| 1431 | if (!ret) {
|
| 1432 | set_mce_nospec(pfn);
|
| 1433 | sync_core();
|
| 1434 | return;
|
| 1435 | }
|
| 1436 |
|
| 1437 | |
| 1438 | * -EHWPOISON from memory_failure() means that it already sent SIGBUS
|
| 1439 | * to the current process with the proper error info,
|
| 1440 | * -EOPNOTSUPP means hwpoison_filter() filtered the error event,
|
| 1441 | *
|
| 1442 | * In both cases, no further processing is required.
|
| 1443 |
|
| 1444 | if (ret == -EHWPOISON || ret == -EOPNOTSUPP)
|
| 1445 | return;
|
| 1446 |
|
| 1447 | pr_err("Memory error not recovered");
|
| 1448 | kill_me_now(cb);
|
| 1449 | }
|
| 1450 |
|
| 1451 | static void kill_me_never(struct callback_head *cb)
|
| 1452 | {
|
| 1453 | struct task_struct *p = container_of(cb, struct task_struct, mce_kill_me);
|
| 1454 | unsigned long pfn;
|
| 1455 |
|
| 1456 | p->mce_count = 0;
|
| 1457 | pr_err("Kernel accessed poison in user space at %llx\n", p->mce_addr);
|
| 1458 | pfn = (p->mce_addr & MCI_ADDR_PHYSADDR) >> PAGE_SHIFT;
|
| 1459 | if (!memory_failure(pfn, 0))
|
| 1460 | set_mce_nospec(pfn);
|
| 1461 | }
|
| 1462 |
|
| 1463 | static void queue_task_work(struct mce_hw_err *err, char *msg, void (*func)(struct callback_head *))
|
| 1464 | {
|
| 1465 | int count = ++current->mce_count;
|
| 1466 | struct mce *m = &err->m;
|
| 1467 |
|
| 1468 |
|
| 1469 | if (count == 1) {
|
| 1470 | current->mce_addr = m->addr;
|
| 1471 | current->mce_kflags = m->kflags;
|
| 1472 | current->mce_ripv = !!(m->mcgstatus & MCG_STATUS_RIPV);
|
| 1473 | current->mce_whole_page = whole_page(m);
|
| 1474 | current->mce_kill_me.func = func;
|
| 1475 | }
|
| 1476 |
|
| 1477 |
|
| 1478 | if (count > 10)
|
| 1479 | mce_panic("Too many consecutive machine checks while accessing user data",
|
| 1480 | err, msg);
|
| 1481 |
|
| 1482 |
|
| 1483 | if (count > 1 && (current->mce_addr >> PAGE_SHIFT) != (m->addr >> PAGE_SHIFT))
|
| 1484 | mce_panic("Consecutive machine checks to different user pages", err, msg);
|
| 1485 |
|
| 1486 |
|
| 1487 | if (count > 1)
|
| 1488 | return;
|
| 1489 |
|
| 1490 | task_work_add(current, ¤t->mce_kill_me, TWA_RESUME);
|
| 1491 | }
|
| 1492 |
|
| 1493 |
|
| 1494 | static noinstr void unexpected_machine_check(struct pt_regs *regs)
|
| 1495 | {
|
| 1496 | instrumentation_begin();
|
| 1497 | pr_err("CPU#%d: Unexpected int18 (Machine Check)\n",
|
| 1498 | smp_processor_id());
|
| 1499 | instrumentation_end();
|
| 1500 | }
|
| 1501 |
|
| 1502 | |
| 1503 | * The actual machine check handler. This only handles real exceptions when
|
| 1504 | * something got corrupted coming in through int 18.
|
| 1505 | *
|
| 1506 | * This is executed in #MC context not subject to normal locking rules.
|
| 1507 | * This implies that most kernel services cannot be safely used. Don't even
|
| 1508 | * think about putting a printk in there!
|
| 1509 | *
|
| 1510 | * On Intel systems this is entered on all CPUs in parallel through
|
| 1511 | * MCE broadcast. However some CPUs might be broken beyond repair,
|
| 1512 | * so be always careful when synchronizing with others.
|
| 1513 | *
|
| 1514 | * Tracing and kprobes are disabled: if we interrupted a kernel context
|
| 1515 | * with IF=1, we need to minimize stack usage. There are also recursion
|
| 1516 | * issues: if the machine check was due to a failure of the memory
|
| 1517 | * backing the user stack, tracing that reads the user stack will cause
|
| 1518 | * potentially infinite recursion.
|
| 1519 | *
|
| 1520 | * Currently, the #MC handler calls out to a number of external facilities
|
| 1521 | * and, therefore, allows instrumentation around them. The optimal thing to
|
| 1522 | * have would be to do the absolutely minimal work required in #MC context
|
| 1523 | * and have instrumentation disabled only around that. Further processing can
|
| 1524 | * then happen in process context where instrumentation is allowed. Achieving
|
| 1525 | * that requires careful auditing and modifications. Until then, the code
|
| 1526 | * allows instrumentation temporarily, where required. *
|
| 1527 |
|
| 1528 | noinstr void do_machine_check(struct pt_regs *regs)
|
| 1529 | {
|
| 1530 | int worst = 0, order, no_way_out, kill_current_task, lmce, taint = 0;
|
| 1531 | DECLARE_BITMAP(valid_banks, MAX_NR_BANKS) = { 0 };
|
| 1532 | DECLARE_BITMAP(toclear, MAX_NR_BANKS) = { 0 };
|
| 1533 | struct mce_hw_err *final;
|
| 1534 | struct mce_hw_err err;
|
| 1535 | char *msg = NULL;
|
| 1536 | struct mce *m;
|
| 1537 |
|
| 1538 | if (unlikely(mce_flags.p5))
|
| 1539 | return pentium_machine_check(regs);
|
| 1540 | else if (unlikely(mce_flags.winchip))
|
| 1541 | return winchip_machine_check(regs);
|
| 1542 | else if (unlikely(!mca_cfg.initialized))
|
| 1543 | return unexpected_machine_check(regs);
|
| 1544 |
|
| 1545 | if (mce_flags.skx_repmov_quirk && quirk_skylake_repmov())
|
| 1546 | goto clear;
|
| 1547 |
|
| 1548 | |
| 1549 | * Establish sequential order between the CPUs entering the machine
|
| 1550 | * check handler.
|
| 1551 |
|
| 1552 | order = -1;
|
| 1553 |
|
| 1554 | |
| 1555 | * If no_way_out gets set, there is no safe way to recover from this
|
| 1556 | * MCE.
|
| 1557 |
|
| 1558 | no_way_out = 0;
|
| 1559 |
|
| 1560 | |
| 1561 | * If kill_current_task is not set, there might be a way to recover from this
|
| 1562 | * error.
|
| 1563 |
|
| 1564 | kill_current_task = 0;
|
| 1565 |
|
| 1566 | |
| 1567 | * MCEs are always local on AMD. Same is determined by MCG_STATUS_LMCES
|
| 1568 | * on Intel.
|
| 1569 |
|
| 1570 | lmce = 1;
|
| 1571 |
|
| 1572 | inc_irq_stat(MCE_EXCEPTION);
|
| 1573 |
|
| 1574 | mce_gather_info(&err, regs);
|
| 1575 | m = &err.m;
|
| 1576 | m->tsc = rdtsc();
|
| 1577 |
|
| 1578 | final = this_cpu_ptr(&hw_errs_seen);
|
| 1579 | *final = err;
|
| 1580 |
|
| 1581 | no_way_out = mce_no_way_out(&err, &msg, valid_banks, regs);
|
| 1582 |
|
| 1583 | barrier();
|
| 1584 |
|
| 1585 | |
| 1586 | * When no restart IP might need to kill or panic.
|
| 1587 | * Assume the worst for now, but if we find the
|
| 1588 | * severity is MCE_AR_SEVERITY we have other options.
|
| 1589 |
|
| 1590 | if (!(m->mcgstatus & MCG_STATUS_RIPV))
|
| 1591 | kill_current_task = 1;
|
| 1592 | |
| 1593 | * Check if this MCE is signaled to only this logical processor,
|
| 1594 | * on Intel, Zhaoxin only.
|
| 1595 |
|
| 1596 | if (m->cpuvendor == X86_VENDOR_INTEL ||
|
| 1597 | m->cpuvendor == X86_VENDOR_ZHAOXIN)
|
| 1598 | lmce = m->mcgstatus & MCG_STATUS_LMCES;
|
| 1599 |
|
| 1600 | |
| 1601 | * Local machine check may already know that we have to panic.
|
| 1602 | * Broadcast machine check begins rendezvous in mce_start()
|
| 1603 | * Go through all banks in exclusion of the other CPUs. This way we
|
| 1604 | * don't report duplicated events on shared banks because the first one
|
| 1605 | * to see it will clear it.
|
| 1606 |
|
| 1607 | if (lmce) {
|
| 1608 | if (no_way_out)
|
| 1609 | mce_panic("Fatal local machine check", &err, msg);
|
| 1610 | } else {
|
| 1611 | order = mce_start(&no_way_out);
|
| 1612 | }
|
| 1613 |
|
| 1614 | taint = __mc_scan_banks(&err, regs, final, toclear, valid_banks, no_way_out, &worst);
|
| 1615 |
|
| 1616 | if (!no_way_out)
|
| 1617 | mce_clear_state(toclear);
|
| 1618 |
|
| 1619 | |
| 1620 | * Do most of the synchronization with other CPUs.
|
| 1621 | * When there's any problem use only local no_way_out state.
|
| 1622 |
|
| 1623 | if (!lmce) {
|
| 1624 | if (mce_end(order) < 0) {
|
| 1625 | if (!no_way_out)
|
| 1626 | no_way_out = worst >= MCE_PANIC_SEVERITY;
|
| 1627 |
|
| 1628 | if (no_way_out)
|
| 1629 | mce_panic("Fatal machine check on current CPU", &err, msg);
|
| 1630 | }
|
| 1631 | } else {
|
| 1632 | |
| 1633 | * If there was a fatal machine check we should have
|
| 1634 | * already called mce_panic earlier in this function.
|
| 1635 | * Since we re-read the banks, we might have found
|
| 1636 | * something new. Check again to see if we found a
|
| 1637 | * fatal error. We call "mce_severity()" again to
|
| 1638 | * make sure we have the right "msg".
|
| 1639 |
|
| 1640 | if (worst >= MCE_PANIC_SEVERITY) {
|
| 1641 | mce_severity(m, regs, &msg, true);
|
| 1642 | mce_panic("Local fatal machine check!", &err, msg);
|
| 1643 | }
|
| 1644 | }
|
| 1645 |
|
| 1646 | |
| 1647 | * Enable instrumentation around the external facilities like task_work_add()
|
| 1648 | * (via queue_task_work()), fixup_exception() etc. For now, that is. Fixing this
|
| 1649 | * properly would need a lot more involved reorganization.
|
| 1650 |
|
| 1651 | instrumentation_begin();
|
| 1652 |
|
| 1653 | if (taint)
|
| 1654 | add_taint(TAINT_MACHINE_CHECK, LOCKDEP_NOW_UNRELIABLE);
|
| 1655 |
|
| 1656 | if (worst != MCE_AR_SEVERITY && !kill_current_task)
|
| 1657 | goto out;
|
| 1658 |
|
| 1659 |
|
| 1660 | if ((m->cs & 3) == 3) {
|
| 1661 |
|
| 1662 | BUG_ON(!on_thread_stack() || !user_mode(regs));
|
| 1663 |
|
| 1664 | if (!mce_usable_address(m))
|
| 1665 | queue_task_work(&err, msg, kill_me_now);
|
| 1666 | else
|
| 1667 | queue_task_work(&err, msg, kill_me_maybe);
|
| 1668 |
|
| 1669 | } else if (m->mcgstatus & MCG_STATUS_SEAM_NR) {
|
| 1670 | |
| 1671 | * Saved RIP on stack makes it look like the machine check
|
| 1672 | * was taken in the kernel on the instruction following
|
| 1673 | * the entry to SEAM mode. But MCG_STATUS_SEAM_NR indicates
|
| 1674 | * that the machine check was taken inside SEAM non-root
|
| 1675 | * mode. CPU core has already marked that guest as dead.
|
| 1676 | * It is OK for the kernel to resume execution at the
|
| 1677 | * apparent point of the machine check as the fault did
|
| 1678 | * not occur there. Mark the page as poisoned so it won't
|
| 1679 | * be added to free list when the guest is terminated.
|
| 1680 |
|
| 1681 | if (mce_usable_address(m)) {
|
| 1682 | struct page *p = pfn_to_online_page(m->addr >> PAGE_SHIFT);
|
| 1683 |
|
| 1684 | if (p)
|
| 1685 | SetPageHWPoison(p);
|
| 1686 | }
|
| 1687 | } else {
|
| 1688 | |
| 1689 | * Handle an MCE which has happened in kernel space but from
|
| 1690 | * which the kernel can recover: ex_has_fault_handler() has
|
| 1691 | * already verified that the rIP at which the error happened is
|
| 1692 | * a rIP from which the kernel can recover (by jumping to
|
| 1693 | * recovery code specified in _ASM_EXTABLE_FAULT()) and the
|
| 1694 | * corresponding exception handler which would do that is the
|
| 1695 | * proper one.
|
| 1696 |
|
| 1697 | if (m->kflags & MCE_IN_KERNEL_RECOV) {
|
| 1698 | if (!fixup_exception(regs, X86_TRAP_MC, 0, 0))
|
| 1699 | mce_panic("Failed kernel mode recovery", &err, msg);
|
| 1700 | }
|
| 1701 |
|
| 1702 | if (m->kflags & MCE_IN_KERNEL_COPYIN)
|
| 1703 | queue_task_work(&err, msg, kill_me_never);
|
| 1704 | }
|
| 1705 |
|
| 1706 | out:
|
| 1707 |
|
| 1708 | hwerr_log_error_type(HWERR_RECOV_OTHERS);
|
| 1709 |
|
| 1710 | instrumentation_end();
|
| 1711 |
|
| 1712 | clear:
|
| 1713 | mce_wrmsrq(MSR_IA32_MCG_STATUS, 0);
|
| 1714 | }
|
| 1715 | EXPORT_SYMBOL_GPL(do_machine_check);
|
| 1716 |
|
| 1717 | #ifndef CONFIG_MEMORY_FAILURE
|
| 1718 | int memory_failure(unsigned long pfn, int flags)
|
| 1719 | {
|
| 1720 |
|
| 1721 | BUG_ON(flags & MF_ACTION_REQUIRED);
|
| 1722 | pr_err("Uncorrected memory error in page 0x%lx ignored\n"
|
| 1723 | "Rebuild kernel with CONFIG_MEMORY_FAILURE=y for smarter handling\n",
|
| 1724 | pfn);
|
| 1725 |
|
| 1726 | return 0;
|
| 1727 | }
|
| 1728 | #endif
|
| 1729 |
|
| 1730 | |
| 1731 | * Periodic polling timer for "silent" machine check errors. If the
|
| 1732 | * poller finds an MCE, poll 2x faster. When the poller finds no more
|
| 1733 | * errors, poll 2x slower (up to check_interval seconds).
|
| 1734 |
|
| 1735 | static unsigned long check_interval = INITIAL_CHECK_INTERVAL;
|
| 1736 |
|
| 1737 | static DEFINE_PER_CPU(unsigned long, mce_next_interval);
|
| 1738 | static DEFINE_PER_CPU(struct timer_list, mce_timer);
|
| 1739 |
|
| 1740 | static void __start_timer(struct timer_list *t, unsigned long interval)
|
| 1741 | {
|
| 1742 | unsigned long when = jiffies + interval;
|
| 1743 | unsigned long flags;
|
| 1744 |
|
| 1745 | local_irq_save(flags);
|
| 1746 |
|
| 1747 | if (!timer_pending(t) || time_before(when, t->expires))
|
| 1748 | mod_timer(t, round_jiffies(when));
|
| 1749 |
|
| 1750 | local_irq_restore(flags);
|
| 1751 | }
|
| 1752 |
|
| 1753 | static void mc_poll_banks_default(void)
|
| 1754 | {
|
| 1755 | machine_check_poll(0, this_cpu_ptr(&mce_poll_banks));
|
| 1756 | }
|
| 1757 |
|
| 1758 | void (*mc_poll_banks)(void) = mc_poll_banks_default;
|
| 1759 |
|
| 1760 | static bool should_enable_timer(unsigned long iv)
|
| 1761 | {
|
| 1762 | return !mca_cfg.ignore_ce && iv;
|
| 1763 | }
|
| 1764 |
|
| 1765 | static void mce_timer_fn(struct timer_list *t)
|
| 1766 | {
|
| 1767 | struct timer_list *cpu_t = this_cpu_ptr(&mce_timer);
|
| 1768 | unsigned long iv;
|
| 1769 |
|
| 1770 | WARN_ON(cpu_t != t);
|
| 1771 |
|
| 1772 | iv = __this_cpu_read(mce_next_interval);
|
| 1773 |
|
| 1774 | if (mce_available(this_cpu_ptr(&cpu_info)))
|
| 1775 | mc_poll_banks();
|
| 1776 |
|
| 1777 | |
| 1778 | * Alert userspace if needed. If we logged an MCE, reduce the polling
|
| 1779 | * interval, otherwise increase the polling interval.
|
| 1780 |
|
| 1781 | if (!mce_gen_pool_empty())
|
| 1782 | iv = max(iv / 2, (unsigned long) HZ/100);
|
| 1783 | else
|
| 1784 | iv = min(iv * 2, round_jiffies_relative(check_interval * HZ));
|
| 1785 |
|
| 1786 | if (mce_get_storm_mode()) {
|
| 1787 | __start_timer(t, HZ);
|
| 1788 | } else if (should_enable_timer(iv)) {
|
| 1789 | __this_cpu_write(mce_next_interval, iv);
|
| 1790 | __start_timer(t, iv);
|
| 1791 | }
|
| 1792 | }
|
| 1793 |
|
| 1794 | |
| 1795 | * When a storm starts on any bank on this CPU, switch to polling
|
| 1796 | * once per second. When the storm ends, revert to the default
|
| 1797 | * polling interval.
|
| 1798 |
|
| 1799 | void mce_timer_kick(bool storm)
|
| 1800 | {
|
| 1801 | struct timer_list *t = this_cpu_ptr(&mce_timer);
|
| 1802 |
|
| 1803 | mce_set_storm_mode(storm);
|
| 1804 |
|
| 1805 | if (storm)
|
| 1806 | __start_timer(t, HZ);
|
| 1807 | else
|
| 1808 | __this_cpu_write(mce_next_interval, check_interval * HZ);
|
| 1809 | }
|
| 1810 |
|
| 1811 |
|
| 1812 | static void mce_timer_delete_all(void)
|
| 1813 | {
|
| 1814 | int cpu;
|
| 1815 |
|
| 1816 | for_each_online_cpu(cpu)
|
| 1817 | timer_delete_sync(&per_cpu(mce_timer, cpu));
|
| 1818 | }
|
| 1819 |
|
| 1820 | static void __mcheck_cpu_mce_banks_init(void)
|
| 1821 | {
|
| 1822 | struct mce_bank *mce_banks = this_cpu_ptr(mce_banks_array);
|
| 1823 | u8 n_banks = this_cpu_read(mce_num_banks);
|
| 1824 | int i;
|
| 1825 |
|
| 1826 | for (i = 0; i < n_banks; i++) {
|
| 1827 | struct mce_bank *b = &mce_banks[i];
|
| 1828 |
|
| 1829 | |
| 1830 | * Init them all by default.
|
| 1831 | *
|
| 1832 | * The required vendor quirks will be applied before
|
| 1833 | * __mcheck_cpu_init_prepare_banks() does the final bank setup.
|
| 1834 |
|
| 1835 | b->ctl = -1ULL;
|
| 1836 | b->init = true;
|
| 1837 | }
|
| 1838 | }
|
| 1839 |
|
| 1840 | |
| 1841 | * Initialize Machine Checks for a CPU.
|
| 1842 |
|
| 1843 | static void __mcheck_cpu_cap_init(void)
|
| 1844 | {
|
| 1845 | u64 cap;
|
| 1846 | u8 b;
|
| 1847 |
|
| 1848 | rdmsrq(MSR_IA32_MCG_CAP, cap);
|
| 1849 |
|
| 1850 | b = cap & MCG_BANKCNT_MASK;
|
| 1851 |
|
| 1852 | if (b > MAX_NR_BANKS) {
|
| 1853 | pr_warn("CPU%d: Using only %u machine check banks out of %u\n",
|
| 1854 | smp_processor_id(), MAX_NR_BANKS, b);
|
| 1855 | b = MAX_NR_BANKS;
|
| 1856 | }
|
| 1857 |
|
| 1858 | this_cpu_write(mce_num_banks, b);
|
| 1859 |
|
| 1860 | __mcheck_cpu_mce_banks_init();
|
| 1861 | }
|
| 1862 |
|
| 1863 | static void __mcheck_cpu_init_generic(void)
|
| 1864 | {
|
| 1865 | u64 cap;
|
| 1866 |
|
| 1867 | rdmsrq(MSR_IA32_MCG_CAP, cap);
|
| 1868 | if (cap & MCG_CTL_P)
|
| 1869 | wrmsrq(MSR_IA32_MCG_CTL, ~0ULL);
|
| 1870 | }
|
| 1871 |
|
| 1872 | static void __mcheck_cpu_init_prepare_banks(void)
|
| 1873 | {
|
| 1874 | struct mce_bank *mce_banks = this_cpu_ptr(mce_banks_array);
|
| 1875 | u64 msrval;
|
| 1876 | int i;
|
| 1877 |
|
| 1878 | |
| 1879 | * Log the machine checks left over from the previous reset. Log them
|
| 1880 | * only, do not start processing them. That will happen in mcheck_late_init()
|
| 1881 | * when all consumers have been registered on the notifier chain.
|
| 1882 |
|
| 1883 | if (mca_cfg.bootlog) {
|
| 1884 | mce_banks_t all_banks;
|
| 1885 |
|
| 1886 | bitmap_fill(all_banks, MAX_NR_BANKS);
|
| 1887 | machine_check_poll(MCP_UC | MCP_QUEUE_LOG, &all_banks);
|
| 1888 | }
|
| 1889 |
|
| 1890 | for (i = 0; i < this_cpu_read(mce_num_banks); i++) {
|
| 1891 | struct mce_bank *b = &mce_banks[i];
|
| 1892 |
|
| 1893 | if (!b->init)
|
| 1894 | continue;
|
| 1895 |
|
| 1896 | wrmsrq(mca_msr_reg(i, MCA_CTL), b->ctl);
|
| 1897 | wrmsrq(mca_msr_reg(i, MCA_STATUS), 0);
|
| 1898 |
|
| 1899 | rdmsrq(mca_msr_reg(i, MCA_CTL), msrval);
|
| 1900 | b->init = !!msrval;
|
| 1901 | }
|
| 1902 | }
|
| 1903 |
|
| 1904 | static void amd_apply_global_quirks(struct cpuinfo_x86 *c)
|
| 1905 | {
|
| 1906 | if (c->x86 < 0x11 && mca_cfg.bootlog < 0) {
|
| 1907 | |
| 1908 | * Lots of broken BIOS around that don't clear them
|
| 1909 | * by default and leave crap in there. Don't log:
|
| 1910 |
|
| 1911 | mca_cfg.bootlog = 0;
|
| 1912 | }
|
| 1913 |
|
| 1914 | |
| 1915 | * overflow_recov is supported for F15h Models 00h-0fh
|
| 1916 | * even though we don't have a CPUID bit for it.
|
| 1917 |
|
| 1918 | if (c->x86 == 0x15 && c->x86_model <= 0xf)
|
| 1919 | mce_flags.overflow_recov = 1;
|
| 1920 |
|
| 1921 | if (c->x86 >= 0x17 && c->x86 <= 0x1A)
|
| 1922 | mce_flags.zen_ifu_quirk = 1;
|
| 1923 | }
|
| 1924 |
|
| 1925 | static void intel_apply_global_quirks(struct cpuinfo_x86 *c)
|
| 1926 | {
|
| 1927 |
|
| 1928 | if (c->x86_vfm < INTEL_PENTIUM_PRO)
|
| 1929 | return;
|
| 1930 |
|
| 1931 | |
| 1932 | * All newer Intel systems support MCE broadcasting. Enable
|
| 1933 | * synchronization with a one second timeout.
|
| 1934 |
|
| 1935 | if (c->x86_vfm >= INTEL_CORE_YONAH && mca_cfg.monarch_timeout < 0)
|
| 1936 | mca_cfg.monarch_timeout = USEC_PER_SEC;
|
| 1937 |
|
| 1938 | |
| 1939 | * There are also broken BIOSes on some Pentium M and
|
| 1940 | * earlier systems:
|
| 1941 |
|
| 1942 | if (c->x86_vfm < INTEL_CORE_YONAH && mca_cfg.bootlog < 0)
|
| 1943 | mca_cfg.bootlog = 0;
|
| 1944 |
|
| 1945 | if (c->x86_vfm == INTEL_SANDYBRIDGE_X)
|
| 1946 | mce_flags.snb_ifu_quirk = 1;
|
| 1947 |
|
| 1948 | |
| 1949 | * Skylake, Cascacde Lake and Cooper Lake require a quirk on
|
| 1950 | * rep movs.
|
| 1951 |
|
| 1952 | if (c->x86_vfm == INTEL_SKYLAKE_X)
|
| 1953 | mce_flags.skx_repmov_quirk = 1;
|
| 1954 | }
|
| 1955 |
|
| 1956 | static void zhaoxin_apply_global_quirks(struct cpuinfo_x86 *c)
|
| 1957 | {
|
| 1958 | |
| 1959 | * All newer Zhaoxin CPUs support MCE broadcasting. Enable
|
| 1960 | * synchronization with a one second timeout.
|
| 1961 |
|
| 1962 | if (c->x86 > 6 || (c->x86_model == 0x19 || c->x86_model == 0x1f)) {
|
| 1963 | if (mca_cfg.monarch_timeout < 0)
|
| 1964 | mca_cfg.monarch_timeout = USEC_PER_SEC;
|
| 1965 | }
|
| 1966 | }
|
| 1967 |
|
| 1968 | static bool __mcheck_cpu_ancient_init(struct cpuinfo_x86 *c)
|
| 1969 | {
|
| 1970 | if (c->x86 != 5)
|
| 1971 | return false;
|
| 1972 |
|
| 1973 | switch (c->x86_vendor) {
|
| 1974 | case X86_VENDOR_INTEL:
|
| 1975 | intel_p5_mcheck_init(c);
|
| 1976 | mce_flags.p5 = 1;
|
| 1977 | return true;
|
| 1978 | case X86_VENDOR_CENTAUR:
|
| 1979 | winchip_mcheck_init(c);
|
| 1980 | mce_flags.winchip = 1;
|
| 1981 | return true;
|
| 1982 | default:
|
| 1983 | return false;
|
| 1984 | }
|
| 1985 |
|
| 1986 | return false;
|
| 1987 | }
|
| 1988 |
|
| 1989 | static void mce_centaur_feature_init(struct cpuinfo_x86 *c)
|
| 1990 | {
|
| 1991 | struct mca_config *cfg = &mca_cfg;
|
| 1992 |
|
| 1993 | |
| 1994 | * All newer Centaur CPUs support MCE broadcasting. Enable
|
| 1995 | * synchronization with a one second timeout.
|
| 1996 |
|
| 1997 | if ((c->x86 == 6 && c->x86_model == 0xf && c->x86_stepping >= 0xe) ||
|
| 1998 | c->x86 > 6) {
|
| 1999 | if (cfg->monarch_timeout < 0)
|
| 2000 | cfg->monarch_timeout = USEC_PER_SEC;
|
| 2001 | }
|
| 2002 | }
|
| 2003 |
|
| 2004 | static void mce_zhaoxin_feature_init(struct cpuinfo_x86 *c)
|
| 2005 | {
|
| 2006 | struct mce_bank *mce_banks = this_cpu_ptr(mce_banks_array);
|
| 2007 |
|
| 2008 | |
| 2009 | * These CPUs have MCA bank 8 which reports only one error type called
|
| 2010 | * SVAD (System View Address Decoder). The reporting of that error is
|
| 2011 | * controlled by IA32_MC8.CTL.0.
|
| 2012 | *
|
| 2013 | * If enabled, prefetching on these CPUs will cause SVAD MCE when
|
| 2014 | * virtual machines start and result in a system panic. Always disable
|
| 2015 | * bank 8 SVAD error by default.
|
| 2016 |
|
| 2017 | if ((c->x86 == 7 && c->x86_model == 0x1b) ||
|
| 2018 | (c->x86_model == 0x19 || c->x86_model == 0x1f)) {
|
| 2019 | if (this_cpu_read(mce_num_banks) > 8)
|
| 2020 | mce_banks[8].ctl = 0;
|
| 2021 | }
|
| 2022 |
|
| 2023 | intel_init_cmci();
|
| 2024 | intel_init_lmce();
|
| 2025 | }
|
| 2026 |
|
| 2027 | static void mce_zhaoxin_feature_clear(struct cpuinfo_x86 *c)
|
| 2028 | {
|
| 2029 | intel_clear_lmce();
|
| 2030 | }
|
| 2031 |
|
| 2032 | static void __mcheck_cpu_init_vendor(struct cpuinfo_x86 *c)
|
| 2033 | {
|
| 2034 | switch (c->x86_vendor) {
|
| 2035 | case X86_VENDOR_INTEL:
|
| 2036 | mce_intel_feature_init(c);
|
| 2037 | break;
|
| 2038 |
|
| 2039 | case X86_VENDOR_AMD:
|
| 2040 | case X86_VENDOR_HYGON:
|
| 2041 | mce_amd_feature_init(c);
|
| 2042 | break;
|
| 2043 |
|
| 2044 | case X86_VENDOR_CENTAUR:
|
| 2045 | mce_centaur_feature_init(c);
|
| 2046 | break;
|
| 2047 |
|
| 2048 | case X86_VENDOR_ZHAOXIN:
|
| 2049 | mce_zhaoxin_feature_init(c);
|
| 2050 | break;
|
| 2051 |
|
| 2052 | default:
|
| 2053 | break;
|
| 2054 | }
|
| 2055 | }
|
| 2056 |
|
| 2057 | static void __mcheck_cpu_clear_vendor(struct cpuinfo_x86 *c)
|
| 2058 | {
|
| 2059 | switch (c->x86_vendor) {
|
| 2060 | case X86_VENDOR_INTEL:
|
| 2061 | mce_intel_feature_clear(c);
|
| 2062 | break;
|
| 2063 |
|
| 2064 | case X86_VENDOR_ZHAOXIN:
|
| 2065 | mce_zhaoxin_feature_clear(c);
|
| 2066 | break;
|
| 2067 |
|
| 2068 | default:
|
| 2069 | break;
|
| 2070 | }
|
| 2071 | }
|
| 2072 |
|
| 2073 | static void mce_start_timer(struct timer_list *t)
|
| 2074 | {
|
| 2075 | unsigned long iv = check_interval * HZ;
|
| 2076 |
|
| 2077 | if (should_enable_timer(iv)) {
|
| 2078 | this_cpu_write(mce_next_interval, iv);
|
| 2079 | __start_timer(t, iv);
|
| 2080 | }
|
| 2081 | }
|
| 2082 |
|
| 2083 | static void __mcheck_cpu_setup_timer(void)
|
| 2084 | {
|
| 2085 | struct timer_list *t = this_cpu_ptr(&mce_timer);
|
| 2086 |
|
| 2087 | timer_setup(t, mce_timer_fn, TIMER_PINNED);
|
| 2088 | }
|
| 2089 |
|
| 2090 | static void __mcheck_cpu_init_timer(void)
|
| 2091 | {
|
| 2092 | struct timer_list *t = this_cpu_ptr(&mce_timer);
|
| 2093 |
|
| 2094 | timer_setup(t, mce_timer_fn, TIMER_PINNED);
|
| 2095 | mce_start_timer(t);
|
| 2096 | }
|
| 2097 |
|
| 2098 | bool filter_mce(struct mce *m)
|
| 2099 | {
|
| 2100 | if (boot_cpu_data.x86_vendor == X86_VENDOR_AMD)
|
| 2101 | return amd_filter_mce(m);
|
| 2102 | if (boot_cpu_data.x86_vendor == X86_VENDOR_INTEL)
|
| 2103 | return intel_filter_mce(m);
|
| 2104 |
|
| 2105 | return false;
|
| 2106 | }
|
| 2107 |
|
| 2108 | static __always_inline void exc_machine_check_kernel(struct pt_regs *regs)
|
| 2109 | {
|
| 2110 | irqentry_state_t irq_state;
|
| 2111 | unsigned long dr7;
|
| 2112 |
|
| 2113 | dr7 = local_db_save();
|
| 2114 |
|
| 2115 | WARN_ON_ONCE(user_mode(regs));
|
| 2116 |
|
| 2117 | |
| 2118 | * Only required when from kernel mode. See
|
| 2119 | * mce_check_crashing_cpu() for details.
|
| 2120 |
|
| 2121 | if (mca_cfg.initialized && mce_check_crashing_cpu())
|
| 2122 | goto out;
|
| 2123 |
|
| 2124 | irq_state = irqentry_nmi_enter(regs);
|
| 2125 |
|
| 2126 | do_machine_check(regs);
|
| 2127 |
|
| 2128 | irqentry_nmi_exit(regs, irq_state);
|
| 2129 | out:
|
| 2130 | local_db_restore(dr7);
|
| 2131 | }
|
| 2132 |
|
| 2133 | static __always_inline void exc_machine_check_user(struct pt_regs *regs)
|
| 2134 | {
|
| 2135 | unsigned long dr7;
|
| 2136 |
|
| 2137 | irqentry_enter_from_user_mode(regs);
|
| 2138 |
|
| 2139 | dr7 = local_db_save();
|
| 2140 | do_machine_check(regs);
|
| 2141 | local_db_restore(dr7);
|
| 2142 |
|
| 2143 | irqentry_exit_to_user_mode(regs);
|
| 2144 | }
|
| 2145 |
|
| 2146 | #ifdef CONFIG_X86_64
|
| 2147 |
|
| 2148 | DEFINE_IDTENTRY_MCE(exc_machine_check)
|
| 2149 | {
|
| 2150 | exc_machine_check_kernel(regs);
|
| 2151 | }
|
| 2152 |
|
| 2153 |
|
| 2154 | DEFINE_IDTENTRY_MCE_USER(exc_machine_check)
|
| 2155 | {
|
| 2156 | exc_machine_check_user(regs);
|
| 2157 | }
|
| 2158 |
|
| 2159 | #ifdef CONFIG_X86_FRED
|
| 2160 | |
| 2161 | * When occurred on different ring level, i.e., from user or kernel
|
| 2162 | * context, #MCE needs to be handled on different stack: User #MCE
|
| 2163 | * on current task stack, while kernel #MCE on a dedicated stack.
|
| 2164 | *
|
| 2165 | * This is exactly how FRED event delivery invokes an exception
|
| 2166 | * handler: ring 3 event on level 0 stack, i.e., current task stack;
|
| 2167 | * ring 0 event on the #MCE dedicated stack specified in the
|
| 2168 | * IA32_FRED_STKLVLS MSR. So unlike IDT, the FRED machine check entry
|
| 2169 | * stub doesn't do stack switch.
|
| 2170 |
|
| 2171 | DEFINE_FREDENTRY_MCE(exc_machine_check)
|
| 2172 | {
|
| 2173 | if (user_mode(regs))
|
| 2174 | exc_machine_check_user(regs);
|
| 2175 | else
|
| 2176 | exc_machine_check_kernel(regs);
|
| 2177 | }
|
| 2178 | #endif
|
| 2179 | #else
|
| 2180 |
|
| 2181 | DEFINE_IDTENTRY_RAW(exc_machine_check)
|
| 2182 | {
|
| 2183 | if (user_mode(regs))
|
| 2184 | exc_machine_check_user(regs);
|
| 2185 | else
|
| 2186 | exc_machine_check_kernel(regs);
|
| 2187 | }
|
| 2188 | #endif
|
| 2189 |
|
| 2190 | void mca_bsp_init(struct cpuinfo_x86 *c)
|
| 2191 | {
|
| 2192 | u64 cap;
|
| 2193 |
|
| 2194 | if (!mce_available(c))
|
| 2195 | return;
|
| 2196 |
|
| 2197 | if (c->x86_vendor == X86_VENDOR_UNKNOWN) {
|
| 2198 | mca_cfg.disabled = 1;
|
| 2199 | pr_info("unknown CPU type - not enabling MCE support\n");
|
| 2200 | return;
|
| 2201 | }
|
| 2202 |
|
| 2203 | mce_flags.overflow_recov = cpu_feature_enabled(X86_FEATURE_OVERFLOW_RECOV);
|
| 2204 | mce_flags.succor = cpu_feature_enabled(X86_FEATURE_SUCCOR);
|
| 2205 | mce_flags.smca = cpu_feature_enabled(X86_FEATURE_SMCA);
|
| 2206 |
|
| 2207 | if (mce_flags.smca)
|
| 2208 | smca_bsp_init();
|
| 2209 |
|
| 2210 | rdmsrq(MSR_IA32_MCG_CAP, cap);
|
| 2211 |
|
| 2212 |
|
| 2213 | if ((cap & MCG_EXT_P) && MCG_EXT_CNT(cap) >= 9)
|
| 2214 | mca_cfg.rip_msr = MSR_IA32_MCG_EIP;
|
| 2215 |
|
| 2216 | if (cap & MCG_SER_P)
|
| 2217 | mca_cfg.ser = 1;
|
| 2218 |
|
| 2219 | switch (c->x86_vendor) {
|
| 2220 | case X86_VENDOR_AMD:
|
| 2221 | amd_apply_global_quirks(c);
|
| 2222 | break;
|
| 2223 | case X86_VENDOR_INTEL:
|
| 2224 | intel_apply_global_quirks(c);
|
| 2225 | break;
|
| 2226 | case X86_VENDOR_ZHAOXIN:
|
| 2227 | zhaoxin_apply_global_quirks(c);
|
| 2228 | break;
|
| 2229 | }
|
| 2230 |
|
| 2231 | if (mca_cfg.monarch_timeout < 0)
|
| 2232 | mca_cfg.monarch_timeout = 0;
|
| 2233 | if (mca_cfg.bootlog != 0)
|
| 2234 | mca_cfg.panic_timeout = 30;
|
| 2235 | }
|
| 2236 |
|
| 2237 | |
| 2238 | * Called for each booted CPU to set up machine checks.
|
| 2239 | * Must be called with preempt off:
|
| 2240 |
|
| 2241 | void mcheck_cpu_init(struct cpuinfo_x86 *c)
|
| 2242 | {
|
| 2243 | if (mca_cfg.disabled)
|
| 2244 | return;
|
| 2245 |
|
| 2246 | if (__mcheck_cpu_ancient_init(c))
|
| 2247 | return;
|
| 2248 |
|
| 2249 | if (!mce_available(c))
|
| 2250 | return;
|
| 2251 |
|
| 2252 | __mcheck_cpu_cap_init();
|
| 2253 |
|
| 2254 | if (!mce_gen_pool_init()) {
|
| 2255 | mca_cfg.disabled = 1;
|
| 2256 | pr_emerg("Couldn't allocate MCE records pool!\n");
|
| 2257 | return;
|
| 2258 | }
|
| 2259 |
|
| 2260 | mca_cfg.initialized = 1;
|
| 2261 |
|
| 2262 | __mcheck_cpu_setup_timer();
|
| 2263 | __mcheck_cpu_init_generic();
|
| 2264 | __mcheck_cpu_init_vendor(c);
|
| 2265 | __mcheck_cpu_init_prepare_banks();
|
| 2266 | cr4_set_bits(X86_CR4_MCE);
|
| 2267 | }
|
| 2268 |
|
| 2269 | |
| 2270 | * Called for each booted CPU to clear some machine checks opt-ins
|
| 2271 |
|
| 2272 | void mcheck_cpu_clear(struct cpuinfo_x86 *c)
|
| 2273 | {
|
| 2274 | if (mca_cfg.disabled)
|
| 2275 | return;
|
| 2276 |
|
| 2277 | if (!mce_available(c))
|
| 2278 | return;
|
| 2279 |
|
| 2280 | |
| 2281 | * Possibly to clear general settings generic to x86
|
| 2282 | * __mcheck_cpu_clear_generic(c);
|
| 2283 |
|
| 2284 | __mcheck_cpu_clear_vendor(c);
|
| 2285 |
|
| 2286 | }
|
| 2287 |
|
| 2288 | static void __mce_disable_bank(void *arg)
|
| 2289 | {
|
| 2290 | int bank = *((int *)arg);
|
| 2291 | __clear_bit(bank, this_cpu_ptr(mce_poll_banks));
|
| 2292 | cmci_disable_bank(bank);
|
| 2293 | }
|
| 2294 |
|
| 2295 | void mce_disable_bank(int bank)
|
| 2296 | {
|
| 2297 | if (bank >= this_cpu_read(mce_num_banks)) {
|
| 2298 | pr_warn(FW_BUG
|
| 2299 | "Ignoring request to disable invalid MCA bank %d.\n",
|
| 2300 | bank);
|
| 2301 | return;
|
| 2302 | }
|
| 2303 | set_bit(bank, mce_banks_ce_disabled);
|
| 2304 | on_each_cpu(__mce_disable_bank, &bank, 1);
|
| 2305 | }
|
| 2306 |
|
| 2307 | |
| 2308 | * mce=off Disables machine check
|
| 2309 | * mce=no_cmci Disables CMCI
|
| 2310 | * mce=no_lmce Disables LMCE
|
| 2311 | * mce=dont_log_ce Clears corrected events silently, no log created for CEs.
|
| 2312 | * mce=print_all Print all machine check logs to console
|
| 2313 | * mce=ignore_ce Disables polling and CMCI, corrected events are not cleared.
|
| 2314 | * mce=TOLERANCELEVEL[,monarchtimeout] (number, see above)
|
| 2315 | * monarchtimeout is how long to wait for other CPUs on machine
|
| 2316 | * check, or 0 to not wait
|
| 2317 | * mce=bootlog Log MCEs from before booting. Disabled by default on AMD Fam10h
|
| 2318 | and older.
|
| 2319 | * mce=nobootlog Don't log MCEs from before booting.
|
| 2320 | * mce=bios_cmci_threshold Don't program the CMCI threshold
|
| 2321 | * mce=recovery force enable copy_mc_fragile()
|
| 2322 |
|
| 2323 | static int __init mcheck_enable(char *str)
|
| 2324 | {
|
| 2325 | struct mca_config *cfg = &mca_cfg;
|
| 2326 |
|
| 2327 | if (*str == 0) {
|
| 2328 | enable_p5_mce();
|
| 2329 | return 1;
|
| 2330 | }
|
| 2331 | if (*str == '=')
|
| 2332 | str++;
|
| 2333 | if (!strcmp(str, "off"))
|
| 2334 | cfg->disabled = 1;
|
| 2335 | else if (!strcmp(str, "no_cmci"))
|
| 2336 | cfg->cmci_disabled = true;
|
| 2337 | else if (!strcmp(str, "no_lmce"))
|
| 2338 | cfg->lmce_disabled = 1;
|
| 2339 | else if (!strcmp(str, "dont_log_ce"))
|
| 2340 | cfg->dont_log_ce = true;
|
| 2341 | else if (!strcmp(str, "print_all"))
|
| 2342 | cfg->print_all = true;
|
| 2343 | else if (!strcmp(str, "ignore_ce"))
|
| 2344 | cfg->ignore_ce = true;
|
| 2345 | else if (!strcmp(str, "bootlog") || !strcmp(str, "nobootlog"))
|
| 2346 | cfg->bootlog = (str[0] == 'b');
|
| 2347 | else if (!strcmp(str, "bios_cmci_threshold"))
|
| 2348 | cfg->bios_cmci_threshold = 1;
|
| 2349 | else if (!strcmp(str, "recovery"))
|
| 2350 | cfg->recovery = 1;
|
| 2351 | else if (isdigit(str[0]))
|
| 2352 | get_option(&str, &(cfg->monarch_timeout));
|
| 2353 | else {
|
| 2354 | pr_info("mce argument %s ignored. Please use /sys\n", str);
|
| 2355 | return 0;
|
| 2356 | }
|
| 2357 | return 1;
|
| 2358 | }
|
| 2359 | __setup("mce", mcheck_enable);
|
| 2360 |
|
| 2361 | int __init mcheck_init(void)
|
| 2362 | {
|
| 2363 | mce_register_decode_chain(&early_nb);
|
| 2364 | mce_register_decode_chain(&mce_uc_nb);
|
| 2365 | mce_register_decode_chain(&mce_default_nb);
|
| 2366 |
|
| 2367 | INIT_WORK(&mce_work, mce_gen_pool_process);
|
| 2368 | init_irq_work(&mce_irq_work, mce_irq_work_cb);
|
| 2369 |
|
| 2370 | return 0;
|
| 2371 | }
|
| 2372 |
|
| 2373 | |
| 2374 | * mce_syscore: PM support
|
| 2375 |
|
| 2376 |
|
| 2377 | |
| 2378 | * Disable machine checks on suspend and shutdown. We can't really handle
|
| 2379 | * them later.
|
| 2380 |
|
| 2381 | static void mce_disable_error_reporting(void)
|
| 2382 | {
|
| 2383 | struct mce_bank *mce_banks = this_cpu_ptr(mce_banks_array);
|
| 2384 | int i;
|
| 2385 |
|
| 2386 | for (i = 0; i < this_cpu_read(mce_num_banks); i++) {
|
| 2387 | struct mce_bank *b = &mce_banks[i];
|
| 2388 |
|
| 2389 | if (b->init)
|
| 2390 | wrmsrq(mca_msr_reg(i, MCA_CTL), 0);
|
| 2391 | }
|
| 2392 | return;
|
| 2393 | }
|
| 2394 |
|
| 2395 | static void vendor_disable_error_reporting(void)
|
| 2396 | {
|
| 2397 | |
| 2398 | * Don't clear on Intel or AMD or Hygon or Zhaoxin CPUs. Some of these
|
| 2399 | * MSRs are socket-wide. Disabling them for just a single offlined CPU
|
| 2400 | * is bad, since it will inhibit reporting for all shared resources on
|
| 2401 | * the socket like the last level cache (LLC), the integrated memory
|
| 2402 | * controller (iMC), etc.
|
| 2403 |
|
| 2404 | if (boot_cpu_data.x86_vendor == X86_VENDOR_INTEL ||
|
| 2405 | boot_cpu_data.x86_vendor == X86_VENDOR_HYGON ||
|
| 2406 | boot_cpu_data.x86_vendor == X86_VENDOR_AMD ||
|
| 2407 | boot_cpu_data.x86_vendor == X86_VENDOR_ZHAOXIN)
|
| 2408 | return;
|
| 2409 |
|
| 2410 | mce_disable_error_reporting();
|
| 2411 | }
|
| 2412 |
|
| 2413 | static int mce_syscore_suspend(void *data)
|
| 2414 | {
|
| 2415 | vendor_disable_error_reporting();
|
| 2416 | return 0;
|
| 2417 | }
|
| 2418 |
|
| 2419 | static void mce_syscore_shutdown(void *data)
|
| 2420 | {
|
| 2421 | vendor_disable_error_reporting();
|
| 2422 | }
|
| 2423 |
|
| 2424 | |
| 2425 | * On resume clear all MCE state. Don't want to see leftovers from the BIOS.
|
| 2426 | * Only one CPU is active at this time, the others get re-added later using
|
| 2427 | * CPU hotplug:
|
| 2428 |
|
| 2429 | static void mce_syscore_resume(void *data)
|
| 2430 | {
|
| 2431 | __mcheck_cpu_init_generic();
|
| 2432 | __mcheck_cpu_init_vendor(raw_cpu_ptr(&cpu_info));
|
| 2433 | __mcheck_cpu_init_prepare_banks();
|
| 2434 | cr4_set_bits(X86_CR4_MCE);
|
| 2435 | }
|
| 2436 |
|
| 2437 | static const struct syscore_ops mce_syscore_ops = {
|
| 2438 | .suspend = mce_syscore_suspend,
|
| 2439 | .shutdown = mce_syscore_shutdown,
|
| 2440 | .resume = mce_syscore_resume,
|
| 2441 | };
|
| 2442 |
|
| 2443 | static struct syscore mce_syscore = {
|
| 2444 | .ops = &mce_syscore_ops,
|
| 2445 | };
|
| 2446 |
|
| 2447 | |
| 2448 | * mce_device: Sysfs support
|
| 2449 |
|
| 2450 |
|
| 2451 | static void mce_cpu_restart(void *data)
|
| 2452 | {
|
| 2453 | if (!mce_available(raw_cpu_ptr(&cpu_info)))
|
| 2454 | return;
|
| 2455 | __mcheck_cpu_init_generic();
|
| 2456 | __mcheck_cpu_init_prepare_banks();
|
| 2457 | __mcheck_cpu_init_timer();
|
| 2458 | cr4_set_bits(X86_CR4_MCE);
|
| 2459 | }
|
| 2460 |
|
| 2461 |
|
| 2462 | static void mce_restart(void)
|
| 2463 | {
|
| 2464 | mce_timer_delete_all();
|
| 2465 | on_each_cpu(mce_cpu_restart, NULL, 1);
|
| 2466 | mce_schedule_work();
|
| 2467 | }
|
| 2468 |
|
| 2469 |
|
| 2470 | static void mce_disable_cmci(void *data)
|
| 2471 | {
|
| 2472 | if (!mce_available(raw_cpu_ptr(&cpu_info)))
|
| 2473 | return;
|
| 2474 | cmci_clear();
|
| 2475 | }
|
| 2476 |
|
| 2477 | static void mce_enable_ce(void *all)
|
| 2478 | {
|
| 2479 | if (!mce_available(raw_cpu_ptr(&cpu_info)))
|
| 2480 | return;
|
| 2481 | cmci_reenable();
|
| 2482 | cmci_recheck();
|
| 2483 | if (all)
|
| 2484 | __mcheck_cpu_init_timer();
|
| 2485 | }
|
| 2486 |
|
| 2487 | static const struct bus_type mce_subsys = {
|
| 2488 | .name = "machinecheck",
|
| 2489 | .dev_name = "machinecheck",
|
| 2490 | };
|
| 2491 |
|
| 2492 | DEFINE_PER_CPU(struct device *, mce_device);
|
| 2493 |
|
| 2494 | static inline struct mce_bank_dev *attr_to_bank(struct device_attribute *attr)
|
| 2495 | {
|
| 2496 | return container_of(attr, struct mce_bank_dev, attr);
|
| 2497 | }
|
| 2498 |
|
| 2499 | static ssize_t show_bank(struct device *s, struct device_attribute *attr,
|
| 2500 | char *buf)
|
| 2501 | {
|
| 2502 | u8 bank = attr_to_bank(attr)->bank;
|
| 2503 | struct mce_bank *b;
|
| 2504 |
|
| 2505 | if (bank >= per_cpu(mce_num_banks, s->id))
|
| 2506 | return -EINVAL;
|
| 2507 |
|
| 2508 | b = &per_cpu(mce_banks_array, s->id)[bank];
|
| 2509 |
|
| 2510 | if (!b->init)
|
| 2511 | return -ENODEV;
|
| 2512 |
|
| 2513 | return sprintf(buf, "%llx\n", b->ctl);
|
| 2514 | }
|
| 2515 |
|
| 2516 | static ssize_t set_bank(struct device *s, struct device_attribute *attr,
|
| 2517 | const char *buf, size_t size)
|
| 2518 | {
|
| 2519 | u8 bank = attr_to_bank(attr)->bank;
|
| 2520 | struct mce_bank *b;
|
| 2521 | u64 new;
|
| 2522 |
|
| 2523 | if (kstrtou64(buf, 0, &new) < 0)
|
| 2524 | return -EINVAL;
|
| 2525 |
|
| 2526 | if (bank >= per_cpu(mce_num_banks, s->id))
|
| 2527 | return -EINVAL;
|
| 2528 |
|
| 2529 | b = &per_cpu(mce_banks_array, s->id)[bank];
|
| 2530 | if (!b->init)
|
| 2531 | return -ENODEV;
|
| 2532 |
|
| 2533 | b->ctl = new;
|
| 2534 |
|
| 2535 | mutex_lock(&mce_sysfs_mutex);
|
| 2536 | mce_restart();
|
| 2537 | mutex_unlock(&mce_sysfs_mutex);
|
| 2538 |
|
| 2539 | return size;
|
| 2540 | }
|
| 2541 |
|
| 2542 | static ssize_t set_ignore_ce(struct device *s,
|
| 2543 | struct device_attribute *attr,
|
| 2544 | const char *buf, size_t size)
|
| 2545 | {
|
| 2546 | u64 new;
|
| 2547 |
|
| 2548 | if (kstrtou64(buf, 0, &new) < 0)
|
| 2549 | return -EINVAL;
|
| 2550 |
|
| 2551 | mutex_lock(&mce_sysfs_mutex);
|
| 2552 | if (mca_cfg.ignore_ce ^ !!new) {
|
| 2553 | if (new) {
|
| 2554 |
|
| 2555 | mce_timer_delete_all();
|
| 2556 | on_each_cpu(mce_disable_cmci, NULL, 1);
|
| 2557 | mca_cfg.ignore_ce = true;
|
| 2558 | } else {
|
| 2559 |
|
| 2560 | mca_cfg.ignore_ce = false;
|
| 2561 | on_each_cpu(mce_enable_ce, (void *)1, 1);
|
| 2562 | }
|
| 2563 | }
|
| 2564 | mutex_unlock(&mce_sysfs_mutex);
|
| 2565 |
|
| 2566 | return size;
|
| 2567 | }
|
| 2568 |
|
| 2569 | static ssize_t set_cmci_disabled(struct device *s,
|
| 2570 | struct device_attribute *attr,
|
| 2571 | const char *buf, size_t size)
|
| 2572 | {
|
| 2573 | u64 new;
|
| 2574 |
|
| 2575 | if (kstrtou64(buf, 0, &new) < 0)
|
| 2576 | return -EINVAL;
|
| 2577 |
|
| 2578 | mutex_lock(&mce_sysfs_mutex);
|
| 2579 | if (mca_cfg.cmci_disabled ^ !!new) {
|
| 2580 | if (new) {
|
| 2581 |
|
| 2582 | on_each_cpu(mce_disable_cmci, NULL, 1);
|
| 2583 | mca_cfg.cmci_disabled = true;
|
| 2584 | } else {
|
| 2585 |
|
| 2586 | mca_cfg.cmci_disabled = false;
|
| 2587 | on_each_cpu(mce_enable_ce, NULL, 1);
|
| 2588 | }
|
| 2589 | }
|
| 2590 | mutex_unlock(&mce_sysfs_mutex);
|
| 2591 |
|
| 2592 | return size;
|
| 2593 | }
|
| 2594 |
|
| 2595 | static ssize_t store_int_with_restart(struct device *s,
|
| 2596 | struct device_attribute *attr,
|
| 2597 | const char *buf, size_t size)
|
| 2598 | {
|
| 2599 | unsigned long old_check_interval = check_interval;
|
| 2600 | ssize_t ret = device_store_ulong(s, attr, buf, size);
|
| 2601 |
|
| 2602 | if (check_interval == old_check_interval)
|
| 2603 | return ret;
|
| 2604 |
|
| 2605 | mutex_lock(&mce_sysfs_mutex);
|
| 2606 | mce_restart();
|
| 2607 | mutex_unlock(&mce_sysfs_mutex);
|
| 2608 |
|
| 2609 | return ret;
|
| 2610 | }
|
| 2611 |
|
| 2612 | static DEVICE_INT_ATTR(monarch_timeout, 0644, mca_cfg.monarch_timeout);
|
| 2613 | static DEVICE_BOOL_ATTR(dont_log_ce, 0644, mca_cfg.dont_log_ce);
|
| 2614 | static DEVICE_BOOL_ATTR(print_all, 0644, mca_cfg.print_all);
|
| 2615 |
|
| 2616 | static struct dev_ext_attribute dev_attr_check_interval = {
|
| 2617 | __ATTR(check_interval, 0644, device_show_int, store_int_with_restart),
|
| 2618 | &check_interval
|
| 2619 | };
|
| 2620 |
|
| 2621 | static struct dev_ext_attribute dev_attr_ignore_ce = {
|
| 2622 | __ATTR(ignore_ce, 0644, device_show_bool, set_ignore_ce),
|
| 2623 | &mca_cfg.ignore_ce
|
| 2624 | };
|
| 2625 |
|
| 2626 | static struct dev_ext_attribute dev_attr_cmci_disabled = {
|
| 2627 | __ATTR(cmci_disabled, 0644, device_show_bool, set_cmci_disabled),
|
| 2628 | &mca_cfg.cmci_disabled
|
| 2629 | };
|
| 2630 |
|
| 2631 | static struct device_attribute *mce_device_attrs[] = {
|
| 2632 | &dev_attr_check_interval.attr,
|
| 2633 | #ifdef CONFIG_X86_MCELOG_LEGACY
|
| 2634 | &dev_attr_trigger,
|
| 2635 | #endif
|
| 2636 | &dev_attr_monarch_timeout.attr,
|
| 2637 | &dev_attr_dont_log_ce.attr,
|
| 2638 | &dev_attr_print_all.attr,
|
| 2639 | &dev_attr_ignore_ce.attr,
|
| 2640 | &dev_attr_cmci_disabled.attr,
|
| 2641 | NULL
|
| 2642 | };
|
| 2643 |
|
| 2644 | static cpumask_var_t mce_device_initialized;
|
| 2645 |
|
| 2646 | static void mce_device_release(struct device *dev)
|
| 2647 | {
|
| 2648 | kfree(dev);
|
| 2649 | }
|
| 2650 |
|
| 2651 |
|
| 2652 | static int mce_device_create(unsigned int cpu)
|
| 2653 | {
|
| 2654 | struct device *dev;
|
| 2655 | int err;
|
| 2656 | int i, j;
|
| 2657 |
|
| 2658 | dev = per_cpu(mce_device, cpu);
|
| 2659 | if (dev)
|
| 2660 | return 0;
|
| 2661 |
|
| 2662 | dev = kzalloc_obj(*dev);
|
| 2663 | if (!dev)
|
| 2664 | return -ENOMEM;
|
| 2665 | dev->id = cpu;
|
| 2666 | dev->bus = &mce_subsys;
|
| 2667 | dev->release = &mce_device_release;
|
| 2668 |
|
| 2669 | err = device_register(dev);
|
| 2670 | if (err) {
|
| 2671 | put_device(dev);
|
| 2672 | return err;
|
| 2673 | }
|
| 2674 |
|
| 2675 | for (i = 0; mce_device_attrs[i]; i++) {
|
| 2676 | err = device_create_file(dev, mce_device_attrs[i]);
|
| 2677 | if (err)
|
| 2678 | goto error;
|
| 2679 | }
|
| 2680 | for (j = 0; j < per_cpu(mce_num_banks, cpu); j++) {
|
| 2681 | err = device_create_file(dev, &mce_bank_devs[j].attr);
|
| 2682 | if (err)
|
| 2683 | goto error2;
|
| 2684 | }
|
| 2685 | cpumask_set_cpu(cpu, mce_device_initialized);
|
| 2686 | per_cpu(mce_device, cpu) = dev;
|
| 2687 |
|
| 2688 | return 0;
|
| 2689 | error2:
|
| 2690 | while (--j >= 0)
|
| 2691 | device_remove_file(dev, &mce_bank_devs[j].attr);
|
| 2692 | error:
|
| 2693 | while (--i >= 0)
|
| 2694 | device_remove_file(dev, mce_device_attrs[i]);
|
| 2695 |
|
| 2696 | device_unregister(dev);
|
| 2697 |
|
| 2698 | return err;
|
| 2699 | }
|
| 2700 |
|
| 2701 | static void mce_device_remove(unsigned int cpu)
|
| 2702 | {
|
| 2703 | struct device *dev = per_cpu(mce_device, cpu);
|
| 2704 | int i;
|
| 2705 |
|
| 2706 | if (!cpumask_test_cpu(cpu, mce_device_initialized))
|
| 2707 | return;
|
| 2708 |
|
| 2709 | for (i = 0; mce_device_attrs[i]; i++)
|
| 2710 | device_remove_file(dev, mce_device_attrs[i]);
|
| 2711 |
|
| 2712 | for (i = 0; i < per_cpu(mce_num_banks, cpu); i++)
|
| 2713 | device_remove_file(dev, &mce_bank_devs[i].attr);
|
| 2714 |
|
| 2715 | device_unregister(dev);
|
| 2716 | cpumask_clear_cpu(cpu, mce_device_initialized);
|
| 2717 | per_cpu(mce_device, cpu) = NULL;
|
| 2718 | }
|
| 2719 |
|
| 2720 |
|
| 2721 | static void mce_disable_cpu(void)
|
| 2722 | {
|
| 2723 | if (!mce_available(raw_cpu_ptr(&cpu_info)))
|
| 2724 | return;
|
| 2725 |
|
| 2726 | if (!cpuhp_tasks_frozen)
|
| 2727 | cmci_clear();
|
| 2728 |
|
| 2729 | vendor_disable_error_reporting();
|
| 2730 | }
|
| 2731 |
|
| 2732 | static void mce_reenable_cpu(void)
|
| 2733 | {
|
| 2734 | struct mce_bank *mce_banks = this_cpu_ptr(mce_banks_array);
|
| 2735 | int i;
|
| 2736 |
|
| 2737 | if (!mce_available(raw_cpu_ptr(&cpu_info)))
|
| 2738 | return;
|
| 2739 |
|
| 2740 | if (!cpuhp_tasks_frozen)
|
| 2741 | cmci_reenable();
|
| 2742 | for (i = 0; i < this_cpu_read(mce_num_banks); i++) {
|
| 2743 | struct mce_bank *b = &mce_banks[i];
|
| 2744 |
|
| 2745 | if (b->init)
|
| 2746 | wrmsrq(mca_msr_reg(i, MCA_CTL), b->ctl);
|
| 2747 | }
|
| 2748 | }
|
| 2749 |
|
| 2750 | static int mce_cpu_dead(unsigned int cpu)
|
| 2751 | {
|
| 2752 |
|
| 2753 | if (!cpuhp_tasks_frozen)
|
| 2754 | cmci_rediscover();
|
| 2755 | return 0;
|
| 2756 | }
|
| 2757 |
|
| 2758 | static int mce_cpu_online(unsigned int cpu)
|
| 2759 | {
|
| 2760 | struct timer_list *t = this_cpu_ptr(&mce_timer);
|
| 2761 |
|
| 2762 | mce_device_create(cpu);
|
| 2763 | mce_threshold_create_device(cpu);
|
| 2764 | mce_reenable_cpu();
|
| 2765 | mce_start_timer(t);
|
| 2766 | return 0;
|
| 2767 | }
|
| 2768 |
|
| 2769 | static int mce_cpu_pre_down(unsigned int cpu)
|
| 2770 | {
|
| 2771 | struct timer_list *t = this_cpu_ptr(&mce_timer);
|
| 2772 |
|
| 2773 | mce_disable_cpu();
|
| 2774 | timer_delete_sync(t);
|
| 2775 | mce_threshold_remove_device(cpu);
|
| 2776 | mce_device_remove(cpu);
|
| 2777 | return 0;
|
| 2778 | }
|
| 2779 |
|
| 2780 | static __init void mce_init_banks(void)
|
| 2781 | {
|
| 2782 | int i;
|
| 2783 |
|
| 2784 | for (i = 0; i < MAX_NR_BANKS; i++) {
|
| 2785 | struct mce_bank_dev *b = &mce_bank_devs[i];
|
| 2786 | struct device_attribute *a = &b->attr;
|
| 2787 |
|
| 2788 | b->bank = i;
|
| 2789 |
|
| 2790 | sysfs_attr_init(&a->attr);
|
| 2791 | a->attr.name = b->attrname;
|
| 2792 | snprintf(b->attrname, ATTR_LEN, "bank%d", i);
|
| 2793 |
|
| 2794 | a->attr.mode = 0644;
|
| 2795 | a->show = show_bank;
|
| 2796 | a->store = set_bank;
|
| 2797 | }
|
| 2798 | }
|
| 2799 |
|
| 2800 | |
| 2801 | * When running on XEN, this initcall is ordered against the XEN mcelog
|
| 2802 | * initcall:
|
| 2803 | *
|
| 2804 | * device_initcall(xen_late_init_mcelog);
|
| 2805 | * device_initcall_sync(mcheck_init_device);
|
| 2806 |
|
| 2807 | static __init int mcheck_init_device(void)
|
| 2808 | {
|
| 2809 | int err;
|
| 2810 |
|
| 2811 | |
| 2812 | * Check if we have a spare virtual bit. This will only become
|
| 2813 | * a problem if/when we move beyond 5-level page tables.
|
| 2814 |
|
| 2815 | MAYBE_BUILD_BUG_ON(__VIRTUAL_MASK_SHIFT >= 63);
|
| 2816 |
|
| 2817 | if (!mce_available(&boot_cpu_data)) {
|
| 2818 | err = -EIO;
|
| 2819 | goto err_out;
|
| 2820 | }
|
| 2821 |
|
| 2822 | if (!zalloc_cpumask_var(&mce_device_initialized, GFP_KERNEL)) {
|
| 2823 | err = -ENOMEM;
|
| 2824 | goto err_out;
|
| 2825 | }
|
| 2826 |
|
| 2827 | mce_init_banks();
|
| 2828 |
|
| 2829 | err = subsys_system_register(&mce_subsys, NULL);
|
| 2830 | if (err)
|
| 2831 | goto err_out_mem;
|
| 2832 |
|
| 2833 | err = cpuhp_setup_state(CPUHP_X86_MCE_DEAD, "x86/mce:dead", NULL,
|
| 2834 | mce_cpu_dead);
|
| 2835 | if (err)
|
| 2836 | goto err_out_mem;
|
| 2837 |
|
| 2838 | |
| 2839 | * Invokes mce_cpu_online() on all CPUs which are online when
|
| 2840 | * the state is installed.
|
| 2841 |
|
| 2842 | err = cpuhp_setup_state(CPUHP_AP_ONLINE_DYN, "x86/mce:online",
|
| 2843 | mce_cpu_online, mce_cpu_pre_down);
|
| 2844 | if (err < 0)
|
| 2845 | goto err_out_online;
|
| 2846 |
|
| 2847 | register_syscore(&mce_syscore);
|
| 2848 |
|
| 2849 | return 0;
|
| 2850 |
|
| 2851 | err_out_online:
|
| 2852 | cpuhp_remove_state(CPUHP_X86_MCE_DEAD);
|
| 2853 |
|
| 2854 | err_out_mem:
|
| 2855 | free_cpumask_var(mce_device_initialized);
|
| 2856 |
|
| 2857 | err_out:
|
| 2858 | pr_err("Unable to init MCE device (rc: %d)\n", err);
|
| 2859 |
|
| 2860 | return err;
|
| 2861 | }
|
| 2862 | device_initcall_sync(mcheck_init_device);
|
| 2863 |
|
| 2864 | |
| 2865 | * Old style boot options parsing. Only for compatibility.
|
| 2866 |
|
| 2867 | static int __init mcheck_disable(char *str)
|
| 2868 | {
|
| 2869 | mca_cfg.disabled = 1;
|
| 2870 | return 1;
|
| 2871 | }
|
| 2872 | __setup("nomce", mcheck_disable);
|
| 2873 |
|
| 2874 | #ifdef CONFIG_DEBUG_FS
|
| 2875 | struct dentry *mce_get_debugfs_dir(void)
|
| 2876 | {
|
| 2877 | static struct dentry *dmce;
|
| 2878 |
|
| 2879 | if (!dmce)
|
| 2880 | dmce = debugfs_create_dir("mce", NULL);
|
| 2881 |
|
| 2882 | return dmce;
|
| 2883 | }
|
| 2884 |
|
| 2885 | static void mce_reset(void)
|
| 2886 | {
|
| 2887 | atomic_set(&mce_fake_panicked, 0);
|
| 2888 | atomic_set(&mce_executing, 0);
|
| 2889 | atomic_set(&mce_callin, 0);
|
| 2890 | atomic_set(&global_nwo, 0);
|
| 2891 | cpumask_setall(&mce_missing_cpus);
|
| 2892 | }
|
| 2893 |
|
| 2894 | static int fake_panic_get(void *data, u64 *val)
|
| 2895 | {
|
| 2896 | *val = fake_panic;
|
| 2897 | return 0;
|
| 2898 | }
|
| 2899 |
|
| 2900 | static int fake_panic_set(void *data, u64 val)
|
| 2901 | {
|
| 2902 | mce_reset();
|
| 2903 | fake_panic = val;
|
| 2904 | return 0;
|
| 2905 | }
|
| 2906 |
|
| 2907 | DEFINE_DEBUGFS_ATTRIBUTE(fake_panic_fops, fake_panic_get, fake_panic_set,
|
| 2908 | "%llu\n");
|
| 2909 |
|
| 2910 | static void __init mcheck_debugfs_init(void)
|
| 2911 | {
|
| 2912 | struct dentry *dmce;
|
| 2913 |
|
| 2914 | dmce = mce_get_debugfs_dir();
|
| 2915 | debugfs_create_file_unsafe("fake_panic", 0444, dmce, NULL,
|
| 2916 | &fake_panic_fops);
|
| 2917 | }
|
| 2918 | #else
|
| 2919 | static void __init mcheck_debugfs_init(void) { }
|
| 2920 | #endif
|
| 2921 |
|
| 2922 | static int __init mcheck_late_init(void)
|
| 2923 | {
|
| 2924 | if (mca_cfg.recovery)
|
| 2925 | enable_copy_mc_fragile();
|
| 2926 |
|
| 2927 | mcheck_debugfs_init();
|
| 2928 |
|
| 2929 | |
| 2930 | * Flush out everything that has been logged during early boot, now that
|
| 2931 | * everything has been initialized (workqueues, decoders, ...).
|
| 2932 |
|
| 2933 | mce_schedule_work();
|
| 2934 |
|
| 2935 | return 0;
|
| 2936 | }
|
| 2937 | late_initcall(mcheck_late_init);
|