projects/linux·blob master · refs

1327 lines · 33.2 KiB

1// SPDX-License-Identifier: GPL-2.0-only
2/*
3 * Copyright (C) 1995 Linus Torvalds
4 *
5 * This file contains the setup_arch() code, which handles the architecture-dependent
6 * parts of early kernel initialization.
7 */
8#include <linux/acpi.h>
9#include <linux/bootconfig.h>
10#include <linux/console.h>
11#include <linux/cpu.h>
12#include <linux/crash_dump.h>
13#include <linux/dma-map-ops.h>
14#include <linux/efi.h>
15#include <linux/hugetlb.h>
16#include <linux/ima.h>
17#include <linux/init_ohci1394_dma.h>
18#include <linux/initrd.h>
19#include <linux/iscsi_ibft.h>
20#include <linux/kexec_handover.h>
21#include <linux/memblock.h>
22#include <linux/panic_notifier.h>
23#include <linux/pci.h>
24#include <linux/random.h>
25#include <linux/root_dev.h>
26#include <linux/static_call.h>
27#include <linux/sysfb.h>
28#include <linux/swiotlb.h>
29#include <linux/tboot.h>
30#include <linux/usb/xhci-dbgp.h>
31#include <linux/vmalloc.h>
32
33#include <uapi/linux/mount.h>
34
35#include <xen/xen.h>
36
37#include <asm/apic.h>
38#include <asm/bios_ebda.h>
39#include <asm/bugs.h>
40#include <asm/cacheinfo.h>
41#include <asm/coco.h>
42#include <asm/cpu.h>
43#include <asm/efi.h>
44#include <asm/gart.h>
45#include <asm/hypervisor.h>
46#include <asm/io_apic.h>
47#include <asm/kasan.h>
48#include <asm/kaslr.h>
49#include <asm/mce.h>
50#include <asm/memtype.h>
51#include <asm/mtrr.h>
52#include <asm/nmi.h>
53#include <asm/numa.h>
54#include <asm/olpc_ofw.h>
55#include <asm/pci-direct.h>
56#include <asm/prom.h>
57#include <asm/proto.h>
58#include <asm/realmode.h>
59#include <asm/thermal.h>
60#include <asm/unwind.h>
61#include <asm/vsyscall.h>
62
63/*
64 * max_low_pfn_mapped: highest directly mapped pfn < 4 GB
65 * max_pfn_mapped: highest directly mapped pfn > 4 GB
66 *
67 * The direct mapping only covers E820_TYPE_RAM regions, so the ranges and gaps are
68 * represented by pfn_mapped[].
69 */
70unsigned long max_low_pfn_mapped;
71unsigned long max_pfn_mapped;
72
73#ifdef CONFIG_DMI
74RESERVE_BRK(dmi_alloc, 65536);
75#endif
76
77
78unsigned long _brk_start = (unsigned long)__brk_base;
79unsigned long _brk_end = (unsigned long)__brk_base;
80
81struct boot_params boot_params;
82
83/*
84 * These are the four main kernel memory regions, we put them into
85 * the resource tree so that kdump tools and other debugging tools
86 * recover it:
87 */
88
89static struct resource rodata_resource = {
90 .name = "Kernel rodata",
91 .start = 0,
92 .end = 0,
93 .flags = IORESOURCE_BUSY | IORESOURCE_SYSTEM_RAM
94};
95
96static struct resource data_resource = {
97 .name = "Kernel data",
98 .start = 0,
99 .end = 0,
100 .flags = IORESOURCE_BUSY | IORESOURCE_SYSTEM_RAM
101};
102
103static struct resource code_resource = {
104 .name = "Kernel code",
105 .start = 0,
106 .end = 0,
107 .flags = IORESOURCE_BUSY | IORESOURCE_SYSTEM_RAM
108};
109
110static struct resource bss_resource = {
111 .name = "Kernel bss",
112 .start = 0,
113 .end = 0,
114 .flags = IORESOURCE_BUSY | IORESOURCE_SYSTEM_RAM
115};
116
117
118#ifdef CONFIG_X86_32
119/* CPU data as detected by the assembly code in head_32.S */
120struct cpuinfo_x86 new_cpu_data;
121
122struct apm_info apm_info;
123EXPORT_SYMBOL(apm_info);
124
125#if IS_ENABLED(CONFIG_X86_SPEEDSTEP_SMI)
126struct ist_info ist_info;
127EXPORT_SYMBOL(ist_info);
128#else
129struct ist_info ist_info;
130#endif
131
132#endif
133
134struct cpuinfo_x86 boot_cpu_data __read_mostly;
135EXPORT_SYMBOL(boot_cpu_data);
136SYM_PIC_ALIAS(boot_cpu_data);
137
138#if !defined(CONFIG_X86_PAE) || defined(CONFIG_X86_64)
139__visible unsigned long mmu_cr4_features __ro_after_init;
140#else
141__visible unsigned long mmu_cr4_features __ro_after_init = X86_CR4_PAE;
142#endif
143
144#ifdef CONFIG_IMA
145static phys_addr_t ima_kexec_buffer_phys;
146static size_t ima_kexec_buffer_size;
147#endif
148
149/* Boot loader ID and version as integers, for the benefit of proc_dointvec */
150int bootloader_type, bootloader_version;
151
152static const struct ctl_table x86_sysctl_table[] = {
153 {
154 .procname = "unknown_nmi_panic",
155 .data = &unknown_nmi_panic,
156 .maxlen = sizeof(int),
157 .mode = 0644,
158 .proc_handler = proc_dointvec,
159 },
160 {
161 .procname = "panic_on_unrecovered_nmi",
162 .data = &panic_on_unrecovered_nmi,
163 .maxlen = sizeof(int),
164 .mode = 0644,
165 .proc_handler = proc_dointvec,
166 },
167 {
168 .procname = "panic_on_io_nmi",
169 .data = &panic_on_io_nmi,
170 .maxlen = sizeof(int),
171 .mode = 0644,
172 .proc_handler = proc_dointvec,
173 },
174 {
175 .procname = "bootloader_type",
176 .data = &bootloader_type,
177 .maxlen = sizeof(int),
178 .mode = 0444,
179 .proc_handler = proc_dointvec,
180 },
181 {
182 .procname = "bootloader_version",
183 .data = &bootloader_version,
184 .maxlen = sizeof(int),
185 .mode = 0444,
186 .proc_handler = proc_dointvec,
187 },
188 {
189 .procname = "io_delay_type",
190 .data = &io_delay_type,
191 .maxlen = sizeof(int),
192 .mode = 0644,
193 .proc_handler = proc_dointvec,
194 },
195#if defined(CONFIG_ACPI_SLEEP)
196 {
197 .procname = "acpi_video_flags",
198 .data = &acpi_realmode_flags,
199 .maxlen = sizeof(unsigned long),
200 .mode = 0644,
201 .proc_handler = proc_doulongvec_minmax,
202 },
203#endif
204};
205
206static int __init init_x86_sysctl(void)
207{
208 register_sysctl_init("kernel", x86_sysctl_table);
209 return 0;
210}
211arch_initcall(init_x86_sysctl);
212
213/*
214 * Setup options
215 */
216
217struct sysfb_display_info sysfb_primary_display;
218EXPORT_SYMBOL(sysfb_primary_display);
219
220extern int root_mountflags;
221
222unsigned long saved_video_mode;
223
224#define RAMDISK_IMAGE_START_MASK 0x07FF
225#define RAMDISK_PROMPT_FLAG 0x8000
226#define RAMDISK_LOAD_FLAG 0x4000
227
228static char __initdata command_line[COMMAND_LINE_SIZE];
229#ifdef CONFIG_CMDLINE_BOOL
230char builtin_cmdline[COMMAND_LINE_SIZE] = CONFIG_CMDLINE;
231bool builtin_cmdline_added __ro_after_init;
232#endif
233
234#if IS_ENABLED(CONFIG_EDD)
235struct edd edd;
236#ifdef CONFIG_EDD_MODULE
237EXPORT_SYMBOL(edd);
238#endif
239/**
240 * copy_edd() - Copy the BIOS EDD information
241 * from boot_params into a safe place.
242 *
243 */
244static inline void __init copy_edd(void)
245{
246 memcpy(edd.mbr_signature, boot_params.edd_mbr_sig_buffer,
247 sizeof(edd.mbr_signature));
248 memcpy(edd.edd_info, boot_params.eddbuf, sizeof(edd.edd_info));
249 edd.mbr_signature_nr = boot_params.edd_mbr_sig_buf_entries;
250 edd.edd_info_nr = boot_params.eddbuf_entries;
251}
252#else
253static inline void __init copy_edd(void)
254{
255}
256#endif
257
258void * __init extend_brk(size_t size, size_t align)
259{
260 size_t mask = align - 1;
261 void *ret;
262
263 BUG_ON(_brk_start == 0);
264 BUG_ON(align & mask);
265
266 _brk_end = (_brk_end + mask) & ~mask;
267 BUG_ON((char *)(_brk_end + size) > __brk_limit);
268
269 ret = (void *)_brk_end;
270 _brk_end += size;
271
272 memset(ret, 0, size);
273
274 return ret;
275}
276
277#ifdef CONFIG_X86_32
278static void __init cleanup_highmap(void)
279{
280}
281#endif
282
283static void __init reserve_brk(void)
284{
285 if (_brk_end > _brk_start)
286 memblock_reserve_kern(__pa_symbol(_brk_start),
287 _brk_end - _brk_start);
288
289 /* Mark brk area as locked down and no longer taking any
290 new allocations */
291 _brk_start = 0;
292}
293
294#ifdef CONFIG_BLK_DEV_INITRD
295
296static u64 __init get_ramdisk_image(void)
297{
298 u64 ramdisk_image = boot_params.hdr.ramdisk_image;
299
300 ramdisk_image |= (u64)boot_params.ext_ramdisk_image << 32;
301
302 if (ramdisk_image == 0)
303 ramdisk_image = phys_initrd_start;
304
305 return ramdisk_image;
306}
307static u64 __init get_ramdisk_size(void)
308{
309 u64 ramdisk_size = boot_params.hdr.ramdisk_size;
310
311 ramdisk_size |= (u64)boot_params.ext_ramdisk_size << 32;
312
313 if (ramdisk_size == 0)
314 ramdisk_size = phys_initrd_size;
315
316 return ramdisk_size;
317}
318
319static void __init relocate_initrd(void)
320{
321 /* Assume only end is not page aligned */
322 u64 ramdisk_image = get_ramdisk_image();
323 u64 ramdisk_size = get_ramdisk_size();
324 u64 area_size = PAGE_ALIGN(ramdisk_size);
325 int ret = 0;
326
327 /* We need to move the initrd down into directly mapped mem */
328 u64 relocated_ramdisk = memblock_phys_alloc_range(area_size, PAGE_SIZE, 0,
329 PFN_PHYS(max_pfn_mapped));
330 if (!relocated_ramdisk)
331 panic("Cannot find place for new RAMDISK of size %lld\n",
332 ramdisk_size);
333
334 initrd_start = relocated_ramdisk + PAGE_OFFSET;
335 initrd_end = initrd_start + ramdisk_size;
336 printk(KERN_INFO "Allocated new RAMDISK: [mem %#010llx-%#010llx]\n",
337 relocated_ramdisk, relocated_ramdisk + ramdisk_size - 1);
338
339 ret = copy_from_early_mem((void *)initrd_start, ramdisk_image, ramdisk_size);
340 if (ret)
341 panic("Copy RAMDISK failed\n");
342
343 printk(KERN_INFO "Move RAMDISK from [mem %#010llx-%#010llx] to"
344 " [mem %#010llx-%#010llx]\n",
345 ramdisk_image, ramdisk_image + ramdisk_size - 1,
346 relocated_ramdisk, relocated_ramdisk + ramdisk_size - 1);
347}
348
349static void __init early_reserve_initrd(void)
350{
351 /* Assume only end is not page aligned */
352 u64 ramdisk_image = get_ramdisk_image();
353 u64 ramdisk_size = get_ramdisk_size();
354 u64 ramdisk_end = PAGE_ALIGN(ramdisk_image + ramdisk_size);
355
356 if (!boot_params.hdr.type_of_loader ||
357 !ramdisk_image || !ramdisk_size)
358 return; /* No initrd provided by bootloader */
359
360 memblock_reserve_kern(ramdisk_image, ramdisk_end - ramdisk_image);
361}
362
363static void __init reserve_initrd(void)
364{
365 /* Assume only end is not page aligned */
366 u64 ramdisk_image = get_ramdisk_image();
367 u64 ramdisk_size = get_ramdisk_size();
368 u64 ramdisk_end = PAGE_ALIGN(ramdisk_image + ramdisk_size);
369
370 if (!boot_params.hdr.type_of_loader ||
371 !ramdisk_image || !ramdisk_size)
372 return; /* No initrd provided by bootloader */
373
374 initrd_start = 0;
375
376 printk(KERN_INFO "RAMDISK: [mem %#010llx-%#010llx]\n", ramdisk_image,
377 ramdisk_end - 1);
378
379 if (pfn_range_is_mapped(PFN_DOWN(ramdisk_image),
380 PFN_DOWN(ramdisk_end))) {
381 /* All are mapped, easy case */
382 initrd_start = ramdisk_image + PAGE_OFFSET;
383 initrd_end = initrd_start + ramdisk_size;
384 return;
385 }
386
387 relocate_initrd();
388
389 memblock_phys_free(ramdisk_image, ramdisk_end - ramdisk_image);
390}
391
392#else
393static void __init early_reserve_initrd(void)
394{
395}
396static void __init reserve_initrd(void)
397{
398}
399#endif /* CONFIG_BLK_DEV_INITRD */
400
401static void __init add_early_ima_buffer(u64 phys_addr)
402{
403#ifdef CONFIG_IMA
404 struct ima_setup_data *data;
405
406 data = early_memremap(phys_addr + sizeof(struct setup_data), sizeof(*data));
407 if (!data) {
408 pr_warn("setup: failed to memremap ima_setup_data entry\n");
409 return;
410 }
411
412 if (data->size) {
413 memblock_reserve_kern(data->addr, data->size);
414 ima_kexec_buffer_phys = data->addr;
415 ima_kexec_buffer_size = data->size;
416 }
417
418 early_memunmap(data, sizeof(*data));
419#else
420 pr_warn("Passed IMA kexec data, but CONFIG_IMA not set. Ignoring.\n");
421#endif
422}
423
424#if defined(CONFIG_HAVE_IMA_KEXEC) && !defined(CONFIG_OF_FLATTREE)
425int __init ima_free_kexec_buffer(void)
426{
427 if (!ima_kexec_buffer_size)
428 return -ENOENT;
429
430 memblock_phys_free(ima_kexec_buffer_phys,
431 ima_kexec_buffer_size);
432
433 ima_kexec_buffer_phys = 0;
434 ima_kexec_buffer_size = 0;
435
436 return 0;
437}
438
439int __init ima_get_kexec_buffer(void **addr, size_t *size)
440{
441 int ret;
442
443 if (!ima_kexec_buffer_size)
444 return -ENOENT;
445
446 ret = ima_validate_range(ima_kexec_buffer_phys, ima_kexec_buffer_size);
447 if (ret)
448 return ret;
449
450 *addr = __va(ima_kexec_buffer_phys);
451 *size = ima_kexec_buffer_size;
452
453 return 0;
454}
455#endif
456
457static void __init add_kho(u64 phys_addr, u32 data_len)
458{
459 struct kho_data *kho;
460 u64 addr = phys_addr + sizeof(struct setup_data);
461 u64 size = data_len - sizeof(struct setup_data);
462
463 if (!IS_ENABLED(CONFIG_KEXEC_HANDOVER)) {
464 pr_warn("Passed KHO data, but CONFIG_KEXEC_HANDOVER not set. Ignoring.\n");
465 return;
466 }
467
468 kho = early_memremap(addr, size);
469 if (!kho) {
470 pr_warn("setup: failed to memremap kho data (0x%llx, 0x%llx)\n",
471 addr, size);
472 return;
473 }
474
475 kho_populate(kho->fdt_addr, kho->fdt_size, kho->scratch_addr, kho->scratch_size);
476
477 early_memunmap(kho, size);
478}
479
480static void __init parse_setup_data(void)
481{
482 struct setup_data *data;
483 u64 pa_data, pa_next;
484
485 pa_data = boot_params.hdr.setup_data;
486 while (pa_data) {
487 u32 data_len, data_type;
488
489 data = early_memremap(pa_data, sizeof(*data));
490 data_len = data->len + sizeof(struct setup_data);
491 data_type = data->type;
492 pa_next = data->next;
493 early_memunmap(data, sizeof(*data));
494
495 switch (data_type) {
496 case SETUP_E820_EXT:
497 e820__memory_setup_extended(pa_data, data_len);
498 break;
499 case SETUP_DTB:
500 add_dtb(pa_data);
501 break;
502 case SETUP_EFI:
503 parse_efi_setup(pa_data, data_len);
504 break;
505 case SETUP_IMA:
506 add_early_ima_buffer(pa_data);
507 break;
508 case SETUP_KEXEC_KHO:
509 add_kho(pa_data, data_len);
510 break;
511 case SETUP_RNG_SEED:
512 data = early_memremap(pa_data, data_len);
513 add_bootloader_randomness(data->data, data->len);
514 /* Zero seed for forward secrecy. */
515 memzero_explicit(data->data, data->len);
516 /* Zero length in case we find ourselves back here by accident. */
517 memzero_explicit(&data->len, sizeof(data->len));
518 early_memunmap(data, data_len);
519 break;
520 default:
521 break;
522 }
523 pa_data = pa_next;
524 }
525}
526
527/*
528 * Translate the fields of 'struct boot_param' into global variables
529 * representing these parameters.
530 */
531static void __init parse_boot_params(void)
532{
533 ROOT_DEV = old_decode_dev(boot_params.hdr.root_dev);
534 sysfb_primary_display.screen = boot_params.screen_info;
535#if defined(CONFIG_FIRMWARE_EDID)
536 sysfb_primary_display.edid = boot_params.edid_info;
537#endif
538#ifdef CONFIG_X86_32
539 apm_info.bios = boot_params.apm_bios_info;
540 ist_info = boot_params.ist_info;
541#endif
542 saved_video_mode = boot_params.hdr.vid_mode;
543 bootloader_type = boot_params.hdr.type_of_loader;
544 if ((bootloader_type >> 4) == 0xe) {
545 bootloader_type &= 0xf;
546 bootloader_type |= (boot_params.hdr.ext_loader_type+0x10) << 4;
547 }
548 bootloader_version = bootloader_type & 0xf;
549 bootloader_version |= boot_params.hdr.ext_loader_ver << 4;
550
551#ifdef CONFIG_BLK_DEV_RAM
552 rd_image_start = boot_params.hdr.ram_size & RAMDISK_IMAGE_START_MASK;
553#endif
554#ifdef CONFIG_EFI
555 if (!strncmp((char *)&boot_params.efi_info.efi_loader_signature,
556 EFI32_LOADER_SIGNATURE, 4)) {
557 set_bit(EFI_BOOT, &efi.flags);
558 } else if (!strncmp((char *)&boot_params.efi_info.efi_loader_signature,
559 EFI64_LOADER_SIGNATURE, 4)) {
560 set_bit(EFI_BOOT, &efi.flags);
561 set_bit(EFI_64BIT, &efi.flags);
562 }
563#endif
564
565 if (!boot_params.hdr.root_flags)
566 root_mountflags &= ~MS_RDONLY;
567}
568
569static void __init memblock_x86_reserve_range_setup_data(void)
570{
571 struct setup_indirect *indirect;
572 struct setup_data *data;
573 u64 pa_data, pa_next;
574 u32 len;
575
576 pa_data = boot_params.hdr.setup_data;
577 while (pa_data) {
578 data = early_memremap(pa_data, sizeof(*data));
579 if (!data) {
580 pr_warn("setup: failed to memremap setup_data entry\n");
581 return;
582 }
583
584 len = sizeof(*data);
585 pa_next = data->next;
586
587 memblock_reserve_kern(pa_data, sizeof(*data) + data->len);
588
589 if (data->type == SETUP_INDIRECT) {
590 len += data->len;
591 early_memunmap(data, sizeof(*data));
592 data = early_memremap(pa_data, len);
593 if (!data) {
594 pr_warn("setup: failed to memremap indirect setup_data\n");
595 return;
596 }
597
598 indirect = (struct setup_indirect *)data->data;
599
600 if (indirect->type != SETUP_INDIRECT)
601 memblock_reserve_kern(indirect->addr, indirect->len);
602 }
603
604 pa_data = pa_next;
605 early_memunmap(data, len);
606 }
607}
608
609static void __init arch_reserve_crashkernel(void)
610{
611 unsigned long long crash_base, crash_size, low_size = 0, cma_size = 0;
612 bool high = false;
613 int ret;
614
615 if (!IS_ENABLED(CONFIG_CRASH_RESERVE))
616 return;
617
618 ret = parse_crashkernel(boot_command_line, memblock_phys_mem_size(),
619 &crash_size, &crash_base,
620 &low_size, &cma_size, &high);
621 if (ret)
622 return;
623
624 if (xen_pv_domain()) {
625 pr_info("Ignoring crashkernel for a Xen PV domain\n");
626 return;
627 }
628
629 reserve_crashkernel_generic(crash_size, crash_base, low_size, high);
630 reserve_crashkernel_cma(cma_size);
631}
632
633static struct resource standard_io_resources[] = {
634 { .name = "dma1", .start = 0x00, .end = 0x1f,
635 .flags = IORESOURCE_BUSY | IORESOURCE_IO },
636 { .name = "pic1", .start = 0x20, .end = 0x21,
637 .flags = IORESOURCE_BUSY | IORESOURCE_IO },
638 { .name = "timer0", .start = 0x40, .end = 0x43,
639 .flags = IORESOURCE_BUSY | IORESOURCE_IO },
640 { .name = "timer1", .start = 0x50, .end = 0x53,
641 .flags = IORESOURCE_BUSY | IORESOURCE_IO },
642 { .name = "keyboard", .start = 0x60, .end = 0x60,
643 .flags = IORESOURCE_BUSY | IORESOURCE_IO },
644 { .name = "keyboard", .start = 0x64, .end = 0x64,
645 .flags = IORESOURCE_BUSY | IORESOURCE_IO },
646 { .name = "dma page reg", .start = 0x80, .end = 0x8f,
647 .flags = IORESOURCE_BUSY | IORESOURCE_IO },
648 { .name = "pic2", .start = 0xa0, .end = 0xa1,
649 .flags = IORESOURCE_BUSY | IORESOURCE_IO },
650 { .name = "dma2", .start = 0xc0, .end = 0xdf,
651 .flags = IORESOURCE_BUSY | IORESOURCE_IO },
652 { .name = "fpu", .start = 0xf0, .end = 0xff,
653 .flags = IORESOURCE_BUSY | IORESOURCE_IO }
654};
655
656void __init reserve_standard_io_resources(void)
657{
658 int i;
659
660 /* request I/O space for devices used on all i[345]86 PCs */
661 for (i = 0; i < ARRAY_SIZE(standard_io_resources); i++)
662 request_resource(&ioport_resource, &standard_io_resources[i]);
663
664}
665
666static void __init setup_kernel_resources(void)
667{
668 code_resource.start = __pa_symbol(_text);
669 code_resource.end = __pa_symbol(_etext)-1;
670 rodata_resource.start = __pa_symbol(__start_rodata);
671 rodata_resource.end = __pa_symbol(__end_rodata)-1;
672 data_resource.start = __pa_symbol(_sdata);
673 data_resource.end = __pa_symbol(_edata)-1;
674 bss_resource.start = __pa_symbol(__bss_start);
675 bss_resource.end = __pa_symbol(__bss_stop)-1;
676
677 insert_resource(&iomem_resource, &code_resource);
678 insert_resource(&iomem_resource, &rodata_resource);
679 insert_resource(&iomem_resource, &data_resource);
680 insert_resource(&iomem_resource, &bss_resource);
681}
682
683static bool __init snb_gfx_workaround_needed(void)
684{
685#ifdef CONFIG_PCI
686 int i;
687 u16 vendor, devid;
688 static const __initconst u16 snb_ids[] = {
689 0x0102,
690 0x0112,
691 0x0122,
692 0x0106,
693 0x0116,
694 0x0126,
695 0x010a,
696 };
697
698 /* Assume no if something weird is going on with PCI */
699 if (!early_pci_allowed())
700 return false;
701
702 vendor = read_pci_config_16(0, 2, 0, PCI_VENDOR_ID);
703 if (vendor != 0x8086)
704 return false;
705
706 devid = read_pci_config_16(0, 2, 0, PCI_DEVICE_ID);
707 for (i = 0; i < ARRAY_SIZE(snb_ids); i++)
708 if (devid == snb_ids[i])
709 return true;
710#endif
711
712 return false;
713}
714
715/*
716 * Sandy Bridge graphics has trouble with certain ranges, exclude
717 * them from allocation.
718 */
719static void __init trim_snb_memory(void)
720{
721 static const __initconst unsigned long bad_pages[] = {
722 0x20050000,
723 0x20110000,
724 0x20130000,
725 0x20138000,
726 0x40004000,
727 };
728 int i;
729
730 if (!snb_gfx_workaround_needed())
731 return;
732
733 printk(KERN_DEBUG "reserving inaccessible SNB gfx pages\n");
734
735 /*
736 * SandyBridge integrated graphics devices have a bug that prevents
737 * them from accessing certain memory ranges, namely anything below
738 * 1M and in the pages listed in bad_pages[] above.
739 *
740 * To avoid these pages being ever accessed by SNB gfx devices reserve
741 * bad_pages that have not already been reserved at boot time.
742 * All memory below the 1 MB mark is anyway reserved later during
743 * setup_arch(), so there is no need to reserve it here.
744 */
745
746 for (i = 0; i < ARRAY_SIZE(bad_pages); i++) {
747 if (memblock_reserve(bad_pages[i], PAGE_SIZE))
748 printk(KERN_WARNING "failed to reserve 0x%08lx\n",
749 bad_pages[i]);
750 }
751}
752
753static void __init trim_bios_range(void)
754{
755 /*
756 * A special case is the first 4Kb of memory;
757 * This is a BIOS owned area, not kernel ram, but generally
758 * not listed as such in the E820 table.
759 *
760 * This typically reserves additional memory (64KiB by default)
761 * since some BIOSes are known to corrupt low memory. See the
762 * Kconfig help text for X86_RESERVE_LOW.
763 */
764 e820__range_update(0, PAGE_SIZE, E820_TYPE_RAM, E820_TYPE_RESERVED);
765
766 /*
767 * special case: Some BIOSes report the PC BIOS
768 * area (640Kb -> 1Mb) as RAM even though it is not.
769 * take them out.
770 */
771 e820__range_remove(BIOS_BEGIN, BIOS_END - BIOS_BEGIN, E820_TYPE_RAM);
772
773 e820__update_table(e820_table);
774}
775
776/* called before trim_bios_range() to spare extra sanitize */
777static void __init e820_add_kernel_range(void)
778{
779 u64 start = __pa_symbol(_text);
780 u64 size = __pa_symbol(_end) - start;
781
782 /*
783 * Complain if .text .data and .bss are not marked as E820_TYPE_RAM and
784 * attempt to fix it by adding the range. We may have a confused BIOS,
785 * or the user may have used memmap=exactmap or memmap=xxM$yyM to
786 * exclude kernel range. If we really are running on top non-RAM,
787 * we will crash later anyways.
788 */
789 if (e820__mapped_all(start, start + size, E820_TYPE_RAM))
790 return;
791
792 pr_warn(".text .data .bss are not marked as E820_TYPE_RAM!\n");
793 e820__range_remove(start, size, 0);
794 e820__range_add(start, size, E820_TYPE_RAM);
795}
796
797static void __init early_reserve_memory(void)
798{
799 /*
800 * Reserve the memory occupied by the kernel between _text and
801 * __end_of_kernel_reserve symbols. Any kernel sections after the
802 * __end_of_kernel_reserve symbol must be explicitly reserved with a
803 * separate memblock_reserve() or they will be discarded.
804 */
805 memblock_reserve_kern(__pa_symbol(_text),
806 (unsigned long)__end_of_kernel_reserve - (unsigned long)_text);
807
808 /*
809 * The first 4Kb of memory is a BIOS owned area, but generally it is
810 * not listed as such in the E820 table.
811 *
812 * Reserve the first 64K of memory since some BIOSes are known to
813 * corrupt low memory. After the real mode trampoline is allocated the
814 * rest of the memory below 640k is reserved.
815 *
816 * In addition, make sure page 0 is always reserved because on
817 * systems with L1TF its contents can be leaked to user processes.
818 */
819 memblock_reserve(0, SZ_64K);
820
821 early_reserve_initrd();
822
823 memblock_x86_reserve_range_setup_data();
824
825 reserve_bios_regions();
826 trim_snb_memory();
827}
828
829/*
830 * Dump out kernel offset information on panic.
831 */
832static int
833dump_kernel_offset(struct notifier_block *self, unsigned long v, void *p)
834{
835 if (kaslr_enabled()) {
836 pr_emerg("Kernel Offset: 0x%lx from 0x%lx (relocation range: 0x%lx-0x%lx)\n",
837 kaslr_offset(),
838 __START_KERNEL,
839 __START_KERNEL_map,
840 MODULES_VADDR-1);
841 } else {
842 pr_emerg("Kernel Offset: disabled\n");
843 }
844
845 return 0;
846}
847
848void x86_configure_nx(void)
849{
850 if (boot_cpu_has(X86_FEATURE_NX))
851 __supported_pte_mask |= _PAGE_NX;
852 else
853 __supported_pte_mask &= ~_PAGE_NX;
854}
855
856static void __init x86_report_nx(void)
857{
858 if (!boot_cpu_has(X86_FEATURE_NX)) {
859 printk(KERN_NOTICE "Notice: NX (Execute Disable) protection "
860 "missing in CPU!\n");
861 } else {
862#if defined(CONFIG_X86_64) || defined(CONFIG_X86_PAE)
863 printk(KERN_INFO "NX (Execute Disable) protection: active\n");
864#else
865 /* 32bit non-PAE kernel, NX cannot be used */
866 printk(KERN_NOTICE "Notice: NX (Execute Disable) protection "
867 "cannot be enabled: non-PAE kernel!\n");
868#endif
869 }
870}
871
872/*
873 * Determine if we were loaded by an EFI loader. If so, then we have also been
874 * passed the efi memmap, systab, etc., so we should use these data structures
875 * for initialization. Note, the efi init code path is determined by the
876 * global efi_enabled. This allows the same kernel image to be used on existing
877 * systems (with a traditional BIOS) as well as on EFI systems.
878 */
879/*
880 * setup_arch - architecture-specific boot-time initializations
881 *
882 * Note: On x86_64, fixmaps are ready for use even before this is called.
883 */
884void __init setup_arch(char **cmdline_p)
885{
886#ifdef CONFIG_X86_32
887 memcpy(&boot_cpu_data, &new_cpu_data, sizeof(new_cpu_data));
888
889 /*
890 * copy kernel address range established so far and switch
891 * to the proper swapper page table
892 */
893 clone_pgd_range(swapper_pg_dir + KERNEL_PGD_BOUNDARY,
894 initial_page_table + KERNEL_PGD_BOUNDARY,
895 KERNEL_PGD_PTRS);
896
897 load_cr3(swapper_pg_dir);
898 /*
899 * Note: Quark X1000 CPUs advertise PGE incorrectly and require
900 * a cr3 based tlb flush, so the following __flush_tlb_all()
901 * will not flush anything because the CPU quirk which clears
902 * X86_FEATURE_PGE has not been invoked yet. Though due to the
903 * load_cr3() above the TLB has been flushed already. The
904 * quirk is invoked before subsequent calls to __flush_tlb_all()
905 * so proper operation is guaranteed.
906 */
907 __flush_tlb_all();
908#else
909 printk(KERN_INFO "Command line: %s\n", boot_command_line);
910 boot_cpu_data.x86_phys_bits = MAX_PHYSMEM_BITS;
911#endif
912
913#ifdef CONFIG_CMDLINE_BOOL
914#ifdef CONFIG_CMDLINE_OVERRIDE
915 strscpy(boot_command_line, builtin_cmdline, COMMAND_LINE_SIZE);
916#else
917 if (builtin_cmdline[0]) {
918 /* append boot loader cmdline to builtin */
919 strlcat(builtin_cmdline, " ", COMMAND_LINE_SIZE);
920 strlcat(builtin_cmdline, boot_command_line, COMMAND_LINE_SIZE);
921 strscpy(boot_command_line, builtin_cmdline, COMMAND_LINE_SIZE);
922 }
923#endif
924 builtin_cmdline_added = true;
925#endif
926
927#ifdef CONFIG_CMDLINE_FROM_BOOTCONFIG
928 /*
929 * Prepend the build-time-rendered embedded "kernel" keys here so
930 * parse_early_param() below sees them, using the same opt-in as the
931 * runtime parser, plus the build-time CONFIG_BOOT_CONFIG_FORCE.
932 */
933 if (bootconfig_cmdline_requested(boot_command_line, NULL) ||
934 IS_ENABLED(CONFIG_BOOT_CONFIG_FORCE))
935 xbc_prepend_embedded_cmdline(boot_command_line,
936 COMMAND_LINE_SIZE);
937#endif
938
939 strscpy(command_line, boot_command_line, COMMAND_LINE_SIZE);
940 *cmdline_p = command_line;
941
942 /*
943 * If we have OLPC OFW, we might end up relocating the fixmap due to
944 * reserve_top(), so do this before touching the ioremap area.
945 */
946 olpc_ofw_detect();
947
948 idt_setup_early_traps();
949 early_cpu_init();
950 jump_label_init();
951 static_call_init();
952 early_ioremap_init();
953
954 setup_olpc_ofw_pgd();
955
956 parse_boot_params();
957
958 x86_init.oem.arch_setup();
959
960 /*
961 * Do some memory reservations *before* memory is added to memblock, so
962 * memblock allocations won't overwrite it.
963 *
964 * After this point, everything still needed from the boot loader or
965 * firmware or kernel text should be early reserved or marked not RAM in
966 * e820. All other memory is free game.
967 *
968 * This call needs to happen before e820__memory_setup() which calls the
969 * xen_memory_setup() on Xen dom0 which relies on the fact that those
970 * early reservations have happened already.
971 */
972 early_reserve_memory();
973
974 iomem_resource.end = (1ULL << boot_cpu_data.x86_phys_bits) - 1;
975 e820__memory_setup();
976 parse_setup_data();
977
978 copy_edd();
979
980 setup_initial_init_mm(_text, _etext, _edata, (void *)_brk_end);
981
982 /*
983 * x86_configure_nx() is called before parse_early_param() to detect
984 * whether hardware doesn't support NX (so that the early EHCI debug
985 * console setup can safely call set_fixmap()).
986 */
987 x86_configure_nx();
988
989 parse_early_param();
990
991 if (efi_enabled(EFI_BOOT))
992 efi_memblock_x86_reserve_range();
993
994 x86_report_nx();
995
996 apic_setup_apic_calls();
997
998 if (acpi_mps_check()) {
999#ifdef CONFIG_X86_LOCAL_APIC
1000 apic_is_disabled = true;
1001#endif
1002 setup_clear_cpu_cap(X86_FEATURE_APIC);
1003 }
1004
1005 e820__finish_early_params();
1006
1007 if (efi_enabled(EFI_BOOT))
1008 efi_init();
1009
1010 reserve_ibft_region();
1011 x86_init.resources.dmi_setup();
1012
1013 /*
1014 * VMware detection requires dmi to be available, so this
1015 * needs to be done after dmi_setup(), for the boot CPU.
1016 * For some guest types (Xen PV, SEV-SNP, TDX) it is required to be
1017 * called before cache_bp_init() for setting up MTRR state.
1018 */
1019 init_hypervisor_platform();
1020
1021 tsc_early_init();
1022 x86_init.resources.probe_roms();
1023
1024 /*
1025 * Add resources for kernel text and data to the iomem_resource.
1026 * Do it after parse_early_param, so it can be debugged.
1027 */
1028 setup_kernel_resources();
1029
1030 e820_add_kernel_range();
1031 trim_bios_range();
1032#ifdef CONFIG_X86_32
1033 if (ppro_with_ram_bug()) {
1034 pr_info("Applying PPro RAM bug workaround: punching 256 kB hole at 1.75 GB physical.\n");
1035 e820__range_update(0x70000000ULL, SZ_256K, E820_TYPE_RAM, E820_TYPE_RESERVED);
1036 e820__update_table(e820_table);
1037 }
1038#else
1039 early_gart_iommu_check();
1040#endif
1041
1042 /*
1043 * partially used pages are not usable - thus
1044 * we are rounding upwards:
1045 */
1046 max_pfn = e820__end_of_ram_pfn();
1047
1048 /* update e820 for memory not covered by WB MTRRs */
1049 cache_bp_init();
1050 if (mtrr_trim_uncached_memory(max_pfn))
1051 max_pfn = e820__end_of_ram_pfn();
1052
1053 max_possible_pfn = max_pfn;
1054
1055 /*
1056 * Define random base addresses for memory sections after max_pfn is
1057 * defined and before each memory section base is used.
1058 */
1059 kernel_randomize_memory();
1060
1061#ifdef CONFIG_X86_32
1062 /* max_low_pfn get updated here */
1063 find_low_pfn_range();
1064#else
1065 check_x2apic();
1066
1067 /* How many end-of-memory variables you have, grandma! */
1068 /* need this before calling reserve_initrd */
1069 if (max_pfn > (1UL<<(32 - PAGE_SHIFT)))
1070 max_low_pfn = e820__end_of_low_ram_pfn();
1071 else
1072 max_low_pfn = max_pfn;
1073#endif
1074
1075 /* Find and reserve MPTABLE area */
1076 x86_init.mpparse.find_mptable();
1077
1078 early_alloc_pgt_buf();
1079
1080 /*
1081 * Need to conclude brk, before e820__memblock_setup()
1082 * it could use memblock_find_in_range, could overlap with
1083 * brk area.
1084 */
1085 reserve_brk();
1086
1087 cleanup_highmap();
1088
1089 e820__memblock_setup();
1090
1091 /*
1092 * Needs to run after memblock setup because it needs the physical
1093 * memory size.
1094 */
1095 mem_encrypt_setup_arch();
1096 cc_random_init();
1097
1098 efi_find_mirror();
1099 efi_esrt_init();
1100 efi_mokvar_table_init();
1101
1102 /*
1103 * The EFI specification says that boot service code won't be
1104 * called after ExitBootServices(). This is, in fact, a lie.
1105 */
1106 efi_reserve_boot_services();
1107
1108 /* preallocate 4k for mptable mpc */
1109 e820__memblock_alloc_reserved_mpc_new();
1110
1111#ifdef CONFIG_X86_CHECK_BIOS_CORRUPTION
1112 setup_bios_corruption_check();
1113#endif
1114
1115#ifdef CONFIG_X86_32
1116 printk(KERN_DEBUG "initial memory mapped: [mem 0x00000000-%#010lx]\n",
1117 (max_pfn_mapped<<PAGE_SHIFT) - 1);
1118#endif
1119
1120 /*
1121 * Find free memory for the real mode trampoline and place it there. If
1122 * there is not enough free memory under 1M, on EFI-enabled systems
1123 * there will be additional attempt to reclaim the memory for the real
1124 * mode trampoline at efi_free_boot_services().
1125 *
1126 * Unconditionally reserve the entire first 1M of RAM because BIOSes
1127 * are known to corrupt low memory and several hundred kilobytes are not
1128 * worth complex detection what memory gets clobbered. Windows does the
1129 * same thing for very similar reasons.
1130 *
1131 * Moreover, on machines with SandyBridge graphics or in setups that use
1132 * crashkernel the entire 1M is reserved anyway.
1133 *
1134 * Note the host kernel TDX also requires the first 1MB being reserved.
1135 */
1136 x86_platform.realmode_reserve();
1137
1138 init_mem_mapping();
1139
1140 /*
1141 * init_mem_mapping() relies on the early IDT page fault handling.
1142 * Now either enable FRED or install the real page fault handler
1143 * for 64-bit in the IDT.
1144 */
1145 cpu_init_replace_early_idt();
1146
1147 /*
1148 * Update mmu_cr4_features (and, indirectly, trampoline_cr4_features)
1149 * with the current CR4 value. This may not be necessary, but
1150 * auditing all the early-boot CR4 manipulation would be needed to
1151 * rule it out.
1152 *
1153 * Mask off features that don't work outside long mode (just
1154 * PCIDE for now).
1155 */
1156 mmu_cr4_features = __read_cr4() & ~X86_CR4_PCIDE;
1157
1158 memblock_set_current_limit(get_max_mapped());
1159
1160 /*
1161 * NOTE: On x86-32, only from this point on, fixmaps are ready for use.
1162 */
1163
1164#ifdef CONFIG_PROVIDE_OHCI1394_DMA_INIT
1165 if (init_ohci1394_dma_early)
1166 init_ohci1394_dma_on_all_controllers();
1167#endif
1168 /* Allocate bigger log buffer */
1169 setup_log_buf(1);
1170
1171 if (efi_enabled(EFI_BOOT)) {
1172 switch (boot_params.secure_boot) {
1173 case efi_secureboot_mode_disabled:
1174 pr_info("Secure boot disabled\n");
1175 break;
1176 case efi_secureboot_mode_enabled:
1177 pr_info("Secure boot enabled\n");
1178 break;
1179 default:
1180 pr_info("Secure boot could not be determined\n");
1181 break;
1182 }
1183 }
1184
1185 reserve_initrd();
1186
1187 acpi_table_upgrade();
1188 /* Look for ACPI tables and reserve memory occupied by them. */
1189 acpi_boot_table_init();
1190
1191 vsmp_init();
1192
1193 io_delay_init();
1194
1195 early_platform_quirks();
1196
1197 /* Some platforms need the APIC registered for NUMA configuration */
1198 early_acpi_boot_init();
1199 x86_init.mpparse.early_parse_smp_cfg();
1200
1201 x86_flattree_get_config();
1202
1203 initmem_init();
1204 dma_contiguous_reserve(max_pfn_mapped << PAGE_SHIFT);
1205
1206 /*
1207 * Reserve memory for crash kernel after SRAT is parsed so that it
1208 * won't consume hotpluggable memory.
1209 */
1210 arch_reserve_crashkernel();
1211
1212 if (!early_xdbc_setup_hardware())
1213 early_xdbc_register_console();
1214
1215 x86_init.paging.pagetable_init();
1216
1217 kasan_init();
1218
1219 /*
1220 * Sync back kernel address range.
1221 *
1222 * FIXME: Can the later sync in setup_cpu_entry_areas() replace
1223 * this call?
1224 */
1225 sync_initial_page_table();
1226
1227 tboot_probe();
1228
1229 map_vsyscall();
1230
1231 x86_32_probe_apic();
1232
1233 early_quirks();
1234
1235 topology_apply_cmdline_limits_early();
1236
1237 /*
1238 * Parse SMP configuration. Try ACPI first and then the platform
1239 * specific parser.
1240 */
1241 acpi_boot_init();
1242 x86_init.mpparse.parse_smp_cfg();
1243
1244 /* Last opportunity to detect and map the local APIC */
1245 init_apic_mappings();
1246
1247 topology_init_possible_cpus();
1248
1249 init_cpu_to_node();
1250 init_gi_nodes();
1251
1252 io_apic_init_mappings();
1253
1254 x86_init.hyper.guest_late_init();
1255
1256 e820__reserve_resources();
1257 e820__register_nosave_regions(max_pfn);
1258
1259 x86_init.resources.reserve_resources();
1260
1261 e820__setup_pci_gap();
1262
1263#ifdef CONFIG_VT
1264#if defined(CONFIG_VGA_CONSOLE)
1265 if (!efi_enabled(EFI_BOOT) || (efi_mem_type(0xa0000) != EFI_CONVENTIONAL_MEMORY))
1266 vgacon_register_screen(&sysfb_primary_display.screen);
1267#endif
1268#endif
1269 x86_init.oem.banner();
1270
1271 x86_init.timers.wallclock_init();
1272
1273 /*
1274 * This needs to run before setup_local_APIC() which soft-disables the
1275 * local APIC temporarily and that masks the thermal LVT interrupt,
1276 * leading to softlockups on machines which have configured SMI
1277 * interrupt delivery.
1278 */
1279 therm_lvt_init();
1280
1281 mcheck_init();
1282
1283 register_refined_jiffies(PIT_TICK_RATE);
1284
1285#ifdef CONFIG_EFI
1286 if (efi_enabled(EFI_BOOT))
1287 efi_apply_memmap_quirks();
1288#endif
1289
1290 unwind_init();
1291}
1292
1293#ifdef CONFIG_X86_32
1294
1295static struct resource video_ram_resource = {
1296 .name = "Video RAM area",
1297 .start = 0xa0000,
1298 .end = 0xbffff,
1299 .flags = IORESOURCE_BUSY | IORESOURCE_MEM
1300};
1301
1302void __init i386_reserve_resources(void)
1303{
1304 request_resource(&iomem_resource, &video_ram_resource);
1305 reserve_standard_io_resources();
1306}
1307
1308#endif /* CONFIG_X86_32 */
1309
1310static struct notifier_block kernel_offset_notifier = {
1311 .notifier_call = dump_kernel_offset
1312};
1313
1314static int __init register_kernel_offset_dumper(void)
1315{
1316 atomic_notifier_chain_register(&panic_notifier_list,
1317 &kernel_offset_notifier);
1318 return 0;
1319}
1320__initcall(register_kernel_offset_dumper);
1321
1322#ifdef CONFIG_HOTPLUG_CPU
1323bool arch_cpu_is_hotpluggable(int cpu)
1324{
1325 return cpu > 0;
1326}
1327#endif /* CONFIG_HOTPLUG_CPU */