projects/linux·blob master · refs

477 lines · 11.4 KiB

1// SPDX-License-Identifier: GPL-2.0-only
2/*
3 * linux/arch/arm/mm/init.c
4 *
5 * Copyright (C) 1995-2005 Russell King
6 */
7#include <linux/kernel.h>
8#include <linux/errno.h>
9#include <linux/swap.h>
10#include <linux/init.h>
11#include <linux/mman.h>
12#include <linux/sched/signal.h>
13#include <linux/sched/task.h>
14#include <linux/export.h>
15#include <linux/nodemask.h>
16#include <linux/initrd.h>
17#include <linux/of_fdt.h>
18#include <linux/highmem.h>
19#include <linux/gfp.h>
20#include <linux/memblock.h>
21#include <linux/dma-map-ops.h>
22#include <linux/sizes.h>
23#include <linux/stop_machine.h>
24#include <linux/swiotlb.h>
25#include <linux/execmem.h>
26
27#include <asm/cp15.h>
28#include <asm/mach-types.h>
29#include <asm/memblock.h>
30#include <asm/page.h>
31#include <asm/prom.h>
32#include <asm/sections.h>
33#include <asm/setup.h>
34#include <asm/set_memory.h>
35#include <asm/system_info.h>
36#include <asm/tlb.h>
37#include <asm/fixmap.h>
38#include <asm/ptdump.h>
39
40#include <asm/mach/arch.h>
41#include <asm/mach/map.h>
42
43#include "mm.h"
44
45#ifdef CONFIG_CPU_CP15_MMU
46unsigned long __init __clear_cr(unsigned long mask)
47{
48 cr_alignment = cr_alignment & ~mask;
49 return cr_alignment;
50}
51#endif
52
53#ifdef CONFIG_BLK_DEV_INITRD
54static int __init parse_tag_initrd(const struct tag *tag)
55{
56 pr_warn("ATAG_INITRD is deprecated; "
57 "please update your bootloader.\n");
58 phys_initrd_start = __virt_to_phys(tag->u.initrd.start);
59 phys_initrd_size = tag->u.initrd.size;
60 return 0;
61}
62
63__tagtable(ATAG_INITRD, parse_tag_initrd);
64
65static int __init parse_tag_initrd2(const struct tag *tag)
66{
67 phys_initrd_start = tag->u.initrd.start;
68 phys_initrd_size = tag->u.initrd.size;
69 return 0;
70}
71
72__tagtable(ATAG_INITRD2, parse_tag_initrd2);
73#endif
74
75static void __init find_limits(unsigned long *min, unsigned long *max_low,
76 unsigned long *max_high)
77{
78 *max_low = PFN_DOWN(memblock_get_current_limit());
79 *min = PFN_UP(memblock_start_of_DRAM());
80 *max_high = PFN_DOWN(memblock_end_of_DRAM());
81}
82
83#ifdef CONFIG_ZONE_DMA
84
85phys_addr_t arm_dma_zone_size __read_mostly;
86EXPORT_SYMBOL(arm_dma_zone_size);
87
88/*
89 * The DMA mask corresponding to the maximum bus address allocatable
90 * using GFP_DMA. The default here places no restriction on DMA
91 * allocations. This must be the smallest DMA mask in the system,
92 * so a successful GFP_DMA allocation will always satisfy this.
93 */
94phys_addr_t arm_dma_limit;
95unsigned long arm_dma_pfn_limit;
96#endif
97
98void __init setup_dma_zone(const struct machine_desc *mdesc)
99{
100#ifdef CONFIG_ZONE_DMA
101 if (mdesc->dma_zone_size) {
102 arm_dma_zone_size = mdesc->dma_zone_size;
103 arm_dma_limit = PHYS_OFFSET + arm_dma_zone_size - 1;
104 } else
105 arm_dma_limit = 0xffffffff;
106 arm_dma_pfn_limit = arm_dma_limit >> PAGE_SHIFT;
107#endif
108}
109
110void __init arch_zone_limits_init(unsigned long *max_zone_pfn)
111{
112#ifdef CONFIG_ZONE_DMA
113 max_zone_pfn[ZONE_DMA] = min(arm_dma_pfn_limit, max_low_pfn);
114#endif
115 max_zone_pfn[ZONE_NORMAL] = max_low_pfn;
116#ifdef CONFIG_HIGHMEM
117 max_zone_pfn[ZONE_HIGHMEM] = max_pfn;
118#endif
119}
120
121#ifdef CONFIG_HAVE_ARCH_PFN_VALID
122int pfn_valid(unsigned long pfn)
123{
124 phys_addr_t addr = __pfn_to_phys(pfn);
125 unsigned long pageblock_size = PAGE_SIZE * pageblock_nr_pages;
126
127 if (__phys_to_pfn(addr) != pfn)
128 return 0;
129
130 /*
131 * If address less than pageblock_size bytes away from a present
132 * memory chunk there still will be a memory map entry for it
133 * because we round freed memory map to the pageblock boundaries.
134 */
135 if (memblock_overlaps_region(&memblock.memory,
136 ALIGN_DOWN(addr, pageblock_size),
137 pageblock_size))
138 return 1;
139
140 return 0;
141}
142EXPORT_SYMBOL(pfn_valid);
143#endif
144
145static bool arm_memblock_steal_permitted = true;
146
147phys_addr_t __init arm_memblock_steal(phys_addr_t size, phys_addr_t align)
148{
149 phys_addr_t phys;
150
151 BUG_ON(!arm_memblock_steal_permitted);
152
153 phys = memblock_phys_alloc(size, align);
154 if (!phys)
155 panic("Failed to steal %pa bytes at %pS\n",
156 &size, (void *)_RET_IP_);
157
158 memblock_phys_free(phys, size);
159 memblock_remove(phys, size);
160
161 return phys;
162}
163
164#ifdef CONFIG_CPU_ICACHE_MISMATCH_WORKAROUND
165void check_cpu_icache_size(int cpuid)
166{
167 u32 size, ctr;
168
169 asm("mrc p15, 0, %0, c0, c0, 1" : "=r" (ctr));
170
171 size = 1 << ((ctr & 0xf) + 2);
172 if (cpuid != 0 && icache_size != size)
173 pr_info("CPU%u: detected I-Cache line size mismatch, workaround enabled\n",
174 cpuid);
175 if (icache_size > size)
176 icache_size = size;
177}
178#endif
179
180void __init arm_memblock_init(const struct machine_desc *mdesc)
181{
182 /* Register the kernel text, kernel data and initrd with memblock. */
183 memblock_reserve(__pa(KERNEL_START), KERNEL_END - KERNEL_START);
184
185 reserve_initrd_mem();
186
187 arm_mm_memblock_reserve();
188
189 /* reserve any platform specific memblock areas */
190 if (mdesc->reserve)
191 mdesc->reserve();
192
193 early_init_fdt_scan_reserved_mem();
194
195 /* reserve memory for DMA contiguous allocations */
196 dma_contiguous_reserve(arm_dma_limit);
197
198 arm_memblock_steal_permitted = false;
199 memblock_dump_all();
200}
201
202void __init bootmem_init(void)
203{
204 memblock_allow_resize();
205
206 find_limits(&min_low_pfn, &max_low_pfn, &max_pfn);
207
208 early_memtest((phys_addr_t)min_low_pfn << PAGE_SHIFT,
209 (phys_addr_t)max_low_pfn << PAGE_SHIFT);
210}
211
212/*
213 * Poison init memory with an undefined instruction (ARM) or a branch to an
214 * undefined instruction (Thumb).
215 */
216static inline void poison_init_mem(void *s, size_t count)
217{
218 u32 *p = (u32 *)s;
219 for (; count != 0; count -= 4)
220 *p++ = 0xe7fddef0;
221}
222
223void __init arch_mm_preinit(void)
224{
225#ifdef CONFIG_ARM_LPAE
226 swiotlb_init(max_pfn > arm_dma_pfn_limit, SWIOTLB_VERBOSE);
227#endif
228
229#ifdef CONFIG_SA1111
230 /* now that our DMA memory is actually so designated, we can free it */
231 memblock_phys_free(PHYS_OFFSET, __pa(swapper_pg_dir) - PHYS_OFFSET);
232#endif
233
234 /*
235 * Check boundaries twice: Some fundamental inconsistencies can
236 * be detected at build time already.
237 */
238#ifdef CONFIG_MMU
239 BUILD_BUG_ON(TASK_SIZE > MODULES_VADDR);
240 BUG_ON(TASK_SIZE > MODULES_VADDR);
241#endif
242
243#ifdef CONFIG_HIGHMEM
244 BUILD_BUG_ON(PKMAP_BASE + LAST_PKMAP * PAGE_SIZE > PAGE_OFFSET);
245 BUG_ON(PKMAP_BASE + LAST_PKMAP * PAGE_SIZE > PAGE_OFFSET);
246#endif
247}
248
249#ifdef CONFIG_STRICT_KERNEL_RWX
250struct section_perm {
251 const char *name;
252 unsigned long start;
253 unsigned long end;
254 pmdval_t mask;
255 pmdval_t prot;
256 pmdval_t clear;
257};
258
259/* First section-aligned location at or after __start_rodata. */
260extern char __start_rodata_section_aligned[];
261
262static struct section_perm nx_perms[] = {
263 /* Make pages tables, etc before _stext RW (set NX). */
264 {
265 .name = "pre-text NX",
266 .start = PAGE_OFFSET,
267 .end = (unsigned long)_stext,
268 .mask = ~PMD_SECT_XN,
269 .prot = PMD_SECT_XN,
270 },
271 /* Make init RW (set NX). */
272 {
273 .name = "init NX",
274 .start = (unsigned long)__init_begin,
275 .end = (unsigned long)_sdata,
276 .mask = ~PMD_SECT_XN,
277 .prot = PMD_SECT_XN,
278 },
279 /* Make rodata NX (set RO in ro_perms below). */
280 {
281 .name = "rodata NX",
282 .start = (unsigned long)__start_rodata_section_aligned,
283 .end = (unsigned long)__init_begin,
284 .mask = ~PMD_SECT_XN,
285 .prot = PMD_SECT_XN,
286 },
287};
288
289static struct section_perm ro_perms[] = {
290 /* Make kernel code and rodata RX (set RO). */
291 {
292 .name = "text/rodata RO",
293 .start = (unsigned long)_stext,
294 .end = (unsigned long)__init_begin,
295#ifdef CONFIG_ARM_LPAE
296 .mask = ~(L_PMD_SECT_RDONLY | PMD_SECT_AP2),
297 .prot = L_PMD_SECT_RDONLY | PMD_SECT_AP2,
298#else
299 .mask = ~(PMD_SECT_APX | PMD_SECT_AP_WRITE),
300 .prot = PMD_SECT_APX | PMD_SECT_AP_WRITE,
301 .clear = PMD_SECT_AP_WRITE,
302#endif
303 },
304};
305
306/*
307 * Updates section permissions only for the current mm (sections are
308 * copied into each mm). During startup, this is the init_mm. Is only
309 * safe to be called with preemption disabled, as under stop_machine().
310 */
311static inline void section_update(unsigned long addr, pmdval_t mask,
312 pmdval_t prot, struct mm_struct *mm)
313{
314 pmd_t *pmd;
315
316 pmd = pmd_offset(pud_offset(p4d_offset(pgd_offset(mm, addr), addr), addr), addr);
317
318#ifdef CONFIG_ARM_LPAE
319 pmd[0] = __pmd((pmd_val(pmd[0]) & mask) | prot);
320#else
321 if (addr & SECTION_SIZE)
322 pmd[1] = __pmd((pmd_val(pmd[1]) & mask) | prot);
323 else
324 pmd[0] = __pmd((pmd_val(pmd[0]) & mask) | prot);
325#endif
326 flush_pmd_entry(pmd);
327 local_flush_tlb_kernel_range(addr, addr + SECTION_SIZE);
328}
329
330/* Make sure extended page tables are in use. */
331static inline bool arch_has_strict_perms(void)
332{
333 if (cpu_architecture() < CPU_ARCH_ARMv6)
334 return false;
335
336 return !!(get_cr() & CR_XP);
337}
338
339static void set_section_perms(struct section_perm *perms, int n, bool set,
340 struct mm_struct *mm)
341{
342 size_t i;
343 unsigned long addr;
344
345 if (!arch_has_strict_perms())
346 return;
347
348 for (i = 0; i < n; i++) {
349 if (!IS_ALIGNED(perms[i].start, SECTION_SIZE) ||
350 !IS_ALIGNED(perms[i].end, SECTION_SIZE)) {
351 pr_err("BUG: %s section %lx-%lx not aligned to %lx\n",
352 perms[i].name, perms[i].start, perms[i].end,
353 SECTION_SIZE);
354 continue;
355 }
356
357 for (addr = perms[i].start;
358 addr < perms[i].end;
359 addr += SECTION_SIZE)
360 section_update(addr, perms[i].mask,
361 set ? perms[i].prot : perms[i].clear, mm);
362 }
363
364}
365
366/*
367 * update_sections_early intended to be called only through stop_machine
368 * framework and executed by only one CPU while all other CPUs will spin and
369 * wait, so no locking is required in this function.
370 */
371static void update_sections_early(struct section_perm perms[], int n)
372{
373 struct task_struct *t, *s;
374
375 for_each_process(t) {
376 if (t->flags & PF_KTHREAD)
377 continue;
378 for_each_thread(t, s)
379 if (s->mm)
380 set_section_perms(perms, n, true, s->mm);
381 }
382 set_section_perms(perms, n, true, current->active_mm);
383 set_section_perms(perms, n, true, &init_mm);
384}
385
386static int __fix_kernmem_perms(void *unused)
387{
388 update_sections_early(nx_perms, ARRAY_SIZE(nx_perms));
389 return 0;
390}
391
392static void fix_kernmem_perms(void)
393{
394 stop_machine(__fix_kernmem_perms, NULL, NULL);
395}
396
397static int __mark_rodata_ro(void *unused)
398{
399 update_sections_early(ro_perms, ARRAY_SIZE(ro_perms));
400 return 0;
401}
402
403void mark_rodata_ro(void)
404{
405 stop_machine(__mark_rodata_ro, NULL, NULL);
406 arm_debug_checkwx();
407}
408
409#else
410static inline void fix_kernmem_perms(void) { }
411#endif /* CONFIG_STRICT_KERNEL_RWX */
412
413void free_initmem(void)
414{
415 fix_kernmem_perms();
416
417 poison_init_mem(__init_begin, __init_end - __init_begin);
418 if (!machine_is_integrator() && !machine_is_cintegrator())
419 free_initmem_default(-1);
420}
421
422#ifdef CONFIG_BLK_DEV_INITRD
423void free_initrd_mem(unsigned long start, unsigned long end)
424{
425 if (start == initrd_start)
426 start = round_down(start, PAGE_SIZE);
427 if (end == initrd_end)
428 end = round_up(end, PAGE_SIZE);
429
430 poison_init_mem((void *)start, PAGE_ALIGN(end) - start);
431 free_reserved_area((void *)start, (void *)end, -1, "initrd");
432}
433#endif
434
435#ifdef CONFIG_EXECMEM
436
437#ifdef CONFIG_XIP_KERNEL
438/*
439 * The XIP kernel text is mapped in the module area for modules and
440 * some other stuff to work without any indirect relocations.
441 * MODULES_VADDR is redefined here and not in asm/memory.h to avoid
442 * recompiling the whole kernel when CONFIG_XIP_KERNEL is turned on/off.
443 */
444#undef MODULES_VADDR
445#define MODULES_VADDR (((unsigned long)_exiprom + ~PMD_MASK) & PMD_MASK)
446#endif
447
448#ifdef CONFIG_MMU
449static struct execmem_info execmem_info __ro_after_init;
450
451struct execmem_info __init *execmem_arch_setup(void)
452{
453 unsigned long fallback_start = 0, fallback_end = 0;
454
455 if (IS_ENABLED(CONFIG_ARM_MODULE_PLTS)) {
456 fallback_start = VMALLOC_START;
457 fallback_end = VMALLOC_END;
458 }
459
460 execmem_info = (struct execmem_info){
461 .ranges = {
462 [EXECMEM_DEFAULT] = {
463 .start = MODULES_VADDR,
464 .end = MODULES_END,
465 .pgprot = PAGE_KERNEL_EXEC,
466 .alignment = 1,
467 .fallback_start = fallback_start,
468 .fallback_end = fallback_end,
469 },
470 },
471 };
472
473 return &execmem_info;
474}
475#endif /* CONFIG_MMU */
476
477#endif /* CONFIG_EXECMEM */