projects/linux·blob assoc-array-fixes · refs

354 lines · 9.4 KiB

1// SPDX-License-Identifier: GPL-2.0
2/*
3 * sparse memory mappings.
4 */
5#include <linux/mm.h>
6#include <linux/slab.h>
7#include <linux/mmzone.h>
8#include <linux/memblock.h>
9#include <linux/compiler.h>
10#include <linux/highmem.h>
11#include <linux/export.h>
12#include <linux/spinlock.h>
13#include <linux/vmalloc.h>
14#include <linux/swap.h>
15#include <linux/swapops.h>
16#include <linux/vmstat.h>
17#include "internal.h"
18#include "mm_init.h"
19#include "sparse.h"
20#include <asm/dma.h>
21
22/*
23 * Permanent SPARSEMEM data:
24 *
25 * 1) mem_section - memory sections, mem_map's for valid memory
26 */
27#ifdef CONFIG_SPARSEMEM_EXTREME
28struct mem_section **mem_section;
29#else
30struct mem_section mem_section[NR_SECTION_ROOTS][SECTIONS_PER_ROOT]
31 ____cacheline_internodealigned_in_smp;
32#endif
33EXPORT_SYMBOL(mem_section);
34
35#ifdef NODE_NOT_IN_PAGE_FLAGS
36/*
37 * If we did not store the node number in the page then we have to
38 * do a lookup in the section_to_node_table in order to find which
39 * node the page belongs to.
40 */
41#if MAX_NUMNODES <= 256
42static u8 section_to_node_table[NR_MEM_SECTIONS] __cacheline_aligned;
43#else
44static u16 section_to_node_table[NR_MEM_SECTIONS] __cacheline_aligned;
45#endif
46
47int memdesc_nid(const memdesc_flags_t *mdf)
48{
49 return section_to_node_table[memdesc_section(mdf)];
50}
51EXPORT_SYMBOL(memdesc_nid);
52
53static void set_section_nid(unsigned long section_nr, int nid)
54{
55 section_to_node_table[section_nr] = nid;
56}
57#else /* !NODE_NOT_IN_PAGE_FLAGS */
58static inline void set_section_nid(unsigned long section_nr, int nid)
59{
60}
61#endif
62
63#ifdef CONFIG_SPARSEMEM_EXTREME
64static noinline struct mem_section __ref *sparse_index_alloc(int nid)
65{
66 struct mem_section *section = NULL;
67 unsigned long array_size = SECTIONS_PER_ROOT *
68 sizeof(struct mem_section);
69
70 if (slab_is_available()) {
71 section = kzalloc_node(array_size, GFP_KERNEL, nid);
72 } else {
73 section = memblock_alloc_node(array_size, SMP_CACHE_BYTES,
74 nid);
75 if (!section)
76 panic("%s: Failed to allocate %lu bytes nid=%d\n",
77 __func__, array_size, nid);
78 }
79
80 return section;
81}
82
83int __meminit sparse_index_init(unsigned long section_nr, int nid)
84{
85 unsigned long root = SECTION_NR_TO_ROOT(section_nr);
86 struct mem_section *section;
87
88 /*
89 * An existing section is possible in the sub-section hotplug
90 * case. First hot-add instantiates, follow-on hot-add reuses
91 * the existing section.
92 *
93 * The mem_hotplug_lock resolves the apparent race below.
94 */
95 if (mem_section[root])
96 return 0;
97
98 section = sparse_index_alloc(nid);
99 if (!section)
100 return -ENOMEM;
101
102 mem_section[root] = section;
103
104 return 0;
105}
106#else /* !SPARSEMEM_EXTREME */
107int __meminit sparse_index_init(unsigned long section_nr, int nid)
108{
109 return 0;
110}
111#endif
112
113/*
114 * During early boot, before section_mem_map is used for an actual
115 * mem_map, we use section_mem_map to store the section's NUMA
116 * node. This keeps us from having to use another data structure. The
117 * node information is cleared just before we store the real mem_map.
118 */
119static inline unsigned long sparse_encode_early_nid(int nid)
120{
121 return ((unsigned long)nid << SECTION_NID_SHIFT);
122}
123
124static inline int sparse_early_nid(struct mem_section *section)
125{
126 return (section->section_mem_map >> SECTION_NID_SHIFT);
127}
128
129/* Validate the physical addressing limitations of the model */
130static void __init mminit_validate_memmodel_limits(unsigned long *start_pfn,
131 unsigned long *end_pfn)
132{
133 unsigned long max_sparsemem_pfn = (DIRECT_MAP_PHYSMEM_END + 1) >> PAGE_SHIFT;
134
135 /*
136 * Sanity checks - do not allow an architecture to pass
137 * in larger pfns than the maximum scope of sparsemem:
138 */
139 if (*start_pfn > max_sparsemem_pfn) {
140 mminit_dprintk(MMINIT_WARNING, "pfnvalidation",
141 "Start of range %lu -> %lu exceeds SPARSEMEM max %lu\n",
142 *start_pfn, *end_pfn, max_sparsemem_pfn);
143 WARN_ON_ONCE(1);
144 *start_pfn = max_sparsemem_pfn;
145 *end_pfn = max_sparsemem_pfn;
146 } else if (*end_pfn > max_sparsemem_pfn) {
147 mminit_dprintk(MMINIT_WARNING, "pfnvalidation",
148 "End of range %lu -> %lu exceeds SPARSEMEM max %lu\n",
149 *start_pfn, *end_pfn, max_sparsemem_pfn);
150 WARN_ON_ONCE(1);
151 *end_pfn = max_sparsemem_pfn;
152 }
153}
154
155/*
156 * There are a number of times that we loop over NR_MEM_SECTIONS,
157 * looking for section_present() on each. But, when we have very
158 * large physical address spaces, NR_MEM_SECTIONS can also be
159 * very large which makes the loops quite long.
160 *
161 * Keeping track of this gives us an easy way to break out of
162 * those loops early.
163 */
164unsigned long __highest_present_section_nr;
165
166static inline unsigned long first_present_section_nr(void)
167{
168 return next_present_section_nr(-1);
169}
170
171/* Record a memory area against a node. */
172static void __init memory_present(int nid, unsigned long start, unsigned long end)
173{
174 unsigned long pfn;
175
176 start &= PAGE_SECTION_MASK;
177 mminit_validate_memmodel_limits(&start, &end);
178 for (pfn = start; pfn < end; pfn += PAGES_PER_SECTION) {
179 unsigned long section_nr = pfn_to_section_nr(pfn);
180 struct mem_section *ms;
181
182 sparse_index_init(section_nr, nid);
183 set_section_nid(section_nr, nid);
184
185 ms = __nr_to_section(section_nr);
186 if (!ms->section_mem_map) {
187 ms->section_mem_map = sparse_encode_early_nid(nid) |
188 SECTION_IS_ONLINE;
189 __section_mark_present(ms, section_nr);
190 }
191 }
192}
193
194/*
195 * Mark all memblocks as present using memory_present().
196 * This is a convenience function that is useful to mark all of the systems
197 * memory as present during initialization.
198 */
199static void __init memblocks_present(void)
200{
201 unsigned long start, end;
202 int i, nid;
203
204#ifdef CONFIG_SPARSEMEM_EXTREME
205 unsigned long size, align;
206
207 size = sizeof(struct mem_section *) * NR_SECTION_ROOTS;
208 align = 1 << (INTERNODE_CACHE_SHIFT);
209 mem_section = memblock_alloc_or_panic(size, align);
210#endif
211
212 for_each_mem_pfn_range(i, MAX_NUMNODES, &start, &end, &nid)
213 memory_present(nid, start, end);
214}
215
216#ifdef CONFIG_SPARSEMEM_VMEMMAP
217unsigned long __init section_map_size(void)
218{
219 return ALIGN(sizeof(struct page) * PAGES_PER_SECTION, PMD_SIZE);
220}
221
222#else
223unsigned long __init section_map_size(void)
224{
225 return PAGE_ALIGN(sizeof(struct page) * PAGES_PER_SECTION);
226}
227
228struct page __init *__populate_section_memmap(unsigned long pfn,
229 unsigned long nr_pages, int nid, struct vmem_altmap *altmap,
230 struct dev_pagemap *pgmap)
231{
232 unsigned long size = section_map_size();
233
234 return memmap_alloc(size, size, __pa(MAX_DMA_ADDRESS), nid, false);
235}
236#endif /* !CONFIG_SPARSEMEM_VMEMMAP */
237
238void __weak __meminit vmemmap_populate_print_last(void)
239{
240}
241
242static void *sparse_usagebuf __initdata;
243static void *sparse_usagebuf_end __initdata;
244
245/*
246 * Helper function that is used for generic section initialization, and
247 * can also be used by any hooks added above.
248 */
249void __init sparse_init_early_section(int nid, struct page *map,
250 unsigned long pnum, unsigned long flags)
251{
252 BUG_ON(!sparse_usagebuf || sparse_usagebuf >= sparse_usagebuf_end);
253 sparse_init_one_section(__nr_to_section(pnum), pnum, map,
254 sparse_usagebuf, SECTION_IS_EARLY | flags);
255 sparse_usagebuf = (void *)sparse_usagebuf + mem_section_usage_size();
256}
257
258static int __init sparse_usage_init(int nid, unsigned long map_count)
259{
260 unsigned long size;
261
262 size = mem_section_usage_size() * map_count;
263 sparse_usagebuf = memblock_alloc_node(size, SMP_CACHE_BYTES, nid);
264 if (!sparse_usagebuf) {
265 sparse_usagebuf_end = NULL;
266 return -ENOMEM;
267 }
268
269 sparse_usagebuf_end = sparse_usagebuf + size;
270 return 0;
271}
272
273static void __init sparse_usage_fini(void)
274{
275 sparse_usagebuf = sparse_usagebuf_end = NULL;
276}
277
278/*
279 * Initialize sparse on a specific node. The node spans [pnum_begin, pnum_end)
280 * And number of present sections in this node is map_count.
281 */
282static void __init sparse_init_nid(int nid, unsigned long pnum_begin,
283 unsigned long pnum_end,
284 unsigned long map_count)
285{
286 unsigned long pnum;
287
288 if (sparse_usage_init(nid, map_count))
289 panic("Failed to allocate usemap for node %d\n", nid);
290
291 sparse_vmemmap_init_nid_early(nid);
292
293 for_each_present_section_nr(pnum_begin, pnum) {
294 struct mem_section *ms;
295 unsigned long pfn = section_nr_to_pfn(pnum);
296
297 if (pnum >= pnum_end)
298 break;
299
300 ms = __nr_to_section(pnum);
301 if (!preinited_vmemmap_section(ms)) {
302 struct page *map;
303
304 map = __populate_section_memmap(pfn, PAGES_PER_SECTION,
305 nid, NULL, NULL);
306 if (!map)
307 panic("Failed to allocate memmap for section %lu\n", pnum);
308 memmap_boot_pages_add(DIV_ROUND_UP(PAGES_PER_SECTION * sizeof(struct page),
309 PAGE_SIZE));
310 sparse_init_early_section(nid, map, pnum, 0);
311 }
312 }
313 sparse_usage_fini();
314}
315
316/*
317 * Allocate the accumulated non-linear sections, allocate a mem_map
318 * for each and record the physical to section mapping.
319 */
320void __init sparse_init(void)
321{
322 unsigned long pnum_end, pnum_begin, map_count = 1;
323 int nid_begin;
324
325 /* see include/linux/mmzone.h 'struct mem_section' definition */
326 BUILD_BUG_ON(!is_power_of_2(sizeof(struct mem_section)));
327 memblocks_present();
328
329 if (compound_info_has_mask()) {
330 VM_WARN_ON_ONCE(!IS_ALIGNED((unsigned long) pfn_to_page(0),
331 MAX_FOLIO_VMEMMAP_ALIGN));
332 }
333
334 pnum_begin = first_present_section_nr();
335 nid_begin = sparse_early_nid(__nr_to_section(pnum_begin));
336
337 for_each_present_section_nr(pnum_begin + 1, pnum_end) {
338 int nid = sparse_early_nid(__nr_to_section(pnum_end));
339
340 if (nid == nid_begin) {
341 map_count++;
342 continue;
343 }
344 /* Init node with sections in range [pnum_begin, pnum_end) */
345 sparse_init_nid(nid_begin, pnum_begin, pnum_end, map_count);
346 nid_begin = nid;
347 pnum_begin = pnum_end;
348 map_count = 1;
349 }
350 /* cover the last node */
351 sparse_init_nid(nid_begin, pnum_begin, pnum_end, map_count);
352 sparse_init_subsection_map();
353 vmemmap_populate_print_last();
354}