projects/linux·blob master · refs

571 lines · 15.0 KiB

1// SPDX-License-Identifier: GPL-2.0
2#include <linux/mm.h>
3#include <linux/mmzone.h>
4#include <linux/memblock.h>
5#include <linux/page_ext.h>
6#include <linux/memory.h>
7#include <linux/vmalloc.h>
8#include <linux/kmemleak.h>
9#include <linux/page_owner.h>
10#include <linux/page_idle.h>
11#include <linux/page_table_check.h>
12#include <linux/rcupdate.h>
13#include <linux/pgalloc_tag.h>
14#include <linux/iommu-debug-pagealloc.h>
15
16/*
17 * struct page extension
18 *
19 * This is the feature to manage memory for extended data per page.
20 *
21 * Until now, we must modify struct page itself to store extra data per page.
22 * This requires rebuilding the kernel and it is really time consuming process.
23 * And, sometimes, rebuild is impossible due to third party module dependency.
24 * At last, enlarging struct page could cause un-wanted system behaviour change.
25 *
26 * This feature is intended to overcome above mentioned problems. This feature
27 * allocates memory for extended data per page in certain place rather than
28 * the struct page itself. This memory can be accessed by the accessor
29 * functions provided by this code. During the boot process, it checks whether
30 * allocation of huge chunk of memory is needed or not. If not, it avoids
31 * allocating memory at all. With this advantage, we can include this feature
32 * into the kernel in default and can avoid rebuild and solve related problems.
33 *
34 * To help these things to work well, there are two callbacks for clients. One
35 * is the need callback which is mandatory if user wants to avoid useless
36 * memory allocation at boot-time. The other is optional, init callback, which
37 * is used to do proper initialization after memory is allocated.
38 *
39 * The need callback is used to decide whether extended memory allocation is
40 * needed or not. Sometimes users want to deactivate some features in this
41 * boot and extra memory would be unnecessary. In this case, to avoid
42 * allocating huge chunk of memory, each clients represent their need of
43 * extra memory through the need callback. If one of the need callbacks
44 * returns true, it means that someone needs extra memory so that
45 * page extension core should allocates memory for page extension. If
46 * none of need callbacks return true, memory isn't needed at all in this boot
47 * and page extension core can skip to allocate memory. As result,
48 * none of memory is wasted.
49 *
50 * When need callback returns true, page_ext checks if there is a request for
51 * extra memory through size in struct page_ext_operations. If it is non-zero,
52 * extra space is allocated for each page_ext entry and offset is returned to
53 * user through offset in struct page_ext_operations.
54 *
55 * The init callback is used to do proper initialization after page extension
56 * is completely initialized. In sparse memory system, extra memory is
57 * allocated some time later than memmap is allocated. In other words, lifetime
58 * of memory for page extension isn't same with memmap for struct page.
59 * Therefore, clients can't store extra data until page extension is
60 * initialized, even if pages are allocated and used freely. This could
61 * cause inadequate state of extra data per page, so, to prevent it, client
62 * can utilize this callback to initialize the state of it correctly.
63 */
64
65#ifdef CONFIG_SPARSEMEM
66#define PAGE_EXT_INVALID (0x1)
67#endif
68
69#if defined(CONFIG_PAGE_IDLE_FLAG) && !defined(CONFIG_64BIT)
70static bool need_page_idle(void)
71{
72 return true;
73}
74static struct page_ext_operations page_idle_ops __initdata = {
75 .need = need_page_idle,
76 .need_shared_flags = true,
77};
78#endif
79
80static struct page_ext_operations *page_ext_ops[] __initdata = {
81#ifdef CONFIG_PAGE_OWNER
82 &page_owner_ops,
83#endif
84#if defined(CONFIG_PAGE_IDLE_FLAG) && !defined(CONFIG_64BIT)
85 &page_idle_ops,
86#endif
87#ifdef CONFIG_MEM_ALLOC_PROFILING
88 &page_alloc_tagging_ops,
89#endif
90#ifdef CONFIG_PAGE_TABLE_CHECK
91 &page_table_check_ops,
92#endif
93#ifdef CONFIG_IOMMU_DEBUG_PAGEALLOC
94 &page_iommu_debug_ops,
95#endif
96};
97
98unsigned long page_ext_size;
99
100static unsigned long total_usage;
101
102#ifdef CONFIG_MEM_ALLOC_PROFILING_DEBUG
103/*
104 * To ensure correct allocation tagging for pages, page_ext should be available
105 * before the first page allocation. Otherwise early task stacks will be
106 * allocated before page_ext initialization and missing tags will be flagged.
107 */
108bool early_page_ext __meminitdata = true;
109#else
110bool early_page_ext __meminitdata;
111#endif
112static int __init setup_early_page_ext(char *str)
113{
114 early_page_ext = true;
115 return 0;
116}
117early_param("early_page_ext", setup_early_page_ext);
118
119static bool __init invoke_need_callbacks(void)
120{
121 int i;
122 int entries = ARRAY_SIZE(page_ext_ops);
123 bool need = false;
124
125 for (i = 0; i < entries; i++) {
126 if (page_ext_ops[i]->need()) {
127 if (page_ext_ops[i]->need_shared_flags) {
128 page_ext_size = sizeof(struct page_ext);
129 break;
130 }
131 }
132 }
133
134 for (i = 0; i < entries; i++) {
135 if (page_ext_ops[i]->need()) {
136 page_ext_ops[i]->offset = page_ext_size;
137 page_ext_size += page_ext_ops[i]->size;
138 need = true;
139 }
140 }
141
142 return need;
143}
144
145static void __init invoke_init_callbacks(void)
146{
147 int i;
148 int entries = ARRAY_SIZE(page_ext_ops);
149
150 for (i = 0; i < entries; i++) {
151 if (page_ext_ops[i]->init)
152 page_ext_ops[i]->init();
153 }
154}
155
156static inline struct page_ext *get_entry(void *base, unsigned long index)
157{
158 return base + page_ext_size * index;
159}
160
161#ifndef CONFIG_SPARSEMEM
162void __init page_ext_init_flatmem_late(void)
163{
164 invoke_init_callbacks();
165}
166
167static struct page_ext *lookup_page_ext(const struct page *page)
168{
169 unsigned long pfn = page_to_pfn(page);
170 unsigned long index;
171 struct page_ext *base;
172
173 WARN_ON_ONCE(!rcu_read_lock_held());
174 base = NODE_DATA(page_to_nid(page))->node_page_ext;
175 /*
176 * The sanity checks the page allocator does upon freeing a
177 * page can reach here before the page_ext arrays are
178 * allocated when feeding a range of pages to the allocator
179 * for the first time during bootup or memory hotplug.
180 */
181 if (unlikely(!base))
182 return NULL;
183 index = pfn - round_down(node_start_pfn(page_to_nid(page)),
184 MAX_ORDER_NR_PAGES);
185 return get_entry(base, index);
186}
187
188static int __init alloc_node_page_ext(int nid)
189{
190 struct page_ext *base;
191 unsigned long table_size;
192 unsigned long nr_pages;
193
194 nr_pages = NODE_DATA(nid)->node_spanned_pages;
195 if (!nr_pages)
196 return 0;
197
198 /*
199 * Need extra space if node range is not aligned with
200 * MAX_ORDER_NR_PAGES. When page allocator's buddy algorithm
201 * checks buddy's status, range could be out of exact node range.
202 */
203 if (!IS_ALIGNED(node_start_pfn(nid), MAX_ORDER_NR_PAGES) ||
204 !IS_ALIGNED(node_end_pfn(nid), MAX_ORDER_NR_PAGES))
205 nr_pages += MAX_ORDER_NR_PAGES;
206
207 table_size = page_ext_size * nr_pages;
208
209 base = memblock_alloc_try_nid(
210 table_size, PAGE_SIZE, __pa(MAX_DMA_ADDRESS),
211 MEMBLOCK_ALLOC_ACCESSIBLE, nid);
212 if (!base)
213 return -ENOMEM;
214 NODE_DATA(nid)->node_page_ext = base;
215 total_usage += table_size;
216 memmap_boot_pages_add(DIV_ROUND_UP(table_size, PAGE_SIZE));
217 return 0;
218}
219
220void __init page_ext_init_flatmem(void)
221{
222
223 int nid, fail;
224
225 if (!invoke_need_callbacks())
226 return;
227
228 for_each_online_node(nid) {
229 fail = alloc_node_page_ext(nid);
230 if (fail)
231 goto fail;
232 }
233 pr_info("allocated %ld bytes of page_ext\n", total_usage);
234 return;
235
236fail:
237 pr_crit("allocation of page_ext failed.\n");
238 panic("Out of memory");
239}
240
241#else /* CONFIG_SPARSEMEM */
242static bool page_ext_invalid(struct page_ext *page_ext)
243{
244 return !page_ext || (((unsigned long)page_ext & PAGE_EXT_INVALID) == PAGE_EXT_INVALID);
245}
246
247static struct page_ext *lookup_page_ext(const struct page *page)
248{
249 unsigned long pfn = page_to_pfn(page);
250 struct mem_section *section = __pfn_to_section(pfn);
251 struct page_ext *page_ext = READ_ONCE(section->page_ext);
252
253 WARN_ON_ONCE(!rcu_read_lock_held());
254 /*
255 * The sanity checks the page allocator does upon freeing a
256 * page can reach here before the page_ext arrays are
257 * allocated when feeding a range of pages to the allocator
258 * for the first time during bootup or memory hotplug.
259 */
260 if (page_ext_invalid(page_ext))
261 return NULL;
262 return get_entry(page_ext, pfn);
263}
264
265static void *__meminit alloc_page_ext(size_t size, int nid)
266{
267 gfp_t flags = GFP_KERNEL | __GFP_ZERO | __GFP_NOWARN;
268 void *addr = NULL;
269
270 addr = alloc_pages_exact_nid(nid, size, flags);
271 if (addr)
272 kmemleak_alloc(addr, size, 1, flags);
273 else
274 addr = vzalloc_node(size, nid);
275
276 if (addr)
277 memmap_pages_add(DIV_ROUND_UP(size, PAGE_SIZE));
278
279 return addr;
280}
281
282static int __meminit init_section_page_ext(unsigned long pfn, int nid)
283{
284 struct mem_section *section;
285 struct page_ext *base;
286 unsigned long table_size;
287
288 section = __pfn_to_section(pfn);
289
290 if (section->page_ext)
291 return 0;
292
293 table_size = page_ext_size * PAGES_PER_SECTION;
294 base = alloc_page_ext(table_size, nid);
295
296 /*
297 * The value stored in section->page_ext is (base - pfn)
298 * and it does not point to the memory block allocated above,
299 * causing kmemleak false positives.
300 */
301 kmemleak_not_leak(base);
302
303 if (!base) {
304 pr_err("page ext allocation failure\n");
305 return -ENOMEM;
306 }
307
308 /*
309 * The passed "pfn" may not be aligned to SECTION. For the calculation
310 * we need to apply a mask.
311 */
312 pfn &= PAGE_SECTION_MASK;
313 section->page_ext = (void *)base - page_ext_size * pfn;
314 total_usage += table_size;
315 return 0;
316}
317
318static void free_page_ext(void *addr)
319{
320 size_t table_size;
321 struct page *page;
322
323 table_size = page_ext_size * PAGES_PER_SECTION;
324 memmap_pages_add(-1L * (DIV_ROUND_UP(table_size, PAGE_SIZE)));
325
326 if (is_vmalloc_addr(addr)) {
327 vfree(addr);
328 } else {
329 page = virt_to_page(addr);
330 BUG_ON(PageReserved(page));
331 kmemleak_free(addr);
332 free_pages_exact(addr, table_size);
333 }
334}
335
336static void __free_page_ext(unsigned long pfn)
337{
338 struct mem_section *ms;
339 struct page_ext *base;
340
341 ms = __pfn_to_section(pfn);
342 if (!ms || !ms->page_ext)
343 return;
344
345 base = READ_ONCE(ms->page_ext);
346 /*
347 * page_ext here can be valid while doing the roll back
348 * operation in online_page_ext().
349 */
350 if (page_ext_invalid(base))
351 base = (void *)base - PAGE_EXT_INVALID;
352 WRITE_ONCE(ms->page_ext, NULL);
353
354 base = get_entry(base, pfn);
355 free_page_ext(base);
356}
357
358static void __invalidate_page_ext(unsigned long pfn)
359{
360 struct mem_section *ms;
361 void *val;
362
363 ms = __pfn_to_section(pfn);
364 if (!ms || !ms->page_ext)
365 return;
366 val = (void *)ms->page_ext + PAGE_EXT_INVALID;
367 WRITE_ONCE(ms->page_ext, val);
368}
369
370static int __meminit online_page_ext(unsigned long start_pfn,
371 unsigned long nr_pages)
372{
373 int nid = pfn_to_nid(start_pfn);
374 unsigned long start, end, pfn;
375 int fail = 0;
376
377 start = SECTION_ALIGN_DOWN(start_pfn);
378 end = SECTION_ALIGN_UP(start_pfn + nr_pages);
379
380 for (pfn = start; !fail && pfn < end; pfn += PAGES_PER_SECTION)
381 fail = init_section_page_ext(pfn, nid);
382 if (!fail)
383 return 0;
384
385 /* rollback */
386 end = pfn - PAGES_PER_SECTION;
387 for (pfn = start; pfn < end; pfn += PAGES_PER_SECTION)
388 __free_page_ext(pfn);
389
390 return -ENOMEM;
391}
392
393static void __meminit offline_page_ext(unsigned long start_pfn,
394 unsigned long nr_pages)
395{
396 unsigned long start, end, pfn;
397
398 start = SECTION_ALIGN_DOWN(start_pfn);
399 end = SECTION_ALIGN_UP(start_pfn + nr_pages);
400
401 /*
402 * Freeing of page_ext is done in 3 steps to avoid
403 * use-after-free of it:
404 * 1) Traverse all the sections and mark their page_ext
405 * as invalid.
406 * 2) Wait for all the existing users of page_ext who
407 * started before invalidation to finish.
408 * 3) Free the page_ext.
409 */
410 for (pfn = start; pfn < end; pfn += PAGES_PER_SECTION)
411 __invalidate_page_ext(pfn);
412
413 synchronize_rcu();
414
415 for (pfn = start; pfn < end; pfn += PAGES_PER_SECTION)
416 __free_page_ext(pfn);
417}
418
419static int __meminit page_ext_callback(struct notifier_block *self,
420 unsigned long action, void *arg)
421{
422 struct memory_notify *mn = arg;
423 int ret = 0;
424
425 switch (action) {
426 case MEM_GOING_ONLINE:
427 ret = online_page_ext(mn->start_pfn, mn->nr_pages);
428 break;
429 case MEM_OFFLINE:
430 offline_page_ext(mn->start_pfn,
431 mn->nr_pages);
432 break;
433 case MEM_CANCEL_ONLINE:
434 offline_page_ext(mn->start_pfn,
435 mn->nr_pages);
436 break;
437 case MEM_GOING_OFFLINE:
438 break;
439 case MEM_ONLINE:
440 case MEM_CANCEL_OFFLINE:
441 break;
442 }
443
444 return notifier_from_errno(ret);
445}
446
447void __init page_ext_init(void)
448{
449 unsigned long pfn;
450 int nid;
451
452 if (!invoke_need_callbacks())
453 return;
454
455 for_each_node_state(nid, N_MEMORY) {
456 unsigned long start_pfn, end_pfn;
457
458 start_pfn = node_start_pfn(nid);
459 end_pfn = node_end_pfn(nid);
460 /*
461 * start_pfn and end_pfn may not be aligned to SECTION and the
462 * page->flags of out of node pages are not initialized. So we
463 * scan [start_pfn, the biggest section's pfn < end_pfn) here.
464 */
465 for (pfn = start_pfn; pfn < end_pfn;
466 pfn = ALIGN(pfn + 1, PAGES_PER_SECTION)) {
467
468 if (!pfn_valid(pfn))
469 continue;
470 /*
471 * Nodes's pfns can be overlapping.
472 * We know some arch can have a nodes layout such as
473 * -------------pfn-------------->
474 * N0 | N1 | N2 | N0 | N1 | N2|....
475 */
476 if (pfn_to_nid(pfn) != nid)
477 continue;
478 if (init_section_page_ext(pfn, nid))
479 goto oom;
480 cond_resched();
481 }
482 }
483 hotplug_memory_notifier(page_ext_callback, DEFAULT_CALLBACK_PRI);
484 pr_info("allocated %ld bytes of page_ext\n", total_usage);
485 invoke_init_callbacks();
486 return;
487
488oom:
489 panic("Out of memory");
490}
491
492#endif
493
494/**
495 * page_ext_lookup() - Lookup a page extension for a PFN.
496 * @pfn: PFN of the page we're interested in.
497 *
498 * Must be called with RCU read lock taken and @pfn must be valid.
499 *
500 * Return: NULL if no page_ext exists for this page.
501 */
502struct page_ext *page_ext_lookup(unsigned long pfn)
503{
504 return lookup_page_ext(pfn_to_page(pfn));
505}
506
507/**
508 * page_ext_get() - Get the extended information for a page.
509 * @page: The page we're interested in.
510 *
511 * Ensures that the page_ext will remain valid until page_ext_put()
512 * is called.
513 *
514 * Return: NULL if no page_ext exists for this page.
515 * Context: Any context. Caller may not sleep until they have called
516 * page_ext_put().
517 */
518struct page_ext *page_ext_get(const struct page *page)
519{
520 struct page_ext *page_ext;
521
522 rcu_read_lock();
523 page_ext = lookup_page_ext(page);
524 if (!page_ext) {
525 rcu_read_unlock();
526 return NULL;
527 }
528
529 return page_ext;
530}
531
532/**
533 * page_ext_from_phys() - Get the page_ext structure for a physical address.
534 * @phys: The physical address to query.
535 *
536 * This function safely gets the `struct page_ext` associated with a given
537 * physical address. It performs validation to ensure the address corresponds
538 * to a valid, online struct page before attempting to access it.
539 * It returns NULL for MMIO, ZONE_DEVICE, holes and offline memory.
540 *
541 * Return: NULL if no page_ext exists for this physical address.
542 * Context: Any context. Caller may not sleep until they have called
543 * page_ext_put().
544 */
545struct page_ext *page_ext_from_phys(phys_addr_t phys)
546{
547 struct page *page = pfn_to_online_page(__phys_to_pfn(phys));
548
549 if (!page)
550 return NULL;
551
552 return page_ext_get(page);
553}
554
555/**
556 * page_ext_put() - Working with page extended information is done.
557 * @page_ext: Page extended information received from page_ext_get().
558 *
559 * The page extended information of the page may not be valid after this
560 * function is called.
561 *
562 * Return: None.
563 * Context: Any context with corresponding page_ext_get() is called.
564 */
565void page_ext_put(struct page_ext *page_ext)
566{
567 if (unlikely(!page_ext))
568 return;
569
570 rcu_read_unlock();
571}