projects/linux·blob v7.2 · refs

1029 lines · 26.6 KiB

1// SPDX-License-Identifier: GPL-2.0-only
2#include <linux/alloc_tag.h>
3#include <linux/execmem.h>
4#include <linux/fs.h>
5#include <linux/gfp.h>
6#include <linux/kallsyms.h>
7#include <linux/module.h>
8#include <linux/page_ext.h>
9#include <linux/pgalloc_tag.h>
10#include <linux/proc_fs.h>
11#include <linux/rcupdate.h>
12#include <linux/seq_buf.h>
13#include <linux/seq_file.h>
14#include <linux/string_choices.h>
15#include <linux/vmalloc.h>
16#include <linux/kmemleak.h>
17
18#define ALLOCINFO_FILE_NAME "allocinfo"
19#define MODULE_ALLOC_TAG_VMAP_SIZE (100000UL * sizeof(struct alloc_tag))
20#define SECTION_START(NAME) (CODETAG_SECTION_START_PREFIX NAME)
21#define SECTION_STOP(NAME) (CODETAG_SECTION_STOP_PREFIX NAME)
22
23#ifdef CONFIG_MEM_ALLOC_PROFILING_ENABLED_BY_DEFAULT
24static bool mem_profiling_support = true;
25#else
26static bool mem_profiling_support;
27#endif
28
29/*
30 * Memory allocation profiling is permanently disabled and cannot be enabled.
31 * Must be called after setup_early_mem_profiling().
32 */
33bool mem_alloc_profiling_permanently_disabled(void)
34{
35 return !mem_profiling_support;
36}
37
38static struct codetag_type *alloc_tag_cttype;
39
40#ifdef CONFIG_ARCH_MODULE_NEEDS_WEAK_PER_CPU
41DEFINE_PER_CPU(struct alloc_tag_counters, _shared_alloc_tag);
42EXPORT_SYMBOL(_shared_alloc_tag);
43#endif
44
45DEFINE_STATIC_KEY_MAYBE(CONFIG_MEM_ALLOC_PROFILING_ENABLED_BY_DEFAULT,
46 mem_alloc_profiling_key);
47EXPORT_SYMBOL(mem_alloc_profiling_key);
48
49DEFINE_STATIC_KEY_FALSE(mem_profiling_compressed);
50
51struct alloc_tag_kernel_section kernel_tags = { NULL, 0 };
52unsigned long alloc_tag_ref_mask;
53int alloc_tag_ref_offs;
54
55struct allocinfo_private {
56 struct codetag_iterator iter;
57 struct codetag_iterator reported_iter;
58 bool print_header;
59};
60
61static void *allocinfo_start(struct seq_file *m, loff_t *pos)
62{
63 struct allocinfo_private *priv;
64 loff_t node = *pos;
65
66 priv = (struct allocinfo_private *)m->private;
67 codetag_lock_module_list(alloc_tag_cttype);
68 if (node == 0) {
69 priv->print_header = true;
70 priv->iter = codetag_get_ct_iter(alloc_tag_cttype);
71 } else {
72 priv->iter = priv->reported_iter;
73 }
74 codetag_next_ct(&priv->iter);
75 return priv->iter.ct ? priv : NULL;
76}
77
78static void *allocinfo_next(struct seq_file *m, void *arg, loff_t *pos)
79{
80 struct allocinfo_private *priv = (struct allocinfo_private *)arg;
81 struct codetag *ct;
82
83 priv->reported_iter = priv->iter;
84 ct = codetag_next_ct(&priv->iter);
85 (*pos)++;
86 if (!ct)
87 return NULL;
88
89 return priv;
90}
91
92static void allocinfo_stop(struct seq_file *m, void *arg)
93{
94 codetag_unlock_module_list(alloc_tag_cttype);
95}
96
97static void print_allocinfo_header(struct seq_buf *buf)
98{
99 /* Output format version, so we can change it. */
100 seq_buf_printf(buf, "allocinfo - version: 2.0\n");
101 seq_buf_printf(buf, "# <size> <calls> <tag info>\n");
102}
103
104static void alloc_tag_to_text(struct seq_buf *out, struct codetag *ct)
105{
106 struct alloc_tag *tag = ct_to_alloc_tag(ct);
107 struct alloc_tag_counters counter = alloc_tag_read(tag);
108 s64 bytes = counter.bytes;
109
110 seq_buf_printf(out, "%12lli %8llu ", bytes, counter.calls);
111 codetag_to_text(out, ct);
112 if (unlikely(alloc_tag_is_inaccurate(tag)))
113 seq_buf_printf(out, " accurate:no");
114 seq_buf_putc(out, ' ');
115 seq_buf_putc(out, '\n');
116}
117
118static int allocinfo_show(struct seq_file *m, void *arg)
119{
120 struct allocinfo_private *priv = (struct allocinfo_private *)arg;
121 char *bufp;
122 size_t n = seq_get_buf(m, &bufp);
123 struct seq_buf buf;
124
125 seq_buf_init(&buf, bufp, n);
126 if (priv->print_header) {
127 print_allocinfo_header(&buf);
128 priv->print_header = false;
129 }
130 alloc_tag_to_text(&buf, priv->iter.ct);
131 seq_commit(m, seq_buf_used(&buf));
132 return 0;
133}
134
135static const struct seq_operations allocinfo_seq_op = {
136 .start = allocinfo_start,
137 .next = allocinfo_next,
138 .stop = allocinfo_stop,
139 .show = allocinfo_show,
140};
141
142size_t alloc_tag_top_users(struct codetag_bytes *tags, size_t count, bool can_sleep)
143{
144 struct codetag_iterator iter;
145 struct codetag *ct;
146 struct codetag_bytes n;
147 unsigned int i, nr = 0;
148
149 if (IS_ERR_OR_NULL(alloc_tag_cttype))
150 return 0;
151
152 if (can_sleep)
153 codetag_lock_module_list(alloc_tag_cttype);
154 else if (!codetag_trylock_module_list(alloc_tag_cttype))
155 return 0;
156
157 iter = codetag_get_ct_iter(alloc_tag_cttype);
158 while ((ct = codetag_next_ct(&iter))) {
159 struct alloc_tag_counters counter = alloc_tag_read(ct_to_alloc_tag(ct));
160
161 n.ct = ct;
162 n.bytes = counter.bytes;
163
164 for (i = 0; i < nr; i++)
165 if (n.bytes > tags[i].bytes)
166 break;
167
168 if (i < count) {
169 nr -= nr == count;
170 memmove(&tags[i + 1],
171 &tags[i],
172 sizeof(tags[0]) * (nr - i));
173 nr++;
174 tags[i] = n;
175 }
176 }
177
178 codetag_unlock_module_list(alloc_tag_cttype);
179
180 return nr;
181}
182
183void pgalloc_tag_split(struct folio *folio, int old_order, int new_order)
184{
185 int i;
186 struct alloc_tag *tag;
187 unsigned int nr_pages = 1 << new_order;
188
189 if (!mem_alloc_profiling_enabled())
190 return;
191
192 tag = __pgalloc_tag_get(&folio->page);
193 if (!tag)
194 return;
195
196 for (i = nr_pages; i < (1 << old_order); i += nr_pages) {
197 union pgtag_ref_handle handle;
198 union codetag_ref ref;
199
200 if (get_page_tag_ref(folio_page(folio, i), &ref, &handle)) {
201 /* Set new reference to point to the original tag */
202 alloc_tag_ref_set(&ref, tag);
203 update_page_tag_ref(handle, &ref);
204 put_page_tag_ref(handle);
205 }
206 }
207}
208
209void pgalloc_tag_swap(struct folio *new, struct folio *old)
210{
211 union pgtag_ref_handle handle_old, handle_new;
212 union codetag_ref ref_old, ref_new;
213 struct alloc_tag *tag_old, *tag_new;
214
215 if (!mem_alloc_profiling_enabled())
216 return;
217
218 tag_old = __pgalloc_tag_get(&old->page);
219 if (!tag_old)
220 return;
221 tag_new = __pgalloc_tag_get(&new->page);
222 if (!tag_new)
223 return;
224
225 if (!get_page_tag_ref(&old->page, &ref_old, &handle_old))
226 return;
227 if (!get_page_tag_ref(&new->page, &ref_new, &handle_new)) {
228 put_page_tag_ref(handle_old);
229 return;
230 }
231
232 /*
233 * Clear tag references to avoid debug warning when using
234 * __alloc_tag_ref_set() with non-empty reference.
235 */
236 set_codetag_empty(&ref_old);
237 set_codetag_empty(&ref_new);
238
239 /* swap tags */
240 __alloc_tag_ref_set(&ref_old, tag_new);
241 update_page_tag_ref(handle_old, &ref_old);
242 __alloc_tag_ref_set(&ref_new, tag_old);
243 update_page_tag_ref(handle_new, &ref_new);
244
245 put_page_tag_ref(handle_old);
246 put_page_tag_ref(handle_new);
247}
248
249static void shutdown_mem_profiling(bool remove_file)
250{
251 if (mem_alloc_profiling_enabled())
252 static_branch_disable(&mem_alloc_profiling_key);
253
254 if (!mem_profiling_support)
255 return;
256
257 if (remove_file)
258 remove_proc_entry(ALLOCINFO_FILE_NAME, NULL);
259 mem_profiling_support = false;
260}
261
262void __init alloc_tag_sec_init(void)
263{
264 struct alloc_tag *last_codetag;
265
266 if (!mem_profiling_support)
267 return;
268
269 if (!static_key_enabled(&mem_profiling_compressed))
270 return;
271
272 kernel_tags.first_tag = (struct alloc_tag *)kallsyms_lookup_name(
273 SECTION_START(ALLOC_TAG_SECTION_NAME));
274 last_codetag = (struct alloc_tag *)kallsyms_lookup_name(
275 SECTION_STOP(ALLOC_TAG_SECTION_NAME));
276 kernel_tags.count = last_codetag - kernel_tags.first_tag;
277
278 /* Check if kernel tags fit into page flags */
279 if (kernel_tags.count > (1UL << NR_UNUSED_PAGEFLAG_BITS)) {
280 shutdown_mem_profiling(false); /* allocinfo file does not exist yet */
281 pr_err("%lu allocation tags cannot be references using %d available page flag bits. Memory allocation profiling is disabled!\n",
282 kernel_tags.count, NR_UNUSED_PAGEFLAG_BITS);
283 return;
284 }
285
286 alloc_tag_ref_offs = (LRU_REFS_PGOFF - NR_UNUSED_PAGEFLAG_BITS);
287 alloc_tag_ref_mask = ((1UL << NR_UNUSED_PAGEFLAG_BITS) - 1);
288 pr_debug("Memory allocation profiling compression is using %d page flag bits!\n",
289 NR_UNUSED_PAGEFLAG_BITS);
290}
291
292#ifdef CONFIG_MODULES
293
294static struct maple_tree mod_area_mt = MTREE_INIT(mod_area_mt, MT_FLAGS_ALLOC_RANGE);
295static struct vm_struct *vm_module_tags;
296/* A dummy object used to indicate an unloaded module */
297static struct module unloaded_mod;
298/* A dummy object used to indicate a module prepended area */
299static struct module prepend_mod;
300
301struct alloc_tag_module_section module_tags;
302
303static inline unsigned long alloc_tag_align(unsigned long val)
304{
305 if (!static_key_enabled(&mem_profiling_compressed)) {
306 /* No alignment requirements when we are not indexing the tags */
307 return val;
308 }
309
310 if (val % sizeof(struct alloc_tag) == 0)
311 return val;
312 return ((val / sizeof(struct alloc_tag)) + 1) * sizeof(struct alloc_tag);
313}
314
315static bool ensure_alignment(unsigned long align, unsigned int *prepend)
316{
317 if (!static_key_enabled(&mem_profiling_compressed)) {
318 /* No alignment requirements when we are not indexing the tags */
319 return true;
320 }
321
322 /*
323 * If alloc_tag size is not a multiple of required alignment, tag
324 * indexing does not work.
325 */
326 if (!IS_ALIGNED(sizeof(struct alloc_tag), align))
327 return false;
328
329 /* Ensure prepend consumes multiple of alloc_tag-sized blocks */
330 if (*prepend)
331 *prepend = alloc_tag_align(*prepend);
332
333 return true;
334}
335
336static inline bool tags_addressable(void)
337{
338 unsigned long tag_idx_count;
339
340 if (!static_key_enabled(&mem_profiling_compressed))
341 return true; /* with page_ext tags are always addressable */
342
343 tag_idx_count = CODETAG_ID_FIRST + kernel_tags.count +
344 module_tags.size / sizeof(struct alloc_tag);
345
346 return tag_idx_count < (1UL << NR_UNUSED_PAGEFLAG_BITS);
347}
348
349static bool needs_section_mem(struct module *mod, unsigned long size)
350{
351 if (!mem_profiling_support)
352 return false;
353
354 return size >= sizeof(struct alloc_tag);
355}
356
357static bool clean_unused_counters(struct alloc_tag *start_tag,
358 struct alloc_tag *end_tag)
359{
360 struct alloc_tag *tag;
361 bool ret = true;
362
363 for (tag = start_tag; tag <= end_tag; tag++) {
364 struct alloc_tag_counters counter;
365
366 if (!tag->counters)
367 continue;
368
369 counter = alloc_tag_read(tag);
370 if (!counter.bytes) {
371 free_percpu(tag->counters);
372 tag->counters = NULL;
373 } else {
374 ret = false;
375 }
376 }
377
378 return ret;
379}
380
381/* Called with mod_area_mt locked */
382static void clean_unused_module_areas_locked(void)
383{
384 MA_STATE(mas, &mod_area_mt, 0, module_tags.size);
385 struct module *val;
386
387 mas_for_each(&mas, val, module_tags.size) {
388 struct alloc_tag *start_tag;
389 struct alloc_tag *end_tag;
390
391 if (val != &unloaded_mod)
392 continue;
393
394 /* Release area if all tags are unused */
395 start_tag = (struct alloc_tag *)(module_tags.start_addr + mas.index);
396 end_tag = (struct alloc_tag *)(module_tags.start_addr + mas.last);
397 if (clean_unused_counters(start_tag, end_tag))
398 mas_erase(&mas);
399 }
400}
401
402/* Called with mod_area_mt locked */
403static bool find_aligned_area(struct ma_state *mas, unsigned long section_size,
404 unsigned long size, unsigned int prepend, unsigned long align)
405{
406 bool cleanup_done = false;
407
408repeat:
409 /* Try finding exact size and hope the start is aligned */
410 if (!mas_empty_area(mas, 0, section_size - 1, prepend + size)) {
411 if (IS_ALIGNED(mas->index + prepend, align))
412 return true;
413
414 /* Try finding larger area to align later */
415 mas_reset(mas);
416 if (!mas_empty_area(mas, 0, section_size - 1,
417 size + prepend + align - 1))
418 return true;
419 }
420
421 /* No free area, try cleanup stale data and repeat the search once */
422 if (!cleanup_done) {
423 clean_unused_module_areas_locked();
424 cleanup_done = true;
425 mas_reset(mas);
426 goto repeat;
427 }
428
429 return false;
430}
431
432static int vm_module_tags_populate(void)
433{
434 unsigned long phys_end = ALIGN_DOWN(module_tags.start_addr, PAGE_SIZE) +
435 (vm_module_tags->nr_pages << PAGE_SHIFT);
436 unsigned long new_end = module_tags.start_addr + module_tags.size;
437
438 if (phys_end < new_end) {
439 struct page **next_page = vm_module_tags->pages + vm_module_tags->nr_pages;
440 unsigned long old_shadow_end = ALIGN(phys_end, MODULE_ALIGN);
441 unsigned long new_shadow_end = ALIGN(new_end, MODULE_ALIGN);
442 unsigned long more_pages;
443 unsigned long nr = 0;
444
445 more_pages = ALIGN(new_end - phys_end, PAGE_SIZE) >> PAGE_SHIFT;
446 while (nr < more_pages) {
447 unsigned long allocated;
448
449 allocated = alloc_pages_bulk_node(GFP_KERNEL | __GFP_NOWARN,
450 NUMA_NO_NODE, more_pages - nr, next_page + nr);
451
452 if (!allocated)
453 break;
454 nr += allocated;
455 }
456
457 if (nr < more_pages ||
458 vmap_pages_range(phys_end, phys_end + (nr << PAGE_SHIFT), PAGE_KERNEL,
459 next_page, PAGE_SHIFT) < 0) {
460 release_pages_arg arg = { .pages = next_page };
461
462 /* Clean up and error out */
463 release_pages(arg, nr);
464 return -ENOMEM;
465 }
466
467 vm_module_tags->nr_pages += nr;
468
469 /*
470 * Kasan allocates 1 byte of shadow for every 8 bytes of data.
471 * When kasan_alloc_module_shadow allocates shadow memory,
472 * its unit of allocation is a page.
473 * Therefore, here we need to align to MODULE_ALIGN.
474 */
475 if (old_shadow_end < new_shadow_end)
476 kasan_alloc_module_shadow((void *)old_shadow_end,
477 new_shadow_end - old_shadow_end,
478 GFP_KERNEL);
479 }
480
481 /*
482 * Mark the pages as accessible, now that they are mapped.
483 * With hardware tag-based KASAN, marking is skipped for
484 * non-VM_ALLOC mappings, see __kasan_unpoison_vmalloc().
485 */
486 kasan_unpoison_vmalloc((void *)module_tags.start_addr,
487 new_end - module_tags.start_addr,
488 KASAN_VMALLOC_PROT_NORMAL);
489
490 return 0;
491}
492
493static void *reserve_module_tags(struct module *mod, unsigned long size,
494 unsigned int prepend, unsigned long align)
495{
496 unsigned long section_size = module_tags.end_addr - module_tags.start_addr;
497 MA_STATE(mas, &mod_area_mt, 0, section_size - 1);
498 unsigned long offset;
499 void *ret = NULL;
500
501 /* If no tags return error */
502 if (size < sizeof(struct alloc_tag))
503 return ERR_PTR(-EINVAL);
504
505 /*
506 * align is always power of 2, so we can use IS_ALIGNED and ALIGN.
507 * align 0 or 1 means no alignment, to simplify set to 1.
508 */
509 if (!align)
510 align = 1;
511
512 if (!ensure_alignment(align, &prepend)) {
513 shutdown_mem_profiling(true);
514 pr_err("%s: alignment %lu is incompatible with allocation tag indexing. Memory allocation profiling is disabled!\n",
515 mod->name, align);
516 return ERR_PTR(-EINVAL);
517 }
518
519 mas_lock(&mas);
520 if (!find_aligned_area(&mas, section_size, size, prepend, align)) {
521 ret = ERR_PTR(-ENOMEM);
522 goto unlock;
523 }
524
525 /* Mark found area as reserved */
526 offset = mas.index;
527 offset += prepend;
528 offset = ALIGN(offset, align);
529 if (offset != mas.index) {
530 unsigned long pad_start = mas.index;
531
532 mas.last = offset - 1;
533 mas_store(&mas, &prepend_mod);
534 if (mas_is_err(&mas)) {
535 ret = ERR_PTR(xa_err(mas.node));
536 goto unlock;
537 }
538 mas.index = offset;
539 mas.last = offset + size - 1;
540 mas_store(&mas, mod);
541 if (mas_is_err(&mas)) {
542 mas.index = pad_start;
543 mas_erase(&mas);
544 ret = ERR_PTR(xa_err(mas.node));
545 }
546 } else {
547 mas.last = offset + size - 1;
548 mas_store(&mas, mod);
549 if (mas_is_err(&mas))
550 ret = ERR_PTR(xa_err(mas.node));
551 }
552unlock:
553 mas_unlock(&mas);
554
555 if (IS_ERR(ret))
556 return ret;
557
558 if (module_tags.size < offset + size) {
559 int grow_res;
560
561 module_tags.size = offset + size;
562 if (mem_alloc_profiling_enabled() && !tags_addressable()) {
563 shutdown_mem_profiling(true);
564 pr_warn("With module %s there are too many tags to fit in %d page flag bits. Memory allocation profiling is disabled!\n",
565 mod->name, NR_UNUSED_PAGEFLAG_BITS);
566 }
567
568 grow_res = vm_module_tags_populate();
569 if (grow_res) {
570 shutdown_mem_profiling(true);
571 pr_err("Failed to allocate memory for allocation tags in the module %s. Memory allocation profiling is disabled!\n",
572 mod->name);
573 return ERR_PTR(grow_res);
574 }
575 }
576
577 return (struct alloc_tag *)(module_tags.start_addr + offset);
578}
579
580static void release_module_tags(struct module *mod, bool used)
581{
582 MA_STATE(mas, &mod_area_mt, module_tags.size, module_tags.size);
583 struct alloc_tag *start_tag;
584 struct alloc_tag *end_tag;
585 struct module *val;
586
587 mas_lock(&mas);
588 mas_for_each_rev(&mas, val, 0)
589 if (val == mod)
590 break;
591
592 if (!val) /* module not found */
593 goto out;
594
595 if (!used)
596 goto release_area;
597
598 start_tag = (struct alloc_tag *)(module_tags.start_addr + mas.index);
599 end_tag = (struct alloc_tag *)(module_tags.start_addr + mas.last);
600 if (!clean_unused_counters(start_tag, end_tag)) {
601 struct alloc_tag *tag;
602
603 for (tag = start_tag; tag <= end_tag; tag++) {
604 struct alloc_tag_counters counter;
605
606 if (!tag->counters)
607 continue;
608
609 counter = alloc_tag_read(tag);
610 pr_info("%s:%u module %s func:%s has %llu allocated at module unload\n",
611 tag->ct.filename, tag->ct.lineno, tag->ct.modname,
612 tag->ct.function, counter.bytes);
613 }
614 } else {
615 used = false;
616 }
617release_area:
618 mas_store(&mas, used ? &unloaded_mod : NULL);
619 val = mas_prev_range(&mas, 0);
620 if (val == &prepend_mod)
621 mas_store(&mas, NULL);
622out:
623 mas_unlock(&mas);
624}
625
626static int load_module(struct module *mod, struct codetag *start, struct codetag *stop)
627{
628 /* Allocate module alloc_tag percpu counters */
629 struct alloc_tag *start_tag;
630 struct alloc_tag *stop_tag;
631 struct alloc_tag *tag;
632
633 /* percpu counters for core allocations are already statically allocated */
634 if (!mod)
635 return 0;
636
637 start_tag = ct_to_alloc_tag(start);
638 stop_tag = ct_to_alloc_tag(stop);
639 for (tag = start_tag; tag < stop_tag; tag++) {
640 WARN_ON(tag->counters);
641 tag->counters = alloc_percpu(struct alloc_tag_counters);
642 if (!tag->counters) {
643 while (--tag >= start_tag) {
644 free_percpu(tag->counters);
645 tag->counters = NULL;
646 }
647 pr_err("Failed to allocate memory for allocation tag percpu counters in the module %s\n",
648 mod->name);
649 return -ENOMEM;
650 }
651
652 /*
653 * Avoid a kmemleak false positive. The pointer to the counters is stored
654 * in the alloc_tag section of the module and cannot be directly accessed.
655 */
656 kmemleak_ignore_percpu(tag->counters);
657 }
658 return 0;
659}
660
661static void replace_module(struct module *mod, struct module *new_mod)
662{
663 MA_STATE(mas, &mod_area_mt, 0, module_tags.size);
664 struct module *val;
665
666 mas_lock(&mas);
667 mas_for_each(&mas, val, module_tags.size) {
668 if (val != mod)
669 continue;
670
671 mas_store_gfp(&mas, new_mod, GFP_KERNEL);
672 break;
673 }
674 mas_unlock(&mas);
675}
676
677static int __init alloc_mod_tags_mem(void)
678{
679 /* Map space to copy allocation tags */
680 vm_module_tags = execmem_vmap(MODULE_ALLOC_TAG_VMAP_SIZE);
681 if (!vm_module_tags) {
682 pr_err("Failed to map %lu bytes for module allocation tags\n",
683 MODULE_ALLOC_TAG_VMAP_SIZE);
684 module_tags.start_addr = 0;
685 return -ENOMEM;
686 }
687
688 vm_module_tags->pages = kmalloc_objs(struct page *,
689 get_vm_area_size(vm_module_tags) >> PAGE_SHIFT,
690 GFP_KERNEL | __GFP_ZERO);
691 if (!vm_module_tags->pages) {
692 free_vm_area(vm_module_tags);
693 return -ENOMEM;
694 }
695
696 module_tags.start_addr = (unsigned long)vm_module_tags->addr;
697 module_tags.end_addr = module_tags.start_addr + MODULE_ALLOC_TAG_VMAP_SIZE;
698 /* Ensure the base is alloc_tag aligned when required for indexing */
699 module_tags.start_addr = alloc_tag_align(module_tags.start_addr);
700
701 return 0;
702}
703
704static void __init free_mod_tags_mem(void)
705{
706 release_pages_arg arg = { .pages = vm_module_tags->pages };
707
708 module_tags.start_addr = 0;
709 release_pages(arg, vm_module_tags->nr_pages);
710 kfree(vm_module_tags->pages);
711 free_vm_area(vm_module_tags);
712}
713
714#else /* CONFIG_MODULES */
715
716static inline int alloc_mod_tags_mem(void) { return 0; }
717static inline void free_mod_tags_mem(void) {}
718
719#endif /* CONFIG_MODULES */
720
721/* See: Documentation/mm/allocation-profiling.rst */
722static int __init setup_early_mem_profiling(char *str)
723{
724 bool compressed = false;
725 bool enable;
726
727 if (!str || !str[0])
728 return -EINVAL;
729
730 if (!strncmp(str, "never", 5)) {
731 enable = false;
732 mem_profiling_support = false;
733 pr_info("Memory allocation profiling is disabled!\n");
734 } else {
735 char *token = strsep(&str, ",");
736
737 if (kstrtobool(token, &enable))
738 return -EINVAL;
739
740 if (str) {
741
742 if (strcmp(str, "compressed"))
743 return -EINVAL;
744
745 compressed = true;
746 }
747 mem_profiling_support = true;
748 pr_info("Memory allocation profiling is enabled %s compression and is turned %s!\n",
749 compressed ? "with" : "without", str_on_off(enable));
750 }
751
752 if (enable != mem_alloc_profiling_enabled()) {
753 if (enable)
754 static_branch_enable(&mem_alloc_profiling_key);
755 else
756 static_branch_disable(&mem_alloc_profiling_key);
757 }
758 if (compressed != static_key_enabled(&mem_profiling_compressed)) {
759 if (compressed)
760 static_branch_enable(&mem_profiling_compressed);
761 else
762 static_branch_disable(&mem_profiling_compressed);
763 }
764
765 return 0;
766}
767early_param("sysctl.vm.mem_profiling", setup_early_mem_profiling);
768
769static __init bool need_page_alloc_tagging(void)
770{
771 if (static_key_enabled(&mem_profiling_compressed))
772 return false;
773
774 return mem_profiling_support;
775}
776
777#ifdef CONFIG_MEM_ALLOC_PROFILING_DEBUG
778/*
779 * Track page allocations before page_ext is initialized.
780 * Some pages are allocated before page_ext becomes available, leaving
781 * their codetag uninitialized. Track these early PFNs so we can clear
782 * their codetag refs later to avoid warnings when they are freed.
783 *
784 * Each page is cast to a pfn_pool: the first few bytes hold metadata
785 * (next pointer and slot count), the remainder stores PFNs.
786 */
787struct pfn_pool {
788 struct pfn_pool *next;
789 atomic_t count;
790 unsigned long pfns[];
791};
792
793#define PFN_POOL_SIZE ((PAGE_SIZE - offsetof(struct pfn_pool, pfns)) / \
794 sizeof(unsigned long))
795
796/*
797 * Skip early PFN recording for a page allocation. Reuses the
798 * %__GFP_NO_OBJ_EXT bit. Used by __alloc_tag_add_early_pfn() to avoid
799 * recursion when allocating pages for the early PFN tracking list
800 * itself.
801 *
802 * Codetags of the pages allocated with __GFP_NO_CODETAG should be
803 * cleared (via clear_page_tag_ref()) before freeing the pages to prevent
804 * alloc_tag_sub_check() from triggering a warning.
805 */
806#define __GFP_NO_CODETAG __GFP_NO_OBJ_EXT
807
808static struct pfn_pool *current_pfn_pool __initdata;
809
810static void __init __alloc_tag_add_early_pfn(unsigned long pfn)
811{
812 struct pfn_pool *pool;
813 int idx;
814
815 do {
816 pool = READ_ONCE(current_pfn_pool);
817 if (!pool || atomic_read(&pool->count) >= PFN_POOL_SIZE) {
818 struct page *new_page = alloc_page(__GFP_HIGH | __GFP_NO_CODETAG);
819 struct pfn_pool *new;
820
821 if (!new_page) {
822 pr_warn_once("early PFN tracking page allocation failed\n");
823 return;
824 }
825 new = page_address(new_page);
826 new->next = pool;
827 atomic_set(&new->count, 0);
828 if (cmpxchg(&current_pfn_pool, pool, new) != pool) {
829 clear_page_tag_ref(new_page);
830 __free_page(new_page);
831 continue;
832 }
833 pool = new;
834 }
835 idx = atomic_read(&pool->count);
836 if (idx >= PFN_POOL_SIZE)
837 continue;
838 if (atomic_cmpxchg(&pool->count, idx, idx + 1) == idx)
839 break;
840 } while (1);
841
842 pool->pfns[idx] = pfn;
843}
844
845typedef void alloc_tag_add_func(unsigned long pfn);
846static alloc_tag_add_func __rcu *alloc_tag_add_early_pfn_ptr __refdata =
847 RCU_INITIALIZER(__alloc_tag_add_early_pfn);
848
849void alloc_tag_add_early_pfn(unsigned long pfn, gfp_t gfp_flags)
850{
851 alloc_tag_add_func *alloc_tag_add;
852
853 if (static_key_enabled(&mem_profiling_compressed))
854 return;
855
856 /* Skip allocations for the tracking list itself to avoid recursion. */
857 if (gfp_flags & __GFP_NO_CODETAG)
858 return;
859
860 rcu_read_lock();
861 alloc_tag_add = rcu_dereference(alloc_tag_add_early_pfn_ptr);
862 if (alloc_tag_add)
863 alloc_tag_add(pfn);
864 rcu_read_unlock();
865}
866
867static void __init clear_early_alloc_pfn_tag_refs(void)
868{
869 struct pfn_pool *pool, *next;
870 struct page *page;
871 int i;
872
873 if (static_key_enabled(&mem_profiling_compressed))
874 return;
875
876 rcu_assign_pointer(alloc_tag_add_early_pfn_ptr, NULL);
877 /* Make sure we are not racing with __alloc_tag_add_early_pfn() */
878 synchronize_rcu();
879
880 for (pool = current_pfn_pool; pool; pool = next) {
881 int nr_pfns = atomic_read(&pool->count);
882
883 for (i = 0; i < nr_pfns; i++) {
884 unsigned long pfn = pool->pfns[i];
885
886 if (pfn_valid(pfn)) {
887 union pgtag_ref_handle handle;
888 union codetag_ref ref;
889
890 if (get_page_tag_ref(pfn_to_page(pfn), &ref, &handle)) {
891 /*
892 * An early-allocated page could be freed and reallocated
893 * after its page_ext is initialized but before we clear it.
894 * In that case, it already has a valid tag set.
895 * We should not overwrite that valid tag
896 * with CODETAG_EMPTY.
897 *
898 * Note: there is still a small race window between checking
899 * ref.ct and calling set_codetag_empty(). We accept this
900 * race as it's unlikely and the extra complexity of atomic
901 * cmpxchg is not worth it for this debug-only code path.
902 */
903 if (ref.ct) {
904 put_page_tag_ref(handle);
905 continue;
906 }
907
908 set_codetag_empty(&ref);
909 update_page_tag_ref(handle, &ref);
910 put_page_tag_ref(handle);
911 }
912 }
913 }
914
915 next = pool->next;
916 page = virt_to_page(pool);
917 clear_page_tag_ref(page);
918 __free_page(page);
919 }
920}
921#else /* !CONFIG_MEM_ALLOC_PROFILING_DEBUG */
922static inline void __init clear_early_alloc_pfn_tag_refs(void) {}
923#endif /* CONFIG_MEM_ALLOC_PROFILING_DEBUG */
924
925static __init void init_page_alloc_tagging(void)
926{
927 clear_early_alloc_pfn_tag_refs();
928}
929
930struct page_ext_operations page_alloc_tagging_ops = {
931 .size = sizeof(union codetag_ref),
932 .need = need_page_alloc_tagging,
933 .init = init_page_alloc_tagging,
934};
935EXPORT_SYMBOL(page_alloc_tagging_ops);
936
937#ifdef CONFIG_SYSCTL
938/*
939 * Not using proc_do_static_key() directly to prevent enabling profiling
940 * after it was shut down.
941 */
942static int proc_mem_profiling_handler(const struct ctl_table *table, int write,
943 void *buffer, size_t *lenp, loff_t *ppos)
944{
945 if (write) {
946 /*
947 * Call from do_sysctl_args() which is a no-op since the same
948 * value was already set by setup_early_mem_profiling.
949 * Return success to avoid warnings from do_sysctl_args().
950 */
951 if (!current->mm)
952 return 0;
953
954#ifdef CONFIG_MEM_ALLOC_PROFILING_DEBUG
955 /* User can't toggle profiling while debugging */
956 return -EACCES;
957#endif
958 if (!mem_profiling_support)
959 return -EINVAL;
960 }
961
962 return proc_do_static_key(table, write, buffer, lenp, ppos);
963}
964
965
966static const struct ctl_table memory_allocation_profiling_sysctls[] = {
967 {
968 .procname = "mem_profiling",
969 .data = &mem_alloc_profiling_key,
970 .mode = 0644,
971 .proc_handler = proc_mem_profiling_handler,
972 },
973};
974
975static void __init sysctl_init(void)
976{
977 register_sysctl_init("vm", memory_allocation_profiling_sysctls);
978}
979#else /* CONFIG_SYSCTL */
980static inline void sysctl_init(void) {}
981#endif /* CONFIG_SYSCTL */
982
983static int __init alloc_tag_init(void)
984{
985 const struct codetag_type_desc desc = {
986 .section = ALLOC_TAG_SECTION_NAME,
987 .tag_size = sizeof(struct alloc_tag),
988#ifdef CONFIG_MODULES
989 .needs_section_mem = needs_section_mem,
990 .alloc_section_mem = reserve_module_tags,
991 .free_section_mem = release_module_tags,
992 .module_load = load_module,
993 .module_replaced = replace_module,
994#endif
995 };
996 int res;
997
998 sysctl_init();
999
1000 if (!mem_profiling_support) {
1001 pr_info("Memory allocation profiling is not supported!\n");
1002 return 0;
1003 }
1004
1005 if (!proc_create_seq_private(ALLOCINFO_FILE_NAME, 0400, NULL, &allocinfo_seq_op,
1006 sizeof(struct allocinfo_private), NULL)) {
1007 pr_err("Failed to create %s file\n", ALLOCINFO_FILE_NAME);
1008 shutdown_mem_profiling(false);
1009 return -ENOMEM;
1010 }
1011
1012 res = alloc_mod_tags_mem();
1013 if (res) {
1014 pr_err("Failed to reserve address space for module tags, errno = %d\n", res);
1015 shutdown_mem_profiling(true);
1016 return res;
1017 }
1018
1019 alloc_tag_cttype = codetag_register_type(&desc);
1020 if (IS_ERR(alloc_tag_cttype)) {
1021 pr_err("Allocation tags registration failed, errno = %pe\n", alloc_tag_cttype);
1022 free_mod_tags_mem();
1023 shutdown_mem_profiling(true);
1024 return PTR_ERR(alloc_tag_cttype);
1025 }
1026
1027 return 0;
1028}
1029module_init(alloc_tag_init);