projects/linux·blob v6.16-rc7 · refs

846 lines · 21.4 KiB

1// SPDX-License-Identifier: GPL-2.0-only
2#include <linux/alloc_tag.h>
3#include <linux/execmem.h>
4#include <linux/fs.h>
5#include <linux/gfp.h>
6#include <linux/kallsyms.h>
7#include <linux/module.h>
8#include <linux/page_ext.h>
9#include <linux/proc_fs.h>
10#include <linux/seq_buf.h>
11#include <linux/seq_file.h>
12#include <linux/vmalloc.h>
13#include <linux/kmemleak.h>
14
15#define ALLOCINFO_FILE_NAME "allocinfo"
16#define MODULE_ALLOC_TAG_VMAP_SIZE (100000UL * sizeof(struct alloc_tag))
17#define SECTION_START(NAME) (CODETAG_SECTION_START_PREFIX NAME)
18#define SECTION_STOP(NAME) (CODETAG_SECTION_STOP_PREFIX NAME)
19
20#ifdef CONFIG_MEM_ALLOC_PROFILING_ENABLED_BY_DEFAULT
21static bool mem_profiling_support = true;
22#else
23static bool mem_profiling_support;
24#endif
25
26static struct codetag_type *alloc_tag_cttype;
27
28DEFINE_PER_CPU(struct alloc_tag_counters, _shared_alloc_tag);
29EXPORT_SYMBOL(_shared_alloc_tag);
30
31DEFINE_STATIC_KEY_MAYBE(CONFIG_MEM_ALLOC_PROFILING_ENABLED_BY_DEFAULT,
32 mem_alloc_profiling_key);
33EXPORT_SYMBOL(mem_alloc_profiling_key);
34
35DEFINE_STATIC_KEY_FALSE(mem_profiling_compressed);
36
37struct alloc_tag_kernel_section kernel_tags = { NULL, 0 };
38unsigned long alloc_tag_ref_mask;
39int alloc_tag_ref_offs;
40
41struct allocinfo_private {
42 struct codetag_iterator iter;
43 bool print_header;
44};
45
46static void *allocinfo_start(struct seq_file *m, loff_t *pos)
47{
48 struct allocinfo_private *priv;
49 struct codetag *ct;
50 loff_t node = *pos;
51
52 priv = kzalloc(sizeof(*priv), GFP_KERNEL);
53 m->private = priv;
54 if (!priv)
55 return NULL;
56
57 priv->print_header = (node == 0);
58 codetag_lock_module_list(alloc_tag_cttype, true);
59 priv->iter = codetag_get_ct_iter(alloc_tag_cttype);
60 while ((ct = codetag_next_ct(&priv->iter)) != NULL && node)
61 node--;
62
63 return ct ? priv : NULL;
64}
65
66static void *allocinfo_next(struct seq_file *m, void *arg, loff_t *pos)
67{
68 struct allocinfo_private *priv = (struct allocinfo_private *)arg;
69 struct codetag *ct = codetag_next_ct(&priv->iter);
70
71 (*pos)++;
72 if (!ct)
73 return NULL;
74
75 return priv;
76}
77
78static void allocinfo_stop(struct seq_file *m, void *arg)
79{
80 struct allocinfo_private *priv = (struct allocinfo_private *)m->private;
81
82 if (priv) {
83 codetag_lock_module_list(alloc_tag_cttype, false);
84 kfree(priv);
85 }
86}
87
88static void print_allocinfo_header(struct seq_buf *buf)
89{
90 /* Output format version, so we can change it. */
91 seq_buf_printf(buf, "allocinfo - version: 1.0\n");
92 seq_buf_printf(buf, "# <size> <calls> <tag info>\n");
93}
94
95static void alloc_tag_to_text(struct seq_buf *out, struct codetag *ct)
96{
97 struct alloc_tag *tag = ct_to_alloc_tag(ct);
98 struct alloc_tag_counters counter = alloc_tag_read(tag);
99 s64 bytes = counter.bytes;
100
101 seq_buf_printf(out, "%12lli %8llu ", bytes, counter.calls);
102 codetag_to_text(out, ct);
103 seq_buf_putc(out, ' ');
104 seq_buf_putc(out, '\n');
105}
106
107static int allocinfo_show(struct seq_file *m, void *arg)
108{
109 struct allocinfo_private *priv = (struct allocinfo_private *)arg;
110 char *bufp;
111 size_t n = seq_get_buf(m, &bufp);
112 struct seq_buf buf;
113
114 seq_buf_init(&buf, bufp, n);
115 if (priv->print_header) {
116 print_allocinfo_header(&buf);
117 priv->print_header = false;
118 }
119 alloc_tag_to_text(&buf, priv->iter.ct);
120 seq_commit(m, seq_buf_used(&buf));
121 return 0;
122}
123
124static const struct seq_operations allocinfo_seq_op = {
125 .start = allocinfo_start,
126 .next = allocinfo_next,
127 .stop = allocinfo_stop,
128 .show = allocinfo_show,
129};
130
131size_t alloc_tag_top_users(struct codetag_bytes *tags, size_t count, bool can_sleep)
132{
133 struct codetag_iterator iter;
134 struct codetag *ct;
135 struct codetag_bytes n;
136 unsigned int i, nr = 0;
137
138 if (IS_ERR_OR_NULL(alloc_tag_cttype))
139 return 0;
140
141 if (can_sleep)
142 codetag_lock_module_list(alloc_tag_cttype, true);
143 else if (!codetag_trylock_module_list(alloc_tag_cttype))
144 return 0;
145
146 iter = codetag_get_ct_iter(alloc_tag_cttype);
147 while ((ct = codetag_next_ct(&iter))) {
148 struct alloc_tag_counters counter = alloc_tag_read(ct_to_alloc_tag(ct));
149
150 n.ct = ct;
151 n.bytes = counter.bytes;
152
153 for (i = 0; i < nr; i++)
154 if (n.bytes > tags[i].bytes)
155 break;
156
157 if (i < count) {
158 nr -= nr == count;
159 memmove(&tags[i + 1],
160 &tags[i],
161 sizeof(tags[0]) * (nr - i));
162 nr++;
163 tags[i] = n;
164 }
165 }
166
167 codetag_lock_module_list(alloc_tag_cttype, false);
168
169 return nr;
170}
171
172void pgalloc_tag_split(struct folio *folio, int old_order, int new_order)
173{
174 int i;
175 struct alloc_tag *tag;
176 unsigned int nr_pages = 1 << new_order;
177
178 if (!mem_alloc_profiling_enabled())
179 return;
180
181 tag = __pgalloc_tag_get(&folio->page);
182 if (!tag)
183 return;
184
185 for (i = nr_pages; i < (1 << old_order); i += nr_pages) {
186 union pgtag_ref_handle handle;
187 union codetag_ref ref;
188
189 if (get_page_tag_ref(folio_page(folio, i), &ref, &handle)) {
190 /* Set new reference to point to the original tag */
191 alloc_tag_ref_set(&ref, tag);
192 update_page_tag_ref(handle, &ref);
193 put_page_tag_ref(handle);
194 }
195 }
196}
197
198void pgalloc_tag_swap(struct folio *new, struct folio *old)
199{
200 union pgtag_ref_handle handle_old, handle_new;
201 union codetag_ref ref_old, ref_new;
202 struct alloc_tag *tag_old, *tag_new;
203
204 if (!mem_alloc_profiling_enabled())
205 return;
206
207 tag_old = __pgalloc_tag_get(&old->page);
208 if (!tag_old)
209 return;
210 tag_new = __pgalloc_tag_get(&new->page);
211 if (!tag_new)
212 return;
213
214 if (!get_page_tag_ref(&old->page, &ref_old, &handle_old))
215 return;
216 if (!get_page_tag_ref(&new->page, &ref_new, &handle_new)) {
217 put_page_tag_ref(handle_old);
218 return;
219 }
220
221 /*
222 * Clear tag references to avoid debug warning when using
223 * __alloc_tag_ref_set() with non-empty reference.
224 */
225 set_codetag_empty(&ref_old);
226 set_codetag_empty(&ref_new);
227
228 /* swap tags */
229 __alloc_tag_ref_set(&ref_old, tag_new);
230 update_page_tag_ref(handle_old, &ref_old);
231 __alloc_tag_ref_set(&ref_new, tag_old);
232 update_page_tag_ref(handle_new, &ref_new);
233
234 put_page_tag_ref(handle_old);
235 put_page_tag_ref(handle_new);
236}
237
238static void shutdown_mem_profiling(bool remove_file)
239{
240 if (mem_alloc_profiling_enabled())
241 static_branch_disable(&mem_alloc_profiling_key);
242
243 if (!mem_profiling_support)
244 return;
245
246 if (remove_file)
247 remove_proc_entry(ALLOCINFO_FILE_NAME, NULL);
248 mem_profiling_support = false;
249}
250
251void __init alloc_tag_sec_init(void)
252{
253 struct alloc_tag *last_codetag;
254
255 if (!mem_profiling_support)
256 return;
257
258 if (!static_key_enabled(&mem_profiling_compressed))
259 return;
260
261 kernel_tags.first_tag = (struct alloc_tag *)kallsyms_lookup_name(
262 SECTION_START(ALLOC_TAG_SECTION_NAME));
263 last_codetag = (struct alloc_tag *)kallsyms_lookup_name(
264 SECTION_STOP(ALLOC_TAG_SECTION_NAME));
265 kernel_tags.count = last_codetag - kernel_tags.first_tag;
266
267 /* Check if kernel tags fit into page flags */
268 if (kernel_tags.count > (1UL << NR_UNUSED_PAGEFLAG_BITS)) {
269 shutdown_mem_profiling(false); /* allocinfo file does not exist yet */
270 pr_err("%lu allocation tags cannot be references using %d available page flag bits. Memory allocation profiling is disabled!\n",
271 kernel_tags.count, NR_UNUSED_PAGEFLAG_BITS);
272 return;
273 }
274
275 alloc_tag_ref_offs = (LRU_REFS_PGOFF - NR_UNUSED_PAGEFLAG_BITS);
276 alloc_tag_ref_mask = ((1UL << NR_UNUSED_PAGEFLAG_BITS) - 1);
277 pr_debug("Memory allocation profiling compression is using %d page flag bits!\n",
278 NR_UNUSED_PAGEFLAG_BITS);
279}
280
281#ifdef CONFIG_MODULES
282
283static struct maple_tree mod_area_mt = MTREE_INIT(mod_area_mt, MT_FLAGS_ALLOC_RANGE);
284static struct vm_struct *vm_module_tags;
285/* A dummy object used to indicate an unloaded module */
286static struct module unloaded_mod;
287/* A dummy object used to indicate a module prepended area */
288static struct module prepend_mod;
289
290struct alloc_tag_module_section module_tags;
291
292static inline unsigned long alloc_tag_align(unsigned long val)
293{
294 if (!static_key_enabled(&mem_profiling_compressed)) {
295 /* No alignment requirements when we are not indexing the tags */
296 return val;
297 }
298
299 if (val % sizeof(struct alloc_tag) == 0)
300 return val;
301 return ((val / sizeof(struct alloc_tag)) + 1) * sizeof(struct alloc_tag);
302}
303
304static bool ensure_alignment(unsigned long align, unsigned int *prepend)
305{
306 if (!static_key_enabled(&mem_profiling_compressed)) {
307 /* No alignment requirements when we are not indexing the tags */
308 return true;
309 }
310
311 /*
312 * If alloc_tag size is not a multiple of required alignment, tag
313 * indexing does not work.
314 */
315 if (!IS_ALIGNED(sizeof(struct alloc_tag), align))
316 return false;
317
318 /* Ensure prepend consumes multiple of alloc_tag-sized blocks */
319 if (*prepend)
320 *prepend = alloc_tag_align(*prepend);
321
322 return true;
323}
324
325static inline bool tags_addressable(void)
326{
327 unsigned long tag_idx_count;
328
329 if (!static_key_enabled(&mem_profiling_compressed))
330 return true; /* with page_ext tags are always addressable */
331
332 tag_idx_count = CODETAG_ID_FIRST + kernel_tags.count +
333 module_tags.size / sizeof(struct alloc_tag);
334
335 return tag_idx_count < (1UL << NR_UNUSED_PAGEFLAG_BITS);
336}
337
338static bool needs_section_mem(struct module *mod, unsigned long size)
339{
340 if (!mem_profiling_support)
341 return false;
342
343 return size >= sizeof(struct alloc_tag);
344}
345
346static bool clean_unused_counters(struct alloc_tag *start_tag,
347 struct alloc_tag *end_tag)
348{
349 struct alloc_tag *tag;
350 bool ret = true;
351
352 for (tag = start_tag; tag <= end_tag; tag++) {
353 struct alloc_tag_counters counter;
354
355 if (!tag->counters)
356 continue;
357
358 counter = alloc_tag_read(tag);
359 if (!counter.bytes) {
360 free_percpu(tag->counters);
361 tag->counters = NULL;
362 } else {
363 ret = false;
364 }
365 }
366
367 return ret;
368}
369
370/* Called with mod_area_mt locked */
371static void clean_unused_module_areas_locked(void)
372{
373 MA_STATE(mas, &mod_area_mt, 0, module_tags.size);
374 struct module *val;
375
376 mas_for_each(&mas, val, module_tags.size) {
377 struct alloc_tag *start_tag;
378 struct alloc_tag *end_tag;
379
380 if (val != &unloaded_mod)
381 continue;
382
383 /* Release area if all tags are unused */
384 start_tag = (struct alloc_tag *)(module_tags.start_addr + mas.index);
385 end_tag = (struct alloc_tag *)(module_tags.start_addr + mas.last);
386 if (clean_unused_counters(start_tag, end_tag))
387 mas_erase(&mas);
388 }
389}
390
391/* Called with mod_area_mt locked */
392static bool find_aligned_area(struct ma_state *mas, unsigned long section_size,
393 unsigned long size, unsigned int prepend, unsigned long align)
394{
395 bool cleanup_done = false;
396
397repeat:
398 /* Try finding exact size and hope the start is aligned */
399 if (!mas_empty_area(mas, 0, section_size - 1, prepend + size)) {
400 if (IS_ALIGNED(mas->index + prepend, align))
401 return true;
402
403 /* Try finding larger area to align later */
404 mas_reset(mas);
405 if (!mas_empty_area(mas, 0, section_size - 1,
406 size + prepend + align - 1))
407 return true;
408 }
409
410 /* No free area, try cleanup stale data and repeat the search once */
411 if (!cleanup_done) {
412 clean_unused_module_areas_locked();
413 cleanup_done = true;
414 mas_reset(mas);
415 goto repeat;
416 }
417
418 return false;
419}
420
421static int vm_module_tags_populate(void)
422{
423 unsigned long phys_end = ALIGN_DOWN(module_tags.start_addr, PAGE_SIZE) +
424 (vm_module_tags->nr_pages << PAGE_SHIFT);
425 unsigned long new_end = module_tags.start_addr + module_tags.size;
426
427 if (phys_end < new_end) {
428 struct page **next_page = vm_module_tags->pages + vm_module_tags->nr_pages;
429 unsigned long old_shadow_end = ALIGN(phys_end, MODULE_ALIGN);
430 unsigned long new_shadow_end = ALIGN(new_end, MODULE_ALIGN);
431 unsigned long more_pages;
432 unsigned long nr = 0;
433
434 more_pages = ALIGN(new_end - phys_end, PAGE_SIZE) >> PAGE_SHIFT;
435 while (nr < more_pages) {
436 unsigned long allocated;
437
438 allocated = alloc_pages_bulk_node(GFP_KERNEL | __GFP_NOWARN,
439 NUMA_NO_NODE, more_pages - nr, next_page + nr);
440
441 if (!allocated)
442 break;
443 nr += allocated;
444 }
445
446 if (nr < more_pages ||
447 vmap_pages_range(phys_end, phys_end + (nr << PAGE_SHIFT), PAGE_KERNEL,
448 next_page, PAGE_SHIFT) < 0) {
449 /* Clean up and error out */
450 for (int i = 0; i < nr; i++)
451 __free_page(next_page[i]);
452 return -ENOMEM;
453 }
454
455 vm_module_tags->nr_pages += nr;
456
457 /*
458 * Kasan allocates 1 byte of shadow for every 8 bytes of data.
459 * When kasan_alloc_module_shadow allocates shadow memory,
460 * its unit of allocation is a page.
461 * Therefore, here we need to align to MODULE_ALIGN.
462 */
463 if (old_shadow_end < new_shadow_end)
464 kasan_alloc_module_shadow((void *)old_shadow_end,
465 new_shadow_end - old_shadow_end,
466 GFP_KERNEL);
467 }
468
469 /*
470 * Mark the pages as accessible, now that they are mapped.
471 * With hardware tag-based KASAN, marking is skipped for
472 * non-VM_ALLOC mappings, see __kasan_unpoison_vmalloc().
473 */
474 kasan_unpoison_vmalloc((void *)module_tags.start_addr,
475 new_end - module_tags.start_addr,
476 KASAN_VMALLOC_PROT_NORMAL);
477
478 return 0;
479}
480
481static void *reserve_module_tags(struct module *mod, unsigned long size,
482 unsigned int prepend, unsigned long align)
483{
484 unsigned long section_size = module_tags.end_addr - module_tags.start_addr;
485 MA_STATE(mas, &mod_area_mt, 0, section_size - 1);
486 unsigned long offset;
487 void *ret = NULL;
488
489 /* If no tags return error */
490 if (size < sizeof(struct alloc_tag))
491 return ERR_PTR(-EINVAL);
492
493 /*
494 * align is always power of 2, so we can use IS_ALIGNED and ALIGN.
495 * align 0 or 1 means no alignment, to simplify set to 1.
496 */
497 if (!align)
498 align = 1;
499
500 if (!ensure_alignment(align, &prepend)) {
501 shutdown_mem_profiling(true);
502 pr_err("%s: alignment %lu is incompatible with allocation tag indexing. Memory allocation profiling is disabled!\n",
503 mod->name, align);
504 return ERR_PTR(-EINVAL);
505 }
506
507 mas_lock(&mas);
508 if (!find_aligned_area(&mas, section_size, size, prepend, align)) {
509 ret = ERR_PTR(-ENOMEM);
510 goto unlock;
511 }
512
513 /* Mark found area as reserved */
514 offset = mas.index;
515 offset += prepend;
516 offset = ALIGN(offset, align);
517 if (offset != mas.index) {
518 unsigned long pad_start = mas.index;
519
520 mas.last = offset - 1;
521 mas_store(&mas, &prepend_mod);
522 if (mas_is_err(&mas)) {
523 ret = ERR_PTR(xa_err(mas.node));
524 goto unlock;
525 }
526 mas.index = offset;
527 mas.last = offset + size - 1;
528 mas_store(&mas, mod);
529 if (mas_is_err(&mas)) {
530 mas.index = pad_start;
531 mas_erase(&mas);
532 ret = ERR_PTR(xa_err(mas.node));
533 }
534 } else {
535 mas.last = offset + size - 1;
536 mas_store(&mas, mod);
537 if (mas_is_err(&mas))
538 ret = ERR_PTR(xa_err(mas.node));
539 }
540unlock:
541 mas_unlock(&mas);
542
543 if (IS_ERR(ret))
544 return ret;
545
546 if (module_tags.size < offset + size) {
547 int grow_res;
548
549 module_tags.size = offset + size;
550 if (mem_alloc_profiling_enabled() && !tags_addressable()) {
551 shutdown_mem_profiling(true);
552 pr_warn("With module %s there are too many tags to fit in %d page flag bits. Memory allocation profiling is disabled!\n",
553 mod->name, NR_UNUSED_PAGEFLAG_BITS);
554 }
555
556 grow_res = vm_module_tags_populate();
557 if (grow_res) {
558 shutdown_mem_profiling(true);
559 pr_err("Failed to allocate memory for allocation tags in the module %s. Memory allocation profiling is disabled!\n",
560 mod->name);
561 return ERR_PTR(grow_res);
562 }
563 }
564
565 return (struct alloc_tag *)(module_tags.start_addr + offset);
566}
567
568static void release_module_tags(struct module *mod, bool used)
569{
570 MA_STATE(mas, &mod_area_mt, module_tags.size, module_tags.size);
571 struct alloc_tag *start_tag;
572 struct alloc_tag *end_tag;
573 struct module *val;
574
575 mas_lock(&mas);
576 mas_for_each_rev(&mas, val, 0)
577 if (val == mod)
578 break;
579
580 if (!val) /* module not found */
581 goto out;
582
583 if (!used)
584 goto release_area;
585
586 start_tag = (struct alloc_tag *)(module_tags.start_addr + mas.index);
587 end_tag = (struct alloc_tag *)(module_tags.start_addr + mas.last);
588 if (!clean_unused_counters(start_tag, end_tag)) {
589 struct alloc_tag *tag;
590
591 for (tag = start_tag; tag <= end_tag; tag++) {
592 struct alloc_tag_counters counter;
593
594 if (!tag->counters)
595 continue;
596
597 counter = alloc_tag_read(tag);
598 pr_info("%s:%u module %s func:%s has %llu allocated at module unload\n",
599 tag->ct.filename, tag->ct.lineno, tag->ct.modname,
600 tag->ct.function, counter.bytes);
601 }
602 } else {
603 used = false;
604 }
605release_area:
606 mas_store(&mas, used ? &unloaded_mod : NULL);
607 val = mas_prev_range(&mas, 0);
608 if (val == &prepend_mod)
609 mas_store(&mas, NULL);
610out:
611 mas_unlock(&mas);
612}
613
614static int load_module(struct module *mod, struct codetag *start, struct codetag *stop)
615{
616 /* Allocate module alloc_tag percpu counters */
617 struct alloc_tag *start_tag;
618 struct alloc_tag *stop_tag;
619 struct alloc_tag *tag;
620
621 /* percpu counters for core allocations are already statically allocated */
622 if (!mod)
623 return 0;
624
625 start_tag = ct_to_alloc_tag(start);
626 stop_tag = ct_to_alloc_tag(stop);
627 for (tag = start_tag; tag < stop_tag; tag++) {
628 WARN_ON(tag->counters);
629 tag->counters = alloc_percpu(struct alloc_tag_counters);
630 if (!tag->counters) {
631 while (--tag >= start_tag) {
632 free_percpu(tag->counters);
633 tag->counters = NULL;
634 }
635 pr_err("Failed to allocate memory for allocation tag percpu counters in the module %s\n",
636 mod->name);
637 return -ENOMEM;
638 }
639
640 /*
641 * Avoid a kmemleak false positive. The pointer to the counters is stored
642 * in the alloc_tag section of the module and cannot be directly accessed.
643 */
644 kmemleak_ignore_percpu(tag->counters);
645 }
646 return 0;
647}
648
649static void replace_module(struct module *mod, struct module *new_mod)
650{
651 MA_STATE(mas, &mod_area_mt, 0, module_tags.size);
652 struct module *val;
653
654 mas_lock(&mas);
655 mas_for_each(&mas, val, module_tags.size) {
656 if (val != mod)
657 continue;
658
659 mas_store_gfp(&mas, new_mod, GFP_KERNEL);
660 break;
661 }
662 mas_unlock(&mas);
663}
664
665static int __init alloc_mod_tags_mem(void)
666{
667 /* Map space to copy allocation tags */
668 vm_module_tags = execmem_vmap(MODULE_ALLOC_TAG_VMAP_SIZE);
669 if (!vm_module_tags) {
670 pr_err("Failed to map %lu bytes for module allocation tags\n",
671 MODULE_ALLOC_TAG_VMAP_SIZE);
672 module_tags.start_addr = 0;
673 return -ENOMEM;
674 }
675
676 vm_module_tags->pages = kmalloc_array(get_vm_area_size(vm_module_tags) >> PAGE_SHIFT,
677 sizeof(struct page *), GFP_KERNEL | __GFP_ZERO);
678 if (!vm_module_tags->pages) {
679 free_vm_area(vm_module_tags);
680 return -ENOMEM;
681 }
682
683 module_tags.start_addr = (unsigned long)vm_module_tags->addr;
684 module_tags.end_addr = module_tags.start_addr + MODULE_ALLOC_TAG_VMAP_SIZE;
685 /* Ensure the base is alloc_tag aligned when required for indexing */
686 module_tags.start_addr = alloc_tag_align(module_tags.start_addr);
687
688 return 0;
689}
690
691static void __init free_mod_tags_mem(void)
692{
693 int i;
694
695 module_tags.start_addr = 0;
696 for (i = 0; i < vm_module_tags->nr_pages; i++)
697 __free_page(vm_module_tags->pages[i]);
698 kfree(vm_module_tags->pages);
699 free_vm_area(vm_module_tags);
700}
701
702#else /* CONFIG_MODULES */
703
704static inline int alloc_mod_tags_mem(void) { return 0; }
705static inline void free_mod_tags_mem(void) {}
706
707#endif /* CONFIG_MODULES */
708
709/* See: Documentation/mm/allocation-profiling.rst */
710static int __init setup_early_mem_profiling(char *str)
711{
712 bool compressed = false;
713 bool enable;
714
715 if (!str || !str[0])
716 return -EINVAL;
717
718 if (!strncmp(str, "never", 5)) {
719 enable = false;
720 mem_profiling_support = false;
721 pr_info("Memory allocation profiling is disabled!\n");
722 } else {
723 char *token = strsep(&str, ",");
724
725 if (kstrtobool(token, &enable))
726 return -EINVAL;
727
728 if (str) {
729
730 if (strcmp(str, "compressed"))
731 return -EINVAL;
732
733 compressed = true;
734 }
735 mem_profiling_support = true;
736 pr_info("Memory allocation profiling is enabled %s compression and is turned %s!\n",
737 compressed ? "with" : "without", enable ? "on" : "off");
738 }
739
740 if (enable != mem_alloc_profiling_enabled()) {
741 if (enable)
742 static_branch_enable(&mem_alloc_profiling_key);
743 else
744 static_branch_disable(&mem_alloc_profiling_key);
745 }
746 if (compressed != static_key_enabled(&mem_profiling_compressed)) {
747 if (compressed)
748 static_branch_enable(&mem_profiling_compressed);
749 else
750 static_branch_disable(&mem_profiling_compressed);
751 }
752
753 return 0;
754}
755early_param("sysctl.vm.mem_profiling", setup_early_mem_profiling);
756
757static __init bool need_page_alloc_tagging(void)
758{
759 if (static_key_enabled(&mem_profiling_compressed))
760 return false;
761
762 return mem_profiling_support;
763}
764
765static __init void init_page_alloc_tagging(void)
766{
767}
768
769struct page_ext_operations page_alloc_tagging_ops = {
770 .size = sizeof(union codetag_ref),
771 .need = need_page_alloc_tagging,
772 .init = init_page_alloc_tagging,
773};
774EXPORT_SYMBOL(page_alloc_tagging_ops);
775
776#ifdef CONFIG_SYSCTL
777static struct ctl_table memory_allocation_profiling_sysctls[] = {
778 {
779 .procname = "mem_profiling",
780 .data = &mem_alloc_profiling_key,
781#ifdef CONFIG_MEM_ALLOC_PROFILING_DEBUG
782 .mode = 0444,
783#else
784 .mode = 0644,
785#endif
786 .proc_handler = proc_do_static_key,
787 },
788};
789
790static void __init sysctl_init(void)
791{
792 if (!mem_profiling_support)
793 memory_allocation_profiling_sysctls[0].mode = 0444;
794
795 register_sysctl_init("vm", memory_allocation_profiling_sysctls);
796}
797#else /* CONFIG_SYSCTL */
798static inline void sysctl_init(void) {}
799#endif /* CONFIG_SYSCTL */
800
801static int __init alloc_tag_init(void)
802{
803 const struct codetag_type_desc desc = {
804 .section = ALLOC_TAG_SECTION_NAME,
805 .tag_size = sizeof(struct alloc_tag),
806#ifdef CONFIG_MODULES
807 .needs_section_mem = needs_section_mem,
808 .alloc_section_mem = reserve_module_tags,
809 .free_section_mem = release_module_tags,
810 .module_load = load_module,
811 .module_replaced = replace_module,
812#endif
813 };
814 int res;
815
816 sysctl_init();
817
818 if (!mem_profiling_support) {
819 pr_info("Memory allocation profiling is not supported!\n");
820 return 0;
821 }
822
823 if (!proc_create_seq(ALLOCINFO_FILE_NAME, 0400, NULL, &allocinfo_seq_op)) {
824 pr_err("Failed to create %s file\n", ALLOCINFO_FILE_NAME);
825 shutdown_mem_profiling(false);
826 return -ENOMEM;
827 }
828
829 res = alloc_mod_tags_mem();
830 if (res) {
831 pr_err("Failed to reserve address space for module tags, errno = %d\n", res);
832 shutdown_mem_profiling(true);
833 return res;
834 }
835
836 alloc_tag_cttype = codetag_register_type(&desc);
837 if (IS_ERR(alloc_tag_cttype)) {
838 pr_err("Allocation tags registration failed, errno = %ld\n", PTR_ERR(alloc_tag_cttype));
839 free_mod_tags_mem();
840 shutdown_mem_profiling(true);
841 return PTR_ERR(alloc_tag_cttype);
842 }
843
844 return 0;
845}
846module_init(alloc_tag_init);