projects/linux·blob v7.0 · refs

864 lines · 22.1 KiB

1// SPDX-License-Identifier: GPL-2.0-only
2#include <linux/alloc_tag.h>
3#include <linux/execmem.h>
4#include <linux/fs.h>
5#include <linux/gfp.h>
6#include <linux/kallsyms.h>
7#include <linux/module.h>
8#include <linux/page_ext.h>
9#include <linux/proc_fs.h>
10#include <linux/seq_buf.h>
11#include <linux/seq_file.h>
12#include <linux/string_choices.h>
13#include <linux/vmalloc.h>
14#include <linux/kmemleak.h>
15
16#define ALLOCINFO_FILE_NAME "allocinfo"
17#define MODULE_ALLOC_TAG_VMAP_SIZE (100000UL * sizeof(struct alloc_tag))
18#define SECTION_START(NAME) (CODETAG_SECTION_START_PREFIX NAME)
19#define SECTION_STOP(NAME) (CODETAG_SECTION_STOP_PREFIX NAME)
20
21#ifdef CONFIG_MEM_ALLOC_PROFILING_ENABLED_BY_DEFAULT
22static bool mem_profiling_support = true;
23#else
24static bool mem_profiling_support;
25#endif
26
27static struct codetag_type *alloc_tag_cttype;
28
29#ifdef CONFIG_ARCH_MODULE_NEEDS_WEAK_PER_CPU
30DEFINE_PER_CPU(struct alloc_tag_counters, _shared_alloc_tag);
31EXPORT_SYMBOL(_shared_alloc_tag);
32#endif
33
34DEFINE_STATIC_KEY_MAYBE(CONFIG_MEM_ALLOC_PROFILING_ENABLED_BY_DEFAULT,
35 mem_alloc_profiling_key);
36EXPORT_SYMBOL(mem_alloc_profiling_key);
37
38DEFINE_STATIC_KEY_FALSE(mem_profiling_compressed);
39
40struct alloc_tag_kernel_section kernel_tags = { NULL, 0 };
41unsigned long alloc_tag_ref_mask;
42int alloc_tag_ref_offs;
43
44struct allocinfo_private {
45 struct codetag_iterator iter;
46 bool print_header;
47};
48
49static void *allocinfo_start(struct seq_file *m, loff_t *pos)
50{
51 struct allocinfo_private *priv;
52 loff_t node = *pos;
53
54 priv = (struct allocinfo_private *)m->private;
55 codetag_lock_module_list(alloc_tag_cttype, true);
56 if (node == 0) {
57 priv->print_header = true;
58 priv->iter = codetag_get_ct_iter(alloc_tag_cttype);
59 codetag_next_ct(&priv->iter);
60 }
61 return priv->iter.ct ? priv : NULL;
62}
63
64static void *allocinfo_next(struct seq_file *m, void *arg, loff_t *pos)
65{
66 struct allocinfo_private *priv = (struct allocinfo_private *)arg;
67 struct codetag *ct = codetag_next_ct(&priv->iter);
68
69 (*pos)++;
70 if (!ct)
71 return NULL;
72
73 return priv;
74}
75
76static void allocinfo_stop(struct seq_file *m, void *arg)
77{
78 codetag_lock_module_list(alloc_tag_cttype, false);
79}
80
81static void print_allocinfo_header(struct seq_buf *buf)
82{
83 /* Output format version, so we can change it. */
84 seq_buf_printf(buf, "allocinfo - version: 2.0\n");
85 seq_buf_printf(buf, "# <size> <calls> <tag info>\n");
86}
87
88static void alloc_tag_to_text(struct seq_buf *out, struct codetag *ct)
89{
90 struct alloc_tag *tag = ct_to_alloc_tag(ct);
91 struct alloc_tag_counters counter = alloc_tag_read(tag);
92 s64 bytes = counter.bytes;
93
94 seq_buf_printf(out, "%12lli %8llu ", bytes, counter.calls);
95 codetag_to_text(out, ct);
96 if (unlikely(alloc_tag_is_inaccurate(tag)))
97 seq_buf_printf(out, " accurate:no");
98 seq_buf_putc(out, ' ');
99 seq_buf_putc(out, '\n');
100}
101
102static int allocinfo_show(struct seq_file *m, void *arg)
103{
104 struct allocinfo_private *priv = (struct allocinfo_private *)arg;
105 char *bufp;
106 size_t n = seq_get_buf(m, &bufp);
107 struct seq_buf buf;
108
109 seq_buf_init(&buf, bufp, n);
110 if (priv->print_header) {
111 print_allocinfo_header(&buf);
112 priv->print_header = false;
113 }
114 alloc_tag_to_text(&buf, priv->iter.ct);
115 seq_commit(m, seq_buf_used(&buf));
116 return 0;
117}
118
119static const struct seq_operations allocinfo_seq_op = {
120 .start = allocinfo_start,
121 .next = allocinfo_next,
122 .stop = allocinfo_stop,
123 .show = allocinfo_show,
124};
125
126size_t alloc_tag_top_users(struct codetag_bytes *tags, size_t count, bool can_sleep)
127{
128 struct codetag_iterator iter;
129 struct codetag *ct;
130 struct codetag_bytes n;
131 unsigned int i, nr = 0;
132
133 if (IS_ERR_OR_NULL(alloc_tag_cttype))
134 return 0;
135
136 if (can_sleep)
137 codetag_lock_module_list(alloc_tag_cttype, true);
138 else if (!codetag_trylock_module_list(alloc_tag_cttype))
139 return 0;
140
141 iter = codetag_get_ct_iter(alloc_tag_cttype);
142 while ((ct = codetag_next_ct(&iter))) {
143 struct alloc_tag_counters counter = alloc_tag_read(ct_to_alloc_tag(ct));
144
145 n.ct = ct;
146 n.bytes = counter.bytes;
147
148 for (i = 0; i < nr; i++)
149 if (n.bytes > tags[i].bytes)
150 break;
151
152 if (i < count) {
153 nr -= nr == count;
154 memmove(&tags[i + 1],
155 &tags[i],
156 sizeof(tags[0]) * (nr - i));
157 nr++;
158 tags[i] = n;
159 }
160 }
161
162 codetag_lock_module_list(alloc_tag_cttype, false);
163
164 return nr;
165}
166
167void pgalloc_tag_split(struct folio *folio, int old_order, int new_order)
168{
169 int i;
170 struct alloc_tag *tag;
171 unsigned int nr_pages = 1 << new_order;
172
173 if (!mem_alloc_profiling_enabled())
174 return;
175
176 tag = __pgalloc_tag_get(&folio->page);
177 if (!tag)
178 return;
179
180 for (i = nr_pages; i < (1 << old_order); i += nr_pages) {
181 union pgtag_ref_handle handle;
182 union codetag_ref ref;
183
184 if (get_page_tag_ref(folio_page(folio, i), &ref, &handle)) {
185 /* Set new reference to point to the original tag */
186 alloc_tag_ref_set(&ref, tag);
187 update_page_tag_ref(handle, &ref);
188 put_page_tag_ref(handle);
189 }
190 }
191}
192
193void pgalloc_tag_swap(struct folio *new, struct folio *old)
194{
195 union pgtag_ref_handle handle_old, handle_new;
196 union codetag_ref ref_old, ref_new;
197 struct alloc_tag *tag_old, *tag_new;
198
199 if (!mem_alloc_profiling_enabled())
200 return;
201
202 tag_old = __pgalloc_tag_get(&old->page);
203 if (!tag_old)
204 return;
205 tag_new = __pgalloc_tag_get(&new->page);
206 if (!tag_new)
207 return;
208
209 if (!get_page_tag_ref(&old->page, &ref_old, &handle_old))
210 return;
211 if (!get_page_tag_ref(&new->page, &ref_new, &handle_new)) {
212 put_page_tag_ref(handle_old);
213 return;
214 }
215
216 /*
217 * Clear tag references to avoid debug warning when using
218 * __alloc_tag_ref_set() with non-empty reference.
219 */
220 set_codetag_empty(&ref_old);
221 set_codetag_empty(&ref_new);
222
223 /* swap tags */
224 __alloc_tag_ref_set(&ref_old, tag_new);
225 update_page_tag_ref(handle_old, &ref_old);
226 __alloc_tag_ref_set(&ref_new, tag_old);
227 update_page_tag_ref(handle_new, &ref_new);
228
229 put_page_tag_ref(handle_old);
230 put_page_tag_ref(handle_new);
231}
232
233static void shutdown_mem_profiling(bool remove_file)
234{
235 if (mem_alloc_profiling_enabled())
236 static_branch_disable(&mem_alloc_profiling_key);
237
238 if (!mem_profiling_support)
239 return;
240
241 if (remove_file)
242 remove_proc_entry(ALLOCINFO_FILE_NAME, NULL);
243 mem_profiling_support = false;
244}
245
246void __init alloc_tag_sec_init(void)
247{
248 struct alloc_tag *last_codetag;
249
250 if (!mem_profiling_support)
251 return;
252
253 if (!static_key_enabled(&mem_profiling_compressed))
254 return;
255
256 kernel_tags.first_tag = (struct alloc_tag *)kallsyms_lookup_name(
257 SECTION_START(ALLOC_TAG_SECTION_NAME));
258 last_codetag = (struct alloc_tag *)kallsyms_lookup_name(
259 SECTION_STOP(ALLOC_TAG_SECTION_NAME));
260 kernel_tags.count = last_codetag - kernel_tags.first_tag;
261
262 /* Check if kernel tags fit into page flags */
263 if (kernel_tags.count > (1UL << NR_UNUSED_PAGEFLAG_BITS)) {
264 shutdown_mem_profiling(false); /* allocinfo file does not exist yet */
265 pr_err("%lu allocation tags cannot be references using %d available page flag bits. Memory allocation profiling is disabled!\n",
266 kernel_tags.count, NR_UNUSED_PAGEFLAG_BITS);
267 return;
268 }
269
270 alloc_tag_ref_offs = (LRU_REFS_PGOFF - NR_UNUSED_PAGEFLAG_BITS);
271 alloc_tag_ref_mask = ((1UL << NR_UNUSED_PAGEFLAG_BITS) - 1);
272 pr_debug("Memory allocation profiling compression is using %d page flag bits!\n",
273 NR_UNUSED_PAGEFLAG_BITS);
274}
275
276#ifdef CONFIG_MODULES
277
278static struct maple_tree mod_area_mt = MTREE_INIT(mod_area_mt, MT_FLAGS_ALLOC_RANGE);
279static struct vm_struct *vm_module_tags;
280/* A dummy object used to indicate an unloaded module */
281static struct module unloaded_mod;
282/* A dummy object used to indicate a module prepended area */
283static struct module prepend_mod;
284
285struct alloc_tag_module_section module_tags;
286
287static inline unsigned long alloc_tag_align(unsigned long val)
288{
289 if (!static_key_enabled(&mem_profiling_compressed)) {
290 /* No alignment requirements when we are not indexing the tags */
291 return val;
292 }
293
294 if (val % sizeof(struct alloc_tag) == 0)
295 return val;
296 return ((val / sizeof(struct alloc_tag)) + 1) * sizeof(struct alloc_tag);
297}
298
299static bool ensure_alignment(unsigned long align, unsigned int *prepend)
300{
301 if (!static_key_enabled(&mem_profiling_compressed)) {
302 /* No alignment requirements when we are not indexing the tags */
303 return true;
304 }
305
306 /*
307 * If alloc_tag size is not a multiple of required alignment, tag
308 * indexing does not work.
309 */
310 if (!IS_ALIGNED(sizeof(struct alloc_tag), align))
311 return false;
312
313 /* Ensure prepend consumes multiple of alloc_tag-sized blocks */
314 if (*prepend)
315 *prepend = alloc_tag_align(*prepend);
316
317 return true;
318}
319
320static inline bool tags_addressable(void)
321{
322 unsigned long tag_idx_count;
323
324 if (!static_key_enabled(&mem_profiling_compressed))
325 return true; /* with page_ext tags are always addressable */
326
327 tag_idx_count = CODETAG_ID_FIRST + kernel_tags.count +
328 module_tags.size / sizeof(struct alloc_tag);
329
330 return tag_idx_count < (1UL << NR_UNUSED_PAGEFLAG_BITS);
331}
332
333static bool needs_section_mem(struct module *mod, unsigned long size)
334{
335 if (!mem_profiling_support)
336 return false;
337
338 return size >= sizeof(struct alloc_tag);
339}
340
341static bool clean_unused_counters(struct alloc_tag *start_tag,
342 struct alloc_tag *end_tag)
343{
344 struct alloc_tag *tag;
345 bool ret = true;
346
347 for (tag = start_tag; tag <= end_tag; tag++) {
348 struct alloc_tag_counters counter;
349
350 if (!tag->counters)
351 continue;
352
353 counter = alloc_tag_read(tag);
354 if (!counter.bytes) {
355 free_percpu(tag->counters);
356 tag->counters = NULL;
357 } else {
358 ret = false;
359 }
360 }
361
362 return ret;
363}
364
365/* Called with mod_area_mt locked */
366static void clean_unused_module_areas_locked(void)
367{
368 MA_STATE(mas, &mod_area_mt, 0, module_tags.size);
369 struct module *val;
370
371 mas_for_each(&mas, val, module_tags.size) {
372 struct alloc_tag *start_tag;
373 struct alloc_tag *end_tag;
374
375 if (val != &unloaded_mod)
376 continue;
377
378 /* Release area if all tags are unused */
379 start_tag = (struct alloc_tag *)(module_tags.start_addr + mas.index);
380 end_tag = (struct alloc_tag *)(module_tags.start_addr + mas.last);
381 if (clean_unused_counters(start_tag, end_tag))
382 mas_erase(&mas);
383 }
384}
385
386/* Called with mod_area_mt locked */
387static bool find_aligned_area(struct ma_state *mas, unsigned long section_size,
388 unsigned long size, unsigned int prepend, unsigned long align)
389{
390 bool cleanup_done = false;
391
392repeat:
393 /* Try finding exact size and hope the start is aligned */
394 if (!mas_empty_area(mas, 0, section_size - 1, prepend + size)) {
395 if (IS_ALIGNED(mas->index + prepend, align))
396 return true;
397
398 /* Try finding larger area to align later */
399 mas_reset(mas);
400 if (!mas_empty_area(mas, 0, section_size - 1,
401 size + prepend + align - 1))
402 return true;
403 }
404
405 /* No free area, try cleanup stale data and repeat the search once */
406 if (!cleanup_done) {
407 clean_unused_module_areas_locked();
408 cleanup_done = true;
409 mas_reset(mas);
410 goto repeat;
411 }
412
413 return false;
414}
415
416static int vm_module_tags_populate(void)
417{
418 unsigned long phys_end = ALIGN_DOWN(module_tags.start_addr, PAGE_SIZE) +
419 (vm_module_tags->nr_pages << PAGE_SHIFT);
420 unsigned long new_end = module_tags.start_addr + module_tags.size;
421
422 if (phys_end < new_end) {
423 struct page **next_page = vm_module_tags->pages + vm_module_tags->nr_pages;
424 unsigned long old_shadow_end = ALIGN(phys_end, MODULE_ALIGN);
425 unsigned long new_shadow_end = ALIGN(new_end, MODULE_ALIGN);
426 unsigned long more_pages;
427 unsigned long nr = 0;
428
429 more_pages = ALIGN(new_end - phys_end, PAGE_SIZE) >> PAGE_SHIFT;
430 while (nr < more_pages) {
431 unsigned long allocated;
432
433 allocated = alloc_pages_bulk_node(GFP_KERNEL | __GFP_NOWARN,
434 NUMA_NO_NODE, more_pages - nr, next_page + nr);
435
436 if (!allocated)
437 break;
438 nr += allocated;
439 }
440
441 if (nr < more_pages ||
442 vmap_pages_range(phys_end, phys_end + (nr << PAGE_SHIFT), PAGE_KERNEL,
443 next_page, PAGE_SHIFT) < 0) {
444 release_pages_arg arg = { .pages = next_page };
445
446 /* Clean up and error out */
447 release_pages(arg, nr);
448 return -ENOMEM;
449 }
450
451 vm_module_tags->nr_pages += nr;
452
453 /*
454 * Kasan allocates 1 byte of shadow for every 8 bytes of data.
455 * When kasan_alloc_module_shadow allocates shadow memory,
456 * its unit of allocation is a page.
457 * Therefore, here we need to align to MODULE_ALIGN.
458 */
459 if (old_shadow_end < new_shadow_end)
460 kasan_alloc_module_shadow((void *)old_shadow_end,
461 new_shadow_end - old_shadow_end,
462 GFP_KERNEL);
463 }
464
465 /*
466 * Mark the pages as accessible, now that they are mapped.
467 * With hardware tag-based KASAN, marking is skipped for
468 * non-VM_ALLOC mappings, see __kasan_unpoison_vmalloc().
469 */
470 kasan_unpoison_vmalloc((void *)module_tags.start_addr,
471 new_end - module_tags.start_addr,
472 KASAN_VMALLOC_PROT_NORMAL);
473
474 return 0;
475}
476
477static void *reserve_module_tags(struct module *mod, unsigned long size,
478 unsigned int prepend, unsigned long align)
479{
480 unsigned long section_size = module_tags.end_addr - module_tags.start_addr;
481 MA_STATE(mas, &mod_area_mt, 0, section_size - 1);
482 unsigned long offset;
483 void *ret = NULL;
484
485 /* If no tags return error */
486 if (size < sizeof(struct alloc_tag))
487 return ERR_PTR(-EINVAL);
488
489 /*
490 * align is always power of 2, so we can use IS_ALIGNED and ALIGN.
491 * align 0 or 1 means no alignment, to simplify set to 1.
492 */
493 if (!align)
494 align = 1;
495
496 if (!ensure_alignment(align, &prepend)) {
497 shutdown_mem_profiling(true);
498 pr_err("%s: alignment %lu is incompatible with allocation tag indexing. Memory allocation profiling is disabled!\n",
499 mod->name, align);
500 return ERR_PTR(-EINVAL);
501 }
502
503 mas_lock(&mas);
504 if (!find_aligned_area(&mas, section_size, size, prepend, align)) {
505 ret = ERR_PTR(-ENOMEM);
506 goto unlock;
507 }
508
509 /* Mark found area as reserved */
510 offset = mas.index;
511 offset += prepend;
512 offset = ALIGN(offset, align);
513 if (offset != mas.index) {
514 unsigned long pad_start = mas.index;
515
516 mas.last = offset - 1;
517 mas_store(&mas, &prepend_mod);
518 if (mas_is_err(&mas)) {
519 ret = ERR_PTR(xa_err(mas.node));
520 goto unlock;
521 }
522 mas.index = offset;
523 mas.last = offset + size - 1;
524 mas_store(&mas, mod);
525 if (mas_is_err(&mas)) {
526 mas.index = pad_start;
527 mas_erase(&mas);
528 ret = ERR_PTR(xa_err(mas.node));
529 }
530 } else {
531 mas.last = offset + size - 1;
532 mas_store(&mas, mod);
533 if (mas_is_err(&mas))
534 ret = ERR_PTR(xa_err(mas.node));
535 }
536unlock:
537 mas_unlock(&mas);
538
539 if (IS_ERR(ret))
540 return ret;
541
542 if (module_tags.size < offset + size) {
543 int grow_res;
544
545 module_tags.size = offset + size;
546 if (mem_alloc_profiling_enabled() && !tags_addressable()) {
547 shutdown_mem_profiling(true);
548 pr_warn("With module %s there are too many tags to fit in %d page flag bits. Memory allocation profiling is disabled!\n",
549 mod->name, NR_UNUSED_PAGEFLAG_BITS);
550 }
551
552 grow_res = vm_module_tags_populate();
553 if (grow_res) {
554 shutdown_mem_profiling(true);
555 pr_err("Failed to allocate memory for allocation tags in the module %s. Memory allocation profiling is disabled!\n",
556 mod->name);
557 return ERR_PTR(grow_res);
558 }
559 }
560
561 return (struct alloc_tag *)(module_tags.start_addr + offset);
562}
563
564static void release_module_tags(struct module *mod, bool used)
565{
566 MA_STATE(mas, &mod_area_mt, module_tags.size, module_tags.size);
567 struct alloc_tag *start_tag;
568 struct alloc_tag *end_tag;
569 struct module *val;
570
571 mas_lock(&mas);
572 mas_for_each_rev(&mas, val, 0)
573 if (val == mod)
574 break;
575
576 if (!val) /* module not found */
577 goto out;
578
579 if (!used)
580 goto release_area;
581
582 start_tag = (struct alloc_tag *)(module_tags.start_addr + mas.index);
583 end_tag = (struct alloc_tag *)(module_tags.start_addr + mas.last);
584 if (!clean_unused_counters(start_tag, end_tag)) {
585 struct alloc_tag *tag;
586
587 for (tag = start_tag; tag <= end_tag; tag++) {
588 struct alloc_tag_counters counter;
589
590 if (!tag->counters)
591 continue;
592
593 counter = alloc_tag_read(tag);
594 pr_info("%s:%u module %s func:%s has %llu allocated at module unload\n",
595 tag->ct.filename, tag->ct.lineno, tag->ct.modname,
596 tag->ct.function, counter.bytes);
597 }
598 } else {
599 used = false;
600 }
601release_area:
602 mas_store(&mas, used ? &unloaded_mod : NULL);
603 val = mas_prev_range(&mas, 0);
604 if (val == &prepend_mod)
605 mas_store(&mas, NULL);
606out:
607 mas_unlock(&mas);
608}
609
610static int load_module(struct module *mod, struct codetag *start, struct codetag *stop)
611{
612 /* Allocate module alloc_tag percpu counters */
613 struct alloc_tag *start_tag;
614 struct alloc_tag *stop_tag;
615 struct alloc_tag *tag;
616
617 /* percpu counters for core allocations are already statically allocated */
618 if (!mod)
619 return 0;
620
621 start_tag = ct_to_alloc_tag(start);
622 stop_tag = ct_to_alloc_tag(stop);
623 for (tag = start_tag; tag < stop_tag; tag++) {
624 WARN_ON(tag->counters);
625 tag->counters = alloc_percpu(struct alloc_tag_counters);
626 if (!tag->counters) {
627 while (--tag >= start_tag) {
628 free_percpu(tag->counters);
629 tag->counters = NULL;
630 }
631 pr_err("Failed to allocate memory for allocation tag percpu counters in the module %s\n",
632 mod->name);
633 return -ENOMEM;
634 }
635
636 /*
637 * Avoid a kmemleak false positive. The pointer to the counters is stored
638 * in the alloc_tag section of the module and cannot be directly accessed.
639 */
640 kmemleak_ignore_percpu(tag->counters);
641 }
642 return 0;
643}
644
645static void replace_module(struct module *mod, struct module *new_mod)
646{
647 MA_STATE(mas, &mod_area_mt, 0, module_tags.size);
648 struct module *val;
649
650 mas_lock(&mas);
651 mas_for_each(&mas, val, module_tags.size) {
652 if (val != mod)
653 continue;
654
655 mas_store_gfp(&mas, new_mod, GFP_KERNEL);
656 break;
657 }
658 mas_unlock(&mas);
659}
660
661static int __init alloc_mod_tags_mem(void)
662{
663 /* Map space to copy allocation tags */
664 vm_module_tags = execmem_vmap(MODULE_ALLOC_TAG_VMAP_SIZE);
665 if (!vm_module_tags) {
666 pr_err("Failed to map %lu bytes for module allocation tags\n",
667 MODULE_ALLOC_TAG_VMAP_SIZE);
668 module_tags.start_addr = 0;
669 return -ENOMEM;
670 }
671
672 vm_module_tags->pages = kmalloc_objs(struct page *,
673 get_vm_area_size(vm_module_tags) >> PAGE_SHIFT,
674 GFP_KERNEL | __GFP_ZERO);
675 if (!vm_module_tags->pages) {
676 free_vm_area(vm_module_tags);
677 return -ENOMEM;
678 }
679
680 module_tags.start_addr = (unsigned long)vm_module_tags->addr;
681 module_tags.end_addr = module_tags.start_addr + MODULE_ALLOC_TAG_VMAP_SIZE;
682 /* Ensure the base is alloc_tag aligned when required for indexing */
683 module_tags.start_addr = alloc_tag_align(module_tags.start_addr);
684
685 return 0;
686}
687
688static void __init free_mod_tags_mem(void)
689{
690 release_pages_arg arg = { .pages = vm_module_tags->pages };
691
692 module_tags.start_addr = 0;
693 release_pages(arg, vm_module_tags->nr_pages);
694 kfree(vm_module_tags->pages);
695 free_vm_area(vm_module_tags);
696}
697
698#else /* CONFIG_MODULES */
699
700static inline int alloc_mod_tags_mem(void) { return 0; }
701static inline void free_mod_tags_mem(void) {}
702
703#endif /* CONFIG_MODULES */
704
705/* See: Documentation/mm/allocation-profiling.rst */
706static int __init setup_early_mem_profiling(char *str)
707{
708 bool compressed = false;
709 bool enable;
710
711 if (!str || !str[0])
712 return -EINVAL;
713
714 if (!strncmp(str, "never", 5)) {
715 enable = false;
716 mem_profiling_support = false;
717 pr_info("Memory allocation profiling is disabled!\n");
718 } else {
719 char *token = strsep(&str, ",");
720
721 if (kstrtobool(token, &enable))
722 return -EINVAL;
723
724 if (str) {
725
726 if (strcmp(str, "compressed"))
727 return -EINVAL;
728
729 compressed = true;
730 }
731 mem_profiling_support = true;
732 pr_info("Memory allocation profiling is enabled %s compression and is turned %s!\n",
733 compressed ? "with" : "without", str_on_off(enable));
734 }
735
736 if (enable != mem_alloc_profiling_enabled()) {
737 if (enable)
738 static_branch_enable(&mem_alloc_profiling_key);
739 else
740 static_branch_disable(&mem_alloc_profiling_key);
741 }
742 if (compressed != static_key_enabled(&mem_profiling_compressed)) {
743 if (compressed)
744 static_branch_enable(&mem_profiling_compressed);
745 else
746 static_branch_disable(&mem_profiling_compressed);
747 }
748
749 return 0;
750}
751early_param("sysctl.vm.mem_profiling", setup_early_mem_profiling);
752
753static __init bool need_page_alloc_tagging(void)
754{
755 if (static_key_enabled(&mem_profiling_compressed))
756 return false;
757
758 return mem_profiling_support;
759}
760
761static __init void init_page_alloc_tagging(void)
762{
763}
764
765struct page_ext_operations page_alloc_tagging_ops = {
766 .size = sizeof(union codetag_ref),
767 .need = need_page_alloc_tagging,
768 .init = init_page_alloc_tagging,
769};
770EXPORT_SYMBOL(page_alloc_tagging_ops);
771
772#ifdef CONFIG_SYSCTL
773/*
774 * Not using proc_do_static_key() directly to prevent enabling profiling
775 * after it was shut down.
776 */
777static int proc_mem_profiling_handler(const struct ctl_table *table, int write,
778 void *buffer, size_t *lenp, loff_t *ppos)
779{
780 if (write) {
781 /*
782 * Call from do_sysctl_args() which is a no-op since the same
783 * value was already set by setup_early_mem_profiling.
784 * Return success to avoid warnings from do_sysctl_args().
785 */
786 if (!current->mm)
787 return 0;
788
789#ifdef CONFIG_MEM_ALLOC_PROFILING_DEBUG
790 /* User can't toggle profiling while debugging */
791 return -EACCES;
792#endif
793 if (!mem_profiling_support)
794 return -EINVAL;
795 }
796
797 return proc_do_static_key(table, write, buffer, lenp, ppos);
798}
799
800
801static const struct ctl_table memory_allocation_profiling_sysctls[] = {
802 {
803 .procname = "mem_profiling",
804 .data = &mem_alloc_profiling_key,
805 .mode = 0644,
806 .proc_handler = proc_mem_profiling_handler,
807 },
808};
809
810static void __init sysctl_init(void)
811{
812 register_sysctl_init("vm", memory_allocation_profiling_sysctls);
813}
814#else /* CONFIG_SYSCTL */
815static inline void sysctl_init(void) {}
816#endif /* CONFIG_SYSCTL */
817
818static int __init alloc_tag_init(void)
819{
820 const struct codetag_type_desc desc = {
821 .section = ALLOC_TAG_SECTION_NAME,
822 .tag_size = sizeof(struct alloc_tag),
823#ifdef CONFIG_MODULES
824 .needs_section_mem = needs_section_mem,
825 .alloc_section_mem = reserve_module_tags,
826 .free_section_mem = release_module_tags,
827 .module_load = load_module,
828 .module_replaced = replace_module,
829#endif
830 };
831 int res;
832
833 sysctl_init();
834
835 if (!mem_profiling_support) {
836 pr_info("Memory allocation profiling is not supported!\n");
837 return 0;
838 }
839
840 if (!proc_create_seq_private(ALLOCINFO_FILE_NAME, 0400, NULL, &allocinfo_seq_op,
841 sizeof(struct allocinfo_private), NULL)) {
842 pr_err("Failed to create %s file\n", ALLOCINFO_FILE_NAME);
843 shutdown_mem_profiling(false);
844 return -ENOMEM;
845 }
846
847 res = alloc_mod_tags_mem();
848 if (res) {
849 pr_err("Failed to reserve address space for module tags, errno = %d\n", res);
850 shutdown_mem_profiling(true);
851 return res;
852 }
853
854 alloc_tag_cttype = codetag_register_type(&desc);
855 if (IS_ERR(alloc_tag_cttype)) {
856 pr_err("Allocation tags registration failed, errno = %pe\n", alloc_tag_cttype);
857 free_mod_tags_mem();
858 shutdown_mem_profiling(true);
859 return PTR_ERR(alloc_tag_cttype);
860 }
861
862 return 0;
863}
864module_init(alloc_tag_init);