projects/linux·blob v6.13 · refs

769 lines · 19.5 KiB

1// SPDX-License-Identifier: GPL-2.0-only
2#include <linux/alloc_tag.h>
3#include <linux/execmem.h>
4#include <linux/fs.h>
5#include <linux/gfp.h>
6#include <linux/kallsyms.h>
7#include <linux/module.h>
8#include <linux/page_ext.h>
9#include <linux/proc_fs.h>
10#include <linux/seq_buf.h>
11#include <linux/seq_file.h>
12#include <linux/vmalloc.h>
13
14#define ALLOCINFO_FILE_NAME "allocinfo"
15#define MODULE_ALLOC_TAG_VMAP_SIZE (100000UL * sizeof(struct alloc_tag))
16#define SECTION_START(NAME) (CODETAG_SECTION_START_PREFIX NAME)
17#define SECTION_STOP(NAME) (CODETAG_SECTION_STOP_PREFIX NAME)
18
19#ifdef CONFIG_MEM_ALLOC_PROFILING_ENABLED_BY_DEFAULT
20static bool mem_profiling_support = true;
21#else
22static bool mem_profiling_support;
23#endif
24
25static struct codetag_type *alloc_tag_cttype;
26
27DEFINE_PER_CPU(struct alloc_tag_counters, _shared_alloc_tag);
28EXPORT_SYMBOL(_shared_alloc_tag);
29
30DEFINE_STATIC_KEY_MAYBE(CONFIG_MEM_ALLOC_PROFILING_ENABLED_BY_DEFAULT,
31 mem_alloc_profiling_key);
32DEFINE_STATIC_KEY_FALSE(mem_profiling_compressed);
33
34struct alloc_tag_kernel_section kernel_tags = { NULL, 0 };
35unsigned long alloc_tag_ref_mask;
36int alloc_tag_ref_offs;
37
38struct allocinfo_private {
39 struct codetag_iterator iter;
40 bool print_header;
41};
42
43static void *allocinfo_start(struct seq_file *m, loff_t *pos)
44{
45 struct allocinfo_private *priv;
46 struct codetag *ct;
47 loff_t node = *pos;
48
49 priv = kzalloc(sizeof(*priv), GFP_KERNEL);
50 m->private = priv;
51 if (!priv)
52 return NULL;
53
54 priv->print_header = (node == 0);
55 codetag_lock_module_list(alloc_tag_cttype, true);
56 priv->iter = codetag_get_ct_iter(alloc_tag_cttype);
57 while ((ct = codetag_next_ct(&priv->iter)) != NULL && node)
58 node--;
59
60 return ct ? priv : NULL;
61}
62
63static void *allocinfo_next(struct seq_file *m, void *arg, loff_t *pos)
64{
65 struct allocinfo_private *priv = (struct allocinfo_private *)arg;
66 struct codetag *ct = codetag_next_ct(&priv->iter);
67
68 (*pos)++;
69 if (!ct)
70 return NULL;
71
72 return priv;
73}
74
75static void allocinfo_stop(struct seq_file *m, void *arg)
76{
77 struct allocinfo_private *priv = (struct allocinfo_private *)m->private;
78
79 if (priv) {
80 codetag_lock_module_list(alloc_tag_cttype, false);
81 kfree(priv);
82 }
83}
84
85static void print_allocinfo_header(struct seq_buf *buf)
86{
87 /* Output format version, so we can change it. */
88 seq_buf_printf(buf, "allocinfo - version: 1.0\n");
89 seq_buf_printf(buf, "# <size> <calls> <tag info>\n");
90}
91
92static void alloc_tag_to_text(struct seq_buf *out, struct codetag *ct)
93{
94 struct alloc_tag *tag = ct_to_alloc_tag(ct);
95 struct alloc_tag_counters counter = alloc_tag_read(tag);
96 s64 bytes = counter.bytes;
97
98 seq_buf_printf(out, "%12lli %8llu ", bytes, counter.calls);
99 codetag_to_text(out, ct);
100 seq_buf_putc(out, ' ');
101 seq_buf_putc(out, '\n');
102}
103
104static int allocinfo_show(struct seq_file *m, void *arg)
105{
106 struct allocinfo_private *priv = (struct allocinfo_private *)arg;
107 char *bufp;
108 size_t n = seq_get_buf(m, &bufp);
109 struct seq_buf buf;
110
111 seq_buf_init(&buf, bufp, n);
112 if (priv->print_header) {
113 print_allocinfo_header(&buf);
114 priv->print_header = false;
115 }
116 alloc_tag_to_text(&buf, priv->iter.ct);
117 seq_commit(m, seq_buf_used(&buf));
118 return 0;
119}
120
121static const struct seq_operations allocinfo_seq_op = {
122 .start = allocinfo_start,
123 .next = allocinfo_next,
124 .stop = allocinfo_stop,
125 .show = allocinfo_show,
126};
127
128size_t alloc_tag_top_users(struct codetag_bytes *tags, size_t count, bool can_sleep)
129{
130 struct codetag_iterator iter;
131 struct codetag *ct;
132 struct codetag_bytes n;
133 unsigned int i, nr = 0;
134
135 if (can_sleep)
136 codetag_lock_module_list(alloc_tag_cttype, true);
137 else if (!codetag_trylock_module_list(alloc_tag_cttype))
138 return 0;
139
140 iter = codetag_get_ct_iter(alloc_tag_cttype);
141 while ((ct = codetag_next_ct(&iter))) {
142 struct alloc_tag_counters counter = alloc_tag_read(ct_to_alloc_tag(ct));
143
144 n.ct = ct;
145 n.bytes = counter.bytes;
146
147 for (i = 0; i < nr; i++)
148 if (n.bytes > tags[i].bytes)
149 break;
150
151 if (i < count) {
152 nr -= nr == count;
153 memmove(&tags[i + 1],
154 &tags[i],
155 sizeof(tags[0]) * (nr - i));
156 nr++;
157 tags[i] = n;
158 }
159 }
160
161 codetag_lock_module_list(alloc_tag_cttype, false);
162
163 return nr;
164}
165
166void pgalloc_tag_split(struct folio *folio, int old_order, int new_order)
167{
168 int i;
169 struct alloc_tag *tag;
170 unsigned int nr_pages = 1 << new_order;
171
172 if (!mem_alloc_profiling_enabled())
173 return;
174
175 tag = pgalloc_tag_get(&folio->page);
176 if (!tag)
177 return;
178
179 for (i = nr_pages; i < (1 << old_order); i += nr_pages) {
180 union pgtag_ref_handle handle;
181 union codetag_ref ref;
182
183 if (get_page_tag_ref(folio_page(folio, i), &ref, &handle)) {
184 /* Set new reference to point to the original tag */
185 alloc_tag_ref_set(&ref, tag);
186 update_page_tag_ref(handle, &ref);
187 put_page_tag_ref(handle);
188 }
189 }
190}
191
192void pgalloc_tag_swap(struct folio *new, struct folio *old)
193{
194 union pgtag_ref_handle handle_old, handle_new;
195 union codetag_ref ref_old, ref_new;
196 struct alloc_tag *tag_old, *tag_new;
197
198 if (!mem_alloc_profiling_enabled())
199 return;
200
201 tag_old = pgalloc_tag_get(&old->page);
202 if (!tag_old)
203 return;
204 tag_new = pgalloc_tag_get(&new->page);
205 if (!tag_new)
206 return;
207
208 if (!get_page_tag_ref(&old->page, &ref_old, &handle_old))
209 return;
210 if (!get_page_tag_ref(&new->page, &ref_new, &handle_new)) {
211 put_page_tag_ref(handle_old);
212 return;
213 }
214
215 /*
216 * Clear tag references to avoid debug warning when using
217 * __alloc_tag_ref_set() with non-empty reference.
218 */
219 set_codetag_empty(&ref_old);
220 set_codetag_empty(&ref_new);
221
222 /* swap tags */
223 __alloc_tag_ref_set(&ref_old, tag_new);
224 update_page_tag_ref(handle_old, &ref_old);
225 __alloc_tag_ref_set(&ref_new, tag_old);
226 update_page_tag_ref(handle_new, &ref_new);
227
228 put_page_tag_ref(handle_old);
229 put_page_tag_ref(handle_new);
230}
231
232static void shutdown_mem_profiling(bool remove_file)
233{
234 if (mem_alloc_profiling_enabled())
235 static_branch_disable(&mem_alloc_profiling_key);
236
237 if (!mem_profiling_support)
238 return;
239
240 if (remove_file)
241 remove_proc_entry(ALLOCINFO_FILE_NAME, NULL);
242 mem_profiling_support = false;
243}
244
245static void __init procfs_init(void)
246{
247 if (!mem_profiling_support)
248 return;
249
250 if (!proc_create_seq(ALLOCINFO_FILE_NAME, 0400, NULL, &allocinfo_seq_op)) {
251 pr_err("Failed to create %s file\n", ALLOCINFO_FILE_NAME);
252 shutdown_mem_profiling(false);
253 }
254}
255
256void __init alloc_tag_sec_init(void)
257{
258 struct alloc_tag *last_codetag;
259
260 if (!mem_profiling_support)
261 return;
262
263 if (!static_key_enabled(&mem_profiling_compressed))
264 return;
265
266 kernel_tags.first_tag = (struct alloc_tag *)kallsyms_lookup_name(
267 SECTION_START(ALLOC_TAG_SECTION_NAME));
268 last_codetag = (struct alloc_tag *)kallsyms_lookup_name(
269 SECTION_STOP(ALLOC_TAG_SECTION_NAME));
270 kernel_tags.count = last_codetag - kernel_tags.first_tag;
271
272 /* Check if kernel tags fit into page flags */
273 if (kernel_tags.count > (1UL << NR_UNUSED_PAGEFLAG_BITS)) {
274 shutdown_mem_profiling(false); /* allocinfo file does not exist yet */
275 pr_err("%lu allocation tags cannot be references using %d available page flag bits. Memory allocation profiling is disabled!\n",
276 kernel_tags.count, NR_UNUSED_PAGEFLAG_BITS);
277 return;
278 }
279
280 alloc_tag_ref_offs = (LRU_REFS_PGOFF - NR_UNUSED_PAGEFLAG_BITS);
281 alloc_tag_ref_mask = ((1UL << NR_UNUSED_PAGEFLAG_BITS) - 1);
282 pr_debug("Memory allocation profiling compression is using %d page flag bits!\n",
283 NR_UNUSED_PAGEFLAG_BITS);
284}
285
286#ifdef CONFIG_MODULES
287
288static struct maple_tree mod_area_mt = MTREE_INIT(mod_area_mt, MT_FLAGS_ALLOC_RANGE);
289static struct vm_struct *vm_module_tags;
290/* A dummy object used to indicate an unloaded module */
291static struct module unloaded_mod;
292/* A dummy object used to indicate a module prepended area */
293static struct module prepend_mod;
294
295struct alloc_tag_module_section module_tags;
296
297static inline unsigned long alloc_tag_align(unsigned long val)
298{
299 if (!static_key_enabled(&mem_profiling_compressed)) {
300 /* No alignment requirements when we are not indexing the tags */
301 return val;
302 }
303
304 if (val % sizeof(struct alloc_tag) == 0)
305 return val;
306 return ((val / sizeof(struct alloc_tag)) + 1) * sizeof(struct alloc_tag);
307}
308
309static bool ensure_alignment(unsigned long align, unsigned int *prepend)
310{
311 if (!static_key_enabled(&mem_profiling_compressed)) {
312 /* No alignment requirements when we are not indexing the tags */
313 return true;
314 }
315
316 /*
317 * If alloc_tag size is not a multiple of required alignment, tag
318 * indexing does not work.
319 */
320 if (!IS_ALIGNED(sizeof(struct alloc_tag), align))
321 return false;
322
323 /* Ensure prepend consumes multiple of alloc_tag-sized blocks */
324 if (*prepend)
325 *prepend = alloc_tag_align(*prepend);
326
327 return true;
328}
329
330static inline bool tags_addressable(void)
331{
332 unsigned long tag_idx_count;
333
334 if (!static_key_enabled(&mem_profiling_compressed))
335 return true; /* with page_ext tags are always addressable */
336
337 tag_idx_count = CODETAG_ID_FIRST + kernel_tags.count +
338 module_tags.size / sizeof(struct alloc_tag);
339
340 return tag_idx_count < (1UL << NR_UNUSED_PAGEFLAG_BITS);
341}
342
343static bool needs_section_mem(struct module *mod, unsigned long size)
344{
345 if (!mem_profiling_support)
346 return false;
347
348 return size >= sizeof(struct alloc_tag);
349}
350
351static struct alloc_tag *find_used_tag(struct alloc_tag *from, struct alloc_tag *to)
352{
353 while (from <= to) {
354 struct alloc_tag_counters counter;
355
356 counter = alloc_tag_read(from);
357 if (counter.bytes)
358 return from;
359 from++;
360 }
361
362 return NULL;
363}
364
365/* Called with mod_area_mt locked */
366static void clean_unused_module_areas_locked(void)
367{
368 MA_STATE(mas, &mod_area_mt, 0, module_tags.size);
369 struct module *val;
370
371 mas_for_each(&mas, val, module_tags.size) {
372 if (val != &unloaded_mod)
373 continue;
374
375 /* Release area if all tags are unused */
376 if (!find_used_tag((struct alloc_tag *)(module_tags.start_addr + mas.index),
377 (struct alloc_tag *)(module_tags.start_addr + mas.last)))
378 mas_erase(&mas);
379 }
380}
381
382/* Called with mod_area_mt locked */
383static bool find_aligned_area(struct ma_state *mas, unsigned long section_size,
384 unsigned long size, unsigned int prepend, unsigned long align)
385{
386 bool cleanup_done = false;
387
388repeat:
389 /* Try finding exact size and hope the start is aligned */
390 if (!mas_empty_area(mas, 0, section_size - 1, prepend + size)) {
391 if (IS_ALIGNED(mas->index + prepend, align))
392 return true;
393
394 /* Try finding larger area to align later */
395 mas_reset(mas);
396 if (!mas_empty_area(mas, 0, section_size - 1,
397 size + prepend + align - 1))
398 return true;
399 }
400
401 /* No free area, try cleanup stale data and repeat the search once */
402 if (!cleanup_done) {
403 clean_unused_module_areas_locked();
404 cleanup_done = true;
405 mas_reset(mas);
406 goto repeat;
407 }
408
409 return false;
410}
411
412static int vm_module_tags_populate(void)
413{
414 unsigned long phys_end = ALIGN_DOWN(module_tags.start_addr, PAGE_SIZE) +
415 (vm_module_tags->nr_pages << PAGE_SHIFT);
416 unsigned long new_end = module_tags.start_addr + module_tags.size;
417
418 if (phys_end < new_end) {
419 struct page **next_page = vm_module_tags->pages + vm_module_tags->nr_pages;
420 unsigned long old_shadow_end = ALIGN(phys_end, MODULE_ALIGN);
421 unsigned long new_shadow_end = ALIGN(new_end, MODULE_ALIGN);
422 unsigned long more_pages;
423 unsigned long nr;
424
425 more_pages = ALIGN(new_end - phys_end, PAGE_SIZE) >> PAGE_SHIFT;
426 nr = alloc_pages_bulk_array_node(GFP_KERNEL | __GFP_NOWARN,
427 NUMA_NO_NODE, more_pages, next_page);
428 if (nr < more_pages ||
429 vmap_pages_range(phys_end, phys_end + (nr << PAGE_SHIFT), PAGE_KERNEL,
430 next_page, PAGE_SHIFT) < 0) {
431 /* Clean up and error out */
432 for (int i = 0; i < nr; i++)
433 __free_page(next_page[i]);
434 return -ENOMEM;
435 }
436
437 vm_module_tags->nr_pages += nr;
438
439 /*
440 * Kasan allocates 1 byte of shadow for every 8 bytes of data.
441 * When kasan_alloc_module_shadow allocates shadow memory,
442 * its unit of allocation is a page.
443 * Therefore, here we need to align to MODULE_ALIGN.
444 */
445 if (old_shadow_end < new_shadow_end)
446 kasan_alloc_module_shadow((void *)old_shadow_end,
447 new_shadow_end - old_shadow_end,
448 GFP_KERNEL);
449 }
450
451 /*
452 * Mark the pages as accessible, now that they are mapped.
453 * With hardware tag-based KASAN, marking is skipped for
454 * non-VM_ALLOC mappings, see __kasan_unpoison_vmalloc().
455 */
456 kasan_unpoison_vmalloc((void *)module_tags.start_addr,
457 new_end - module_tags.start_addr,
458 KASAN_VMALLOC_PROT_NORMAL);
459
460 return 0;
461}
462
463static void *reserve_module_tags(struct module *mod, unsigned long size,
464 unsigned int prepend, unsigned long align)
465{
466 unsigned long section_size = module_tags.end_addr - module_tags.start_addr;
467 MA_STATE(mas, &mod_area_mt, 0, section_size - 1);
468 unsigned long offset;
469 void *ret = NULL;
470
471 /* If no tags return error */
472 if (size < sizeof(struct alloc_tag))
473 return ERR_PTR(-EINVAL);
474
475 /*
476 * align is always power of 2, so we can use IS_ALIGNED and ALIGN.
477 * align 0 or 1 means no alignment, to simplify set to 1.
478 */
479 if (!align)
480 align = 1;
481
482 if (!ensure_alignment(align, &prepend)) {
483 shutdown_mem_profiling(true);
484 pr_err("%s: alignment %lu is incompatible with allocation tag indexing. Memory allocation profiling is disabled!\n",
485 mod->name, align);
486 return ERR_PTR(-EINVAL);
487 }
488
489 mas_lock(&mas);
490 if (!find_aligned_area(&mas, section_size, size, prepend, align)) {
491 ret = ERR_PTR(-ENOMEM);
492 goto unlock;
493 }
494
495 /* Mark found area as reserved */
496 offset = mas.index;
497 offset += prepend;
498 offset = ALIGN(offset, align);
499 if (offset != mas.index) {
500 unsigned long pad_start = mas.index;
501
502 mas.last = offset - 1;
503 mas_store(&mas, &prepend_mod);
504 if (mas_is_err(&mas)) {
505 ret = ERR_PTR(xa_err(mas.node));
506 goto unlock;
507 }
508 mas.index = offset;
509 mas.last = offset + size - 1;
510 mas_store(&mas, mod);
511 if (mas_is_err(&mas)) {
512 mas.index = pad_start;
513 mas_erase(&mas);
514 ret = ERR_PTR(xa_err(mas.node));
515 }
516 } else {
517 mas.last = offset + size - 1;
518 mas_store(&mas, mod);
519 if (mas_is_err(&mas))
520 ret = ERR_PTR(xa_err(mas.node));
521 }
522unlock:
523 mas_unlock(&mas);
524
525 if (IS_ERR(ret))
526 return ret;
527
528 if (module_tags.size < offset + size) {
529 int grow_res;
530
531 module_tags.size = offset + size;
532 if (mem_alloc_profiling_enabled() && !tags_addressable()) {
533 shutdown_mem_profiling(true);
534 pr_warn("With module %s there are too many tags to fit in %d page flag bits. Memory allocation profiling is disabled!\n",
535 mod->name, NR_UNUSED_PAGEFLAG_BITS);
536 }
537
538 grow_res = vm_module_tags_populate();
539 if (grow_res) {
540 shutdown_mem_profiling(true);
541 pr_err("Failed to allocate memory for allocation tags in the module %s. Memory allocation profiling is disabled!\n",
542 mod->name);
543 return ERR_PTR(grow_res);
544 }
545 }
546
547 return (struct alloc_tag *)(module_tags.start_addr + offset);
548}
549
550static void release_module_tags(struct module *mod, bool used)
551{
552 MA_STATE(mas, &mod_area_mt, module_tags.size, module_tags.size);
553 struct alloc_tag *tag;
554 struct module *val;
555
556 mas_lock(&mas);
557 mas_for_each_rev(&mas, val, 0)
558 if (val == mod)
559 break;
560
561 if (!val) /* module not found */
562 goto out;
563
564 if (!used)
565 goto release_area;
566
567 /* Find out if the area is used */
568 tag = find_used_tag((struct alloc_tag *)(module_tags.start_addr + mas.index),
569 (struct alloc_tag *)(module_tags.start_addr + mas.last));
570 if (tag) {
571 struct alloc_tag_counters counter = alloc_tag_read(tag);
572
573 pr_info("%s:%u module %s func:%s has %llu allocated at module unload\n",
574 tag->ct.filename, tag->ct.lineno, tag->ct.modname,
575 tag->ct.function, counter.bytes);
576 } else {
577 used = false;
578 }
579release_area:
580 mas_store(&mas, used ? &unloaded_mod : NULL);
581 val = mas_prev_range(&mas, 0);
582 if (val == &prepend_mod)
583 mas_store(&mas, NULL);
584out:
585 mas_unlock(&mas);
586}
587
588static void replace_module(struct module *mod, struct module *new_mod)
589{
590 MA_STATE(mas, &mod_area_mt, 0, module_tags.size);
591 struct module *val;
592
593 mas_lock(&mas);
594 mas_for_each(&mas, val, module_tags.size) {
595 if (val != mod)
596 continue;
597
598 mas_store_gfp(&mas, new_mod, GFP_KERNEL);
599 break;
600 }
601 mas_unlock(&mas);
602}
603
604static int __init alloc_mod_tags_mem(void)
605{
606 /* Map space to copy allocation tags */
607 vm_module_tags = execmem_vmap(MODULE_ALLOC_TAG_VMAP_SIZE);
608 if (!vm_module_tags) {
609 pr_err("Failed to map %lu bytes for module allocation tags\n",
610 MODULE_ALLOC_TAG_VMAP_SIZE);
611 module_tags.start_addr = 0;
612 return -ENOMEM;
613 }
614
615 vm_module_tags->pages = kmalloc_array(get_vm_area_size(vm_module_tags) >> PAGE_SHIFT,
616 sizeof(struct page *), GFP_KERNEL | __GFP_ZERO);
617 if (!vm_module_tags->pages) {
618 free_vm_area(vm_module_tags);
619 return -ENOMEM;
620 }
621
622 module_tags.start_addr = (unsigned long)vm_module_tags->addr;
623 module_tags.end_addr = module_tags.start_addr + MODULE_ALLOC_TAG_VMAP_SIZE;
624 /* Ensure the base is alloc_tag aligned when required for indexing */
625 module_tags.start_addr = alloc_tag_align(module_tags.start_addr);
626
627 return 0;
628}
629
630static void __init free_mod_tags_mem(void)
631{
632 int i;
633
634 module_tags.start_addr = 0;
635 for (i = 0; i < vm_module_tags->nr_pages; i++)
636 __free_page(vm_module_tags->pages[i]);
637 kfree(vm_module_tags->pages);
638 free_vm_area(vm_module_tags);
639}
640
641#else /* CONFIG_MODULES */
642
643static inline int alloc_mod_tags_mem(void) { return 0; }
644static inline void free_mod_tags_mem(void) {}
645
646#endif /* CONFIG_MODULES */
647
648/* See: Documentation/mm/allocation-profiling.rst */
649static int __init setup_early_mem_profiling(char *str)
650{
651 bool compressed = false;
652 bool enable;
653
654 if (!str || !str[0])
655 return -EINVAL;
656
657 if (!strncmp(str, "never", 5)) {
658 enable = false;
659 mem_profiling_support = false;
660 pr_info("Memory allocation profiling is disabled!\n");
661 } else {
662 char *token = strsep(&str, ",");
663
664 if (kstrtobool(token, &enable))
665 return -EINVAL;
666
667 if (str) {
668
669 if (strcmp(str, "compressed"))
670 return -EINVAL;
671
672 compressed = true;
673 }
674 mem_profiling_support = true;
675 pr_info("Memory allocation profiling is enabled %s compression and is turned %s!\n",
676 compressed ? "with" : "without", enable ? "on" : "off");
677 }
678
679 if (enable != mem_alloc_profiling_enabled()) {
680 if (enable)
681 static_branch_enable(&mem_alloc_profiling_key);
682 else
683 static_branch_disable(&mem_alloc_profiling_key);
684 }
685 if (compressed != static_key_enabled(&mem_profiling_compressed)) {
686 if (compressed)
687 static_branch_enable(&mem_profiling_compressed);
688 else
689 static_branch_disable(&mem_profiling_compressed);
690 }
691
692 return 0;
693}
694early_param("sysctl.vm.mem_profiling", setup_early_mem_profiling);
695
696static __init bool need_page_alloc_tagging(void)
697{
698 if (static_key_enabled(&mem_profiling_compressed))
699 return false;
700
701 return mem_profiling_support;
702}
703
704static __init void init_page_alloc_tagging(void)
705{
706}
707
708struct page_ext_operations page_alloc_tagging_ops = {
709 .size = sizeof(union codetag_ref),
710 .need = need_page_alloc_tagging,
711 .init = init_page_alloc_tagging,
712};
713EXPORT_SYMBOL(page_alloc_tagging_ops);
714
715#ifdef CONFIG_SYSCTL
716static struct ctl_table memory_allocation_profiling_sysctls[] = {
717 {
718 .procname = "mem_profiling",
719 .data = &mem_alloc_profiling_key,
720#ifdef CONFIG_MEM_ALLOC_PROFILING_DEBUG
721 .mode = 0444,
722#else
723 .mode = 0644,
724#endif
725 .proc_handler = proc_do_static_key,
726 },
727};
728
729static void __init sysctl_init(void)
730{
731 if (!mem_profiling_support)
732 memory_allocation_profiling_sysctls[0].mode = 0444;
733
734 register_sysctl_init("vm", memory_allocation_profiling_sysctls);
735}
736#else /* CONFIG_SYSCTL */
737static inline void sysctl_init(void) {}
738#endif /* CONFIG_SYSCTL */
739
740static int __init alloc_tag_init(void)
741{
742 const struct codetag_type_desc desc = {
743 .section = ALLOC_TAG_SECTION_NAME,
744 .tag_size = sizeof(struct alloc_tag),
745#ifdef CONFIG_MODULES
746 .needs_section_mem = needs_section_mem,
747 .alloc_section_mem = reserve_module_tags,
748 .free_section_mem = release_module_tags,
749 .module_replaced = replace_module,
750#endif
751 };
752 int res;
753
754 res = alloc_mod_tags_mem();
755 if (res)
756 return res;
757
758 alloc_tag_cttype = codetag_register_type(&desc);
759 if (IS_ERR(alloc_tag_cttype)) {
760 free_mod_tags_mem();
761 return PTR_ERR(alloc_tag_cttype);
762 }
763
764 sysctl_init();
765 procfs_init();
766
767 return 0;
768}
769module_init(alloc_tag_init);