projects/linux·blob master · refs

1367 lines · 35.5 KiB

1// SPDX-License-Identifier: GPL-2.0-only
2#include <linux/alloc_tag.h>
3#include <linux/execmem.h>
4#include <linux/fs.h>
5#include <linux/gfp.h>
6#include <linux/kallsyms.h>
7#include <linux/module.h>
8#include <linux/mutex.h>
9#include <linux/compat.h>
10#include <linux/page_ext.h>
11#include <linux/pgalloc_tag.h>
12#include <linux/proc_fs.h>
13#include <linux/rcupdate.h>
14#include <linux/seq_buf.h>
15#include <linux/seq_file.h>
16#include <linux/string_choices.h>
17#include <linux/vmalloc.h>
18#include <linux/kmemleak.h>
19#include <uapi/linux/alloc_tag.h>
20
21#include "internal.h"
22#include "page_alloc.h"
23
24#define ALLOCINFO_FILE_NAME "allocinfo"
25#define MODULE_ALLOC_TAG_VMAP_SIZE (100000UL * sizeof(struct alloc_tag))
26#define SECTION_START(NAME) (CODETAG_SECTION_START_PREFIX NAME)
27#define SECTION_STOP(NAME) (CODETAG_SECTION_STOP_PREFIX NAME)
28
29#ifdef CONFIG_MEM_ALLOC_PROFILING_ENABLED_BY_DEFAULT
30static bool mem_profiling_support = true;
31#else
32static bool mem_profiling_support;
33#endif
34
35/*
36 * Memory allocation profiling is permanently disabled and cannot be enabled.
37 * Must be called after setup_early_mem_profiling().
38 */
39bool mem_alloc_profiling_permanently_disabled(void)
40{
41 return !mem_profiling_support;
42}
43
44static struct codetag_type *alloc_tag_cttype;
45
46#ifdef CONFIG_ARCH_MODULE_NEEDS_WEAK_PER_CPU
47DEFINE_PER_CPU(struct alloc_tag_counters, _shared_alloc_tag);
48EXPORT_SYMBOL(_shared_alloc_tag);
49#endif
50
51DEFINE_STATIC_KEY_MAYBE(CONFIG_MEM_ALLOC_PROFILING_ENABLED_BY_DEFAULT,
52 mem_alloc_profiling_key);
53EXPORT_SYMBOL(mem_alloc_profiling_key);
54
55DEFINE_STATIC_KEY_FALSE(mem_profiling_compressed);
56
57struct alloc_tag_kernel_section kernel_tags = { NULL, 0 };
58unsigned long alloc_tag_ref_mask;
59int alloc_tag_ref_offs;
60
61struct allocinfo_private {
62 struct codetag_iterator iter;
63 struct codetag_iterator reported_iter;
64 bool print_header;
65 struct allocinfo_filter filter;
66 /* ioctl uses a separate iterator not to interfere with reads */
67 struct codetag_iterator ioctl_iter;
68 bool positioned; /* seq_open_private() sets to 0 */
69 struct mutex ioctl_lock;
70};
71
72static void *allocinfo_start(struct seq_file *m, loff_t *pos)
73{
74 struct allocinfo_private *priv;
75 loff_t node = *pos;
76
77 priv = (struct allocinfo_private *)m->private;
78 codetag_lock_module_list(alloc_tag_cttype);
79 if (node == 0) {
80 priv->print_header = true;
81 priv->iter = codetag_get_ct_iter(alloc_tag_cttype);
82 } else {
83 priv->iter = priv->reported_iter;
84 }
85 codetag_next_ct(&priv->iter);
86 return priv->iter.ct ? priv : NULL;
87}
88
89static void *allocinfo_next(struct seq_file *m, void *arg, loff_t *pos)
90{
91 struct allocinfo_private *priv = (struct allocinfo_private *)arg;
92 struct codetag *ct;
93
94 priv->reported_iter = priv->iter;
95 ct = codetag_next_ct(&priv->iter);
96 (*pos)++;
97 if (!ct)
98 return NULL;
99
100 return priv;
101}
102
103static void allocinfo_stop(struct seq_file *m, void *arg)
104{
105 codetag_unlock_module_list(alloc_tag_cttype);
106}
107
108static void print_allocinfo_header(struct seq_buf *buf)
109{
110 /* Output format version, so we can change it. */
111 seq_buf_printf(buf, "allocinfo - version: 2.0\n");
112 seq_buf_printf(buf, "# <size> <calls> <tag info>\n");
113}
114
115static void alloc_tag_to_text(struct seq_buf *out, struct codetag *ct)
116{
117 struct alloc_tag *tag = ct_to_alloc_tag(ct);
118 struct alloc_tag_counters counter = alloc_tag_read(tag);
119 s64 bytes = counter.bytes;
120
121 seq_buf_printf(out, "%12lli %8llu ", bytes, counter.calls);
122 codetag_to_text(out, ct);
123 if (unlikely(alloc_tag_is_inaccurate(tag)))
124 seq_buf_printf(out, " accurate:no");
125 seq_buf_putc(out, ' ');
126 seq_buf_putc(out, '\n');
127}
128
129static int allocinfo_show(struct seq_file *m, void *arg)
130{
131 struct allocinfo_private *priv = (struct allocinfo_private *)arg;
132 char *bufp;
133 size_t n = seq_get_buf(m, &bufp);
134 struct seq_buf buf;
135
136 seq_buf_init(&buf, bufp, n);
137 if (priv->print_header) {
138 print_allocinfo_header(&buf);
139 priv->print_header = false;
140 }
141 alloc_tag_to_text(&buf, priv->iter.ct);
142 seq_commit(m, seq_buf_used(&buf));
143 return 0;
144}
145
146static const struct seq_operations allocinfo_seq_op = {
147 .start = allocinfo_start,
148 .next = allocinfo_next,
149 .stop = allocinfo_stop,
150 .show = allocinfo_show,
151};
152
153/*
154 * Initializes seq_file operations and allocates private state when opening
155 * the /proc/allocinfo procfs entry.
156 */
157static int allocinfo_open(struct inode *inode, struct file *file)
158{
159 int ret;
160
161 ret = seq_open_private(file, &allocinfo_seq_op,
162 sizeof(struct allocinfo_private));
163 if (!ret) {
164 struct seq_file *m = file->private_data;
165 struct allocinfo_private *priv = m->private;
166
167 mutex_init(&priv->ioctl_lock);
168 }
169 return ret;
170}
171
172/*
173 * Cleans up the seq_file state and frees up the private state allocated in
174 * allocinfo_open() when closing the /proc/allocinfo file descriptor.
175 */
176static int allocinfo_release(struct inode *inode, struct file *file)
177{
178 struct seq_file *m = file->private_data;
179 struct allocinfo_private *priv = m->private;
180
181 mutex_destroy(&priv->ioctl_lock);
182 return seq_release_private(inode, file);
183}
184
185/*
186 * Returns a pointer to the suffix of a string so that its length fits within
187 * ALLOCINFO_STR_SIZE, preserving the trailing characters.
188 * Function, file and module names often have the same prefixes, therefore
189 * when filtering by these criteria, we compare the last 64 characters to
190 * minimize the chances of name collisions
191 */
192static const char *allocinfo_str(const char *str)
193{
194 size_t len = strlen(str);
195
196 /* Keep an extra space for the trailing NULL. */
197 if (len >= ALLOCINFO_STR_SIZE)
198 str += (len - ALLOCINFO_STR_SIZE) + 1;
199 return str;
200}
201
202/* Copy a string and trim from the beginning if it's too long */
203static void allocinfo_copy_str(char *dest, const char *src)
204{
205 strscpy_pad(dest, allocinfo_str(src), ALLOCINFO_STR_SIZE);
206}
207
208/* Compare two strings and only consider the trimmed suffix if s1 is too long */
209static int allocinfo_cmp_str(const char *str, const char *template)
210{
211 return strncmp(allocinfo_str(str), template, ALLOCINFO_STR_SIZE);
212}
213
214/* Fetch the per-CPU counters */
215static inline struct alloc_tag_counters allocinfo_prefetch_counters(struct codetag *ct)
216{
217 return alloc_tag_read(ct_to_alloc_tag(ct));
218}
219
220/*
221 * Populates the UAPI allocinfo_tag_data structure with active runtime
222 * profiling counters extracted from the given kernel codetag.
223 */
224static void allocinfo_to_params(struct codetag *ct,
225 struct allocinfo_tag_data *data,
226 struct alloc_tag_counters *counters)
227{
228 if (ct->modname)
229 allocinfo_copy_str(data->tag.modname, ct->modname);
230 else
231 data->tag.modname[0] = '\0';
232 allocinfo_copy_str(data->tag.function, ct->function);
233 allocinfo_copy_str(data->tag.filename, ct->filename);
234 data->tag.lineno = ct->lineno;
235 data->counter.bytes = counters->bytes;
236 data->counter.calls = counters->calls;
237 data->counter.accurate = !alloc_tag_is_inaccurate(ct_to_alloc_tag(ct));
238}
239
240/*
241 * Retrieves the unique content ID representing the current allocation tag module
242 * layout, allowing userspace to detect if modules were loaded / unloaded.
243 */
244static int allocinfo_ioctl_get_content_id(struct seq_file *m, void __user *arg)
245{
246 struct allocinfo_content_id params;
247
248 codetag_lock_module_list(alloc_tag_cttype);
249 params.id = codetag_get_content_id(alloc_tag_cttype);
250 codetag_unlock_module_list(alloc_tag_cttype);
251 if (copy_to_user(arg, &params, sizeof(params)))
252 return -EFAULT;
253
254 return 0;
255}
256
257/*
258 * Verifies whether a given codetag satisfies the active filtering criteria by
259 * matching its characteristics against the specified filter.
260 */
261static bool matches_filter(struct codetag *ct, struct allocinfo_filter *filter,
262 struct alloc_tag_counters *counters,
263 bool *fetched_counters)
264{
265 bool inaccurate;
266
267 if (!filter || !filter->mask)
268 return true;
269
270 if (filter->mask & ALLOCINFO_FILTER_MASK_MODNAME) {
271 /* user wants to filter by modname but ct->modname is NULL */
272 if (!ct->modname) {
273 /* validate if user was attempting to filter for built-in allocations */
274 if (filter->fields.modname[0] != '\0')
275 return false;
276 } else if (allocinfo_cmp_str(ct->modname, filter->fields.modname))
277 return false;
278 }
279
280 if ((filter->mask & ALLOCINFO_FILTER_MASK_FUNCTION) &&
281 ct->function && allocinfo_cmp_str(ct->function, filter->fields.function))
282 return false;
283
284 if ((filter->mask & ALLOCINFO_FILTER_MASK_FILENAME) &&
285 ct->filename && allocinfo_cmp_str(ct->filename, filter->fields.filename))
286 return false;
287
288 if ((filter->mask & ALLOCINFO_FILTER_MASK_LINENO) &&
289 ct->lineno != filter->fields.lineno)
290 return false;
291
292 if (filter->mask & ALLOCINFO_FILTER_MASK_INACCURATE) {
293 inaccurate = !!(ct->flags & CODETAG_FLAG_INACCURATE);
294 if (inaccurate != !!(filter->inaccurate))
295 return false;
296 }
297
298 if (filter->mask & (ALLOCINFO_FILTER_MASK_MIN_SIZE | ALLOCINFO_FILTER_MASK_MAX_SIZE)) {
299 if (!*fetched_counters) {
300 *counters = allocinfo_prefetch_counters(ct);
301 *fetched_counters = true;
302 }
303 if ((filter->mask & ALLOCINFO_FILTER_MASK_MIN_SIZE) &&
304 counters->bytes < filter->min_size)
305 return false;
306 if ((filter->mask & ALLOCINFO_FILTER_MASK_MAX_SIZE) &&
307 counters->bytes > filter->max_size)
308 return false;
309 }
310
311 return true;
312}
313
314/*
315 * Seeks the ioctl iterator to the specified 0-indexed tag position, reads its
316 * profiling data and returns it to userspace.
317 */
318static int allocinfo_ioctl_get_at(struct seq_file *m, void __user *arg)
319{
320 struct allocinfo_private *priv;
321 struct codetag *ct;
322 struct allocinfo_get_at params = {0};
323 __u64 skip_count;
324 struct alloc_tag_counters counters;
325 bool fetched_counters;
326
327 if (copy_from_user(&params, arg, sizeof(params)))
328 return -EFAULT;
329
330 if (params.filter.mask & ~ALLOCINFO_FILTER_MASKS)
331 return -EINVAL;
332
333 if ((params.filter.mask & ALLOCINFO_FILTER_MASK_MIN_SIZE) &&
334 (params.filter.mask & ALLOCINFO_FILTER_MASK_MAX_SIZE) &&
335 params.filter.min_size > params.filter.max_size)
336 return -EINVAL;
337
338 priv = m->private;
339
340 mutex_lock(&priv->ioctl_lock);
341 codetag_lock_module_list(alloc_tag_cttype);
342
343 if (params.pos >= codetag_get_count(alloc_tag_cttype)) {
344 codetag_unlock_module_list(alloc_tag_cttype);
345 mutex_unlock(&priv->ioctl_lock);
346 return -ENOENT;
347 }
348
349 skip_count = params.pos;
350
351 if (params.filter.mask)
352 priv->filter = params.filter;
353 else
354 priv->filter.mask = 0;
355
356 /* Find the codetag */
357 priv->ioctl_iter = codetag_get_ct_iter(alloc_tag_cttype);
358 ct = codetag_next_ct(&priv->ioctl_iter);
359
360 while (ct) {
361 fetched_counters = false;
362 if (matches_filter(ct, &priv->filter, &counters, &fetched_counters)) {
363 if (skip_count == 0)
364 break;
365 skip_count--;
366 }
367 ct = codetag_next_ct(&priv->ioctl_iter);
368 }
369
370 if (ct) {
371 if (!fetched_counters)
372 counters = allocinfo_prefetch_counters(ct);
373 allocinfo_to_params(ct, &params.data, &counters);
374 priv->positioned = true;
375 }
376
377 codetag_unlock_module_list(alloc_tag_cttype);
378 mutex_unlock(&priv->ioctl_lock);
379
380 if (!ct)
381 return -ENOENT;
382
383 if (copy_to_user(arg, &params, sizeof(params)))
384 return -EFAULT;
385
386 return 0;
387}
388
389/*
390 * Advances the ioctl iterator to the next allocation tag in the sequence and
391 * returns its profiling data to userspace.
392 */
393static int allocinfo_ioctl_get_next(struct seq_file *m, void __user *arg)
394{
395 struct allocinfo_private *priv;
396 struct codetag *ct;
397 struct allocinfo_tag_data params;
398 int ret = 0;
399 struct alloc_tag_counters counters;
400 bool fetched_counters;
401
402 memset(&params, 0, sizeof(params));
403 priv = m->private;
404
405 mutex_lock(&priv->ioctl_lock);
406 codetag_lock_module_list(alloc_tag_cttype);
407
408 if (!priv->positioned) {
409 priv->ioctl_iter = codetag_get_ct_iter(alloc_tag_cttype);
410 priv->positioned = true;
411 }
412
413 ct = codetag_next_ct(&priv->ioctl_iter);
414 while (ct) {
415 fetched_counters = false;
416 if (matches_filter(ct, &priv->filter, &counters, &fetched_counters))
417 break;
418 ct = codetag_next_ct(&priv->ioctl_iter);
419 }
420
421 if (ct) {
422 if (!fetched_counters)
423 counters = allocinfo_prefetch_counters(ct);
424 allocinfo_to_params(ct, &params, &counters);
425 }
426 if (!ct) {
427 priv->positioned = false;
428 ret = -ENOENT;
429 }
430 codetag_unlock_module_list(alloc_tag_cttype);
431 mutex_unlock(&priv->ioctl_lock);
432
433 if (ret == 0) {
434 if (copy_to_user(arg, &params, sizeof(params)))
435 return -EFAULT;
436 }
437 return ret;
438}
439
440/*
441 * Entry point ioctl function for /proc/allocinfo routing requests to fetch the
442 * layout content ID, seek to a specific tag, or read sequential tags.
443 */
444static long allocinfo_ioctl(struct file *file, unsigned int cmd,
445 unsigned long __arg)
446{
447 void __user *arg = (void __user *)__arg;
448 int ret;
449
450 switch (cmd) {
451 case ALLOCINFO_IOC_CONTENT_ID:
452 ret = allocinfo_ioctl_get_content_id(file->private_data, arg);
453 break;
454 case ALLOCINFO_IOC_GET_AT:
455 ret = allocinfo_ioctl_get_at(file->private_data, arg);
456 break;
457 case ALLOCINFO_IOC_GET_NEXT:
458 ret = allocinfo_ioctl_get_next(file->private_data, arg);
459 break;
460 default:
461 ret = -ENOIOCTLCMD;
462 break;
463 }
464
465 return ret;
466}
467
468#ifdef CONFIG_COMPAT
469static long allocinfo_compat_ioctl(struct file *file, unsigned int cmd,
470 unsigned long arg)
471{
472 return allocinfo_ioctl(file, cmd, (unsigned long)compat_ptr(arg));
473}
474#endif
475
476static const struct proc_ops allocinfo_proc_ops = {
477 .proc_open = allocinfo_open,
478 .proc_read_iter = seq_read_iter,
479 .proc_lseek = seq_lseek,
480 .proc_release = allocinfo_release,
481 .proc_ioctl = allocinfo_ioctl,
482#ifdef CONFIG_COMPAT
483 .proc_compat_ioctl = allocinfo_compat_ioctl,
484#endif
485};
486
487size_t alloc_tag_top_users(struct codetag_bytes *tags, size_t count, bool can_sleep)
488{
489 struct codetag_iterator iter;
490 struct codetag *ct;
491 struct codetag_bytes n;
492 unsigned int i, nr = 0;
493
494 if (IS_ERR_OR_NULL(alloc_tag_cttype))
495 return 0;
496
497 if (can_sleep)
498 codetag_lock_module_list(alloc_tag_cttype);
499 else if (!codetag_trylock_module_list(alloc_tag_cttype))
500 return 0;
501
502 iter = codetag_get_ct_iter(alloc_tag_cttype);
503 while ((ct = codetag_next_ct(&iter))) {
504 struct alloc_tag_counters counter = alloc_tag_read(ct_to_alloc_tag(ct));
505
506 n.ct = ct;
507 n.bytes = counter.bytes;
508
509 for (i = 0; i < nr; i++)
510 if (n.bytes > tags[i].bytes)
511 break;
512
513 if (i < count) {
514 nr -= nr == count;
515 memmove(&tags[i + 1],
516 &tags[i],
517 sizeof(tags[0]) * (nr - i));
518 nr++;
519 tags[i] = n;
520 }
521 }
522
523 codetag_unlock_module_list(alloc_tag_cttype);
524
525 return nr;
526}
527
528void pgalloc_tag_split(struct folio *folio, int old_order, int new_order)
529{
530 int i;
531 struct alloc_tag *tag;
532 unsigned int nr_pages = 1 << new_order;
533
534 if (!mem_alloc_profiling_enabled())
535 return;
536
537 tag = __pgalloc_tag_get(&folio->page);
538 if (!tag)
539 return;
540
541 for (i = nr_pages; i < (1 << old_order); i += nr_pages) {
542 union pgtag_ref_handle handle;
543 union codetag_ref ref;
544
545 if (get_page_tag_ref(folio_page(folio, i), &ref, &handle)) {
546 /* Set new reference to point to the original tag */
547 alloc_tag_ref_set(&ref, tag);
548 update_page_tag_ref(handle, &ref);
549 put_page_tag_ref(handle);
550 }
551 }
552}
553
554void pgalloc_tag_swap(struct folio *new, struct folio *old)
555{
556 union pgtag_ref_handle handle_old, handle_new;
557 union codetag_ref ref_old, ref_new;
558 struct alloc_tag *tag_old, *tag_new;
559
560 if (!mem_alloc_profiling_enabled())
561 return;
562
563 tag_old = __pgalloc_tag_get(&old->page);
564 if (!tag_old)
565 return;
566 tag_new = __pgalloc_tag_get(&new->page);
567 if (!tag_new)
568 return;
569
570 if (!get_page_tag_ref(&old->page, &ref_old, &handle_old))
571 return;
572 if (!get_page_tag_ref(&new->page, &ref_new, &handle_new)) {
573 put_page_tag_ref(handle_old);
574 return;
575 }
576
577 /*
578 * Clear tag references to avoid debug warning when using
579 * __alloc_tag_ref_set() with non-empty reference.
580 */
581 set_codetag_empty(&ref_old);
582 set_codetag_empty(&ref_new);
583
584 /* swap tags */
585 __alloc_tag_ref_set(&ref_old, tag_new);
586 update_page_tag_ref(handle_old, &ref_old);
587 __alloc_tag_ref_set(&ref_new, tag_old);
588 update_page_tag_ref(handle_new, &ref_new);
589
590 put_page_tag_ref(handle_old);
591 put_page_tag_ref(handle_new);
592}
593
594static void shutdown_mem_profiling(bool remove_file)
595{
596 if (mem_alloc_profiling_enabled())
597 static_branch_disable(&mem_alloc_profiling_key);
598
599 if (!mem_profiling_support)
600 return;
601
602 if (remove_file)
603 remove_proc_entry(ALLOCINFO_FILE_NAME, NULL);
604 mem_profiling_support = false;
605}
606
607void __init alloc_tag_sec_init(void)
608{
609 struct alloc_tag *last_codetag;
610
611 if (!mem_profiling_support)
612 return;
613
614 if (!static_key_enabled(&mem_profiling_compressed))
615 return;
616
617 kernel_tags.first_tag = (struct alloc_tag *)kallsyms_lookup_name(
618 SECTION_START(ALLOC_TAG_SECTION_NAME));
619 last_codetag = (struct alloc_tag *)kallsyms_lookup_name(
620 SECTION_STOP(ALLOC_TAG_SECTION_NAME));
621 kernel_tags.count = last_codetag - kernel_tags.first_tag;
622
623 /* Check if kernel tags fit into page flags */
624 if (kernel_tags.count > (1UL << NR_UNUSED_PAGEFLAG_BITS)) {
625 shutdown_mem_profiling(false); /* allocinfo file does not exist yet */
626 pr_err("%lu allocation tags cannot be references using %d available page flag bits. Memory allocation profiling is disabled!\n",
627 kernel_tags.count, NR_UNUSED_PAGEFLAG_BITS);
628 return;
629 }
630
631 alloc_tag_ref_offs = (LRU_REFS_PGOFF - NR_UNUSED_PAGEFLAG_BITS);
632 alloc_tag_ref_mask = ((1UL << NR_UNUSED_PAGEFLAG_BITS) - 1);
633 pr_debug("Memory allocation profiling compression is using %d page flag bits!\n",
634 NR_UNUSED_PAGEFLAG_BITS);
635}
636
637#ifdef CONFIG_MODULES
638
639static struct maple_tree mod_area_mt = MTREE_INIT(mod_area_mt, MT_FLAGS_ALLOC_RANGE);
640static struct vm_struct *vm_module_tags;
641/* A dummy object used to indicate an unloaded module */
642static struct module unloaded_mod;
643/* A dummy object used to indicate a module prepended area */
644static struct module prepend_mod;
645
646struct alloc_tag_module_section module_tags;
647
648static inline unsigned long alloc_tag_align(unsigned long val)
649{
650 if (!static_key_enabled(&mem_profiling_compressed)) {
651 /* No alignment requirements when we are not indexing the tags */
652 return val;
653 }
654
655 if (val % sizeof(struct alloc_tag) == 0)
656 return val;
657 return ((val / sizeof(struct alloc_tag)) + 1) * sizeof(struct alloc_tag);
658}
659
660static bool ensure_alignment(unsigned long align, unsigned int *prepend)
661{
662 if (!static_key_enabled(&mem_profiling_compressed)) {
663 /* No alignment requirements when we are not indexing the tags */
664 return true;
665 }
666
667 /*
668 * If alloc_tag size is not a multiple of required alignment, tag
669 * indexing does not work.
670 */
671 if (!IS_ALIGNED(sizeof(struct alloc_tag), align))
672 return false;
673
674 /* Ensure prepend consumes multiple of alloc_tag-sized blocks */
675 if (*prepend)
676 *prepend = alloc_tag_align(*prepend);
677
678 return true;
679}
680
681static inline bool tags_addressable(void)
682{
683 unsigned long tag_idx_count;
684
685 if (!static_key_enabled(&mem_profiling_compressed))
686 return true; /* with page_ext tags are always addressable */
687
688 tag_idx_count = CODETAG_ID_FIRST + kernel_tags.count +
689 module_tags.size / sizeof(struct alloc_tag);
690
691 return tag_idx_count < (1UL << NR_UNUSED_PAGEFLAG_BITS);
692}
693
694static bool needs_section_mem(struct module *mod, unsigned long size)
695{
696 if (!mem_profiling_support)
697 return false;
698
699 return size >= sizeof(struct alloc_tag);
700}
701
702static bool clean_unused_counters(struct alloc_tag *start_tag,
703 struct alloc_tag *end_tag)
704{
705 struct alloc_tag *tag;
706 bool ret = true;
707
708 for (tag = start_tag; tag <= end_tag; tag++) {
709 struct alloc_tag_counters counter;
710
711 if (!tag->counters)
712 continue;
713
714 counter = alloc_tag_read(tag);
715 if (!counter.bytes) {
716 free_percpu(tag->counters);
717 tag->counters = NULL;
718 } else {
719 ret = false;
720 }
721 }
722
723 return ret;
724}
725
726/* Called with mod_area_mt locked */
727static void clean_unused_module_areas_locked(void)
728{
729 MA_STATE(mas, &mod_area_mt, 0, module_tags.size);
730 struct module *val;
731
732 mas_for_each(&mas, val, module_tags.size) {
733 struct alloc_tag *start_tag;
734 struct alloc_tag *end_tag;
735
736 if (val != &unloaded_mod)
737 continue;
738
739 /* Release area if all tags are unused */
740 start_tag = (struct alloc_tag *)(module_tags.start_addr + mas.index);
741 end_tag = (struct alloc_tag *)(module_tags.start_addr + mas.last);
742 if (clean_unused_counters(start_tag, end_tag))
743 mas_erase(&mas);
744 }
745}
746
747/* Called with mod_area_mt locked */
748static bool find_aligned_area(struct ma_state *mas, unsigned long section_size,
749 unsigned long size, unsigned int prepend, unsigned long align)
750{
751 bool cleanup_done = false;
752
753repeat:
754 /* Try finding exact size and hope the start is aligned */
755 if (!mas_empty_area(mas, 0, section_size - 1, prepend + size)) {
756 if (IS_ALIGNED(mas->index + prepend, align))
757 return true;
758
759 /* Try finding larger area to align later */
760 mas_reset(mas);
761 if (!mas_empty_area(mas, 0, section_size - 1,
762 size + prepend + align - 1))
763 return true;
764 }
765
766 /* No free area, try cleanup stale data and repeat the search once */
767 if (!cleanup_done) {
768 clean_unused_module_areas_locked();
769 cleanup_done = true;
770 mas_reset(mas);
771 goto repeat;
772 }
773
774 return false;
775}
776
777static int vm_module_tags_populate(void)
778{
779 unsigned long phys_end = ALIGN_DOWN(module_tags.start_addr, PAGE_SIZE) +
780 (vm_module_tags->nr_pages << PAGE_SHIFT);
781 unsigned long new_end = module_tags.start_addr + module_tags.size;
782
783 if (phys_end < new_end) {
784 struct page **next_page = vm_module_tags->pages + vm_module_tags->nr_pages;
785 unsigned long old_shadow_end = ALIGN(phys_end, MODULE_ALIGN);
786 unsigned long new_shadow_end = ALIGN(new_end, MODULE_ALIGN);
787 unsigned long more_pages;
788 unsigned long nr = 0;
789
790 more_pages = ALIGN(new_end - phys_end, PAGE_SIZE) >> PAGE_SHIFT;
791 while (nr < more_pages) {
792 unsigned long allocated;
793
794 allocated = alloc_pages_bulk_node(GFP_KERNEL | __GFP_NOWARN,
795 NUMA_NO_NODE, more_pages - nr, next_page + nr);
796
797 if (!allocated)
798 break;
799 nr += allocated;
800 }
801
802 if (nr < more_pages ||
803 vmap_pages_range(phys_end, phys_end + (nr << PAGE_SHIFT), PAGE_KERNEL,
804 next_page, PAGE_SHIFT) < 0) {
805 release_pages_arg arg = { .pages = next_page };
806
807 /* Clean up and error out */
808 release_pages(arg, nr);
809 return -ENOMEM;
810 }
811
812 vm_module_tags->nr_pages += nr;
813
814 /*
815 * Kasan allocates 1 byte of shadow for every 8 bytes of data.
816 * When kasan_alloc_module_shadow allocates shadow memory,
817 * its unit of allocation is a page.
818 * Therefore, here we need to align to MODULE_ALIGN.
819 */
820 if (old_shadow_end < new_shadow_end)
821 kasan_alloc_module_shadow((void *)old_shadow_end,
822 new_shadow_end - old_shadow_end,
823 GFP_KERNEL);
824 }
825
826 /*
827 * Mark the pages as accessible, now that they are mapped.
828 * With hardware tag-based KASAN, marking is skipped for
829 * non-VM_ALLOC mappings, see __kasan_unpoison_vmalloc().
830 */
831 kasan_unpoison_vmalloc((void *)module_tags.start_addr,
832 new_end - module_tags.start_addr,
833 KASAN_VMALLOC_PROT_NORMAL);
834
835 return 0;
836}
837
838static void *reserve_module_tags(struct module *mod, unsigned long size,
839 unsigned int prepend, unsigned long align)
840{
841 unsigned long section_size = module_tags.end_addr - module_tags.start_addr;
842 MA_STATE(mas, &mod_area_mt, 0, section_size - 1);
843 unsigned long offset;
844 void *ret = NULL;
845
846 /* If no tags return error */
847 if (size < sizeof(struct alloc_tag))
848 return ERR_PTR(-EINVAL);
849
850 /*
851 * align is always power of 2, so we can use IS_ALIGNED and ALIGN.
852 * align 0 or 1 means no alignment, to simplify set to 1.
853 */
854 if (!align)
855 align = 1;
856
857 if (!ensure_alignment(align, &prepend)) {
858 shutdown_mem_profiling(true);
859 pr_err("%s: alignment %lu is incompatible with allocation tag indexing. Memory allocation profiling is disabled!\n",
860 mod->name, align);
861 return ERR_PTR(-EINVAL);
862 }
863
864 mas_lock(&mas);
865 if (!find_aligned_area(&mas, section_size, size, prepend, align)) {
866 ret = ERR_PTR(-ENOMEM);
867 goto unlock;
868 }
869
870 /* Mark found area as reserved */
871 offset = mas.index;
872 offset += prepend;
873 offset = ALIGN(offset, align);
874 if (offset != mas.index) {
875 unsigned long pad_start = mas.index;
876
877 mas.last = offset - 1;
878 mas_store(&mas, &prepend_mod);
879 if (mas_is_err(&mas)) {
880 ret = ERR_PTR(xa_err(mas.node));
881 goto unlock;
882 }
883 mas.index = offset;
884 mas.last = offset + size - 1;
885 mas_store(&mas, mod);
886 if (mas_is_err(&mas)) {
887 mas.index = pad_start;
888 mas_erase(&mas);
889 ret = ERR_PTR(xa_err(mas.node));
890 }
891 } else {
892 mas.last = offset + size - 1;
893 mas_store(&mas, mod);
894 if (mas_is_err(&mas))
895 ret = ERR_PTR(xa_err(mas.node));
896 }
897unlock:
898 mas_unlock(&mas);
899
900 if (IS_ERR(ret))
901 return ret;
902
903 if (module_tags.size < offset + size) {
904 int grow_res;
905
906 module_tags.size = offset + size;
907 if (mem_alloc_profiling_enabled() && !tags_addressable()) {
908 shutdown_mem_profiling(true);
909 pr_warn("With module %s there are too many tags to fit in %d page flag bits. Memory allocation profiling is disabled!\n",
910 mod->name, NR_UNUSED_PAGEFLAG_BITS);
911 }
912
913 grow_res = vm_module_tags_populate();
914 if (grow_res) {
915 shutdown_mem_profiling(true);
916 pr_err("Failed to allocate memory for allocation tags in the module %s. Memory allocation profiling is disabled!\n",
917 mod->name);
918 return ERR_PTR(grow_res);
919 }
920 }
921
922 return (struct alloc_tag *)(module_tags.start_addr + offset);
923}
924
925static void release_module_tags(struct module *mod, bool used)
926{
927 MA_STATE(mas, &mod_area_mt, module_tags.size, module_tags.size);
928 struct alloc_tag *start_tag;
929 struct alloc_tag *end_tag;
930 struct module *val;
931
932 mas_lock(&mas);
933 mas_for_each_rev(&mas, val, 0)
934 if (val == mod)
935 break;
936
937 if (!val) /* module not found */
938 goto out;
939
940 if (!used)
941 goto release_area;
942
943 start_tag = (struct alloc_tag *)(module_tags.start_addr + mas.index);
944 end_tag = (struct alloc_tag *)(module_tags.start_addr + mas.last);
945 if (!clean_unused_counters(start_tag, end_tag)) {
946 struct alloc_tag *tag;
947
948 for (tag = start_tag; tag <= end_tag; tag++) {
949 struct alloc_tag_counters counter;
950
951 if (!tag->counters)
952 continue;
953
954 counter = alloc_tag_read(tag);
955 pr_info("%s:%u module %s func:%s has %llu allocated at module unload\n",
956 tag->ct.filename, tag->ct.lineno, tag->ct.modname,
957 tag->ct.function, counter.bytes);
958 }
959 } else {
960 used = false;
961 }
962release_area:
963 mas_store(&mas, used ? &unloaded_mod : NULL);
964 val = mas_prev_range(&mas, 0);
965 if (val == &prepend_mod)
966 mas_store(&mas, NULL);
967out:
968 mas_unlock(&mas);
969}
970
971static int load_module(struct module *mod, struct codetag *start, struct codetag *stop)
972{
973 /* Allocate module alloc_tag percpu counters */
974 struct alloc_tag *start_tag;
975 struct alloc_tag *stop_tag;
976 struct alloc_tag *tag;
977
978 /* percpu counters for core allocations are already statically allocated */
979 if (!mod)
980 return 0;
981
982 start_tag = ct_to_alloc_tag(start);
983 stop_tag = ct_to_alloc_tag(stop);
984 for (tag = start_tag; tag < stop_tag; tag++) {
985 WARN_ON(tag->counters);
986 tag->counters = alloc_percpu(struct alloc_tag_counters);
987 if (!tag->counters) {
988 while (--tag >= start_tag) {
989 free_percpu(tag->counters);
990 tag->counters = NULL;
991 }
992 pr_err("Failed to allocate memory for allocation tag percpu counters in the module %s\n",
993 mod->name);
994 return -ENOMEM;
995 }
996
997 /*
998 * Avoid a kmemleak false positive. The pointer to the counters is stored
999 * in the alloc_tag section of the module and cannot be directly accessed.
1000 */
1001 kmemleak_ignore_percpu(tag->counters);
1002 }
1003 return 0;
1004}
1005
1006static void replace_module(struct module *mod, struct module *new_mod)
1007{
1008 MA_STATE(mas, &mod_area_mt, 0, module_tags.size);
1009 struct module *val;
1010
1011 mas_lock(&mas);
1012 mas_for_each(&mas, val, module_tags.size) {
1013 if (val != mod)
1014 continue;
1015
1016 mas_store_gfp(&mas, new_mod, GFP_KERNEL);
1017 break;
1018 }
1019 mas_unlock(&mas);
1020}
1021
1022static int __init alloc_mod_tags_mem(void)
1023{
1024 /* Map space to copy allocation tags */
1025 vm_module_tags = execmem_vmap(MODULE_ALLOC_TAG_VMAP_SIZE);
1026 if (!vm_module_tags) {
1027 pr_err("Failed to map %lu bytes for module allocation tags\n",
1028 MODULE_ALLOC_TAG_VMAP_SIZE);
1029 module_tags.start_addr = 0;
1030 return -ENOMEM;
1031 }
1032
1033 vm_module_tags->pages = kmalloc_objs(struct page *,
1034 get_vm_area_size(vm_module_tags) >> PAGE_SHIFT,
1035 GFP_KERNEL | __GFP_ZERO);
1036 if (!vm_module_tags->pages) {
1037 free_vm_area(vm_module_tags);
1038 return -ENOMEM;
1039 }
1040
1041 module_tags.start_addr = (unsigned long)vm_module_tags->addr;
1042 module_tags.end_addr = module_tags.start_addr + MODULE_ALLOC_TAG_VMAP_SIZE;
1043 /* Ensure the base is alloc_tag aligned when required for indexing */
1044 module_tags.start_addr = alloc_tag_align(module_tags.start_addr);
1045
1046 return 0;
1047}
1048
1049static void __init free_mod_tags_mem(void)
1050{
1051 release_pages_arg arg = { .pages = vm_module_tags->pages };
1052
1053 module_tags.start_addr = 0;
1054 release_pages(arg, vm_module_tags->nr_pages);
1055 kfree(vm_module_tags->pages);
1056 free_vm_area(vm_module_tags);
1057}
1058
1059#else /* CONFIG_MODULES */
1060
1061static inline int alloc_mod_tags_mem(void) { return 0; }
1062static inline void free_mod_tags_mem(void) {}
1063
1064#endif /* CONFIG_MODULES */
1065
1066/* See: Documentation/mm/allocation-profiling.rst */
1067static int __init setup_early_mem_profiling(char *str)
1068{
1069 bool compressed = false;
1070 bool enable;
1071
1072 if (!str || !str[0])
1073 return -EINVAL;
1074
1075 if (!strncmp(str, "never", 5)) {
1076 enable = false;
1077 mem_profiling_support = false;
1078 pr_info("Memory allocation profiling is disabled!\n");
1079 } else {
1080 char *token = strsep(&str, ",");
1081
1082 if (kstrtobool(token, &enable))
1083 return -EINVAL;
1084
1085 if (str) {
1086
1087 if (strcmp(str, "compressed"))
1088 return -EINVAL;
1089
1090 compressed = true;
1091 }
1092 mem_profiling_support = true;
1093 pr_info("Memory allocation profiling is enabled %s compression and is turned %s!\n",
1094 compressed ? "with" : "without", str_on_off(enable));
1095 }
1096
1097 if (enable != mem_alloc_profiling_enabled()) {
1098 if (enable)
1099 static_branch_enable(&mem_alloc_profiling_key);
1100 else
1101 static_branch_disable(&mem_alloc_profiling_key);
1102 }
1103 if (compressed != static_key_enabled(&mem_profiling_compressed)) {
1104 if (compressed)
1105 static_branch_enable(&mem_profiling_compressed);
1106 else
1107 static_branch_disable(&mem_profiling_compressed);
1108 }
1109
1110 return 0;
1111}
1112early_param("sysctl.vm.mem_profiling", setup_early_mem_profiling);
1113
1114static __init bool need_page_alloc_tagging(void)
1115{
1116 if (static_key_enabled(&mem_profiling_compressed))
1117 return false;
1118
1119 return mem_profiling_support;
1120}
1121
1122#ifdef CONFIG_MEM_ALLOC_PROFILING_DEBUG
1123/*
1124 * Track page allocations before page_ext is initialized.
1125 * Some pages are allocated before page_ext becomes available, leaving
1126 * their codetag uninitialized. Track these early PFNs so we can clear
1127 * their codetag refs later to avoid warnings when they are freed.
1128 *
1129 * Each page is cast to a pfn_pool: the first few bytes hold metadata
1130 * (next pointer and slot count), the remainder stores PFNs.
1131 */
1132struct pfn_pool {
1133 struct pfn_pool *next;
1134 atomic_t count;
1135 unsigned long pfns[];
1136};
1137
1138#define PFN_POOL_SIZE ((PAGE_SIZE - offsetof(struct pfn_pool, pfns)) / \
1139 sizeof(unsigned long))
1140static struct pfn_pool *current_pfn_pool __initdata;
1141
1142static void __init __alloc_tag_add_early_pfn(unsigned long pfn)
1143{
1144 struct pfn_pool *pool;
1145 int idx;
1146
1147 do {
1148 pool = READ_ONCE(current_pfn_pool);
1149 if (!pool || atomic_read(&pool->count) >= PFN_POOL_SIZE) {
1150 struct page *new_page = __alloc_pages(__GFP_HIGH, 0, numa_mem_id(),
1151 NULL, ALLOC_NO_CODETAG);
1152 struct pfn_pool *new;
1153
1154 if (!new_page) {
1155 pr_warn_once("early PFN tracking page allocation failed\n");
1156 return;
1157 }
1158 new = page_address(new_page);
1159 new->next = pool;
1160 atomic_set(&new->count, 0);
1161 if (cmpxchg(&current_pfn_pool, pool, new) != pool) {
1162 clear_page_tag_ref(new_page);
1163 __free_page(new_page);
1164 continue;
1165 }
1166 pool = new;
1167 }
1168 idx = atomic_read(&pool->count);
1169 if (idx >= PFN_POOL_SIZE)
1170 continue;
1171 if (atomic_cmpxchg(&pool->count, idx, idx + 1) == idx)
1172 break;
1173 } while (1);
1174
1175 pool->pfns[idx] = pfn;
1176}
1177
1178typedef void alloc_tag_add_func(unsigned long pfn);
1179static alloc_tag_add_func __rcu *alloc_tag_add_early_pfn_ptr __refdata =
1180 RCU_INITIALIZER(__alloc_tag_add_early_pfn);
1181
1182void alloc_tag_add_early_pfn(unsigned long pfn, unsigned int alloc_flags)
1183{
1184 alloc_tag_add_func *alloc_tag_add;
1185
1186 if (static_key_enabled(&mem_profiling_compressed))
1187 return;
1188
1189 /* Skip allocations for the tracking list itself to avoid recursion. */
1190 if (alloc_flags & ALLOC_NO_CODETAG)
1191 return;
1192
1193 rcu_read_lock();
1194 alloc_tag_add = rcu_dereference(alloc_tag_add_early_pfn_ptr);
1195 if (alloc_tag_add)
1196 alloc_tag_add(pfn);
1197 rcu_read_unlock();
1198}
1199
1200static void __init clear_early_alloc_pfn_tag_refs(void)
1201{
1202 struct pfn_pool *pool, *next;
1203 struct page *page;
1204 int i;
1205
1206 if (static_key_enabled(&mem_profiling_compressed))
1207 return;
1208
1209 rcu_assign_pointer(alloc_tag_add_early_pfn_ptr, NULL);
1210 /* Make sure we are not racing with __alloc_tag_add_early_pfn() */
1211 synchronize_rcu();
1212
1213 for (pool = current_pfn_pool; pool; pool = next) {
1214 int nr_pfns = atomic_read(&pool->count);
1215
1216 for (i = 0; i < nr_pfns; i++) {
1217 unsigned long pfn = pool->pfns[i];
1218
1219 if (pfn_valid(pfn)) {
1220 union pgtag_ref_handle handle;
1221 union codetag_ref ref;
1222
1223 if (get_page_tag_ref(pfn_to_page(pfn), &ref, &handle)) {
1224 /*
1225 * An early-allocated page could be freed and reallocated
1226 * after its page_ext is initialized but before we clear it.
1227 * In that case, it already has a valid tag set.
1228 * We should not overwrite that valid tag
1229 * with CODETAG_EMPTY.
1230 *
1231 * Note: there is still a small race window between checking
1232 * ref.ct and calling set_codetag_empty(). We accept this
1233 * race as it's unlikely and the extra complexity of atomic
1234 * cmpxchg is not worth it for this debug-only code path.
1235 */
1236 if (ref.ct) {
1237 put_page_tag_ref(handle);
1238 continue;
1239 }
1240
1241 set_codetag_empty(&ref);
1242 update_page_tag_ref(handle, &ref);
1243 put_page_tag_ref(handle);
1244 }
1245 }
1246 }
1247
1248 next = pool->next;
1249 page = virt_to_page(pool);
1250 clear_page_tag_ref(page);
1251 __free_page(page);
1252 }
1253}
1254#else /* !CONFIG_MEM_ALLOC_PROFILING_DEBUG */
1255static inline void __init clear_early_alloc_pfn_tag_refs(void) {}
1256#endif /* CONFIG_MEM_ALLOC_PROFILING_DEBUG */
1257
1258static __init void init_page_alloc_tagging(void)
1259{
1260 clear_early_alloc_pfn_tag_refs();
1261}
1262
1263struct page_ext_operations page_alloc_tagging_ops = {
1264 .size = sizeof(union codetag_ref),
1265 .need = need_page_alloc_tagging,
1266 .init = init_page_alloc_tagging,
1267};
1268EXPORT_SYMBOL(page_alloc_tagging_ops);
1269
1270#ifdef CONFIG_SYSCTL
1271/*
1272 * Not using proc_do_static_key() directly to prevent enabling profiling
1273 * after it was shut down.
1274 */
1275static int proc_mem_profiling_handler(const struct ctl_table *table, int write,
1276 void *buffer, size_t *lenp, loff_t *ppos)
1277{
1278 if (write) {
1279 /*
1280 * Call from do_sysctl_args() which is a no-op since the same
1281 * value was already set by setup_early_mem_profiling.
1282 * Return success to avoid warnings from do_sysctl_args().
1283 */
1284 if (!current->mm)
1285 return 0;
1286
1287#ifdef CONFIG_MEM_ALLOC_PROFILING_DEBUG
1288 /* User can't toggle profiling while debugging */
1289 return -EACCES;
1290#endif
1291 if (!mem_profiling_support)
1292 return -EINVAL;
1293 }
1294
1295 return proc_do_static_key(table, write, buffer, lenp, ppos);
1296}
1297
1298
1299static const struct ctl_table memory_allocation_profiling_sysctls[] = {
1300 {
1301 .procname = "mem_profiling",
1302 .data = &mem_alloc_profiling_key,
1303 .mode = 0644,
1304 .proc_handler = proc_mem_profiling_handler,
1305 },
1306 {
1307 .procname = "mem_profiling_compressed",
1308 .data = &mem_profiling_compressed,
1309 .mode = 0444,
1310 .proc_handler = proc_do_static_key,
1311 },
1312};
1313
1314static void __init sysctl_init(void)
1315{
1316 register_sysctl_init("vm", memory_allocation_profiling_sysctls);
1317}
1318#else /* CONFIG_SYSCTL */
1319static inline void sysctl_init(void) {}
1320#endif /* CONFIG_SYSCTL */
1321
1322static int __init alloc_tag_init(void)
1323{
1324 const struct codetag_type_desc desc = {
1325 .section = ALLOC_TAG_SECTION_NAME,
1326 .tag_size = sizeof(struct alloc_tag),
1327#ifdef CONFIG_MODULES
1328 .needs_section_mem = needs_section_mem,
1329 .alloc_section_mem = reserve_module_tags,
1330 .free_section_mem = release_module_tags,
1331 .module_load = load_module,
1332 .module_replaced = replace_module,
1333#endif
1334 };
1335 int res;
1336
1337 sysctl_init();
1338
1339 if (!mem_profiling_support) {
1340 pr_info("Memory allocation profiling is not supported!\n");
1341 return 0;
1342 }
1343
1344 if (!proc_create(ALLOCINFO_FILE_NAME, 0400, NULL, &allocinfo_proc_ops)) {
1345 pr_err("Failed to create %s file\n", ALLOCINFO_FILE_NAME);
1346 shutdown_mem_profiling(false);
1347 return -ENOMEM;
1348 }
1349
1350 res = alloc_mod_tags_mem();
1351 if (res) {
1352 pr_err("Failed to reserve address space for module tags, errno = %d\n", res);
1353 shutdown_mem_profiling(true);
1354 return res;
1355 }
1356
1357 alloc_tag_cttype = codetag_register_type(&desc);
1358 if (IS_ERR(alloc_tag_cttype)) {
1359 pr_err("Allocation tags registration failed, errno = %pe\n", alloc_tag_cttype);
1360 free_mod_tags_mem();
1361 shutdown_mem_profiling(true);
1362 return PTR_ERR(alloc_tag_cttype);
1363 }
1364
1365 return 0;
1366}
1367module_init(alloc_tag_init);