linux - mm/kmemleak.c

2601 lines · 73.2 KiB

1	// SPDX-License-Identifier: GPL-2.0-only
2	/*
3	* mm/kmemleak.c
4	*
5	* Copyright (C) 2008 ARM Limited
6	* Written by Catalin Marinas <[email protected]>
7	*
8	* For more information on the algorithm and kmemleak usage, please see
9	* Documentation/dev-tools/kmemleak.rst.
10	*
11	* Notes on locking
12	* ----------------
13	*
14	* The following locks and mutexes are used by kmemleak:
15	*
16	* - kmemleak_lock (raw_spinlock_t): protects the object_list as well as
17	* del_state modifications and accesses to the object trees
18	* (object_tree_root, object_phys_tree_root, object_percpu_tree_root). The
19	* object_list is the main list holding the metadata (struct
20	* kmemleak_object) for the allocated memory blocks. The object trees are
21	* red black trees used to look-up metadata based on a pointer to the
22	* corresponding memory block. The kmemleak_object structures are added to
23	* the object_list and the object tree root in the create_object() function
24	* called from the kmemleak_alloc{,_phys,_percpu}() callback and removed in
25	* delete_object() called from the kmemleak_free{,_phys,_percpu}() callback
26	* - kmemleak_object.lock (raw_spinlock_t): protects a kmemleak_object.
27	* Accesses to the metadata (e.g. count) are protected by this lock. Note
28	* that some members of this structure may be protected by other means
29	* (atomic or kmemleak_lock). This lock is also held when scanning the
30	* corresponding memory block to avoid the kernel freeing it via the
31	* kmemleak_free() callback. This is less heavyweight than holding a global
32	* lock like kmemleak_lock during scanning.
33	* - scan_mutex (mutex): ensures that only one thread may scan the memory for
34	* unreferenced objects at a time. The gray_list contains the objects which
35	* are already referenced or marked as false positives and need to be
36	* scanned. This list is only modified during a scanning episode when the
37	* scan_mutex is held. At the end of a scan, the gray_list is always empty.
38	* Note that the kmemleak_object.use_count is incremented when an object is
39	* added to the gray_list and therefore cannot be freed. This mutex also
40	* prevents multiple users of the "kmemleak" debugfs file together with
41	* modifications to the memory scanning parameters including the scan_thread
42	* pointer
43	*
44	* Locks and mutexes are acquired/nested in the following order:
45	*
46	* scan_mutex [-> object->lock] -> kmemleak_lock -> other_object->lock (SINGLE_DEPTH_NESTING)
47	*
48	* No kmemleak_lock and object->lock nesting is allowed outside scan_mutex
49	* regions.
50	*
51	* The kmemleak_object structures have a use_count incremented or decremented
52	* using the get_object()/put_object() functions. When the use_count becomes
53	* 0, this count can no longer be incremented and put_object() schedules the
54	* kmemleak_object freeing via an RCU callback. All calls to the get_object()
55	* function must be protected by rcu_read_lock() to avoid accessing a freed
56	* structure.
57	*/
58
59	#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
60
61	#include <linux/init.h>
62	#include <linux/kernel.h>
63	#include <linux/list.h>
64	#include <linux/sched/signal.h>
65	#include <linux/sched/task.h>
66	#include <linux/sched/task_stack.h>
67	#include <linux/jiffies.h>
68	#include <linux/delay.h>
69	#include <linux/export.h>
70	#include <linux/kthread.h>
71	#include <linux/rbtree.h>
72	#include <linux/fs.h>
73	#include <linux/debugfs.h>
74	#include <linux/seq_file.h>
75	#include <linux/cpumask.h>
76	#include <linux/spinlock.h>
77	#include <linux/module.h>
78	#include <linux/mutex.h>
79	#include <linux/rcupdate.h>
80	#include <linux/stacktrace.h>
81	#include <linux/stackdepot.h>
82	#include <linux/cache.h>
83	#include <linux/percpu.h>
84	#include <linux/memblock.h>
85	#include <linux/pfn.h>
86	#include <linux/mmzone.h>
87	#include <linux/slab.h>
88	#include <linux/thread_info.h>
89	#include <linux/err.h>
90	#include <linux/uaccess.h>
91	#include <linux/string.h>
92	#include <linux/nodemask.h>
93	#include <linux/mm.h>
94	#include <linux/workqueue.h>
95	#include <linux/xarray.h>
96	#include <linux/crc32.h>
97
98	#include <asm/sections.h>
99	#include <asm/processor.h>
100	#include <linux/atomic.h>
101
102	#include <linux/kasan.h>
103	#include <linux/kfence.h>
104	#include <linux/kmemleak.h>
105	#include <linux/memory_hotplug.h>
106
107	/*
108	* Kmemleak configuration and common defines.
109	*/
110	#define MAX_TRACE 16 /* stack trace length */
111	#define MSECS_MIN_AGE 5000 /* minimum object age for reporting */
112	#define SECS_FIRST_SCAN 60 /* delay before the first scan */
113	#define SECS_SCAN_WAIT 600 /* subsequent auto scanning delay */
114	#define MAX_SCAN_SIZE 4096 /* maximum size of a scanned block */
115
116	#define BYTES_PER_POINTER sizeof(void *)
117
118	/* scanning area inside a memory block */
119	struct kmemleak_scan_area {
120	struct hlist_node node;
121	unsigned long start;
122	size_t size;
123	};
124
125	#define KMEMLEAK_GREY 0
126	#define KMEMLEAK_BLACK -1
127
128	/*
129	* Structure holding the metadata for each allocated memory block.
130	* Modifications to such objects should be made while holding the
131	* object->lock. Insertions or deletions from object_list, gray_list or
132	* rb_node are already protected by the corresponding locks or mutex (see
133	* the notes on locking above). These objects are reference-counted
134	* (use_count) and freed using the RCU mechanism.
135	*/
136	struct kmemleak_object {
137	raw_spinlock_t lock;
138	unsigned int flags; /* object status flags */
139	struct list_head object_list;
140	struct list_head gray_list;
141	struct rb_node rb_node;
142	struct rcu_head rcu; /* object_list lockless traversal */
143	/* object usage count; object freed when use_count == 0 */
144	atomic_t use_count;
145	unsigned int del_state; /* deletion state */
146	unsigned long pointer;
147	size_t size;
148	/* pass surplus references to this pointer */
149	unsigned long excess_ref;
150	/* minimum number of a pointers found before it is considered leak */
151	int min_count;
152	/* the total number of pointers found pointing to this object */
153	int count;
154	/* consecutive scans the object has been seen unreferenced */
155	unsigned int unref_scans;
156	/* checksum for detecting modified objects */
157	u32 checksum;
158	depot_stack_handle_t trace_handle;
159	/* memory ranges to be scanned inside an object (empty for all) */
160	struct hlist_head area_list;
161	unsigned long jiffies; /* creation timestamp */
162	pid_t pid; /* pid of the current task */
163	/* per-scan dedup count, valid only while in scan-local dedup xarray */
164	unsigned int dup_count;
165	char comm[TASK_COMM_LEN]; /* executable name */
166	};
167
168	/* flag representing the memory block allocation status */
169	#define OBJECT_ALLOCATED (1 << 0)
170	/* flag set after the first reporting of an unreference object */
171	#define OBJECT_REPORTED (1 << 1)
172	/* flag set to not scan the object */
173	#define OBJECT_NO_SCAN (1 << 2)
174	/* flag set to fully scan the object when scan_area allocation failed */
175	#define OBJECT_FULL_SCAN (1 << 3)
176	/* flag set for object allocated with physical address */
177	#define OBJECT_PHYS (1 << 4)
178	/* flag set for per-CPU pointers */
179	#define OBJECT_PERCPU (1 << 5)
180	/* flag set on an object left unreferenced by the full scan, pending confirmation */
181	#define OBJECT_SUSPECT (1 << 6)
182
183	/* set when __remove_object() called */
184	#define DELSTATE_REMOVED (1 << 0)
185	/* set to temporarily prevent deletion from object_list */
186	#define DELSTATE_NO_DELETE (1 << 1)
187
188	#define HEX_PREFIX " "
189	/* number of bytes to print per line; must be 16 or 32 */
190	#define HEX_ROW_SIZE 16
191	/* number of bytes to print at a time (1, 2, 4, 8) */
192	#define HEX_GROUP_SIZE 1
193	/* include ASCII after the hex output */
194	#define HEX_ASCII 1
195	/* max number of lines to be printed */
196	#define HEX_MAX_LINES 2
197
198	/* the list of all allocated objects */
199	static LIST_HEAD(object_list);
200	/* the list of gray-colored objects (see color_gray comment below) */
201	static LIST_HEAD(gray_list);
202	/* memory pool allocation */
203	static struct kmemleak_object mem_pool[CONFIG_DEBUG_KMEMLEAK_MEM_POOL_SIZE];
204	static int mem_pool_free_count = ARRAY_SIZE(mem_pool);
205	static LIST_HEAD(mem_pool_free_list);
206	/* search tree for object boundaries */
207	static struct rb_root object_tree_root = RB_ROOT;
208	/* search tree for object (with OBJECT_PHYS flag) boundaries */
209	static struct rb_root object_phys_tree_root = RB_ROOT;
210	/* search tree for object (with OBJECT_PERCPU flag) boundaries */
211	static struct rb_root object_percpu_tree_root = RB_ROOT;
212	/* protecting the access to object_list, object_tree_root (or object_phys_tree_root) */
213	static DEFINE_RAW_SPINLOCK(kmemleak_lock);
214
215	/* allocation caches for kmemleak internal data */
216	static struct kmem_cache *object_cache;
217	static struct kmem_cache *scan_area_cache;
218
219	/* set if tracing memory operations is enabled */
220	static int kmemleak_enabled __read_mostly = 1;
221	/* same as above but only for the kmemleak_free() callback */
222	static int kmemleak_free_enabled __read_mostly = 1;
223	/* set in the late_initcall if there were no errors */
224	static int kmemleak_late_initialized;
225	/* set if a fatal kmemleak error has occurred */
226	static int kmemleak_error;
227
228	/* minimum and maximum address that may be valid pointers */
229	static unsigned long min_addr = ULONG_MAX;
230	static unsigned long max_addr;
231
232	/* minimum and maximum address that may be valid per-CPU pointers */
233	static unsigned long min_percpu_addr = ULONG_MAX;
234	static unsigned long max_percpu_addr;
235
236	static struct task_struct *scan_thread;
237	/* used to avoid reporting of recently allocated objects */
238	static unsigned long jiffies_min_age;
239	/* consecutive scans an object must stay unreferenced before reporting */
240	static unsigned int min_unref_scans =
241	IS_ENABLED(CONFIG_DEBUG_KMEMLEAK_VERBOSE) ? 2 : 1;
242	module_param(min_unref_scans, uint, 0644);
243	static unsigned long jiffies_last_scan;
244	/* delay between automatic memory scannings */
245	static unsigned long jiffies_scan_wait;
246	/* number of objects flagged OBJECT_SUSPECT during the current scan */
247	static int nr_suspects;
248	/* enables or disables the task stacks scanning */
249	static int kmemleak_stack_scan = 1;
250	/* protects the memory scanning, parameters and debug/kmemleak file access */
251	static DEFINE_MUTEX(scan_mutex);
252	/* setting kmemleak=on, will set this var, skipping the disable */
253	static int kmemleak_skip_disable;
254	/* If there are leaks that can be reported */
255	static bool kmemleak_found_leaks;
256
257	static bool kmemleak_verbose = IS_ENABLED(CONFIG_DEBUG_KMEMLEAK_VERBOSE);
258	module_param_named(verbose, kmemleak_verbose, bool, 0600);
259
260	static void kmemleak_disable(void);
261
262	/*
263	* Print a warning and dump the stack trace.
264	*/
265	#define kmemleak_warn(x...) do { \
266	pr_warn(x); \
267	dump_stack(); \
268	} while (0)
269
270	/*
271	* Macro invoked when a serious kmemleak condition occurred and cannot be
272	* recovered from. Kmemleak will be disabled and further allocation/freeing
273	* tracing no longer available.
274	*/
275	#define kmemleak_stop(x...) do { \
276	kmemleak_warn(x); \
277	kmemleak_disable(); \
278	} while (0)
279
280	#define warn_or_seq_printf(seq, fmt, ...) do { \
281	if (seq) \
282	seq_printf(seq, fmt, ##__VA_ARGS__); \
283	else \
284	pr_warn(fmt, ##__VA_ARGS__); \
285	} while (0)
286
287	static void warn_or_seq_hex_dump(struct seq_file *seq, int prefix_type,
288	int rowsize, int groupsize, const void *buf,
289	size_t len, bool ascii)
290	{
291	if (seq)
292	seq_hex_dump(seq, HEX_PREFIX, prefix_type, rowsize, groupsize,
293	buf, len, ascii);
294	else
295	print_hex_dump(KERN_WARNING, pr_fmt(HEX_PREFIX), prefix_type,
296	rowsize, groupsize, buf, len, ascii);
297	}
298
299	/*
300	* Printing of the objects hex dump to the seq file. The number of lines to be
301	* printed is limited to HEX_MAX_LINES to prevent seq file spamming. The
302	* actual number of printed bytes depends on HEX_ROW_SIZE. It must be called
303	* with the object->lock held.
304	*/
305	static void hex_dump_object(struct seq_file *seq,
306	struct kmemleak_object *object)
307	{
308	const u8 ptr = (const u8 )object->pointer;
309	size_t len;
310
311	if (WARN_ON_ONCE(object->flags & OBJECT_PHYS))
312	return;
313
314	if (object->flags & OBJECT_PERCPU)
315	ptr = (const u8 )this_cpu_ptr((void __percpu )object->pointer);
316
317	/* limit the number of lines to HEX_MAX_LINES */
318	len = min_t(size_t, object->size, HEX_MAX_LINES * HEX_ROW_SIZE);
319
320	if (object->flags & OBJECT_PERCPU)
321	warn_or_seq_printf(seq, " hex dump (first %zu bytes on cpu %d):\n",
322	len, raw_smp_processor_id());
323	else
324	warn_or_seq_printf(seq, " hex dump (first %zu bytes):\n", len);
325	kasan_disable_current();
326	warn_or_seq_hex_dump(seq, DUMP_PREFIX_NONE, HEX_ROW_SIZE,
327	HEX_GROUP_SIZE, kasan_reset_tag((void *)ptr), len, HEX_ASCII);
328	kasan_enable_current();
329	}
330
331	/*
332	* Object colors, encoded with count and min_count:
333	* - white - orphan object, not enough references to it (count < min_count)
334	* - gray - not orphan, not marked as false positive (min_count == 0) or
335	* sufficient references to it (count >= min_count)
336	* - black - ignore, it doesn't contain references (e.g. text section)
337	* (min_count == -1). No function defined for this color.
338	*/
339	static bool color_white(const struct kmemleak_object *object)
340	{
341	return object->count != KMEMLEAK_BLACK &&
342	object->count < object->min_count;
343	}
344
345	static bool color_gray(const struct kmemleak_object *object)
346	{
347	return object->min_count != KMEMLEAK_BLACK &&
348	object->count >= object->min_count;
349	}
350
351	/*
352	* Objects are considered unreferenced only if their color is white, they have
353	* not be deleted and have a minimum age to avoid false positives caused by
354	* pointers temporarily stored in CPU registers.
355	*/
356	static bool unreferenced_object(struct kmemleak_object *object)
357	{
358	return (color_white(object) && object->flags & OBJECT_ALLOCATED) &&
359	time_before_eq(object->jiffies + jiffies_min_age,
360	jiffies_last_scan);
361	}
362
363	static const char __object_type_str(struct kmemleak_object object)
364	{
365	if (object->flags & OBJECT_PHYS)
366	return " (phys)";
367	if (object->flags & OBJECT_PERCPU)
368	return " (percpu)";
369	return "";
370	}
371
372	/*
373	* Printing of the unreferenced objects information to the seq file. The
374	* print_unreferenced function must be called with the object->lock held.
375	*/
376	static void __print_unreferenced(struct seq_file *seq,
377	struct kmemleak_object *object,
378	bool hex_dump)
379	{
380	int i;
381	unsigned long *entries;
382	unsigned int nr_entries;
383
384	nr_entries = stack_depot_fetch(object->trace_handle, &entries);
385	warn_or_seq_printf(seq, "unreferenced object%s 0x%08lx (size %zu):\n",
386	__object_type_str(object),
387	object->pointer, object->size);
388	warn_or_seq_printf(seq, " comm \"%s\", pid %d, jiffies %lu\n",
389	object->comm, object->pid, object->jiffies);
390	if (hex_dump)
391	hex_dump_object(seq, object);
392	warn_or_seq_printf(seq, " backtrace (crc %x):\n", object->checksum);
393
394	for (i = 0; i < nr_entries; i++) {
395	void ptr = (void )entries[i];
396	warn_or_seq_printf(seq, " %pS\n", ptr);
397	}
398	}
399
400	static void print_unreferenced(struct seq_file *seq,
401	struct kmemleak_object *object)
402	{
403	__print_unreferenced(seq, object, true);
404	}
405
406	/*
407	* Print the kmemleak_object information. This function is used mainly for
408	* debugging special cases when kmemleak operations. It must be called with
409	* the object->lock held.
410	*/
411	static void dump_object_info(struct kmemleak_object *object)
412	{
413	pr_notice("Object%s 0x%08lx (size %zu):\n",
414	__object_type_str(object), object->pointer, object->size);
415	pr_notice(" comm \"%s\", pid %d, jiffies %lu\n",
416	object->comm, object->pid, object->jiffies);
417	pr_notice(" min_count = %d\n", object->min_count);
418	pr_notice(" count = %d\n", object->count);
419	pr_notice(" flags = 0x%x\n", object->flags);
420	pr_notice(" checksum = %u\n", object->checksum);
421	pr_notice(" backtrace:\n");
422	if (object->trace_handle)
423	stack_depot_print(object->trace_handle);
424	}
425
426	static struct rb_root *object_tree(unsigned long objflags)
427	{
428	if (objflags & OBJECT_PHYS)
429	return &object_phys_tree_root;
430	if (objflags & OBJECT_PERCPU)
431	return &object_percpu_tree_root;
432	return &object_tree_root;
433	}
434
435	/*
436	* Look-up a memory block metadata (kmemleak_object) in the object search
437	* tree based on a pointer value. If alias is 0, only values pointing to the
438	* beginning of the memory block are allowed. The kmemleak_lock must be held
439	* when calling this function.
440	*/
441	static struct kmemleak_object *__lookup_object(unsigned long ptr, int alias,
442	unsigned int objflags)
443	{
444	struct rb_node *rb = object_tree(objflags)->rb_node;
445	unsigned long untagged_ptr = (unsigned long)kasan_reset_tag((void *)ptr);
446
447	while (rb) {
448	struct kmemleak_object *object;
449	unsigned long untagged_objp;
450
451	object = rb_entry(rb, struct kmemleak_object, rb_node);
452	untagged_objp = (unsigned long)kasan_reset_tag((void *)object->pointer);
453
454	if (untagged_ptr < untagged_objp)
455	rb = object->rb_node.rb_left;
456	else if (untagged_objp + object->size <= untagged_ptr)
457	rb = object->rb_node.rb_right;
458	else if (untagged_objp == untagged_ptr \|\| alias)
459	return object;
460	else {
461	/*
462	* Printk deferring due to the kmemleak_lock held.
463	* This is done to avoid deadlock.
464	*/
465	printk_deferred_enter();
466	kmemleak_warn("Found object by alias at 0x%08lx\n",
467	ptr);
468	dump_object_info(object);
469	printk_deferred_exit();
470	break;
471	}
472	}
473	return NULL;
474	}
475
476	/* Look-up a kmemleak object which allocated with virtual address. */
477	static struct kmemleak_object *lookup_object(unsigned long ptr, int alias)
478	{
479	return __lookup_object(ptr, alias, 0);
480	}
481
482	/*
483	* Increment the object use_count. Return 1 if successful or 0 otherwise. Note
484	* that once an object's use_count reached 0, the RCU freeing was already
485	* registered and the object should no longer be used. This function must be
486	* called under the protection of rcu_read_lock().
487	*/
488	static int get_object(struct kmemleak_object *object)
489	{
490	return atomic_inc_not_zero(&object->use_count);
491	}
492
493	/*
494	* Memory pool allocation and freeing. kmemleak_lock must not be held.
495	*/
496	static struct kmemleak_object *mem_pool_alloc(gfp_t gfp)
497	{
498	unsigned long flags;
499	struct kmemleak_object *object;
500	bool warn = false;
501
502	/* try the slab allocator first */
503	if (object_cache) {
504	object = kmem_cache_alloc_noprof(object_cache,
505	gfp_nested_mask(gfp));
506	if (object)
507	return object;
508	}
509
510	/* slab allocation failed, try the memory pool */
511	raw_spin_lock_irqsave(&kmemleak_lock, flags);
512	object = list_first_entry_or_null(&mem_pool_free_list,
513	typeof(*object), object_list);
514	if (object)
515	list_del(&object->object_list);
516	else if (mem_pool_free_count)
517	object = &mem_pool[--mem_pool_free_count];
518	else
519	warn = true;
520	raw_spin_unlock_irqrestore(&kmemleak_lock, flags);
521	if (warn)
522	pr_warn_once("Memory pool empty, consider increasing CONFIG_DEBUG_KMEMLEAK_MEM_POOL_SIZE\n");
523
524	return object;
525	}
526
527	/*
528	* Return the object to either the slab allocator or the memory pool.
529	*/
530	static void mem_pool_free(struct kmemleak_object *object)
531	{
532	unsigned long flags;
533
534	if (object < mem_pool \|\| object >= ARRAY_END(mem_pool)) {
535	kmem_cache_free(object_cache, object);
536	return;
537	}
538
539	/* add the object to the memory pool free list */
540	raw_spin_lock_irqsave(&kmemleak_lock, flags);
541	list_add(&object->object_list, &mem_pool_free_list);
542	raw_spin_unlock_irqrestore(&kmemleak_lock, flags);
543	}
544
545	/*
546	* RCU callback to free a kmemleak_object.
547	*/
548	static void free_object_rcu(struct rcu_head *rcu)
549	{
550	struct hlist_node *tmp;
551	struct kmemleak_scan_area *area;
552	struct kmemleak_object *object =
553	container_of(rcu, struct kmemleak_object, rcu);
554
555	/*
556	* Once use_count is 0 (guaranteed by put_object), there is no other
557	* code accessing this object, hence no need for locking.
558	*/
559	hlist_for_each_entry_safe(area, tmp, &object->area_list, node) {
560	hlist_del(&area->node);
561	kmem_cache_free(scan_area_cache, area);
562	}
563	mem_pool_free(object);
564	}
565
566	/*
567	* Decrement the object use_count. Once the count is 0, free the object using
568	* an RCU callback. Since put_object() may be called via the kmemleak_free() ->
569	* delete_object() path, the delayed RCU freeing ensures that there is no
570	* recursive call to the kernel allocator. Lock-less RCU object_list traversal
571	* is also possible.
572	*/
573	static void put_object(struct kmemleak_object *object)
574	{
575	if (!atomic_dec_and_test(&object->use_count))
576	return;
577
578	/* should only get here after delete_object was called */
579	WARN_ON(object->flags & OBJECT_ALLOCATED);
580
581	/*
582	* It may be too early for the RCU callbacks, however, there is no
583	* concurrent object_list traversal when !object_cache and all objects
584	* came from the memory pool. Free the object directly.
585	*/
586	if (object_cache)
587	call_rcu(&object->rcu, free_object_rcu);
588	else
589	free_object_rcu(&object->rcu);
590	}
591
592	/*
593	* Look up an object in the object search tree and increase its use_count.
594	*/
595	static struct kmemleak_object *__find_and_get_object(unsigned long ptr, int alias,
596	unsigned int objflags)
597	{
598	unsigned long flags;
599	struct kmemleak_object *object;
600
601	rcu_read_lock();
602	raw_spin_lock_irqsave(&kmemleak_lock, flags);
603	object = __lookup_object(ptr, alias, objflags);
604	raw_spin_unlock_irqrestore(&kmemleak_lock, flags);
605
606	/* check whether the object is still available */
607	if (object && !get_object(object))
608	object = NULL;
609	rcu_read_unlock();
610
611	return object;
612	}
613
614	/* Look up and get an object which allocated with virtual address. */
615	static struct kmemleak_object *find_and_get_object(unsigned long ptr, int alias)
616	{
617	return __find_and_get_object(ptr, alias, 0);
618	}
619
620	/*
621	* Remove an object from its object tree and object_list. Must be called with
622	* the kmemleak_lock held _if_ kmemleak is still enabled.
623	*/
624	static void __remove_object(struct kmemleak_object *object)
625	{
626	rb_erase(&object->rb_node, object_tree(object->flags));
627	if (!(object->del_state & DELSTATE_NO_DELETE))
628	list_del_rcu(&object->object_list);
629	object->del_state \|= DELSTATE_REMOVED;
630	}
631
632	static struct kmemleak_object *__find_and_remove_object(unsigned long ptr,
633	int alias,
634	unsigned int objflags)
635	{
636	struct kmemleak_object *object;
637
638	object = __lookup_object(ptr, alias, objflags);
639	if (object)
640	__remove_object(object);
641
642	return object;
643	}
644
645	/*
646	* Look up an object in the object search tree and remove it from both object
647	* tree root and object_list. The returned object's use_count should be at
648	* least 1, as initially set by create_object().
649	*/
650	static struct kmemleak_object *find_and_remove_object(unsigned long ptr, int alias,
651	unsigned int objflags)
652	{
653	unsigned long flags;
654	struct kmemleak_object *object;
655
656	raw_spin_lock_irqsave(&kmemleak_lock, flags);
657	object = __find_and_remove_object(ptr, alias, objflags);
658	raw_spin_unlock_irqrestore(&kmemleak_lock, flags);
659
660	return object;
661	}
662
663	static noinline depot_stack_handle_t set_track_prepare(void)
664	{
665	depot_stack_handle_t trace_handle;
666	unsigned long entries[MAX_TRACE];
667	unsigned int nr_entries;
668
669	/*
670	* Use object_cache to determine whether kmemleak_init() has
671	* been invoked. stack_depot_early_init() is called before
672	* kmemleak_init() in mm_core_init().
673	*/
674	if (!object_cache)
675	return 0;
676	nr_entries = stack_trace_save(entries, ARRAY_SIZE(entries), 3);
677	trace_handle = stack_depot_save(entries, nr_entries, GFP_NOWAIT);
678
679	return trace_handle;
680	}
681
682	static struct kmemleak_object *__alloc_object(gfp_t gfp)
683	{
684	struct kmemleak_object *object;
685
686	object = mem_pool_alloc(gfp);
687	if (!object) {
688	pr_warn("Cannot allocate a kmemleak_object structure\n");
689	kmemleak_disable();
690	return NULL;
691	}
692
693	INIT_LIST_HEAD(&object->object_list);
694	INIT_LIST_HEAD(&object->gray_list);
695	INIT_HLIST_HEAD(&object->area_list);
696	raw_spin_lock_init(&object->lock);
697	atomic_set(&object->use_count, 1);
698	object->excess_ref = 0;
699	object->count = 0; /* white color initially */
700	object->checksum = ~0;
701	object->unref_scans = 0;
702	object->del_state = 0;
703
704	/* task information */
705	if (in_hardirq()) {
706	object->pid = 0;
707	strscpy(object->comm, "hardirq");
708	} else if (in_serving_softirq()) {
709	object->pid = 0;
710	strscpy(object->comm, "softirq");
711	} else {
712	object->pid = current->pid;
713	/*
714	* There is a small chance of a race with set_task_comm(),
715	* however using get_task_comm() here may cause locking
716	* dependency issues with current->alloc_lock. In the worst
717	* case, the command line is not correct.
718	*/
719	strscpy(object->comm, current->comm);
720	}
721
722	/* kernel backtrace */
723	object->trace_handle = set_track_prepare();
724
725	return object;
726	}
727
728	static int __link_object(struct kmemleak_object *object, unsigned long ptr,
729	size_t size, int min_count, unsigned int objflags)
730	{
731
732	struct kmemleak_object *parent;
733	struct rb_node *link, rb_parent;
734	unsigned long untagged_ptr;
735	unsigned long untagged_objp;
736
737	object->flags = OBJECT_ALLOCATED \| objflags;
738	object->pointer = ptr;
739	object->size = kfence_ksize((void *)ptr) ?: size;
740	object->min_count = min_count;
741	object->jiffies = jiffies;
742
743	untagged_ptr = (unsigned long)kasan_reset_tag((void *)ptr);
744	/*
745	* Only update min_addr and max_addr with object storing virtual
746	* address. And update min_percpu_addr max_percpu_addr for per-CPU
747	* objects.
748	*/
749	if (objflags & OBJECT_PERCPU) {
750	min_percpu_addr = min(min_percpu_addr, untagged_ptr);
751	max_percpu_addr = max(max_percpu_addr, untagged_ptr + size);
752	} else if (!(objflags & OBJECT_PHYS)) {
753	min_addr = min(min_addr, untagged_ptr);
754	max_addr = max(max_addr, untagged_ptr + size);
755	}
756	link = &object_tree(objflags)->rb_node;
757	rb_parent = NULL;
758	while (*link) {
759	rb_parent = *link;
760	parent = rb_entry(rb_parent, struct kmemleak_object, rb_node);
761	untagged_objp = (unsigned long)kasan_reset_tag((void *)parent->pointer);
762	if (untagged_ptr + size <= untagged_objp)
763	link = &parent->rb_node.rb_left;
764	else if (untagged_objp + parent->size <= untagged_ptr)
765	link = &parent->rb_node.rb_right;
766	else {
767	/*
768	* Printk deferring due to the kmemleak_lock held.
769	* This is done to avoid deadlock.
770	*/
771	printk_deferred_enter();
772	kmemleak_stop("Cannot insert 0x%lx into the object search tree (overlaps existing)\n",
773	ptr);
774	/*
775	* No need for parent->lock here since "parent" cannot
776	* be freed while the kmemleak_lock is held.
777	*/
778	dump_object_info(parent);
779	printk_deferred_exit();
780	return -EEXIST;
781	}
782	}
783	rb_link_node(&object->rb_node, rb_parent, link);
784	rb_insert_color(&object->rb_node, object_tree(objflags));
785	list_add_tail_rcu(&object->object_list, &object_list);
786
787	return 0;
788	}
789
790	/*
791	* Create the metadata (struct kmemleak_object) corresponding to an allocated
792	* memory block and add it to the object_list and object tree.
793	*/
794	static void __create_object(unsigned long ptr, size_t size,
795	int min_count, gfp_t gfp, unsigned int objflags)
796	{
797	struct kmemleak_object *object;
798	unsigned long flags;
799	int ret;
800
801	object = __alloc_object(gfp);
802	if (!object)
803	return;
804
805	raw_spin_lock_irqsave(&kmemleak_lock, flags);
806	ret = __link_object(object, ptr, size, min_count, objflags);
807	raw_spin_unlock_irqrestore(&kmemleak_lock, flags);
808	if (ret)
809	mem_pool_free(object);
810	}
811
812	/* Create kmemleak object which allocated with virtual address. */
813	static void create_object(unsigned long ptr, size_t size,
814	int min_count, gfp_t gfp)
815	{
816	__create_object(ptr, size, min_count, gfp, 0);
817	}
818
819	/* Create kmemleak object which allocated with physical address. */
820	static void create_object_phys(unsigned long ptr, size_t size,
821	int min_count, gfp_t gfp)
822	{
823	__create_object(ptr, size, min_count, gfp, OBJECT_PHYS);
824	}
825
826	/* Create kmemleak object corresponding to a per-CPU allocation. */
827	static void create_object_percpu(unsigned long ptr, size_t size,
828	int min_count, gfp_t gfp)
829	{
830	__create_object(ptr, size, min_count, gfp, OBJECT_PERCPU);
831	}
832
833	/*
834	* Mark the object as not allocated and schedule RCU freeing via put_object().
835	*/
836	static void __delete_object(struct kmemleak_object *object)
837	{
838	unsigned long flags;
839
840	WARN_ON(!(object->flags & OBJECT_ALLOCATED));
841	WARN_ON(atomic_read(&object->use_count) < 1);
842
843	/*
844	* Locking here also ensures that the corresponding memory block
845	* cannot be freed when it is being scanned.
846	*/
847	raw_spin_lock_irqsave(&object->lock, flags);
848	object->flags &= ~OBJECT_ALLOCATED;
849	raw_spin_unlock_irqrestore(&object->lock, flags);
850	put_object(object);
851	}
852
853	/*
854	* Look up the metadata (struct kmemleak_object) corresponding to ptr and
855	* delete it.
856	*/
857	static void delete_object_full(unsigned long ptr, unsigned int objflags)
858	{
859	struct kmemleak_object *object;
860
861	object = find_and_remove_object(ptr, 0, objflags);
862	if (!object)
863	/*
864	* kmalloc_nolock() -> kfree() calls kmemleak_free()
865	* without kmemleak_alloc().
866	*/
867	return;
868	__delete_object(object);
869	}
870
871	/*
872	* Look up the metadata (struct kmemleak_object) corresponding to ptr and
873	* delete it. If the memory block is partially freed, the function may create
874	* additional metadata for the remaining parts of the block.
875	*/
876	static void delete_object_part(unsigned long ptr, size_t size,
877	unsigned int objflags)
878	{
879	struct kmemleak_object object, object_l, *object_r;
880	unsigned long start, end, flags;
881
882	object_l = __alloc_object(GFP_KERNEL);
883	if (!object_l)
884	return;
885
886	object_r = __alloc_object(GFP_KERNEL);
887	if (!object_r)
888	goto out;
889
890	raw_spin_lock_irqsave(&kmemleak_lock, flags);
891	object = __find_and_remove_object(ptr, 1, objflags);
892	if (!object)
893	goto unlock;
894
895	/*
896	* Create one or two objects that may result from the memory block
897	* split. Note that partial freeing is only done by free_bootmem() and
898	* this happens before kmemleak_init() is called.
899	*/
900	start = object->pointer;
901	end = object->pointer + object->size;
902	if ((ptr > start) &&
903	!__link_object(object_l, start, ptr - start,
904	object->min_count, objflags))
905	object_l = NULL;
906	if ((ptr + size < end) &&
907	!__link_object(object_r, ptr + size, end - ptr - size,
908	object->min_count, objflags))
909	object_r = NULL;
910
911	unlock:
912	raw_spin_unlock_irqrestore(&kmemleak_lock, flags);
913	if (object) {
914	__delete_object(object);
915	} else {
916	#ifdef DEBUG
917	kmemleak_warn("Partially freeing unknown object at 0x%08lx (size %zu)\n",
918	ptr, size);
919	#endif
920	}
921
922	out:
923	if (object_l)
924	mem_pool_free(object_l);
925	if (object_r)
926	mem_pool_free(object_r);
927	}
928
929	static void __paint_it(struct kmemleak_object *object, int color)
930	{
931	object->min_count = color;
932	if (color == KMEMLEAK_BLACK)
933	object->flags \|= OBJECT_NO_SCAN;
934	}
935
936	static void paint_it(struct kmemleak_object *object, int color)
937	{
938	unsigned long flags;
939
940	raw_spin_lock_irqsave(&object->lock, flags);
941	__paint_it(object, color);
942	raw_spin_unlock_irqrestore(&object->lock, flags);
943	}
944
945	static void paint_ptr(unsigned long ptr, int color, unsigned int objflags)
946	{
947	struct kmemleak_object *object;
948
949	object = __find_and_get_object(ptr, 0, objflags);
950	if (!object)
951	/*
952	* kmalloc_nolock() -> kfree_rcu() calls kmemleak_ignore()
953	* without kmemleak_alloc().
954	*/
955	return;
956	paint_it(object, color);
957	put_object(object);
958	}
959
960	/*
961	* Mark an object permanently as gray-colored so that it can no longer be
962	* reported as a leak. This is used in general to mark a false positive.
963	*/
964	static void make_gray_object(unsigned long ptr)
965	{
966	paint_ptr(ptr, KMEMLEAK_GREY, 0);
967	}
968
969	/*
970	* Mark the object as black-colored so that it is ignored from scans and
971	* reporting.
972	*/
973	static void make_black_object(unsigned long ptr, unsigned int objflags)
974	{
975	paint_ptr(ptr, KMEMLEAK_BLACK, objflags);
976	}
977
978	/*
979	* Reset the checksum of an object. The immediate effect is that it will not
980	* be reported as a leak during the next scan until its checksum is updated.
981	*/
982	static void reset_checksum(unsigned long ptr)
983	{
984	unsigned long flags;
985	struct kmemleak_object *object;
986
987	object = find_and_get_object(ptr, 0);
988	if (!object) {
989	kmemleak_warn("Not resetting the checksum of an unknown object at 0x%08lx\n",
990	ptr);
991	return;
992	}
993
994	raw_spin_lock_irqsave(&object->lock, flags);
995	object->checksum = ~0;
996	raw_spin_unlock_irqrestore(&object->lock, flags);
997	put_object(object);
998	}
999
1000	/*
1001	* Add a scanning area to the object. If at least one such area is added,
1002	* kmemleak will only scan these ranges rather than the whole memory block.
1003	*/
1004	static void add_scan_area(unsigned long ptr, size_t size, gfp_t gfp)
1005	{
1006	unsigned long flags;
1007	struct kmemleak_object *object;
1008	struct kmemleak_scan_area *area = NULL;
1009	unsigned long untagged_ptr;
1010	unsigned long untagged_objp;
1011
1012	object = find_and_get_object(ptr, 1);
1013	if (!object) {
1014	kmemleak_warn("Adding scan area to unknown object at 0x%08lx\n",
1015	ptr);
1016	return;
1017	}
1018
1019	untagged_ptr = (unsigned long)kasan_reset_tag((void *)ptr);
1020	untagged_objp = (unsigned long)kasan_reset_tag((void *)object->pointer);
1021
1022	if (scan_area_cache)
1023	area = kmem_cache_alloc_noprof(scan_area_cache,
1024	gfp_nested_mask(gfp));
1025
1026	raw_spin_lock_irqsave(&object->lock, flags);
1027	if (!area) {
1028	pr_warn_once("Cannot allocate a scan area, scanning the full object\n");
1029	/* mark the object for full scan to avoid false positives */
1030	object->flags \|= OBJECT_FULL_SCAN;
1031	goto out_unlock;
1032	}
1033	if (size == SIZE_MAX) {
1034	size = untagged_objp + object->size - untagged_ptr;
1035	} else if (untagged_ptr + size > untagged_objp + object->size) {
1036	kmemleak_warn("Scan area larger than object 0x%08lx\n", ptr);
1037	dump_object_info(object);
1038	kmem_cache_free(scan_area_cache, area);
1039	goto out_unlock;
1040	}
1041
1042	INIT_HLIST_NODE(&area->node);
1043	area->start = ptr;
1044	area->size = size;
1045
1046	hlist_add_head(&area->node, &object->area_list);
1047	out_unlock:
1048	raw_spin_unlock_irqrestore(&object->lock, flags);
1049	put_object(object);
1050	}
1051
1052	/*
1053	* Any surplus references (object already gray) to 'ptr' are passed to
1054	* 'excess_ref'. This is used in the vmalloc() case where a pointer to
1055	* vm_struct may be used as an alternative reference to the vmalloc'ed object
1056	* (see free_thread_stack()).
1057	*/
1058	static void object_set_excess_ref(unsigned long ptr, unsigned long excess_ref)
1059	{
1060	unsigned long flags;
1061	struct kmemleak_object *object;
1062
1063	object = find_and_get_object(ptr, 0);
1064	if (!object) {
1065	kmemleak_warn("Setting excess_ref on unknown object at 0x%08lx\n",
1066	ptr);
1067	return;
1068	}
1069
1070	raw_spin_lock_irqsave(&object->lock, flags);
1071	object->excess_ref = excess_ref;
1072	raw_spin_unlock_irqrestore(&object->lock, flags);
1073	put_object(object);
1074	}
1075
1076	/*
1077	* Set the OBJECT_NO_SCAN flag for the object corresponding to the given
1078	* pointer. Such object will not be scanned by kmemleak but references to it
1079	* are searched.
1080	*/
1081	static void object_no_scan(unsigned long ptr)
1082	{
1083	unsigned long flags;
1084	struct kmemleak_object *object;
1085
1086	object = find_and_get_object(ptr, 0);
1087	if (!object) {
1088	kmemleak_warn("Not scanning unknown object at 0x%08lx\n", ptr);
1089	return;
1090	}
1091
1092	raw_spin_lock_irqsave(&object->lock, flags);
1093	object->flags \|= OBJECT_NO_SCAN;
1094	raw_spin_unlock_irqrestore(&object->lock, flags);
1095	put_object(object);
1096	}
1097
1098	/**
1099	* kmemleak_alloc - register a newly allocated object
1100	* @ptr: pointer to beginning of the object
1101	* @size: size of the object
1102	* @min_count: minimum number of references to this object. If during memory
1103	* scanning a number of references less than @min_count is found,
1104	* the object is reported as a memory leak. If @min_count is 0,
1105	* the object is never reported as a leak. If @min_count is -1,
1106	* the object is ignored (not scanned and not reported as a leak)
1107	* @gfp: kmalloc() flags used for kmemleak internal memory allocations
1108	*
1109	* This function is called from the kernel allocators when a new object
1110	* (memory block) is allocated (kmem_cache_alloc, kmalloc etc.).
1111	*/
1112	void __ref kmemleak_alloc(const void *ptr, size_t size, int min_count,
1113	gfp_t gfp)
1114	{
1115	pr_debug("%s(0x%px, %zu, %d)\n", __func__, ptr, size, min_count);
1116
1117	if (kmemleak_enabled && ptr && !IS_ERR(ptr))
1118	create_object((unsigned long)ptr, size, min_count, gfp);
1119	}
1120	EXPORT_SYMBOL_GPL(kmemleak_alloc);
1121
1122	/**
1123	* kmemleak_alloc_percpu - register a newly allocated __percpu object
1124	* @ptr: __percpu pointer to beginning of the object
1125	* @size: size of the object
1126	* @gfp: flags used for kmemleak internal memory allocations
1127	*
1128	* This function is called from the kernel percpu allocator when a new object
1129	* (memory block) is allocated (alloc_percpu).
1130	*/
1131	void __ref kmemleak_alloc_percpu(const void __percpu *ptr, size_t size,
1132	gfp_t gfp)
1133	{
1134	pr_debug("%s(0x%px, %zu)\n", __func__, ptr, size);
1135
1136	if (kmemleak_enabled && ptr && !IS_ERR_PCPU(ptr))
1137	create_object_percpu((__force unsigned long)ptr, size, 1, gfp);
1138	}
1139	EXPORT_SYMBOL_GPL(kmemleak_alloc_percpu);
1140
1141	/**
1142	* kmemleak_vmalloc - register a newly vmalloc'ed object
1143	* @area: pointer to vm_struct
1144	* @size: size of the object
1145	* @gfp: __vmalloc() flags used for kmemleak internal memory allocations
1146	*
1147	* This function is called from the vmalloc() kernel allocator when a new
1148	* object (memory block) is allocated.
1149	*/
1150	void __ref kmemleak_vmalloc(const struct vm_struct *area, size_t size, gfp_t gfp)
1151	{
1152	pr_debug("%s(0x%px, %zu)\n", __func__, area, size);
1153
1154	/*
1155	* A min_count = 2 is needed because vm_struct contains a reference to
1156	* the virtual address of the vmalloc'ed block.
1157	*/
1158	if (kmemleak_enabled) {
1159	create_object((unsigned long)area->addr, size, 2, gfp);
1160	object_set_excess_ref((unsigned long)area,
1161	(unsigned long)area->addr);
1162	}
1163	}
1164	EXPORT_SYMBOL_GPL(kmemleak_vmalloc);
1165
1166	/**
1167	* kmemleak_free - unregister a previously registered object
1168	* @ptr: pointer to beginning of the object
1169	*
1170	* This function is called from the kernel allocators when an object (memory
1171	* block) is freed (kmem_cache_free, kfree, vfree etc.).
1172	*/
1173	void __ref kmemleak_free(const void *ptr)
1174	{
1175	pr_debug("%s(0x%px)\n", __func__, ptr);
1176
1177	if (kmemleak_free_enabled && ptr && !IS_ERR(ptr))
1178	delete_object_full((unsigned long)ptr, 0);
1179	}
1180	EXPORT_SYMBOL_GPL(kmemleak_free);
1181
1182	/**
1183	* kmemleak_free_part - partially unregister a previously registered object
1184	* @ptr: pointer to the beginning or inside the object. This also
1185	* represents the start of the range to be freed
1186	* @size: size to be unregistered
1187	*
1188	* This function is called when only a part of a memory block is freed
1189	* (usually from the bootmem allocator).
1190	*/
1191	void __ref kmemleak_free_part(const void *ptr, size_t size)
1192	{
1193	pr_debug("%s(0x%px)\n", __func__, ptr);
1194
1195	if (kmemleak_enabled && ptr && !IS_ERR(ptr))
1196	delete_object_part((unsigned long)ptr, size, 0);
1197	}
1198	EXPORT_SYMBOL_GPL(kmemleak_free_part);
1199
1200	/**
1201	* kmemleak_free_percpu - unregister a previously registered __percpu object
1202	* @ptr: __percpu pointer to beginning of the object
1203	*
1204	* This function is called from the kernel percpu allocator when an object
1205	* (memory block) is freed (free_percpu).
1206	*/
1207	void __ref kmemleak_free_percpu(const void __percpu *ptr)
1208	{
1209	pr_debug("%s(0x%px)\n", __func__, ptr);
1210
1211	if (kmemleak_free_enabled && ptr && !IS_ERR_PCPU(ptr))
1212	delete_object_full((__force unsigned long)ptr, OBJECT_PERCPU);
1213	}
1214	EXPORT_SYMBOL_GPL(kmemleak_free_percpu);
1215
1216	/**
1217	* kmemleak_update_trace - update object allocation stack trace
1218	* @ptr: pointer to beginning of the object
1219	*
1220	* Override the object allocation stack trace for cases where the actual
1221	* allocation place is not always useful.
1222	*/
1223	void __ref kmemleak_update_trace(const void *ptr)
1224	{
1225	struct kmemleak_object *object;
1226	depot_stack_handle_t trace_handle;
1227	unsigned long flags;
1228
1229	pr_debug("%s(0x%px)\n", __func__, ptr);
1230
1231	if (!kmemleak_enabled \|\| IS_ERR_OR_NULL(ptr))
1232	return;
1233
1234	object = find_and_get_object((unsigned long)ptr, 1);
1235	if (!object) {
1236	#ifdef DEBUG
1237	kmemleak_warn("Updating stack trace for unknown object at %p\n",
1238	ptr);
1239	#endif
1240	return;
1241	}
1242
1243	trace_handle = set_track_prepare();
1244	raw_spin_lock_irqsave(&object->lock, flags);
1245	object->trace_handle = trace_handle;
1246	raw_spin_unlock_irqrestore(&object->lock, flags);
1247
1248	put_object(object);
1249	}
1250	EXPORT_SYMBOL(kmemleak_update_trace);
1251
1252	/**
1253	* kmemleak_not_leak - mark an allocated object as false positive
1254	* @ptr: pointer to beginning of the object
1255	*
1256	* Calling this function on an object will cause the memory block to no longer
1257	* be reported as leak and always be scanned.
1258	*/
1259	void __ref kmemleak_not_leak(const void *ptr)
1260	{
1261	pr_debug("%s(0x%px)\n", __func__, ptr);
1262
1263	if (kmemleak_enabled && ptr && !IS_ERR(ptr))
1264	make_gray_object((unsigned long)ptr);
1265	}
1266	EXPORT_SYMBOL(kmemleak_not_leak);
1267
1268	/**
1269	* kmemleak_transient_leak - mark an allocated object as transient false positive
1270	* @ptr: pointer to beginning of the object
1271	*
1272	* Calling this function on an object will cause the memory block to not be
1273	* reported as a leak temporarily. This may happen, for example, if the object
1274	* is part of a singly linked list and the ->next reference to it is changed.
1275	*/
1276	void __ref kmemleak_transient_leak(const void *ptr)
1277	{
1278	pr_debug("%s(0x%px)\n", __func__, ptr);
1279
1280	if (kmemleak_enabled && ptr && !IS_ERR(ptr))
1281	reset_checksum((unsigned long)ptr);
1282	}
1283	EXPORT_SYMBOL(kmemleak_transient_leak);
1284
1285	/**
1286	* kmemleak_ignore_percpu - similar to kmemleak_ignore but taking a percpu
1287	* address argument
1288	* @ptr: percpu address of the object
1289	*/
1290	void __ref kmemleak_ignore_percpu(const void __percpu *ptr)
1291	{
1292	pr_debug("%s(0x%px)\n", __func__, ptr);
1293
1294	if (kmemleak_enabled && ptr && !IS_ERR_PCPU(ptr))
1295	make_black_object((unsigned long)ptr, OBJECT_PERCPU);
1296	}
1297	EXPORT_SYMBOL_GPL(kmemleak_ignore_percpu);
1298
1299	/**
1300	* kmemleak_ignore - ignore an allocated object
1301	* @ptr: pointer to beginning of the object
1302	*
1303	* Calling this function on an object will cause the memory block to be
1304	* ignored (not scanned and not reported as a leak). This is usually done when
1305	* it is known that the corresponding block is not a leak and does not contain
1306	* any references to other allocated memory blocks.
1307	*/
1308	void __ref kmemleak_ignore(const void *ptr)
1309	{
1310	pr_debug("%s(0x%px)\n", __func__, ptr);
1311
1312	if (kmemleak_enabled && ptr && !IS_ERR(ptr))
1313	make_black_object((unsigned long)ptr, 0);
1314	}
1315	EXPORT_SYMBOL(kmemleak_ignore);
1316
1317	/**
1318	* kmemleak_scan_area - limit the range to be scanned in an allocated object
1319	* @ptr: pointer to beginning or inside the object. This also
1320	* represents the start of the scan area
1321	* @size: size of the scan area
1322	* @gfp: kmalloc() flags used for kmemleak internal memory allocations
1323	*
1324	* This function is used when it is known that only certain parts of an object
1325	* contain references to other objects. Kmemleak will only scan these areas
1326	* reducing the number false negatives.
1327	*/
1328	void __ref kmemleak_scan_area(const void *ptr, size_t size, gfp_t gfp)
1329	{
1330	pr_debug("%s(0x%px)\n", __func__, ptr);
1331
1332	if (kmemleak_enabled && ptr && size && !IS_ERR(ptr))
1333	add_scan_area((unsigned long)ptr, size, gfp);
1334	}
1335	EXPORT_SYMBOL(kmemleak_scan_area);
1336
1337	/**
1338	* kmemleak_no_scan - do not scan an allocated object
1339	* @ptr: pointer to beginning of the object
1340	*
1341	* This function notifies kmemleak not to scan the given memory block. Useful
1342	* in situations where it is known that the given object does not contain any
1343	* references to other objects. Kmemleak will not scan such objects reducing
1344	* the number of false negatives.
1345	*/
1346	void __ref kmemleak_no_scan(const void *ptr)
1347	{
1348	pr_debug("%s(0x%px)\n", __func__, ptr);
1349
1350	if (kmemleak_enabled && ptr && !IS_ERR(ptr))
1351	object_no_scan((unsigned long)ptr);
1352	}
1353	EXPORT_SYMBOL(kmemleak_no_scan);
1354
1355	/**
1356	* kmemleak_alloc_phys - similar to kmemleak_alloc but taking a physical
1357	* address argument
1358	* @phys: physical address of the object
1359	* @size: size of the object
1360	* @gfp: kmalloc() flags used for kmemleak internal memory allocations
1361	*/
1362	void __ref kmemleak_alloc_phys(phys_addr_t phys, size_t size, gfp_t gfp)
1363	{
1364	pr_debug("%s(0x%px, %zu)\n", __func__, &phys, size);
1365
1366	if (kmemleak_enabled)
1367	/*
1368	* Create object with OBJECT_PHYS flag and
1369	* assume min_count 0.
1370	*/
1371	create_object_phys((unsigned long)phys, size, 0, gfp);
1372	}
1373	EXPORT_SYMBOL(kmemleak_alloc_phys);
1374
1375	/**
1376	* kmemleak_free_part_phys - similar to kmemleak_free_part but taking a
1377	* physical address argument
1378	* @phys: physical address if the beginning or inside an object. This
1379	* also represents the start of the range to be freed
1380	* @size: size to be unregistered
1381	*/
1382	void __ref kmemleak_free_part_phys(phys_addr_t phys, size_t size)
1383	{
1384	pr_debug("%s(0x%px)\n", __func__, &phys);
1385
1386	if (kmemleak_enabled)
1387	delete_object_part((unsigned long)phys, size, OBJECT_PHYS);
1388	}
1389	EXPORT_SYMBOL(kmemleak_free_part_phys);
1390
1391	/**
1392	* kmemleak_ignore_phys - similar to kmemleak_ignore but taking a physical
1393	* address argument
1394	* @phys: physical address of the object
1395	*/
1396	void __ref kmemleak_ignore_phys(phys_addr_t phys)
1397	{
1398	pr_debug("%s(0x%px)\n", __func__, &phys);
1399
1400	if (kmemleak_enabled)
1401	make_black_object((unsigned long)phys, OBJECT_PHYS);
1402	}
1403	EXPORT_SYMBOL(kmemleak_ignore_phys);
1404
1405	/*
1406	* Update an object's checksum and return true if it was modified.
1407	*/
1408	static bool update_checksum(struct kmemleak_object *object)
1409	{
1410	u32 old_csum = object->checksum;
1411
1412	if (WARN_ON_ONCE(object->flags & OBJECT_PHYS))
1413	return false;
1414
1415	kasan_disable_current();
1416	kcsan_disable_current();
1417	if (object->flags & OBJECT_PERCPU) {
1418	unsigned int cpu;
1419
1420	object->checksum = 0;
1421	for_each_possible_cpu(cpu) {
1422	void ptr = per_cpu_ptr((void __percpu )object->pointer, cpu);
1423
1424	object->checksum = crc32(object->checksum,
1425	kasan_reset_tag((void *)ptr), object->size);
1426	}
1427	} else {
1428	object->checksum = crc32(0, kasan_reset_tag((void *)object->pointer), object->size);
1429	}
1430	kasan_enable_current();
1431	kcsan_enable_current();
1432
1433	return object->checksum != old_csum;
1434	}
1435
1436	/*
1437	* Update an object's references. object->lock must be held by the caller.
1438	*/
1439	static void update_refs(struct kmemleak_object *object)
1440	{
1441	if (!color_white(object)) {
1442	/* non-orphan, ignored or new */
1443	return;
1444	}
1445
1446	/*
1447	* Increase the object's reference count (number of pointers to the
1448	* memory block). If this count reaches the required minimum, the
1449	* object's color will become gray and it will be added to the
1450	* gray_list.
1451	*/
1452	object->count++;
1453	if (color_gray(object)) {
1454	/* referenced after all, no longer a suspect */
1455	if (object->flags & OBJECT_SUSPECT) {
1456	object->flags &= ~OBJECT_SUSPECT;
1457	nr_suspects--;
1458	}
1459	/* put_object() called when removing from gray_list */
1460	WARN_ON(!get_object(object));
1461	list_add_tail(&object->gray_list, &gray_list);
1462	}
1463	}
1464
1465	static void pointer_update_refs(struct kmemleak_object *scanned,
1466	unsigned long pointer, unsigned int objflags)
1467	{
1468	struct kmemleak_object *object;
1469	unsigned long untagged_ptr;
1470	unsigned long excess_ref;
1471
1472	untagged_ptr = (unsigned long)kasan_reset_tag((void *)pointer);
1473	if (objflags & OBJECT_PERCPU) {
1474	if (untagged_ptr < min_percpu_addr \|\| untagged_ptr >= max_percpu_addr)
1475	return;
1476	} else {
1477	if (untagged_ptr < min_addr \|\| untagged_ptr >= max_addr)
1478	return;
1479	}
1480
1481	/*
1482	* No need for get_object() here since we hold kmemleak_lock.
1483	* object->use_count cannot be dropped to 0 while the object
1484	* is still present in object_tree_root and object_list
1485	* (with updates protected by kmemleak_lock).
1486	*/
1487	object = __lookup_object(pointer, 1, objflags);
1488	if (!object)
1489	return;
1490	if (object == scanned)
1491	/* self referenced, ignore */
1492	return;
1493
1494	/*
1495	* Avoid the lockdep recursive warning on object->lock being
1496	* previously acquired in scan_object(). These locks are
1497	* enclosed by scan_mutex.
1498	*/
1499	raw_spin_lock_nested(&object->lock, SINGLE_DEPTH_NESTING);
1500	/* only pass surplus references (object already gray) */
1501	if (color_gray(object)) {
1502	excess_ref = object->excess_ref;
1503	/* no need for update_refs() if object already gray */
1504	} else {
1505	excess_ref = 0;
1506	update_refs(object);
1507	}
1508	raw_spin_unlock(&object->lock);
1509
1510	if (excess_ref) {
1511	object = lookup_object(excess_ref, 0);
1512	if (!object)
1513	return;
1514	if (object == scanned)
1515	/* circular reference, ignore */
1516	return;
1517	raw_spin_lock_nested(&object->lock, SINGLE_DEPTH_NESTING);
1518	update_refs(object);
1519	raw_spin_unlock(&object->lock);
1520	}
1521	}
1522
1523	/*
1524	* Memory scanning is a long process and it needs to be interruptible. This
1525	* function checks whether such interrupt condition occurred.
1526	*/
1527	static int scan_should_stop(void)
1528	{
1529	if (!kmemleak_enabled)
1530	return 1;
1531
1532	/*
1533	* This function may be called from either process or kthread context,
1534	* hence the need to check for both stop conditions.
1535	*/
1536	if (current->flags & PF_KTHREAD)
1537	return kthread_should_stop();
1538
1539	return signal_pending(current);
1540	}
1541
1542	/*
1543	* Scan a memory block (exclusive range) for valid pointers and add those
1544	* found to the gray list. Return non-zero if the scan was interrupted.
1545	*/
1546	static int scan_block(void _start, void _end,
1547	struct kmemleak_object *scanned)
1548	{
1549	unsigned long *ptr;
1550	unsigned long *start = PTR_ALIGN(_start, BYTES_PER_POINTER);
1551	unsigned long *end = _end - (BYTES_PER_POINTER - 1);
1552	unsigned long flags;
1553	int stop = 0;
1554
1555	raw_spin_lock_irqsave(&kmemleak_lock, flags);
1556	for (ptr = start; ptr < end; ptr++) {
1557	unsigned long pointer;
1558
1559	if (scan_should_stop()) {
1560	stop = 1;
1561	break;
1562	}
1563
1564	kasan_disable_current();
1565	pointer = (unsigned long )kasan_reset_tag((void *)ptr);
1566	kasan_enable_current();
1567
1568	pointer_update_refs(scanned, pointer, 0);
1569	pointer_update_refs(scanned, pointer, OBJECT_PERCPU);
1570	}
1571	raw_spin_unlock_irqrestore(&kmemleak_lock, flags);
1572
1573	return stop;
1574	}
1575
1576	/*
1577	* Scan a large memory block in MAX_SCAN_SIZE chunks to reduce the latency.
1578	* Return non-zero if the scan was interrupted.
1579	*/
1580	#ifdef CONFIG_SMP
1581	static int scan_large_block(void start, void end)
1582	{
1583	void *next;
1584
1585	while (start < end) {
1586	next = min(start + MAX_SCAN_SIZE, end);
1587	if (scan_block(start, next, NULL))
1588	return 1;
1589	start = next;
1590	cond_resched_tasks_rcu_qs();
1591	}
1592
1593	return 0;
1594	}
1595	#endif
1596
1597	/*
1598	* Scan a memory block corresponding to a kmemleak_object. A condition is
1599	* that object->use_count >= 1.
1600	*/
1601	static void scan_object(struct kmemleak_object *object)
1602	{
1603	struct kmemleak_scan_area *area;
1604	unsigned long flags;
1605
1606	/*
1607	* Once the object->lock is acquired, the corresponding memory block
1608	* cannot be freed (the same lock is acquired in delete_object).
1609	*/
1610	raw_spin_lock_irqsave(&object->lock, flags);
1611	if (object->flags & OBJECT_NO_SCAN)
1612	goto out;
1613	if (!(object->flags & OBJECT_ALLOCATED))
1614	/* already freed object */
1615	goto out;
1616
1617	if (object->flags & OBJECT_PERCPU) {
1618	unsigned int cpu;
1619
1620	for_each_possible_cpu(cpu) {
1621	void start = per_cpu_ptr((void __percpu )object->pointer, cpu);
1622	void *end = start + object->size;
1623
1624	scan_block(start, end, object);
1625
1626	raw_spin_unlock_irqrestore(&object->lock, flags);
1627	cond_resched_tasks_rcu_qs();
1628	raw_spin_lock_irqsave(&object->lock, flags);
1629	if (!(object->flags & OBJECT_ALLOCATED))
1630	break;
1631	}
1632	} else if (hlist_empty(&object->area_list) \|\|
1633	object->flags & OBJECT_FULL_SCAN) {
1634	void *start = object->flags & OBJECT_PHYS ?
1635	__va((phys_addr_t)object->pointer) :
1636	(void *)object->pointer;
1637	void *end = start + object->size;
1638	void *next;
1639
1640	do {
1641	next = min(start + MAX_SCAN_SIZE, end);
1642	scan_block(start, next, object);
1643
1644	start = next;
1645	if (start >= end)
1646	break;
1647
1648	raw_spin_unlock_irqrestore(&object->lock, flags);
1649	cond_resched_tasks_rcu_qs();
1650	raw_spin_lock_irqsave(&object->lock, flags);
1651	} while (object->flags & OBJECT_ALLOCATED);
1652	} else {
1653	hlist_for_each_entry(area, &object->area_list, node)
1654	scan_block((void *)area->start,
1655	(void *)(area->start + area->size),
1656	object);
1657	}
1658	out:
1659	raw_spin_unlock_irqrestore(&object->lock, flags);
1660	}
1661
1662	/*
1663	* Scan the objects already referenced (gray objects). More objects will be
1664	* referenced and, if there are no memory leaks, all the objects are scanned.
1665	*/
1666	static void scan_gray_list(void)
1667	{
1668	struct kmemleak_object object, tmp;
1669
1670	/*
1671	* The list traversal is safe for both tail additions and removals
1672	* from inside the loop. The kmemleak objects cannot be freed from
1673	* outside the loop because their use_count was incremented.
1674	*/
1675	object = list_entry(gray_list.next, typeof(*object), gray_list);
1676	while (&object->gray_list != &gray_list) {
1677	cond_resched_tasks_rcu_qs();
1678
1679	/* may add new objects to the list */
1680	if (!scan_should_stop())
1681	scan_object(object);
1682
1683	tmp = list_entry(object->gray_list.next, typeof(*object),
1684	gray_list);
1685
1686	/* remove the object from the list and release it */
1687	list_del(&object->gray_list);
1688	put_object(object);
1689
1690	object = tmp;
1691	}
1692	WARN_ON(!list_empty(&gray_list));
1693	}
1694
1695	/*
1696	* Conditionally call resched() in an object iteration loop while making sure
1697	* that the given object won't go away without RCU read lock by performing a
1698	* get_object() if necessaary.
1699	*/
1700	static void kmemleak_cond_resched(struct kmemleak_object *object)
1701	{
1702	if (!get_object(object))
1703	return; /* Try next object */
1704
1705	raw_spin_lock_irq(&kmemleak_lock);
1706	if (object->del_state & DELSTATE_REMOVED)
1707	goto unlock_put; /* Object removed */
1708	object->del_state \|= DELSTATE_NO_DELETE;
1709	raw_spin_unlock_irq(&kmemleak_lock);
1710
1711	rcu_read_unlock();
1712	cond_resched_tasks_rcu_qs();
1713	rcu_read_lock();
1714
1715	raw_spin_lock_irq(&kmemleak_lock);
1716	if (object->del_state & DELSTATE_REMOVED)
1717	list_del_rcu(&object->object_list);
1718	object->del_state &= ~DELSTATE_NO_DELETE;
1719	unlock_put:
1720	raw_spin_unlock_irq(&kmemleak_lock);
1721	put_object(object);
1722	}
1723
1724	/*
1725	* Scan all task kernel stacks, rescheduling between tasks. Each task is looked
1726	* up and pinned within its own RCU read-side section, so no lock is held across
1727	* the scan and the walk cannot trip the soft lockup watchdog.
1728	*/
1729	static void kmemleak_scan_task_stacks(void)
1730	{
1731	struct pid *pid;
1732	int nr = 1;
1733	int stop = 0;
1734
1735	do {
1736	struct task_struct *p = NULL;
1737
1738	rcu_read_lock();
1739	pid = find_ge_pid(nr, &init_pid_ns);
1740	if (pid) {
1741	nr = pid_nr(pid) + 1;
1742	p = pid_task(pid, PIDTYPE_PID);
1743	if (p)
1744	get_task_struct(p);
1745	}
1746	rcu_read_unlock();
1747
1748	if (p) {
1749	void *stack = try_get_task_stack(p);
1750
1751	if (stack) {
1752	stop = scan_block(stack, stack + THREAD_SIZE, NULL);
1753	put_task_stack(p);
1754	}
1755	put_task_struct(p);
1756	}
1757	cond_resched_tasks_rcu_qs();
1758	} while (pid && !stop);
1759	}
1760
1761	/*
1762	* Print one leak inline. The hex dump is gated on OBJECT_ALLOCATED so it
1763	* does not touch user memory that was freed concurrently; the rest of the
1764	* report (backtrace, comm, pid) is always emitted since the kmemleak_object
1765	* metadata is pinned by the caller.
1766	*/
1767	static void print_leak_locked(struct kmemleak_object *object, bool hex_dump)
1768	{
1769	raw_spin_lock_irq(&object->lock);
1770	__print_unreferenced(NULL, object,
1771	hex_dump && (object->flags & OBJECT_ALLOCATED));
1772	raw_spin_unlock_irq(&object->lock);
1773	}
1774
1775	/*
1776	* Per-scan dedup table for verbose leak printing. The xarray is keyed by
1777	* stackdepot trace_handle and stores a pointer to the representative
1778	* kmemleak_object. The per-scan repeat count lives in object->dup_count.
1779	*
1780	* dedup_record() must run outside object->lock: xa_store() may take
1781	* mutexes (xa_node slab allocation) which lockdep would flag against the
1782	* raw spinlock object->lock.
1783	*/
1784	static void dedup_record(struct xarray dedup, struct kmemleak_object object,
1785	depot_stack_handle_t trace_handle)
1786	{
1787	struct kmemleak_object *rep;
1788	void *old;
1789
1790	/*
1791	* No stack trace to dedup against: early-boot allocation tracked
1792	* before kmemleak_init() set up object_cache, or stack_depot_save()
1793	* failure under memory pressure.
1794	*/
1795	if (!trace_handle) {
1796	print_leak_locked(object, true);
1797	return;
1798	}
1799
1800	/* stack is available, now we can de-dup */
1801	rep = xa_load(dedup, trace_handle);
1802	if (rep) {
1803	rep->dup_count++;
1804	return;
1805	}
1806
1807	/*
1808	* Object is being torn down (use_count already hit zero); the
1809	* tracked memory at object->pointer is unsafe to read, so skip.
1810	*/
1811	if (!get_object(object))
1812	return;
1813
1814	object->dup_count = 1;
1815	old = xa_store(dedup, trace_handle, object, GFP_ATOMIC);
1816	if (xa_is_err(old)) {
1817	/* xa_node allocation failed; fall back to inline print. */
1818	print_leak_locked(object, true);
1819	put_object(object);
1820	return;
1821	}
1822	/*
1823	* scan_mutex serialises all writers to the dedup xarray, so xa_store()
1824	* after a NULL xa_load() must always overwrite an empty slot.
1825	*/
1826	WARN_ON_ONCE(old);
1827	}
1828
1829	/*
1830	* Drain the dedup table. Re-acquires object->lock and re-checks
1831	* OBJECT_ALLOCATED before printing: while get_object() pins the
1832	* kmemleak_object metadata, the underlying tracked allocation may have
1833	* been freed since the scan walked it (kmemleak_free clears
1834	* OBJECT_ALLOCATED under object->lock before the user memory goes away).
1835	* The hex dump is skipped for coalesced entries since the bytes would
1836	* differ across objects anyway.
1837	*/
1838	static void dedup_flush(struct xarray *dedup)
1839	{
1840	struct kmemleak_object *object;
1841	unsigned long idx;
1842	unsigned int dup;
1843	bool coalesced;
1844
1845	xa_for_each(dedup, idx, object) {
1846	dup = object->dup_count;
1847	coalesced = dup > 1;
1848
1849	print_leak_locked(object, !coalesced);
1850	if (coalesced)
1851	pr_warn(" ... and %u more object(s) with the same backtrace\n",
1852	dup - 1);
1853	put_object(object);
1854	xa_erase(dedup, idx);
1855	}
1856	}
1857
1858	/*
1859	* Scan data sections and all the referenced memory blocks allocated via the
1860	* kernel's standard allocators. This function must be called with the
1861	* scan_mutex held.
1862	*/
1863	static int __kmemleak_scan(bool full)
1864	{
1865	struct kmemleak_object *object;
1866	struct zone *zone;
1867	int __maybe_unused i;
1868	int stop = 0;
1869
1870	jiffies_last_scan = jiffies;
1871	if (full)
1872	nr_suspects = 0;
1873
1874	/* prepare the kmemleak_object's */
1875	rcu_read_lock();
1876	list_for_each_entry_rcu(object, &object_list, object_list) {
1877	raw_spin_lock_irq(&object->lock);
1878	#ifdef DEBUG
1879	/*
1880	* With a few exceptions there should be a maximum of
1881	* 1 reference to any object at this point.
1882	*/
1883	if (atomic_read(&object->use_count) > 1) {
1884	pr_debug("object->use_count = %d\n",
1885	atomic_read(&object->use_count));
1886	dump_object_info(object);
1887	}
1888	#endif
1889
1890	/* ignore objects outside lowmem (paint them black) */
1891	if ((object->flags & OBJECT_PHYS) &&
1892	!(object->flags & OBJECT_NO_SCAN)) {
1893	unsigned long phys = object->pointer;
1894
1895	if (PHYS_PFN(phys) < min_low_pfn \|\|
1896	PHYS_PFN(phys + object->size) > max_low_pfn)
1897	__paint_it(object, KMEMLEAK_BLACK);
1898	}
1899
1900	/* referenced last scan: restart the unreferenced run */
1901	if (!color_white(object))
1902	object->unref_scans = 0;
1903	/* reset the reference count (whiten the object) */
1904	object->count = 0;
1905	if (full)
1906	object->flags &= ~OBJECT_SUSPECT;
1907	if (color_gray(object) && get_object(object))
1908	list_add_tail(&object->gray_list, &gray_list);
1909
1910	raw_spin_unlock_irq(&object->lock);
1911
1912	if (need_resched())
1913	kmemleak_cond_resched(object);
1914	}
1915	rcu_read_unlock();
1916
1917	#ifdef CONFIG_SMP
1918	/* per-cpu sections scanning */
1919	for_each_possible_cpu(i) {
1920	if (scan_large_block(__per_cpu_start + per_cpu_offset(i),
1921	__per_cpu_end + per_cpu_offset(i)))
1922	goto scan_gray;
1923	}
1924	#endif
1925
1926	/*
1927	* Struct page scanning for each node.
1928	*/
1929	get_online_mems();
1930	for_each_populated_zone(zone) {
1931	unsigned long start_pfn = zone->zone_start_pfn;
1932	unsigned long end_pfn = zone_end_pfn(zone);
1933	unsigned long pfn;
1934
1935	for (pfn = start_pfn; pfn < end_pfn; pfn++) {
1936	struct page *page = pfn_to_online_page(pfn);
1937
1938	if (!(pfn & 63))
1939	cond_resched_tasks_rcu_qs();
1940
1941	if (!page)
1942	continue;
1943
1944	/* only scan pages belonging to this zone */
1945	if (page_zone(page) != zone)
1946	continue;
1947	/* only scan if page is in use */
1948	if (page_count(page) == 0)
1949	continue;
1950	stop = scan_block(page, page + 1, NULL);
1951	if (stop)
1952	break;
1953	}
1954	if (stop)
1955	break;
1956	}
1957	put_online_mems();
1958	if (stop)
1959	goto scan_gray;
1960
1961	/*
1962	* Scanning the task stacks (may introduce false negatives).
1963	*/
1964	if (kmemleak_stack_scan)
1965	kmemleak_scan_task_stacks();
1966
1967	/*
1968	* Scan the objects already referenced from the sections scanned
1969	* above.
1970	*/
1971	scan_gray:
1972	scan_gray_list();
1973
1974	/* a confirmation scan does not look for modified objects */
1975	if (!full)
1976	return nr_suspects;
1977
1978	/*
1979	* Check for new or unreferenced objects modified since the previous
1980	* scan and color them gray until the next scan.
1981	*/
1982	rcu_read_lock();
1983	list_for_each_entry_rcu(object, &object_list, object_list) {
1984	if (need_resched())
1985	kmemleak_cond_resched(object);
1986
1987	/*
1988	* This is racy but we can save the overhead of lock/unlock
1989	* calls. The missed objects, if any, should be caught in
1990	* the next scan.
1991	*/
1992	if (!color_white(object))
1993	continue;
1994	raw_spin_lock_irq(&object->lock);
1995	if (color_white(object) && (object->flags & OBJECT_ALLOCATED)
1996	&& update_checksum(object) && get_object(object)) {
1997	/* color it gray temporarily */
1998	object->count = object->min_count;
1999	list_add_tail(&object->gray_list, &gray_list);
2000	} else if (unreferenced_object(object) &&
2001	!(object->flags & OBJECT_REPORTED)) {
2002	/* flag the objects left unreferenced by this scan */
2003	object->flags \|= OBJECT_SUSPECT;
2004	nr_suspects++;
2005	}
2006	raw_spin_unlock_irq(&object->lock);
2007	}
2008	rcu_read_unlock();
2009
2010	/*
2011	* Re-scan the gray list for modified unreferenced objects.
2012	*/
2013	scan_gray_list();
2014
2015	return nr_suspects;
2016	}
2017
2018	/*
2019	* Promote a suspected object to a reported leak once it has stayed
2020	* unreferenced for min_unref_scans consecutive scans. Called with
2021	* object->lock held; returns true when the object is newly reported.
2022	*/
2023	static bool confirm_leak(struct kmemleak_object *object)
2024	{
2025	if (!unreferenced_object(object) \|\|
2026	!(object->flags & OBJECT_SUSPECT) \|\|
2027	(object->flags & OBJECT_REPORTED))
2028	return false;
2029
2030	object->unref_scans += 1;
2031	if (object->unref_scans < min_unref_scans)
2032	return false;
2033
2034	object->flags \|= OBJECT_REPORTED;
2035	return true;
2036	}
2037
2038	/*
2039	* Scan the memory and report the unreferenced objects as leaks. Must be
2040	* called with the scan_mutex held.
2041	*/
2042	static void kmemleak_scan(void)
2043	{
2044	struct kmemleak_object *object;
2045	struct xarray dedup;
2046	int new_leaks = 0;
2047
2048	/*
2049	* Full scan. Objects left unreferenced are flagged OBJECT_SUSPECT and
2050	* counted in the return value; nothing to confirm or report otherwise.
2051	*/
2052	if (!__kmemleak_scan(true))
2053	return;
2054
2055	/*
2056	* If scanning was stopped do not report any new unreferenced objects.
2057	*/
2058	if (scan_should_stop())
2059	return;
2060
2061	/*
2062	* A live object whose only reference is moved by, for example, a
2063	* concurrent RCU update can be missed for one scan and reported as a
2064	* transient false positive. Scan again and only report the objects
2065	* left unreferenced (still flagged OBJECT_SUSPECT) by both scans.
2066	*/
2067	__kmemleak_scan(false);
2068	if (scan_should_stop())
2069	return;
2070
2071	/*
2072	* Scanning result reporting. When verbose printing is enabled, dedupe
2073	* by stackdepot trace_handle so each unique backtrace is logged once
2074	* per scan, annotated with the number of objects that share it. The
2075	* per-leak count below still reflects every object, and
2076	* /sys/kernel/debug/kmemleak still lists them individually.
2077	*/
2078	xa_init(&dedup);
2079	rcu_read_lock();
2080	list_for_each_entry_rcu(object, &object_list, object_list) {
2081	depot_stack_handle_t trace_handle;
2082	bool dedup_print;
2083
2084	if (need_resched())
2085	kmemleak_cond_resched(object);
2086
2087	/*
2088	* This is racy but we can save the overhead of lock/unlock
2089	* calls. The missed objects, if any, should be caught in
2090	* the next scan.
2091	*/
2092	if (!color_white(object))
2093	continue;
2094	raw_spin_lock_irq(&object->lock);
2095	trace_handle = 0;
2096	dedup_print = false;
2097
2098	if (confirm_leak(object)) {
2099	if (kmemleak_verbose) {
2100	trace_handle = object->trace_handle;
2101	dedup_print = true;
2102	}
2103	new_leaks++;
2104	}
2105	raw_spin_unlock_irq(&object->lock);
2106
2107	/*
2108	* Defer the verbose print outside object->lock: xa_store()
2109	* may take xa_node slab locks at a higher wait-context level
2110	* which lockdep would flag against the raw_spinlock_t
2111	* object->lock. rcu_read_lock() keeps the kmemleak_object
2112	* alive across the call.
2113	*/
2114	if (dedup_print)
2115	dedup_record(&dedup, object, trace_handle);
2116	}
2117	rcu_read_unlock();
2118	/* Flush'em all */
2119	dedup_flush(&dedup);
2120	xa_destroy(&dedup);
2121
2122	if (new_leaks) {
2123	kmemleak_found_leaks = true;
2124
2125	pr_info("%d new suspected memory leaks (see /sys/kernel/debug/kmemleak)\n",
2126	new_leaks);
2127	}
2128
2129	}
2130
2131	/*
2132	* Thread function performing automatic memory scanning. Unreferenced objects
2133	* at the end of a memory scan are reported but only the first time.
2134	*/
2135	static int kmemleak_scan_thread(void *arg)
2136	{
2137	static int first_run = IS_ENABLED(CONFIG_DEBUG_KMEMLEAK_AUTO_SCAN);
2138
2139	pr_info("Automatic memory scanning thread started\n");
2140	set_user_nice(current, 10);
2141
2142	/*
2143	* Wait before the first scan to allow the system to fully initialize.
2144	*/
2145	if (first_run) {
2146	signed long timeout = secs_to_jiffies(SECS_FIRST_SCAN);
2147	first_run = 0;
2148	while (timeout && !kthread_should_stop())
2149	timeout = schedule_timeout_interruptible(timeout);
2150	}
2151
2152	while (!kthread_should_stop()) {
2153	signed long timeout = READ_ONCE(jiffies_scan_wait);
2154
2155	mutex_lock(&scan_mutex);
2156	kmemleak_scan();
2157	mutex_unlock(&scan_mutex);
2158
2159	/* wait before the next scan */
2160	while (timeout && !kthread_should_stop())
2161	timeout = schedule_timeout_interruptible(timeout);
2162	}
2163
2164	pr_info("Automatic memory scanning thread ended\n");
2165
2166	return 0;
2167	}
2168
2169	/*
2170	* Start the automatic memory scanning thread. This function must be called
2171	* with the scan_mutex held.
2172	*/
2173	static void start_scan_thread(void)
2174	{
2175	if (scan_thread)
2176	return;
2177	scan_thread = kthread_run(kmemleak_scan_thread, NULL, "kmemleak");
2178	if (IS_ERR(scan_thread)) {
2179	pr_warn("Failed to create the scan thread\n");
2180	scan_thread = NULL;
2181	}
2182	}
2183
2184	/*
2185	* Stop the automatic memory scanning thread.
2186	*/
2187	static void stop_scan_thread(void)
2188	{
2189	if (scan_thread) {
2190	kthread_stop(scan_thread);
2191	scan_thread = NULL;
2192	}
2193	}
2194
2195	/*
2196	* Iterate over the object_list and return the first valid object at or after
2197	* the required position with its use_count incremented. The function triggers
2198	* a memory scanning when the pos argument points to the first position.
2199	*/
2200	static void kmemleak_seq_start(struct seq_file seq, loff_t *pos)
2201	{
2202	struct kmemleak_object *object;
2203	loff_t n = *pos;
2204	int err;
2205
2206	err = mutex_lock_interruptible(&scan_mutex);
2207	if (err < 0)
2208	return ERR_PTR(err);
2209
2210	rcu_read_lock();
2211	list_for_each_entry_rcu(object, &object_list, object_list) {
2212	if (n-- > 0)
2213	continue;
2214	if (get_object(object))
2215	goto out;
2216	}
2217	object = NULL;
2218	out:
2219	return object;
2220	}
2221
2222	/*
2223	* Return the next object in the object_list. The function decrements the
2224	* use_count of the previous object and increases that of the next one.
2225	*/
2226	static void kmemleak_seq_next(struct seq_file seq, void v, loff_t pos)
2227	{
2228	struct kmemleak_object *prev_obj = v;
2229	struct kmemleak_object *next_obj = NULL;
2230	struct kmemleak_object *obj = prev_obj;
2231
2232	++(*pos);
2233
2234	list_for_each_entry_continue_rcu(obj, &object_list, object_list) {
2235	if (get_object(obj)) {
2236	next_obj = obj;
2237	break;
2238	}
2239	}
2240
2241	put_object(prev_obj);
2242	return next_obj;
2243	}
2244
2245	/*
2246	* Decrement the use_count of the last object required, if any.
2247	*/
2248	static void kmemleak_seq_stop(struct seq_file seq, void v)
2249	{
2250	if (!IS_ERR(v)) {
2251	/*
2252	* kmemleak_seq_start may return ERR_PTR if the scan_mutex
2253	* waiting was interrupted, so only release it if !IS_ERR.
2254	*/
2255	rcu_read_unlock();
2256	mutex_unlock(&scan_mutex);
2257	if (v)
2258	put_object(v);
2259	}
2260	}
2261
2262	/*
2263	* Print the information for an unreferenced object to the seq file.
2264	*/
2265	static int kmemleak_seq_show(struct seq_file seq, void v)
2266	{
2267	struct kmemleak_object *object = v;
2268	unsigned long flags;
2269
2270	raw_spin_lock_irqsave(&object->lock, flags);
2271	if ((object->flags & OBJECT_REPORTED) && unreferenced_object(object))
2272	print_unreferenced(seq, object);
2273	raw_spin_unlock_irqrestore(&object->lock, flags);
2274	return 0;
2275	}
2276
2277	static const struct seq_operations kmemleak_seq_ops = {
2278	.start = kmemleak_seq_start,
2279	.next = kmemleak_seq_next,
2280	.stop = kmemleak_seq_stop,
2281	.show = kmemleak_seq_show,
2282	};
2283
2284	static int kmemleak_open(struct inode inode, struct file file)
2285	{
2286	return seq_open(file, &kmemleak_seq_ops);
2287	}
2288
2289	static bool __dump_str_object_info(unsigned long addr, unsigned int objflags)
2290	{
2291	unsigned long flags;
2292	struct kmemleak_object *object;
2293
2294	object = __find_and_get_object(addr, 1, objflags);
2295	if (!object)
2296	return false;
2297
2298	raw_spin_lock_irqsave(&object->lock, flags);
2299	dump_object_info(object);
2300	raw_spin_unlock_irqrestore(&object->lock, flags);
2301
2302	put_object(object);
2303
2304	return true;
2305	}
2306
2307	static int dump_str_object_info(const char *str)
2308	{
2309	unsigned long addr;
2310	bool found = false;
2311
2312	if (kstrtoul(str, 0, &addr))
2313	return -EINVAL;
2314
2315	found \|= __dump_str_object_info(addr, 0);
2316	found \|= __dump_str_object_info(addr, OBJECT_PHYS);
2317	found \|= __dump_str_object_info(addr, OBJECT_PERCPU);
2318
2319	if (!found) {
2320	pr_info("Unknown object at 0x%08lx\n", addr);
2321	return -EINVAL;
2322	}
2323
2324	return 0;
2325	}
2326
2327	/*
2328	* We use grey instead of black to ensure we can do future scans on the same
2329	* objects. If we did not do future scans these black objects could
2330	* potentially contain references to newly allocated objects in the future and
2331	* we'd end up with false positives.
2332	*/
2333	static void kmemleak_clear(void)
2334	{
2335	struct kmemleak_object *object;
2336
2337	rcu_read_lock();
2338	list_for_each_entry_rcu(object, &object_list, object_list) {
2339	raw_spin_lock_irq(&object->lock);
2340	if ((object->flags & OBJECT_REPORTED) &&
2341	unreferenced_object(object))
2342	__paint_it(object, KMEMLEAK_GREY);
2343	raw_spin_unlock_irq(&object->lock);
2344	}
2345	rcu_read_unlock();
2346
2347	kmemleak_found_leaks = false;
2348	}
2349
2350	static void __kmemleak_do_cleanup(void);
2351
2352	/*
2353	* File write operation to configure kmemleak at run-time. The following
2354	* commands can be written to the /sys/kernel/debug/kmemleak file:
2355	* off - disable kmemleak (irreversible)
2356	* stack=on - enable the task stacks scanning
2357	* stack=off - disable the tasks stacks scanning
2358	* scan=on - start the automatic memory scanning thread
2359	* scan=off - stop the automatic memory scanning thread
2360	* scan=... - set the automatic memory scanning period in seconds (0 to
2361	* disable it)
2362	* scan - trigger a memory scan
2363	* clear - mark all current reported unreferenced kmemleak objects as
2364	* grey to ignore printing them, or free all kmemleak objects
2365	* if kmemleak has been disabled.
2366	* dump=... - dump information about the object found at the given address
2367	*/
2368	static ssize_t kmemleak_write(struct file file, const char __user user_buf,
2369	size_t size, loff_t *ppos)
2370	{
2371	char buf[64];
2372	int buf_size;
2373	int ret;
2374
2375	buf_size = min(size, (sizeof(buf) - 1));
2376	if (strncpy_from_user(buf, user_buf, buf_size) < 0)
2377	return -EFAULT;
2378	buf[buf_size] = 0;
2379
2380	ret = mutex_lock_interruptible(&scan_mutex);
2381	if (ret < 0)
2382	return ret;
2383
2384	if (strncmp(buf, "clear", 5) == 0) {
2385	if (kmemleak_enabled)
2386	kmemleak_clear();
2387	else
2388	__kmemleak_do_cleanup();
2389	goto out;
2390	}
2391
2392	if (!kmemleak_enabled) {
2393	ret = -EPERM;
2394	goto out;
2395	}
2396
2397	if (strncmp(buf, "off", 3) == 0)
2398	kmemleak_disable();
2399	else if (strncmp(buf, "stack=on", 8) == 0)
2400	kmemleak_stack_scan = 1;
2401	else if (strncmp(buf, "stack=off", 9) == 0)
2402	kmemleak_stack_scan = 0;
2403	else if (strncmp(buf, "scan=on", 7) == 0)
2404	start_scan_thread();
2405	else if (strncmp(buf, "scan=off", 8) == 0)
2406	stop_scan_thread();
2407	else if (strncmp(buf, "scan=", 5) == 0) {
2408	unsigned secs;
2409	unsigned long msecs;
2410
2411	ret = kstrtouint(buf + 5, 0, &secs);
2412	if (ret < 0)
2413	goto out;
2414
2415	msecs = secs * MSEC_PER_SEC;
2416	if (msecs > UINT_MAX)
2417	msecs = UINT_MAX;
2418
2419	stop_scan_thread();
2420	if (msecs) {
2421	WRITE_ONCE(jiffies_scan_wait, msecs_to_jiffies(msecs));
2422	start_scan_thread();
2423	}
2424	} else if (strncmp(buf, "scan", 4) == 0)
2425	kmemleak_scan();
2426	else if (strncmp(buf, "dump=", 5) == 0)
2427	ret = dump_str_object_info(buf + 5);
2428	else
2429	ret = -EINVAL;
2430
2431	out:
2432	mutex_unlock(&scan_mutex);
2433	if (ret < 0)
2434	return ret;
2435
2436	/* ignore the rest of the buffer, only one command at a time */
2437	*ppos += size;
2438	return size;
2439	}
2440
2441	static const struct file_operations kmemleak_fops = {
2442	.owner = THIS_MODULE,
2443	.open = kmemleak_open,
2444	.read = seq_read,
2445	.write = kmemleak_write,
2446	.llseek = seq_lseek,
2447	.release = seq_release,
2448	};
2449
2450	static void __kmemleak_do_cleanup(void)
2451	{
2452	struct kmemleak_object object, tmp;
2453	unsigned int cnt = 0;
2454
2455	/*
2456	* Kmemleak has already been disabled, no need for RCU list traversal
2457	* or kmemleak_lock held.
2458	*/
2459	list_for_each_entry_safe(object, tmp, &object_list, object_list) {
2460	__remove_object(object);
2461	__delete_object(object);
2462
2463	/* Call cond_resched() once per 64 iterations to avoid soft lockup */
2464	if (!(++cnt & 0x3f))
2465	cond_resched();
2466	}
2467	}
2468
2469	/*
2470	* Stop the memory scanning thread and free the kmemleak internal objects if
2471	* no previous scan thread (otherwise, kmemleak may still have some useful
2472	* information on memory leaks).
2473	*/
2474	static void kmemleak_do_cleanup(struct work_struct *work)
2475	{
2476	stop_scan_thread();
2477
2478	mutex_lock(&scan_mutex);
2479	/*
2480	* Once it is made sure that kmemleak_scan has stopped, it is safe to no
2481	* longer track object freeing. Ordering of the scan thread stopping and
2482	* the memory accesses below is guaranteed by the kthread_stop()
2483	* function.
2484	*/
2485	kmemleak_free_enabled = 0;
2486	mutex_unlock(&scan_mutex);
2487
2488	if (!kmemleak_found_leaks)
2489	__kmemleak_do_cleanup();
2490	else
2491	pr_info("Kmemleak disabled without freeing internal data. Reclaim the memory with \"echo clear > /sys/kernel/debug/kmemleak\".\n");
2492	}
2493
2494	static DECLARE_WORK(cleanup_work, kmemleak_do_cleanup);
2495
2496	/*
2497	* Disable kmemleak. No memory allocation/freeing will be traced once this
2498	* function is called. Disabling kmemleak is an irreversible operation.
2499	*/
2500	static void kmemleak_disable(void)
2501	{
2502	/* atomically check whether it was already invoked */
2503	if (cmpxchg(&kmemleak_error, 0, 1))
2504	return;
2505
2506	/* stop any memory operation tracing */
2507	kmemleak_enabled = 0;
2508
2509	/* check whether it is too early for a kernel thread */
2510	if (kmemleak_late_initialized)
2511	schedule_work(&cleanup_work);
2512	else
2513	kmemleak_free_enabled = 0;
2514
2515	pr_info("Kernel memory leak detector disabled\n");
2516	}
2517
2518	/*
2519	* Allow boot-time kmemleak disabling (enabled by default).
2520	*/
2521	static int __init kmemleak_boot_config(char *str)
2522	{
2523	if (!str)
2524	return -EINVAL;
2525	if (strcmp(str, "off") == 0)
2526	kmemleak_disable();
2527	else if (strcmp(str, "on") == 0) {
2528	kmemleak_skip_disable = 1;
2529	stack_depot_request_early_init();
2530	}
2531	else
2532	return -EINVAL;
2533	return 0;
2534	}
2535	early_param("kmemleak", kmemleak_boot_config);
2536
2537	/*
2538	* Kmemleak initialization.
2539	*/
2540	void __init kmemleak_init(void)
2541	{
2542	#ifdef CONFIG_DEBUG_KMEMLEAK_DEFAULT_OFF
2543	if (!kmemleak_skip_disable) {
2544	kmemleak_disable();
2545	return;
2546	}
2547	#endif
2548
2549	if (kmemleak_error)
2550	return;
2551
2552	jiffies_min_age = msecs_to_jiffies(MSECS_MIN_AGE);
2553	jiffies_scan_wait = secs_to_jiffies(SECS_SCAN_WAIT);
2554
2555	object_cache = KMEM_CACHE(kmemleak_object, SLAB_NOLEAKTRACE);
2556	scan_area_cache = KMEM_CACHE(kmemleak_scan_area, SLAB_NOLEAKTRACE);
2557
2558	/* register the data/bss sections */
2559	create_object((unsigned long)_sdata, _edata - _sdata,
2560	KMEMLEAK_GREY, GFP_ATOMIC);
2561	create_object((unsigned long)__bss_start, __bss_stop - __bss_start,
2562	KMEMLEAK_GREY, GFP_ATOMIC);
2563	/* only register .data..ro_after_init if not within .data */
2564	if (&__start_ro_after_init < &_sdata \|\| &__end_ro_after_init > &_edata)
2565	create_object((unsigned long)__start_ro_after_init,
2566	__end_ro_after_init - __start_ro_after_init,
2567	KMEMLEAK_GREY, GFP_ATOMIC);
2568	}
2569
2570	/*
2571	* Late initialization function.
2572	*/
2573	static int __init kmemleak_late_init(void)
2574	{
2575	kmemleak_late_initialized = 1;
2576
2577	debugfs_create_file("kmemleak", 0644, NULL, NULL, &kmemleak_fops);
2578
2579	if (kmemleak_error) {
2580	/*
2581	* Some error occurred and kmemleak was disabled. There is a
2582	* small chance that kmemleak_disable() was called immediately
2583	* after setting kmemleak_late_initialized and we may end up with
2584	* two clean-up threads but serialized by scan_mutex.
2585	*/
2586	schedule_work(&cleanup_work);
2587	return -ENOMEM;
2588	}
2589
2590	if (IS_ENABLED(CONFIG_DEBUG_KMEMLEAK_AUTO_SCAN)) {
2591	mutex_lock(&scan_mutex);
2592	start_scan_thread();
2593	mutex_unlock(&scan_mutex);
2594	}
2595
2596	pr_info("Kernel memory leak detector initialized (mem pool available: %d)\n",
2597	mem_pool_free_count);
2598
2599	return 0;
2600	}
2601	late_initcall(kmemleak_late_init);