projects/linux·blob v4.20 · refs

1201 lines · 29.5 KiB

1/*
2 * Generic infrastructure for lifetime debugging of objects.
3 *
4 * Started by Thomas Gleixner
5 *
6 * Copyright (C) 2008, Thomas Gleixner <[email protected]>
7 *
8 * For licencing details see kernel-base/COPYING
9 */
10
11#define pr_fmt(fmt) "ODEBUG: " fmt
12
13#include <linux/debugobjects.h>
14#include <linux/interrupt.h>
15#include <linux/sched.h>
16#include <linux/sched/task_stack.h>
17#include <linux/seq_file.h>
18#include <linux/debugfs.h>
19#include <linux/slab.h>
20#include <linux/hash.h>
21#include <linux/kmemleak.h>
22
23#define ODEBUG_HASH_BITS 14
24#define ODEBUG_HASH_SIZE (1 << ODEBUG_HASH_BITS)
25
26#define ODEBUG_POOL_SIZE 1024
27#define ODEBUG_POOL_MIN_LEVEL 256
28
29#define ODEBUG_CHUNK_SHIFT PAGE_SHIFT
30#define ODEBUG_CHUNK_SIZE (1 << ODEBUG_CHUNK_SHIFT)
31#define ODEBUG_CHUNK_MASK (~(ODEBUG_CHUNK_SIZE - 1))
32
33struct debug_bucket {
34 struct hlist_head list;
35 raw_spinlock_t lock;
36};
37
38static struct debug_bucket obj_hash[ODEBUG_HASH_SIZE];
39
40static struct debug_obj obj_static_pool[ODEBUG_POOL_SIZE] __initdata;
41
42static DEFINE_RAW_SPINLOCK(pool_lock);
43
44static HLIST_HEAD(obj_pool);
45static HLIST_HEAD(obj_to_free);
46
47static int obj_pool_min_free = ODEBUG_POOL_SIZE;
48static int obj_pool_free = ODEBUG_POOL_SIZE;
49static int obj_pool_used;
50static int obj_pool_max_used;
51/* The number of objs on the global free list */
52static int obj_nr_tofree;
53static struct kmem_cache *obj_cache;
54
55static int debug_objects_maxchain __read_mostly;
56static int __maybe_unused debug_objects_maxchecked __read_mostly;
57static int debug_objects_fixups __read_mostly;
58static int debug_objects_warnings __read_mostly;
59static int debug_objects_enabled __read_mostly
60 = CONFIG_DEBUG_OBJECTS_ENABLE_DEFAULT;
61static int debug_objects_pool_size __read_mostly
62 = ODEBUG_POOL_SIZE;
63static int debug_objects_pool_min_level __read_mostly
64 = ODEBUG_POOL_MIN_LEVEL;
65static struct debug_obj_descr *descr_test __read_mostly;
66
67/*
68 * Track numbers of kmem_cache_alloc()/free() calls done.
69 */
70static int debug_objects_allocated;
71static int debug_objects_freed;
72
73static void free_obj_work(struct work_struct *work);
74static DECLARE_WORK(debug_obj_work, free_obj_work);
75
76static int __init enable_object_debug(char *str)
77{
78 debug_objects_enabled = 1;
79 return 0;
80}
81
82static int __init disable_object_debug(char *str)
83{
84 debug_objects_enabled = 0;
85 return 0;
86}
87
88early_param("debug_objects", enable_object_debug);
89early_param("no_debug_objects", disable_object_debug);
90
91static const char *obj_states[ODEBUG_STATE_MAX] = {
92 [ODEBUG_STATE_NONE] = "none",
93 [ODEBUG_STATE_INIT] = "initialized",
94 [ODEBUG_STATE_INACTIVE] = "inactive",
95 [ODEBUG_STATE_ACTIVE] = "active",
96 [ODEBUG_STATE_DESTROYED] = "destroyed",
97 [ODEBUG_STATE_NOTAVAILABLE] = "not available",
98};
99
100static void fill_pool(void)
101{
102 gfp_t gfp = GFP_ATOMIC | __GFP_NORETRY | __GFP_NOWARN;
103 struct debug_obj *new, *obj;
104 unsigned long flags;
105
106 if (likely(obj_pool_free >= debug_objects_pool_min_level))
107 return;
108
109 /*
110 * Reuse objs from the global free list; they will be reinitialized
111 * when allocating.
112 */
113 while (obj_nr_tofree && (obj_pool_free < obj_pool_min_free)) {
114 raw_spin_lock_irqsave(&pool_lock, flags);
115 /*
116 * Recheck with the lock held as the worker thread might have
117 * won the race and freed the global free list already.
118 */
119 if (obj_nr_tofree) {
120 obj = hlist_entry(obj_to_free.first, typeof(*obj), node);
121 hlist_del(&obj->node);
122 obj_nr_tofree--;
123 hlist_add_head(&obj->node, &obj_pool);
124 obj_pool_free++;
125 }
126 raw_spin_unlock_irqrestore(&pool_lock, flags);
127 }
128
129 if (unlikely(!obj_cache))
130 return;
131
132 while (obj_pool_free < debug_objects_pool_min_level) {
133
134 new = kmem_cache_zalloc(obj_cache, gfp);
135 if (!new)
136 return;
137
138 raw_spin_lock_irqsave(&pool_lock, flags);
139 hlist_add_head(&new->node, &obj_pool);
140 debug_objects_allocated++;
141 obj_pool_free++;
142 raw_spin_unlock_irqrestore(&pool_lock, flags);
143 }
144}
145
146/*
147 * Lookup an object in the hash bucket.
148 */
149static struct debug_obj *lookup_object(void *addr, struct debug_bucket *b)
150{
151 struct debug_obj *obj;
152 int cnt = 0;
153
154 hlist_for_each_entry(obj, &b->list, node) {
155 cnt++;
156 if (obj->object == addr)
157 return obj;
158 }
159 if (cnt > debug_objects_maxchain)
160 debug_objects_maxchain = cnt;
161
162 return NULL;
163}
164
165/*
166 * Allocate a new object. If the pool is empty, switch off the debugger.
167 * Must be called with interrupts disabled.
168 */
169static struct debug_obj *
170alloc_object(void *addr, struct debug_bucket *b, struct debug_obj_descr *descr)
171{
172 struct debug_obj *obj = NULL;
173
174 raw_spin_lock(&pool_lock);
175 if (obj_pool.first) {
176 obj = hlist_entry(obj_pool.first, typeof(*obj), node);
177
178 obj->object = addr;
179 obj->descr = descr;
180 obj->state = ODEBUG_STATE_NONE;
181 obj->astate = 0;
182 hlist_del(&obj->node);
183
184 hlist_add_head(&obj->node, &b->list);
185
186 obj_pool_used++;
187 if (obj_pool_used > obj_pool_max_used)
188 obj_pool_max_used = obj_pool_used;
189
190 obj_pool_free--;
191 if (obj_pool_free < obj_pool_min_free)
192 obj_pool_min_free = obj_pool_free;
193 }
194 raw_spin_unlock(&pool_lock);
195
196 return obj;
197}
198
199/*
200 * workqueue function to free objects.
201 *
202 * To reduce contention on the global pool_lock, the actual freeing of
203 * debug objects will be delayed if the pool_lock is busy.
204 */
205static void free_obj_work(struct work_struct *work)
206{
207 struct hlist_node *tmp;
208 struct debug_obj *obj;
209 unsigned long flags;
210 HLIST_HEAD(tofree);
211
212 if (!raw_spin_trylock_irqsave(&pool_lock, flags))
213 return;
214
215 /*
216 * The objs on the pool list might be allocated before the work is
217 * run, so recheck if pool list it full or not, if not fill pool
218 * list from the global free list
219 */
220 while (obj_nr_tofree && obj_pool_free < debug_objects_pool_size) {
221 obj = hlist_entry(obj_to_free.first, typeof(*obj), node);
222 hlist_del(&obj->node);
223 hlist_add_head(&obj->node, &obj_pool);
224 obj_pool_free++;
225 obj_nr_tofree--;
226 }
227
228 /*
229 * Pool list is already full and there are still objs on the free
230 * list. Move remaining free objs to a temporary list to free the
231 * memory outside the pool_lock held region.
232 */
233 if (obj_nr_tofree) {
234 hlist_move_list(&obj_to_free, &tofree);
235 debug_objects_freed += obj_nr_tofree;
236 obj_nr_tofree = 0;
237 }
238 raw_spin_unlock_irqrestore(&pool_lock, flags);
239
240 hlist_for_each_entry_safe(obj, tmp, &tofree, node) {
241 hlist_del(&obj->node);
242 kmem_cache_free(obj_cache, obj);
243 }
244}
245
246static bool __free_object(struct debug_obj *obj)
247{
248 unsigned long flags;
249 bool work;
250
251 raw_spin_lock_irqsave(&pool_lock, flags);
252 work = (obj_pool_free > debug_objects_pool_size) && obj_cache;
253 obj_pool_used--;
254
255 if (work) {
256 obj_nr_tofree++;
257 hlist_add_head(&obj->node, &obj_to_free);
258 } else {
259 obj_pool_free++;
260 hlist_add_head(&obj->node, &obj_pool);
261 }
262 raw_spin_unlock_irqrestore(&pool_lock, flags);
263 return work;
264}
265
266/*
267 * Put the object back into the pool and schedule work to free objects
268 * if necessary.
269 */
270static void free_object(struct debug_obj *obj)
271{
272 if (__free_object(obj))
273 schedule_work(&debug_obj_work);
274}
275
276/*
277 * We run out of memory. That means we probably have tons of objects
278 * allocated.
279 */
280static void debug_objects_oom(void)
281{
282 struct debug_bucket *db = obj_hash;
283 struct hlist_node *tmp;
284 HLIST_HEAD(freelist);
285 struct debug_obj *obj;
286 unsigned long flags;
287 int i;
288
289 pr_warn("Out of memory. ODEBUG disabled\n");
290
291 for (i = 0; i < ODEBUG_HASH_SIZE; i++, db++) {
292 raw_spin_lock_irqsave(&db->lock, flags);
293 hlist_move_list(&db->list, &freelist);
294 raw_spin_unlock_irqrestore(&db->lock, flags);
295
296 /* Now free them */
297 hlist_for_each_entry_safe(obj, tmp, &freelist, node) {
298 hlist_del(&obj->node);
299 free_object(obj);
300 }
301 }
302}
303
304/*
305 * We use the pfn of the address for the hash. That way we can check
306 * for freed objects simply by checking the affected bucket.
307 */
308static struct debug_bucket *get_bucket(unsigned long addr)
309{
310 unsigned long hash;
311
312 hash = hash_long((addr >> ODEBUG_CHUNK_SHIFT), ODEBUG_HASH_BITS);
313 return &obj_hash[hash];
314}
315
316static void debug_print_object(struct debug_obj *obj, char *msg)
317{
318 struct debug_obj_descr *descr = obj->descr;
319 static int limit;
320
321 if (limit < 5 && descr != descr_test) {
322 void *hint = descr->debug_hint ?
323 descr->debug_hint(obj->object) : NULL;
324 limit++;
325 WARN(1, KERN_ERR "ODEBUG: %s %s (active state %u) "
326 "object type: %s hint: %pS\n",
327 msg, obj_states[obj->state], obj->astate,
328 descr->name, hint);
329 }
330 debug_objects_warnings++;
331}
332
333/*
334 * Try to repair the damage, so we have a better chance to get useful
335 * debug output.
336 */
337static bool
338debug_object_fixup(bool (*fixup)(void *addr, enum debug_obj_state state),
339 void * addr, enum debug_obj_state state)
340{
341 if (fixup && fixup(addr, state)) {
342 debug_objects_fixups++;
343 return true;
344 }
345 return false;
346}
347
348static void debug_object_is_on_stack(void *addr, int onstack)
349{
350 int is_on_stack;
351 static int limit;
352
353 if (limit > 4)
354 return;
355
356 is_on_stack = object_is_on_stack(addr);
357 if (is_on_stack == onstack)
358 return;
359
360 limit++;
361 if (is_on_stack)
362 pr_warn("object %p is on stack %p, but NOT annotated.\n", addr,
363 task_stack_page(current));
364 else
365 pr_warn("object %p is NOT on stack %p, but annotated.\n", addr,
366 task_stack_page(current));
367
368 WARN_ON(1);
369}
370
371static void
372__debug_object_init(void *addr, struct debug_obj_descr *descr, int onstack)
373{
374 enum debug_obj_state state;
375 struct debug_bucket *db;
376 struct debug_obj *obj;
377 unsigned long flags;
378
379 fill_pool();
380
381 db = get_bucket((unsigned long) addr);
382
383 raw_spin_lock_irqsave(&db->lock, flags);
384
385 obj = lookup_object(addr, db);
386 if (!obj) {
387 obj = alloc_object(addr, db, descr);
388 if (!obj) {
389 debug_objects_enabled = 0;
390 raw_spin_unlock_irqrestore(&db->lock, flags);
391 debug_objects_oom();
392 return;
393 }
394 debug_object_is_on_stack(addr, onstack);
395 }
396
397 switch (obj->state) {
398 case ODEBUG_STATE_NONE:
399 case ODEBUG_STATE_INIT:
400 case ODEBUG_STATE_INACTIVE:
401 obj->state = ODEBUG_STATE_INIT;
402 break;
403
404 case ODEBUG_STATE_ACTIVE:
405 debug_print_object(obj, "init");
406 state = obj->state;
407 raw_spin_unlock_irqrestore(&db->lock, flags);
408 debug_object_fixup(descr->fixup_init, addr, state);
409 return;
410
411 case ODEBUG_STATE_DESTROYED:
412 debug_print_object(obj, "init");
413 break;
414 default:
415 break;
416 }
417
418 raw_spin_unlock_irqrestore(&db->lock, flags);
419}
420
421/**
422 * debug_object_init - debug checks when an object is initialized
423 * @addr: address of the object
424 * @descr: pointer to an object specific debug description structure
425 */
426void debug_object_init(void *addr, struct debug_obj_descr *descr)
427{
428 if (!debug_objects_enabled)
429 return;
430
431 __debug_object_init(addr, descr, 0);
432}
433EXPORT_SYMBOL_GPL(debug_object_init);
434
435/**
436 * debug_object_init_on_stack - debug checks when an object on stack is
437 * initialized
438 * @addr: address of the object
439 * @descr: pointer to an object specific debug description structure
440 */
441void debug_object_init_on_stack(void *addr, struct debug_obj_descr *descr)
442{
443 if (!debug_objects_enabled)
444 return;
445
446 __debug_object_init(addr, descr, 1);
447}
448EXPORT_SYMBOL_GPL(debug_object_init_on_stack);
449
450/**
451 * debug_object_activate - debug checks when an object is activated
452 * @addr: address of the object
453 * @descr: pointer to an object specific debug description structure
454 * Returns 0 for success, -EINVAL for check failed.
455 */
456int debug_object_activate(void *addr, struct debug_obj_descr *descr)
457{
458 enum debug_obj_state state;
459 struct debug_bucket *db;
460 struct debug_obj *obj;
461 unsigned long flags;
462 int ret;
463 struct debug_obj o = { .object = addr,
464 .state = ODEBUG_STATE_NOTAVAILABLE,
465 .descr = descr };
466
467 if (!debug_objects_enabled)
468 return 0;
469
470 db = get_bucket((unsigned long) addr);
471
472 raw_spin_lock_irqsave(&db->lock, flags);
473
474 obj = lookup_object(addr, db);
475 if (obj) {
476 switch (obj->state) {
477 case ODEBUG_STATE_INIT:
478 case ODEBUG_STATE_INACTIVE:
479 obj->state = ODEBUG_STATE_ACTIVE;
480 ret = 0;
481 break;
482
483 case ODEBUG_STATE_ACTIVE:
484 debug_print_object(obj, "activate");
485 state = obj->state;
486 raw_spin_unlock_irqrestore(&db->lock, flags);
487 ret = debug_object_fixup(descr->fixup_activate, addr, state);
488 return ret ? 0 : -EINVAL;
489
490 case ODEBUG_STATE_DESTROYED:
491 debug_print_object(obj, "activate");
492 ret = -EINVAL;
493 break;
494 default:
495 ret = 0;
496 break;
497 }
498 raw_spin_unlock_irqrestore(&db->lock, flags);
499 return ret;
500 }
501
502 raw_spin_unlock_irqrestore(&db->lock, flags);
503 /*
504 * We are here when a static object is activated. We
505 * let the type specific code confirm whether this is
506 * true or not. if true, we just make sure that the
507 * static object is tracked in the object tracker. If
508 * not, this must be a bug, so we try to fix it up.
509 */
510 if (descr->is_static_object && descr->is_static_object(addr)) {
511 /* track this static object */
512 debug_object_init(addr, descr);
513 debug_object_activate(addr, descr);
514 } else {
515 debug_print_object(&o, "activate");
516 ret = debug_object_fixup(descr->fixup_activate, addr,
517 ODEBUG_STATE_NOTAVAILABLE);
518 return ret ? 0 : -EINVAL;
519 }
520 return 0;
521}
522EXPORT_SYMBOL_GPL(debug_object_activate);
523
524/**
525 * debug_object_deactivate - debug checks when an object is deactivated
526 * @addr: address of the object
527 * @descr: pointer to an object specific debug description structure
528 */
529void debug_object_deactivate(void *addr, struct debug_obj_descr *descr)
530{
531 struct debug_bucket *db;
532 struct debug_obj *obj;
533 unsigned long flags;
534
535 if (!debug_objects_enabled)
536 return;
537
538 db = get_bucket((unsigned long) addr);
539
540 raw_spin_lock_irqsave(&db->lock, flags);
541
542 obj = lookup_object(addr, db);
543 if (obj) {
544 switch (obj->state) {
545 case ODEBUG_STATE_INIT:
546 case ODEBUG_STATE_INACTIVE:
547 case ODEBUG_STATE_ACTIVE:
548 if (!obj->astate)
549 obj->state = ODEBUG_STATE_INACTIVE;
550 else
551 debug_print_object(obj, "deactivate");
552 break;
553
554 case ODEBUG_STATE_DESTROYED:
555 debug_print_object(obj, "deactivate");
556 break;
557 default:
558 break;
559 }
560 } else {
561 struct debug_obj o = { .object = addr,
562 .state = ODEBUG_STATE_NOTAVAILABLE,
563 .descr = descr };
564
565 debug_print_object(&o, "deactivate");
566 }
567
568 raw_spin_unlock_irqrestore(&db->lock, flags);
569}
570EXPORT_SYMBOL_GPL(debug_object_deactivate);
571
572/**
573 * debug_object_destroy - debug checks when an object is destroyed
574 * @addr: address of the object
575 * @descr: pointer to an object specific debug description structure
576 */
577void debug_object_destroy(void *addr, struct debug_obj_descr *descr)
578{
579 enum debug_obj_state state;
580 struct debug_bucket *db;
581 struct debug_obj *obj;
582 unsigned long flags;
583
584 if (!debug_objects_enabled)
585 return;
586
587 db = get_bucket((unsigned long) addr);
588
589 raw_spin_lock_irqsave(&db->lock, flags);
590
591 obj = lookup_object(addr, db);
592 if (!obj)
593 goto out_unlock;
594
595 switch (obj->state) {
596 case ODEBUG_STATE_NONE:
597 case ODEBUG_STATE_INIT:
598 case ODEBUG_STATE_INACTIVE:
599 obj->state = ODEBUG_STATE_DESTROYED;
600 break;
601 case ODEBUG_STATE_ACTIVE:
602 debug_print_object(obj, "destroy");
603 state = obj->state;
604 raw_spin_unlock_irqrestore(&db->lock, flags);
605 debug_object_fixup(descr->fixup_destroy, addr, state);
606 return;
607
608 case ODEBUG_STATE_DESTROYED:
609 debug_print_object(obj, "destroy");
610 break;
611 default:
612 break;
613 }
614out_unlock:
615 raw_spin_unlock_irqrestore(&db->lock, flags);
616}
617EXPORT_SYMBOL_GPL(debug_object_destroy);
618
619/**
620 * debug_object_free - debug checks when an object is freed
621 * @addr: address of the object
622 * @descr: pointer to an object specific debug description structure
623 */
624void debug_object_free(void *addr, struct debug_obj_descr *descr)
625{
626 enum debug_obj_state state;
627 struct debug_bucket *db;
628 struct debug_obj *obj;
629 unsigned long flags;
630
631 if (!debug_objects_enabled)
632 return;
633
634 db = get_bucket((unsigned long) addr);
635
636 raw_spin_lock_irqsave(&db->lock, flags);
637
638 obj = lookup_object(addr, db);
639 if (!obj)
640 goto out_unlock;
641
642 switch (obj->state) {
643 case ODEBUG_STATE_ACTIVE:
644 debug_print_object(obj, "free");
645 state = obj->state;
646 raw_spin_unlock_irqrestore(&db->lock, flags);
647 debug_object_fixup(descr->fixup_free, addr, state);
648 return;
649 default:
650 hlist_del(&obj->node);
651 raw_spin_unlock_irqrestore(&db->lock, flags);
652 free_object(obj);
653 return;
654 }
655out_unlock:
656 raw_spin_unlock_irqrestore(&db->lock, flags);
657}
658EXPORT_SYMBOL_GPL(debug_object_free);
659
660/**
661 * debug_object_assert_init - debug checks when object should be init-ed
662 * @addr: address of the object
663 * @descr: pointer to an object specific debug description structure
664 */
665void debug_object_assert_init(void *addr, struct debug_obj_descr *descr)
666{
667 struct debug_bucket *db;
668 struct debug_obj *obj;
669 unsigned long flags;
670
671 if (!debug_objects_enabled)
672 return;
673
674 db = get_bucket((unsigned long) addr);
675
676 raw_spin_lock_irqsave(&db->lock, flags);
677
678 obj = lookup_object(addr, db);
679 if (!obj) {
680 struct debug_obj o = { .object = addr,
681 .state = ODEBUG_STATE_NOTAVAILABLE,
682 .descr = descr };
683
684 raw_spin_unlock_irqrestore(&db->lock, flags);
685 /*
686 * Maybe the object is static, and we let the type specific
687 * code confirm. Track this static object if true, else invoke
688 * fixup.
689 */
690 if (descr->is_static_object && descr->is_static_object(addr)) {
691 /* Track this static object */
692 debug_object_init(addr, descr);
693 } else {
694 debug_print_object(&o, "assert_init");
695 debug_object_fixup(descr->fixup_assert_init, addr,
696 ODEBUG_STATE_NOTAVAILABLE);
697 }
698 return;
699 }
700
701 raw_spin_unlock_irqrestore(&db->lock, flags);
702}
703EXPORT_SYMBOL_GPL(debug_object_assert_init);
704
705/**
706 * debug_object_active_state - debug checks object usage state machine
707 * @addr: address of the object
708 * @descr: pointer to an object specific debug description structure
709 * @expect: expected state
710 * @next: state to move to if expected state is found
711 */
712void
713debug_object_active_state(void *addr, struct debug_obj_descr *descr,
714 unsigned int expect, unsigned int next)
715{
716 struct debug_bucket *db;
717 struct debug_obj *obj;
718 unsigned long flags;
719
720 if (!debug_objects_enabled)
721 return;
722
723 db = get_bucket((unsigned long) addr);
724
725 raw_spin_lock_irqsave(&db->lock, flags);
726
727 obj = lookup_object(addr, db);
728 if (obj) {
729 switch (obj->state) {
730 case ODEBUG_STATE_ACTIVE:
731 if (obj->astate == expect)
732 obj->astate = next;
733 else
734 debug_print_object(obj, "active_state");
735 break;
736
737 default:
738 debug_print_object(obj, "active_state");
739 break;
740 }
741 } else {
742 struct debug_obj o = { .object = addr,
743 .state = ODEBUG_STATE_NOTAVAILABLE,
744 .descr = descr };
745
746 debug_print_object(&o, "active_state");
747 }
748
749 raw_spin_unlock_irqrestore(&db->lock, flags);
750}
751EXPORT_SYMBOL_GPL(debug_object_active_state);
752
753#ifdef CONFIG_DEBUG_OBJECTS_FREE
754static void __debug_check_no_obj_freed(const void *address, unsigned long size)
755{
756 unsigned long flags, oaddr, saddr, eaddr, paddr, chunks;
757 struct debug_obj_descr *descr;
758 enum debug_obj_state state;
759 struct debug_bucket *db;
760 struct hlist_node *tmp;
761 struct debug_obj *obj;
762 int cnt, objs_checked = 0;
763 bool work = false;
764
765 saddr = (unsigned long) address;
766 eaddr = saddr + size;
767 paddr = saddr & ODEBUG_CHUNK_MASK;
768 chunks = ((eaddr - paddr) + (ODEBUG_CHUNK_SIZE - 1));
769 chunks >>= ODEBUG_CHUNK_SHIFT;
770
771 for (;chunks > 0; chunks--, paddr += ODEBUG_CHUNK_SIZE) {
772 db = get_bucket(paddr);
773
774repeat:
775 cnt = 0;
776 raw_spin_lock_irqsave(&db->lock, flags);
777 hlist_for_each_entry_safe(obj, tmp, &db->list, node) {
778 cnt++;
779 oaddr = (unsigned long) obj->object;
780 if (oaddr < saddr || oaddr >= eaddr)
781 continue;
782
783 switch (obj->state) {
784 case ODEBUG_STATE_ACTIVE:
785 debug_print_object(obj, "free");
786 descr = obj->descr;
787 state = obj->state;
788 raw_spin_unlock_irqrestore(&db->lock, flags);
789 debug_object_fixup(descr->fixup_free,
790 (void *) oaddr, state);
791 goto repeat;
792 default:
793 hlist_del(&obj->node);
794 work |= __free_object(obj);
795 break;
796 }
797 }
798 raw_spin_unlock_irqrestore(&db->lock, flags);
799
800 if (cnt > debug_objects_maxchain)
801 debug_objects_maxchain = cnt;
802
803 objs_checked += cnt;
804 }
805
806 if (objs_checked > debug_objects_maxchecked)
807 debug_objects_maxchecked = objs_checked;
808
809 /* Schedule work to actually kmem_cache_free() objects */
810 if (work)
811 schedule_work(&debug_obj_work);
812}
813
814void debug_check_no_obj_freed(const void *address, unsigned long size)
815{
816 if (debug_objects_enabled)
817 __debug_check_no_obj_freed(address, size);
818}
819#endif
820
821#ifdef CONFIG_DEBUG_FS
822
823static int debug_stats_show(struct seq_file *m, void *v)
824{
825 seq_printf(m, "max_chain :%d\n", debug_objects_maxchain);
826 seq_printf(m, "max_checked :%d\n", debug_objects_maxchecked);
827 seq_printf(m, "warnings :%d\n", debug_objects_warnings);
828 seq_printf(m, "fixups :%d\n", debug_objects_fixups);
829 seq_printf(m, "pool_free :%d\n", obj_pool_free);
830 seq_printf(m, "pool_min_free :%d\n", obj_pool_min_free);
831 seq_printf(m, "pool_used :%d\n", obj_pool_used);
832 seq_printf(m, "pool_max_used :%d\n", obj_pool_max_used);
833 seq_printf(m, "on_free_list :%d\n", obj_nr_tofree);
834 seq_printf(m, "objs_allocated:%d\n", debug_objects_allocated);
835 seq_printf(m, "objs_freed :%d\n", debug_objects_freed);
836 return 0;
837}
838
839static int debug_stats_open(struct inode *inode, struct file *filp)
840{
841 return single_open(filp, debug_stats_show, NULL);
842}
843
844static const struct file_operations debug_stats_fops = {
845 .open = debug_stats_open,
846 .read = seq_read,
847 .llseek = seq_lseek,
848 .release = single_release,
849};
850
851static int __init debug_objects_init_debugfs(void)
852{
853 struct dentry *dbgdir, *dbgstats;
854
855 if (!debug_objects_enabled)
856 return 0;
857
858 dbgdir = debugfs_create_dir("debug_objects", NULL);
859 if (!dbgdir)
860 return -ENOMEM;
861
862 dbgstats = debugfs_create_file("stats", 0444, dbgdir, NULL,
863 &debug_stats_fops);
864 if (!dbgstats)
865 goto err;
866
867 return 0;
868
869err:
870 debugfs_remove(dbgdir);
871
872 return -ENOMEM;
873}
874__initcall(debug_objects_init_debugfs);
875
876#else
877static inline void debug_objects_init_debugfs(void) { }
878#endif
879
880#ifdef CONFIG_DEBUG_OBJECTS_SELFTEST
881
882/* Random data structure for the self test */
883struct self_test {
884 unsigned long dummy1[6];
885 int static_init;
886 unsigned long dummy2[3];
887};
888
889static __initdata struct debug_obj_descr descr_type_test;
890
891static bool __init is_static_object(void *addr)
892{
893 struct self_test *obj = addr;
894
895 return obj->static_init;
896}
897
898/*
899 * fixup_init is called when:
900 * - an active object is initialized
901 */
902static bool __init fixup_init(void *addr, enum debug_obj_state state)
903{
904 struct self_test *obj = addr;
905
906 switch (state) {
907 case ODEBUG_STATE_ACTIVE:
908 debug_object_deactivate(obj, &descr_type_test);
909 debug_object_init(obj, &descr_type_test);
910 return true;
911 default:
912 return false;
913 }
914}
915
916/*
917 * fixup_activate is called when:
918 * - an active object is activated
919 * - an unknown non-static object is activated
920 */
921static bool __init fixup_activate(void *addr, enum debug_obj_state state)
922{
923 struct self_test *obj = addr;
924
925 switch (state) {
926 case ODEBUG_STATE_NOTAVAILABLE:
927 return true;
928 case ODEBUG_STATE_ACTIVE:
929 debug_object_deactivate(obj, &descr_type_test);
930 debug_object_activate(obj, &descr_type_test);
931 return true;
932
933 default:
934 return false;
935 }
936}
937
938/*
939 * fixup_destroy is called when:
940 * - an active object is destroyed
941 */
942static bool __init fixup_destroy(void *addr, enum debug_obj_state state)
943{
944 struct self_test *obj = addr;
945
946 switch (state) {
947 case ODEBUG_STATE_ACTIVE:
948 debug_object_deactivate(obj, &descr_type_test);
949 debug_object_destroy(obj, &descr_type_test);
950 return true;
951 default:
952 return false;
953 }
954}
955
956/*
957 * fixup_free is called when:
958 * - an active object is freed
959 */
960static bool __init fixup_free(void *addr, enum debug_obj_state state)
961{
962 struct self_test *obj = addr;
963
964 switch (state) {
965 case ODEBUG_STATE_ACTIVE:
966 debug_object_deactivate(obj, &descr_type_test);
967 debug_object_free(obj, &descr_type_test);
968 return true;
969 default:
970 return false;
971 }
972}
973
974static int __init
975check_results(void *addr, enum debug_obj_state state, int fixups, int warnings)
976{
977 struct debug_bucket *db;
978 struct debug_obj *obj;
979 unsigned long flags;
980 int res = -EINVAL;
981
982 db = get_bucket((unsigned long) addr);
983
984 raw_spin_lock_irqsave(&db->lock, flags);
985
986 obj = lookup_object(addr, db);
987 if (!obj && state != ODEBUG_STATE_NONE) {
988 WARN(1, KERN_ERR "ODEBUG: selftest object not found\n");
989 goto out;
990 }
991 if (obj && obj->state != state) {
992 WARN(1, KERN_ERR "ODEBUG: selftest wrong state: %d != %d\n",
993 obj->state, state);
994 goto out;
995 }
996 if (fixups != debug_objects_fixups) {
997 WARN(1, KERN_ERR "ODEBUG: selftest fixups failed %d != %d\n",
998 fixups, debug_objects_fixups);
999 goto out;
1000 }
1001 if (warnings != debug_objects_warnings) {
1002 WARN(1, KERN_ERR "ODEBUG: selftest warnings failed %d != %d\n",
1003 warnings, debug_objects_warnings);
1004 goto out;
1005 }
1006 res = 0;
1007out:
1008 raw_spin_unlock_irqrestore(&db->lock, flags);
1009 if (res)
1010 debug_objects_enabled = 0;
1011 return res;
1012}
1013
1014static __initdata struct debug_obj_descr descr_type_test = {
1015 .name = "selftest",
1016 .is_static_object = is_static_object,
1017 .fixup_init = fixup_init,
1018 .fixup_activate = fixup_activate,
1019 .fixup_destroy = fixup_destroy,
1020 .fixup_free = fixup_free,
1021};
1022
1023static __initdata struct self_test obj = { .static_init = 0 };
1024
1025static void __init debug_objects_selftest(void)
1026{
1027 int fixups, oldfixups, warnings, oldwarnings;
1028 unsigned long flags;
1029
1030 local_irq_save(flags);
1031
1032 fixups = oldfixups = debug_objects_fixups;
1033 warnings = oldwarnings = debug_objects_warnings;
1034 descr_test = &descr_type_test;
1035
1036 debug_object_init(&obj, &descr_type_test);
1037 if (check_results(&obj, ODEBUG_STATE_INIT, fixups, warnings))
1038 goto out;
1039 debug_object_activate(&obj, &descr_type_test);
1040 if (check_results(&obj, ODEBUG_STATE_ACTIVE, fixups, warnings))
1041 goto out;
1042 debug_object_activate(&obj, &descr_type_test);
1043 if (check_results(&obj, ODEBUG_STATE_ACTIVE, ++fixups, ++warnings))
1044 goto out;
1045 debug_object_deactivate(&obj, &descr_type_test);
1046 if (check_results(&obj, ODEBUG_STATE_INACTIVE, fixups, warnings))
1047 goto out;
1048 debug_object_destroy(&obj, &descr_type_test);
1049 if (check_results(&obj, ODEBUG_STATE_DESTROYED, fixups, warnings))
1050 goto out;
1051 debug_object_init(&obj, &descr_type_test);
1052 if (check_results(&obj, ODEBUG_STATE_DESTROYED, fixups, ++warnings))
1053 goto out;
1054 debug_object_activate(&obj, &descr_type_test);
1055 if (check_results(&obj, ODEBUG_STATE_DESTROYED, fixups, ++warnings))
1056 goto out;
1057 debug_object_deactivate(&obj, &descr_type_test);
1058 if (check_results(&obj, ODEBUG_STATE_DESTROYED, fixups, ++warnings))
1059 goto out;
1060 debug_object_free(&obj, &descr_type_test);
1061 if (check_results(&obj, ODEBUG_STATE_NONE, fixups, warnings))
1062 goto out;
1063
1064 obj.static_init = 1;
1065 debug_object_activate(&obj, &descr_type_test);
1066 if (check_results(&obj, ODEBUG_STATE_ACTIVE, fixups, warnings))
1067 goto out;
1068 debug_object_init(&obj, &descr_type_test);
1069 if (check_results(&obj, ODEBUG_STATE_INIT, ++fixups, ++warnings))
1070 goto out;
1071 debug_object_free(&obj, &descr_type_test);
1072 if (check_results(&obj, ODEBUG_STATE_NONE, fixups, warnings))
1073 goto out;
1074
1075#ifdef CONFIG_DEBUG_OBJECTS_FREE
1076 debug_object_init(&obj, &descr_type_test);
1077 if (check_results(&obj, ODEBUG_STATE_INIT, fixups, warnings))
1078 goto out;
1079 debug_object_activate(&obj, &descr_type_test);
1080 if (check_results(&obj, ODEBUG_STATE_ACTIVE, fixups, warnings))
1081 goto out;
1082 __debug_check_no_obj_freed(&obj, sizeof(obj));
1083 if (check_results(&obj, ODEBUG_STATE_NONE, ++fixups, ++warnings))
1084 goto out;
1085#endif
1086 pr_info("selftest passed\n");
1087
1088out:
1089 debug_objects_fixups = oldfixups;
1090 debug_objects_warnings = oldwarnings;
1091 descr_test = NULL;
1092
1093 local_irq_restore(flags);
1094}
1095#else
1096static inline void debug_objects_selftest(void) { }
1097#endif
1098
1099/*
1100 * Called during early boot to initialize the hash buckets and link
1101 * the static object pool objects into the poll list. After this call
1102 * the object tracker is fully operational.
1103 */
1104void __init debug_objects_early_init(void)
1105{
1106 int i;
1107
1108 for (i = 0; i < ODEBUG_HASH_SIZE; i++)
1109 raw_spin_lock_init(&obj_hash[i].lock);
1110
1111 for (i = 0; i < ODEBUG_POOL_SIZE; i++)
1112 hlist_add_head(&obj_static_pool[i].node, &obj_pool);
1113}
1114
1115/*
1116 * Convert the statically allocated objects to dynamic ones:
1117 */
1118static int __init debug_objects_replace_static_objects(void)
1119{
1120 struct debug_bucket *db = obj_hash;
1121 struct hlist_node *tmp;
1122 struct debug_obj *obj, *new;
1123 HLIST_HEAD(objects);
1124 int i, cnt = 0;
1125
1126 for (i = 0; i < ODEBUG_POOL_SIZE; i++) {
1127 obj = kmem_cache_zalloc(obj_cache, GFP_KERNEL);
1128 if (!obj)
1129 goto free;
1130 hlist_add_head(&obj->node, &objects);
1131 }
1132
1133 /*
1134 * When debug_objects_mem_init() is called we know that only
1135 * one CPU is up, so disabling interrupts is enough
1136 * protection. This avoids the lockdep hell of lock ordering.
1137 */
1138 local_irq_disable();
1139
1140 /* Remove the statically allocated objects from the pool */
1141 hlist_for_each_entry_safe(obj, tmp, &obj_pool, node)
1142 hlist_del(&obj->node);
1143 /* Move the allocated objects to the pool */
1144 hlist_move_list(&objects, &obj_pool);
1145
1146 /* Replace the active object references */
1147 for (i = 0; i < ODEBUG_HASH_SIZE; i++, db++) {
1148 hlist_move_list(&db->list, &objects);
1149
1150 hlist_for_each_entry(obj, &objects, node) {
1151 new = hlist_entry(obj_pool.first, typeof(*obj), node);
1152 hlist_del(&new->node);
1153 /* copy object data */
1154 *new = *obj;
1155 hlist_add_head(&new->node, &db->list);
1156 cnt++;
1157 }
1158 }
1159 local_irq_enable();
1160
1161 pr_debug("%d of %d active objects replaced\n",
1162 cnt, obj_pool_used);
1163 return 0;
1164free:
1165 hlist_for_each_entry_safe(obj, tmp, &objects, node) {
1166 hlist_del(&obj->node);
1167 kmem_cache_free(obj_cache, obj);
1168 }
1169 return -ENOMEM;
1170}
1171
1172/*
1173 * Called after the kmem_caches are functional to setup a dedicated
1174 * cache pool, which has the SLAB_DEBUG_OBJECTS flag set. This flag
1175 * prevents that the debug code is called on kmem_cache_free() for the
1176 * debug tracker objects to avoid recursive calls.
1177 */
1178void __init debug_objects_mem_init(void)
1179{
1180 if (!debug_objects_enabled)
1181 return;
1182
1183 obj_cache = kmem_cache_create("debug_objects_cache",
1184 sizeof (struct debug_obj), 0,
1185 SLAB_DEBUG_OBJECTS | SLAB_NOLEAKTRACE,
1186 NULL);
1187
1188 if (!obj_cache || debug_objects_replace_static_objects()) {
1189 debug_objects_enabled = 0;
1190 kmem_cache_destroy(obj_cache);
1191 pr_warn("out of memory.\n");
1192 } else
1193 debug_objects_selftest();
1194
1195 /*
1196 * Increase the thresholds for allocating and freeing objects
1197 * according to the number of possible CPUs available in the system.
1198 */
1199 debug_objects_pool_size += num_possible_cpus() * 32;
1200 debug_objects_pool_min_level += num_possible_cpus() * 4;
1201}