projects/linux·blob master · refs

509 lines · 12.0 KiB

1// SPDX-License-Identifier: GPL-2.0
2#include <kunit/test.h>
3#include <kunit/test-bug.h>
4#include <linux/mm.h>
5#include <linux/slab.h>
6#include <linux/module.h>
7#include <linux/kernel.h>
8#include <linux/rcupdate.h>
9#include <linux/delay.h>
10#include <linux/perf_event.h>
11#include <linux/kprobes.h>
12#include "../mm/slab.h"
13
14static struct kunit_resource resource;
15static int slab_errors;
16
17/*
18 * Wrapper function for kmem_cache_create(), which reduces 2 parameters:
19 * 'align' and 'ctor', and sets SLAB_SKIP_KFENCE flag to avoid getting an
20 * object from kfence pool, where the operation could be caught by both
21 * our test and kfence sanity check.
22 */
23static struct kmem_cache *test_kmem_cache_create(const char *name,
24 unsigned int size, slab_flags_t flags)
25{
26 struct kmem_cache *s = kmem_cache_create(name, size, 0,
27 (flags | SLAB_NO_USER_FLAGS), NULL);
28 s->flags |= SLAB_SKIP_KFENCE;
29 return s;
30}
31
32static void test_clobber_zone(struct kunit *test)
33{
34 struct kmem_cache *s = test_kmem_cache_create("TestSlub_RZ_alloc", 64,
35 SLAB_RED_ZONE);
36 u8 *p = kmem_cache_alloc(s, GFP_KERNEL);
37
38 kasan_disable_current();
39 p[64] = 0x12;
40
41 validate_slab_cache(s);
42 KUNIT_EXPECT_EQ(test, 2, slab_errors);
43
44 kasan_enable_current();
45 kmem_cache_free(s, p);
46 kmem_cache_destroy(s);
47}
48
49#ifndef CONFIG_KASAN
50static void test_next_pointer(struct kunit *test)
51{
52 struct kmem_cache *s = test_kmem_cache_create("TestSlub_next_ptr_free",
53 64, SLAB_POISON);
54 u8 *p = kmem_cache_alloc(s, GFP_KERNEL);
55 unsigned long tmp;
56 unsigned long *ptr_addr;
57
58 kmem_cache_free(s, p);
59
60 ptr_addr = (unsigned long *)(p + s->offset);
61 tmp = *ptr_addr;
62 p[s->offset] = ~p[s->offset];
63
64 /*
65 * Expecting three errors.
66 * One for the corrupted freechain and the other one for the wrong
67 * count of objects in use. The third error is fixing broken cache.
68 */
69 validate_slab_cache(s);
70 KUNIT_EXPECT_EQ(test, 3, slab_errors);
71
72 /*
73 * Try to repair corrupted freepointer.
74 * Still expecting two errors. The first for the wrong count
75 * of objects in use.
76 * The second error is for fixing broken cache.
77 */
78 *ptr_addr = tmp;
79 slab_errors = 0;
80
81 validate_slab_cache(s);
82 KUNIT_EXPECT_EQ(test, 2, slab_errors);
83
84 /*
85 * Previous validation repaired the count of objects in use.
86 * Now expecting no error.
87 */
88 slab_errors = 0;
89 validate_slab_cache(s);
90 KUNIT_EXPECT_EQ(test, 0, slab_errors);
91
92 kmem_cache_destroy(s);
93}
94
95static void test_first_word(struct kunit *test)
96{
97 struct kmem_cache *s = test_kmem_cache_create("TestSlub_1th_word_free",
98 64, SLAB_POISON);
99 u8 *p = kmem_cache_alloc(s, GFP_KERNEL);
100
101 kmem_cache_free(s, p);
102 *p = 0x78;
103
104 validate_slab_cache(s);
105 KUNIT_EXPECT_EQ(test, 2, slab_errors);
106
107 kmem_cache_destroy(s);
108}
109
110static void test_clobber_50th_byte(struct kunit *test)
111{
112 struct kmem_cache *s = test_kmem_cache_create("TestSlub_50th_word_free",
113 64, SLAB_POISON);
114 u8 *p = kmem_cache_alloc(s, GFP_KERNEL);
115
116 kmem_cache_free(s, p);
117 p[50] = 0x9a;
118
119 validate_slab_cache(s);
120 KUNIT_EXPECT_EQ(test, 2, slab_errors);
121
122 kmem_cache_destroy(s);
123}
124#endif
125
126static void test_clobber_redzone_free(struct kunit *test)
127{
128 struct kmem_cache *s = test_kmem_cache_create("TestSlub_RZ_free", 64,
129 SLAB_RED_ZONE);
130 u8 *p = kmem_cache_alloc(s, GFP_KERNEL);
131
132 kasan_disable_current();
133 kmem_cache_free(s, p);
134 p[64] = 0xab;
135
136 validate_slab_cache(s);
137 KUNIT_EXPECT_EQ(test, 2, slab_errors);
138
139 kasan_enable_current();
140 kmem_cache_destroy(s);
141}
142
143static void test_kmalloc_redzone_access(struct kunit *test)
144{
145 struct kmem_cache *s = test_kmem_cache_create("TestSlub_RZ_kmalloc", 32,
146 SLAB_KMALLOC|SLAB_STORE_USER|SLAB_RED_ZONE);
147 u8 *p = alloc_hooks(__kmalloc_cache_noprof(s, GFP_KERNEL, 18));
148
149 kasan_disable_current();
150
151 /* Suppress the -Warray-bounds warning */
152 OPTIMIZER_HIDE_VAR(p);
153 p[18] = 0xab;
154 p[19] = 0xab;
155
156 validate_slab_cache(s);
157 KUNIT_EXPECT_EQ(test, 2, slab_errors);
158
159 kasan_enable_current();
160 kmem_cache_free(s, p);
161 kmem_cache_destroy(s);
162}
163
164struct test_kfree_rcu_struct {
165 union {
166 struct rcu_head rcu;
167 struct kvfree_rcu_head kvrcu;
168 };
169};
170
171static void test_kfree_rcu(struct kunit *test)
172{
173 struct kmem_cache *s;
174 struct test_kfree_rcu_struct *p;
175
176 if (IS_BUILTIN(CONFIG_SLUB_KUNIT_TEST))
177 kunit_skip(test, "can't do kfree_rcu() when test is built-in");
178
179 s = test_kmem_cache_create("TestSlub_kfree_rcu",
180 sizeof(struct test_kfree_rcu_struct),
181 SLAB_NO_MERGE);
182 p = kmem_cache_alloc(s, GFP_KERNEL);
183
184 kfree_rcu(p, rcu);
185 kmem_cache_destroy(s);
186
187 KUNIT_EXPECT_EQ(test, 0, slab_errors);
188}
189
190struct cache_destroy_work {
191 struct work_struct work;
192 struct kmem_cache *s;
193};
194
195static void cache_destroy_workfn(struct work_struct *w)
196{
197 struct cache_destroy_work *cdw;
198
199 cdw = container_of(w, struct cache_destroy_work, work);
200 kmem_cache_destroy(cdw->s);
201}
202
203#define KMEM_CACHE_DESTROY_NR 10
204
205static void test_kfree_rcu_wq_destroy(struct kunit *test)
206{
207 struct test_kfree_rcu_struct *p;
208 struct cache_destroy_work cdw;
209 struct workqueue_struct *wq;
210 struct kmem_cache *s;
211 unsigned int delay;
212 int i;
213
214 if (IS_BUILTIN(CONFIG_SLUB_KUNIT_TEST))
215 kunit_skip(test, "can't do kfree_rcu() when test is built-in");
216
217 INIT_WORK_ONSTACK(&cdw.work, cache_destroy_workfn);
218 wq = alloc_workqueue("test_kfree_rcu_destroy_wq",
219 WQ_HIGHPRI | WQ_UNBOUND | WQ_MEM_RECLAIM, 0);
220
221 if (!wq)
222 kunit_skip(test, "failed to alloc wq");
223
224 for (i = 0; i < KMEM_CACHE_DESTROY_NR; i++) {
225 s = test_kmem_cache_create("TestSlub_kfree_rcu_wq_destroy",
226 sizeof(struct test_kfree_rcu_struct),
227 SLAB_NO_MERGE);
228
229 if (!s)
230 kunit_skip(test, "failed to create cache");
231
232 delay = get_random_u8();
233 p = kmem_cache_alloc(s, GFP_KERNEL);
234 kfree_rcu(p, rcu);
235
236 cdw.s = s;
237
238 msleep(delay);
239 queue_work(wq, &cdw.work);
240 flush_work(&cdw.work);
241 }
242
243 destroy_workqueue(wq);
244 KUNIT_EXPECT_EQ(test, 0, slab_errors);
245}
246
247static void test_leak_destroy(struct kunit *test)
248{
249 struct kmem_cache *s = test_kmem_cache_create("TestSlub_leak_destroy",
250 64, SLAB_NO_MERGE);
251 kmem_cache_alloc(s, GFP_KERNEL);
252
253 kmem_cache_destroy(s);
254
255 KUNIT_EXPECT_EQ(test, 2, slab_errors);
256}
257
258static void test_krealloc_redzone_zeroing(struct kunit *test)
259{
260 u8 *p;
261 int i;
262 struct kmem_cache *s = test_kmem_cache_create("TestSlub_krealloc", 64,
263 SLAB_KMALLOC|SLAB_STORE_USER|SLAB_RED_ZONE);
264
265 p = alloc_hooks(__kmalloc_cache_noprof(s, GFP_KERNEL, 48));
266 memset(p, 0xff, 48);
267
268 kasan_disable_current();
269 OPTIMIZER_HIDE_VAR(p);
270
271 /* Test shrink */
272 p = krealloc(p, 40, GFP_KERNEL | __GFP_ZERO);
273 for (i = 40; i < 64; i++)
274 KUNIT_EXPECT_EQ(test, p[i], SLUB_RED_ACTIVE);
275
276 /* Test grow within the same 64B kmalloc object */
277 p = krealloc(p, 56, GFP_KERNEL | __GFP_ZERO);
278 for (i = 40; i < 56; i++)
279 KUNIT_EXPECT_EQ(test, p[i], 0);
280 for (i = 56; i < 64; i++)
281 KUNIT_EXPECT_EQ(test, p[i], SLUB_RED_ACTIVE);
282
283 validate_slab_cache(s);
284 KUNIT_EXPECT_EQ(test, 0, slab_errors);
285
286 memset(p, 0xff, 56);
287 /* Test grow with allocating a bigger 128B object */
288 p = krealloc(p, 112, GFP_KERNEL | __GFP_ZERO);
289 for (i = 0; i < 56; i++)
290 KUNIT_EXPECT_EQ(test, p[i], 0xff);
291 for (i = 56; i < 112; i++)
292 KUNIT_EXPECT_EQ(test, p[i], 0);
293
294 kfree(p);
295 kasan_enable_current();
296 kmem_cache_destroy(s);
297}
298
299#if defined(CONFIG_PERF_EVENTS) || (defined(CONFIG_KPROBES) && defined(CONFIG_SMP))
300#define NR_ITERATIONS 1000
301#define NR_OBJECTS 1000
302static struct test_kfree_rcu_struct *objects[NR_OBJECTS];
303
304struct test_nolock_context {
305 struct kunit *test;
306 int callback_count;
307 int alloc_ok;
308 int alloc_fail;
309#ifdef CONFIG_PERF_EVENTS
310 struct perf_event *event;
311#endif
312#if defined(CONFIG_KPROBES) && defined(CONFIG_SMP)
313 struct kprobe kprobe;
314#endif
315};
316
317static void test_kmalloc_and_friends(void)
318{
319 int i, j;
320 bool can_use_kfree_rcu = !IS_BUILTIN(CONFIG_SLUB_KUNIT_TEST);
321
322 for (i = 0; i < NR_ITERATIONS; i++) {
323 for (j = 0; j < NR_OBJECTS; j++) {
324 gfp_t gfp = (i & 1) ? GFP_KERNEL : GFP_KERNEL_ACCOUNT;
325
326 objects[j] = kmalloc_obj(*objects[j], gfp);
327 if (!objects[j]) {
328 j--;
329 while (j >= 0)
330 kfree(objects[j--]);
331 return;
332 }
333 }
334
335 for (j = 0; j < NR_OBJECTS; j++) {
336 if (can_use_kfree_rcu && (i & 2))
337 kfree_rcu(objects[j], rcu);
338 else
339 kfree(objects[j]);
340 }
341 }
342}
343
344static void test_nolock(struct test_nolock_context *ctx)
345{
346 struct test_kfree_rcu_struct *objp;
347 gfp_t gfp;
348 bool can_use_kfree_rcu = !IS_BUILTIN(CONFIG_SLUB_KUNIT_TEST);
349
350 /* __GFP_ACCOUNT to test kmalloc_nolock() in alloc_slab_obj_exts() */
351 gfp = (ctx->callback_count & 1) ? 0 : __GFP_ACCOUNT;
352 objp = kmalloc_nolock(sizeof(*objp), gfp, NUMA_NO_NODE);
353
354 if (objp)
355 ctx->alloc_ok++;
356 else
357 ctx->alloc_fail++;
358
359 if (can_use_kfree_rcu && (ctx->callback_count & 2))
360 kfree_rcu_nolock(objp, kvrcu);
361 else
362 kfree_nolock(objp);
363
364 ctx->callback_count++;
365}
366#endif
367
368#ifdef CONFIG_PERF_EVENTS
369static struct perf_event_attr hw_attr = {
370 .type = PERF_TYPE_HARDWARE,
371 .config = PERF_COUNT_HW_CPU_CYCLES,
372 .size = sizeof(struct perf_event_attr),
373 .pinned = 1,
374 .disabled = 1,
375 .freq = 1,
376 .sample_freq = 100000,
377};
378
379static void overflow_handler_test_nolock(struct perf_event *event,
380 struct perf_sample_data *data,
381 struct pt_regs *regs)
382{
383 struct test_nolock_context *ctx = event->overflow_handler_context;
384
385 test_nolock(ctx);
386}
387
388static bool enable_perf_events(struct test_nolock_context *ctx)
389{
390 struct perf_event *event;
391
392 event = perf_event_create_kernel_counter(&hw_attr, -1, current,
393 overflow_handler_test_nolock,
394 ctx);
395
396 if (IS_ERR(event))
397 return false;
398
399 ctx->event = event;
400 perf_event_enable(ctx->event);
401 return true;
402}
403
404static void disable_perf_events(struct test_nolock_context *ctx)
405{
406 kunit_info(ctx->test, "HW perf events: callback_count: %d, alloc_ok: %d, alloc_fail: %d\n",
407 ctx->callback_count, ctx->alloc_ok, ctx->alloc_fail);
408
409 perf_event_disable(ctx->event);
410 perf_event_release_kernel(ctx->event);
411}
412
413static void test_kmalloc_nolock_and_friends_perf(struct kunit *test)
414{
415 struct test_nolock_context ctx = { .test = test };
416
417 if (!enable_perf_events(&ctx))
418 kunit_skip(test, "Failed to enable perf event, skipping");
419
420 test_kmalloc_and_friends();
421
422 disable_perf_events(&ctx);
423 KUNIT_EXPECT_EQ(test, 0, slab_errors);
424}
425#endif
426
427#if defined(CONFIG_KPROBES) && defined(CONFIG_SMP)
428static int slab_kprobe_pre_handler(struct kprobe *p, struct pt_regs *regs)
429{
430 struct test_nolock_context *ctx;
431
432 ctx = container_of(p, struct test_nolock_context, kprobe);
433 test_nolock(ctx);
434 return 0;
435}
436
437static bool register_slab_kprobes(struct test_nolock_context *ctx)
438{
439 ctx->kprobe.symbol_name = "slab_attach_kprobe_locked";
440 ctx->kprobe.pre_handler = slab_kprobe_pre_handler;
441
442 if (register_kprobe(&ctx->kprobe))
443 return false;
444 return true;
445}
446
447static void unregister_slab_kprobes(struct test_nolock_context *ctx)
448{
449 kunit_info(ctx->test, "kprobes: callback_count: %d, alloc_ok: %d, alloc_fail: %d\n",
450 ctx->callback_count, ctx->alloc_ok, ctx->alloc_fail);
451 unregister_kprobe(&ctx->kprobe);
452}
453
454static void test_kmalloc_nolock_and_friends_kprobe(struct kunit *test)
455{
456 struct test_nolock_context ctx = { .test = test };
457
458 if (!register_slab_kprobes(&ctx))
459 kunit_skip(test, "Failed to register kprobe, skipping");
460
461 test_kmalloc_and_friends();
462
463 unregister_slab_kprobes(&ctx);
464 KUNIT_EXPECT_EQ(test, 0, slab_errors);
465}
466#endif
467
468static int test_init(struct kunit *test)
469{
470 slab_errors = 0;
471
472 kunit_add_named_resource(test, NULL, NULL, &resource,
473 "slab_errors", &slab_errors);
474 return 0;
475}
476
477static struct kunit_case test_cases[] = {
478 KUNIT_CASE(test_clobber_zone),
479
480#ifndef CONFIG_KASAN
481 KUNIT_CASE(test_next_pointer),
482 KUNIT_CASE(test_first_word),
483 KUNIT_CASE(test_clobber_50th_byte),
484#endif
485
486 KUNIT_CASE(test_clobber_redzone_free),
487 KUNIT_CASE(test_kmalloc_redzone_access),
488 KUNIT_CASE(test_kfree_rcu),
489 KUNIT_CASE(test_kfree_rcu_wq_destroy),
490 KUNIT_CASE(test_leak_destroy),
491 KUNIT_CASE(test_krealloc_redzone_zeroing),
492#ifdef CONFIG_PERF_EVENTS
493 KUNIT_CASE_SLOW(test_kmalloc_nolock_and_friends_perf),
494#endif
495#if defined(CONFIG_KPROBES) && defined(CONFIG_SMP)
496 KUNIT_CASE_SLOW(test_kmalloc_nolock_and_friends_kprobe),
497#endif
498 {}
499};
500
501static struct kunit_suite test_suite = {
502 .name = "slub_test",
503 .init = test_init,
504 .test_cases = test_cases,
505};
506kunit_test_suite(test_suite);
507
508MODULE_DESCRIPTION("Kunit tests for slub allocator");
509MODULE_LICENSE("GPL");