projects/linux·blob master · refs

568 lines · 16.5 KiB

1// SPDX-License-Identifier: GPL-2.0
2#define _GNU_SOURCE
3#include <sched.h>
4#include <test_progs.h>
5#include "cgroup_helpers.h"
6#include "percpu_alloc_array.skel.h"
7#include "percpu_alloc_cgrp_local_storage.skel.h"
8#include "percpu_alloc_fail.skel.h"
9
10static void test_array(void)
11{
12 struct percpu_alloc_array *skel;
13 int err, prog_fd;
14 LIBBPF_OPTS(bpf_test_run_opts, topts);
15
16 skel = percpu_alloc_array__open();
17 if (!ASSERT_OK_PTR(skel, "percpu_alloc_array__open"))
18 return;
19
20 bpf_program__set_autoload(skel->progs.test_array_map_1, true);
21 bpf_program__set_autoload(skel->progs.test_array_map_2, true);
22 bpf_program__set_autoload(skel->progs.test_array_map_3, true);
23 bpf_program__set_autoload(skel->progs.test_array_map_4, true);
24
25 skel->bss->my_pid = getpid();
26 skel->rodata->nr_cpus = libbpf_num_possible_cpus();
27
28 err = percpu_alloc_array__load(skel);
29 if (!ASSERT_OK(err, "percpu_alloc_array__load"))
30 goto out;
31
32 err = percpu_alloc_array__attach(skel);
33 if (!ASSERT_OK(err, "percpu_alloc_array__attach"))
34 goto out;
35
36 prog_fd = bpf_program__fd(skel->progs.test_array_map_1);
37 err = bpf_prog_test_run_opts(prog_fd, &topts);
38 ASSERT_OK(err, "test_run array_map 1-4");
39 ASSERT_EQ(topts.retval, 0, "test_run array_map 1-4");
40 ASSERT_EQ(skel->bss->cpu0_field_d, 2, "cpu0_field_d");
41 ASSERT_EQ(skel->bss->sum_field_c, 1, "sum_field_c");
42out:
43 percpu_alloc_array__destroy(skel);
44}
45
46static void test_array_sleepable(void)
47{
48 struct percpu_alloc_array *skel;
49 int err, prog_fd;
50 LIBBPF_OPTS(bpf_test_run_opts, topts);
51
52 skel = percpu_alloc_array__open();
53 if (!ASSERT_OK_PTR(skel, "percpu_alloc__open"))
54 return;
55
56 bpf_program__set_autoload(skel->progs.test_array_map_10, true);
57
58 skel->bss->my_pid = getpid();
59 skel->rodata->nr_cpus = libbpf_num_possible_cpus();
60
61 err = percpu_alloc_array__load(skel);
62 if (!ASSERT_OK(err, "percpu_alloc_array__load"))
63 goto out;
64
65 err = percpu_alloc_array__attach(skel);
66 if (!ASSERT_OK(err, "percpu_alloc_array__attach"))
67 goto out;
68
69 prog_fd = bpf_program__fd(skel->progs.test_array_map_10);
70 err = bpf_prog_test_run_opts(prog_fd, &topts);
71 ASSERT_OK(err, "test_run array_map_10");
72 ASSERT_EQ(topts.retval, 0, "test_run array_map_10");
73 ASSERT_EQ(skel->bss->cpu0_field_d, 2, "cpu0_field_d");
74 ASSERT_EQ(skel->bss->sum_field_c, 1, "sum_field_c");
75out:
76 percpu_alloc_array__destroy(skel);
77}
78
79static void test_cgrp_local_storage(void)
80{
81 struct percpu_alloc_cgrp_local_storage *skel;
82 int err, cgroup_fd, prog_fd;
83 LIBBPF_OPTS(bpf_test_run_opts, topts);
84
85 cgroup_fd = test__join_cgroup("/percpu_alloc");
86 if (!ASSERT_GE(cgroup_fd, 0, "join_cgroup /percpu_alloc"))
87 return;
88
89 skel = percpu_alloc_cgrp_local_storage__open();
90 if (!ASSERT_OK_PTR(skel, "percpu_alloc_cgrp_local_storage__open"))
91 goto close_fd;
92
93 skel->bss->my_pid = getpid();
94 skel->rodata->nr_cpus = libbpf_num_possible_cpus();
95
96 err = percpu_alloc_cgrp_local_storage__load(skel);
97 if (!ASSERT_OK(err, "percpu_alloc_cgrp_local_storage__load"))
98 goto destroy_skel;
99
100 err = percpu_alloc_cgrp_local_storage__attach(skel);
101 if (!ASSERT_OK(err, "percpu_alloc_cgrp_local_storage__attach"))
102 goto destroy_skel;
103
104 prog_fd = bpf_program__fd(skel->progs.test_cgrp_local_storage_1);
105 err = bpf_prog_test_run_opts(prog_fd, &topts);
106 ASSERT_OK(err, "test_run cgrp_local_storage 1-3");
107 ASSERT_EQ(topts.retval, 0, "test_run cgrp_local_storage 1-3");
108 ASSERT_EQ(skel->bss->cpu0_field_d, 2, "cpu0_field_d");
109 ASSERT_EQ(skel->bss->sum_field_c, 1, "sum_field_c");
110
111destroy_skel:
112 percpu_alloc_cgrp_local_storage__destroy(skel);
113close_fd:
114 close(cgroup_fd);
115}
116
117static void test_failure(void) {
118 RUN_TESTS(percpu_alloc_fail);
119}
120
121static void test_percpu_map_op_cpu_flag(struct bpf_map *map, void *keys, size_t key_sz, u32 entries,
122 int nr_cpus, bool test_batch)
123{
124 size_t value_sz = sizeof(u32), value_sz_cpus, value_sz_total;
125 u32 *values = NULL, *values_percpu = NULL;
126 const u32 value = 0xDEADC0DE;
127 int i, j, cpu, map_fd, err;
128 u64 batch = 0, flags;
129 void *values_row;
130 u32 count, v;
131 LIBBPF_OPTS(bpf_map_batch_opts, batch_opts);
132
133 value_sz_cpus = value_sz * nr_cpus;
134 values = calloc(entries, value_sz_cpus);
135 if (!ASSERT_OK_PTR(values, "calloc values"))
136 return;
137
138 values_percpu = calloc(entries, roundup(value_sz, 8) * nr_cpus);
139 if (!ASSERT_OK_PTR(values_percpu, "calloc values_percpu")) {
140 free(values);
141 return;
142 }
143
144 value_sz_total = value_sz_cpus * entries;
145 memset(values, 0, value_sz_total);
146
147 map_fd = bpf_map__fd(map);
148 flags = BPF_F_CPU | BPF_F_ALL_CPUS;
149 err = bpf_map_lookup_elem_flags(map_fd, keys, values, flags);
150 if (!ASSERT_ERR(err, "bpf_map_lookup_elem_flags cpu|all_cpus"))
151 goto out;
152
153 err = bpf_map_update_elem(map_fd, keys, values, flags);
154 if (!ASSERT_ERR(err, "bpf_map_update_elem cpu|all_cpus"))
155 goto out;
156
157 flags = BPF_F_ALL_CPUS;
158 err = bpf_map_lookup_elem_flags(map_fd, keys, values, flags);
159 if (!ASSERT_ERR(err, "bpf_map_lookup_elem_flags all_cpus"))
160 goto out;
161
162 flags = BPF_F_LOCK | BPF_F_CPU;
163 err = bpf_map_lookup_elem_flags(map_fd, keys, values, flags);
164 if (!ASSERT_ERR(err, "bpf_map_lookup_elem_flags BPF_F_LOCK"))
165 goto out;
166
167 flags = BPF_F_LOCK | BPF_F_ALL_CPUS;
168 err = bpf_map_update_elem(map_fd, keys, values, flags);
169 if (!ASSERT_ERR(err, "bpf_map_update_elem BPF_F_LOCK"))
170 goto out;
171
172 flags = (u64)nr_cpus << 32 | BPF_F_CPU;
173 err = bpf_map_update_elem(map_fd, keys, values, flags);
174 if (!ASSERT_EQ(err, -ERANGE, "bpf_map_update_elem -ERANGE"))
175 goto out;
176
177 err = bpf_map__update_elem(map, keys, key_sz, values, value_sz, flags);
178 if (!ASSERT_EQ(err, -ERANGE, "bpf_map__update_elem -ERANGE"))
179 goto out;
180
181 err = bpf_map_lookup_elem_flags(map_fd, keys, values, flags);
182 if (!ASSERT_EQ(err, -ERANGE, "bpf_map_lookup_elem_flags -ERANGE"))
183 goto out;
184
185 err = bpf_map__lookup_elem(map, keys, key_sz, values, value_sz, flags);
186 if (!ASSERT_EQ(err, -ERANGE, "bpf_map__lookup_elem -ERANGE"))
187 goto out;
188
189 for (cpu = 0; cpu < nr_cpus; cpu++) {
190 /* clear value on all cpus */
191 values[0] = 0;
192 flags = BPF_F_ALL_CPUS;
193 for (i = 0; i < entries; i++) {
194 err = bpf_map__update_elem(map, keys + i * key_sz, key_sz, values,
195 value_sz, flags);
196 if (!ASSERT_OK(err, "bpf_map__update_elem all_cpus"))
197 goto out;
198 }
199
200 /* update value on specified cpu */
201 for (i = 0; i < entries; i++) {
202 values[0] = value;
203 flags = (u64)cpu << 32 | BPF_F_CPU;
204 err = bpf_map__update_elem(map, keys + i * key_sz, key_sz, values,
205 value_sz, flags);
206 if (!ASSERT_OK(err, "bpf_map__update_elem specified cpu"))
207 goto out;
208
209 /* lookup then check value on CPUs */
210 for (j = 0; j < nr_cpus; j++) {
211 flags = (u64)j << 32 | BPF_F_CPU;
212 err = bpf_map__lookup_elem(map, keys + i * key_sz, key_sz, values,
213 value_sz, flags);
214 if (!ASSERT_OK(err, "bpf_map__lookup_elem specified cpu"))
215 goto out;
216 if (!ASSERT_EQ(values[0], j != cpu ? 0 : value,
217 "bpf_map__lookup_elem value on specified cpu"))
218 goto out;
219 }
220 }
221 }
222
223 if (!test_batch)
224 goto out;
225
226 count = entries;
227 batch_opts.elem_flags = (u64)nr_cpus << 32 | BPF_F_CPU;
228 err = bpf_map_update_batch(map_fd, keys, values, &count, &batch_opts);
229 if (!ASSERT_EQ(err, -ERANGE, "bpf_map_update_batch -ERANGE"))
230 goto out;
231
232 for (cpu = 0; cpu < nr_cpus; cpu++) {
233 memset(values, 0, value_sz_total);
234
235 /* clear values across all CPUs */
236 count = entries;
237 batch_opts.elem_flags = BPF_F_ALL_CPUS;
238 err = bpf_map_update_batch(map_fd, keys, values, &count, &batch_opts);
239 if (!ASSERT_OK(err, "bpf_map_update_batch all_cpus"))
240 goto out;
241 if (!ASSERT_EQ(count, entries, "bpf_map_update_batch count"))
242 goto out;
243
244 /* update values on specified CPU */
245 for (i = 0; i < entries; i++)
246 values[i] = value;
247
248 count = entries;
249 batch_opts.elem_flags = (u64)cpu << 32 | BPF_F_CPU;
250 err = bpf_map_update_batch(map_fd, keys, values, &count, &batch_opts);
251 if (!ASSERT_OK(err, "bpf_map_update_batch specified cpu"))
252 goto out;
253 if (!ASSERT_EQ(count, entries, "bpf_map_update_batch count"))
254 goto out;
255
256 /* lookup values on specified CPU */
257 batch = 0;
258 count = entries;
259 memset(values, 0, entries * value_sz);
260 err = bpf_map_lookup_batch(map_fd, NULL, &batch, keys, values, &count, &batch_opts);
261 if (!ASSERT_TRUE(!err || err == -ENOENT, "bpf_map_lookup_batch specified cpu"))
262 goto out;
263 if (!ASSERT_EQ(count, entries, "bpf_map_lookup_batch count"))
264 goto out;
265
266 for (i = 0; i < entries; i++)
267 if (!ASSERT_EQ(values[i], value,
268 "bpf_map_lookup_batch value on specified cpu"))
269 goto out;
270
271 /* lookup values from all CPUs */
272 batch = 0;
273 count = entries;
274 batch_opts.elem_flags = 0;
275 memset(values_percpu, 0, roundup(value_sz, 8) * nr_cpus * entries);
276 err = bpf_map_lookup_batch(map_fd, NULL, &batch, keys, values_percpu, &count,
277 &batch_opts);
278 if (!ASSERT_TRUE(!err || err == -ENOENT, "bpf_map_lookup_batch all_cpus"))
279 goto out;
280 if (!ASSERT_EQ(count, entries, "bpf_map_lookup_batch count"))
281 goto out;
282
283 for (i = 0; i < entries; i++) {
284 values_row = (void *) values_percpu +
285 roundup(value_sz, 8) * i * nr_cpus;
286 for (j = 0; j < nr_cpus; j++) {
287 v = *(u32 *) (values_row + roundup(value_sz, 8) * j);
288 if (!ASSERT_EQ(v, j != cpu ? 0 : value,
289 "bpf_map_lookup_batch value all_cpus"))
290 goto out;
291 }
292 }
293 }
294
295out:
296 free(values_percpu);
297 free(values);
298}
299
300static void test_percpu_map_cpu_flag(enum bpf_map_type map_type)
301{
302 struct percpu_alloc_array *skel;
303 size_t key_sz = sizeof(int);
304 int *keys, nr_cpus, i, err;
305 struct bpf_map *map;
306 u32 max_entries;
307
308 nr_cpus = libbpf_num_possible_cpus();
309 if (!ASSERT_GT(nr_cpus, 0, "libbpf_num_possible_cpus"))
310 return;
311
312 max_entries = nr_cpus * 2;
313 keys = calloc(max_entries, key_sz);
314 if (!ASSERT_OK_PTR(keys, "calloc keys"))
315 return;
316
317 for (i = 0; i < max_entries; i++)
318 keys[i] = i;
319
320 skel = percpu_alloc_array__open();
321 if (!ASSERT_OK_PTR(skel, "percpu_alloc_array__open")) {
322 free(keys);
323 return;
324 }
325
326 map = skel->maps.percpu;
327 bpf_map__set_type(map, map_type);
328 bpf_map__set_max_entries(map, max_entries);
329
330 err = percpu_alloc_array__load(skel);
331 if (!ASSERT_OK(err, "test_percpu_alloc__load"))
332 goto out;
333
334 test_percpu_map_op_cpu_flag(map, keys, key_sz, nr_cpus, nr_cpus, true);
335out:
336 percpu_alloc_array__destroy(skel);
337 free(keys);
338}
339
340static void test_percpu_array_cpu_flag(void)
341{
342 test_percpu_map_cpu_flag(BPF_MAP_TYPE_PERCPU_ARRAY);
343}
344
345static void test_percpu_hash_cpu_flag(void)
346{
347 test_percpu_map_cpu_flag(BPF_MAP_TYPE_PERCPU_HASH);
348}
349
350static void test_lru_percpu_hash_cpu_flag(void)
351{
352 test_percpu_map_cpu_flag(BPF_MAP_TYPE_LRU_PERCPU_HASH);
353}
354
355/*
356 * A BPF_F_CPU update that creates an element must zero the value on the other
357 * cpus, rather than leave them holding whatever the recycled element last
358 * contained. max_entries is 1 so the second key can only reuse the element
359 * the first one released.
360 */
361static void test_percpu_map_cpu_flag_create(enum bpf_map_type map_type, __u32 map_flags)
362{
363 LIBBPF_OPTS(bpf_map_create_opts, opts, .map_flags = map_flags);
364 const u32 stale = 0xDEADC0DE, fresh = 0xC0FFEE;
365 int nr_cpus, cpu, map_fd, err, key;
366 int pinned_cpu, value_cpu;
367 cpu_set_t old_mask, new_mask;
368 bool restore_mask = false;
369 u32 value;
370 u64 flags;
371
372 nr_cpus = libbpf_num_possible_cpus();
373 if (!ASSERT_GT(nr_cpus, 0, "libbpf_num_possible_cpus"))
374 return;
375
376 if (nr_cpus < 2) {
377 test__skip();
378 return;
379 }
380
381 map_fd = bpf_map_create(map_type, "cpu_flag_create", sizeof(key), sizeof(value), 1, &opts);
382 if (!ASSERT_GE(map_fd, 0, "bpf_map_create"))
383 return;
384
385 /* NO_PREALLOC recycles per cpu, so keep the delete and the create on one cpu. */
386 err = sched_getaffinity(0, sizeof(old_mask), &old_mask);
387 if (!ASSERT_OK(err, "sched_getaffinity"))
388 goto out;
389
390 pinned_cpu = sched_getcpu();
391 if (!ASSERT_GE(pinned_cpu, 0, "sched_getcpu"))
392 goto out;
393
394 CPU_ZERO(&new_mask);
395 CPU_SET(pinned_cpu, &new_mask);
396 err = sched_setaffinity(0, sizeof(new_mask), &new_mask);
397 if (!ASSERT_OK(err, "sched_setaffinity"))
398 goto out;
399 restore_mask = true;
400
401 value_cpu = pinned_cpu ? 0 : 1;
402
403 key = 1;
404 value = stale;
405 err = bpf_map_update_elem(map_fd, &key, &value, BPF_F_ALL_CPUS);
406 if (!ASSERT_OK(err, "bpf_map_update_elem all_cpus"))
407 goto out;
408
409 err = bpf_map_delete_elem(map_fd, &key);
410 if (!ASSERT_OK(err, "bpf_map_delete_elem"))
411 goto out;
412
413 key = 2;
414 value = fresh;
415 flags = (u64)value_cpu << 32 | BPF_F_CPU;
416 err = bpf_map_update_elem(map_fd, &key, &value, flags);
417 if (!ASSERT_OK(err, "bpf_map_update_elem specified cpu"))
418 goto out;
419
420 for (cpu = 0; cpu < nr_cpus; cpu++) {
421 value = 0;
422 flags = (u64)cpu << 32 | BPF_F_CPU;
423 err = bpf_map_lookup_elem_flags(map_fd, &key, &value, flags);
424 if (!ASSERT_OK(err, "bpf_map_lookup_elem_flags specified cpu"))
425 goto out;
426 if (!ASSERT_EQ(value, cpu == value_cpu ? fresh : 0, "value on specified cpu"))
427 goto out;
428 }
429
430out:
431 if (restore_mask)
432 sched_setaffinity(0, sizeof(old_mask), &old_mask);
433 close(map_fd);
434}
435
436static void test_percpu_hash_cpu_flag_create(void)
437{
438 test_percpu_map_cpu_flag_create(BPF_MAP_TYPE_PERCPU_HASH, 0);
439}
440
441static void test_percpu_hash_cpu_flag_create_malloc(void)
442{
443 test_percpu_map_cpu_flag_create(BPF_MAP_TYPE_PERCPU_HASH, BPF_F_NO_PREALLOC);
444}
445
446static void test_lru_percpu_hash_cpu_flag_create(void)
447{
448 /* lru without prealloc is -ENOTSUPP, so there is no malloc variant */
449 test_percpu_map_cpu_flag_create(BPF_MAP_TYPE_LRU_PERCPU_HASH, 0);
450}
451
452static void test_percpu_cgroup_storage_cpu_flag(void)
453{
454 struct percpu_alloc_array *skel = NULL;
455 struct bpf_cgroup_storage_key key;
456 int cgroup, prog_fd, nr_cpus, err;
457 struct bpf_map *map;
458
459 nr_cpus = libbpf_num_possible_cpus();
460 if (!ASSERT_GT(nr_cpus, 0, "libbpf_num_possible_cpus"))
461 return;
462
463 err = setup_cgroup_environment();
464 if (!ASSERT_OK(err, "setup_cgroup_environment"))
465 return;
466
467 cgroup = create_and_get_cgroup("/cg_percpu");
468 if (!ASSERT_GE(cgroup, 0, "create_and_get_cgroup")) {
469 cleanup_cgroup_environment();
470 return;
471 }
472
473 err = join_cgroup("/cg_percpu");
474 if (!ASSERT_OK(err, "join_cgroup"))
475 goto out;
476
477 skel = percpu_alloc_array__open_and_load();
478 if (!ASSERT_OK_PTR(skel, "percpu_alloc_array__open_and_load"))
479 goto out;
480
481 prog_fd = bpf_program__fd(skel->progs.cgroup_egress);
482 err = bpf_prog_attach(prog_fd, cgroup, BPF_CGROUP_INET_EGRESS, 0);
483 if (!ASSERT_OK(err, "bpf_prog_attach"))
484 goto out;
485
486 map = skel->maps.percpu_cgroup_storage;
487 err = bpf_map_get_next_key(bpf_map__fd(map), NULL, &key);
488 if (!ASSERT_OK(err, "bpf_map_get_next_key"))
489 goto out;
490
491 test_percpu_map_op_cpu_flag(map, &key, sizeof(key), 1, nr_cpus, false);
492out:
493 bpf_prog_detach2(-1, cgroup, BPF_CGROUP_INET_EGRESS);
494 close(cgroup);
495 cleanup_cgroup_environment();
496 percpu_alloc_array__destroy(skel);
497}
498
499static void test_map_op_cpu_flag(enum bpf_map_type map_type)
500{
501 u32 max_entries = 1, count = max_entries;
502 u64 flags, batch = 0, val = 0;
503 int err, map_fd, key = 0;
504 LIBBPF_OPTS(bpf_map_batch_opts, batch_opts);
505
506 map_fd = bpf_map_create(map_type, "test_cpu_flag", sizeof(int), sizeof(u64), max_entries,
507 NULL);
508 if (!ASSERT_GE(map_fd, 0, "bpf_map_create"))
509 return;
510
511 flags = BPF_F_ALL_CPUS;
512 err = bpf_map_update_elem(map_fd, &key, &val, flags);
513 ASSERT_ERR(err, "bpf_map_update_elem all_cpus");
514
515 batch_opts.elem_flags = BPF_F_ALL_CPUS;
516 err = bpf_map_update_batch(map_fd, &key, &val, &count, &batch_opts);
517 ASSERT_ERR(err, "bpf_map_update_batch all_cpus");
518
519 flags = BPF_F_CPU;
520 err = bpf_map_lookup_elem_flags(map_fd, &key, &val, flags);
521 ASSERT_ERR(err, "bpf_map_lookup_elem_flags cpu");
522
523 batch_opts.elem_flags = BPF_F_CPU;
524 err = bpf_map_lookup_batch(map_fd, NULL, &batch, &key, &val, &count, &batch_opts);
525 ASSERT_ERR(err, "bpf_map_lookup_batch cpu");
526
527 close(map_fd);
528}
529
530static void test_array_cpu_flag(void)
531{
532 test_map_op_cpu_flag(BPF_MAP_TYPE_ARRAY);
533}
534
535static void test_hash_cpu_flag(void)
536{
537 test_map_op_cpu_flag(BPF_MAP_TYPE_HASH);
538}
539
540void test_percpu_alloc(void)
541{
542 if (test__start_subtest("array"))
543 test_array();
544 if (test__start_subtest("array_sleepable"))
545 test_array_sleepable();
546 if (test__start_subtest("cgrp_local_storage"))
547 test_cgrp_local_storage();
548 if (test__start_subtest("failure_tests"))
549 test_failure();
550 if (test__start_subtest("cpu_flag_percpu_array"))
551 test_percpu_array_cpu_flag();
552 if (test__start_subtest("cpu_flag_percpu_hash"))
553 test_percpu_hash_cpu_flag();
554 if (test__start_subtest("cpu_flag_lru_percpu_hash"))
555 test_lru_percpu_hash_cpu_flag();
556 if (test__start_subtest("cpu_flag_create_percpu_hash"))
557 test_percpu_hash_cpu_flag_create();
558 if (test__start_subtest("cpu_flag_create_percpu_hash_malloc"))
559 test_percpu_hash_cpu_flag_create_malloc();
560 if (test__start_subtest("cpu_flag_create_lru_percpu_hash"))
561 test_lru_percpu_hash_cpu_flag_create();
562 if (test__start_subtest("cpu_flag_percpu_cgroup_storage"))
563 test_percpu_cgroup_storage_cpu_flag();
564 if (test__start_subtest("cpu_flag_array"))
565 test_array_cpu_flag();
566 if (test__start_subtest("cpu_flag_hash"))
567 test_hash_cpu_flag();
568}