projects/linux·blob master · refs

1521 lines · 40.6 KiB

1// SPDX-License-Identifier: GPL-2.0-only
2/* Copyright (c) 2011-2014 PLUMgrid, http://plumgrid.com
3 * Copyright (c) 2016,2017 Facebook
4 */
5#include <linux/bpf.h>
6#include <linux/btf.h>
7#include <linux/err.h>
8#include <linux/slab.h>
9#include <linux/mm.h>
10#include <linux/filter.h>
11#include <linux/perf_event.h>
12#include <uapi/linux/btf.h>
13#include <linux/rcupdate_trace.h>
14#include <linux/btf_ids.h>
15#include <crypto/sha2.h>
16
17#include "map_in_map.h"
18
19#define ARRAY_CREATE_FLAG_MASK \
20 (BPF_F_NUMA_NODE | BPF_F_MMAPABLE | BPF_F_ACCESS_MASK | \
21 BPF_F_PRESERVE_ELEMS | BPF_F_INNER_MAP)
22
23static void bpf_array_free_percpu(struct bpf_array *array)
24{
25 int i;
26
27 for (i = 0; i < array->map.max_entries; i++) {
28 free_percpu(array->pptrs[i]);
29 cond_resched();
30 }
31}
32
33static int bpf_array_alloc_percpu(struct bpf_array *array)
34{
35 void __percpu *ptr;
36 int i;
37
38 for (i = 0; i < array->map.max_entries; i++) {
39 ptr = bpf_map_alloc_percpu(&array->map, array->elem_size, 8,
40 GFP_USER | __GFP_NOWARN);
41 if (!ptr) {
42 bpf_array_free_percpu(array);
43 return -ENOMEM;
44 }
45 array->pptrs[i] = ptr;
46 cond_resched();
47 }
48
49 return 0;
50}
51
52/* Called from syscall */
53int array_map_alloc_check(union bpf_attr *attr)
54{
55 bool percpu = attr->map_type == BPF_MAP_TYPE_PERCPU_ARRAY;
56 int numa_node = bpf_map_attr_numa_node(attr);
57
58 /* check sanity of attributes */
59 if (attr->max_entries == 0 || attr->key_size != 4 ||
60 attr->value_size == 0 ||
61 attr->map_flags & ~ARRAY_CREATE_FLAG_MASK ||
62 !bpf_map_flags_access_ok(attr->map_flags) ||
63 (percpu && numa_node != NUMA_NO_NODE))
64 return -EINVAL;
65
66 if (attr->map_type != BPF_MAP_TYPE_ARRAY &&
67 attr->map_flags & (BPF_F_MMAPABLE | BPF_F_INNER_MAP))
68 return -EINVAL;
69
70 if (attr->map_type != BPF_MAP_TYPE_PERF_EVENT_ARRAY &&
71 attr->map_flags & BPF_F_PRESERVE_ELEMS)
72 return -EINVAL;
73
74 /* avoid overflow on round_up(map->value_size) */
75 if (attr->value_size > INT_MAX)
76 return -E2BIG;
77 /* percpu map value size is bound by PCPU_MIN_UNIT_SIZE */
78 if (percpu && round_up(attr->value_size, 8) > PCPU_MIN_UNIT_SIZE)
79 return -E2BIG;
80
81 return 0;
82}
83
84static struct bpf_map *array_map_alloc(union bpf_attr *attr)
85{
86 bool percpu = attr->map_type == BPF_MAP_TYPE_PERCPU_ARRAY;
87 int numa_node = bpf_map_attr_numa_node(attr);
88 u32 elem_size, index_mask, max_entries;
89 bool bypass_spec_v1 = bpf_bypass_spec_v1(NULL);
90 u64 array_size, mask64;
91 struct bpf_array *array;
92
93 elem_size = round_up(attr->value_size, 8);
94
95 max_entries = attr->max_entries;
96
97 /* On 32 bit archs roundup_pow_of_two() with max_entries that has
98 * upper most bit set in u32 space is undefined behavior due to
99 * resulting 1U << 32, so do it manually here in u64 space.
100 */
101 mask64 = fls_long(max_entries - 1);
102 mask64 = 1ULL << mask64;
103 mask64 -= 1;
104
105 index_mask = mask64;
106 if (!bypass_spec_v1) {
107 /* round up array size to nearest power of 2,
108 * since cpu will speculate within index_mask limits
109 */
110 max_entries = index_mask + 1;
111 /* Check for overflows. */
112 if (max_entries < attr->max_entries)
113 return ERR_PTR(-E2BIG);
114 }
115
116 array_size = sizeof(*array);
117 if (percpu) {
118 array_size += (u64) max_entries * sizeof(void *);
119 } else {
120 /* rely on vmalloc() to return page-aligned memory and
121 * ensure array->value is exactly page-aligned
122 */
123 if (attr->map_flags & BPF_F_MMAPABLE) {
124 array_size = PAGE_ALIGN(array_size);
125 array_size += PAGE_ALIGN((u64) max_entries * elem_size);
126 } else {
127 array_size += (u64) max_entries * elem_size;
128 }
129 }
130
131 /* allocate all map elements and zero-initialize them */
132 if (attr->map_flags & BPF_F_MMAPABLE) {
133 void *data;
134
135 /* kmalloc'ed memory can't be mmap'ed, use explicit vmalloc */
136 data = bpf_map_area_mmapable_alloc(array_size, numa_node);
137 if (!data)
138 return ERR_PTR(-ENOMEM);
139 array = data + PAGE_ALIGN(sizeof(struct bpf_array))
140 - offsetof(struct bpf_array, value);
141 } else {
142 array = bpf_map_area_alloc(array_size, numa_node);
143 }
144 if (!array)
145 return ERR_PTR(-ENOMEM);
146 array->index_mask = index_mask;
147 array->map.bypass_spec_v1 = bypass_spec_v1;
148
149 /* copy mandatory map attributes */
150 bpf_map_init_from_attr(&array->map, attr);
151 array->elem_size = elem_size;
152
153 if (percpu && bpf_array_alloc_percpu(array)) {
154 bpf_map_area_free(array);
155 return ERR_PTR(-ENOMEM);
156 }
157
158 return &array->map;
159}
160
161static void *array_map_elem_ptr(struct bpf_array* array, u32 index)
162{
163 return array->value + (u64)array->elem_size * index;
164}
165
166/* Called from syscall or from eBPF program */
167static void *array_map_lookup_elem(struct bpf_map *map, void *key)
168{
169 struct bpf_array *array = container_of(map, struct bpf_array, map);
170 u32 index = *(u32 *)key;
171
172 if (unlikely(index >= array->map.max_entries))
173 return NULL;
174
175 return array->value + (u64)array->elem_size * (index & array->index_mask);
176}
177
178static int array_map_get_hash(struct bpf_map *map)
179{
180 struct bpf_array *array = container_of(map, struct bpf_array, map);
181
182 sha256(array->value, (u64)array->elem_size * array->map.max_entries,
183 array->map.sha);
184 return 0;
185}
186
187static int array_map_direct_value_addr(const struct bpf_map *map, u64 *imm,
188 u32 off)
189{
190 struct bpf_array *array = container_of(map, struct bpf_array, map);
191
192 if (map->max_entries != 1)
193 return -ENOTSUPP;
194 if (off >= map->value_size)
195 return -EINVAL;
196
197 *imm = (unsigned long)array->value;
198 return 0;
199}
200
201static int array_map_direct_value_meta(const struct bpf_map *map, u64 imm,
202 u32 *off)
203{
204 struct bpf_array *array = container_of(map, struct bpf_array, map);
205 u64 base = (unsigned long)array->value;
206 u64 range = array->elem_size;
207
208 if (map->max_entries != 1)
209 return -ENOTSUPP;
210 if (imm < base || imm >= base + range)
211 return -ENOENT;
212
213 *off = imm - base;
214 return 0;
215}
216
217/* emit BPF instructions equivalent to C code of array_map_lookup_elem() */
218static int array_map_gen_lookup(struct bpf_map *map, struct bpf_insn *insn_buf)
219{
220 struct bpf_array *array = container_of(map, struct bpf_array, map);
221 struct bpf_insn *insn = insn_buf;
222 u32 elem_size = array->elem_size;
223 const int ret = BPF_REG_0;
224 const int map_ptr = BPF_REG_1;
225 const int index = BPF_REG_2;
226
227 if (map->map_flags & BPF_F_INNER_MAP)
228 return -EOPNOTSUPP;
229
230 *insn++ = BPF_ALU64_IMM(BPF_ADD, map_ptr, offsetof(struct bpf_array, value));
231 *insn++ = BPF_LDX_MEM(BPF_W, ret, index, 0);
232 if (!map->bypass_spec_v1) {
233 *insn++ = BPF_JMP_IMM(BPF_JGE, ret, map->max_entries, 4);
234 *insn++ = BPF_ALU32_IMM(BPF_AND, ret, array->index_mask);
235 } else {
236 *insn++ = BPF_JMP_IMM(BPF_JGE, ret, map->max_entries, 3);
237 }
238
239 if (is_power_of_2(elem_size)) {
240 *insn++ = BPF_ALU64_IMM(BPF_LSH, ret, ilog2(elem_size));
241 } else {
242 *insn++ = BPF_ALU64_IMM(BPF_MUL, ret, elem_size);
243 }
244 *insn++ = BPF_ALU64_REG(BPF_ADD, ret, map_ptr);
245 *insn++ = BPF_JMP_IMM(BPF_JA, 0, 0, 1);
246 *insn++ = BPF_MOV64_IMM(ret, 0);
247 return insn - insn_buf;
248}
249
250/* Called from eBPF program */
251static void *percpu_array_map_lookup_elem(struct bpf_map *map, void *key)
252{
253 struct bpf_array *array = container_of(map, struct bpf_array, map);
254 u32 index = *(u32 *)key;
255
256 if (unlikely(index >= array->map.max_entries))
257 return NULL;
258
259 return this_cpu_ptr(array->pptrs[index & array->index_mask]);
260}
261
262static int percpu_array_map_direct_value_addr(const struct bpf_map *map, u64 *imm, u32 off)
263{
264 struct bpf_array *array = container_of(map, struct bpf_array, map);
265
266 if (!bpf_jit_supports_percpu_insn())
267 return -EOPNOTSUPP;
268 if (map->max_entries != 1)
269 return -EOPNOTSUPP;
270 if (off >= map->value_size)
271 return -EINVAL;
272
273 *imm = (u64)(__force unsigned long) array->pptrs[0];
274 return 0;
275}
276
277static int percpu_array_map_direct_value_meta(const struct bpf_map *map, u64 imm, u32 *off)
278{
279 struct bpf_array *array = container_of(map, struct bpf_array, map);
280 u64 base = (u64)(__force unsigned long) array->pptrs[0];
281
282 if (!bpf_jit_supports_percpu_insn())
283 return -EOPNOTSUPP;
284 if (map->max_entries != 1)
285 return -EOPNOTSUPP;
286 if (imm < base || imm >= base + array->elem_size)
287 return -ENOENT;
288
289 *off = imm - base;
290 return 0;
291}
292
293/* emit BPF instructions equivalent to C code of percpu_array_map_lookup_elem() */
294static int percpu_array_map_gen_lookup(struct bpf_map *map, struct bpf_insn *insn_buf)
295{
296 struct bpf_array *array = container_of(map, struct bpf_array, map);
297 struct bpf_insn *insn = insn_buf;
298
299 if (!bpf_jit_supports_percpu_insn())
300 return -EOPNOTSUPP;
301
302 if (map->map_flags & BPF_F_INNER_MAP)
303 return -EOPNOTSUPP;
304
305 BUILD_BUG_ON(offsetof(struct bpf_array, map) != 0);
306 *insn++ = BPF_ALU64_IMM(BPF_ADD, BPF_REG_1, offsetof(struct bpf_array, pptrs));
307
308 *insn++ = BPF_LDX_MEM(BPF_W, BPF_REG_0, BPF_REG_2, 0);
309 if (!map->bypass_spec_v1) {
310 *insn++ = BPF_JMP_IMM(BPF_JGE, BPF_REG_0, map->max_entries, 6);
311 *insn++ = BPF_ALU32_IMM(BPF_AND, BPF_REG_0, array->index_mask);
312 } else {
313 *insn++ = BPF_JMP_IMM(BPF_JGE, BPF_REG_0, map->max_entries, 5);
314 }
315
316 *insn++ = BPF_ALU64_IMM(BPF_LSH, BPF_REG_0, 3);
317 *insn++ = BPF_ALU64_REG(BPF_ADD, BPF_REG_0, BPF_REG_1);
318 *insn++ = BPF_LDX_MEM(BPF_DW, BPF_REG_0, BPF_REG_0, 0);
319 *insn++ = BPF_MOV64_PERCPU_REG(BPF_REG_0, BPF_REG_0);
320 *insn++ = BPF_JMP_IMM(BPF_JA, 0, 0, 1);
321 *insn++ = BPF_MOV64_IMM(BPF_REG_0, 0);
322 return insn - insn_buf;
323}
324
325static void *percpu_array_map_lookup_percpu_elem(struct bpf_map *map, void *key, u32 cpu)
326{
327 struct bpf_array *array = container_of(map, struct bpf_array, map);
328 u32 index = *(u32 *)key;
329
330 if (cpu >= nr_cpu_ids)
331 return NULL;
332
333 if (unlikely(index >= array->map.max_entries))
334 return NULL;
335
336 return per_cpu_ptr(array->pptrs[index & array->index_mask], cpu);
337}
338
339int bpf_percpu_array_copy(struct bpf_map *map, void *key, void *value, u64 map_flags)
340{
341 struct bpf_array *array = container_of(map, struct bpf_array, map);
342 u32 index = *(u32 *)key;
343 void __percpu *pptr;
344 int cpu, off = 0;
345 u32 size;
346
347 if (unlikely(index >= array->map.max_entries))
348 return -ENOENT;
349
350 /* per_cpu areas are zero-filled and bpf programs can only
351 * access 'value_size' of them, so copying rounded areas
352 * will not leak any kernel data
353 */
354 size = array->elem_size;
355 rcu_read_lock();
356 pptr = array->pptrs[index & array->index_mask];
357 if (map_flags & BPF_F_CPU) {
358 cpu = map_flags >> 32;
359 copy_map_value(map, value, per_cpu_ptr(pptr, cpu));
360 check_and_init_map_value(map, value);
361 goto unlock;
362 }
363 for_each_possible_cpu(cpu) {
364 copy_map_value_long(map, value + off, per_cpu_ptr(pptr, cpu));
365 check_and_init_map_value(map, value + off);
366 off += size;
367 }
368unlock:
369 rcu_read_unlock();
370 return 0;
371}
372
373/* Called from syscall */
374int bpf_array_get_next_key(struct bpf_map *map, void *key, void *next_key)
375{
376 u32 index = key ? *(u32 *)key : U32_MAX;
377 u32 *next = (u32 *)next_key;
378
379 if (index >= map->max_entries) {
380 *next = 0;
381 return 0;
382 }
383
384 if (index == map->max_entries - 1)
385 return -ENOENT;
386
387 *next = index + 1;
388 return 0;
389}
390
391/* Called from syscall or from eBPF program */
392static long array_map_update_elem(struct bpf_map *map, void *key, void *value,
393 u64 map_flags)
394{
395 struct bpf_array *array = container_of(map, struct bpf_array, map);
396 u32 index = *(u32 *)key;
397 char *val;
398
399 if (unlikely((map_flags & ~BPF_F_LOCK) > BPF_EXIST))
400 /* unknown flags */
401 return -EINVAL;
402
403 if (unlikely(index >= array->map.max_entries))
404 /* all elements were pre-allocated, cannot insert a new one */
405 return -E2BIG;
406
407 if (unlikely(map_flags & BPF_NOEXIST))
408 /* all elements already exist */
409 return -EEXIST;
410
411 if (unlikely((map_flags & BPF_F_LOCK) &&
412 !btf_record_has_field(map->record, BPF_SPIN_LOCK)))
413 return -EINVAL;
414
415 if (array->map.map_type == BPF_MAP_TYPE_PERCPU_ARRAY) {
416 val = this_cpu_ptr(array->pptrs[index & array->index_mask]);
417 copy_map_value(map, val, value);
418 bpf_obj_cancel_fields(map, val);
419 } else {
420 val = array->value +
421 (u64)array->elem_size * (index & array->index_mask);
422 if (map_flags & BPF_F_LOCK)
423 copy_map_value_locked(map, val, value, false);
424 else
425 copy_map_value(map, val, value);
426 bpf_obj_cancel_fields(map, val);
427 }
428 return 0;
429}
430
431int bpf_percpu_array_update(struct bpf_map *map, void *key, void *value,
432 u64 map_flags)
433{
434 struct bpf_array *array = container_of(map, struct bpf_array, map);
435 u32 index = *(u32 *)key;
436 void __percpu *pptr;
437 void *ptr, *val;
438 u32 size;
439 int cpu, off = 0;
440
441 if (unlikely((map_flags & BPF_F_LOCK) || (u32)map_flags > BPF_F_ALL_CPUS))
442 /* unknown flags */
443 return -EINVAL;
444
445 if (unlikely(index >= array->map.max_entries))
446 /* all elements were pre-allocated, cannot insert a new one */
447 return -E2BIG;
448
449 if (unlikely(map_flags == BPF_NOEXIST))
450 /* all elements already exist */
451 return -EEXIST;
452
453 /* the user space will provide round_up(value_size, 8) bytes that
454 * will be copied into per-cpu area. bpf programs can only access
455 * value_size of it. During lookup the same extra bytes will be
456 * returned or zeros which were zero-filled by percpu_alloc,
457 * so no kernel data leaks possible
458 */
459 size = array->elem_size;
460 rcu_read_lock();
461 pptr = array->pptrs[index & array->index_mask];
462 if (map_flags & BPF_F_CPU) {
463 cpu = map_flags >> 32;
464 ptr = per_cpu_ptr(pptr, cpu);
465 copy_map_value(map, ptr, value);
466 bpf_obj_cancel_fields(map, ptr);
467 goto unlock;
468 }
469 for_each_possible_cpu(cpu) {
470 ptr = per_cpu_ptr(pptr, cpu);
471 val = (map_flags & BPF_F_ALL_CPUS) ? value : value + off;
472 copy_map_value(map, ptr, val);
473 bpf_obj_cancel_fields(map, ptr);
474 off += size;
475 }
476unlock:
477 rcu_read_unlock();
478 return 0;
479}
480
481/* Called from syscall or from eBPF program */
482static long array_map_delete_elem(struct bpf_map *map, void *key)
483{
484 return -EINVAL;
485}
486
487static void *array_map_vmalloc_addr(struct bpf_array *array)
488{
489 return (void *)round_down((unsigned long)array, PAGE_SIZE);
490}
491
492static void array_map_free_internal_structs(struct bpf_map *map)
493{
494 struct bpf_array *array = container_of(map, struct bpf_array, map);
495 int i;
496
497 /* We only free internal structs on uref dropping to zero */
498 if (!bpf_map_has_internal_structs(map))
499 return;
500
501 for (i = 0; i < array->map.max_entries; i++)
502 bpf_map_free_internal_structs(map, array_map_elem_ptr(array, i));
503}
504
505/* Called when map->refcnt goes to zero, either from workqueue or from syscall */
506static void array_map_free(struct bpf_map *map)
507{
508 struct bpf_array *array = container_of(map, struct bpf_array, map);
509 int i;
510
511 if (!IS_ERR_OR_NULL(map->record)) {
512 if (array->map.map_type == BPF_MAP_TYPE_PERCPU_ARRAY) {
513 for (i = 0; i < array->map.max_entries; i++) {
514 void __percpu *pptr = array->pptrs[i & array->index_mask];
515 int cpu;
516
517 for_each_possible_cpu(cpu) {
518 bpf_obj_free_fields(map->record, per_cpu_ptr(pptr, cpu));
519 cond_resched();
520 }
521 }
522 } else {
523 for (i = 0; i < array->map.max_entries; i++)
524 bpf_obj_free_fields(map->record, array_map_elem_ptr(array, i));
525 }
526 }
527
528 if (array->map.map_type == BPF_MAP_TYPE_PERCPU_ARRAY)
529 bpf_array_free_percpu(array);
530
531 if (array->map.map_flags & BPF_F_MMAPABLE)
532 bpf_map_area_free(array_map_vmalloc_addr(array));
533 else
534 bpf_map_area_free(array);
535}
536
537static void array_map_seq_show_elem(struct bpf_map *map, void *key,
538 struct seq_file *m)
539{
540 void *value;
541
542 rcu_read_lock();
543
544 value = array_map_lookup_elem(map, key);
545 if (!value) {
546 rcu_read_unlock();
547 return;
548 }
549
550 if (map->btf_key_type_id)
551 seq_printf(m, "%u: ", *(u32 *)key);
552 btf_type_seq_show(map->btf, map->btf_value_type_id, value, m);
553 seq_putc(m, '\n');
554
555 rcu_read_unlock();
556}
557
558static void percpu_array_map_seq_show_elem(struct bpf_map *map, void *key,
559 struct seq_file *m)
560{
561 struct bpf_array *array = container_of(map, struct bpf_array, map);
562 u32 index = *(u32 *)key;
563 void __percpu *pptr;
564 int cpu;
565
566 rcu_read_lock();
567
568 seq_printf(m, "%u: {\n", *(u32 *)key);
569 pptr = array->pptrs[index & array->index_mask];
570 for_each_possible_cpu(cpu) {
571 seq_printf(m, "\tcpu%d: ", cpu);
572 btf_type_seq_show(map->btf, map->btf_value_type_id,
573 per_cpu_ptr(pptr, cpu), m);
574 seq_putc(m, '\n');
575 }
576 seq_puts(m, "}\n");
577
578 rcu_read_unlock();
579}
580
581static int array_map_check_btf(struct bpf_map *map,
582 const struct btf *btf,
583 const struct btf_type *key_type,
584 const struct btf_type *value_type)
585{
586 /* One exception for keyless BTF: .bss/.data/.rodata/.percpu map */
587 if (btf_type_is_void(key_type)) {
588 if ((map->map_type != BPF_MAP_TYPE_ARRAY &&
589 map->map_type != BPF_MAP_TYPE_PERCPU_ARRAY) ||
590 map->max_entries != 1)
591 return -EINVAL;
592
593 if (BTF_INFO_KIND(value_type->info) != BTF_KIND_DATASEC)
594 return -EINVAL;
595
596 return 0;
597 }
598
599 /*
600 * Bpf array can only take a u32 key. This check makes sure
601 * that the btf matches the attr used during map_create.
602 */
603 if (!btf_type_is_i32(key_type))
604 return -EINVAL;
605
606 return 0;
607}
608
609static int array_map_mmap(struct bpf_map *map, struct vm_area_struct *vma)
610{
611 struct bpf_array *array = container_of(map, struct bpf_array, map);
612
613 if (!(map->map_flags & BPF_F_MMAPABLE))
614 return -EINVAL;
615
616 /* use u64 math so the offset cannot overflow on 32-bit archs */
617 if ((u64)vma->vm_pgoff * PAGE_SIZE + (vma->vm_end - vma->vm_start) >
618 PAGE_ALIGN((u64)array->map.max_entries * array->elem_size))
619 return -EINVAL;
620
621 /*
622 * Pages are faulted in on demand by array_map_mmap_fault(). Set the
623 * same flags that the eager remap_vmalloc_range() path used to set
624 * via vm_insert_page(), so that e.g. NUMA balancing keeps skipping
625 * these VMAs.
626 */
627 vm_flags_set(vma, VM_DONTEXPAND | VM_DONTDUMP | VM_MIXEDMAP);
628
629 return 0;
630}
631
632static vm_fault_t array_map_mmap_fault(struct bpf_map *map,
633 struct vm_fault *vmf)
634{
635 struct bpf_array *array = container_of(map, struct bpf_array, map);
636 struct page *page;
637
638 page = vmalloc_to_page(array->value + ((u64)vmf->pgoff << PAGE_SHIFT));
639 if (!page)
640 return VM_FAULT_SIGBUS;
641
642 /* the eager remap_vmalloc_range() flushed via vm_insert_page() */
643 flush_dcache_folio(page_folio(page));
644 get_page(page);
645 vmf->page = page;
646
647 return 0;
648}
649
650static bool array_map_meta_equal(const struct bpf_map *meta0,
651 const struct bpf_map *meta1)
652{
653 if (!bpf_map_meta_equal(meta0, meta1))
654 return false;
655 return meta0->map_flags & BPF_F_INNER_MAP ? true :
656 meta0->max_entries == meta1->max_entries;
657}
658
659struct bpf_iter_seq_array_map_info {
660 struct bpf_map *map;
661 void *percpu_value_buf;
662 u32 index;
663};
664
665static void *bpf_array_map_seq_start(struct seq_file *seq, loff_t *pos)
666{
667 struct bpf_iter_seq_array_map_info *info = seq->private;
668 struct bpf_map *map = info->map;
669 struct bpf_array *array;
670 u32 index;
671
672 if (info->index >= map->max_entries)
673 return NULL;
674
675 if (*pos == 0)
676 ++*pos;
677 array = container_of(map, struct bpf_array, map);
678 index = info->index & array->index_mask;
679 if (info->percpu_value_buf)
680 return (void *)(uintptr_t)array->pptrs[index];
681 return array_map_elem_ptr(array, index);
682}
683
684static void *bpf_array_map_seq_next(struct seq_file *seq, void *v, loff_t *pos)
685{
686 struct bpf_iter_seq_array_map_info *info = seq->private;
687 struct bpf_map *map = info->map;
688 struct bpf_array *array;
689 u32 index;
690
691 ++*pos;
692 ++info->index;
693 if (info->index >= map->max_entries)
694 return NULL;
695
696 array = container_of(map, struct bpf_array, map);
697 index = info->index & array->index_mask;
698 if (info->percpu_value_buf)
699 return (void *)(uintptr_t)array->pptrs[index];
700 return array_map_elem_ptr(array, index);
701}
702
703static int __bpf_array_map_seq_show(struct seq_file *seq, void *v)
704{
705 struct bpf_iter_seq_array_map_info *info = seq->private;
706 struct bpf_iter__bpf_map_elem ctx = {};
707 struct bpf_map *map = info->map;
708 struct bpf_array *array = container_of(map, struct bpf_array, map);
709 struct bpf_iter_meta meta;
710 struct bpf_prog *prog;
711 int off = 0, cpu = 0;
712 void __percpu *pptr;
713 u32 size;
714
715 meta.seq = seq;
716 prog = bpf_iter_get_info(&meta, v == NULL);
717 if (!prog)
718 return 0;
719
720 ctx.meta = &meta;
721 ctx.map = info->map;
722 if (v) {
723 ctx.key = &info->index;
724
725 if (!info->percpu_value_buf) {
726 ctx.value = v;
727 } else {
728 pptr = (void __percpu *)(uintptr_t)v;
729 size = array->elem_size;
730 for_each_possible_cpu(cpu) {
731 copy_map_value_long(map, info->percpu_value_buf + off,
732 per_cpu_ptr(pptr, cpu));
733 check_and_init_map_value(map, info->percpu_value_buf + off);
734 off += size;
735 }
736 ctx.value = info->percpu_value_buf;
737 }
738 }
739
740 return bpf_iter_run_prog(prog, &ctx);
741}
742
743static int bpf_array_map_seq_show(struct seq_file *seq, void *v)
744{
745 return __bpf_array_map_seq_show(seq, v);
746}
747
748static void bpf_array_map_seq_stop(struct seq_file *seq, void *v)
749{
750 if (!v)
751 (void)__bpf_array_map_seq_show(seq, NULL);
752}
753
754static int bpf_iter_init_array_map(void *priv_data,
755 struct bpf_iter_aux_info *aux)
756{
757 struct bpf_iter_seq_array_map_info *seq_info = priv_data;
758 struct bpf_map *map = aux->map;
759 struct bpf_array *array = container_of(map, struct bpf_array, map);
760 void *value_buf;
761 u32 buf_size;
762
763 if (map->map_type == BPF_MAP_TYPE_PERCPU_ARRAY) {
764 buf_size = array->elem_size * num_possible_cpus();
765 value_buf = kmalloc(buf_size, GFP_USER | __GFP_NOWARN);
766 if (!value_buf)
767 return -ENOMEM;
768
769 seq_info->percpu_value_buf = value_buf;
770 }
771
772 /* bpf_iter_attach_map() acquires a map uref, and the uref may be
773 * released before or in the middle of iterating map elements, so
774 * acquire an extra map uref for iterator.
775 */
776 bpf_map_inc_with_uref(map);
777 seq_info->map = map;
778 return 0;
779}
780
781static void bpf_iter_fini_array_map(void *priv_data)
782{
783 struct bpf_iter_seq_array_map_info *seq_info = priv_data;
784
785 bpf_map_put_with_uref(seq_info->map);
786 kfree(seq_info->percpu_value_buf);
787}
788
789static const struct seq_operations bpf_array_map_seq_ops = {
790 .start = bpf_array_map_seq_start,
791 .next = bpf_array_map_seq_next,
792 .stop = bpf_array_map_seq_stop,
793 .show = bpf_array_map_seq_show,
794};
795
796static const struct bpf_iter_seq_info iter_seq_info = {
797 .seq_ops = &bpf_array_map_seq_ops,
798 .init_seq_private = bpf_iter_init_array_map,
799 .fini_seq_private = bpf_iter_fini_array_map,
800 .seq_priv_size = sizeof(struct bpf_iter_seq_array_map_info),
801};
802
803static long bpf_for_each_array_elem(struct bpf_map *map, bpf_callback_t callback_fn,
804 void *callback_ctx, u64 flags)
805{
806 u32 i, key, num_elems = 0;
807 struct bpf_array *array;
808 bool is_percpu;
809 u64 ret = 0;
810 void *val;
811
812 cant_migrate();
813
814 if (flags != 0)
815 return -EINVAL;
816
817 is_percpu = map->map_type == BPF_MAP_TYPE_PERCPU_ARRAY;
818 array = container_of(map, struct bpf_array, map);
819 for (i = 0; i < map->max_entries; i++) {
820 if (is_percpu)
821 val = this_cpu_ptr(array->pptrs[i]);
822 else
823 val = array_map_elem_ptr(array, i);
824 num_elems++;
825 key = i;
826 ret = callback_fn((u64)(long)map, (u64)(long)&key,
827 (u64)(long)val, (u64)(long)callback_ctx, 0);
828 /* return value: 0 - continue, 1 - stop and return */
829 if (ret)
830 break;
831 }
832
833 return num_elems;
834}
835
836static u64 array_map_mem_usage(const struct bpf_map *map)
837{
838 struct bpf_array *array = container_of(map, struct bpf_array, map);
839 bool percpu = map->map_type == BPF_MAP_TYPE_PERCPU_ARRAY;
840 u32 elem_size = array->elem_size;
841 u64 entries = map->max_entries;
842 u64 usage = sizeof(*array);
843
844 if (percpu) {
845 usage += entries * sizeof(void *);
846 usage += entries * elem_size * num_possible_cpus();
847 } else {
848 if (map->map_flags & BPF_F_MMAPABLE) {
849 usage = PAGE_ALIGN(usage);
850 usage += PAGE_ALIGN(entries * elem_size);
851 } else {
852 usage += entries * elem_size;
853 }
854 }
855 return usage;
856}
857
858BTF_ID_LIST_SINGLE(array_map_btf_ids, struct, bpf_array)
859const struct bpf_map_ops array_map_ops = {
860 .map_meta_equal = array_map_meta_equal,
861 .map_alloc_check = array_map_alloc_check,
862 .map_alloc = array_map_alloc,
863 .map_free = array_map_free,
864 .map_get_next_key = bpf_array_get_next_key,
865 .map_release_uref = array_map_free_internal_structs,
866 .map_lookup_elem = array_map_lookup_elem,
867 .map_update_elem = array_map_update_elem,
868 .map_delete_elem = array_map_delete_elem,
869 .map_gen_lookup = array_map_gen_lookup,
870 .map_direct_value_addr = array_map_direct_value_addr,
871 .map_direct_value_meta = array_map_direct_value_meta,
872 .map_mmap = array_map_mmap,
873 .map_mmap_fault = array_map_mmap_fault,
874 .map_seq_show_elem = array_map_seq_show_elem,
875 .map_check_btf = array_map_check_btf,
876 .map_lookup_batch = generic_map_lookup_batch,
877 .map_update_batch = generic_map_update_batch,
878 .map_set_for_each_callback_args = map_set_for_each_callback_args,
879 .map_for_each_callback = bpf_for_each_array_elem,
880 .map_mem_usage = array_map_mem_usage,
881 .map_btf_id = &array_map_btf_ids[0],
882 .iter_seq_info = &iter_seq_info,
883 .map_get_hash = &array_map_get_hash,
884};
885
886const struct bpf_map_ops percpu_array_map_ops = {
887 .map_meta_equal = array_map_meta_equal,
888 .map_alloc_check = array_map_alloc_check,
889 .map_alloc = array_map_alloc,
890 .map_free = array_map_free,
891 .map_get_next_key = bpf_array_get_next_key,
892 .map_lookup_elem = percpu_array_map_lookup_elem,
893 .map_gen_lookup = percpu_array_map_gen_lookup,
894 .map_direct_value_addr = percpu_array_map_direct_value_addr,
895 .map_direct_value_meta = percpu_array_map_direct_value_meta,
896 .map_update_elem = array_map_update_elem,
897 .map_delete_elem = array_map_delete_elem,
898 .map_lookup_percpu_elem = percpu_array_map_lookup_percpu_elem,
899 .map_seq_show_elem = percpu_array_map_seq_show_elem,
900 .map_check_btf = array_map_check_btf,
901 .map_lookup_batch = generic_map_lookup_batch,
902 .map_update_batch = generic_map_update_batch,
903 .map_set_for_each_callback_args = map_set_for_each_callback_args,
904 .map_for_each_callback = bpf_for_each_array_elem,
905 .map_mem_usage = array_map_mem_usage,
906 .map_btf_id = &array_map_btf_ids[0],
907 .iter_seq_info = &iter_seq_info,
908};
909
910static int fd_array_map_alloc_check(union bpf_attr *attr)
911{
912 /* only file descriptors can be stored in this type of map */
913 if (attr->value_size != sizeof(u32))
914 return -EINVAL;
915 /* Program read-only/write-only not supported for special maps yet. */
916 if (attr->map_flags & (BPF_F_RDONLY_PROG | BPF_F_WRONLY_PROG))
917 return -EINVAL;
918 return array_map_alloc_check(attr);
919}
920
921static void fd_array_map_free(struct bpf_map *map)
922{
923 struct bpf_array *array = container_of(map, struct bpf_array, map);
924 int i;
925
926 /* make sure it's empty */
927 for (i = 0; i < array->map.max_entries; i++)
928 BUG_ON(array->ptrs[i] != NULL);
929
930 bpf_map_area_free(array);
931}
932
933static void *fd_array_map_lookup_elem(struct bpf_map *map, void *key)
934{
935 return ERR_PTR(-EOPNOTSUPP);
936}
937
938/* only called from syscall */
939int bpf_fd_array_map_lookup_elem(struct bpf_map *map, void *key, u32 *value)
940{
941 void **elem, *ptr;
942 int ret = 0;
943
944 if (!map->ops->map_fd_sys_lookup_elem)
945 return -ENOTSUPP;
946
947 rcu_read_lock();
948 elem = array_map_lookup_elem(map, key);
949 if (elem && (ptr = READ_ONCE(*elem)))
950 *value = map->ops->map_fd_sys_lookup_elem(ptr);
951 else
952 ret = -ENOENT;
953 rcu_read_unlock();
954
955 return ret;
956}
957
958/* only called from syscall */
959int bpf_fd_array_map_update_elem(struct bpf_map *map, struct file *map_file,
960 void *key, void *value, u64 map_flags)
961{
962 struct bpf_array *array = container_of(map, struct bpf_array, map);
963 void *new_ptr, *old_ptr;
964 u32 index = *(u32 *)key, ufd;
965
966 if (map_flags != BPF_ANY)
967 return -EINVAL;
968
969 if (index >= array->map.max_entries)
970 return -E2BIG;
971
972 ufd = *(u32 *)value;
973 new_ptr = map->ops->map_fd_get_ptr(map, map_file, ufd);
974 if (IS_ERR(new_ptr))
975 return PTR_ERR(new_ptr);
976
977 if (map->ops->map_poke_run) {
978 mutex_lock(&array->aux->poke_mutex);
979 old_ptr = xchg(array->ptrs + index, new_ptr);
980 map->ops->map_poke_run(map, index, old_ptr, new_ptr);
981 mutex_unlock(&array->aux->poke_mutex);
982 } else {
983 old_ptr = xchg(array->ptrs + index, new_ptr);
984 }
985
986 if (old_ptr)
987 map->ops->map_fd_put_ptr(map, old_ptr, true);
988 return 0;
989}
990
991static long __fd_array_map_delete_elem(struct bpf_map *map, void *key, bool need_defer)
992{
993 struct bpf_array *array = container_of(map, struct bpf_array, map);
994 void *old_ptr;
995 u32 index = *(u32 *)key;
996
997 if (index >= array->map.max_entries)
998 return -E2BIG;
999
1000 if (map->ops->map_poke_run) {
1001 mutex_lock(&array->aux->poke_mutex);
1002 old_ptr = xchg(array->ptrs + index, NULL);
1003 map->ops->map_poke_run(map, index, old_ptr, NULL);
1004 mutex_unlock(&array->aux->poke_mutex);
1005 } else {
1006 old_ptr = xchg(array->ptrs + index, NULL);
1007 }
1008
1009 if (old_ptr) {
1010 map->ops->map_fd_put_ptr(map, old_ptr, need_defer);
1011 return 0;
1012 } else {
1013 return -ENOENT;
1014 }
1015}
1016
1017static long fd_array_map_delete_elem(struct bpf_map *map, void *key)
1018{
1019 return __fd_array_map_delete_elem(map, key, true);
1020}
1021
1022static void *prog_fd_array_get_ptr(struct bpf_map *map,
1023 struct file *map_file, int fd)
1024{
1025 struct bpf_prog *prog = bpf_prog_get(fd);
1026 bool is_extended;
1027
1028 if (IS_ERR(prog))
1029 return prog;
1030
1031 if (prog->type == BPF_PROG_TYPE_EXT ||
1032 !bpf_prog_map_compatible(map, prog)) {
1033 bpf_prog_put(prog);
1034 return ERR_PTR(-EINVAL);
1035 }
1036
1037 mutex_lock(&prog->aux->ext_mutex);
1038 is_extended = prog->aux->is_extended;
1039 if (!is_extended)
1040 prog->aux->prog_array_member_cnt++;
1041 mutex_unlock(&prog->aux->ext_mutex);
1042 if (is_extended) {
1043 /* Extended prog can not be tail callee. It's to prevent a
1044 * potential infinite loop like:
1045 * tail callee prog entry -> tail callee prog subprog ->
1046 * freplace prog entry --tailcall-> tail callee prog entry.
1047 */
1048 bpf_prog_put(prog);
1049 return ERR_PTR(-EBUSY);
1050 }
1051
1052 return prog;
1053}
1054
1055static void prog_fd_array_put_ptr(struct bpf_map *map, void *ptr, bool need_defer)
1056{
1057 struct bpf_prog *prog = ptr;
1058
1059 mutex_lock(&prog->aux->ext_mutex);
1060 prog->aux->prog_array_member_cnt--;
1061 mutex_unlock(&prog->aux->ext_mutex);
1062 /* bpf_prog is freed after one RCU or tasks trace grace period */
1063 bpf_prog_put(prog);
1064}
1065
1066static u32 prog_fd_array_sys_lookup_elem(void *ptr)
1067{
1068 return ((struct bpf_prog *)ptr)->aux->id;
1069}
1070
1071/* decrement refcnt of all bpf_progs that are stored in this map */
1072static void bpf_fd_array_map_clear(struct bpf_map *map, bool need_defer)
1073{
1074 struct bpf_array *array = container_of(map, struct bpf_array, map);
1075 int i;
1076
1077 for (i = 0; i < array->map.max_entries; i++) {
1078 __fd_array_map_delete_elem(map, &i, need_defer);
1079 cond_resched();
1080 }
1081}
1082
1083static void prog_array_map_seq_show_elem(struct bpf_map *map, void *key,
1084 struct seq_file *m)
1085{
1086 void **elem, *ptr;
1087 u32 prog_id;
1088
1089 rcu_read_lock();
1090
1091 elem = array_map_lookup_elem(map, key);
1092 if (elem) {
1093 ptr = READ_ONCE(*elem);
1094 if (ptr) {
1095 seq_printf(m, "%u: ", *(u32 *)key);
1096 prog_id = prog_fd_array_sys_lookup_elem(ptr);
1097 btf_type_seq_show(map->btf, map->btf_value_type_id,
1098 &prog_id, m);
1099 seq_putc(m, '\n');
1100 }
1101 }
1102
1103 rcu_read_unlock();
1104}
1105
1106struct prog_poke_elem {
1107 struct list_head list;
1108 struct bpf_prog_aux *aux;
1109};
1110
1111static int prog_array_map_poke_track(struct bpf_map *map,
1112 struct bpf_prog_aux *prog_aux)
1113{
1114 struct prog_poke_elem *elem;
1115 struct bpf_array_aux *aux;
1116 int ret = 0;
1117
1118 aux = container_of(map, struct bpf_array, map)->aux;
1119 mutex_lock(&aux->poke_mutex);
1120 list_for_each_entry(elem, &aux->poke_progs, list) {
1121 if (elem->aux == prog_aux)
1122 goto out;
1123 }
1124
1125 elem = kmalloc_obj(*elem);
1126 if (!elem) {
1127 ret = -ENOMEM;
1128 goto out;
1129 }
1130
1131 INIT_LIST_HEAD(&elem->list);
1132 /* We must track the program's aux info at this point in time
1133 * since the program pointer itself may not be stable yet, see
1134 * also comment in prog_array_map_poke_run().
1135 */
1136 elem->aux = prog_aux;
1137
1138 list_add_tail(&elem->list, &aux->poke_progs);
1139out:
1140 mutex_unlock(&aux->poke_mutex);
1141 return ret;
1142}
1143
1144static void prog_array_map_poke_untrack(struct bpf_map *map,
1145 struct bpf_prog_aux *prog_aux)
1146{
1147 struct prog_poke_elem *elem, *tmp;
1148 struct bpf_array_aux *aux;
1149
1150 aux = container_of(map, struct bpf_array, map)->aux;
1151 mutex_lock(&aux->poke_mutex);
1152 list_for_each_entry_safe(elem, tmp, &aux->poke_progs, list) {
1153 if (elem->aux == prog_aux) {
1154 list_del_init(&elem->list);
1155 kfree(elem);
1156 break;
1157 }
1158 }
1159 mutex_unlock(&aux->poke_mutex);
1160}
1161
1162void __weak bpf_arch_poke_desc_update(struct bpf_jit_poke_descriptor *poke,
1163 struct bpf_prog *new, struct bpf_prog *old)
1164{
1165 WARN_ON_ONCE(1);
1166}
1167
1168static void prog_array_map_poke_run(struct bpf_map *map, u32 key,
1169 struct bpf_prog *old,
1170 struct bpf_prog *new)
1171{
1172 struct prog_poke_elem *elem;
1173 struct bpf_array_aux *aux;
1174
1175 aux = container_of(map, struct bpf_array, map)->aux;
1176 WARN_ON_ONCE(!mutex_is_locked(&aux->poke_mutex));
1177
1178 list_for_each_entry(elem, &aux->poke_progs, list) {
1179 struct bpf_jit_poke_descriptor *poke;
1180 int i;
1181
1182 for (i = 0; i < elem->aux->size_poke_tab; i++) {
1183 poke = &elem->aux->poke_tab[i];
1184
1185 /* Few things to be aware of:
1186 *
1187 * 1) We can only ever access aux in this context, but
1188 * not aux->prog since it might not be stable yet and
1189 * there could be danger of use after free otherwise.
1190 * 2) Initially when we start tracking aux, the program
1191 * is not JITed yet and also does not have a kallsyms
1192 * entry. We skip these as poke->tailcall_target_stable
1193 * is not active yet. The JIT will do the final fixup
1194 * before setting it stable. The various
1195 * poke->tailcall_target_stable are successively
1196 * activated, so tail call updates can arrive from here
1197 * while JIT is still finishing its final fixup for
1198 * non-activated poke entries.
1199 * 3) Also programs reaching refcount of zero while patching
1200 * is in progress is okay since we're protected under
1201 * poke_mutex and untrack the programs before the JIT
1202 * buffer is freed.
1203 */
1204 if (!READ_ONCE(poke->tailcall_target_stable))
1205 continue;
1206 if (poke->reason != BPF_POKE_REASON_TAIL_CALL)
1207 continue;
1208 if (poke->tail_call.map != map ||
1209 poke->tail_call.key != key)
1210 continue;
1211
1212 bpf_arch_poke_desc_update(poke, new, old);
1213 }
1214 }
1215}
1216
1217static void prog_array_map_clear_deferred(struct work_struct *work)
1218{
1219 struct bpf_map *map = container_of(work, struct bpf_array_aux,
1220 work)->map;
1221 bpf_fd_array_map_clear(map, true);
1222 bpf_map_put(map);
1223}
1224
1225static void prog_array_map_clear(struct bpf_map *map)
1226{
1227 struct bpf_array_aux *aux = container_of(map, struct bpf_array,
1228 map)->aux;
1229 bpf_map_inc(map);
1230 schedule_work(&aux->work);
1231}
1232
1233static struct bpf_map *prog_array_map_alloc(union bpf_attr *attr)
1234{
1235 struct bpf_array_aux *aux;
1236 struct bpf_map *map;
1237
1238 aux = kzalloc_obj(*aux, GFP_KERNEL_ACCOUNT);
1239 if (!aux)
1240 return ERR_PTR(-ENOMEM);
1241
1242 INIT_WORK(&aux->work, prog_array_map_clear_deferred);
1243 INIT_LIST_HEAD(&aux->poke_progs);
1244 mutex_init(&aux->poke_mutex);
1245
1246 map = array_map_alloc(attr);
1247 if (IS_ERR(map)) {
1248 kfree(aux);
1249 return map;
1250 }
1251
1252 container_of(map, struct bpf_array, map)->aux = aux;
1253 aux->map = map;
1254
1255 return map;
1256}
1257
1258static void prog_array_map_free(struct bpf_map *map)
1259{
1260 struct prog_poke_elem *elem, *tmp;
1261 struct bpf_array_aux *aux;
1262
1263 aux = container_of(map, struct bpf_array, map)->aux;
1264 list_for_each_entry_safe(elem, tmp, &aux->poke_progs, list) {
1265 list_del_init(&elem->list);
1266 kfree(elem);
1267 }
1268 kfree(aux);
1269 fd_array_map_free(map);
1270}
1271
1272/* prog_array->aux->{type,jited} is a runtime binding.
1273 * Doing static check alone in the verifier is not enough.
1274 * Thus, prog_array_map cannot be used as an inner_map
1275 * and map_meta_equal is not implemented.
1276 */
1277const struct bpf_map_ops prog_array_map_ops = {
1278 .map_alloc_check = fd_array_map_alloc_check,
1279 .map_alloc = prog_array_map_alloc,
1280 .map_free = prog_array_map_free,
1281 .map_poke_track = prog_array_map_poke_track,
1282 .map_poke_untrack = prog_array_map_poke_untrack,
1283 .map_poke_run = prog_array_map_poke_run,
1284 .map_get_next_key = bpf_array_get_next_key,
1285 .map_lookup_elem = fd_array_map_lookup_elem,
1286 .map_delete_elem = fd_array_map_delete_elem,
1287 .map_fd_get_ptr = prog_fd_array_get_ptr,
1288 .map_fd_put_ptr = prog_fd_array_put_ptr,
1289 .map_fd_sys_lookup_elem = prog_fd_array_sys_lookup_elem,
1290 .map_release_uref = prog_array_map_clear,
1291 .map_seq_show_elem = prog_array_map_seq_show_elem,
1292 .map_mem_usage = array_map_mem_usage,
1293 .map_btf_id = &array_map_btf_ids[0],
1294};
1295
1296static struct bpf_event_entry *bpf_event_entry_gen(struct file *perf_file,
1297 struct file *map_file)
1298{
1299 struct bpf_event_entry *ee;
1300
1301 ee = kzalloc_obj(*ee);
1302 if (ee) {
1303 ee->event = perf_file->private_data;
1304 ee->perf_file = perf_file;
1305 ee->map_file = map_file;
1306 }
1307
1308 return ee;
1309}
1310
1311static void __bpf_event_entry_free(struct rcu_head *rcu)
1312{
1313 struct bpf_event_entry *ee;
1314
1315 ee = container_of(rcu, struct bpf_event_entry, rcu);
1316 fput(ee->perf_file);
1317 kfree(ee);
1318}
1319
1320static void bpf_event_entry_free_rcu(struct bpf_event_entry *ee)
1321{
1322 call_rcu(&ee->rcu, __bpf_event_entry_free);
1323}
1324
1325static void *perf_event_fd_array_get_ptr(struct bpf_map *map,
1326 struct file *map_file, int fd)
1327{
1328 struct bpf_event_entry *ee;
1329 struct perf_event *event;
1330 struct file *perf_file;
1331 u64 value;
1332
1333 perf_file = perf_event_get(fd);
1334 if (IS_ERR(perf_file))
1335 return perf_file;
1336
1337 ee = ERR_PTR(-EOPNOTSUPP);
1338 event = perf_file->private_data;
1339 if (perf_event_read_local(event, &value, NULL, NULL) == -EOPNOTSUPP)
1340 goto err_out;
1341
1342 ee = bpf_event_entry_gen(perf_file, map_file);
1343 if (ee)
1344 return ee;
1345 ee = ERR_PTR(-ENOMEM);
1346err_out:
1347 fput(perf_file);
1348 return ee;
1349}
1350
1351static void perf_event_fd_array_put_ptr(struct bpf_map *map, void *ptr, bool need_defer)
1352{
1353 /* bpf_perf_event is freed after one RCU grace period */
1354 bpf_event_entry_free_rcu(ptr);
1355}
1356
1357static void perf_event_fd_array_release(struct bpf_map *map,
1358 struct file *map_file)
1359{
1360 struct bpf_array *array = container_of(map, struct bpf_array, map);
1361 struct bpf_event_entry *ee;
1362 int i;
1363
1364 if (map->map_flags & BPF_F_PRESERVE_ELEMS)
1365 return;
1366
1367 rcu_read_lock();
1368 for (i = 0; i < array->map.max_entries; i++) {
1369 ee = READ_ONCE(array->ptrs[i]);
1370 if (ee && ee->map_file == map_file)
1371 __fd_array_map_delete_elem(map, &i, true);
1372 }
1373 rcu_read_unlock();
1374}
1375
1376static void perf_event_fd_array_map_free(struct bpf_map *map)
1377{
1378 if (map->map_flags & BPF_F_PRESERVE_ELEMS)
1379 bpf_fd_array_map_clear(map, false);
1380 fd_array_map_free(map);
1381}
1382
1383const struct bpf_map_ops perf_event_array_map_ops = {
1384 .map_meta_equal = bpf_map_meta_equal,
1385 .map_alloc_check = fd_array_map_alloc_check,
1386 .map_alloc = array_map_alloc,
1387 .map_free = perf_event_fd_array_map_free,
1388 .map_get_next_key = bpf_array_get_next_key,
1389 .map_lookup_elem = fd_array_map_lookup_elem,
1390 .map_delete_elem = fd_array_map_delete_elem,
1391 .map_fd_get_ptr = perf_event_fd_array_get_ptr,
1392 .map_fd_put_ptr = perf_event_fd_array_put_ptr,
1393 .map_release = perf_event_fd_array_release,
1394 .map_check_btf = map_check_no_btf,
1395 .map_mem_usage = array_map_mem_usage,
1396 .map_btf_id = &array_map_btf_ids[0],
1397};
1398
1399#ifdef CONFIG_CGROUPS
1400static void *cgroup_fd_array_get_ptr(struct bpf_map *map,
1401 struct file *map_file /* not used */,
1402 int fd)
1403{
1404 return cgroup_get_from_fd(fd);
1405}
1406
1407static void cgroup_fd_array_put_ptr(struct bpf_map *map, void *ptr, bool need_defer)
1408{
1409 /* cgroup_put free cgrp after a rcu grace period */
1410 cgroup_put(ptr);
1411}
1412
1413static void cgroup_fd_array_free(struct bpf_map *map)
1414{
1415 bpf_fd_array_map_clear(map, false);
1416 fd_array_map_free(map);
1417}
1418
1419const struct bpf_map_ops cgroup_array_map_ops = {
1420 .map_meta_equal = bpf_map_meta_equal,
1421 .map_alloc_check = fd_array_map_alloc_check,
1422 .map_alloc = array_map_alloc,
1423 .map_free = cgroup_fd_array_free,
1424 .map_get_next_key = bpf_array_get_next_key,
1425 .map_lookup_elem = fd_array_map_lookup_elem,
1426 .map_delete_elem = fd_array_map_delete_elem,
1427 .map_fd_get_ptr = cgroup_fd_array_get_ptr,
1428 .map_fd_put_ptr = cgroup_fd_array_put_ptr,
1429 .map_check_btf = map_check_no_btf,
1430 .map_mem_usage = array_map_mem_usage,
1431 .map_btf_id = &array_map_btf_ids[0],
1432};
1433#endif
1434
1435static struct bpf_map *array_of_map_alloc(union bpf_attr *attr)
1436{
1437 struct bpf_map *map, *inner_map_meta;
1438
1439 inner_map_meta = bpf_map_meta_alloc(attr->inner_map_fd);
1440 if (IS_ERR(inner_map_meta))
1441 return inner_map_meta;
1442
1443 map = array_map_alloc(attr);
1444 if (IS_ERR(map)) {
1445 bpf_map_meta_free(inner_map_meta);
1446 return map;
1447 }
1448
1449 map->inner_map_meta = inner_map_meta;
1450
1451 return map;
1452}
1453
1454static void array_of_map_free(struct bpf_map *map)
1455{
1456 /* map->inner_map_meta is only accessed by syscall which
1457 * is protected by fdget/fdput.
1458 */
1459 bpf_map_meta_free(map->inner_map_meta);
1460 bpf_fd_array_map_clear(map, false);
1461 fd_array_map_free(map);
1462}
1463
1464static void *array_of_map_lookup_elem(struct bpf_map *map, void *key)
1465{
1466 struct bpf_map **inner_map = array_map_lookup_elem(map, key);
1467
1468 if (!inner_map)
1469 return NULL;
1470
1471 return READ_ONCE(*inner_map);
1472}
1473
1474static int array_of_map_gen_lookup(struct bpf_map *map,
1475 struct bpf_insn *insn_buf)
1476{
1477 struct bpf_array *array = container_of(map, struct bpf_array, map);
1478 u32 elem_size = array->elem_size;
1479 struct bpf_insn *insn = insn_buf;
1480 const int ret = BPF_REG_0;
1481 const int map_ptr = BPF_REG_1;
1482 const int index = BPF_REG_2;
1483
1484 *insn++ = BPF_ALU64_IMM(BPF_ADD, map_ptr, offsetof(struct bpf_array, value));
1485 *insn++ = BPF_LDX_MEM(BPF_W, ret, index, 0);
1486 if (!map->bypass_spec_v1) {
1487 *insn++ = BPF_JMP_IMM(BPF_JGE, ret, map->max_entries, 6);
1488 *insn++ = BPF_ALU32_IMM(BPF_AND, ret, array->index_mask);
1489 } else {
1490 *insn++ = BPF_JMP_IMM(BPF_JGE, ret, map->max_entries, 5);
1491 }
1492 if (is_power_of_2(elem_size))
1493 *insn++ = BPF_ALU64_IMM(BPF_LSH, ret, ilog2(elem_size));
1494 else
1495 *insn++ = BPF_ALU64_IMM(BPF_MUL, ret, elem_size);
1496 *insn++ = BPF_ALU64_REG(BPF_ADD, ret, map_ptr);
1497 *insn++ = BPF_LDX_MEM(BPF_DW, ret, ret, 0);
1498 *insn++ = BPF_JMP_IMM(BPF_JEQ, ret, 0, 1);
1499 *insn++ = BPF_JMP_IMM(BPF_JA, 0, 0, 1);
1500 *insn++ = BPF_MOV64_IMM(ret, 0);
1501
1502 return insn - insn_buf;
1503}
1504
1505const struct bpf_map_ops array_of_maps_map_ops = {
1506 .map_alloc_check = fd_array_map_alloc_check,
1507 .map_alloc = array_of_map_alloc,
1508 .map_free = array_of_map_free,
1509 .map_get_next_key = bpf_array_get_next_key,
1510 .map_lookup_elem = array_of_map_lookup_elem,
1511 .map_delete_elem = fd_array_map_delete_elem,
1512 .map_fd_get_ptr = bpf_map_fd_get_ptr,
1513 .map_fd_put_ptr = bpf_map_fd_put_ptr,
1514 .map_fd_sys_lookup_elem = bpf_map_fd_sys_lookup_elem,
1515 .map_gen_lookup = array_of_map_gen_lookup,
1516 .map_lookup_batch = generic_map_lookup_batch,
1517 .map_update_batch = generic_map_update_batch,
1518 .map_check_btf = map_check_no_btf,
1519 .map_mem_usage = array_map_mem_usage,
1520 .map_btf_id = &array_map_btf_ids[0],
1521};