projects/linux·blob v5.18-rc4 · refs

1350 lines · 35.5 KiB

1// SPDX-License-Identifier: GPL-2.0-only
2/* Copyright (c) 2011-2014 PLUMgrid, http://plumgrid.com
3 * Copyright (c) 2016,2017 Facebook
4 */
5#include <linux/bpf.h>
6#include <linux/btf.h>
7#include <linux/err.h>
8#include <linux/slab.h>
9#include <linux/mm.h>
10#include <linux/filter.h>
11#include <linux/perf_event.h>
12#include <uapi/linux/btf.h>
13#include <linux/rcupdate_trace.h>
14
15#include "map_in_map.h"
16
17#define ARRAY_CREATE_FLAG_MASK \
18 (BPF_F_NUMA_NODE | BPF_F_MMAPABLE | BPF_F_ACCESS_MASK | \
19 BPF_F_PRESERVE_ELEMS | BPF_F_INNER_MAP)
20
21static void bpf_array_free_percpu(struct bpf_array *array)
22{
23 int i;
24
25 for (i = 0; i < array->map.max_entries; i++) {
26 free_percpu(array->pptrs[i]);
27 cond_resched();
28 }
29}
30
31static int bpf_array_alloc_percpu(struct bpf_array *array)
32{
33 void __percpu *ptr;
34 int i;
35
36 for (i = 0; i < array->map.max_entries; i++) {
37 ptr = bpf_map_alloc_percpu(&array->map, array->elem_size, 8,
38 GFP_USER | __GFP_NOWARN);
39 if (!ptr) {
40 bpf_array_free_percpu(array);
41 return -ENOMEM;
42 }
43 array->pptrs[i] = ptr;
44 cond_resched();
45 }
46
47 return 0;
48}
49
50/* Called from syscall */
51int array_map_alloc_check(union bpf_attr *attr)
52{
53 bool percpu = attr->map_type == BPF_MAP_TYPE_PERCPU_ARRAY;
54 int numa_node = bpf_map_attr_numa_node(attr);
55
56 /* check sanity of attributes */
57 if (attr->max_entries == 0 || attr->key_size != 4 ||
58 attr->value_size == 0 ||
59 attr->map_flags & ~ARRAY_CREATE_FLAG_MASK ||
60 !bpf_map_flags_access_ok(attr->map_flags) ||
61 (percpu && numa_node != NUMA_NO_NODE))
62 return -EINVAL;
63
64 if (attr->map_type != BPF_MAP_TYPE_ARRAY &&
65 attr->map_flags & (BPF_F_MMAPABLE | BPF_F_INNER_MAP))
66 return -EINVAL;
67
68 if (attr->map_type != BPF_MAP_TYPE_PERF_EVENT_ARRAY &&
69 attr->map_flags & BPF_F_PRESERVE_ELEMS)
70 return -EINVAL;
71
72 if (attr->value_size > KMALLOC_MAX_SIZE)
73 /* if value_size is bigger, the user space won't be able to
74 * access the elements.
75 */
76 return -E2BIG;
77
78 return 0;
79}
80
81static struct bpf_map *array_map_alloc(union bpf_attr *attr)
82{
83 bool percpu = attr->map_type == BPF_MAP_TYPE_PERCPU_ARRAY;
84 int numa_node = bpf_map_attr_numa_node(attr);
85 u32 elem_size, index_mask, max_entries;
86 bool bypass_spec_v1 = bpf_bypass_spec_v1();
87 u64 array_size, mask64;
88 struct bpf_array *array;
89
90 elem_size = round_up(attr->value_size, 8);
91
92 max_entries = attr->max_entries;
93
94 /* On 32 bit archs roundup_pow_of_two() with max_entries that has
95 * upper most bit set in u32 space is undefined behavior due to
96 * resulting 1U << 32, so do it manually here in u64 space.
97 */
98 mask64 = fls_long(max_entries - 1);
99 mask64 = 1ULL << mask64;
100 mask64 -= 1;
101
102 index_mask = mask64;
103 if (!bypass_spec_v1) {
104 /* round up array size to nearest power of 2,
105 * since cpu will speculate within index_mask limits
106 */
107 max_entries = index_mask + 1;
108 /* Check for overflows. */
109 if (max_entries < attr->max_entries)
110 return ERR_PTR(-E2BIG);
111 }
112
113 array_size = sizeof(*array);
114 if (percpu) {
115 array_size += (u64) max_entries * sizeof(void *);
116 } else {
117 /* rely on vmalloc() to return page-aligned memory and
118 * ensure array->value is exactly page-aligned
119 */
120 if (attr->map_flags & BPF_F_MMAPABLE) {
121 array_size = PAGE_ALIGN(array_size);
122 array_size += PAGE_ALIGN((u64) max_entries * elem_size);
123 } else {
124 array_size += (u64) max_entries * elem_size;
125 }
126 }
127
128 /* allocate all map elements and zero-initialize them */
129 if (attr->map_flags & BPF_F_MMAPABLE) {
130 void *data;
131
132 /* kmalloc'ed memory can't be mmap'ed, use explicit vmalloc */
133 data = bpf_map_area_mmapable_alloc(array_size, numa_node);
134 if (!data)
135 return ERR_PTR(-ENOMEM);
136 array = data + PAGE_ALIGN(sizeof(struct bpf_array))
137 - offsetof(struct bpf_array, value);
138 } else {
139 array = bpf_map_area_alloc(array_size, numa_node);
140 }
141 if (!array)
142 return ERR_PTR(-ENOMEM);
143 array->index_mask = index_mask;
144 array->map.bypass_spec_v1 = bypass_spec_v1;
145
146 /* copy mandatory map attributes */
147 bpf_map_init_from_attr(&array->map, attr);
148 array->elem_size = elem_size;
149
150 if (percpu && bpf_array_alloc_percpu(array)) {
151 bpf_map_area_free(array);
152 return ERR_PTR(-ENOMEM);
153 }
154
155 return &array->map;
156}
157
158/* Called from syscall or from eBPF program */
159static void *array_map_lookup_elem(struct bpf_map *map, void *key)
160{
161 struct bpf_array *array = container_of(map, struct bpf_array, map);
162 u32 index = *(u32 *)key;
163
164 if (unlikely(index >= array->map.max_entries))
165 return NULL;
166
167 return array->value + array->elem_size * (index & array->index_mask);
168}
169
170static int array_map_direct_value_addr(const struct bpf_map *map, u64 *imm,
171 u32 off)
172{
173 struct bpf_array *array = container_of(map, struct bpf_array, map);
174
175 if (map->max_entries != 1)
176 return -ENOTSUPP;
177 if (off >= map->value_size)
178 return -EINVAL;
179
180 *imm = (unsigned long)array->value;
181 return 0;
182}
183
184static int array_map_direct_value_meta(const struct bpf_map *map, u64 imm,
185 u32 *off)
186{
187 struct bpf_array *array = container_of(map, struct bpf_array, map);
188 u64 base = (unsigned long)array->value;
189 u64 range = array->elem_size;
190
191 if (map->max_entries != 1)
192 return -ENOTSUPP;
193 if (imm < base || imm >= base + range)
194 return -ENOENT;
195
196 *off = imm - base;
197 return 0;
198}
199
200/* emit BPF instructions equivalent to C code of array_map_lookup_elem() */
201static int array_map_gen_lookup(struct bpf_map *map, struct bpf_insn *insn_buf)
202{
203 struct bpf_array *array = container_of(map, struct bpf_array, map);
204 struct bpf_insn *insn = insn_buf;
205 u32 elem_size = round_up(map->value_size, 8);
206 const int ret = BPF_REG_0;
207 const int map_ptr = BPF_REG_1;
208 const int index = BPF_REG_2;
209
210 if (map->map_flags & BPF_F_INNER_MAP)
211 return -EOPNOTSUPP;
212
213 *insn++ = BPF_ALU64_IMM(BPF_ADD, map_ptr, offsetof(struct bpf_array, value));
214 *insn++ = BPF_LDX_MEM(BPF_W, ret, index, 0);
215 if (!map->bypass_spec_v1) {
216 *insn++ = BPF_JMP_IMM(BPF_JGE, ret, map->max_entries, 4);
217 *insn++ = BPF_ALU32_IMM(BPF_AND, ret, array->index_mask);
218 } else {
219 *insn++ = BPF_JMP_IMM(BPF_JGE, ret, map->max_entries, 3);
220 }
221
222 if (is_power_of_2(elem_size)) {
223 *insn++ = BPF_ALU64_IMM(BPF_LSH, ret, ilog2(elem_size));
224 } else {
225 *insn++ = BPF_ALU64_IMM(BPF_MUL, ret, elem_size);
226 }
227 *insn++ = BPF_ALU64_REG(BPF_ADD, ret, map_ptr);
228 *insn++ = BPF_JMP_IMM(BPF_JA, 0, 0, 1);
229 *insn++ = BPF_MOV64_IMM(ret, 0);
230 return insn - insn_buf;
231}
232
233/* Called from eBPF program */
234static void *percpu_array_map_lookup_elem(struct bpf_map *map, void *key)
235{
236 struct bpf_array *array = container_of(map, struct bpf_array, map);
237 u32 index = *(u32 *)key;
238
239 if (unlikely(index >= array->map.max_entries))
240 return NULL;
241
242 return this_cpu_ptr(array->pptrs[index & array->index_mask]);
243}
244
245int bpf_percpu_array_copy(struct bpf_map *map, void *key, void *value)
246{
247 struct bpf_array *array = container_of(map, struct bpf_array, map);
248 u32 index = *(u32 *)key;
249 void __percpu *pptr;
250 int cpu, off = 0;
251 u32 size;
252
253 if (unlikely(index >= array->map.max_entries))
254 return -ENOENT;
255
256 /* per_cpu areas are zero-filled and bpf programs can only
257 * access 'value_size' of them, so copying rounded areas
258 * will not leak any kernel data
259 */
260 size = round_up(map->value_size, 8);
261 rcu_read_lock();
262 pptr = array->pptrs[index & array->index_mask];
263 for_each_possible_cpu(cpu) {
264 bpf_long_memcpy(value + off, per_cpu_ptr(pptr, cpu), size);
265 off += size;
266 }
267 rcu_read_unlock();
268 return 0;
269}
270
271/* Called from syscall */
272static int array_map_get_next_key(struct bpf_map *map, void *key, void *next_key)
273{
274 struct bpf_array *array = container_of(map, struct bpf_array, map);
275 u32 index = key ? *(u32 *)key : U32_MAX;
276 u32 *next = (u32 *)next_key;
277
278 if (index >= array->map.max_entries) {
279 *next = 0;
280 return 0;
281 }
282
283 if (index == array->map.max_entries - 1)
284 return -ENOENT;
285
286 *next = index + 1;
287 return 0;
288}
289
290static void check_and_free_timer_in_array(struct bpf_array *arr, void *val)
291{
292 if (unlikely(map_value_has_timer(&arr->map)))
293 bpf_timer_cancel_and_free(val + arr->map.timer_off);
294}
295
296/* Called from syscall or from eBPF program */
297static int array_map_update_elem(struct bpf_map *map, void *key, void *value,
298 u64 map_flags)
299{
300 struct bpf_array *array = container_of(map, struct bpf_array, map);
301 u32 index = *(u32 *)key;
302 char *val;
303
304 if (unlikely((map_flags & ~BPF_F_LOCK) > BPF_EXIST))
305 /* unknown flags */
306 return -EINVAL;
307
308 if (unlikely(index >= array->map.max_entries))
309 /* all elements were pre-allocated, cannot insert a new one */
310 return -E2BIG;
311
312 if (unlikely(map_flags & BPF_NOEXIST))
313 /* all elements already exist */
314 return -EEXIST;
315
316 if (unlikely((map_flags & BPF_F_LOCK) &&
317 !map_value_has_spin_lock(map)))
318 return -EINVAL;
319
320 if (array->map.map_type == BPF_MAP_TYPE_PERCPU_ARRAY) {
321 memcpy(this_cpu_ptr(array->pptrs[index & array->index_mask]),
322 value, map->value_size);
323 } else {
324 val = array->value +
325 array->elem_size * (index & array->index_mask);
326 if (map_flags & BPF_F_LOCK)
327 copy_map_value_locked(map, val, value, false);
328 else
329 copy_map_value(map, val, value);
330 check_and_free_timer_in_array(array, val);
331 }
332 return 0;
333}
334
335int bpf_percpu_array_update(struct bpf_map *map, void *key, void *value,
336 u64 map_flags)
337{
338 struct bpf_array *array = container_of(map, struct bpf_array, map);
339 u32 index = *(u32 *)key;
340 void __percpu *pptr;
341 int cpu, off = 0;
342 u32 size;
343
344 if (unlikely(map_flags > BPF_EXIST))
345 /* unknown flags */
346 return -EINVAL;
347
348 if (unlikely(index >= array->map.max_entries))
349 /* all elements were pre-allocated, cannot insert a new one */
350 return -E2BIG;
351
352 if (unlikely(map_flags == BPF_NOEXIST))
353 /* all elements already exist */
354 return -EEXIST;
355
356 /* the user space will provide round_up(value_size, 8) bytes that
357 * will be copied into per-cpu area. bpf programs can only access
358 * value_size of it. During lookup the same extra bytes will be
359 * returned or zeros which were zero-filled by percpu_alloc,
360 * so no kernel data leaks possible
361 */
362 size = round_up(map->value_size, 8);
363 rcu_read_lock();
364 pptr = array->pptrs[index & array->index_mask];
365 for_each_possible_cpu(cpu) {
366 bpf_long_memcpy(per_cpu_ptr(pptr, cpu), value + off, size);
367 off += size;
368 }
369 rcu_read_unlock();
370 return 0;
371}
372
373/* Called from syscall or from eBPF program */
374static int array_map_delete_elem(struct bpf_map *map, void *key)
375{
376 return -EINVAL;
377}
378
379static void *array_map_vmalloc_addr(struct bpf_array *array)
380{
381 return (void *)round_down((unsigned long)array, PAGE_SIZE);
382}
383
384static void array_map_free_timers(struct bpf_map *map)
385{
386 struct bpf_array *array = container_of(map, struct bpf_array, map);
387 int i;
388
389 if (likely(!map_value_has_timer(map)))
390 return;
391
392 for (i = 0; i < array->map.max_entries; i++)
393 bpf_timer_cancel_and_free(array->value + array->elem_size * i +
394 map->timer_off);
395}
396
397/* Called when map->refcnt goes to zero, either from workqueue or from syscall */
398static void array_map_free(struct bpf_map *map)
399{
400 struct bpf_array *array = container_of(map, struct bpf_array, map);
401
402 if (array->map.map_type == BPF_MAP_TYPE_PERCPU_ARRAY)
403 bpf_array_free_percpu(array);
404
405 if (array->map.map_flags & BPF_F_MMAPABLE)
406 bpf_map_area_free(array_map_vmalloc_addr(array));
407 else
408 bpf_map_area_free(array);
409}
410
411static void array_map_seq_show_elem(struct bpf_map *map, void *key,
412 struct seq_file *m)
413{
414 void *value;
415
416 rcu_read_lock();
417
418 value = array_map_lookup_elem(map, key);
419 if (!value) {
420 rcu_read_unlock();
421 return;
422 }
423
424 if (map->btf_key_type_id)
425 seq_printf(m, "%u: ", *(u32 *)key);
426 btf_type_seq_show(map->btf, map->btf_value_type_id, value, m);
427 seq_puts(m, "\n");
428
429 rcu_read_unlock();
430}
431
432static void percpu_array_map_seq_show_elem(struct bpf_map *map, void *key,
433 struct seq_file *m)
434{
435 struct bpf_array *array = container_of(map, struct bpf_array, map);
436 u32 index = *(u32 *)key;
437 void __percpu *pptr;
438 int cpu;
439
440 rcu_read_lock();
441
442 seq_printf(m, "%u: {\n", *(u32 *)key);
443 pptr = array->pptrs[index & array->index_mask];
444 for_each_possible_cpu(cpu) {
445 seq_printf(m, "\tcpu%d: ", cpu);
446 btf_type_seq_show(map->btf, map->btf_value_type_id,
447 per_cpu_ptr(pptr, cpu), m);
448 seq_puts(m, "\n");
449 }
450 seq_puts(m, "}\n");
451
452 rcu_read_unlock();
453}
454
455static int array_map_check_btf(const struct bpf_map *map,
456 const struct btf *btf,
457 const struct btf_type *key_type,
458 const struct btf_type *value_type)
459{
460 u32 int_data;
461
462 /* One exception for keyless BTF: .bss/.data/.rodata map */
463 if (btf_type_is_void(key_type)) {
464 if (map->map_type != BPF_MAP_TYPE_ARRAY ||
465 map->max_entries != 1)
466 return -EINVAL;
467
468 if (BTF_INFO_KIND(value_type->info) != BTF_KIND_DATASEC)
469 return -EINVAL;
470
471 return 0;
472 }
473
474 if (BTF_INFO_KIND(key_type->info) != BTF_KIND_INT)
475 return -EINVAL;
476
477 int_data = *(u32 *)(key_type + 1);
478 /* bpf array can only take a u32 key. This check makes sure
479 * that the btf matches the attr used during map_create.
480 */
481 if (BTF_INT_BITS(int_data) != 32 || BTF_INT_OFFSET(int_data))
482 return -EINVAL;
483
484 return 0;
485}
486
487static int array_map_mmap(struct bpf_map *map, struct vm_area_struct *vma)
488{
489 struct bpf_array *array = container_of(map, struct bpf_array, map);
490 pgoff_t pgoff = PAGE_ALIGN(sizeof(*array)) >> PAGE_SHIFT;
491
492 if (!(map->map_flags & BPF_F_MMAPABLE))
493 return -EINVAL;
494
495 if (vma->vm_pgoff * PAGE_SIZE + (vma->vm_end - vma->vm_start) >
496 PAGE_ALIGN((u64)array->map.max_entries * array->elem_size))
497 return -EINVAL;
498
499 return remap_vmalloc_range(vma, array_map_vmalloc_addr(array),
500 vma->vm_pgoff + pgoff);
501}
502
503static bool array_map_meta_equal(const struct bpf_map *meta0,
504 const struct bpf_map *meta1)
505{
506 if (!bpf_map_meta_equal(meta0, meta1))
507 return false;
508 return meta0->map_flags & BPF_F_INNER_MAP ? true :
509 meta0->max_entries == meta1->max_entries;
510}
511
512struct bpf_iter_seq_array_map_info {
513 struct bpf_map *map;
514 void *percpu_value_buf;
515 u32 index;
516};
517
518static void *bpf_array_map_seq_start(struct seq_file *seq, loff_t *pos)
519{
520 struct bpf_iter_seq_array_map_info *info = seq->private;
521 struct bpf_map *map = info->map;
522 struct bpf_array *array;
523 u32 index;
524
525 if (info->index >= map->max_entries)
526 return NULL;
527
528 if (*pos == 0)
529 ++*pos;
530 array = container_of(map, struct bpf_array, map);
531 index = info->index & array->index_mask;
532 if (info->percpu_value_buf)
533 return array->pptrs[index];
534 return array->value + array->elem_size * index;
535}
536
537static void *bpf_array_map_seq_next(struct seq_file *seq, void *v, loff_t *pos)
538{
539 struct bpf_iter_seq_array_map_info *info = seq->private;
540 struct bpf_map *map = info->map;
541 struct bpf_array *array;
542 u32 index;
543
544 ++*pos;
545 ++info->index;
546 if (info->index >= map->max_entries)
547 return NULL;
548
549 array = container_of(map, struct bpf_array, map);
550 index = info->index & array->index_mask;
551 if (info->percpu_value_buf)
552 return array->pptrs[index];
553 return array->value + array->elem_size * index;
554}
555
556static int __bpf_array_map_seq_show(struct seq_file *seq, void *v)
557{
558 struct bpf_iter_seq_array_map_info *info = seq->private;
559 struct bpf_iter__bpf_map_elem ctx = {};
560 struct bpf_map *map = info->map;
561 struct bpf_iter_meta meta;
562 struct bpf_prog *prog;
563 int off = 0, cpu = 0;
564 void __percpu **pptr;
565 u32 size;
566
567 meta.seq = seq;
568 prog = bpf_iter_get_info(&meta, v == NULL);
569 if (!prog)
570 return 0;
571
572 ctx.meta = &meta;
573 ctx.map = info->map;
574 if (v) {
575 ctx.key = &info->index;
576
577 if (!info->percpu_value_buf) {
578 ctx.value = v;
579 } else {
580 pptr = v;
581 size = round_up(map->value_size, 8);
582 for_each_possible_cpu(cpu) {
583 bpf_long_memcpy(info->percpu_value_buf + off,
584 per_cpu_ptr(pptr, cpu),
585 size);
586 off += size;
587 }
588 ctx.value = info->percpu_value_buf;
589 }
590 }
591
592 return bpf_iter_run_prog(prog, &ctx);
593}
594
595static int bpf_array_map_seq_show(struct seq_file *seq, void *v)
596{
597 return __bpf_array_map_seq_show(seq, v);
598}
599
600static void bpf_array_map_seq_stop(struct seq_file *seq, void *v)
601{
602 if (!v)
603 (void)__bpf_array_map_seq_show(seq, NULL);
604}
605
606static int bpf_iter_init_array_map(void *priv_data,
607 struct bpf_iter_aux_info *aux)
608{
609 struct bpf_iter_seq_array_map_info *seq_info = priv_data;
610 struct bpf_map *map = aux->map;
611 void *value_buf;
612 u32 buf_size;
613
614 if (map->map_type == BPF_MAP_TYPE_PERCPU_ARRAY) {
615 buf_size = round_up(map->value_size, 8) * num_possible_cpus();
616 value_buf = kmalloc(buf_size, GFP_USER | __GFP_NOWARN);
617 if (!value_buf)
618 return -ENOMEM;
619
620 seq_info->percpu_value_buf = value_buf;
621 }
622
623 seq_info->map = map;
624 return 0;
625}
626
627static void bpf_iter_fini_array_map(void *priv_data)
628{
629 struct bpf_iter_seq_array_map_info *seq_info = priv_data;
630
631 kfree(seq_info->percpu_value_buf);
632}
633
634static const struct seq_operations bpf_array_map_seq_ops = {
635 .start = bpf_array_map_seq_start,
636 .next = bpf_array_map_seq_next,
637 .stop = bpf_array_map_seq_stop,
638 .show = bpf_array_map_seq_show,
639};
640
641static const struct bpf_iter_seq_info iter_seq_info = {
642 .seq_ops = &bpf_array_map_seq_ops,
643 .init_seq_private = bpf_iter_init_array_map,
644 .fini_seq_private = bpf_iter_fini_array_map,
645 .seq_priv_size = sizeof(struct bpf_iter_seq_array_map_info),
646};
647
648static int bpf_for_each_array_elem(struct bpf_map *map, bpf_callback_t callback_fn,
649 void *callback_ctx, u64 flags)
650{
651 u32 i, key, num_elems = 0;
652 struct bpf_array *array;
653 bool is_percpu;
654 u64 ret = 0;
655 void *val;
656
657 if (flags != 0)
658 return -EINVAL;
659
660 is_percpu = map->map_type == BPF_MAP_TYPE_PERCPU_ARRAY;
661 array = container_of(map, struct bpf_array, map);
662 if (is_percpu)
663 migrate_disable();
664 for (i = 0; i < map->max_entries; i++) {
665 if (is_percpu)
666 val = this_cpu_ptr(array->pptrs[i]);
667 else
668 val = array->value + array->elem_size * i;
669 num_elems++;
670 key = i;
671 ret = callback_fn((u64)(long)map, (u64)(long)&key,
672 (u64)(long)val, (u64)(long)callback_ctx, 0);
673 /* return value: 0 - continue, 1 - stop and return */
674 if (ret)
675 break;
676 }
677
678 if (is_percpu)
679 migrate_enable();
680 return num_elems;
681}
682
683static int array_map_btf_id;
684const struct bpf_map_ops array_map_ops = {
685 .map_meta_equal = array_map_meta_equal,
686 .map_alloc_check = array_map_alloc_check,
687 .map_alloc = array_map_alloc,
688 .map_free = array_map_free,
689 .map_get_next_key = array_map_get_next_key,
690 .map_release_uref = array_map_free_timers,
691 .map_lookup_elem = array_map_lookup_elem,
692 .map_update_elem = array_map_update_elem,
693 .map_delete_elem = array_map_delete_elem,
694 .map_gen_lookup = array_map_gen_lookup,
695 .map_direct_value_addr = array_map_direct_value_addr,
696 .map_direct_value_meta = array_map_direct_value_meta,
697 .map_mmap = array_map_mmap,
698 .map_seq_show_elem = array_map_seq_show_elem,
699 .map_check_btf = array_map_check_btf,
700 .map_lookup_batch = generic_map_lookup_batch,
701 .map_update_batch = generic_map_update_batch,
702 .map_set_for_each_callback_args = map_set_for_each_callback_args,
703 .map_for_each_callback = bpf_for_each_array_elem,
704 .map_btf_name = "bpf_array",
705 .map_btf_id = &array_map_btf_id,
706 .iter_seq_info = &iter_seq_info,
707};
708
709static int percpu_array_map_btf_id;
710const struct bpf_map_ops percpu_array_map_ops = {
711 .map_meta_equal = bpf_map_meta_equal,
712 .map_alloc_check = array_map_alloc_check,
713 .map_alloc = array_map_alloc,
714 .map_free = array_map_free,
715 .map_get_next_key = array_map_get_next_key,
716 .map_lookup_elem = percpu_array_map_lookup_elem,
717 .map_update_elem = array_map_update_elem,
718 .map_delete_elem = array_map_delete_elem,
719 .map_seq_show_elem = percpu_array_map_seq_show_elem,
720 .map_check_btf = array_map_check_btf,
721 .map_lookup_batch = generic_map_lookup_batch,
722 .map_update_batch = generic_map_update_batch,
723 .map_set_for_each_callback_args = map_set_for_each_callback_args,
724 .map_for_each_callback = bpf_for_each_array_elem,
725 .map_btf_name = "bpf_array",
726 .map_btf_id = &percpu_array_map_btf_id,
727 .iter_seq_info = &iter_seq_info,
728};
729
730static int fd_array_map_alloc_check(union bpf_attr *attr)
731{
732 /* only file descriptors can be stored in this type of map */
733 if (attr->value_size != sizeof(u32))
734 return -EINVAL;
735 /* Program read-only/write-only not supported for special maps yet. */
736 if (attr->map_flags & (BPF_F_RDONLY_PROG | BPF_F_WRONLY_PROG))
737 return -EINVAL;
738 return array_map_alloc_check(attr);
739}
740
741static void fd_array_map_free(struct bpf_map *map)
742{
743 struct bpf_array *array = container_of(map, struct bpf_array, map);
744 int i;
745
746 /* make sure it's empty */
747 for (i = 0; i < array->map.max_entries; i++)
748 BUG_ON(array->ptrs[i] != NULL);
749
750 bpf_map_area_free(array);
751}
752
753static void *fd_array_map_lookup_elem(struct bpf_map *map, void *key)
754{
755 return ERR_PTR(-EOPNOTSUPP);
756}
757
758/* only called from syscall */
759int bpf_fd_array_map_lookup_elem(struct bpf_map *map, void *key, u32 *value)
760{
761 void **elem, *ptr;
762 int ret = 0;
763
764 if (!map->ops->map_fd_sys_lookup_elem)
765 return -ENOTSUPP;
766
767 rcu_read_lock();
768 elem = array_map_lookup_elem(map, key);
769 if (elem && (ptr = READ_ONCE(*elem)))
770 *value = map->ops->map_fd_sys_lookup_elem(ptr);
771 else
772 ret = -ENOENT;
773 rcu_read_unlock();
774
775 return ret;
776}
777
778/* only called from syscall */
779int bpf_fd_array_map_update_elem(struct bpf_map *map, struct file *map_file,
780 void *key, void *value, u64 map_flags)
781{
782 struct bpf_array *array = container_of(map, struct bpf_array, map);
783 void *new_ptr, *old_ptr;
784 u32 index = *(u32 *)key, ufd;
785
786 if (map_flags != BPF_ANY)
787 return -EINVAL;
788
789 if (index >= array->map.max_entries)
790 return -E2BIG;
791
792 ufd = *(u32 *)value;
793 new_ptr = map->ops->map_fd_get_ptr(map, map_file, ufd);
794 if (IS_ERR(new_ptr))
795 return PTR_ERR(new_ptr);
796
797 if (map->ops->map_poke_run) {
798 mutex_lock(&array->aux->poke_mutex);
799 old_ptr = xchg(array->ptrs + index, new_ptr);
800 map->ops->map_poke_run(map, index, old_ptr, new_ptr);
801 mutex_unlock(&array->aux->poke_mutex);
802 } else {
803 old_ptr = xchg(array->ptrs + index, new_ptr);
804 }
805
806 if (old_ptr)
807 map->ops->map_fd_put_ptr(old_ptr);
808 return 0;
809}
810
811static int fd_array_map_delete_elem(struct bpf_map *map, void *key)
812{
813 struct bpf_array *array = container_of(map, struct bpf_array, map);
814 void *old_ptr;
815 u32 index = *(u32 *)key;
816
817 if (index >= array->map.max_entries)
818 return -E2BIG;
819
820 if (map->ops->map_poke_run) {
821 mutex_lock(&array->aux->poke_mutex);
822 old_ptr = xchg(array->ptrs + index, NULL);
823 map->ops->map_poke_run(map, index, old_ptr, NULL);
824 mutex_unlock(&array->aux->poke_mutex);
825 } else {
826 old_ptr = xchg(array->ptrs + index, NULL);
827 }
828
829 if (old_ptr) {
830 map->ops->map_fd_put_ptr(old_ptr);
831 return 0;
832 } else {
833 return -ENOENT;
834 }
835}
836
837static void *prog_fd_array_get_ptr(struct bpf_map *map,
838 struct file *map_file, int fd)
839{
840 struct bpf_prog *prog = bpf_prog_get(fd);
841
842 if (IS_ERR(prog))
843 return prog;
844
845 if (!bpf_prog_map_compatible(map, prog)) {
846 bpf_prog_put(prog);
847 return ERR_PTR(-EINVAL);
848 }
849
850 return prog;
851}
852
853static void prog_fd_array_put_ptr(void *ptr)
854{
855 bpf_prog_put(ptr);
856}
857
858static u32 prog_fd_array_sys_lookup_elem(void *ptr)
859{
860 return ((struct bpf_prog *)ptr)->aux->id;
861}
862
863/* decrement refcnt of all bpf_progs that are stored in this map */
864static void bpf_fd_array_map_clear(struct bpf_map *map)
865{
866 struct bpf_array *array = container_of(map, struct bpf_array, map);
867 int i;
868
869 for (i = 0; i < array->map.max_entries; i++)
870 fd_array_map_delete_elem(map, &i);
871}
872
873static void prog_array_map_seq_show_elem(struct bpf_map *map, void *key,
874 struct seq_file *m)
875{
876 void **elem, *ptr;
877 u32 prog_id;
878
879 rcu_read_lock();
880
881 elem = array_map_lookup_elem(map, key);
882 if (elem) {
883 ptr = READ_ONCE(*elem);
884 if (ptr) {
885 seq_printf(m, "%u: ", *(u32 *)key);
886 prog_id = prog_fd_array_sys_lookup_elem(ptr);
887 btf_type_seq_show(map->btf, map->btf_value_type_id,
888 &prog_id, m);
889 seq_puts(m, "\n");
890 }
891 }
892
893 rcu_read_unlock();
894}
895
896struct prog_poke_elem {
897 struct list_head list;
898 struct bpf_prog_aux *aux;
899};
900
901static int prog_array_map_poke_track(struct bpf_map *map,
902 struct bpf_prog_aux *prog_aux)
903{
904 struct prog_poke_elem *elem;
905 struct bpf_array_aux *aux;
906 int ret = 0;
907
908 aux = container_of(map, struct bpf_array, map)->aux;
909 mutex_lock(&aux->poke_mutex);
910 list_for_each_entry(elem, &aux->poke_progs, list) {
911 if (elem->aux == prog_aux)
912 goto out;
913 }
914
915 elem = kmalloc(sizeof(*elem), GFP_KERNEL);
916 if (!elem) {
917 ret = -ENOMEM;
918 goto out;
919 }
920
921 INIT_LIST_HEAD(&elem->list);
922 /* We must track the program's aux info at this point in time
923 * since the program pointer itself may not be stable yet, see
924 * also comment in prog_array_map_poke_run().
925 */
926 elem->aux = prog_aux;
927
928 list_add_tail(&elem->list, &aux->poke_progs);
929out:
930 mutex_unlock(&aux->poke_mutex);
931 return ret;
932}
933
934static void prog_array_map_poke_untrack(struct bpf_map *map,
935 struct bpf_prog_aux *prog_aux)
936{
937 struct prog_poke_elem *elem, *tmp;
938 struct bpf_array_aux *aux;
939
940 aux = container_of(map, struct bpf_array, map)->aux;
941 mutex_lock(&aux->poke_mutex);
942 list_for_each_entry_safe(elem, tmp, &aux->poke_progs, list) {
943 if (elem->aux == prog_aux) {
944 list_del_init(&elem->list);
945 kfree(elem);
946 break;
947 }
948 }
949 mutex_unlock(&aux->poke_mutex);
950}
951
952static void prog_array_map_poke_run(struct bpf_map *map, u32 key,
953 struct bpf_prog *old,
954 struct bpf_prog *new)
955{
956 u8 *old_addr, *new_addr, *old_bypass_addr;
957 struct prog_poke_elem *elem;
958 struct bpf_array_aux *aux;
959
960 aux = container_of(map, struct bpf_array, map)->aux;
961 WARN_ON_ONCE(!mutex_is_locked(&aux->poke_mutex));
962
963 list_for_each_entry(elem, &aux->poke_progs, list) {
964 struct bpf_jit_poke_descriptor *poke;
965 int i, ret;
966
967 for (i = 0; i < elem->aux->size_poke_tab; i++) {
968 poke = &elem->aux->poke_tab[i];
969
970 /* Few things to be aware of:
971 *
972 * 1) We can only ever access aux in this context, but
973 * not aux->prog since it might not be stable yet and
974 * there could be danger of use after free otherwise.
975 * 2) Initially when we start tracking aux, the program
976 * is not JITed yet and also does not have a kallsyms
977 * entry. We skip these as poke->tailcall_target_stable
978 * is not active yet. The JIT will do the final fixup
979 * before setting it stable. The various
980 * poke->tailcall_target_stable are successively
981 * activated, so tail call updates can arrive from here
982 * while JIT is still finishing its final fixup for
983 * non-activated poke entries.
984 * 3) On program teardown, the program's kallsym entry gets
985 * removed out of RCU callback, but we can only untrack
986 * from sleepable context, therefore bpf_arch_text_poke()
987 * might not see that this is in BPF text section and
988 * bails out with -EINVAL. As these are unreachable since
989 * RCU grace period already passed, we simply skip them.
990 * 4) Also programs reaching refcount of zero while patching
991 * is in progress is okay since we're protected under
992 * poke_mutex and untrack the programs before the JIT
993 * buffer is freed. When we're still in the middle of
994 * patching and suddenly kallsyms entry of the program
995 * gets evicted, we just skip the rest which is fine due
996 * to point 3).
997 * 5) Any other error happening below from bpf_arch_text_poke()
998 * is a unexpected bug.
999 */
1000 if (!READ_ONCE(poke->tailcall_target_stable))
1001 continue;
1002 if (poke->reason != BPF_POKE_REASON_TAIL_CALL)
1003 continue;
1004 if (poke->tail_call.map != map ||
1005 poke->tail_call.key != key)
1006 continue;
1007
1008 old_bypass_addr = old ? NULL : poke->bypass_addr;
1009 old_addr = old ? (u8 *)old->bpf_func + poke->adj_off : NULL;
1010 new_addr = new ? (u8 *)new->bpf_func + poke->adj_off : NULL;
1011
1012 if (new) {
1013 ret = bpf_arch_text_poke(poke->tailcall_target,
1014 BPF_MOD_JUMP,
1015 old_addr, new_addr);
1016 BUG_ON(ret < 0 && ret != -EINVAL);
1017 if (!old) {
1018 ret = bpf_arch_text_poke(poke->tailcall_bypass,
1019 BPF_MOD_JUMP,
1020 poke->bypass_addr,
1021 NULL);
1022 BUG_ON(ret < 0 && ret != -EINVAL);
1023 }
1024 } else {
1025 ret = bpf_arch_text_poke(poke->tailcall_bypass,
1026 BPF_MOD_JUMP,
1027 old_bypass_addr,
1028 poke->bypass_addr);
1029 BUG_ON(ret < 0 && ret != -EINVAL);
1030 /* let other CPUs finish the execution of program
1031 * so that it will not possible to expose them
1032 * to invalid nop, stack unwind, nop state
1033 */
1034 if (!ret)
1035 synchronize_rcu();
1036 ret = bpf_arch_text_poke(poke->tailcall_target,
1037 BPF_MOD_JUMP,
1038 old_addr, NULL);
1039 BUG_ON(ret < 0 && ret != -EINVAL);
1040 }
1041 }
1042 }
1043}
1044
1045static void prog_array_map_clear_deferred(struct work_struct *work)
1046{
1047 struct bpf_map *map = container_of(work, struct bpf_array_aux,
1048 work)->map;
1049 bpf_fd_array_map_clear(map);
1050 bpf_map_put(map);
1051}
1052
1053static void prog_array_map_clear(struct bpf_map *map)
1054{
1055 struct bpf_array_aux *aux = container_of(map, struct bpf_array,
1056 map)->aux;
1057 bpf_map_inc(map);
1058 schedule_work(&aux->work);
1059}
1060
1061static struct bpf_map *prog_array_map_alloc(union bpf_attr *attr)
1062{
1063 struct bpf_array_aux *aux;
1064 struct bpf_map *map;
1065
1066 aux = kzalloc(sizeof(*aux), GFP_KERNEL_ACCOUNT);
1067 if (!aux)
1068 return ERR_PTR(-ENOMEM);
1069
1070 INIT_WORK(&aux->work, prog_array_map_clear_deferred);
1071 INIT_LIST_HEAD(&aux->poke_progs);
1072 mutex_init(&aux->poke_mutex);
1073
1074 map = array_map_alloc(attr);
1075 if (IS_ERR(map)) {
1076 kfree(aux);
1077 return map;
1078 }
1079
1080 container_of(map, struct bpf_array, map)->aux = aux;
1081 aux->map = map;
1082
1083 return map;
1084}
1085
1086static void prog_array_map_free(struct bpf_map *map)
1087{
1088 struct prog_poke_elem *elem, *tmp;
1089 struct bpf_array_aux *aux;
1090
1091 aux = container_of(map, struct bpf_array, map)->aux;
1092 list_for_each_entry_safe(elem, tmp, &aux->poke_progs, list) {
1093 list_del_init(&elem->list);
1094 kfree(elem);
1095 }
1096 kfree(aux);
1097 fd_array_map_free(map);
1098}
1099
1100/* prog_array->aux->{type,jited} is a runtime binding.
1101 * Doing static check alone in the verifier is not enough.
1102 * Thus, prog_array_map cannot be used as an inner_map
1103 * and map_meta_equal is not implemented.
1104 */
1105static int prog_array_map_btf_id;
1106const struct bpf_map_ops prog_array_map_ops = {
1107 .map_alloc_check = fd_array_map_alloc_check,
1108 .map_alloc = prog_array_map_alloc,
1109 .map_free = prog_array_map_free,
1110 .map_poke_track = prog_array_map_poke_track,
1111 .map_poke_untrack = prog_array_map_poke_untrack,
1112 .map_poke_run = prog_array_map_poke_run,
1113 .map_get_next_key = array_map_get_next_key,
1114 .map_lookup_elem = fd_array_map_lookup_elem,
1115 .map_delete_elem = fd_array_map_delete_elem,
1116 .map_fd_get_ptr = prog_fd_array_get_ptr,
1117 .map_fd_put_ptr = prog_fd_array_put_ptr,
1118 .map_fd_sys_lookup_elem = prog_fd_array_sys_lookup_elem,
1119 .map_release_uref = prog_array_map_clear,
1120 .map_seq_show_elem = prog_array_map_seq_show_elem,
1121 .map_btf_name = "bpf_array",
1122 .map_btf_id = &prog_array_map_btf_id,
1123};
1124
1125static struct bpf_event_entry *bpf_event_entry_gen(struct file *perf_file,
1126 struct file *map_file)
1127{
1128 struct bpf_event_entry *ee;
1129
1130 ee = kzalloc(sizeof(*ee), GFP_ATOMIC);
1131 if (ee) {
1132 ee->event = perf_file->private_data;
1133 ee->perf_file = perf_file;
1134 ee->map_file = map_file;
1135 }
1136
1137 return ee;
1138}
1139
1140static void __bpf_event_entry_free(struct rcu_head *rcu)
1141{
1142 struct bpf_event_entry *ee;
1143
1144 ee = container_of(rcu, struct bpf_event_entry, rcu);
1145 fput(ee->perf_file);
1146 kfree(ee);
1147}
1148
1149static void bpf_event_entry_free_rcu(struct bpf_event_entry *ee)
1150{
1151 call_rcu(&ee->rcu, __bpf_event_entry_free);
1152}
1153
1154static void *perf_event_fd_array_get_ptr(struct bpf_map *map,
1155 struct file *map_file, int fd)
1156{
1157 struct bpf_event_entry *ee;
1158 struct perf_event *event;
1159 struct file *perf_file;
1160 u64 value;
1161
1162 perf_file = perf_event_get(fd);
1163 if (IS_ERR(perf_file))
1164 return perf_file;
1165
1166 ee = ERR_PTR(-EOPNOTSUPP);
1167 event = perf_file->private_data;
1168 if (perf_event_read_local(event, &value, NULL, NULL) == -EOPNOTSUPP)
1169 goto err_out;
1170
1171 ee = bpf_event_entry_gen(perf_file, map_file);
1172 if (ee)
1173 return ee;
1174 ee = ERR_PTR(-ENOMEM);
1175err_out:
1176 fput(perf_file);
1177 return ee;
1178}
1179
1180static void perf_event_fd_array_put_ptr(void *ptr)
1181{
1182 bpf_event_entry_free_rcu(ptr);
1183}
1184
1185static void perf_event_fd_array_release(struct bpf_map *map,
1186 struct file *map_file)
1187{
1188 struct bpf_array *array = container_of(map, struct bpf_array, map);
1189 struct bpf_event_entry *ee;
1190 int i;
1191
1192 if (map->map_flags & BPF_F_PRESERVE_ELEMS)
1193 return;
1194
1195 rcu_read_lock();
1196 for (i = 0; i < array->map.max_entries; i++) {
1197 ee = READ_ONCE(array->ptrs[i]);
1198 if (ee && ee->map_file == map_file)
1199 fd_array_map_delete_elem(map, &i);
1200 }
1201 rcu_read_unlock();
1202}
1203
1204static void perf_event_fd_array_map_free(struct bpf_map *map)
1205{
1206 if (map->map_flags & BPF_F_PRESERVE_ELEMS)
1207 bpf_fd_array_map_clear(map);
1208 fd_array_map_free(map);
1209}
1210
1211static int perf_event_array_map_btf_id;
1212const struct bpf_map_ops perf_event_array_map_ops = {
1213 .map_meta_equal = bpf_map_meta_equal,
1214 .map_alloc_check = fd_array_map_alloc_check,
1215 .map_alloc = array_map_alloc,
1216 .map_free = perf_event_fd_array_map_free,
1217 .map_get_next_key = array_map_get_next_key,
1218 .map_lookup_elem = fd_array_map_lookup_elem,
1219 .map_delete_elem = fd_array_map_delete_elem,
1220 .map_fd_get_ptr = perf_event_fd_array_get_ptr,
1221 .map_fd_put_ptr = perf_event_fd_array_put_ptr,
1222 .map_release = perf_event_fd_array_release,
1223 .map_check_btf = map_check_no_btf,
1224 .map_btf_name = "bpf_array",
1225 .map_btf_id = &perf_event_array_map_btf_id,
1226};
1227
1228#ifdef CONFIG_CGROUPS
1229static void *cgroup_fd_array_get_ptr(struct bpf_map *map,
1230 struct file *map_file /* not used */,
1231 int fd)
1232{
1233 return cgroup_get_from_fd(fd);
1234}
1235
1236static void cgroup_fd_array_put_ptr(void *ptr)
1237{
1238 /* cgroup_put free cgrp after a rcu grace period */
1239 cgroup_put(ptr);
1240}
1241
1242static void cgroup_fd_array_free(struct bpf_map *map)
1243{
1244 bpf_fd_array_map_clear(map);
1245 fd_array_map_free(map);
1246}
1247
1248static int cgroup_array_map_btf_id;
1249const struct bpf_map_ops cgroup_array_map_ops = {
1250 .map_meta_equal = bpf_map_meta_equal,
1251 .map_alloc_check = fd_array_map_alloc_check,
1252 .map_alloc = array_map_alloc,
1253 .map_free = cgroup_fd_array_free,
1254 .map_get_next_key = array_map_get_next_key,
1255 .map_lookup_elem = fd_array_map_lookup_elem,
1256 .map_delete_elem = fd_array_map_delete_elem,
1257 .map_fd_get_ptr = cgroup_fd_array_get_ptr,
1258 .map_fd_put_ptr = cgroup_fd_array_put_ptr,
1259 .map_check_btf = map_check_no_btf,
1260 .map_btf_name = "bpf_array",
1261 .map_btf_id = &cgroup_array_map_btf_id,
1262};
1263#endif
1264
1265static struct bpf_map *array_of_map_alloc(union bpf_attr *attr)
1266{
1267 struct bpf_map *map, *inner_map_meta;
1268
1269 inner_map_meta = bpf_map_meta_alloc(attr->inner_map_fd);
1270 if (IS_ERR(inner_map_meta))
1271 return inner_map_meta;
1272
1273 map = array_map_alloc(attr);
1274 if (IS_ERR(map)) {
1275 bpf_map_meta_free(inner_map_meta);
1276 return map;
1277 }
1278
1279 map->inner_map_meta = inner_map_meta;
1280
1281 return map;
1282}
1283
1284static void array_of_map_free(struct bpf_map *map)
1285{
1286 /* map->inner_map_meta is only accessed by syscall which
1287 * is protected by fdget/fdput.
1288 */
1289 bpf_map_meta_free(map->inner_map_meta);
1290 bpf_fd_array_map_clear(map);
1291 fd_array_map_free(map);
1292}
1293
1294static void *array_of_map_lookup_elem(struct bpf_map *map, void *key)
1295{
1296 struct bpf_map **inner_map = array_map_lookup_elem(map, key);
1297
1298 if (!inner_map)
1299 return NULL;
1300
1301 return READ_ONCE(*inner_map);
1302}
1303
1304static int array_of_map_gen_lookup(struct bpf_map *map,
1305 struct bpf_insn *insn_buf)
1306{
1307 struct bpf_array *array = container_of(map, struct bpf_array, map);
1308 u32 elem_size = round_up(map->value_size, 8);
1309 struct bpf_insn *insn = insn_buf;
1310 const int ret = BPF_REG_0;
1311 const int map_ptr = BPF_REG_1;
1312 const int index = BPF_REG_2;
1313
1314 *insn++ = BPF_ALU64_IMM(BPF_ADD, map_ptr, offsetof(struct bpf_array, value));
1315 *insn++ = BPF_LDX_MEM(BPF_W, ret, index, 0);
1316 if (!map->bypass_spec_v1) {
1317 *insn++ = BPF_JMP_IMM(BPF_JGE, ret, map->max_entries, 6);
1318 *insn++ = BPF_ALU32_IMM(BPF_AND, ret, array->index_mask);
1319 } else {
1320 *insn++ = BPF_JMP_IMM(BPF_JGE, ret, map->max_entries, 5);
1321 }
1322 if (is_power_of_2(elem_size))
1323 *insn++ = BPF_ALU64_IMM(BPF_LSH, ret, ilog2(elem_size));
1324 else
1325 *insn++ = BPF_ALU64_IMM(BPF_MUL, ret, elem_size);
1326 *insn++ = BPF_ALU64_REG(BPF_ADD, ret, map_ptr);
1327 *insn++ = BPF_LDX_MEM(BPF_DW, ret, ret, 0);
1328 *insn++ = BPF_JMP_IMM(BPF_JEQ, ret, 0, 1);
1329 *insn++ = BPF_JMP_IMM(BPF_JA, 0, 0, 1);
1330 *insn++ = BPF_MOV64_IMM(ret, 0);
1331
1332 return insn - insn_buf;
1333}
1334
1335static int array_of_maps_map_btf_id;
1336const struct bpf_map_ops array_of_maps_map_ops = {
1337 .map_alloc_check = fd_array_map_alloc_check,
1338 .map_alloc = array_of_map_alloc,
1339 .map_free = array_of_map_free,
1340 .map_get_next_key = array_map_get_next_key,
1341 .map_lookup_elem = array_of_map_lookup_elem,
1342 .map_delete_elem = fd_array_map_delete_elem,
1343 .map_fd_get_ptr = bpf_map_fd_get_ptr,
1344 .map_fd_put_ptr = bpf_map_fd_put_ptr,
1345 .map_fd_sys_lookup_elem = bpf_map_fd_sys_lookup_elem,
1346 .map_gen_lookup = array_of_map_gen_lookup,
1347 .map_check_btf = map_check_no_btf,
1348 .map_btf_name = "bpf_array",
1349 .map_btf_id = &array_of_maps_map_btf_id,
1350};