projects/linux·blob master · refs

1484 lines · 41.9 KiB

1// SPDX-License-Identifier: (LGPL-2.1 OR BSD-2-Clause)
2
3/*
4 * common eBPF ELF operations.
5 *
6 * Copyright (C) 2013-2015 Alexei Starovoitov <[email protected]>
7 * Copyright (C) 2015 Wang Nan <[email protected]>
8 * Copyright (C) 2015 Huawei Inc.
9 *
10 * This program is free software; you can redistribute it and/or
11 * modify it under the terms of the GNU Lesser General Public
12 * License as published by the Free Software Foundation;
13 * version 2.1 of the License (not later!)
14 *
15 * This program is distributed in the hope that it will be useful,
16 * but WITHOUT ANY WARRANTY; without even the implied warranty of
17 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
18 * GNU Lesser General Public License for more details.
19 *
20 * You should have received a copy of the GNU Lesser General Public
21 * License along with this program; if not, see <http://www.gnu.org/licenses>
22 */
23
24#include <stdlib.h>
25#include <string.h>
26#include <memory.h>
27#include <unistd.h>
28#include <asm/unistd.h>
29#include <errno.h>
30#include <linux/bpf.h>
31#include <linux/filter.h>
32#include <linux/kernel.h>
33#include <limits.h>
34#include <sys/resource.h>
35#include "bpf.h"
36#include "libbpf.h"
37#include "libbpf_internal.h"
38
39/*
40 * When building perf, unistd.h is overridden. __NR_bpf is
41 * required to be defined explicitly.
42 */
43#ifndef __NR_bpf
44# if defined(__i386__)
45# define __NR_bpf 357
46# elif defined(__x86_64__)
47# define __NR_bpf 321
48# elif defined(__aarch64__)
49# define __NR_bpf 280
50# elif defined(__sparc__)
51# define __NR_bpf 349
52# elif defined(__s390__)
53# define __NR_bpf 351
54# elif defined(__arc__)
55# define __NR_bpf 280
56# elif defined(__mips__) && defined(_ABIO32)
57# define __NR_bpf 4355
58# elif defined(__mips__) && defined(_ABIN32)
59# define __NR_bpf 6319
60# elif defined(__mips__) && defined(_ABI64)
61# define __NR_bpf 5315
62# elif defined(__loongarch__)
63# define __NR_bpf 280
64# else
65# error __NR_bpf not defined. libbpf does not support your arch.
66# endif
67#endif
68
69static inline __u64 ptr_to_u64(const void *ptr)
70{
71 return (__u64) (unsigned long) ptr;
72}
73
74static inline int sys_bpf_ext(enum bpf_cmd cmd, union bpf_attr *attr,
75 unsigned int size,
76 struct bpf_common_attr *attr_common,
77 unsigned int size_common)
78{
79 cmd = attr_common ? (cmd | BPF_COMMON_ATTRS) : (cmd & ~BPF_COMMON_ATTRS);
80 return syscall(__NR_bpf, cmd, attr, size, attr_common, size_common);
81}
82
83static inline int sys_bpf_ext_fd(enum bpf_cmd cmd, union bpf_attr *attr,
84 unsigned int size,
85 struct bpf_common_attr *attr_common,
86 unsigned int size_common)
87{
88 int fd;
89
90 fd = sys_bpf_ext(cmd, attr, size, attr_common, size_common);
91 return ensure_good_fd(fd);
92}
93
94int probe_sys_bpf_ext(void)
95{
96 const size_t attr_sz = offsetofend(union bpf_attr, prog_token_fd);
97 union bpf_attr attr;
98 int fd;
99
100 memset(&attr, 0, attr_sz);
101 fd = syscall(__NR_bpf, BPF_PROG_LOAD | BPF_COMMON_ATTRS, &attr, attr_sz, NULL,
102 sizeof(struct bpf_common_attr));
103 if (fd >= 0) {
104 close(fd);
105 return -EINVAL;
106 }
107 return errno == EFAULT ? 1 : 0;
108}
109
110static inline int sys_bpf(enum bpf_cmd cmd, union bpf_attr *attr,
111 unsigned int size)
112{
113 return syscall(__NR_bpf, cmd, attr, size);
114}
115
116static inline int sys_bpf_fd(enum bpf_cmd cmd, union bpf_attr *attr,
117 unsigned int size)
118{
119 int fd;
120
121 fd = sys_bpf(cmd, attr, size);
122 return ensure_good_fd(fd);
123}
124
125int sys_bpf_prog_load(union bpf_attr *attr, unsigned int size, int attempts)
126{
127 int fd;
128
129 do {
130 fd = sys_bpf_fd(BPF_PROG_LOAD, attr, size);
131 } while (fd < 0 && errno == EAGAIN && --attempts > 0);
132
133 return fd;
134}
135
136/* Probe whether kernel switched from memlock-based (RLIMIT_MEMLOCK) to
137 * memcg-based memory accounting for BPF maps and progs. This was done in [0].
138 * We use the support for bpf_ktime_get_coarse_ns() helper, which was added in
139 * the same 5.11 Linux release ([1]), to detect memcg-based accounting for BPF.
140 *
141 * [0] https://lore.kernel.org/bpf/[email protected]/
142 * [1] d05512618056 ("bpf: Add bpf_ktime_get_coarse_ns helper")
143 */
144int probe_memcg_account(int token_fd)
145{
146 const size_t attr_sz = offsetofend(union bpf_attr, prog_token_fd);
147 struct bpf_insn insns[] = {
148 BPF_EMIT_CALL(BPF_FUNC_ktime_get_coarse_ns),
149 BPF_EXIT_INSN(),
150 };
151 size_t insn_cnt = ARRAY_SIZE(insns);
152 union bpf_attr attr;
153 int prog_fd;
154
155 /* attempt loading freplace trying to use custom BTF */
156 memset(&attr, 0, attr_sz);
157 attr.prog_type = BPF_PROG_TYPE_SOCKET_FILTER;
158 attr.insns = ptr_to_u64(insns);
159 attr.insn_cnt = insn_cnt;
160 attr.license = ptr_to_u64("GPL");
161 attr.prog_token_fd = token_fd;
162 if (token_fd)
163 attr.prog_flags |= BPF_F_TOKEN_FD;
164
165 prog_fd = sys_bpf_fd(BPF_PROG_LOAD, &attr, attr_sz);
166 if (prog_fd >= 0) {
167 close(prog_fd);
168 return 1;
169 }
170 return 0;
171}
172
173static bool memlock_bumped;
174static rlim_t memlock_rlim = RLIM_INFINITY;
175
176int libbpf_set_memlock_rlim(size_t memlock_bytes)
177{
178 if (memlock_bumped)
179 return libbpf_err(-EBUSY);
180
181 memlock_rlim = memlock_bytes;
182 return 0;
183}
184
185int bump_rlimit_memlock(void)
186{
187 struct rlimit rlim;
188
189 /* if kernel supports memcg-based accounting, skip bumping RLIMIT_MEMLOCK */
190 if (memlock_bumped || feat_supported(NULL, FEAT_MEMCG_ACCOUNT))
191 return 0;
192
193 memlock_bumped = true;
194
195 /* zero memlock_rlim disables auto-bumping RLIMIT_MEMLOCK */
196 if (memlock_rlim == 0)
197 return 0;
198
199 rlim.rlim_cur = rlim.rlim_max = memlock_rlim;
200 if (setrlimit(RLIMIT_MEMLOCK, &rlim))
201 return -errno;
202
203 return 0;
204}
205
206int bpf_map_create(enum bpf_map_type map_type,
207 const char *map_name,
208 __u32 key_size,
209 __u32 value_size,
210 __u32 max_entries,
211 const struct bpf_map_create_opts *opts)
212{
213 const size_t attr_sz = offsetofend(union bpf_attr, excl_prog_hash_size);
214 const size_t attr_common_sz = sizeof(struct bpf_common_attr);
215 struct bpf_common_attr attr_common;
216 struct bpf_log_opts *log_opts;
217 union bpf_attr attr;
218 int fd;
219
220 bump_rlimit_memlock();
221
222 memset(&attr, 0, attr_sz);
223
224 if (!OPTS_VALID(opts, bpf_map_create_opts))
225 return libbpf_err(-EINVAL);
226
227 attr.map_type = map_type;
228 if (map_name && feat_supported(NULL, FEAT_PROG_NAME))
229 libbpf_strlcpy(attr.map_name, map_name, sizeof(attr.map_name));
230 attr.key_size = key_size;
231 attr.value_size = value_size;
232 attr.max_entries = max_entries;
233
234 attr.btf_fd = OPTS_GET(opts, btf_fd, 0);
235 attr.btf_key_type_id = OPTS_GET(opts, btf_key_type_id, 0);
236 attr.btf_value_type_id = OPTS_GET(opts, btf_value_type_id, 0);
237 attr.btf_vmlinux_value_type_id = OPTS_GET(opts, btf_vmlinux_value_type_id, 0);
238 attr.value_type_btf_obj_fd = OPTS_GET(opts, value_type_btf_obj_fd, 0);
239
240 attr.inner_map_fd = OPTS_GET(opts, inner_map_fd, 0);
241 attr.map_flags = OPTS_GET(opts, map_flags, 0);
242 attr.map_extra = OPTS_GET(opts, map_extra, 0);
243 attr.numa_node = OPTS_GET(opts, numa_node, 0);
244 attr.map_ifindex = OPTS_GET(opts, map_ifindex, 0);
245
246 attr.map_token_fd = OPTS_GET(opts, token_fd, 0);
247 attr.excl_prog_hash = ptr_to_u64(OPTS_GET(opts, excl_prog_hash, NULL));
248 attr.excl_prog_hash_size = OPTS_GET(opts, excl_prog_hash_size, 0);
249
250 log_opts = OPTS_GET(opts, log_opts, NULL);
251 if (!OPTS_VALID(log_opts, bpf_log_opts))
252 return libbpf_err(-EINVAL);
253
254 if (log_opts && feat_supported(NULL, FEAT_BPF_SYSCALL_COMMON_ATTRS)) {
255 memset(&attr_common, 0, attr_common_sz);
256 attr_common.log_buf = ptr_to_u64(OPTS_GET(log_opts, buf, NULL));
257 attr_common.log_size = OPTS_GET(log_opts, size, 0);
258 attr_common.log_level = OPTS_GET(log_opts, level, 0);
259 fd = sys_bpf_ext_fd(BPF_MAP_CREATE, &attr, attr_sz, &attr_common, attr_common_sz);
260 OPTS_SET(log_opts, true_size, attr_common.log_true_size);
261 } else {
262 fd = sys_bpf_fd(BPF_MAP_CREATE, &attr, attr_sz);
263 OPTS_SET(log_opts, true_size, 0);
264 }
265 return libbpf_err_errno(fd);
266}
267
268static void *
269alloc_zero_tailing_info(const void *orecord, __u32 cnt,
270 __u32 actual_rec_size, __u32 expected_rec_size)
271{
272 __u64 info_len = (__u64)actual_rec_size * cnt;
273 void *info, *nrecord;
274 int i;
275
276 info = malloc(info_len);
277 if (!info)
278 return NULL;
279
280 /* zero out bytes kernel does not understand */
281 nrecord = info;
282 for (i = 0; i < cnt; i++) {
283 memcpy(nrecord, orecord, expected_rec_size);
284 memset(nrecord + expected_rec_size, 0,
285 actual_rec_size - expected_rec_size);
286 orecord += actual_rec_size;
287 nrecord += actual_rec_size;
288 }
289
290 return info;
291}
292
293int bpf_prog_load(enum bpf_prog_type prog_type,
294 const char *prog_name, const char *license,
295 const struct bpf_insn *insns, size_t insn_cnt,
296 struct bpf_prog_load_opts *opts)
297{
298 const size_t attr_sz = offsetofend(union bpf_attr, keyring_id);
299 void *finfo = NULL, *linfo = NULL;
300 const char *func_info, *line_info;
301 __u32 log_size, log_level, attach_prog_fd, attach_btf_obj_fd;
302 __u32 func_info_rec_size, line_info_rec_size;
303 int fd, attempts;
304 union bpf_attr attr;
305 char *log_buf;
306
307 bump_rlimit_memlock();
308
309 if (!OPTS_VALID(opts, bpf_prog_load_opts))
310 return libbpf_err(-EINVAL);
311
312 attempts = OPTS_GET(opts, attempts, 0);
313 if (attempts < 0)
314 return libbpf_err(-EINVAL);
315 if (attempts == 0)
316 attempts = PROG_LOAD_ATTEMPTS;
317
318 memset(&attr, 0, attr_sz);
319
320 attr.prog_type = prog_type;
321 attr.expected_attach_type = OPTS_GET(opts, expected_attach_type, 0);
322
323 attr.prog_btf_fd = OPTS_GET(opts, prog_btf_fd, 0);
324 attr.prog_flags = OPTS_GET(opts, prog_flags, 0);
325 attr.prog_ifindex = OPTS_GET(opts, prog_ifindex, 0);
326 attr.kern_version = OPTS_GET(opts, kern_version, 0);
327 attr.prog_token_fd = OPTS_GET(opts, token_fd, 0);
328
329 if (prog_name && feat_supported(NULL, FEAT_PROG_NAME))
330 libbpf_strlcpy(attr.prog_name, prog_name, sizeof(attr.prog_name));
331 attr.license = ptr_to_u64(license);
332
333 if (insn_cnt > UINT_MAX)
334 return libbpf_err(-E2BIG);
335
336 attr.insns = ptr_to_u64(insns);
337 attr.insn_cnt = (__u32)insn_cnt;
338
339 attach_prog_fd = OPTS_GET(opts, attach_prog_fd, 0);
340 attach_btf_obj_fd = OPTS_GET(opts, attach_btf_obj_fd, 0);
341
342 if (attach_prog_fd && attach_btf_obj_fd)
343 return libbpf_err(-EINVAL);
344
345 attr.attach_btf_id = OPTS_GET(opts, attach_btf_id, 0);
346 if (attach_prog_fd)
347 attr.attach_prog_fd = attach_prog_fd;
348 else
349 attr.attach_btf_obj_fd = attach_btf_obj_fd;
350
351 log_buf = OPTS_GET(opts, log_buf, NULL);
352 log_size = OPTS_GET(opts, log_size, 0);
353 log_level = OPTS_GET(opts, log_level, 0);
354
355 if (!!log_buf != !!log_size)
356 return libbpf_err(-EINVAL);
357
358 func_info_rec_size = OPTS_GET(opts, func_info_rec_size, 0);
359 func_info = OPTS_GET(opts, func_info, NULL);
360 attr.func_info_rec_size = func_info_rec_size;
361 attr.func_info = ptr_to_u64(func_info);
362 attr.func_info_cnt = OPTS_GET(opts, func_info_cnt, 0);
363
364 line_info_rec_size = OPTS_GET(opts, line_info_rec_size, 0);
365 line_info = OPTS_GET(opts, line_info, NULL);
366 attr.line_info_rec_size = line_info_rec_size;
367 attr.line_info = ptr_to_u64(line_info);
368 attr.line_info_cnt = OPTS_GET(opts, line_info_cnt, 0);
369
370 attr.fd_array = ptr_to_u64(OPTS_GET(opts, fd_array, NULL));
371 attr.fd_array_cnt = OPTS_GET(opts, fd_array_cnt, 0);
372
373 if (log_level) {
374 attr.log_buf = ptr_to_u64(log_buf);
375 attr.log_size = log_size;
376 attr.log_level = log_level;
377 }
378
379 fd = sys_bpf_prog_load(&attr, attr_sz, attempts);
380 OPTS_SET(opts, log_true_size, attr.log_true_size);
381 if (fd >= 0)
382 return fd;
383
384 /* After bpf_prog_load, the kernel may modify certain attributes
385 * to give user space a hint how to deal with loading failure.
386 * Check to see whether we can make some changes and load again.
387 */
388 while (errno == E2BIG && (!finfo || !linfo)) {
389 if (!finfo && attr.func_info_cnt &&
390 attr.func_info_rec_size < func_info_rec_size) {
391 /* try with corrected func info records */
392 finfo = alloc_zero_tailing_info(func_info,
393 attr.func_info_cnt,
394 func_info_rec_size,
395 attr.func_info_rec_size);
396 if (!finfo) {
397 errno = E2BIG;
398 goto done;
399 }
400
401 attr.func_info = ptr_to_u64(finfo);
402 attr.func_info_rec_size = func_info_rec_size;
403 } else if (!linfo && attr.line_info_cnt &&
404 attr.line_info_rec_size < line_info_rec_size) {
405 linfo = alloc_zero_tailing_info(line_info,
406 attr.line_info_cnt,
407 line_info_rec_size,
408 attr.line_info_rec_size);
409 if (!linfo) {
410 errno = E2BIG;
411 goto done;
412 }
413
414 attr.line_info = ptr_to_u64(linfo);
415 attr.line_info_rec_size = line_info_rec_size;
416 } else {
417 break;
418 }
419
420 fd = sys_bpf_prog_load(&attr, attr_sz, attempts);
421 OPTS_SET(opts, log_true_size, attr.log_true_size);
422 if (fd >= 0)
423 goto done;
424 }
425
426 if (log_level == 0 && log_buf) {
427 /* log_level == 0 with non-NULL log_buf requires retrying on error
428 * with log_level == 1 and log_buf/log_buf_size set, to get details of
429 * failure
430 */
431 attr.log_buf = ptr_to_u64(log_buf);
432 attr.log_size = log_size;
433 attr.log_level = 1;
434
435 fd = sys_bpf_prog_load(&attr, attr_sz, attempts);
436 OPTS_SET(opts, log_true_size, attr.log_true_size);
437 }
438done:
439 /* free() doesn't affect errno, so we don't need to restore it */
440 free(finfo);
441 free(linfo);
442 return libbpf_err_errno(fd);
443}
444
445int bpf_map_update_elem(int fd, const void *key, const void *value,
446 __u64 flags)
447{
448 const size_t attr_sz = offsetofend(union bpf_attr, flags);
449 union bpf_attr attr;
450 int ret;
451
452 memset(&attr, 0, attr_sz);
453 attr.map_fd = fd;
454 attr.key = ptr_to_u64(key);
455 attr.value = ptr_to_u64(value);
456 attr.flags = flags;
457
458 ret = sys_bpf(BPF_MAP_UPDATE_ELEM, &attr, attr_sz);
459 return libbpf_err_errno(ret);
460}
461
462int bpf_map_lookup_elem(int fd, const void *key, void *value)
463{
464 const size_t attr_sz = offsetofend(union bpf_attr, flags);
465 union bpf_attr attr;
466 int ret;
467
468 memset(&attr, 0, attr_sz);
469 attr.map_fd = fd;
470 attr.key = ptr_to_u64(key);
471 attr.value = ptr_to_u64(value);
472
473 ret = sys_bpf(BPF_MAP_LOOKUP_ELEM, &attr, attr_sz);
474 return libbpf_err_errno(ret);
475}
476
477int bpf_map_lookup_elem_flags(int fd, const void *key, void *value, __u64 flags)
478{
479 const size_t attr_sz = offsetofend(union bpf_attr, flags);
480 union bpf_attr attr;
481 int ret;
482
483 memset(&attr, 0, attr_sz);
484 attr.map_fd = fd;
485 attr.key = ptr_to_u64(key);
486 attr.value = ptr_to_u64(value);
487 attr.flags = flags;
488
489 ret = sys_bpf(BPF_MAP_LOOKUP_ELEM, &attr, attr_sz);
490 return libbpf_err_errno(ret);
491}
492
493int bpf_map_lookup_and_delete_elem(int fd, const void *key, void *value)
494{
495 const size_t attr_sz = offsetofend(union bpf_attr, flags);
496 union bpf_attr attr;
497 int ret;
498
499 memset(&attr, 0, attr_sz);
500 attr.map_fd = fd;
501 attr.key = ptr_to_u64(key);
502 attr.value = ptr_to_u64(value);
503
504 ret = sys_bpf(BPF_MAP_LOOKUP_AND_DELETE_ELEM, &attr, attr_sz);
505 return libbpf_err_errno(ret);
506}
507
508int bpf_map_lookup_and_delete_elem_flags(int fd, const void *key, void *value, __u64 flags)
509{
510 const size_t attr_sz = offsetofend(union bpf_attr, flags);
511 union bpf_attr attr;
512 int ret;
513
514 memset(&attr, 0, attr_sz);
515 attr.map_fd = fd;
516 attr.key = ptr_to_u64(key);
517 attr.value = ptr_to_u64(value);
518 attr.flags = flags;
519
520 ret = sys_bpf(BPF_MAP_LOOKUP_AND_DELETE_ELEM, &attr, attr_sz);
521 return libbpf_err_errno(ret);
522}
523
524int bpf_map_delete_elem(int fd, const void *key)
525{
526 const size_t attr_sz = offsetofend(union bpf_attr, flags);
527 union bpf_attr attr;
528 int ret;
529
530 memset(&attr, 0, attr_sz);
531 attr.map_fd = fd;
532 attr.key = ptr_to_u64(key);
533
534 ret = sys_bpf(BPF_MAP_DELETE_ELEM, &attr, attr_sz);
535 return libbpf_err_errno(ret);
536}
537
538int bpf_map_delete_elem_flags(int fd, const void *key, __u64 flags)
539{
540 const size_t attr_sz = offsetofend(union bpf_attr, flags);
541 union bpf_attr attr;
542 int ret;
543
544 memset(&attr, 0, attr_sz);
545 attr.map_fd = fd;
546 attr.key = ptr_to_u64(key);
547 attr.flags = flags;
548
549 ret = sys_bpf(BPF_MAP_DELETE_ELEM, &attr, attr_sz);
550 return libbpf_err_errno(ret);
551}
552
553int bpf_map_get_next_key(int fd, const void *key, void *next_key)
554{
555 const size_t attr_sz = offsetofend(union bpf_attr, next_key);
556 union bpf_attr attr;
557 int ret;
558
559 memset(&attr, 0, attr_sz);
560 attr.map_fd = fd;
561 attr.key = ptr_to_u64(key);
562 attr.next_key = ptr_to_u64(next_key);
563
564 ret = sys_bpf(BPF_MAP_GET_NEXT_KEY, &attr, attr_sz);
565 return libbpf_err_errno(ret);
566}
567
568int bpf_map_freeze(int fd)
569{
570 const size_t attr_sz = offsetofend(union bpf_attr, map_fd);
571 union bpf_attr attr;
572 int ret;
573
574 memset(&attr, 0, attr_sz);
575 attr.map_fd = fd;
576
577 ret = sys_bpf(BPF_MAP_FREEZE, &attr, attr_sz);
578 return libbpf_err_errno(ret);
579}
580
581static int bpf_map_batch_common(int cmd, int fd, void *in_batch,
582 void *out_batch, void *keys, void *values,
583 __u32 *count,
584 const struct bpf_map_batch_opts *opts)
585{
586 const size_t attr_sz = offsetofend(union bpf_attr, batch);
587 union bpf_attr attr;
588 int ret;
589
590 if (!OPTS_VALID(opts, bpf_map_batch_opts))
591 return libbpf_err(-EINVAL);
592
593 memset(&attr, 0, attr_sz);
594 attr.batch.map_fd = fd;
595 attr.batch.in_batch = ptr_to_u64(in_batch);
596 attr.batch.out_batch = ptr_to_u64(out_batch);
597 attr.batch.keys = ptr_to_u64(keys);
598 attr.batch.values = ptr_to_u64(values);
599 attr.batch.count = *count;
600 attr.batch.elem_flags = OPTS_GET(opts, elem_flags, 0);
601 attr.batch.flags = OPTS_GET(opts, flags, 0);
602
603 ret = sys_bpf(cmd, &attr, attr_sz);
604 *count = attr.batch.count;
605
606 return libbpf_err_errno(ret);
607}
608
609int bpf_map_delete_batch(int fd, const void *keys, __u32 *count,
610 const struct bpf_map_batch_opts *opts)
611{
612 return bpf_map_batch_common(BPF_MAP_DELETE_BATCH, fd, NULL,
613 NULL, (void *)keys, NULL, count, opts);
614}
615
616int bpf_map_lookup_batch(int fd, void *in_batch, void *out_batch, void *keys,
617 void *values, __u32 *count,
618 const struct bpf_map_batch_opts *opts)
619{
620 return bpf_map_batch_common(BPF_MAP_LOOKUP_BATCH, fd, in_batch,
621 out_batch, keys, values, count, opts);
622}
623
624int bpf_map_lookup_and_delete_batch(int fd, void *in_batch, void *out_batch,
625 void *keys, void *values, __u32 *count,
626 const struct bpf_map_batch_opts *opts)
627{
628 return bpf_map_batch_common(BPF_MAP_LOOKUP_AND_DELETE_BATCH,
629 fd, in_batch, out_batch, keys, values,
630 count, opts);
631}
632
633int bpf_map_update_batch(int fd, const void *keys, const void *values, __u32 *count,
634 const struct bpf_map_batch_opts *opts)
635{
636 return bpf_map_batch_common(BPF_MAP_UPDATE_BATCH, fd, NULL, NULL,
637 (void *)keys, (void *)values, count, opts);
638}
639
640int bpf_obj_pin_opts(int fd, const char *pathname, const struct bpf_obj_pin_opts *opts)
641{
642 const size_t attr_sz = offsetofend(union bpf_attr, path_fd);
643 union bpf_attr attr;
644 int ret;
645
646 if (!OPTS_VALID(opts, bpf_obj_pin_opts))
647 return libbpf_err(-EINVAL);
648
649 memset(&attr, 0, attr_sz);
650 attr.path_fd = OPTS_GET(opts, path_fd, 0);
651 attr.pathname = ptr_to_u64((void *)pathname);
652 attr.file_flags = OPTS_GET(opts, file_flags, 0);
653 attr.bpf_fd = fd;
654
655 ret = sys_bpf(BPF_OBJ_PIN, &attr, attr_sz);
656 return libbpf_err_errno(ret);
657}
658
659int bpf_obj_pin(int fd, const char *pathname)
660{
661 return bpf_obj_pin_opts(fd, pathname, NULL);
662}
663
664int bpf_obj_get(const char *pathname)
665{
666 return bpf_obj_get_opts(pathname, NULL);
667}
668
669int bpf_obj_get_opts(const char *pathname, const struct bpf_obj_get_opts *opts)
670{
671 const size_t attr_sz = offsetofend(union bpf_attr, path_fd);
672 union bpf_attr attr;
673 int fd;
674
675 if (!OPTS_VALID(opts, bpf_obj_get_opts))
676 return libbpf_err(-EINVAL);
677
678 memset(&attr, 0, attr_sz);
679 attr.path_fd = OPTS_GET(opts, path_fd, 0);
680 attr.pathname = ptr_to_u64((void *)pathname);
681 attr.file_flags = OPTS_GET(opts, file_flags, 0);
682
683 fd = sys_bpf_fd(BPF_OBJ_GET, &attr, attr_sz);
684 return libbpf_err_errno(fd);
685}
686
687int bpf_prog_attach(int prog_fd, int target_fd, enum bpf_attach_type type,
688 unsigned int flags)
689{
690 DECLARE_LIBBPF_OPTS(bpf_prog_attach_opts, opts,
691 .flags = flags,
692 );
693
694 return bpf_prog_attach_opts(prog_fd, target_fd, type, &opts);
695}
696
697int bpf_prog_attach_opts(int prog_fd, int target, enum bpf_attach_type type,
698 const struct bpf_prog_attach_opts *opts)
699{
700 const size_t attr_sz = offsetofend(union bpf_attr, expected_revision);
701 __u32 relative_id, flags;
702 int ret, relative_fd;
703 union bpf_attr attr;
704
705 if (!OPTS_VALID(opts, bpf_prog_attach_opts))
706 return libbpf_err(-EINVAL);
707
708 relative_id = OPTS_GET(opts, relative_id, 0);
709 relative_fd = OPTS_GET(opts, relative_fd, 0);
710 flags = OPTS_GET(opts, flags, 0);
711
712 /* validate we don't have unexpected combinations of non-zero fields */
713 if (relative_fd && relative_id)
714 return libbpf_err(-EINVAL);
715
716 memset(&attr, 0, attr_sz);
717 attr.target_fd = target;
718 attr.attach_bpf_fd = prog_fd;
719 attr.attach_type = type;
720 attr.replace_bpf_fd = OPTS_GET(opts, replace_fd, 0);
721 attr.expected_revision = OPTS_GET(opts, expected_revision, 0);
722
723 if (relative_id) {
724 attr.attach_flags = flags | BPF_F_ID;
725 attr.relative_id = relative_id;
726 } else {
727 attr.attach_flags = flags;
728 attr.relative_fd = relative_fd;
729 }
730
731 ret = sys_bpf(BPF_PROG_ATTACH, &attr, attr_sz);
732 return libbpf_err_errno(ret);
733}
734
735int bpf_prog_detach_opts(int prog_fd, int target, enum bpf_attach_type type,
736 const struct bpf_prog_detach_opts *opts)
737{
738 const size_t attr_sz = offsetofend(union bpf_attr, expected_revision);
739 __u32 relative_id, flags;
740 int ret, relative_fd;
741 union bpf_attr attr;
742
743 if (!OPTS_VALID(opts, bpf_prog_detach_opts))
744 return libbpf_err(-EINVAL);
745
746 relative_id = OPTS_GET(opts, relative_id, 0);
747 relative_fd = OPTS_GET(opts, relative_fd, 0);
748 flags = OPTS_GET(opts, flags, 0);
749
750 /* validate we don't have unexpected combinations of non-zero fields */
751 if (relative_fd && relative_id)
752 return libbpf_err(-EINVAL);
753
754 memset(&attr, 0, attr_sz);
755 attr.target_fd = target;
756 attr.attach_bpf_fd = prog_fd;
757 attr.attach_type = type;
758 attr.expected_revision = OPTS_GET(opts, expected_revision, 0);
759
760 if (relative_id) {
761 attr.attach_flags = flags | BPF_F_ID;
762 attr.relative_id = relative_id;
763 } else {
764 attr.attach_flags = flags;
765 attr.relative_fd = relative_fd;
766 }
767
768 ret = sys_bpf(BPF_PROG_DETACH, &attr, attr_sz);
769 return libbpf_err_errno(ret);
770}
771
772int bpf_prog_detach(int target_fd, enum bpf_attach_type type)
773{
774 return bpf_prog_detach_opts(0, target_fd, type, NULL);
775}
776
777int bpf_prog_detach2(int prog_fd, int target_fd, enum bpf_attach_type type)
778{
779 return bpf_prog_detach_opts(prog_fd, target_fd, type, NULL);
780}
781
782int bpf_link_create(int prog_fd, int target_fd,
783 enum bpf_attach_type attach_type,
784 const struct bpf_link_create_opts *opts)
785{
786 const size_t attr_sz = offsetofend(union bpf_attr, link_create);
787 __u32 target_btf_id, iter_info_len, relative_id;
788 int fd, err, relative_fd;
789 union bpf_attr attr;
790
791 if (!OPTS_VALID(opts, bpf_link_create_opts))
792 return libbpf_err(-EINVAL);
793
794 iter_info_len = OPTS_GET(opts, iter_info_len, 0);
795 target_btf_id = OPTS_GET(opts, target_btf_id, 0);
796
797 /* validate we don't have unexpected combinations of non-zero fields */
798 if (iter_info_len || target_btf_id) {
799 if (iter_info_len && target_btf_id)
800 return libbpf_err(-EINVAL);
801 if (!OPTS_ZEROED(opts, target_btf_id))
802 return libbpf_err(-EINVAL);
803 }
804
805 memset(&attr, 0, attr_sz);
806 attr.link_create.prog_fd = prog_fd;
807 attr.link_create.target_fd = target_fd;
808 attr.link_create.attach_type = attach_type;
809 attr.link_create.flags = OPTS_GET(opts, flags, 0);
810
811 if (target_btf_id) {
812 attr.link_create.target_btf_id = target_btf_id;
813 goto proceed;
814 }
815
816 switch (attach_type) {
817 case BPF_TRACE_ITER:
818 attr.link_create.iter_info = ptr_to_u64(OPTS_GET(opts, iter_info, (void *)0));
819 attr.link_create.iter_info_len = iter_info_len;
820 break;
821 case BPF_PERF_EVENT:
822 attr.link_create.perf_event.bpf_cookie = OPTS_GET(opts, perf_event.bpf_cookie, 0);
823 if (!OPTS_ZEROED(opts, perf_event))
824 return libbpf_err(-EINVAL);
825 break;
826 case BPF_TRACE_KPROBE_MULTI:
827 case BPF_TRACE_KPROBE_SESSION:
828 attr.link_create.kprobe_multi.flags = OPTS_GET(opts, kprobe_multi.flags, 0);
829 attr.link_create.kprobe_multi.cnt = OPTS_GET(opts, kprobe_multi.cnt, 0);
830 attr.link_create.kprobe_multi.syms = ptr_to_u64(OPTS_GET(opts, kprobe_multi.syms, 0));
831 attr.link_create.kprobe_multi.addrs = ptr_to_u64(OPTS_GET(opts, kprobe_multi.addrs, 0));
832 attr.link_create.kprobe_multi.cookies = ptr_to_u64(OPTS_GET(opts, kprobe_multi.cookies, 0));
833 if (!OPTS_ZEROED(opts, kprobe_multi))
834 return libbpf_err(-EINVAL);
835 break;
836 case BPF_TRACE_UPROBE_MULTI:
837 case BPF_TRACE_UPROBE_SESSION:
838 attr.link_create.uprobe_multi.flags = OPTS_GET(opts, uprobe_multi.flags, 0);
839 attr.link_create.uprobe_multi.cnt = OPTS_GET(opts, uprobe_multi.cnt, 0);
840 attr.link_create.uprobe_multi.path = ptr_to_u64(OPTS_GET(opts, uprobe_multi.path, 0));
841 attr.link_create.uprobe_multi.offsets = ptr_to_u64(OPTS_GET(opts, uprobe_multi.offsets, 0));
842 attr.link_create.uprobe_multi.ref_ctr_offsets = ptr_to_u64(OPTS_GET(opts, uprobe_multi.ref_ctr_offsets, 0));
843 attr.link_create.uprobe_multi.cookies = ptr_to_u64(OPTS_GET(opts, uprobe_multi.cookies, 0));
844 attr.link_create.uprobe_multi.pid = OPTS_GET(opts, uprobe_multi.pid, 0);
845 attr.link_create.uprobe_multi.path_fd = OPTS_GET(opts, uprobe_multi.path_fd, 0);
846 if (!OPTS_ZEROED(opts, uprobe_multi))
847 return libbpf_err(-EINVAL);
848 break;
849 case BPF_TRACE_FENTRY_MULTI:
850 case BPF_TRACE_FEXIT_MULTI:
851 case BPF_TRACE_FSESSION_MULTI:
852 attr.link_create.tracing_multi.ids = ptr_to_u64(OPTS_GET(opts, tracing_multi.ids, 0));
853 attr.link_create.tracing_multi.cookies = ptr_to_u64(OPTS_GET(opts, tracing_multi.cookies, 0));
854 attr.link_create.tracing_multi.cnt = OPTS_GET(opts, tracing_multi.cnt, 0);
855 if (!OPTS_ZEROED(opts, tracing_multi))
856 return libbpf_err(-EINVAL);
857 break;
858 case BPF_TRACE_RAW_TP:
859 case BPF_TRACE_FENTRY:
860 case BPF_TRACE_FEXIT:
861 case BPF_MODIFY_RETURN:
862 case BPF_TRACE_FSESSION:
863 case BPF_LSM_MAC:
864 attr.link_create.tracing.cookie = OPTS_GET(opts, tracing.cookie, 0);
865 if (!OPTS_ZEROED(opts, tracing))
866 return libbpf_err(-EINVAL);
867 break;
868 case BPF_NETFILTER:
869 attr.link_create.netfilter.pf = OPTS_GET(opts, netfilter.pf, 0);
870 attr.link_create.netfilter.hooknum = OPTS_GET(opts, netfilter.hooknum, 0);
871 attr.link_create.netfilter.priority = OPTS_GET(opts, netfilter.priority, 0);
872 attr.link_create.netfilter.flags = OPTS_GET(opts, netfilter.flags, 0);
873 if (!OPTS_ZEROED(opts, netfilter))
874 return libbpf_err(-EINVAL);
875 break;
876 case BPF_TCX_INGRESS:
877 case BPF_TCX_EGRESS:
878 relative_fd = OPTS_GET(opts, tcx.relative_fd, 0);
879 relative_id = OPTS_GET(opts, tcx.relative_id, 0);
880 if (relative_fd && relative_id)
881 return libbpf_err(-EINVAL);
882 if (relative_id) {
883 attr.link_create.tcx.relative_id = relative_id;
884 attr.link_create.flags |= BPF_F_ID;
885 } else {
886 attr.link_create.tcx.relative_fd = relative_fd;
887 }
888 attr.link_create.tcx.expected_revision = OPTS_GET(opts, tcx.expected_revision, 0);
889 if (!OPTS_ZEROED(opts, tcx))
890 return libbpf_err(-EINVAL);
891 break;
892 case BPF_NETKIT_PRIMARY:
893 case BPF_NETKIT_PEER:
894 relative_fd = OPTS_GET(opts, netkit.relative_fd, 0);
895 relative_id = OPTS_GET(opts, netkit.relative_id, 0);
896 if (relative_fd && relative_id)
897 return libbpf_err(-EINVAL);
898 if (relative_id) {
899 attr.link_create.netkit.relative_id = relative_id;
900 attr.link_create.flags |= BPF_F_ID;
901 } else {
902 attr.link_create.netkit.relative_fd = relative_fd;
903 }
904 attr.link_create.netkit.expected_revision = OPTS_GET(opts, netkit.expected_revision, 0);
905 if (!OPTS_ZEROED(opts, netkit))
906 return libbpf_err(-EINVAL);
907 break;
908 case BPF_CGROUP_INET_INGRESS:
909 case BPF_CGROUP_INET_EGRESS:
910 case BPF_CGROUP_INET_SOCK_CREATE:
911 case BPF_CGROUP_INET_SOCK_RELEASE:
912 case BPF_CGROUP_INET4_BIND:
913 case BPF_CGROUP_INET6_BIND:
914 case BPF_CGROUP_INET4_POST_BIND:
915 case BPF_CGROUP_INET6_POST_BIND:
916 case BPF_CGROUP_INET4_CONNECT:
917 case BPF_CGROUP_INET6_CONNECT:
918 case BPF_CGROUP_UNIX_CONNECT:
919 case BPF_CGROUP_INET4_GETPEERNAME:
920 case BPF_CGROUP_INET6_GETPEERNAME:
921 case BPF_CGROUP_UNIX_GETPEERNAME:
922 case BPF_CGROUP_INET4_GETSOCKNAME:
923 case BPF_CGROUP_INET6_GETSOCKNAME:
924 case BPF_CGROUP_UNIX_GETSOCKNAME:
925 case BPF_CGROUP_UDP4_SENDMSG:
926 case BPF_CGROUP_UDP6_SENDMSG:
927 case BPF_CGROUP_UNIX_SENDMSG:
928 case BPF_CGROUP_UDP4_RECVMSG:
929 case BPF_CGROUP_UDP6_RECVMSG:
930 case BPF_CGROUP_UNIX_RECVMSG:
931 case BPF_CGROUP_SOCK_OPS:
932 case BPF_CGROUP_DEVICE:
933 case BPF_CGROUP_SYSCTL:
934 case BPF_CGROUP_GETSOCKOPT:
935 case BPF_CGROUP_SETSOCKOPT:
936 case BPF_LSM_CGROUP:
937 relative_fd = OPTS_GET(opts, cgroup.relative_fd, 0);
938 relative_id = OPTS_GET(opts, cgroup.relative_id, 0);
939 if (relative_fd && relative_id)
940 return libbpf_err(-EINVAL);
941 if (relative_id) {
942 attr.link_create.cgroup.relative_id = relative_id;
943 attr.link_create.flags |= BPF_F_ID;
944 } else {
945 attr.link_create.cgroup.relative_fd = relative_fd;
946 }
947 attr.link_create.cgroup.expected_revision =
948 OPTS_GET(opts, cgroup.expected_revision, 0);
949 if (!OPTS_ZEROED(opts, cgroup))
950 return libbpf_err(-EINVAL);
951 break;
952 default:
953 if (!OPTS_ZEROED(opts, flags))
954 return libbpf_err(-EINVAL);
955 break;
956 }
957proceed:
958 fd = sys_bpf_fd(BPF_LINK_CREATE, &attr, attr_sz);
959 if (fd >= 0)
960 return fd;
961 /* we'll get EINVAL if LINK_CREATE doesn't support attaching fentry
962 * and other similar programs
963 */
964 err = -errno;
965 if (err != -EINVAL)
966 return libbpf_err(err);
967
968 /* if user used features not supported by
969 * BPF_RAW_TRACEPOINT_OPEN command, then just give up immediately
970 */
971 if (attr.link_create.target_fd || attr.link_create.target_btf_id)
972 return libbpf_err(err);
973 if (!OPTS_ZEROED(opts, sz))
974 return libbpf_err(err);
975
976 /* otherwise, for few select kinds of programs that can be
977 * attached using BPF_RAW_TRACEPOINT_OPEN command, try that as
978 * a fallback for older kernels
979 */
980 switch (attach_type) {
981 case BPF_TRACE_RAW_TP:
982 case BPF_LSM_MAC:
983 case BPF_TRACE_FENTRY:
984 case BPF_TRACE_FEXIT:
985 case BPF_MODIFY_RETURN:
986 return bpf_raw_tracepoint_open(NULL, prog_fd);
987 default:
988 return libbpf_err(err);
989 }
990}
991
992int bpf_link_detach(int link_fd)
993{
994 const size_t attr_sz = offsetofend(union bpf_attr, link_detach);
995 union bpf_attr attr;
996 int ret;
997
998 memset(&attr, 0, attr_sz);
999 attr.link_detach.link_fd = link_fd;
1000
1001 ret = sys_bpf(BPF_LINK_DETACH, &attr, attr_sz);
1002 return libbpf_err_errno(ret);
1003}
1004
1005int bpf_link_update(int link_fd, int new_prog_fd,
1006 const struct bpf_link_update_opts *opts)
1007{
1008 const size_t attr_sz = offsetofend(union bpf_attr, link_update);
1009 union bpf_attr attr;
1010 int ret;
1011
1012 if (!OPTS_VALID(opts, bpf_link_update_opts))
1013 return libbpf_err(-EINVAL);
1014
1015 if (OPTS_GET(opts, old_prog_fd, 0) && OPTS_GET(opts, old_map_fd, 0))
1016 return libbpf_err(-EINVAL);
1017
1018 memset(&attr, 0, attr_sz);
1019 attr.link_update.link_fd = link_fd;
1020 attr.link_update.new_prog_fd = new_prog_fd;
1021 attr.link_update.flags = OPTS_GET(opts, flags, 0);
1022 if (OPTS_GET(opts, old_prog_fd, 0))
1023 attr.link_update.old_prog_fd = OPTS_GET(opts, old_prog_fd, 0);
1024 else if (OPTS_GET(opts, old_map_fd, 0))
1025 attr.link_update.old_map_fd = OPTS_GET(opts, old_map_fd, 0);
1026
1027 ret = sys_bpf(BPF_LINK_UPDATE, &attr, attr_sz);
1028 return libbpf_err_errno(ret);
1029}
1030
1031int bpf_iter_create(int link_fd)
1032{
1033 const size_t attr_sz = offsetofend(union bpf_attr, iter_create);
1034 union bpf_attr attr;
1035 int fd;
1036
1037 memset(&attr, 0, attr_sz);
1038 attr.iter_create.link_fd = link_fd;
1039
1040 fd = sys_bpf_fd(BPF_ITER_CREATE, &attr, attr_sz);
1041 return libbpf_err_errno(fd);
1042}
1043
1044int bpf_prog_query_opts(int target, enum bpf_attach_type type,
1045 struct bpf_prog_query_opts *opts)
1046{
1047 const size_t attr_sz = offsetofend(union bpf_attr, query);
1048 union bpf_attr attr;
1049 int ret;
1050
1051 if (!OPTS_VALID(opts, bpf_prog_query_opts))
1052 return libbpf_err(-EINVAL);
1053
1054 memset(&attr, 0, attr_sz);
1055 attr.query.target_fd = target;
1056 attr.query.attach_type = type;
1057 attr.query.query_flags = OPTS_GET(opts, query_flags, 0);
1058 attr.query.count = OPTS_GET(opts, count, 0);
1059 attr.query.prog_ids = ptr_to_u64(OPTS_GET(opts, prog_ids, NULL));
1060 attr.query.link_ids = ptr_to_u64(OPTS_GET(opts, link_ids, NULL));
1061 attr.query.prog_attach_flags = ptr_to_u64(OPTS_GET(opts, prog_attach_flags, NULL));
1062 attr.query.link_attach_flags = ptr_to_u64(OPTS_GET(opts, link_attach_flags, NULL));
1063
1064 ret = sys_bpf(BPF_PROG_QUERY, &attr, attr_sz);
1065
1066 OPTS_SET(opts, attach_flags, attr.query.attach_flags);
1067 OPTS_SET(opts, revision, attr.query.revision);
1068 OPTS_SET(opts, count, attr.query.count);
1069
1070 return libbpf_err_errno(ret);
1071}
1072
1073int bpf_prog_query(int target_fd, enum bpf_attach_type type, __u32 query_flags,
1074 __u32 *attach_flags, __u32 *prog_ids, __u32 *prog_cnt)
1075{
1076 LIBBPF_OPTS(bpf_prog_query_opts, opts);
1077 int ret;
1078
1079 opts.query_flags = query_flags;
1080 opts.prog_ids = prog_ids;
1081 opts.prog_cnt = *prog_cnt;
1082
1083 ret = bpf_prog_query_opts(target_fd, type, &opts);
1084
1085 if (attach_flags)
1086 *attach_flags = opts.attach_flags;
1087 *prog_cnt = opts.prog_cnt;
1088
1089 return libbpf_err_errno(ret);
1090}
1091
1092int bpf_prog_test_run_opts(int prog_fd, struct bpf_test_run_opts *opts)
1093{
1094 const size_t attr_sz = offsetofend(union bpf_attr, test);
1095 union bpf_attr attr;
1096 int ret;
1097
1098 if (!OPTS_VALID(opts, bpf_test_run_opts))
1099 return libbpf_err(-EINVAL);
1100
1101 memset(&attr, 0, attr_sz);
1102 attr.test.prog_fd = prog_fd;
1103 attr.test.batch_size = OPTS_GET(opts, batch_size, 0);
1104 attr.test.cpu = OPTS_GET(opts, cpu, 0);
1105 attr.test.flags = OPTS_GET(opts, flags, 0);
1106 attr.test.repeat = OPTS_GET(opts, repeat, 0);
1107 attr.test.duration = OPTS_GET(opts, duration, 0);
1108 attr.test.ctx_size_in = OPTS_GET(opts, ctx_size_in, 0);
1109 attr.test.ctx_size_out = OPTS_GET(opts, ctx_size_out, 0);
1110 attr.test.data_size_in = OPTS_GET(opts, data_size_in, 0);
1111 attr.test.data_size_out = OPTS_GET(opts, data_size_out, 0);
1112 attr.test.ctx_in = ptr_to_u64(OPTS_GET(opts, ctx_in, NULL));
1113 attr.test.ctx_out = ptr_to_u64(OPTS_GET(opts, ctx_out, NULL));
1114 attr.test.data_in = ptr_to_u64(OPTS_GET(opts, data_in, NULL));
1115 attr.test.data_out = ptr_to_u64(OPTS_GET(opts, data_out, NULL));
1116
1117 ret = sys_bpf(BPF_PROG_TEST_RUN, &attr, attr_sz);
1118
1119 OPTS_SET(opts, data_size_out, attr.test.data_size_out);
1120 OPTS_SET(opts, ctx_size_out, attr.test.ctx_size_out);
1121 OPTS_SET(opts, duration, attr.test.duration);
1122 OPTS_SET(opts, retval, attr.test.retval);
1123
1124 return libbpf_err_errno(ret);
1125}
1126
1127static int bpf_obj_get_next_id(__u32 start_id, __u32 *next_id, int cmd)
1128{
1129 const size_t attr_sz = offsetofend(union bpf_attr, open_flags);
1130 union bpf_attr attr;
1131 int err;
1132
1133 memset(&attr, 0, attr_sz);
1134 attr.start_id = start_id;
1135
1136 err = sys_bpf(cmd, &attr, attr_sz);
1137 if (!err)
1138 *next_id = attr.next_id;
1139
1140 return libbpf_err_errno(err);
1141}
1142
1143int bpf_prog_get_next_id(__u32 start_id, __u32 *next_id)
1144{
1145 return bpf_obj_get_next_id(start_id, next_id, BPF_PROG_GET_NEXT_ID);
1146}
1147
1148int bpf_map_get_next_id(__u32 start_id, __u32 *next_id)
1149{
1150 return bpf_obj_get_next_id(start_id, next_id, BPF_MAP_GET_NEXT_ID);
1151}
1152
1153int bpf_btf_get_next_id(__u32 start_id, __u32 *next_id)
1154{
1155 return bpf_obj_get_next_id(start_id, next_id, BPF_BTF_GET_NEXT_ID);
1156}
1157
1158int bpf_link_get_next_id(__u32 start_id, __u32 *next_id)
1159{
1160 return bpf_obj_get_next_id(start_id, next_id, BPF_LINK_GET_NEXT_ID);
1161}
1162
1163int bpf_prog_get_fd_by_id_opts(__u32 id,
1164 const struct bpf_get_fd_by_id_opts *opts)
1165{
1166 const size_t attr_sz = offsetofend(union bpf_attr, open_flags);
1167 union bpf_attr attr;
1168 int fd;
1169
1170 if (!OPTS_VALID(opts, bpf_get_fd_by_id_opts))
1171 return libbpf_err(-EINVAL);
1172
1173 memset(&attr, 0, attr_sz);
1174 attr.prog_id = id;
1175 attr.open_flags = OPTS_GET(opts, open_flags, 0);
1176
1177 fd = sys_bpf_fd(BPF_PROG_GET_FD_BY_ID, &attr, attr_sz);
1178 return libbpf_err_errno(fd);
1179}
1180
1181int bpf_prog_get_fd_by_id(__u32 id)
1182{
1183 return bpf_prog_get_fd_by_id_opts(id, NULL);
1184}
1185
1186int bpf_map_get_fd_by_id_opts(__u32 id,
1187 const struct bpf_get_fd_by_id_opts *opts)
1188{
1189 const size_t attr_sz = offsetofend(union bpf_attr, open_flags);
1190 union bpf_attr attr;
1191 int fd;
1192
1193 if (!OPTS_VALID(opts, bpf_get_fd_by_id_opts))
1194 return libbpf_err(-EINVAL);
1195
1196 memset(&attr, 0, attr_sz);
1197 attr.map_id = id;
1198 attr.open_flags = OPTS_GET(opts, open_flags, 0);
1199
1200 fd = sys_bpf_fd(BPF_MAP_GET_FD_BY_ID, &attr, attr_sz);
1201 return libbpf_err_errno(fd);
1202}
1203
1204int bpf_map_get_fd_by_id(__u32 id)
1205{
1206 return bpf_map_get_fd_by_id_opts(id, NULL);
1207}
1208
1209int bpf_btf_get_fd_by_id_opts(__u32 id,
1210 const struct bpf_get_fd_by_id_opts *opts)
1211{
1212 const size_t attr_sz = offsetofend(union bpf_attr, fd_by_id_token_fd);
1213 union bpf_attr attr;
1214 int fd;
1215
1216 if (!OPTS_VALID(opts, bpf_get_fd_by_id_opts))
1217 return libbpf_err(-EINVAL);
1218
1219 memset(&attr, 0, attr_sz);
1220 attr.btf_id = id;
1221 attr.open_flags = OPTS_GET(opts, open_flags, 0);
1222 attr.fd_by_id_token_fd = OPTS_GET(opts, token_fd, 0);
1223
1224 fd = sys_bpf_fd(BPF_BTF_GET_FD_BY_ID, &attr, attr_sz);
1225 return libbpf_err_errno(fd);
1226}
1227
1228int bpf_btf_get_fd_by_id(__u32 id)
1229{
1230 return bpf_btf_get_fd_by_id_opts(id, NULL);
1231}
1232
1233int bpf_link_get_fd_by_id_opts(__u32 id,
1234 const struct bpf_get_fd_by_id_opts *opts)
1235{
1236 const size_t attr_sz = offsetofend(union bpf_attr, open_flags);
1237 union bpf_attr attr;
1238 int fd;
1239
1240 if (!OPTS_VALID(opts, bpf_get_fd_by_id_opts))
1241 return libbpf_err(-EINVAL);
1242
1243 memset(&attr, 0, attr_sz);
1244 attr.link_id = id;
1245 attr.open_flags = OPTS_GET(opts, open_flags, 0);
1246
1247 fd = sys_bpf_fd(BPF_LINK_GET_FD_BY_ID, &attr, attr_sz);
1248 return libbpf_err_errno(fd);
1249}
1250
1251int bpf_link_get_fd_by_id(__u32 id)
1252{
1253 return bpf_link_get_fd_by_id_opts(id, NULL);
1254}
1255
1256int bpf_obj_get_info_by_fd(int bpf_fd, void *info, __u32 *info_len)
1257{
1258 const size_t attr_sz = offsetofend(union bpf_attr, info);
1259 union bpf_attr attr;
1260 int err;
1261
1262 memset(&attr, 0, attr_sz);
1263 attr.info.bpf_fd = bpf_fd;
1264 attr.info.info_len = *info_len;
1265 attr.info.info = ptr_to_u64(info);
1266
1267 err = sys_bpf(BPF_OBJ_GET_INFO_BY_FD, &attr, attr_sz);
1268 if (!err)
1269 *info_len = attr.info.info_len;
1270 return libbpf_err_errno(err);
1271}
1272
1273int bpf_prog_get_info_by_fd(int prog_fd, struct bpf_prog_info *info, __u32 *info_len)
1274{
1275 return bpf_obj_get_info_by_fd(prog_fd, info, info_len);
1276}
1277
1278int bpf_map_get_info_by_fd(int map_fd, struct bpf_map_info *info, __u32 *info_len)
1279{
1280 return bpf_obj_get_info_by_fd(map_fd, info, info_len);
1281}
1282
1283int bpf_btf_get_info_by_fd(int btf_fd, struct bpf_btf_info *info, __u32 *info_len)
1284{
1285 return bpf_obj_get_info_by_fd(btf_fd, info, info_len);
1286}
1287
1288int bpf_link_get_info_by_fd(int link_fd, struct bpf_link_info *info, __u32 *info_len)
1289{
1290 return bpf_obj_get_info_by_fd(link_fd, info, info_len);
1291}
1292
1293int bpf_raw_tracepoint_open_opts(int prog_fd, struct bpf_raw_tp_opts *opts)
1294{
1295 const size_t attr_sz = offsetofend(union bpf_attr, raw_tracepoint);
1296 union bpf_attr attr;
1297 int fd;
1298
1299 if (!OPTS_VALID(opts, bpf_raw_tp_opts))
1300 return libbpf_err(-EINVAL);
1301
1302 memset(&attr, 0, attr_sz);
1303 attr.raw_tracepoint.prog_fd = prog_fd;
1304 attr.raw_tracepoint.name = ptr_to_u64(OPTS_GET(opts, tp_name, NULL));
1305 attr.raw_tracepoint.cookie = OPTS_GET(opts, cookie, 0);
1306
1307 fd = sys_bpf_fd(BPF_RAW_TRACEPOINT_OPEN, &attr, attr_sz);
1308 return libbpf_err_errno(fd);
1309}
1310
1311int bpf_raw_tracepoint_open(const char *name, int prog_fd)
1312{
1313 LIBBPF_OPTS(bpf_raw_tp_opts, opts, .tp_name = name);
1314
1315 return bpf_raw_tracepoint_open_opts(prog_fd, &opts);
1316}
1317
1318int bpf_btf_load(const void *btf_data, size_t btf_size, struct bpf_btf_load_opts *opts)
1319{
1320 const size_t attr_sz = offsetofend(union bpf_attr, btf_token_fd);
1321 union bpf_attr attr;
1322 char *log_buf;
1323 size_t log_size;
1324 __u32 log_level;
1325 int fd;
1326
1327 bump_rlimit_memlock();
1328
1329 memset(&attr, 0, attr_sz);
1330
1331 if (!OPTS_VALID(opts, bpf_btf_load_opts))
1332 return libbpf_err(-EINVAL);
1333
1334 log_buf = OPTS_GET(opts, log_buf, NULL);
1335 log_size = OPTS_GET(opts, log_size, 0);
1336 log_level = OPTS_GET(opts, log_level, 0);
1337
1338 if (log_size > UINT_MAX)
1339 return libbpf_err(-EINVAL);
1340 if (log_size && !log_buf)
1341 return libbpf_err(-EINVAL);
1342
1343 attr.btf = ptr_to_u64(btf_data);
1344 attr.btf_size = btf_size;
1345
1346 attr.btf_flags = OPTS_GET(opts, btf_flags, 0);
1347 attr.btf_token_fd = OPTS_GET(opts, token_fd, 0);
1348
1349 /* log_level == 0 and log_buf != NULL means "try loading without
1350 * log_buf, but retry with log_buf and log_level=1 on error", which is
1351 * consistent across low-level and high-level BTF and program loading
1352 * APIs within libbpf and provides a sensible behavior in practice
1353 */
1354 if (log_level) {
1355 attr.btf_log_buf = ptr_to_u64(log_buf);
1356 attr.btf_log_size = (__u32)log_size;
1357 attr.btf_log_level = log_level;
1358 }
1359
1360 fd = sys_bpf_fd(BPF_BTF_LOAD, &attr, attr_sz);
1361 if (fd < 0 && log_buf && log_level == 0) {
1362 attr.btf_log_buf = ptr_to_u64(log_buf);
1363 attr.btf_log_size = (__u32)log_size;
1364 attr.btf_log_level = 1;
1365 fd = sys_bpf_fd(BPF_BTF_LOAD, &attr, attr_sz);
1366 }
1367
1368 OPTS_SET(opts, log_true_size, attr.btf_log_true_size);
1369 return libbpf_err_errno(fd);
1370}
1371
1372int bpf_task_fd_query(int pid, int fd, __u32 flags, char *buf, __u32 *buf_len,
1373 __u32 *prog_id, __u32 *fd_type, __u64 *probe_offset,
1374 __u64 *probe_addr)
1375{
1376 const size_t attr_sz = offsetofend(union bpf_attr, task_fd_query);
1377 union bpf_attr attr;
1378 int err;
1379
1380 memset(&attr, 0, attr_sz);
1381 attr.task_fd_query.pid = pid;
1382 attr.task_fd_query.fd = fd;
1383 attr.task_fd_query.flags = flags;
1384 attr.task_fd_query.buf = ptr_to_u64(buf);
1385 attr.task_fd_query.buf_len = *buf_len;
1386
1387 err = sys_bpf(BPF_TASK_FD_QUERY, &attr, attr_sz);
1388
1389 *buf_len = attr.task_fd_query.buf_len;
1390 *prog_id = attr.task_fd_query.prog_id;
1391 *fd_type = attr.task_fd_query.fd_type;
1392 *probe_offset = attr.task_fd_query.probe_offset;
1393 *probe_addr = attr.task_fd_query.probe_addr;
1394
1395 return libbpf_err_errno(err);
1396}
1397
1398int bpf_enable_stats(enum bpf_stats_type type)
1399{
1400 const size_t attr_sz = offsetofend(union bpf_attr, enable_stats);
1401 union bpf_attr attr;
1402 int fd;
1403
1404 memset(&attr, 0, attr_sz);
1405 attr.enable_stats.type = type;
1406
1407 fd = sys_bpf_fd(BPF_ENABLE_STATS, &attr, attr_sz);
1408 return libbpf_err_errno(fd);
1409}
1410
1411int bpf_prog_bind_map(int prog_fd, int map_fd,
1412 const struct bpf_prog_bind_opts *opts)
1413{
1414 const size_t attr_sz = offsetofend(union bpf_attr, prog_bind_map);
1415 union bpf_attr attr;
1416 int ret;
1417
1418 if (!OPTS_VALID(opts, bpf_prog_bind_opts))
1419 return libbpf_err(-EINVAL);
1420
1421 memset(&attr, 0, attr_sz);
1422 attr.prog_bind_map.prog_fd = prog_fd;
1423 attr.prog_bind_map.map_fd = map_fd;
1424 attr.prog_bind_map.flags = OPTS_GET(opts, flags, 0);
1425
1426 ret = sys_bpf(BPF_PROG_BIND_MAP, &attr, attr_sz);
1427 return libbpf_err_errno(ret);
1428}
1429
1430int bpf_token_create(int bpffs_fd, struct bpf_token_create_opts *opts)
1431{
1432 const size_t attr_sz = offsetofend(union bpf_attr, token_create);
1433 union bpf_attr attr;
1434 int fd;
1435
1436 if (!OPTS_VALID(opts, bpf_token_create_opts))
1437 return libbpf_err(-EINVAL);
1438
1439 memset(&attr, 0, attr_sz);
1440 attr.token_create.bpffs_fd = bpffs_fd;
1441 attr.token_create.flags = OPTS_GET(opts, flags, 0);
1442
1443 fd = sys_bpf_fd(BPF_TOKEN_CREATE, &attr, attr_sz);
1444 return libbpf_err_errno(fd);
1445}
1446
1447int bpf_prog_stream_read(int prog_fd, __u32 stream_id, void *buf, __u32 buf_len,
1448 struct bpf_prog_stream_read_opts *opts)
1449{
1450 const size_t attr_sz = offsetofend(union bpf_attr, prog_stream_read);
1451 union bpf_attr attr;
1452 int err;
1453
1454 if (!OPTS_VALID(opts, bpf_prog_stream_read_opts))
1455 return libbpf_err(-EINVAL);
1456
1457 memset(&attr, 0, attr_sz);
1458 attr.prog_stream_read.stream_buf = ptr_to_u64(buf);
1459 attr.prog_stream_read.stream_buf_len = buf_len;
1460 attr.prog_stream_read.stream_id = stream_id;
1461 attr.prog_stream_read.prog_fd = prog_fd;
1462
1463 err = sys_bpf(BPF_PROG_STREAM_READ_BY_FD, &attr, attr_sz);
1464 return libbpf_err_errno(err);
1465}
1466
1467int bpf_prog_assoc_struct_ops(int prog_fd, int map_fd,
1468 struct bpf_prog_assoc_struct_ops_opts *opts)
1469{
1470 const size_t attr_sz = offsetofend(union bpf_attr, prog_assoc_struct_ops);
1471 union bpf_attr attr;
1472 int err;
1473
1474 if (!OPTS_VALID(opts, bpf_prog_assoc_struct_ops_opts))
1475 return libbpf_err(-EINVAL);
1476
1477 memset(&attr, 0, attr_sz);
1478 attr.prog_assoc_struct_ops.map_fd = map_fd;
1479 attr.prog_assoc_struct_ops.prog_fd = prog_fd;
1480 attr.prog_assoc_struct_ops.flags = OPTS_GET(opts, flags, 0);
1481
1482 err = sys_bpf(BPF_PROG_ASSOC_STRUCT_OPS, &attr, attr_sz);
1483 return libbpf_err_errno(err);
1484}