linux - tools/lib/bpf/libbpf.c

15230 lines · 408.3 KiB

1	// SPDX-License-Identifier: (LGPL-2.1 OR BSD-2-Clause)
2
3	/*
4	* Common eBPF ELF object loading operations.
5	*
6	* Copyright (C) 2013-2015 Alexei Starovoitov <[email protected]>
7	* Copyright (C) 2015 Wang Nan <[email protected]>
8	* Copyright (C) 2015 Huawei Inc.
9	* Copyright (C) 2017 Nicira, Inc.
10	* Copyright (C) 2019 Isovalent, Inc.
11	*/
12
13	#ifndef _GNU_SOURCE
14	#define _GNU_SOURCE
15	#endif
16	#include <stdlib.h>
17	#include <stdio.h>
18	#include <stdarg.h>
19	#include <libgen.h>
20	#include <inttypes.h>
21	#include <limits.h>
22	#include <string.h>
23	#include <unistd.h>
24	#include <endian.h>
25	#include <fcntl.h>
26	#include <errno.h>
27	#include <ctype.h>
28	#include <asm/unistd.h>
29	#include <linux/err.h>
30	#include <linux/kernel.h>
31	#include <linux/bpf.h>
32	#include <linux/btf.h>
33	#include <linux/filter.h>
34	#include <linux/limits.h>
35	#include <linux/perf_event.h>
36	#include <linux/bpf_perf_event.h>
37	#include <linux/ring_buffer.h>
38	#include <sys/epoll.h>
39	#include <sys/ioctl.h>
40	#include <sys/mman.h>
41	#include <sys/stat.h>
42	#include <sys/types.h>
43	#include <sys/vfs.h>
44	#include <sys/utsname.h>
45	#include <sys/resource.h>
46	#include <libelf.h>
47	#include <gelf.h>
48	#include <zlib.h>
49
50	#include "libbpf.h"
51	#include "bpf.h"
52	#include "btf.h"
53	#include "libbpf_internal.h"
54	#include "hashmap.h"
55	#include "bpf_gen_internal.h"
56	#include "zip.h"
57
58	#ifndef BPF_FS_MAGIC
59	#define BPF_FS_MAGIC 0xcafe4a11
60	#endif
61
62	#define MAX_EVENT_NAME_LEN 64
63
64	#define BPF_FS_DEFAULT_PATH "/sys/fs/bpf"
65
66	#define BPF_INSN_SZ (sizeof(struct bpf_insn))
67
68	/* vsprintf() in __base_pr() uses nonliteral format string. It may break
69	* compilation if user enables corresponding warning. Disable it explicitly.
70	*/
71	#pragma GCC diagnostic ignored "-Wformat-nonliteral"
72
73	#define __printf(a, b) __attribute__((format(printf, a, b)))
74
75	static struct bpf_map bpf_object__add_map(struct bpf_object obj);
76	static bool prog_is_subprog(const struct bpf_object obj, const struct bpf_program prog);
77	static int map_set_def_max_entries(struct bpf_map *map);
78
79	static const char * const attach_type_name[] = {
80	[BPF_CGROUP_INET_INGRESS] = "cgroup_inet_ingress",
81	[BPF_CGROUP_INET_EGRESS] = "cgroup_inet_egress",
82	[BPF_CGROUP_INET_SOCK_CREATE] = "cgroup_inet_sock_create",
83	[BPF_CGROUP_INET_SOCK_RELEASE] = "cgroup_inet_sock_release",
84	[BPF_CGROUP_SOCK_OPS] = "cgroup_sock_ops",
85	[BPF_CGROUP_DEVICE] = "cgroup_device",
86	[BPF_CGROUP_INET4_BIND] = "cgroup_inet4_bind",
87	[BPF_CGROUP_INET6_BIND] = "cgroup_inet6_bind",
88	[BPF_CGROUP_INET4_CONNECT] = "cgroup_inet4_connect",
89	[BPF_CGROUP_INET6_CONNECT] = "cgroup_inet6_connect",
90	[BPF_CGROUP_UNIX_CONNECT] = "cgroup_unix_connect",
91	[BPF_CGROUP_INET4_POST_BIND] = "cgroup_inet4_post_bind",
92	[BPF_CGROUP_INET6_POST_BIND] = "cgroup_inet6_post_bind",
93	[BPF_CGROUP_INET4_GETPEERNAME] = "cgroup_inet4_getpeername",
94	[BPF_CGROUP_INET6_GETPEERNAME] = "cgroup_inet6_getpeername",
95	[BPF_CGROUP_UNIX_GETPEERNAME] = "cgroup_unix_getpeername",
96	[BPF_CGROUP_INET4_GETSOCKNAME] = "cgroup_inet4_getsockname",
97	[BPF_CGROUP_INET6_GETSOCKNAME] = "cgroup_inet6_getsockname",
98	[BPF_CGROUP_UNIX_GETSOCKNAME] = "cgroup_unix_getsockname",
99	[BPF_CGROUP_UDP4_SENDMSG] = "cgroup_udp4_sendmsg",
100	[BPF_CGROUP_UDP6_SENDMSG] = "cgroup_udp6_sendmsg",
101	[BPF_CGROUP_UNIX_SENDMSG] = "cgroup_unix_sendmsg",
102	[BPF_CGROUP_SYSCTL] = "cgroup_sysctl",
103	[BPF_CGROUP_UDP4_RECVMSG] = "cgroup_udp4_recvmsg",
104	[BPF_CGROUP_UDP6_RECVMSG] = "cgroup_udp6_recvmsg",
105	[BPF_CGROUP_UNIX_RECVMSG] = "cgroup_unix_recvmsg",
106	[BPF_CGROUP_GETSOCKOPT] = "cgroup_getsockopt",
107	[BPF_CGROUP_SETSOCKOPT] = "cgroup_setsockopt",
108	[BPF_SK_SKB_STREAM_PARSER] = "sk_skb_stream_parser",
109	[BPF_SK_SKB_STREAM_VERDICT] = "sk_skb_stream_verdict",
110	[BPF_SK_SKB_VERDICT] = "sk_skb_verdict",
111	[BPF_SK_MSG_VERDICT] = "sk_msg_verdict",
112	[BPF_LIRC_MODE2] = "lirc_mode2",
113	[BPF_FLOW_DISSECTOR] = "flow_dissector",
114	[BPF_TRACE_RAW_TP] = "trace_raw_tp",
115	[BPF_TRACE_FENTRY] = "trace_fentry",
116	[BPF_TRACE_FEXIT] = "trace_fexit",
117	[BPF_MODIFY_RETURN] = "modify_return",
118	[BPF_TRACE_FSESSION] = "trace_fsession",
119	[BPF_LSM_MAC] = "lsm_mac",
120	[BPF_LSM_CGROUP] = "lsm_cgroup",
121	[BPF_SK_LOOKUP] = "sk_lookup",
122	[BPF_TRACE_ITER] = "trace_iter",
123	[BPF_XDP_DEVMAP] = "xdp_devmap",
124	[BPF_XDP_CPUMAP] = "xdp_cpumap",
125	[BPF_XDP] = "xdp",
126	[BPF_SK_REUSEPORT_SELECT] = "sk_reuseport_select",
127	[BPF_SK_REUSEPORT_SELECT_OR_MIGRATE] = "sk_reuseport_select_or_migrate",
128	[BPF_PERF_EVENT] = "perf_event",
129	[BPF_TRACE_KPROBE_MULTI] = "trace_kprobe_multi",
130	[BPF_STRUCT_OPS] = "struct_ops",
131	[BPF_NETFILTER] = "netfilter",
132	[BPF_TCX_INGRESS] = "tcx_ingress",
133	[BPF_TCX_EGRESS] = "tcx_egress",
134	[BPF_TRACE_UPROBE_MULTI] = "trace_uprobe_multi",
135	[BPF_NETKIT_PRIMARY] = "netkit_primary",
136	[BPF_NETKIT_PEER] = "netkit_peer",
137	[BPF_TRACE_KPROBE_SESSION] = "trace_kprobe_session",
138	[BPF_TRACE_UPROBE_SESSION] = "trace_uprobe_session",
139	[BPF_TRACE_FENTRY_MULTI] = "trace_fentry_multi",
140	[BPF_TRACE_FEXIT_MULTI] = "trace_fexit_multi",
141	[BPF_TRACE_FSESSION_MULTI] = "trace_fsession_multi",
142	};
143
144	static const char * const link_type_name[] = {
145	[BPF_LINK_TYPE_UNSPEC] = "unspec",
146	[BPF_LINK_TYPE_RAW_TRACEPOINT] = "raw_tracepoint",
147	[BPF_LINK_TYPE_TRACING] = "tracing",
148	[BPF_LINK_TYPE_CGROUP] = "cgroup",
149	[BPF_LINK_TYPE_ITER] = "iter",
150	[BPF_LINK_TYPE_NETNS] = "netns",
151	[BPF_LINK_TYPE_XDP] = "xdp",
152	[BPF_LINK_TYPE_PERF_EVENT] = "perf_event",
153	[BPF_LINK_TYPE_KPROBE_MULTI] = "kprobe_multi",
154	[BPF_LINK_TYPE_STRUCT_OPS] = "struct_ops",
155	[BPF_LINK_TYPE_NETFILTER] = "netfilter",
156	[BPF_LINK_TYPE_TCX] = "tcx",
157	[BPF_LINK_TYPE_UPROBE_MULTI] = "uprobe_multi",
158	[BPF_LINK_TYPE_NETKIT] = "netkit",
159	[BPF_LINK_TYPE_SOCKMAP] = "sockmap",
160	[BPF_LINK_TYPE_TRACING_MULTI] = "tracing_multi",
161	};
162
163	static const char * const map_type_name[] = {
164	[BPF_MAP_TYPE_UNSPEC] = "unspec",
165	[BPF_MAP_TYPE_HASH] = "hash",
166	[BPF_MAP_TYPE_ARRAY] = "array",
167	[BPF_MAP_TYPE_PROG_ARRAY] = "prog_array",
168	[BPF_MAP_TYPE_PERF_EVENT_ARRAY] = "perf_event_array",
169	[BPF_MAP_TYPE_PERCPU_HASH] = "percpu_hash",
170	[BPF_MAP_TYPE_PERCPU_ARRAY] = "percpu_array",
171	[BPF_MAP_TYPE_STACK_TRACE] = "stack_trace",
172	[BPF_MAP_TYPE_CGROUP_ARRAY] = "cgroup_array",
173	[BPF_MAP_TYPE_LRU_HASH] = "lru_hash",
174	[BPF_MAP_TYPE_LRU_PERCPU_HASH] = "lru_percpu_hash",
175	[BPF_MAP_TYPE_LPM_TRIE] = "lpm_trie",
176	[BPF_MAP_TYPE_ARRAY_OF_MAPS] = "array_of_maps",
177	[BPF_MAP_TYPE_HASH_OF_MAPS] = "hash_of_maps",
178	[BPF_MAP_TYPE_DEVMAP] = "devmap",
179	[BPF_MAP_TYPE_DEVMAP_HASH] = "devmap_hash",
180	[BPF_MAP_TYPE_SOCKMAP] = "sockmap",
181	[BPF_MAP_TYPE_CPUMAP] = "cpumap",
182	[BPF_MAP_TYPE_XSKMAP] = "xskmap",
183	[BPF_MAP_TYPE_SOCKHASH] = "sockhash",
184	[BPF_MAP_TYPE_CGROUP_STORAGE] = "cgroup_storage",
185	[BPF_MAP_TYPE_REUSEPORT_SOCKARRAY] = "reuseport_sockarray",
186	[BPF_MAP_TYPE_PERCPU_CGROUP_STORAGE] = "percpu_cgroup_storage",
187	[BPF_MAP_TYPE_QUEUE] = "queue",
188	[BPF_MAP_TYPE_STACK] = "stack",
189	[BPF_MAP_TYPE_SK_STORAGE] = "sk_storage",
190	[BPF_MAP_TYPE_STRUCT_OPS] = "struct_ops",
191	[BPF_MAP_TYPE_RINGBUF] = "ringbuf",
192	[BPF_MAP_TYPE_INODE_STORAGE] = "inode_storage",
193	[BPF_MAP_TYPE_TASK_STORAGE] = "task_storage",
194	[BPF_MAP_TYPE_BLOOM_FILTER] = "bloom_filter",
195	[BPF_MAP_TYPE_USER_RINGBUF] = "user_ringbuf",
196	[BPF_MAP_TYPE_CGRP_STORAGE] = "cgrp_storage",
197	[BPF_MAP_TYPE_ARENA] = "arena",
198	[BPF_MAP_TYPE_INSN_ARRAY] = "insn_array",
199	[BPF_MAP_TYPE_RHASH] = "rhash",
200	};
201
202	static const char * const prog_type_name[] = {
203	[BPF_PROG_TYPE_UNSPEC] = "unspec",
204	[BPF_PROG_TYPE_SOCKET_FILTER] = "socket_filter",
205	[BPF_PROG_TYPE_KPROBE] = "kprobe",
206	[BPF_PROG_TYPE_SCHED_CLS] = "sched_cls",
207	[BPF_PROG_TYPE_SCHED_ACT] = "sched_act",
208	[BPF_PROG_TYPE_TRACEPOINT] = "tracepoint",
209	[BPF_PROG_TYPE_XDP] = "xdp",
210	[BPF_PROG_TYPE_PERF_EVENT] = "perf_event",
211	[BPF_PROG_TYPE_CGROUP_SKB] = "cgroup_skb",
212	[BPF_PROG_TYPE_CGROUP_SOCK] = "cgroup_sock",
213	[BPF_PROG_TYPE_LWT_IN] = "lwt_in",
214	[BPF_PROG_TYPE_LWT_OUT] = "lwt_out",
215	[BPF_PROG_TYPE_LWT_XMIT] = "lwt_xmit",
216	[BPF_PROG_TYPE_SOCK_OPS] = "sock_ops",
217	[BPF_PROG_TYPE_SK_SKB] = "sk_skb",
218	[BPF_PROG_TYPE_CGROUP_DEVICE] = "cgroup_device",
219	[BPF_PROG_TYPE_SK_MSG] = "sk_msg",
220	[BPF_PROG_TYPE_RAW_TRACEPOINT] = "raw_tracepoint",
221	[BPF_PROG_TYPE_CGROUP_SOCK_ADDR] = "cgroup_sock_addr",
222	[BPF_PROG_TYPE_LWT_SEG6LOCAL] = "lwt_seg6local",
223	[BPF_PROG_TYPE_LIRC_MODE2] = "lirc_mode2",
224	[BPF_PROG_TYPE_SK_REUSEPORT] = "sk_reuseport",
225	[BPF_PROG_TYPE_FLOW_DISSECTOR] = "flow_dissector",
226	[BPF_PROG_TYPE_CGROUP_SYSCTL] = "cgroup_sysctl",
227	[BPF_PROG_TYPE_RAW_TRACEPOINT_WRITABLE] = "raw_tracepoint_writable",
228	[BPF_PROG_TYPE_CGROUP_SOCKOPT] = "cgroup_sockopt",
229	[BPF_PROG_TYPE_TRACING] = "tracing",
230	[BPF_PROG_TYPE_STRUCT_OPS] = "struct_ops",
231	[BPF_PROG_TYPE_EXT] = "ext",
232	[BPF_PROG_TYPE_LSM] = "lsm",
233	[BPF_PROG_TYPE_SK_LOOKUP] = "sk_lookup",
234	[BPF_PROG_TYPE_SYSCALL] = "syscall",
235	[BPF_PROG_TYPE_NETFILTER] = "netfilter",
236	};
237
238	static int __base_pr(enum libbpf_print_level level, const char *format,
239	va_list args)
240	{
241	const char *env_var = "LIBBPF_LOG_LEVEL";
242	static enum libbpf_print_level min_level = LIBBPF_INFO;
243	static bool initialized;
244
245	if (!initialized) {
246	char *verbosity;
247
248	initialized = true;
249	verbosity = getenv(env_var);
250	if (verbosity) {
251	if (strcasecmp(verbosity, "warn") == 0)
252	min_level = LIBBPF_WARN;
253	else if (strcasecmp(verbosity, "debug") == 0)
254	min_level = LIBBPF_DEBUG;
255	else if (strcasecmp(verbosity, "info") == 0)
256	min_level = LIBBPF_INFO;
257	else
258	fprintf(stderr, "libbpf: unrecognized '%s' envvar value: '%s', should be one of 'warn', 'debug', or 'info'.\n",
259	env_var, verbosity);
260	}
261	}
262
263	/* if too verbose, skip logging */
264	if (level > min_level)
265	return 0;
266
267	return vfprintf(stderr, format, args);
268	}
269
270	static libbpf_print_fn_t __libbpf_pr = __base_pr;
271
272	libbpf_print_fn_t libbpf_set_print(libbpf_print_fn_t fn)
273	{
274	libbpf_print_fn_t old_print_fn;
275
276	old_print_fn = __atomic_exchange_n(&__libbpf_pr, fn, __ATOMIC_RELAXED);
277
278	return old_print_fn;
279	}
280
281	__printf(2, 3)
282	void libbpf_print(enum libbpf_print_level level, const char *format, ...)
283	{
284	va_list args;
285	int old_errno;
286	libbpf_print_fn_t print_fn;
287
288	print_fn = __atomic_load_n(&__libbpf_pr, __ATOMIC_RELAXED);
289	if (!print_fn)
290	return;
291
292	old_errno = errno;
293
294	va_start(args, format);
295	print_fn(level, format, args);
296	va_end(args);
297
298	errno = old_errno;
299	}
300
301	static void pr_perm_msg(int err)
302	{
303	struct rlimit limit;
304	char buf[100];
305
306	if (err != -EPERM \|\| geteuid() != 0)
307	return;
308
309	err = getrlimit(RLIMIT_MEMLOCK, &limit);
310	if (err)
311	return;
312
313	if (limit.rlim_cur == RLIM_INFINITY)
314	return;
315
316	if (limit.rlim_cur < 1024)
317	snprintf(buf, sizeof(buf), "%zu bytes", (size_t)limit.rlim_cur);
318	else if (limit.rlim_cur < 1024*1024)
319	snprintf(buf, sizeof(buf), "%.1f KiB", (double)limit.rlim_cur / 1024);
320	else
321	snprintf(buf, sizeof(buf), "%.1f MiB", (double)limit.rlim_cur / (1024*1024));
322
323	pr_warn("permission error while running as root; try raising 'ulimit -l'? current value: %s\n",
324	buf);
325	}
326
327	/* Copied from tools/perf/util/util.h */
328	#ifndef zfree
329	# define zfree(ptr) ({ free(ptr); ptr = NULL; })
330	#endif
331
332	#ifndef zclose
333	# define zclose(fd) ({ \
334	int ___err = 0; \
335	if ((fd) >= 0) \
336	___err = close((fd)); \
337	fd = -1; \
338	___err; })
339	#endif
340
341	static inline __u64 ptr_to_u64(const void *ptr)
342	{
343	return (__u64) (unsigned long) ptr;
344	}
345
346	int libbpf_set_strict_mode(enum libbpf_strict_mode mode)
347	{
348	/* as of v1.0 libbpf_set_strict_mode() is a no-op */
349	return 0;
350	}
351
352	__u32 libbpf_major_version(void)
353	{
354	return LIBBPF_MAJOR_VERSION;
355	}
356
357	__u32 libbpf_minor_version(void)
358	{
359	return LIBBPF_MINOR_VERSION;
360	}
361
362	const char *libbpf_version_string(void)
363	{
364	#define __S(X) #X
365	#define _S(X) __S(X)
366	return "v" _S(LIBBPF_MAJOR_VERSION) "." _S(LIBBPF_MINOR_VERSION);
367	#undef _S
368	#undef __S
369	}
370
371	enum reloc_type {
372	RELO_LD64,
373	RELO_CALL,
374	RELO_DATA,
375	RELO_EXTERN_LD64,
376	RELO_EXTERN_CALL,
377	RELO_SUBPROG_ADDR,
378	RELO_CORE,
379	RELO_INSN_ARRAY,
380	};
381
382	struct reloc_desc {
383	enum reloc_type type;
384	int insn_idx;
385	union {
386	const struct bpf_core_relo core_relo; / used when type == RELO_CORE */
387	struct {
388	int map_idx;
389	unsigned int sym_off;
390	/*
391	* The following two fields can be unionized, as the
392	* ext_idx field is used for extern symbols, and the
393	* sym_size is used for jump tables, which are never
394	* extern
395	*/
396	union {
397	int ext_idx;
398	int sym_size;
399	};
400	};
401	};
402	};
403
404	/* stored as sec_def->cookie for all libbpf-supported SEC()s */
405	enum sec_def_flags {
406	SEC_NONE = 0,
407	/* expected_attach_type is optional, if kernel doesn't support that */
408	SEC_EXP_ATTACH_OPT = 1,
409	/* legacy, only used by libbpf_get_type_names() and
410	* libbpf_attach_type_by_name(), not used by libbpf itself at all.
411	* This used to be associated with cgroup (and few other) BPF programs
412	* that were attachable through BPF_PROG_ATTACH command. Pretty
413	* meaningless nowadays, though.
414	*/
415	SEC_ATTACHABLE = 2,
416	SEC_ATTACHABLE_OPT = SEC_ATTACHABLE \| SEC_EXP_ATTACH_OPT,
417	/* attachment target is specified through BTF ID in either kernel or
418	* other BPF program's BTF object
419	*/
420	SEC_ATTACH_BTF = 4,
421	/* BPF program type allows sleeping/blocking in kernel */
422	SEC_SLEEPABLE = 8,
423	/* BPF program support non-linear XDP buffer */
424	SEC_XDP_FRAGS = 16,
425	/* Setup proper attach type for usdt probes. */
426	SEC_USDT = 32,
427	};
428
429	struct bpf_sec_def {
430	char *sec;
431	enum bpf_prog_type prog_type;
432	enum bpf_attach_type expected_attach_type;
433	long cookie;
434	int handler_id;
435
436	libbpf_prog_setup_fn_t prog_setup_fn;
437	libbpf_prog_prepare_load_fn_t prog_prepare_load_fn;
438	libbpf_prog_attach_fn_t prog_attach_fn;
439	};
440
441	struct bpf_light_subprog {
442	__u32 sec_insn_off;
443	__u32 sub_insn_off;
444	};
445
446	/*
447	* bpf_prog should be a better name but it has been used in
448	* linux/filter.h.
449	*/
450	struct bpf_program {
451	char *name;
452	char *sec_name;
453	size_t sec_idx;
454	const struct bpf_sec_def *sec_def;
455	/* this program's instruction offset (in number of instructions)
456	* within its containing ELF section
457	*/
458	size_t sec_insn_off;
459	/* number of original instructions in ELF section belonging to this
460	* program, not taking into account subprogram instructions possible
461	* appended later during relocation
462	*/
463	size_t sec_insn_cnt;
464	/* Offset (in number of instructions) of the start of instruction
465	* belonging to this BPF program within its containing main BPF
466	* program. For the entry-point (main) BPF program, this is always
467	* zero. For a sub-program, this gets reset before each of main BPF
468	* programs are processed and relocated and is used to determined
469	* whether sub-program was already appended to the main program, and
470	* if yes, at which instruction offset.
471	*/
472	size_t sub_insn_off;
473
474	/* instructions that belong to BPF program; insns[0] is located at
475	* sec_insn_off instruction within its ELF section in ELF file, so
476	* when mapping ELF file instruction index to the local instruction,
477	* one needs to subtract sec_insn_off; and vice versa.
478	*/
479	struct bpf_insn *insns;
480	/* actual number of instruction in this BPF program's image; for
481	* entry-point BPF programs this includes the size of main program
482	* itself plus all the used sub-programs, appended at the end
483	*/
484	size_t insns_cnt;
485
486	struct reloc_desc *reloc_desc;
487	int nr_reloc;
488
489	/* BPF verifier log settings */
490	char *log_buf;
491	size_t log_size;
492	__u32 log_level;
493
494	struct bpf_object *obj;
495
496	int fd;
497	bool autoload;
498	bool autoattach;
499	bool sym_global;
500	bool mark_btf_static;
501	enum bpf_prog_type type;
502	enum bpf_attach_type expected_attach_type;
503	int exception_cb_idx;
504
505	int prog_ifindex;
506	__u32 attach_btf_obj_fd;
507	__u32 attach_btf_id;
508	__u32 attach_prog_fd;
509
510	void *func_info;
511	__u32 func_info_rec_size;
512	__u32 func_info_cnt;
513
514	void *line_info;
515	__u32 line_info_rec_size;
516	__u32 line_info_cnt;
517	__u32 prog_flags;
518	__u8 hash[SHA256_DIGEST_LENGTH];
519
520	struct bpf_light_subprog *subprogs;
521	__u32 subprog_cnt;
522	};
523
524	struct bpf_struct_ops {
525	struct bpf_program **progs;
526	__u32 *kern_func_off;
527	/* e.g. struct tcp_congestion_ops in bpf_prog's btf format */
528	void *data;
529	/* e.g. struct bpf_struct_ops_tcp_congestion_ops in
530	* btf_vmlinux's format.
531	* struct bpf_struct_ops_tcp_congestion_ops {
532	* [... some other kernel fields ...]
533	* struct tcp_congestion_ops data;
534	* }
535	* kern_vdata-size == sizeof(struct bpf_struct_ops_tcp_congestion_ops)
536	* bpf_map__init_kern_struct_ops() will populate the "kern_vdata"
537	* from "data".
538	*/
539	void *kern_vdata;
540	__u32 type_id;
541	};
542
543	#define DATA_SEC ".data"
544	#define PERCPU_SEC ".percpu"
545	#define BSS_SEC ".bss"
546	#define RODATA_SEC ".rodata"
547	#define KCONFIG_SEC ".kconfig"
548	#define KSYMS_SEC ".ksyms"
549	#define STRUCT_OPS_SEC ".struct_ops"
550	#define STRUCT_OPS_LINK_SEC ".struct_ops.link"
551	#define ARENA_SEC ".addr_space.1"
552
553	enum libbpf_map_type {
554	LIBBPF_MAP_UNSPEC,
555	LIBBPF_MAP_DATA,
556	LIBBPF_MAP_BSS,
557	LIBBPF_MAP_RODATA,
558	LIBBPF_MAP_KCONFIG,
559	LIBBPF_MAP_PERCPU,
560	};
561
562	struct bpf_map_def {
563	unsigned int type;
564	unsigned int key_size;
565	unsigned int value_size;
566	unsigned int max_entries;
567	unsigned int map_flags;
568	};
569
570	struct bpf_map {
571	struct bpf_object *obj;
572	char *name;
573	/* real_name is defined for special internal maps (.rodata*,
574	* .data*, .bss, .kconfig) and preserves their original ELF section
575	* name. This is important to be able to find corresponding BTF
576	* DATASEC information.
577	*/
578	char *real_name;
579	int fd;
580	int sec_idx;
581	size_t sec_offset;
582	int map_ifindex;
583	int inner_map_fd;
584	struct bpf_map_def def;
585	__u32 numa_node;
586	__u32 btf_var_idx;
587	int mod_btf_fd;
588	__u32 btf_key_type_id;
589	__u32 btf_value_type_id;
590	__u32 btf_vmlinux_value_type_id;
591	enum libbpf_map_type libbpf_type;
592	void *mmaped;
593	struct bpf_struct_ops *st_ops;
594	struct bpf_map *inner_map;
595	void **init_slots;
596	int init_slots_sz;
597	char *pin_path;
598	bool pinned;
599	bool reused;
600	bool autocreate;
601	bool autoattach;
602	__u64 map_extra;
603	struct bpf_program *excl_prog;
604	};
605
606	enum extern_type {
607	EXT_UNKNOWN,
608	EXT_KCFG,
609	EXT_KSYM,
610	};
611
612	enum kcfg_type {
613	KCFG_UNKNOWN,
614	KCFG_CHAR,
615	KCFG_BOOL,
616	KCFG_INT,
617	KCFG_TRISTATE,
618	KCFG_CHAR_ARR,
619	};
620
621	struct extern_desc {
622	enum extern_type type;
623	int sym_idx;
624	int btf_id;
625	int sec_btf_id;
626	char *name;
627	char *essent_name;
628	bool is_set;
629	bool is_weak;
630	union {
631	struct {
632	enum kcfg_type type;
633	int sz;
634	int align;
635	int data_off;
636	bool is_signed;
637	} kcfg;
638	struct {
639	unsigned long long addr;
640
641	/* target btf_id of the corresponding kernel var. */
642	int kernel_btf_obj_fd;
643	int kernel_btf_id;
644
645	/* local btf_id of the ksym extern's type. */
646	__u32 type_id;
647	/* BTF fd index to be patched in for insn->off, this is
648	* 0 for vmlinux BTF, index in obj->fd_array for module
649	* BTF
650	*/
651	__s16 btf_fd_idx;
652	} ksym;
653	};
654	};
655
656	struct module_btf {
657	struct btf *btf;
658	char *name;
659	__u32 id;
660	int fd;
661	int fd_array_idx;
662	};
663
664	enum sec_type {
665	SEC_UNUSED = 0,
666	SEC_RELO,
667	SEC_BSS,
668	SEC_DATA,
669	SEC_RODATA,
670	SEC_ST_OPS,
671	SEC_PERCPU,
672	};
673
674	struct elf_sec_desc {
675	enum sec_type sec_type;
676	Elf64_Shdr *shdr;
677	Elf_Data *data;
678	};
679
680	struct elf_state {
681	int fd;
682	const void *obj_buf;
683	size_t obj_buf_sz;
684	Elf *elf;
685	Elf64_Ehdr *ehdr;
686	Elf_Data *symbols;
687	Elf_Data *arena_data;
688	size_t shstrndx; /* section index for section name strings */
689	size_t strtabidx;
690	struct elf_sec_desc *secs;
691	size_t sec_cnt;
692	int btf_maps_shndx;
693	__u32 btf_maps_sec_btf_id;
694	int text_shndx;
695	int symbols_shndx;
696	bool has_st_ops;
697	int arena_data_shndx;
698	int jumptables_data_shndx;
699	};
700
701	struct usdt_manager;
702
703	enum bpf_object_state {
704	OBJ_OPEN,
705	OBJ_PREPARED,
706	OBJ_LOADED,
707	};
708
709	struct bpf_object {
710	char name[BPF_OBJ_NAME_LEN];
711	char license[64];
712	__u32 kern_version;
713
714	enum bpf_object_state state;
715	struct bpf_program *programs;
716	size_t nr_programs;
717	struct bpf_map *maps;
718	size_t nr_maps;
719	size_t maps_cap;
720
721	char *kconfig;
722	struct extern_desc *externs;
723	int nr_extern;
724	int kconfig_map_idx;
725
726	bool has_subcalls;
727	bool has_rodata;
728
729	struct bpf_gen *gen_loader;
730
731	/* Information when doing ELF related work. Only valid if efile.elf is not NULL */
732	struct elf_state efile;
733
734	unsigned char byteorder;
735
736	struct btf *btf;
737	struct btf_ext *btf_ext;
738
739	/* Parse and load BTF vmlinux if any of the programs in the object need
740	* it at load time.
741	*/
742	struct btf *btf_vmlinux;
743	/* Path to the custom BTF to be used for BPF CO-RE relocations as an
744	* override for vmlinux BTF.
745	*/
746	char *btf_custom_path;
747	/* vmlinux BTF override for CO-RE relocations */
748	struct btf *btf_vmlinux_override;
749	/* Lazily initialized kernel module BTFs */
750	struct module_btf *btf_modules;
751	bool btf_modules_loaded;
752	size_t btf_module_cnt;
753	size_t btf_module_cap;
754
755	/* optional log settings passed to BPF_BTF_LOAD and BPF_PROG_LOAD commands */
756	char *log_buf;
757	size_t log_size;
758	__u32 log_level;
759
760	int *fd_array;
761	size_t fd_array_cap;
762	size_t fd_array_cnt;
763
764	struct usdt_manager *usdt_man;
765
766	int arena_map_idx;
767	void *arena_data;
768	size_t arena_data_sz;
769	size_t arena_data_off;
770
771	void *jumptables_data;
772	size_t jumptables_data_sz;
773
774	struct {
775	struct bpf_program *prog;
776	unsigned int sym_off;
777	int fd;
778	} *jumptable_maps;
779	size_t jumptable_map_cnt;
780
781	struct kern_feature_cache *feat_cache;
782	char *token_path;
783	int token_fd;
784
785	char path[];
786	};
787
788	static const char elf_sym_str(const struct bpf_object obj, size_t off);
789	static const char elf_sec_str(const struct bpf_object obj, size_t off);
790	static Elf_Scn elf_sec_by_idx(const struct bpf_object obj, size_t idx);
791	static Elf_Scn elf_sec_by_name(const struct bpf_object obj, const char *name);
792	static Elf64_Shdr elf_sec_hdr(const struct bpf_object obj, Elf_Scn *scn);
793	static const char elf_sec_name(const struct bpf_object obj, Elf_Scn *scn);
794	static Elf_Data elf_sec_data(const struct bpf_object obj, Elf_Scn *scn);
795	static Elf64_Sym elf_sym_by_idx(const struct bpf_object obj, size_t idx);
796	static Elf64_Rel elf_rel_by_idx(Elf_Data data, size_t idx);
797
798	void bpf_program__unload(struct bpf_program *prog)
799	{
800	if (!prog)
801	return;
802
803	zclose(prog->fd);
804
805	zfree(&prog->func_info);
806	zfree(&prog->line_info);
807	zfree(&prog->subprogs);
808	}
809
810	static void bpf_program__exit(struct bpf_program *prog)
811	{
812	if (!prog)
813	return;
814
815	bpf_program__unload(prog);
816	zfree(&prog->name);
817	zfree(&prog->sec_name);
818	zfree(&prog->insns);
819	zfree(&prog->reloc_desc);
820
821	prog->nr_reloc = 0;
822	prog->insns_cnt = 0;
823	prog->sec_idx = -1;
824	}
825
826	static bool insn_is_subprog_call(const struct bpf_insn *insn)
827	{
828	return BPF_CLASS(insn->code) == BPF_JMP &&
829	BPF_OP(insn->code) == BPF_CALL &&
830	BPF_SRC(insn->code) == BPF_K &&
831	insn->src_reg == BPF_PSEUDO_CALL &&
832	insn->dst_reg == 0 &&
833	insn->off == 0;
834	}
835
836	static bool is_call_insn(const struct bpf_insn *insn)
837	{
838	return insn->code == (BPF_JMP \| BPF_CALL);
839	}
840
841	static bool insn_is_pseudo_func(struct bpf_insn *insn)
842	{
843	return is_ldimm64_insn(insn) && insn->src_reg == BPF_PSEUDO_FUNC;
844	}
845
846	static int
847	bpf_object__init_prog(struct bpf_object obj, struct bpf_program prog,
848	const char name, size_t sec_idx, const char sec_name,
849	size_t sec_off, void *insn_data, size_t insn_data_sz)
850	{
851	if (insn_data_sz == 0 \|\| insn_data_sz % BPF_INSN_SZ \|\| sec_off % BPF_INSN_SZ) {
852	pr_warn("sec '%s': corrupted program '%s', offset %zu, size %zu\n",
853	sec_name, name, sec_off, insn_data_sz);
854	return -EINVAL;
855	}
856
857	memset(prog, 0, sizeof(*prog));
858	prog->obj = obj;
859
860	prog->sec_idx = sec_idx;
861	prog->sec_insn_off = sec_off / BPF_INSN_SZ;
862	prog->sec_insn_cnt = insn_data_sz / BPF_INSN_SZ;
863	/* insns_cnt can later be increased by appending used subprograms */
864	prog->insns_cnt = prog->sec_insn_cnt;
865
866	prog->type = BPF_PROG_TYPE_UNSPEC;
867	prog->fd = -1;
868	prog->exception_cb_idx = -1;
869
870	/* libbpf's convention for SEC("?abc...") is that it's just like
871	* SEC("abc...") but the corresponding bpf_program starts out with
872	* autoload set to false.
873	*/
874	if (sec_name[0] == '?') {
875	prog->autoload = false;
876	/* from now on forget there was ? in section name */
877	sec_name++;
878	} else {
879	prog->autoload = true;
880	}
881
882	prog->autoattach = true;
883
884	/* inherit object's log_level */
885	prog->log_level = obj->log_level;
886
887	prog->sec_name = strdup(sec_name);
888	if (!prog->sec_name)
889	goto errout;
890
891	prog->name = strdup(name);
892	if (!prog->name)
893	goto errout;
894
895	prog->insns = malloc(insn_data_sz);
896	if (!prog->insns)
897	goto errout;
898	memcpy(prog->insns, insn_data, insn_data_sz);
899
900	return 0;
901	errout:
902	pr_warn("sec '%s': failed to allocate memory for prog '%s'\n", sec_name, name);
903	bpf_program__exit(prog);
904	return -ENOMEM;
905	}
906
907	static int
908	bpf_object__add_programs(struct bpf_object obj, Elf_Data sec_data,
909	const char *sec_name, int sec_idx)
910	{
911	Elf_Data *symbols = obj->efile.symbols;
912	struct bpf_program prog, progs;
913	void *data = sec_data->d_buf;
914	size_t sec_sz = sec_data->d_size, sec_off, prog_sz, nr_syms;
915	int nr_progs, err, i;
916	const char *name;
917	Elf64_Sym *sym;
918
919	progs = obj->programs;
920	nr_progs = obj->nr_programs;
921	nr_syms = symbols->d_size / sizeof(Elf64_Sym);
922
923	for (i = 0; i < nr_syms; i++) {
924	sym = elf_sym_by_idx(obj, i);
925
926	if (sym->st_shndx != sec_idx)
927	continue;
928	if (ELF64_ST_TYPE(sym->st_info) != STT_FUNC)
929	continue;
930
931	prog_sz = sym->st_size;
932	sec_off = sym->st_value;
933
934	name = elf_sym_str(obj, sym->st_name);
935	if (!name) {
936	pr_warn("sec '%s': failed to get symbol name for offset %zu\n",
937	sec_name, sec_off);
938	return -LIBBPF_ERRNO__FORMAT;
939	}
940
941	if (sec_off + prog_sz > sec_sz \|\| sec_off + prog_sz < sec_off) {
942	pr_warn("sec '%s': program at offset %zu crosses section boundary\n",
943	sec_name, sec_off);
944	return -LIBBPF_ERRNO__FORMAT;
945	}
946
947	if (sec_idx != obj->efile.text_shndx && ELF64_ST_BIND(sym->st_info) == STB_LOCAL) {
948	pr_warn("sec '%s': program '%s' is static and not supported\n", sec_name, name);
949	return -ENOTSUP;
950	}
951
952	pr_debug("sec '%s': found program '%s' at insn offset %zu (%zu bytes), code size %zu insns (%zu bytes)\n",
953	sec_name, name, sec_off / BPF_INSN_SZ, sec_off, prog_sz / BPF_INSN_SZ, prog_sz);
954
955	progs = libbpf_reallocarray(progs, nr_progs + 1, sizeof(*progs));
956	if (!progs) {
957	/*
958	* In this case the original obj->programs
959	* is still valid, so don't need special treat for
960	* bpf_close_object().
961	*/
962	pr_warn("sec '%s': failed to alloc memory for new program '%s'\n",
963	sec_name, name);
964	return -ENOMEM;
965	}
966	obj->programs = progs;
967
968	prog = &progs[nr_progs];
969
970	err = bpf_object__init_prog(obj, prog, name, sec_idx, sec_name,
971	sec_off, data + sec_off, prog_sz);
972	if (err)
973	return err;
974
975	if (ELF64_ST_BIND(sym->st_info) != STB_LOCAL)
976	prog->sym_global = true;
977
978	/* if function is a global/weak symbol, but has restricted
979	* (STV_HIDDEN or STV_INTERNAL) visibility, mark its BTF FUNC
980	* as static to enable more permissive BPF verification mode
981	* with more outside context available to BPF verifier
982	*/
983	if (prog->sym_global && (ELF64_ST_VISIBILITY(sym->st_other) == STV_HIDDEN
984	\|\| ELF64_ST_VISIBILITY(sym->st_other) == STV_INTERNAL))
985	prog->mark_btf_static = true;
986
987	nr_progs++;
988	obj->nr_programs = nr_progs;
989	}
990
991	return 0;
992	}
993
994	static void bpf_object_bswap_progs(struct bpf_object *obj)
995	{
996	struct bpf_program *prog = obj->programs;
997	struct bpf_insn *insn;
998	int p, i;
999
1000	for (p = 0; p < obj->nr_programs; p++, prog++) {
1001	insn = prog->insns;
1002	for (i = 0; i < prog->insns_cnt; i++, insn++)
1003	bpf_insn_bswap(insn);
1004	}
1005	pr_debug("converted %zu BPF programs to native byte order\n", obj->nr_programs);
1006	}
1007
1008	static const struct btf_member *
1009	find_member_by_offset(const struct btf_type *t, __u32 bit_offset)
1010	{
1011	struct btf_member *m;
1012	int i;
1013
1014	for (i = 0, m = btf_members(t); i < btf_vlen(t); i++, m++) {
1015	if (btf_member_bit_offset(t, i) == bit_offset)
1016	return m;
1017	}
1018
1019	return NULL;
1020	}
1021
1022	static const struct btf_member *
1023	find_member_by_name(const struct btf btf, const struct btf_type t,
1024	const char *name)
1025	{
1026	struct btf_member *m;
1027	int i;
1028
1029	for (i = 0, m = btf_members(t); i < btf_vlen(t); i++, m++) {
1030	if (!strcmp(btf__name_by_offset(btf, m->name_off), name))
1031	return m;
1032	}
1033
1034	return NULL;
1035	}
1036
1037	static int find_ksym_btf_id(struct bpf_object obj, const char ksym_name,
1038	__u16 kind, struct btf **res_btf,
1039	struct module_btf **res_mod_btf);
1040
1041	#define STRUCT_OPS_VALUE_PREFIX "bpf_struct_ops_"
1042	static int find_btf_by_prefix_kind(const struct btf btf, const char prefix,
1043	const char *name, __u32 kind);
1044
1045	static int
1046	find_struct_ops_kern_types(struct bpf_object obj, const char tname_raw,
1047	struct module_btf **mod_btf,
1048	const struct btf_type *type, __u32 type_id,
1049	const struct btf_type *vtype, __u32 vtype_id,
1050	const struct btf_member **data_member)
1051	{
1052	const struct btf_type kern_type, kern_vtype;
1053	const struct btf_member *kern_data_member;
1054	struct btf *btf = NULL;
1055	__s32 kern_vtype_id, kern_type_id;
1056	char tname[192], stname[256];
1057	__u32 i;
1058
1059	snprintf(tname, sizeof(tname), "%.*s",
1060	(int)bpf_core_essential_name_len(tname_raw), tname_raw);
1061
1062	snprintf(stname, sizeof(stname), "%s%s", STRUCT_OPS_VALUE_PREFIX, tname);
1063
1064	/* Look for the corresponding "map_value" type that will be used
1065	* in map_update(BPF_MAP_TYPE_STRUCT_OPS) first, figure out the btf
1066	* and the mod_btf.
1067	* For example, find "struct bpf_struct_ops_tcp_congestion_ops".
1068	*/
1069	kern_vtype_id = find_ksym_btf_id(obj, stname, BTF_KIND_STRUCT, &btf, mod_btf);
1070	if (kern_vtype_id < 0) {
1071	pr_warn("struct_ops init_kern: struct %s is not found in kernel BTF\n", stname);
1072	return kern_vtype_id;
1073	}
1074	kern_vtype = btf__type_by_id(btf, kern_vtype_id);
1075
1076	kern_type_id = btf__find_by_name_kind(btf, tname, BTF_KIND_STRUCT);
1077	if (kern_type_id < 0) {
1078	pr_warn("struct_ops init_kern: struct %s is not found in kernel BTF\n", tname);
1079	return kern_type_id;
1080	}
1081	kern_type = btf__type_by_id(btf, kern_type_id);
1082
1083	/* Find "struct tcp_congestion_ops" from
1084	* struct bpf_struct_ops_tcp_congestion_ops {
1085	* [ ... ]
1086	* struct tcp_congestion_ops data;
1087	* }
1088	*/
1089	kern_data_member = btf_members(kern_vtype);
1090	for (i = 0; i < btf_vlen(kern_vtype); i++, kern_data_member++) {
1091	if (kern_data_member->type == kern_type_id)
1092	break;
1093	}
1094	if (i == btf_vlen(kern_vtype)) {
1095	pr_warn("struct_ops init_kern: struct %s data is not found in struct %s\n",
1096	tname, stname);
1097	return -EINVAL;
1098	}
1099
1100	*type = kern_type;
1101	*type_id = kern_type_id;
1102	*vtype = kern_vtype;
1103	*vtype_id = kern_vtype_id;
1104	*data_member = kern_data_member;
1105
1106	return 0;
1107	}
1108
1109	static bool bpf_map__is_struct_ops(const struct bpf_map *map)
1110	{
1111	return map->def.type == BPF_MAP_TYPE_STRUCT_OPS;
1112	}
1113
1114	static bool is_valid_st_ops_program(struct bpf_object *obj,
1115	const struct bpf_program *prog)
1116	{
1117	int i;
1118
1119	for (i = 0; i < obj->nr_programs; i++) {
1120	if (&obj->programs[i] == prog)
1121	return prog->type == BPF_PROG_TYPE_STRUCT_OPS;
1122	}
1123
1124	return false;
1125	}
1126
1127	/* For each struct_ops program P, referenced from some struct_ops map M,
1128	* enable P.autoload if there are Ms for which M.autocreate is true,
1129	* disable P.autoload if for all Ms M.autocreate is false.
1130	* Don't change P.autoload for programs that are not referenced from any maps.
1131	*/
1132	static int bpf_object_adjust_struct_ops_autoload(struct bpf_object *obj)
1133	{
1134	struct bpf_program prog, slot_prog;
1135	struct bpf_map *map;
1136	int i, j, k, vlen;
1137
1138	for (i = 0; i < obj->nr_programs; ++i) {
1139	int should_load = false;
1140	int use_cnt = 0;
1141
1142	prog = &obj->programs[i];
1143	if (prog->type != BPF_PROG_TYPE_STRUCT_OPS)
1144	continue;
1145
1146	for (j = 0; j < obj->nr_maps; ++j) {
1147	const struct btf_type *type;
1148
1149	map = &obj->maps[j];
1150	if (!bpf_map__is_struct_ops(map))
1151	continue;
1152
1153	type = btf__type_by_id(obj->btf, map->st_ops->type_id);
1154	vlen = btf_vlen(type);
1155	for (k = 0; k < vlen; ++k) {
1156	slot_prog = map->st_ops->progs[k];
1157	if (prog != slot_prog)
1158	continue;
1159
1160	use_cnt++;
1161	if (map->autocreate)
1162	should_load = true;
1163	}
1164	}
1165	if (use_cnt)
1166	prog->autoload = should_load;
1167	}
1168
1169	return 0;
1170	}
1171
1172	/* Init the map's fields that depend on kern_btf */
1173	static int bpf_map__init_kern_struct_ops(struct bpf_map *map)
1174	{
1175	const struct btf_member member, kern_member, *kern_data_member;
1176	const struct btf_type type, kern_type, *kern_vtype;
1177	__u32 i, kern_type_id, kern_vtype_id, kern_data_off;
1178	struct bpf_object *obj = map->obj;
1179	const struct btf *btf = obj->btf;
1180	struct bpf_struct_ops *st_ops;
1181	const struct btf *kern_btf;
1182	struct module_btf *mod_btf = NULL;
1183	void data, kern_data;
1184	const char *tname;
1185	int err;
1186
1187	st_ops = map->st_ops;
1188	type = btf__type_by_id(btf, st_ops->type_id);
1189	tname = btf__name_by_offset(btf, type->name_off);
1190	err = find_struct_ops_kern_types(obj, tname, &mod_btf,
1191	&kern_type, &kern_type_id,
1192	&kern_vtype, &kern_vtype_id,
1193	&kern_data_member);
1194	if (err)
1195	return err;
1196
1197	kern_btf = mod_btf ? mod_btf->btf : obj->btf_vmlinux;
1198
1199	pr_debug("struct_ops init_kern %s: type_id:%u kern_type_id:%u kern_vtype_id:%u\n",
1200	map->name, st_ops->type_id, kern_type_id, kern_vtype_id);
1201
1202	map->mod_btf_fd = mod_btf ? mod_btf->fd : -1;
1203	map->def.value_size = kern_vtype->size;
1204	map->btf_vmlinux_value_type_id = kern_vtype_id;
1205
1206	st_ops->kern_vdata = calloc(1, kern_vtype->size);
1207	if (!st_ops->kern_vdata)
1208	return -ENOMEM;
1209
1210	data = st_ops->data;
1211	kern_data_off = kern_data_member->offset / 8;
1212	kern_data = st_ops->kern_vdata + kern_data_off;
1213
1214	member = btf_members(type);
1215	for (i = 0; i < btf_vlen(type); i++, member++) {
1216	const struct btf_type mtype, kern_mtype;
1217	__u32 mtype_id, kern_mtype_id;
1218	void mdata, kern_mdata;
1219	struct bpf_program *prog;
1220	__s64 msize, kern_msize;
1221	__u32 moff, kern_moff;
1222	__u32 kern_member_idx;
1223	const char *mname;
1224
1225	mname = btf__name_by_offset(btf, member->name_off);
1226	moff = member->offset / 8;
1227	mdata = data + moff;
1228	msize = btf__resolve_size(btf, member->type);
1229	if (msize < 0) {
1230	pr_warn("struct_ops init_kern %s: failed to resolve the size of member %s\n",
1231	map->name, mname);
1232	return msize;
1233	}
1234
1235	kern_member = find_member_by_name(kern_btf, kern_type, mname);
1236	if (!kern_member) {
1237	if (!libbpf_is_mem_zeroed(mdata, msize)) {
1238	pr_warn("struct_ops init_kern %s: Cannot find member %s in kernel BTF\n",
1239	map->name, mname);
1240	return -ENOTSUP;
1241	}
1242
1243	if (st_ops->progs[i]) {
1244	/* If we had declaratively set struct_ops callback, we need to
1245	* force its autoload to false, because it doesn't have
1246	* a chance of succeeding from POV of the current struct_ops map.
1247	* If this program is still referenced somewhere else, though,
1248	* then bpf_object_adjust_struct_ops_autoload() will update its
1249	* autoload accordingly.
1250	*/
1251	st_ops->progs[i]->autoload = false;
1252	st_ops->progs[i] = NULL;
1253	}
1254
1255	/* Skip all-zero/NULL fields if they are not present in the kernel BTF */
1256	pr_info("struct_ops %s: member %s not found in kernel, skipping it as it's set to zero\n",
1257	map->name, mname);
1258	continue;
1259	}
1260
1261	kern_member_idx = kern_member - btf_members(kern_type);
1262	if (btf_member_bitfield_size(type, i) \|\|
1263	btf_member_bitfield_size(kern_type, kern_member_idx)) {
1264	pr_warn("struct_ops init_kern %s: bitfield %s is not supported\n",
1265	map->name, mname);
1266	return -ENOTSUP;
1267	}
1268
1269	kern_moff = kern_member->offset / 8;
1270	kern_mdata = kern_data + kern_moff;
1271
1272	mtype = skip_mods_and_typedefs(btf, member->type, &mtype_id);
1273	kern_mtype = skip_mods_and_typedefs(kern_btf, kern_member->type,
1274	&kern_mtype_id);
1275	if (BTF_INFO_KIND(mtype->info) !=
1276	BTF_INFO_KIND(kern_mtype->info)) {
1277	pr_warn("struct_ops init_kern %s: Unmatched member type %s %u != %u(kernel)\n",
1278	map->name, mname, BTF_INFO_KIND(mtype->info),
1279	BTF_INFO_KIND(kern_mtype->info));
1280	return -ENOTSUP;
1281	}
1282
1283	if (btf_is_ptr(mtype)) {
1284	prog = (void *)mdata;
1285	/* just like for !kern_member case above, reset declaratively
1286	* set (at compile time) program's autload to false,
1287	* if user replaced it with another program or NULL
1288	*/
1289	if (st_ops->progs[i] && st_ops->progs[i] != prog)
1290	st_ops->progs[i]->autoload = false;
1291
1292	/* Update the value from the shadow type */
1293	st_ops->progs[i] = prog;
1294	if (!prog)
1295	continue;
1296
1297	if (!is_valid_st_ops_program(obj, prog)) {
1298	pr_warn("struct_ops init_kern %s: member %s is not a struct_ops program\n",
1299	map->name, mname);
1300	return -ENOTSUP;
1301	}
1302
1303	kern_mtype = skip_mods_and_typedefs(kern_btf,
1304	kern_mtype->type,
1305	&kern_mtype_id);
1306
1307	/* mtype->type must be a func_proto which was
1308	* guaranteed in bpf_object__collect_st_ops_relos(),
1309	* so only check kern_mtype for func_proto here.
1310	*/
1311	if (!btf_is_func_proto(kern_mtype)) {
1312	pr_warn("struct_ops init_kern %s: kernel member %s is not a func ptr\n",
1313	map->name, mname);
1314	return -ENOTSUP;
1315	}
1316
1317	if (mod_btf)
1318	prog->attach_btf_obj_fd = mod_btf->fd;
1319
1320	/* if we haven't yet processed this BPF program, record proper
1321	* attach_btf_id and member_idx
1322	*/
1323	if (!prog->attach_btf_id) {
1324	prog->attach_btf_id = kern_type_id;
1325	prog->expected_attach_type = kern_member_idx;
1326	}
1327
1328	/* struct_ops BPF prog can be re-used between multiple
1329	* .struct_ops & .struct_ops.link as long as it's the
1330	* same struct_ops struct definition and the same
1331	* function pointer field
1332	*/
1333	if (prog->attach_btf_id != kern_type_id) {
1334	pr_warn("struct_ops init_kern %s func ptr %s: invalid reuse of prog %s in sec %s with type %u: attach_btf_id %u != kern_type_id %u\n",
1335	map->name, mname, prog->name, prog->sec_name, prog->type,
1336	prog->attach_btf_id, kern_type_id);
1337	return -EINVAL;
1338	}
1339	if (prog->expected_attach_type != kern_member_idx) {
1340	pr_warn("struct_ops init_kern %s func ptr %s: invalid reuse of prog %s in sec %s with type %u: expected_attach_type %u != kern_member_idx %u\n",
1341	map->name, mname, prog->name, prog->sec_name, prog->type,
1342	prog->expected_attach_type, kern_member_idx);
1343	return -EINVAL;
1344	}
1345
1346	st_ops->kern_func_off[i] = kern_data_off + kern_moff;
1347
1348	pr_debug("struct_ops init_kern %s: func ptr %s is set to prog %s from data(+%u) to kern_data(+%u)\n",
1349	map->name, mname, prog->name, moff,
1350	kern_moff);
1351
1352	continue;
1353	}
1354
1355	kern_msize = btf__resolve_size(kern_btf, kern_mtype_id);
1356	if (kern_msize < 0 \|\| msize != kern_msize) {
1357	pr_warn("struct_ops init_kern %s: Error in size of member %s: %zd != %zd(kernel)\n",
1358	map->name, mname, (ssize_t)msize,
1359	(ssize_t)kern_msize);
1360	return -ENOTSUP;
1361	}
1362
1363	pr_debug("struct_ops init_kern %s: copy %s %u bytes from data(+%u) to kern_data(+%u)\n",
1364	map->name, mname, (unsigned int)msize,
1365	moff, kern_moff);
1366	memcpy(kern_mdata, mdata, msize);
1367	}
1368
1369	return 0;
1370	}
1371
1372	static int bpf_object__init_kern_struct_ops_maps(struct bpf_object *obj)
1373	{
1374	struct bpf_map *map;
1375	size_t i;
1376	int err;
1377
1378	for (i = 0; i < obj->nr_maps; i++) {
1379	map = &obj->maps[i];
1380
1381	if (!bpf_map__is_struct_ops(map))
1382	continue;
1383
1384	if (!map->autocreate)
1385	continue;
1386
1387	err = bpf_map__init_kern_struct_ops(map);
1388	if (err)
1389	return err;
1390	}
1391
1392	return 0;
1393	}
1394
1395	static int init_struct_ops_maps(struct bpf_object obj, const char sec_name,
1396	int shndx, Elf_Data *data)
1397	{
1398	const struct btf_type type, datasec;
1399	const struct btf_var_secinfo *vsi;
1400	struct bpf_struct_ops *st_ops;
1401	const char tname, var_name;
1402	__s32 type_id, datasec_id;
1403	const struct btf *btf;
1404	struct bpf_map *map;
1405	__u32 i;
1406
1407	if (shndx == -1)
1408	return 0;
1409
1410	btf = obj->btf;
1411	datasec_id = btf__find_by_name_kind(btf, sec_name,
1412	BTF_KIND_DATASEC);
1413	if (datasec_id < 0) {
1414	pr_warn("struct_ops init: DATASEC %s not found\n",
1415	sec_name);
1416	return -EINVAL;
1417	}
1418
1419	datasec = btf__type_by_id(btf, datasec_id);
1420	vsi = btf_var_secinfos(datasec);
1421	for (i = 0; i < btf_vlen(datasec); i++, vsi++) {
1422	type = btf__type_by_id(obj->btf, vsi->type);
1423	var_name = btf__name_by_offset(obj->btf, type->name_off);
1424
1425	type_id = btf__resolve_type(obj->btf, vsi->type);
1426	if (type_id < 0) {
1427	pr_warn("struct_ops init: Cannot resolve var type_id %u in DATASEC %s\n",
1428	vsi->type, sec_name);
1429	return -EINVAL;
1430	}
1431
1432	type = btf__type_by_id(obj->btf, type_id);
1433	tname = btf__name_by_offset(obj->btf, type->name_off);
1434	if (!tname[0]) {
1435	pr_warn("struct_ops init: anonymous type is not supported\n");
1436	return -ENOTSUP;
1437	}
1438	if (!btf_is_struct(type)) {
1439	pr_warn("struct_ops init: %s is not a struct\n", tname);
1440	return -EINVAL;
1441	}
1442
1443	map = bpf_object__add_map(obj);
1444	if (IS_ERR(map))
1445	return PTR_ERR(map);
1446
1447	map->sec_idx = shndx;
1448	map->sec_offset = vsi->offset;
1449	map->name = strdup(var_name);
1450	if (!map->name)
1451	return -ENOMEM;
1452	map->btf_value_type_id = type_id;
1453
1454	/* Follow same convention as for programs autoload:
1455	* SEC("?.struct_ops") means map is not created by default.
1456	*/
1457	if (sec_name[0] == '?') {
1458	map->autocreate = false;
1459	/* from now on forget there was ? in section name */
1460	sec_name++;
1461	}
1462
1463	map->def.type = BPF_MAP_TYPE_STRUCT_OPS;
1464	map->def.key_size = sizeof(int);
1465	map->def.value_size = type->size;
1466	map->def.max_entries = 1;
1467	map->def.map_flags = strcmp(sec_name, STRUCT_OPS_LINK_SEC) == 0 ? BPF_F_LINK : 0;
1468	map->autoattach = true;
1469
1470	map->st_ops = calloc(1, sizeof(*map->st_ops));
1471	if (!map->st_ops)
1472	return -ENOMEM;
1473	st_ops = map->st_ops;
1474	st_ops->data = malloc(type->size);
1475	st_ops->progs = calloc(btf_vlen(type), sizeof(*st_ops->progs));
1476	st_ops->kern_func_off = malloc(btf_vlen(type) *
1477	sizeof(*st_ops->kern_func_off));
1478	if (!st_ops->data \|\| !st_ops->progs \|\| !st_ops->kern_func_off)
1479	return -ENOMEM;
1480
1481	if (vsi->offset + type->size > data->d_size) {
1482	pr_warn("struct_ops init: var %s is beyond the end of DATASEC %s\n",
1483	var_name, sec_name);
1484	return -EINVAL;
1485	}
1486
1487	memcpy(st_ops->data,
1488	data->d_buf + vsi->offset,
1489	type->size);
1490	st_ops->type_id = type_id;
1491
1492	pr_debug("struct_ops init: struct %s(type_id=%d) %s found at offset %u\n",
1493	tname, type_id, var_name, vsi->offset);
1494	}
1495
1496	return 0;
1497	}
1498
1499	static int bpf_object_init_struct_ops(struct bpf_object *obj)
1500	{
1501	const char *sec_name;
1502	int sec_idx, err;
1503
1504	for (sec_idx = 0; sec_idx < obj->efile.sec_cnt; ++sec_idx) {
1505	struct elf_sec_desc *desc = &obj->efile.secs[sec_idx];
1506
1507	if (desc->sec_type != SEC_ST_OPS)
1508	continue;
1509
1510	sec_name = elf_sec_name(obj, elf_sec_by_idx(obj, sec_idx));
1511	if (!sec_name)
1512	return -LIBBPF_ERRNO__FORMAT;
1513
1514	err = init_struct_ops_maps(obj, sec_name, sec_idx, desc->data);
1515	if (err)
1516	return err;
1517	}
1518
1519	return 0;
1520	}
1521
1522	static struct bpf_object bpf_object__new(const char path,
1523	const void *obj_buf,
1524	size_t obj_buf_sz,
1525	const char *obj_name)
1526	{
1527	struct bpf_object *obj;
1528	char *end;
1529
1530	obj = calloc(1, sizeof(struct bpf_object) + strlen(path) + 1);
1531	if (!obj) {
1532	pr_warn("alloc memory failed for %s\n", path);
1533	return ERR_PTR(-ENOMEM);
1534	}
1535
1536	strcpy(obj->path, path);
1537	if (obj_name) {
1538	libbpf_strlcpy(obj->name, obj_name, sizeof(obj->name));
1539	} else {
1540	/* Using basename() GNU version which doesn't modify arg. */
1541	libbpf_strlcpy(obj->name, basename((void *)path), sizeof(obj->name));
1542	end = strchr(obj->name, '.');
1543	if (end)
1544	*end = 0;
1545	}
1546
1547	obj->efile.fd = -1;
1548	/*
1549	* Caller of this function should also call
1550	* bpf_object__elf_finish() after data collection to return
1551	* obj_buf to user. If not, we should duplicate the buffer to
1552	* avoid user freeing them before elf finish.
1553	*/
1554	obj->efile.obj_buf = obj_buf;
1555	obj->efile.obj_buf_sz = obj_buf_sz;
1556	obj->efile.btf_maps_shndx = -1;
1557	obj->kconfig_map_idx = -1;
1558	obj->arena_map_idx = -1;
1559
1560	obj->kern_version = get_kernel_version();
1561	obj->state = OBJ_OPEN;
1562
1563	return obj;
1564	}
1565
1566	static void bpf_object__elf_finish(struct bpf_object *obj)
1567	{
1568	if (!obj->efile.elf)
1569	return;
1570
1571	elf_end(obj->efile.elf);
1572	obj->efile.elf = NULL;
1573	obj->efile.ehdr = NULL;
1574	obj->efile.symbols = NULL;
1575	obj->efile.arena_data = NULL;
1576
1577	zfree(&obj->efile.secs);
1578	obj->efile.sec_cnt = 0;
1579	zclose(obj->efile.fd);
1580	obj->efile.obj_buf = NULL;
1581	obj->efile.obj_buf_sz = 0;
1582	}
1583
1584	static int bpf_object__elf_init(struct bpf_object *obj)
1585	{
1586	Elf64_Ehdr *ehdr;
1587	int err = 0;
1588	Elf *elf;
1589
1590	if (obj->efile.elf) {
1591	pr_warn("elf: init internal error\n");
1592	return -LIBBPF_ERRNO__LIBELF;
1593	}
1594
1595	if (obj->efile.obj_buf_sz > 0) {
1596	/* obj_buf should have been validated by bpf_object__open_mem(). */
1597	elf = elf_memory((char *)obj->efile.obj_buf, obj->efile.obj_buf_sz);
1598	} else {
1599	obj->efile.fd = open(obj->path, O_RDONLY \| O_CLOEXEC);
1600	if (obj->efile.fd < 0) {
1601	err = -errno;
1602	pr_warn("elf: failed to open %s: %s\n", obj->path, errstr(err));
1603	return err;
1604	}
1605
1606	elf = elf_begin(obj->efile.fd, ELF_C_READ_MMAP, NULL);
1607	}
1608
1609	if (!elf) {
1610	pr_warn("elf: failed to open %s as ELF file: %s\n", obj->path, elf_errmsg(-1));
1611	err = -LIBBPF_ERRNO__LIBELF;
1612	goto errout;
1613	}
1614
1615	obj->efile.elf = elf;
1616
1617	if (elf_kind(elf) != ELF_K_ELF) {
1618	err = -LIBBPF_ERRNO__FORMAT;
1619	pr_warn("elf: '%s' is not a proper ELF object\n", obj->path);
1620	goto errout;
1621	}
1622
1623	if (gelf_getclass(elf) != ELFCLASS64) {
1624	err = -LIBBPF_ERRNO__FORMAT;
1625	pr_warn("elf: '%s' is not a 64-bit ELF object\n", obj->path);
1626	goto errout;
1627	}
1628
1629	obj->efile.ehdr = ehdr = elf64_getehdr(elf);
1630	if (!obj->efile.ehdr) {
1631	pr_warn("elf: failed to get ELF header from %s: %s\n", obj->path, elf_errmsg(-1));
1632	err = -LIBBPF_ERRNO__FORMAT;
1633	goto errout;
1634	}
1635
1636	/* Validate ELF object endianness... */
1637	if (ehdr->e_ident[EI_DATA] != ELFDATA2LSB &&
1638	ehdr->e_ident[EI_DATA] != ELFDATA2MSB) {
1639	err = -LIBBPF_ERRNO__ENDIAN;
1640	pr_warn("elf: '%s' has unknown byte order\n", obj->path);
1641	goto errout;
1642	}
1643	/* and save after bpf_object_open() frees ELF data */
1644	obj->byteorder = ehdr->e_ident[EI_DATA];
1645
1646	if (elf_getshdrstrndx(elf, &obj->efile.shstrndx)) {
1647	pr_warn("elf: failed to get section names section index for %s: %s\n",
1648	obj->path, elf_errmsg(-1));
1649	err = -LIBBPF_ERRNO__FORMAT;
1650	goto errout;
1651	}
1652
1653	/* ELF is corrupted/truncated, avoid calling elf_strptr. */
1654	if (!elf_rawdata(elf_getscn(elf, obj->efile.shstrndx), NULL)) {
1655	pr_warn("elf: failed to get section names strings from %s: %s\n",
1656	obj->path, elf_errmsg(-1));
1657	err = -LIBBPF_ERRNO__FORMAT;
1658	goto errout;
1659	}
1660
1661	/* Old LLVM set e_machine to EM_NONE */
1662	if (ehdr->e_type != ET_REL \|\| (ehdr->e_machine && ehdr->e_machine != EM_BPF)) {
1663	pr_warn("elf: %s is not a valid eBPF object file\n", obj->path);
1664	err = -LIBBPF_ERRNO__FORMAT;
1665	goto errout;
1666	}
1667
1668	return 0;
1669	errout:
1670	bpf_object__elf_finish(obj);
1671	return err;
1672	}
1673
1674	static bool is_native_endianness(struct bpf_object *obj)
1675	{
1676	#if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__
1677	return obj->byteorder == ELFDATA2LSB;
1678	#elif __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__
1679	return obj->byteorder == ELFDATA2MSB;
1680	#else
1681	# error "Unrecognized __BYTE_ORDER__"
1682	#endif
1683	}
1684
1685	static int
1686	bpf_object__init_license(struct bpf_object obj, void data, size_t size)
1687	{
1688	if (!data) {
1689	pr_warn("invalid license section in %s\n", obj->path);
1690	return -LIBBPF_ERRNO__FORMAT;
1691	}
1692	/* libbpf_strlcpy() only copies first N - 1 bytes, so size + 1 won't
1693	* go over allowed ELF data section buffer
1694	*/
1695	libbpf_strlcpy(obj->license, data, min(size + 1, sizeof(obj->license)));
1696	pr_debug("license of %s is %s\n", obj->path, obj->license);
1697	return 0;
1698	}
1699
1700	static int
1701	bpf_object__init_kversion(struct bpf_object obj, void data, size_t size)
1702	{
1703	__u32 kver;
1704
1705	if (!data \|\| size != sizeof(kver)) {
1706	pr_warn("invalid kver section in %s\n", obj->path);
1707	return -LIBBPF_ERRNO__FORMAT;
1708	}
1709	memcpy(&kver, data, sizeof(kver));
1710	obj->kern_version = kver;
1711	pr_debug("kernel version of %s is %x\n", obj->path, obj->kern_version);
1712	return 0;
1713	}
1714
1715	static bool bpf_map_type__is_map_in_map(enum bpf_map_type type)
1716	{
1717	if (type == BPF_MAP_TYPE_ARRAY_OF_MAPS \|\|
1718	type == BPF_MAP_TYPE_HASH_OF_MAPS)
1719	return true;
1720	return false;
1721	}
1722
1723	static int find_elf_sec_sz(const struct bpf_object obj, const char name, __u32 *size)
1724	{
1725	Elf_Data *data;
1726	Elf_Scn *scn;
1727
1728	if (!name)
1729	return -EINVAL;
1730
1731	scn = elf_sec_by_name(obj, name);
1732	data = elf_sec_data(obj, scn);
1733	if (data) {
1734	*size = data->d_size;
1735	return 0; /* found it */
1736	}
1737
1738	return -ENOENT;
1739	}
1740
1741	static Elf64_Sym find_elf_var_sym(const struct bpf_object obj, const char *name)
1742	{
1743	Elf_Data *symbols = obj->efile.symbols;
1744	const char *sname;
1745	size_t si;
1746
1747	for (si = 0; si < symbols->d_size / sizeof(Elf64_Sym); si++) {
1748	Elf64_Sym *sym = elf_sym_by_idx(obj, si);
1749
1750	if (ELF64_ST_TYPE(sym->st_info) != STT_OBJECT)
1751	continue;
1752
1753	if (ELF64_ST_BIND(sym->st_info) != STB_GLOBAL &&
1754	ELF64_ST_BIND(sym->st_info) != STB_WEAK)
1755	continue;
1756
1757	sname = elf_sym_str(obj, sym->st_name);
1758	if (!sname) {
1759	pr_warn("failed to get sym name string for var %s\n", name);
1760	return ERR_PTR(-EIO);
1761	}
1762	if (strcmp(name, sname) == 0)
1763	return sym;
1764	}
1765
1766	return ERR_PTR(-ENOENT);
1767	}
1768
1769	#ifndef MFD_CLOEXEC
1770	#define MFD_CLOEXEC 0x0001U
1771	#endif
1772	#ifndef MFD_NOEXEC_SEAL
1773	#define MFD_NOEXEC_SEAL 0x0008U
1774	#endif
1775
1776	static int create_placeholder_fd(void)
1777	{
1778	unsigned int flags = MFD_CLOEXEC \| MFD_NOEXEC_SEAL;
1779	const char *name = "libbpf-placeholder-fd";
1780	int fd;
1781
1782	fd = ensure_good_fd(sys_memfd_create(name, flags));
1783	if (fd >= 0)
1784	return fd;
1785	else if (errno != EINVAL)
1786	return -errno;
1787
1788	/* Possibly running on kernel without MFD_NOEXEC_SEAL */
1789	fd = ensure_good_fd(sys_memfd_create(name, flags & ~MFD_NOEXEC_SEAL));
1790	if (fd < 0)
1791	return -errno;
1792	return fd;
1793	}
1794
1795	static struct bpf_map bpf_object__add_map(struct bpf_object obj)
1796	{
1797	struct bpf_map *map;
1798	int err;
1799
1800	err = libbpf_ensure_mem((void **)&obj->maps, &obj->maps_cap,
1801	sizeof(*obj->maps), obj->nr_maps + 1);
1802	if (err)
1803	return ERR_PTR(err);
1804
1805	map = &obj->maps[obj->nr_maps++];
1806	map->obj = obj;
1807	/* Preallocate map FD without actually creating BPF map just yet.
1808	* These map FD "placeholders" will be reused later without changing
1809	* FD value when map is actually created in the kernel.
1810	*
1811	* This is useful to be able to perform BPF program relocations
1812	* without having to create BPF maps before that step. This allows us
1813	* to finalize and load BTF very late in BPF object's loading phase,
1814	* right before BPF maps have to be created and BPF programs have to
1815	* be loaded. By having these map FD placeholders we can perform all
1816	* the sanitizations, relocations, and any other adjustments before we
1817	* start creating actual BPF kernel objects (BTF, maps, progs).
1818	*/
1819	map->fd = create_placeholder_fd();
1820	if (map->fd < 0)
1821	return ERR_PTR(map->fd);
1822	map->inner_map_fd = -1;
1823	map->autocreate = true;
1824
1825	return map;
1826	}
1827
1828	static size_t array_map_mmap_sz(unsigned int value_sz, unsigned int max_entries)
1829	{
1830	const long page_sz = sysconf(_SC_PAGE_SIZE);
1831	size_t map_sz;
1832
1833	map_sz = (size_t)roundup(value_sz, 8) * max_entries;
1834	map_sz = roundup(map_sz, page_sz);
1835	return map_sz;
1836	}
1837
1838	static size_t bpf_map_mmap_sz(const struct bpf_map *map)
1839	{
1840	const long page_sz = sysconf(_SC_PAGE_SIZE);
1841
1842	switch (map->def.type) {
1843	case BPF_MAP_TYPE_ARRAY:
1844	case BPF_MAP_TYPE_PERCPU_ARRAY:
1845	return array_map_mmap_sz(map->def.value_size, map->def.max_entries);
1846	case BPF_MAP_TYPE_ARENA:
1847	return page_sz * map->def.max_entries;
1848	default:
1849	return 0; /* not supported */
1850	}
1851	}
1852
1853	static int bpf_map_mmap_resize(struct bpf_map *map, size_t old_sz, size_t new_sz)
1854	{
1855	void *mmaped;
1856
1857	if (!map->mmaped)
1858	return -EINVAL;
1859
1860	if (old_sz == new_sz)
1861	return 0;
1862
1863	mmaped = mmap(NULL, new_sz, PROT_READ \| PROT_WRITE, MAP_SHARED \| MAP_ANONYMOUS, -1, 0);
1864	if (mmaped == MAP_FAILED)
1865	return -errno;
1866
1867	memcpy(mmaped, map->mmaped, min(old_sz, new_sz));
1868	munmap(map->mmaped, old_sz);
1869	map->mmaped = mmaped;
1870	return 0;
1871	}
1872
1873	static char internal_map_name(struct bpf_object obj, const char *real_name,
1874	enum libbpf_map_type type)
1875	{
1876	char map_name[BPF_OBJ_NAME_LEN], *p;
1877	int pfx_len, sfx_len = max((size_t)7, strlen(real_name));
1878
1879	/* This is one of the more confusing parts of libbpf for various
1880	* reasons, some of which are historical. The original idea for naming
1881	* internal names was to include as much of BPF object name prefix as
1882	* possible, so that it can be distinguished from similar internal
1883	* maps of a different BPF object.
1884	* As an example, let's say we have bpf_object named 'my_object_name'
1885	* and internal map corresponding to '.rodata' ELF section. The final
1886	* map name advertised to user and to the kernel will be
1887	* 'my_objec.rodata', taking first 8 characters of object name and
1888	* entire 7 characters of '.rodata'.
1889	* Somewhat confusingly, if internal map ELF section name is shorter
1890	* than 7 characters, e.g., '.bss', we still reserve 7 characters
1891	* for the suffix, even though we only have 4 actual characters, and
1892	* resulting map will be called 'my_objec.bss', not even using all 15
1893	* characters allowed by the kernel. Oh well, at least the truncated
1894	* object name is somewhat consistent in this case. But if the map
1895	* name is '.kconfig', we'll still have entirety of '.kconfig' added
1896	* (8 chars) and thus will be left with only first 7 characters of the
1897	* object name ('my_obje'). Happy guessing, user, that the final map
1898	* name will be "my_obje.kconfig".
1899	* Now, with libbpf starting to support arbitrarily named .rodata.*
1900	* and .data.* data sections, it's possible that ELF section name is
1901	* longer than allowed 15 chars, so we now need to be careful to take
1902	* only up to 15 first characters of ELF name, taking no BPF object
1903	* name characters at all. So '.rodata.abracadabra' will result in
1904	* '.rodata.abracad' kernel and user-visible name.
1905	* We need to keep this convoluted logic intact for .data, .bss and
1906	* .rodata maps, but for new custom .data.custom and .rodata.custom
1907	* maps we use their ELF names as is, not prepending bpf_object name
1908	* in front. We still need to truncate them to 15 characters for the
1909	* kernel. Full name can be recovered for such maps by using DATASEC
1910	* BTF type associated with such map's value type, though.
1911	*/
1912	if (sfx_len >= BPF_OBJ_NAME_LEN)
1913	sfx_len = BPF_OBJ_NAME_LEN - 1;
1914
1915	/*
1916	* Don't prefix the bpf_object name if this is a custom dot map
1917	* (containing two or more dots) or a percpu data map.
1918	*/
1919	if (strchr(real_name + 1, '.') != NULL \|\| type == LIBBPF_MAP_PERCPU)
1920	pfx_len = 0;
1921	else
1922	pfx_len = min((size_t)BPF_OBJ_NAME_LEN - sfx_len - 1, strlen(obj->name));
1923
1924	snprintf(map_name, sizeof(map_name), "%.s%.s", pfx_len, obj->name,
1925	sfx_len, real_name);
1926
1927	/* sanities map name to characters allowed by kernel */
1928	for (p = map_name; *p && p < map_name + sizeof(map_name); p++)
1929	if (!isalnum(p) && p != '_' && *p != '.')
1930	*p = '_';
1931
1932	return strdup(map_name);
1933	}
1934
1935	static int
1936	map_fill_btf_type_info(struct bpf_object obj, struct bpf_map map);
1937
1938	/* Internal BPF map is mmap()'able only if at least one of corresponding
1939	* DATASEC's VARs are to be exposed through BPF skeleton. I.e., it's a GLOBAL
1940	* variable and it's not marked as __hidden (which turns it into, effectively,
1941	* a STATIC variable).
1942	*/
1943	static bool map_is_mmapable(struct bpf_object obj, struct bpf_map map)
1944	{
1945	const struct btf_type t, vt;
1946	struct btf_var_secinfo *vsi;
1947	int i, n;
1948
1949	if (!map->btf_value_type_id)
1950	return false;
1951
1952	/*
1953	* The internal PERCPU maps are not mmapable because the underlying
1954	* percpu_array maps do not have mmap support.
1955	*/
1956	if (map->libbpf_type == LIBBPF_MAP_PERCPU)
1957	return false;
1958
1959	t = btf__type_by_id(obj->btf, map->btf_value_type_id);
1960	if (!btf_is_datasec(t))
1961	return false;
1962
1963	vsi = btf_var_secinfos(t);
1964	for (i = 0, n = btf_vlen(t); i < n; i++, vsi++) {
1965	vt = btf__type_by_id(obj->btf, vsi->type);
1966	if (!btf_is_var(vt))
1967	continue;
1968
1969	if (btf_var(vt)->linkage != BTF_VAR_STATIC)
1970	return true;
1971	}
1972
1973	return false;
1974	}
1975
1976	static int
1977	bpf_object__init_internal_map(struct bpf_object *obj, enum libbpf_map_type type,
1978	const char real_name, int sec_idx, void data, size_t data_sz)
1979	{
1980	bool is_percpu = type == LIBBPF_MAP_PERCPU;
1981	struct bpf_map_def *def;
1982	struct bpf_map *map;
1983	size_t mmap_sz;
1984	int err;
1985
1986	map = bpf_object__add_map(obj);
1987	if (IS_ERR(map))
1988	return PTR_ERR(map);
1989
1990	map->libbpf_type = type;
1991	map->sec_idx = sec_idx;
1992	map->sec_offset = 0;
1993	map->real_name = strdup(real_name);
1994	map->name = internal_map_name(obj, real_name, type);
1995	if (!map->real_name \|\| !map->name) {
1996	zfree(&map->real_name);
1997	zfree(&map->name);
1998	return -ENOMEM;
1999	}
2000
2001	def = &map->def;
2002	def->type = is_percpu ? BPF_MAP_TYPE_PERCPU_ARRAY : BPF_MAP_TYPE_ARRAY;
2003	def->key_size = sizeof(int);
2004	def->value_size = data_sz;
2005	def->max_entries = 1;
2006	def->map_flags = type == LIBBPF_MAP_RODATA \|\| type == LIBBPF_MAP_KCONFIG
2007	? BPF_F_RDONLY_PROG : 0;
2008
2009	/* failures are fine because of maps like .rodata.str1.1 */
2010	(void) map_fill_btf_type_info(obj, map);
2011
2012	if (map_is_mmapable(obj, map))
2013	def->map_flags \|= BPF_F_MMAPABLE;
2014
2015	pr_debug("map '%s' (global %sdata): at sec_idx %d, offset %zu, flags %x.\n",
2016	map->name, is_percpu ? "percpu " : "", map->sec_idx,
2017	map->sec_offset, def->map_flags);
2018
2019	mmap_sz = bpf_map_mmap_sz(map);
2020	map->mmaped = mmap(NULL, mmap_sz, PROT_READ \| PROT_WRITE,
2021	MAP_SHARED \| MAP_ANONYMOUS, -1, 0);
2022	if (map->mmaped == MAP_FAILED) {
2023	err = -errno;
2024	map->mmaped = NULL;
2025	pr_warn("failed to alloc map '%s' content buffer: %s\n", map->name, errstr(err));
2026	zfree(&map->real_name);
2027	zfree(&map->name);
2028	return err;
2029	}
2030
2031	if (data)
2032	memcpy(map->mmaped, data, data_sz);
2033
2034	pr_debug("map %td is \"%s\"\n", map - obj->maps, map->name);
2035	return 0;
2036	}
2037
2038	static int bpf_object__init_global_data_maps(struct bpf_object *obj)
2039	{
2040	struct elf_sec_desc *sec_desc;
2041	const char *sec_name;
2042	int err = 0, sec_idx;
2043
2044	/*
2045	* Populate obj->maps with libbpf internal maps.
2046	*/
2047	for (sec_idx = 1; sec_idx < obj->efile.sec_cnt; sec_idx++) {
2048	sec_desc = &obj->efile.secs[sec_idx];
2049
2050	/* Skip recognized sections with size 0. */
2051	if (!sec_desc->data \|\| sec_desc->data->d_size == 0)
2052	continue;
2053
2054	switch (sec_desc->sec_type) {
2055	case SEC_DATA:
2056	sec_name = elf_sec_name(obj, elf_sec_by_idx(obj, sec_idx));
2057	err = bpf_object__init_internal_map(obj, LIBBPF_MAP_DATA,
2058	sec_name, sec_idx,
2059	sec_desc->data->d_buf,
2060	sec_desc->data->d_size);
2061	break;
2062	case SEC_RODATA:
2063	obj->has_rodata = true;
2064	sec_name = elf_sec_name(obj, elf_sec_by_idx(obj, sec_idx));
2065	err = bpf_object__init_internal_map(obj, LIBBPF_MAP_RODATA,
2066	sec_name, sec_idx,
2067	sec_desc->data->d_buf,
2068	sec_desc->data->d_size);
2069	break;
2070	case SEC_BSS:
2071	sec_name = elf_sec_name(obj, elf_sec_by_idx(obj, sec_idx));
2072	err = bpf_object__init_internal_map(obj, LIBBPF_MAP_BSS,
2073	sec_name, sec_idx,
2074	NULL,
2075	sec_desc->data->d_size);
2076	break;
2077	case SEC_PERCPU:
2078	sec_name = elf_sec_name(obj, elf_sec_by_idx(obj, sec_idx));
2079	err = bpf_object__init_internal_map(obj, LIBBPF_MAP_PERCPU,
2080	sec_name, sec_idx,
2081	sec_desc->data->d_buf,
2082	sec_desc->data->d_size);
2083	break;
2084	default:
2085	/* skip */
2086	break;
2087	}
2088	if (err)
2089	return err;
2090	}
2091	return 0;
2092	}
2093
2094
2095	static struct extern_desc find_extern_by_name(const struct bpf_object obj,
2096	const void *name)
2097	{
2098	int i;
2099
2100	for (i = 0; i < obj->nr_extern; i++) {
2101	if (strcmp(obj->externs[i].name, name) == 0)
2102	return &obj->externs[i];
2103	}
2104	return NULL;
2105	}
2106
2107	static struct extern_desc find_extern_by_name_with_len(const struct bpf_object obj,
2108	const void *name, int len)
2109	{
2110	const char *ext_name;
2111	int i;
2112
2113	for (i = 0; i < obj->nr_extern; i++) {
2114	ext_name = obj->externs[i].name;
2115	if (strlen(ext_name) == len && strncmp(ext_name, name, len) == 0)
2116	return &obj->externs[i];
2117	}
2118	return NULL;
2119	}
2120
2121	static int set_kcfg_value_tri(struct extern_desc ext, void ext_val,
2122	char value)
2123	{
2124	switch (ext->kcfg.type) {
2125	case KCFG_BOOL:
2126	if (value == 'm') {
2127	pr_warn("extern (kcfg) '%s': value '%c' implies tristate or char type\n",
2128	ext->name, value);
2129	return -EINVAL;
2130	}
2131	(bool )ext_val = value == 'y' ? true : false;
2132	break;
2133	case KCFG_TRISTATE:
2134	if (value == 'y')
2135	(enum libbpf_tristate )ext_val = TRI_YES;
2136	else if (value == 'm')
2137	(enum libbpf_tristate )ext_val = TRI_MODULE;
2138	else /* value == 'n' */
2139	(enum libbpf_tristate )ext_val = TRI_NO;
2140	break;
2141	case KCFG_CHAR:
2142	(char )ext_val = value;
2143	break;
2144	case KCFG_UNKNOWN:
2145	case KCFG_INT:
2146	case KCFG_CHAR_ARR:
2147	default:
2148	pr_warn("extern (kcfg) '%s': value '%c' implies bool, tristate, or char type\n",
2149	ext->name, value);
2150	return -EINVAL;
2151	}
2152	ext->is_set = true;
2153	return 0;
2154	}
2155
2156	static int set_kcfg_value_str(struct extern_desc ext, char ext_val,
2157	const char *value)
2158	{
2159	size_t len;
2160
2161	if (ext->kcfg.type != KCFG_CHAR_ARR) {
2162	pr_warn("extern (kcfg) '%s': value '%s' implies char array type\n",
2163	ext->name, value);
2164	return -EINVAL;
2165	}
2166
2167	len = strlen(value);
2168	if (len < 2 \|\| value[len - 1] != '"') {
2169	pr_warn("extern (kcfg) '%s': invalid string config '%s'\n",
2170	ext->name, value);
2171	return -EINVAL;
2172	}
2173
2174	/* strip quotes */
2175	len -= 2;
2176	if (len >= ext->kcfg.sz) {
2177	pr_warn("extern (kcfg) '%s': long string '%s' of (%zu bytes) truncated to %d bytes\n",
2178	ext->name, value, len, ext->kcfg.sz - 1);
2179	len = ext->kcfg.sz - 1;
2180	}
2181	memcpy(ext_val, value + 1, len);
2182	ext_val[len] = '\0';
2183	ext->is_set = true;
2184	return 0;
2185	}
2186
2187	static int parse_u64(const char value, __u64 res)
2188	{
2189	char *value_end;
2190	int err;
2191
2192	errno = 0;
2193	*res = strtoull(value, &value_end, 0);
2194	if (errno) {
2195	err = -errno;
2196	pr_warn("failed to parse '%s': %s\n", value, errstr(err));
2197	return err;
2198	}
2199	if (*value_end) {
2200	pr_warn("failed to parse '%s' as integer completely\n", value);
2201	return -EINVAL;
2202	}
2203	return 0;
2204	}
2205
2206	static bool is_kcfg_value_in_range(const struct extern_desc *ext, __u64 v)
2207	{
2208	int bit_sz = ext->kcfg.sz * 8;
2209
2210	if (ext->kcfg.sz == 8)
2211	return true;
2212
2213	/* Validate that value stored in u64 fits in integer of `ext->sz`
2214	* bytes size without any loss of information. If the target integer
2215	* is signed, we rely on the following limits of integer type of
2216	* Y bits and subsequent transformation:
2217	*
2218	* -2^(Y-1) <= X <= 2^(Y-1) - 1
2219	* 0 <= X + 2^(Y-1) <= 2^Y - 1
2220	* 0 <= X + 2^(Y-1) < 2^Y
2221	*
2222	* For unsigned target integer, check that all the (64 - Y) bits are
2223	* zero.
2224	*/
2225	if (ext->kcfg.is_signed)
2226	return v + (1ULL << (bit_sz - 1)) < (1ULL << bit_sz);
2227	else
2228	return (v >> bit_sz) == 0;
2229	}
2230
2231	static int set_kcfg_value_num(struct extern_desc ext, void ext_val,
2232	__u64 value)
2233	{
2234	if (ext->kcfg.type != KCFG_INT && ext->kcfg.type != KCFG_CHAR &&
2235	ext->kcfg.type != KCFG_BOOL) {
2236	pr_warn("extern (kcfg) '%s': value '%llu' implies integer, char, or boolean type\n",
2237	ext->name, (unsigned long long)value);
2238	return -EINVAL;
2239	}
2240	if (ext->kcfg.type == KCFG_BOOL && value > 1) {
2241	pr_warn("extern (kcfg) '%s': value '%llu' isn't boolean compatible\n",
2242	ext->name, (unsigned long long)value);
2243	return -EINVAL;
2244
2245	}
2246	if (!is_kcfg_value_in_range(ext, value)) {
2247	pr_warn("extern (kcfg) '%s': value '%llu' doesn't fit in %d bytes\n",
2248	ext->name, (unsigned long long)value, ext->kcfg.sz);
2249	return -ERANGE;
2250	}
2251	switch (ext->kcfg.sz) {
2252	case 1:
2253	(__u8 )ext_val = value;
2254	break;
2255	case 2:
2256	(__u16 )ext_val = value;
2257	break;
2258	case 4:
2259	(__u32 )ext_val = value;
2260	break;
2261	case 8:
2262	(__u64 )ext_val = value;
2263	break;
2264	default:
2265	return -EINVAL;
2266	}
2267	ext->is_set = true;
2268	return 0;
2269	}
2270
2271	static int bpf_object__process_kconfig_line(struct bpf_object *obj,
2272	char buf, void data)
2273	{
2274	struct extern_desc *ext;
2275	char sep, value;
2276	int len, err = 0;
2277	void *ext_val;
2278	__u64 num;
2279
2280	if (!str_has_pfx(buf, "CONFIG_"))
2281	return 0;
2282
2283	sep = strchr(buf, '=');
2284	if (!sep) {
2285	pr_warn("failed to parse '%s': no separator\n", buf);
2286	return -EINVAL;
2287	}
2288
2289	/* Trim ending '\n' */
2290	len = strlen(buf);
2291	if (buf[len - 1] == '\n')
2292	buf[len - 1] = '\0';
2293	/* Split on '=' and ensure that a value is present. */
2294	*sep = '\0';
2295	if (!sep[1]) {
2296	*sep = '=';
2297	pr_warn("failed to parse '%s': no value\n", buf);
2298	return -EINVAL;
2299	}
2300
2301	ext = find_extern_by_name(obj, buf);
2302	if (!ext \|\| ext->is_set)
2303	return 0;
2304
2305	ext_val = data + ext->kcfg.data_off;
2306	value = sep + 1;
2307
2308	switch (*value) {
2309	case 'y': case 'n': case 'm':
2310	err = set_kcfg_value_tri(ext, ext_val, *value);
2311	break;
2312	case '"':
2313	err = set_kcfg_value_str(ext, ext_val, value);
2314	break;
2315	default:
2316	/* assume integer */
2317	err = parse_u64(value, &num);
2318	if (err) {
2319	pr_warn("extern (kcfg) '%s': value '%s' isn't a valid integer\n", ext->name, value);
2320	return err;
2321	}
2322	if (ext->kcfg.type != KCFG_INT && ext->kcfg.type != KCFG_CHAR) {
2323	pr_warn("extern (kcfg) '%s': value '%s' implies integer type\n", ext->name, value);
2324	return -EINVAL;
2325	}
2326	err = set_kcfg_value_num(ext, ext_val, num);
2327	break;
2328	}
2329	if (err)
2330	return err;
2331	pr_debug("extern (kcfg) '%s': set to %s\n", ext->name, value);
2332	return 0;
2333	}
2334
2335	static int bpf_object__read_kconfig_file(struct bpf_object obj, void data)
2336	{
2337	char buf[PATH_MAX];
2338	struct utsname uts;
2339	int len, err = 0;
2340	gzFile file;
2341
2342	uname(&uts);
2343	len = snprintf(buf, PATH_MAX, "/boot/config-%s", uts.release);
2344	if (len < 0)
2345	return -EINVAL;
2346	else if (len >= PATH_MAX)
2347	return -ENAMETOOLONG;
2348
2349	/* gzopen also accepts uncompressed files. */
2350	file = gzopen(buf, "re");
2351	if (!file)
2352	file = gzopen("/proc/config.gz", "re");
2353
2354	if (!file) {
2355	pr_warn("failed to open system Kconfig\n");
2356	return -ENOENT;
2357	}
2358
2359	while (gzgets(file, buf, sizeof(buf))) {
2360	err = bpf_object__process_kconfig_line(obj, buf, data);
2361	if (err) {
2362	pr_warn("error parsing system Kconfig line '%s': %s\n",
2363	buf, errstr(err));
2364	goto out;
2365	}
2366	}
2367
2368	out:
2369	gzclose(file);
2370	return err;
2371	}
2372
2373	static int bpf_object__read_kconfig_mem(struct bpf_object *obj,
2374	const char config, void data)
2375	{
2376	char buf[PATH_MAX];
2377	int err = 0;
2378	FILE *file;
2379
2380	file = fmemopen((void *)config, strlen(config), "r");
2381	if (!file) {
2382	err = -errno;
2383	pr_warn("failed to open in-memory Kconfig: %s\n", errstr(err));
2384	return err;
2385	}
2386
2387	while (fgets(buf, sizeof(buf), file)) {
2388	err = bpf_object__process_kconfig_line(obj, buf, data);
2389	if (err) {
2390	pr_warn("error parsing in-memory Kconfig line '%s': %s\n",
2391	buf, errstr(err));
2392	break;
2393	}
2394	}
2395
2396	fclose(file);
2397	return err;
2398	}
2399
2400	static int bpf_object__init_kconfig_map(struct bpf_object *obj)
2401	{
2402	struct extern_desc last_ext = NULL, ext;
2403	size_t map_sz;
2404	int i, err;
2405
2406	for (i = 0; i < obj->nr_extern; i++) {
2407	ext = &obj->externs[i];
2408	if (ext->type == EXT_KCFG)
2409	last_ext = ext;
2410	}
2411
2412	if (!last_ext)
2413	return 0;
2414
2415	map_sz = last_ext->kcfg.data_off + last_ext->kcfg.sz;
2416	err = bpf_object__init_internal_map(obj, LIBBPF_MAP_KCONFIG,
2417	".kconfig", obj->efile.symbols_shndx,
2418	NULL, map_sz);
2419	if (err)
2420	return err;
2421
2422	obj->kconfig_map_idx = obj->nr_maps - 1;
2423
2424	return 0;
2425	}
2426
2427	const struct btf_type *
2428	skip_mods_and_typedefs(const struct btf btf, __u32 id, __u32 res_id)
2429	{
2430	const struct btf_type *t = btf__type_by_id(btf, id);
2431
2432	if (res_id)
2433	*res_id = id;
2434
2435	while (btf_is_mod(t) \|\| btf_is_typedef(t)) {
2436	if (res_id)
2437	*res_id = t->type;
2438	t = btf__type_by_id(btf, t->type);
2439	}
2440
2441	return t;
2442	}
2443
2444	static const struct btf_type *
2445	resolve_func_ptr(const struct btf btf, __u32 id, __u32 res_id)
2446	{
2447	const struct btf_type *t;
2448
2449	t = skip_mods_and_typedefs(btf, id, NULL);
2450	if (!btf_is_ptr(t))
2451	return NULL;
2452
2453	t = skip_mods_and_typedefs(btf, t->type, res_id);
2454
2455	return btf_is_func_proto(t) ? t : NULL;
2456	}
2457
2458	static const char *__btf_kind_str(__u16 kind)
2459	{
2460	switch (kind) {
2461	case BTF_KIND_UNKN: return "void";
2462	case BTF_KIND_INT: return "int";
2463	case BTF_KIND_PTR: return "ptr";
2464	case BTF_KIND_ARRAY: return "array";
2465	case BTF_KIND_STRUCT: return "struct";
2466	case BTF_KIND_UNION: return "union";
2467	case BTF_KIND_ENUM: return "enum";
2468	case BTF_KIND_FWD: return "fwd";
2469	case BTF_KIND_TYPEDEF: return "typedef";
2470	case BTF_KIND_VOLATILE: return "volatile";
2471	case BTF_KIND_CONST: return "const";
2472	case BTF_KIND_RESTRICT: return "restrict";
2473	case BTF_KIND_FUNC: return "func";
2474	case BTF_KIND_FUNC_PROTO: return "func_proto";
2475	case BTF_KIND_VAR: return "var";
2476	case BTF_KIND_DATASEC: return "datasec";
2477	case BTF_KIND_FLOAT: return "float";
2478	case BTF_KIND_DECL_TAG: return "decl_tag";
2479	case BTF_KIND_TYPE_TAG: return "type_tag";
2480	case BTF_KIND_ENUM64: return "enum64";
2481	default: return "unknown";
2482	}
2483	}
2484
2485	const char btf_kind_str(const struct btf_type t)
2486	{
2487	return __btf_kind_str(btf_kind(t));
2488	}
2489
2490	/*
2491	* Fetch integer attribute of BTF map definition. Such attributes are
2492	* represented using a pointer to an array, in which dimensionality of array
2493	* encodes specified integer value. E.g., int (*type)[BPF_MAP_TYPE_ARRAY];
2494	* encodes `type => BPF_MAP_TYPE_ARRAY` key/value pair completely using BTF
2495	* type definition, while using only sizeof(void *) space in ELF data section.
2496	*/
2497	static bool get_map_field_int(const char map_name, const struct btf btf,
2498	const struct btf_member m, __u32 res)
2499	{
2500	const struct btf_type *t = skip_mods_and_typedefs(btf, m->type, NULL);
2501	const char *name = btf__name_by_offset(btf, m->name_off);
2502	const struct btf_array *arr_info;
2503	const struct btf_type *arr_t;
2504
2505	if (!btf_is_ptr(t)) {
2506	pr_warn("map '%s': attr '%s': expected PTR, got %s.\n",
2507	map_name, name, btf_kind_str(t));
2508	return false;
2509	}
2510
2511	arr_t = btf__type_by_id(btf, t->type);
2512	if (!arr_t) {
2513	pr_warn("map '%s': attr '%s': type [%u] not found.\n",
2514	map_name, name, t->type);
2515	return false;
2516	}
2517	if (!btf_is_array(arr_t)) {
2518	pr_warn("map '%s': attr '%s': expected ARRAY, got %s.\n",
2519	map_name, name, btf_kind_str(arr_t));
2520	return false;
2521	}
2522	arr_info = btf_array(arr_t);
2523	*res = arr_info->nelems;
2524	return true;
2525	}
2526
2527	static bool get_map_field_long(const char map_name, const struct btf btf,
2528	const struct btf_member m, __u64 res)
2529	{
2530	const struct btf_type *t = skip_mods_and_typedefs(btf, m->type, NULL);
2531	const char *name = btf__name_by_offset(btf, m->name_off);
2532
2533	if (btf_is_ptr(t)) {
2534	__u32 res32;
2535	bool ret;
2536
2537	ret = get_map_field_int(map_name, btf, m, &res32);
2538	if (ret)
2539	*res = (__u64)res32;
2540	return ret;
2541	}
2542
2543	if (!btf_is_enum(t) && !btf_is_enum64(t)) {
2544	pr_warn("map '%s': attr '%s': expected ENUM or ENUM64, got %s.\n",
2545	map_name, name, btf_kind_str(t));
2546	return false;
2547	}
2548
2549	if (btf_vlen(t) != 1) {
2550	pr_warn("map '%s': attr '%s': invalid __ulong\n",
2551	map_name, name);
2552	return false;
2553	}
2554
2555	if (btf_is_enum(t)) {
2556	const struct btf_enum *e = btf_enum(t);
2557
2558	*res = e->val;
2559	} else {
2560	const struct btf_enum64 *e = btf_enum64(t);
2561
2562	*res = btf_enum64_value(e);
2563	}
2564	return true;
2565	}
2566
2567	static int pathname_concat(char buf, size_t buf_sz, const char path, const char *name)
2568	{
2569	int len;
2570
2571	len = snprintf(buf, buf_sz, "%s/%s", path, name);
2572	if (len < 0)
2573	return -EINVAL;
2574	if (len >= buf_sz)
2575	return -ENAMETOOLONG;
2576
2577	return 0;
2578	}
2579
2580	static int build_map_pin_path(struct bpf_map map, const char path)
2581	{
2582	char buf[PATH_MAX];
2583	int err;
2584
2585	if (!path)
2586	path = BPF_FS_DEFAULT_PATH;
2587
2588	err = pathname_concat(buf, sizeof(buf), path, bpf_map__name(map));
2589	if (err)
2590	return err;
2591
2592	return bpf_map__set_pin_path(map, buf);
2593	}
2594
2595	/* should match definition in bpf_helpers.h */
2596	enum libbpf_pin_type {
2597	LIBBPF_PIN_NONE,
2598	/* PIN_BY_NAME: pin maps by name (in /sys/fs/bpf by default) */
2599	LIBBPF_PIN_BY_NAME,
2600	};
2601
2602	int parse_btf_map_def(const char map_name, struct btf btf,
2603	const struct btf_type *def_t, bool strict,
2604	struct btf_map_def map_def, struct btf_map_def inner_def)
2605	{
2606	const struct btf_type *t;
2607	const struct btf_member *m;
2608	bool is_inner = inner_def == NULL;
2609	int vlen, i;
2610
2611	vlen = btf_vlen(def_t);
2612	m = btf_members(def_t);
2613	for (i = 0; i < vlen; i++, m++) {
2614	const char *name = btf__name_by_offset(btf, m->name_off);
2615
2616	if (!name) {
2617	pr_warn("map '%s': invalid field #%d.\n", map_name, i);
2618	return -EINVAL;
2619	}
2620	if (strcmp(name, "type") == 0) {
2621	if (!get_map_field_int(map_name, btf, m, &map_def->map_type))
2622	return -EINVAL;
2623	map_def->parts \|= MAP_DEF_MAP_TYPE;
2624	} else if (strcmp(name, "max_entries") == 0) {
2625	if (!get_map_field_int(map_name, btf, m, &map_def->max_entries))
2626	return -EINVAL;
2627	map_def->parts \|= MAP_DEF_MAX_ENTRIES;
2628	} else if (strcmp(name, "map_flags") == 0) {
2629	if (!get_map_field_int(map_name, btf, m, &map_def->map_flags))
2630	return -EINVAL;
2631	map_def->parts \|= MAP_DEF_MAP_FLAGS;
2632	} else if (strcmp(name, "numa_node") == 0) {
2633	if (!get_map_field_int(map_name, btf, m, &map_def->numa_node))
2634	return -EINVAL;
2635	map_def->parts \|= MAP_DEF_NUMA_NODE;
2636	} else if (strcmp(name, "key_size") == 0) {
2637	__u32 sz;
2638
2639	if (!get_map_field_int(map_name, btf, m, &sz))
2640	return -EINVAL;
2641	if (map_def->key_size && map_def->key_size != sz) {
2642	pr_warn("map '%s': conflicting key size %u != %u.\n",
2643	map_name, map_def->key_size, sz);
2644	return -EINVAL;
2645	}
2646	map_def->key_size = sz;
2647	map_def->parts \|= MAP_DEF_KEY_SIZE;
2648	} else if (strcmp(name, "key") == 0) {
2649	__s64 sz;
2650
2651	t = btf__type_by_id(btf, m->type);
2652	if (!t) {
2653	pr_warn("map '%s': key type [%u] not found.\n",
2654	map_name, m->type);
2655	return -EINVAL;
2656	}
2657	if (!btf_is_ptr(t)) {
2658	pr_warn("map '%s': key spec is not PTR: %s.\n",
2659	map_name, btf_kind_str(t));
2660	return -EINVAL;
2661	}
2662	sz = btf__resolve_size(btf, t->type);
2663	if (sz < 0) {
2664	pr_warn("map '%s': can't determine key size for type [%u]: %zd.\n",
2665	map_name, t->type, (ssize_t)sz);
2666	return sz;
2667	}
2668	if (map_def->key_size && map_def->key_size != sz) {
2669	pr_warn("map '%s': conflicting key size %u != %zd.\n",
2670	map_name, map_def->key_size, (ssize_t)sz);
2671	return -EINVAL;
2672	}
2673	map_def->key_size = sz;
2674	map_def->key_type_id = t->type;
2675	map_def->parts \|= MAP_DEF_KEY_SIZE \| MAP_DEF_KEY_TYPE;
2676	} else if (strcmp(name, "value_size") == 0) {
2677	__u32 sz;
2678
2679	if (!get_map_field_int(map_name, btf, m, &sz))
2680	return -EINVAL;
2681	if (map_def->value_size && map_def->value_size != sz) {
2682	pr_warn("map '%s': conflicting value size %u != %u.\n",
2683	map_name, map_def->value_size, sz);
2684	return -EINVAL;
2685	}
2686	map_def->value_size = sz;
2687	map_def->parts \|= MAP_DEF_VALUE_SIZE;
2688	} else if (strcmp(name, "value") == 0) {
2689	__s64 sz;
2690
2691	t = btf__type_by_id(btf, m->type);
2692	if (!t) {
2693	pr_warn("map '%s': value type [%u] not found.\n",
2694	map_name, m->type);
2695	return -EINVAL;
2696	}
2697	if (!btf_is_ptr(t)) {
2698	pr_warn("map '%s': value spec is not PTR: %s.\n",
2699	map_name, btf_kind_str(t));
2700	return -EINVAL;
2701	}
2702	sz = btf__resolve_size(btf, t->type);
2703	if (sz < 0) {
2704	pr_warn("map '%s': can't determine value size for type [%u]: %zd.\n",
2705	map_name, t->type, (ssize_t)sz);
2706	return sz;
2707	}
2708	if (map_def->value_size && map_def->value_size != sz) {
2709	pr_warn("map '%s': conflicting value size %u != %zd.\n",
2710	map_name, map_def->value_size, (ssize_t)sz);
2711	return -EINVAL;
2712	}
2713	map_def->value_size = sz;
2714	map_def->value_type_id = t->type;
2715	map_def->parts \|= MAP_DEF_VALUE_SIZE \| MAP_DEF_VALUE_TYPE;
2716	}
2717	else if (strcmp(name, "values") == 0) {
2718	bool is_map_in_map = bpf_map_type__is_map_in_map(map_def->map_type);
2719	bool is_prog_array = map_def->map_type == BPF_MAP_TYPE_PROG_ARRAY;
2720	const char *desc = is_map_in_map ? "map-in-map inner" : "prog-array value";
2721	char inner_map_name[128];
2722	int err;
2723
2724	if (is_inner) {
2725	pr_warn("map '%s': multi-level inner maps not supported.\n",
2726	map_name);
2727	return -ENOTSUP;
2728	}
2729	if (i != vlen - 1) {
2730	pr_warn("map '%s': '%s' member should be last.\n",
2731	map_name, name);
2732	return -EINVAL;
2733	}
2734	if (!is_map_in_map && !is_prog_array) {
2735	pr_warn("map '%s': should be map-in-map or prog-array.\n",
2736	map_name);
2737	return -ENOTSUP;
2738	}
2739	if (map_def->value_size && map_def->value_size != 4) {
2740	pr_warn("map '%s': conflicting value size %u != 4.\n",
2741	map_name, map_def->value_size);
2742	return -EINVAL;
2743	}
2744	map_def->value_size = 4;
2745	t = btf__type_by_id(btf, m->type);
2746	if (!t) {
2747	pr_warn("map '%s': %s type [%u] not found.\n",
2748	map_name, desc, m->type);
2749	return -EINVAL;
2750	}
2751	if (!btf_is_array(t) \|\| btf_array(t)->nelems) {
2752	pr_warn("map '%s': %s spec is not a zero-sized array.\n",
2753	map_name, desc);
2754	return -EINVAL;
2755	}
2756	t = skip_mods_and_typedefs(btf, btf_array(t)->type, NULL);
2757	if (!btf_is_ptr(t)) {
2758	pr_warn("map '%s': %s def is of unexpected kind %s.\n",
2759	map_name, desc, btf_kind_str(t));
2760	return -EINVAL;
2761	}
2762	t = skip_mods_and_typedefs(btf, t->type, NULL);
2763	if (is_prog_array) {
2764	if (!btf_is_func_proto(t)) {
2765	pr_warn("map '%s': prog-array value def is of unexpected kind %s.\n",
2766	map_name, btf_kind_str(t));
2767	return -EINVAL;
2768	}
2769	continue;
2770	}
2771	if (!btf_is_struct(t)) {
2772	pr_warn("map '%s': map-in-map inner def is of unexpected kind %s.\n",
2773	map_name, btf_kind_str(t));
2774	return -EINVAL;
2775	}
2776
2777	snprintf(inner_map_name, sizeof(inner_map_name), "%s.inner", map_name);
2778	err = parse_btf_map_def(inner_map_name, btf, t, strict, inner_def, NULL);
2779	if (err)
2780	return err;
2781
2782	map_def->parts \|= MAP_DEF_INNER_MAP;
2783	} else if (strcmp(name, "pinning") == 0) {
2784	__u32 val;
2785
2786	if (is_inner) {
2787	pr_warn("map '%s': inner def can't be pinned.\n", map_name);
2788	return -EINVAL;
2789	}
2790	if (!get_map_field_int(map_name, btf, m, &val))
2791	return -EINVAL;
2792	if (val != LIBBPF_PIN_NONE && val != LIBBPF_PIN_BY_NAME) {
2793	pr_warn("map '%s': invalid pinning value %u.\n",
2794	map_name, val);
2795	return -EINVAL;
2796	}
2797	map_def->pinning = val;
2798	map_def->parts \|= MAP_DEF_PINNING;
2799	} else if (strcmp(name, "map_extra") == 0) {
2800	__u64 map_extra;
2801
2802	if (!get_map_field_long(map_name, btf, m, &map_extra))
2803	return -EINVAL;
2804	map_def->map_extra = map_extra;
2805	map_def->parts \|= MAP_DEF_MAP_EXTRA;
2806	} else {
2807	if (strict) {
2808	pr_warn("map '%s': unknown field '%s'.\n", map_name, name);
2809	return -ENOTSUP;
2810	}
2811	pr_debug("map '%s': ignoring unknown field '%s'.\n", map_name, name);
2812	}
2813	}
2814
2815	if (map_def->map_type == BPF_MAP_TYPE_UNSPEC) {
2816	pr_warn("map '%s': map type isn't specified.\n", map_name);
2817	return -EINVAL;
2818	}
2819
2820	return 0;
2821	}
2822
2823	static size_t adjust_ringbuf_sz(size_t sz)
2824	{
2825	__u32 page_sz = sysconf(_SC_PAGE_SIZE);
2826	__u32 mul;
2827
2828	/* if user forgot to set any size, make sure they see error */
2829	if (sz == 0)
2830	return 0;
2831	/* Kernel expects BPF_MAP_TYPE_RINGBUF's max_entries to be
2832	* a power-of-2 multiple of kernel's page size. If user diligently
2833	* satisfied these conditions, pass the size through.
2834	*/
2835	if ((sz % page_sz) == 0 && is_pow_of_2(sz / page_sz))
2836	return sz;
2837
2838	/* Otherwise find closest (page_sz * power_of_2) product bigger than
2839	* user-set size to satisfy both user size request and kernel
2840	* requirements and substitute correct max_entries for map creation.
2841	*/
2842	for (mul = 1; mul <= UINT_MAX / page_sz; mul <<= 1) {
2843	if (mul * page_sz > sz)
2844	return mul * page_sz;
2845	}
2846
2847	/* if it's impossible to satisfy the conditions (i.e., user size is
2848	* very close to UINT_MAX but is not a power-of-2 multiple of
2849	* page_size) then just return original size and let kernel reject it
2850	*/
2851	return sz;
2852	}
2853
2854	static bool map_is_ringbuf(const struct bpf_map *map)
2855	{
2856	return map->def.type == BPF_MAP_TYPE_RINGBUF \|\|
2857	map->def.type == BPF_MAP_TYPE_USER_RINGBUF;
2858	}
2859
2860	static void fill_map_from_def(struct bpf_map map, const struct btf_map_def def)
2861	{
2862	map->def.type = def->map_type;
2863	map->def.key_size = def->key_size;
2864	map->def.value_size = def->value_size;
2865	map->def.max_entries = def->max_entries;
2866	map->def.map_flags = def->map_flags;
2867	map->map_extra = def->map_extra;
2868
2869	map->numa_node = def->numa_node;
2870	map->btf_key_type_id = def->key_type_id;
2871	map->btf_value_type_id = def->value_type_id;
2872
2873	/* auto-adjust BPF ringbuf map max_entries to be a multiple of page size */
2874	if (map_is_ringbuf(map))
2875	map->def.max_entries = adjust_ringbuf_sz(map->def.max_entries);
2876
2877	if (def->parts & MAP_DEF_MAP_TYPE)
2878	pr_debug("map '%s': found type = %u.\n", map->name, def->map_type);
2879
2880	if (def->parts & MAP_DEF_KEY_TYPE)
2881	pr_debug("map '%s': found key [%u], sz = %u.\n",
2882	map->name, def->key_type_id, def->key_size);
2883	else if (def->parts & MAP_DEF_KEY_SIZE)
2884	pr_debug("map '%s': found key_size = %u.\n", map->name, def->key_size);
2885
2886	if (def->parts & MAP_DEF_VALUE_TYPE)
2887	pr_debug("map '%s': found value [%u], sz = %u.\n",
2888	map->name, def->value_type_id, def->value_size);
2889	else if (def->parts & MAP_DEF_VALUE_SIZE)
2890	pr_debug("map '%s': found value_size = %u.\n", map->name, def->value_size);
2891
2892	if (def->parts & MAP_DEF_MAX_ENTRIES)
2893	pr_debug("map '%s': found max_entries = %u.\n", map->name, def->max_entries);
2894	if (def->parts & MAP_DEF_MAP_FLAGS)
2895	pr_debug("map '%s': found map_flags = 0x%x.\n", map->name, def->map_flags);
2896	if (def->parts & MAP_DEF_MAP_EXTRA)
2897	pr_debug("map '%s': found map_extra = 0x%llx.\n", map->name,
2898	(unsigned long long)def->map_extra);
2899	if (def->parts & MAP_DEF_PINNING)
2900	pr_debug("map '%s': found pinning = %u.\n", map->name, def->pinning);
2901	if (def->parts & MAP_DEF_NUMA_NODE)
2902	pr_debug("map '%s': found numa_node = %u.\n", map->name, def->numa_node);
2903
2904	if (def->parts & MAP_DEF_INNER_MAP)
2905	pr_debug("map '%s': found inner map definition.\n", map->name);
2906	}
2907
2908	static const char *btf_var_linkage_str(__u32 linkage)
2909	{
2910	switch (linkage) {
2911	case BTF_VAR_STATIC: return "static";
2912	case BTF_VAR_GLOBAL_ALLOCATED: return "global";
2913	case BTF_VAR_GLOBAL_EXTERN: return "extern";
2914	default: return "unknown";
2915	}
2916	}
2917
2918	static int bpf_object__init_user_btf_map(struct bpf_object *obj,
2919	const struct btf_type *sec,
2920	int var_idx, int sec_idx,
2921	const Elf_Data *data, bool strict,
2922	const char *pin_root_path)
2923	{
2924	struct btf_map_def map_def = {}, inner_def = {};
2925	const struct btf_type var, def;
2926	const struct btf_var_secinfo *vi;
2927	const struct btf_var *var_extra;
2928	const char *map_name;
2929	struct bpf_map *map;
2930	int err;
2931
2932	vi = btf_var_secinfos(sec) + var_idx;
2933	var = btf__type_by_id(obj->btf, vi->type);
2934	var_extra = btf_var(var);
2935	map_name = btf__name_by_offset(obj->btf, var->name_off);
2936
2937	if (str_is_empty(map_name)) {
2938	pr_warn("map #%d: empty name.\n", var_idx);
2939	return -EINVAL;
2940	}
2941	if ((__u64)vi->offset + vi->size > data->d_size) {
2942	pr_warn("map '%s' BTF data is corrupted.\n", map_name);
2943	return -EINVAL;
2944	}
2945	if (!btf_is_var(var)) {
2946	pr_warn("map '%s': unexpected var kind %s.\n",
2947	map_name, btf_kind_str(var));
2948	return -EINVAL;
2949	}
2950	if (var_extra->linkage != BTF_VAR_GLOBAL_ALLOCATED) {
2951	pr_warn("map '%s': unsupported map linkage %s.\n",
2952	map_name, btf_var_linkage_str(var_extra->linkage));
2953	return -EOPNOTSUPP;
2954	}
2955
2956	def = skip_mods_and_typedefs(obj->btf, var->type, NULL);
2957	if (!btf_is_struct(def)) {
2958	pr_warn("map '%s': unexpected def kind %s.\n",
2959	map_name, btf_kind_str(var));
2960	return -EINVAL;
2961	}
2962	if (def->size > vi->size) {
2963	pr_warn("map '%s': invalid def size.\n", map_name);
2964	return -EINVAL;
2965	}
2966
2967	map = bpf_object__add_map(obj);
2968	if (IS_ERR(map))
2969	return PTR_ERR(map);
2970	map->name = strdup(map_name);
2971	if (!map->name) {
2972	pr_warn("map '%s': failed to alloc map name.\n", map_name);
2973	return -ENOMEM;
2974	}
2975	map->libbpf_type = LIBBPF_MAP_UNSPEC;
2976	map->def.type = BPF_MAP_TYPE_UNSPEC;
2977	map->sec_idx = sec_idx;
2978	map->sec_offset = vi->offset;
2979	map->btf_var_idx = var_idx;
2980	pr_debug("map '%s': at sec_idx %d, offset %zu.\n",
2981	map_name, map->sec_idx, map->sec_offset);
2982
2983	err = parse_btf_map_def(map->name, obj->btf, def, strict, &map_def, &inner_def);
2984	if (err)
2985	return err;
2986
2987	fill_map_from_def(map, &map_def);
2988
2989	if (map_def.pinning == LIBBPF_PIN_BY_NAME) {
2990	err = build_map_pin_path(map, pin_root_path);
2991	if (err) {
2992	pr_warn("map '%s': couldn't build pin path.\n", map->name);
2993	return err;
2994	}
2995	}
2996
2997	if (map_def.parts & MAP_DEF_INNER_MAP) {
2998	map->inner_map = calloc(1, sizeof(*map->inner_map));
2999	if (!map->inner_map)
3000	return -ENOMEM;
3001	map->inner_map->fd = create_placeholder_fd();
3002	if (map->inner_map->fd < 0)
3003	return map->inner_map->fd;
3004	map->inner_map->sec_idx = sec_idx;
3005	map->inner_map->name = malloc(strlen(map_name) + sizeof(".inner") + 1);
3006	if (!map->inner_map->name)
3007	return -ENOMEM;
3008	sprintf(map->inner_map->name, "%s.inner", map_name);
3009
3010	fill_map_from_def(map->inner_map, &inner_def);
3011	}
3012
3013	err = map_fill_btf_type_info(obj, map);
3014	if (err)
3015	return err;
3016
3017	return 0;
3018	}
3019
3020	static int init_arena_map_data(struct bpf_object obj, struct bpf_map map,
3021	const char *sec_name, int sec_idx,
3022	void *data, size_t data_sz)
3023	{
3024	const long page_sz = sysconf(_SC_PAGE_SIZE);
3025	const size_t data_alloc_sz = roundup(data_sz, page_sz);
3026	size_t mmap_sz;
3027
3028	mmap_sz = bpf_map_mmap_sz(map);
3029	if (data_alloc_sz > mmap_sz) {
3030	pr_warn("elf: sec '%s': declared ARENA map size (%zu) is too small to hold global __arena variables of size %zu\n",
3031	sec_name, mmap_sz, data_sz);
3032	return -E2BIG;
3033	}
3034
3035	obj->arena_data = malloc(data_sz);
3036	if (!obj->arena_data)
3037	return -ENOMEM;
3038	memcpy(obj->arena_data, data, data_sz);
3039	obj->arena_data_sz = data_sz;
3040
3041	/* make bpf_map__init_value() work for ARENA maps */
3042	map->mmaped = obj->arena_data;
3043
3044	return 0;
3045	}
3046
3047	static int bpf_object__init_user_btf_maps(struct bpf_object *obj, bool strict,
3048	const char *pin_root_path)
3049	{
3050	const struct btf_type *sec = NULL;
3051	int nr_types, i, vlen, err;
3052	const struct btf_type *t;
3053	const char *name;
3054	Elf_Data *data;
3055	Elf_Scn *scn;
3056
3057	if (obj->efile.btf_maps_shndx < 0)
3058	return 0;
3059
3060	scn = elf_sec_by_idx(obj, obj->efile.btf_maps_shndx);
3061	data = elf_sec_data(obj, scn);
3062	if (!data) {
3063	pr_warn("elf: failed to get %s map definitions for %s\n",
3064	MAPS_ELF_SEC, obj->path);
3065	return -EINVAL;
3066	}
3067
3068	nr_types = btf__type_cnt(obj->btf);
3069	for (i = 1; i < nr_types; i++) {
3070	t = btf__type_by_id(obj->btf, i);
3071	if (!btf_is_datasec(t))
3072	continue;
3073	name = btf__name_by_offset(obj->btf, t->name_off);
3074	if (strcmp(name, MAPS_ELF_SEC) == 0) {
3075	sec = t;
3076	obj->efile.btf_maps_sec_btf_id = i;
3077	break;
3078	}
3079	}
3080
3081	if (!sec) {
3082	pr_warn("DATASEC '%s' not found.\n", MAPS_ELF_SEC);
3083	return -ENOENT;
3084	}
3085
3086	vlen = btf_vlen(sec);
3087	for (i = 0; i < vlen; i++) {
3088	err = bpf_object__init_user_btf_map(obj, sec, i,
3089	obj->efile.btf_maps_shndx,
3090	data, strict,
3091	pin_root_path);
3092	if (err)
3093	return err;
3094	}
3095
3096	for (i = 0; i < obj->nr_maps; i++) {
3097	struct bpf_map *map = &obj->maps[i];
3098
3099	if (map->def.type != BPF_MAP_TYPE_ARENA)
3100	continue;
3101
3102	if (obj->arena_map_idx >= 0) {
3103	pr_warn("map '%s': only single ARENA map is supported (map '%s' is also ARENA)\n",
3104	map->name, obj->maps[obj->arena_map_idx].name);
3105	return -EINVAL;
3106	}
3107	obj->arena_map_idx = i;
3108
3109	if (obj->efile.arena_data) {
3110	err = init_arena_map_data(obj, map, ARENA_SEC, obj->efile.arena_data_shndx,
3111	obj->efile.arena_data->d_buf,
3112	obj->efile.arena_data->d_size);
3113	if (err)
3114	return err;
3115	}
3116	}
3117	if (obj->efile.arena_data && obj->arena_map_idx < 0) {
3118	pr_warn("elf: sec '%s': to use global __arena variables the ARENA map should be explicitly declared in SEC(\".maps\")\n",
3119	ARENA_SEC);
3120	return -ENOENT;
3121	}
3122
3123	return 0;
3124	}
3125
3126	static int bpf_object__init_maps(struct bpf_object *obj,
3127	const struct bpf_object_open_opts *opts)
3128	{
3129	const char *pin_root_path;
3130	bool strict;
3131	int err = 0;
3132
3133	strict = !OPTS_GET(opts, relaxed_maps, false);
3134	pin_root_path = OPTS_GET(opts, pin_root_path, NULL);
3135
3136	err = bpf_object__init_user_btf_maps(obj, strict, pin_root_path);
3137	err = err ?: bpf_object__init_global_data_maps(obj);
3138	err = err ?: bpf_object__init_kconfig_map(obj);
3139	err = err ?: bpf_object_init_struct_ops(obj);
3140
3141	return err;
3142	}
3143
3144	static bool section_have_execinstr(struct bpf_object *obj, int idx)
3145	{
3146	Elf64_Shdr *sh;
3147
3148	sh = elf_sec_hdr(obj, elf_sec_by_idx(obj, idx));
3149	if (!sh)
3150	return false;
3151
3152	return sh->sh_flags & SHF_EXECINSTR;
3153	}
3154
3155	static bool starts_with_qmark(const char *s)
3156	{
3157	return s && s[0] == '?';
3158	}
3159
3160	static bool btf_needs_sanitization(struct bpf_object *obj)
3161	{
3162	bool has_func_global = kernel_supports(obj, FEAT_BTF_GLOBAL_FUNC);
3163	bool has_datasec = kernel_supports(obj, FEAT_BTF_DATASEC);
3164	bool has_float = kernel_supports(obj, FEAT_BTF_FLOAT);
3165	bool has_func = kernel_supports(obj, FEAT_BTF_FUNC);
3166	bool has_decl_tag = kernel_supports(obj, FEAT_BTF_DECL_TAG);
3167	bool has_type_tag = kernel_supports(obj, FEAT_BTF_TYPE_TAG);
3168	bool has_enum64 = kernel_supports(obj, FEAT_BTF_ENUM64);
3169	bool has_qmark_datasec = kernel_supports(obj, FEAT_BTF_QMARK_DATASEC);
3170	bool has_layout = kernel_supports(obj, FEAT_BTF_LAYOUT);
3171
3172	return !has_func \|\| !has_datasec \|\| !has_func_global \|\| !has_float \|\|
3173	!has_decl_tag \|\| !has_type_tag \|\| !has_enum64 \|\| !has_qmark_datasec \|\|
3174	!has_layout;
3175	}
3176
3177	struct btf bpf_object__sanitize_btf(struct bpf_object obj, struct btf *orig_btf)
3178	{
3179	bool has_func_global = kernel_supports(obj, FEAT_BTF_GLOBAL_FUNC);
3180	bool has_datasec = kernel_supports(obj, FEAT_BTF_DATASEC);
3181	bool has_float = kernel_supports(obj, FEAT_BTF_FLOAT);
3182	bool has_func = kernel_supports(obj, FEAT_BTF_FUNC);
3183	bool has_decl_tag = kernel_supports(obj, FEAT_BTF_DECL_TAG);
3184	bool has_type_tag = kernel_supports(obj, FEAT_BTF_TYPE_TAG);
3185	bool has_enum64 = kernel_supports(obj, FEAT_BTF_ENUM64);
3186	bool has_qmark_datasec = kernel_supports(obj, FEAT_BTF_QMARK_DATASEC);
3187	bool has_layout = kernel_supports(obj, FEAT_BTF_LAYOUT);
3188	int enum64_placeholder_id = 0;
3189	const struct btf_header *hdr;
3190	struct btf *btf = NULL;
3191	const void *raw_data;
3192	struct btf_type *t;
3193	int i, j, vlen;
3194	__u32 sz;
3195	int err;
3196
3197	/* clone BTF to sanitize a copy and leave the original intact */
3198	raw_data = btf__raw_data(orig_btf, &sz);
3199	if (!raw_data)
3200	return ERR_PTR(-ENOMEM);
3201	/* btf_header() gives us endian-safe header info */
3202	hdr = btf_header(orig_btf);
3203
3204	if (!has_layout && hdr->hdr_len >= sizeof(struct btf_header) &&
3205	(hdr->layout_len != 0 \|\| hdr->layout_off != 0)) {
3206	const struct btf_header *old_hdr = raw_data;
3207	struct btf_header *new_hdr;
3208	void *new_raw_data;
3209	__u32 new_str_off;
3210
3211	/*
3212	* Need to rewrite BTF to exclude layout information and
3213	* move string section to immediately after types.
3214	*/
3215	new_raw_data = malloc(sz);
3216	if (!new_raw_data)
3217	return ERR_PTR(-ENOMEM);
3218
3219	memcpy(new_raw_data, raw_data, sz);
3220	new_hdr = new_raw_data;
3221	new_hdr->layout_off = 0;
3222	new_hdr->layout_len = 0;
3223	new_str_off = hdr->type_off + hdr->type_len;
3224	/* Handle swapped endian case */
3225	if (old_hdr->magic != hdr->magic)
3226	new_hdr->str_off = bswap_32(new_str_off);
3227	else
3228	new_hdr->str_off = new_str_off;
3229
3230	memmove(new_raw_data + hdr->hdr_len + new_str_off,
3231	new_raw_data + hdr->hdr_len + hdr->str_off,
3232	hdr->str_len);
3233	sz = hdr->hdr_len + hdr->type_off + hdr->type_len + hdr->str_len;
3234	btf = btf__new(new_raw_data, sz);
3235	free(new_raw_data);
3236	} else {
3237	btf = btf__new(raw_data, sz);
3238	}
3239	err = libbpf_get_error(btf);
3240	if (err)
3241	return ERR_PTR(err);
3242
3243	/* enforce 8-byte pointers for BPF-targeted BTFs */
3244	btf__set_pointer_size(btf, 8);
3245
3246	for (i = 1; i < btf__type_cnt(btf); i++) {
3247	t = (struct btf_type *)btf__type_by_id(btf, i);
3248
3249	if ((!has_datasec && btf_is_var(t)) \|\| (!has_decl_tag && btf_is_decl_tag(t))) {
3250	/* replace VAR/DECL_TAG with INT */
3251	t->info = BTF_INFO_ENC(BTF_KIND_INT, 0, 0);
3252	/*
3253	* using size = 1 is the safest choice, 4 will be too
3254	* big and cause kernel BTF validation failure if
3255	* original variable took less than 4 bytes
3256	*/
3257	t->size = 1;
3258	(int )(t + 1) = BTF_INT_ENC(0, 0, 8);
3259	} else if (!has_datasec && btf_is_datasec(t)) {
3260	/* replace DATASEC with STRUCT */
3261	const struct btf_var_secinfo *v = btf_var_secinfos(t);
3262	struct btf_member *m = btf_members(t);
3263	struct btf_type *vt;
3264	char *name;
3265
3266	name = (char *)btf__name_by_offset(btf, t->name_off);
3267	while (*name) {
3268	if (name == '.' \|\| name == '?')
3269	*name = '_';
3270	name++;
3271	}
3272
3273	vlen = btf_vlen(t);
3274	t->info = BTF_INFO_ENC(BTF_KIND_STRUCT, 0, vlen);
3275	for (j = 0; j < vlen; j++, v++, m++) {
3276	/* order of field assignments is important */
3277	m->offset = v->offset * 8;
3278	m->type = v->type;
3279	/* preserve variable name as member name */
3280	vt = (void *)btf__type_by_id(btf, v->type);
3281	m->name_off = vt->name_off;
3282	}
3283	} else if (!has_qmark_datasec && btf_is_datasec(t) &&
3284	starts_with_qmark(btf__name_by_offset(btf, t->name_off))) {
3285	/* replace '?' prefix with '_' for DATASEC names */
3286	char *name;
3287
3288	name = (char *)btf__name_by_offset(btf, t->name_off);
3289	if (name[0] == '?')
3290	name[0] = '_';
3291	} else if (!has_func && btf_is_func_proto(t)) {
3292	/* replace FUNC_PROTO with ENUM */
3293	vlen = btf_vlen(t);
3294	t->info = BTF_INFO_ENC(BTF_KIND_ENUM, 0, vlen);
3295	t->size = sizeof(__u32); /* kernel enforced */
3296	} else if (!has_func && btf_is_func(t)) {
3297	/* replace FUNC with TYPEDEF */
3298	t->info = BTF_INFO_ENC(BTF_KIND_TYPEDEF, 0, 0);
3299	} else if (!has_func_global && btf_is_func(t)) {
3300	/* replace BTF_FUNC_GLOBAL with BTF_FUNC_STATIC */
3301	t->info = BTF_INFO_ENC(BTF_KIND_FUNC, 0, 0);
3302	} else if (!has_float && btf_is_float(t)) {
3303	/* replace FLOAT with an equally-sized empty STRUCT;
3304	* since C compilers do not accept e.g. "float" as a
3305	* valid struct name, make it anonymous
3306	*/
3307	t->name_off = 0;
3308	t->info = BTF_INFO_ENC(BTF_KIND_STRUCT, 0, 0);
3309	} else if (!has_type_tag && btf_is_type_tag(t)) {
3310	/* replace TYPE_TAG with a CONST */
3311	t->name_off = 0;
3312	t->info = BTF_INFO_ENC(BTF_KIND_CONST, 0, 0);
3313	} else if (!has_enum64 && btf_is_enum(t)) {
3314	/* clear the kflag */
3315	t->info = btf_type_info(btf_kind(t), btf_vlen(t), false);
3316	} else if (!has_enum64 && btf_is_enum64(t)) {
3317	/* replace ENUM64 with a union */
3318	struct btf_member *m;
3319
3320	if (enum64_placeholder_id == 0) {
3321	enum64_placeholder_id = btf__add_int(btf, "enum64_placeholder", 1, 0);
3322	if (enum64_placeholder_id < 0) {
3323	btf__free(btf);
3324	return ERR_PTR(enum64_placeholder_id);
3325	}
3326	t = (struct btf_type *)btf__type_by_id(btf, i);
3327	}
3328
3329	m = btf_members(t);
3330	vlen = btf_vlen(t);
3331	t->info = BTF_INFO_ENC(BTF_KIND_UNION, 0, vlen);
3332	for (j = 0; j < vlen; j++, m++) {
3333	m->type = enum64_placeholder_id;
3334	m->offset = 0;
3335	}
3336	}
3337	}
3338
3339	return btf;
3340	}
3341
3342	static bool libbpf_needs_btf(const struct bpf_object *obj)
3343	{
3344	return obj->efile.btf_maps_shndx >= 0 \|\|
3345	obj->efile.has_st_ops \|\|
3346	obj->nr_extern > 0;
3347	}
3348
3349	static bool kernel_needs_btf(const struct bpf_object *obj)
3350	{
3351	return obj->efile.has_st_ops;
3352	}
3353
3354	static int bpf_object__init_btf(struct bpf_object *obj,
3355	Elf_Data *btf_data,
3356	Elf_Data *btf_ext_data)
3357	{
3358	int err = -ENOENT;
3359
3360	if (btf_data) {
3361	obj->btf = btf__new(btf_data->d_buf, btf_data->d_size);
3362	err = libbpf_get_error(obj->btf);
3363	if (err) {
3364	obj->btf = NULL;
3365	pr_warn("Error loading ELF section %s: %s.\n", BTF_ELF_SEC, errstr(err));
3366	goto out;
3367	}
3368	/* enforce 8-byte pointers for BPF-targeted BTFs */
3369	btf__set_pointer_size(obj->btf, 8);
3370	}
3371	if (btf_ext_data) {
3372	struct btf_ext_info *ext_segs[3];
3373	int seg_num, sec_num;
3374
3375	if (!obj->btf) {
3376	pr_debug("Ignore ELF section %s because its depending ELF section %s is not found.\n",
3377	BTF_EXT_ELF_SEC, BTF_ELF_SEC);
3378	goto out;
3379	}
3380	obj->btf_ext = btf_ext__new(btf_ext_data->d_buf, btf_ext_data->d_size);
3381	err = libbpf_get_error(obj->btf_ext);
3382	if (err) {
3383	pr_warn("Error loading ELF section %s: %s. Ignored and continue.\n",
3384	BTF_EXT_ELF_SEC, errstr(err));
3385	obj->btf_ext = NULL;
3386	goto out;
3387	}
3388
3389	/* setup .BTF.ext to ELF section mapping */
3390	ext_segs[0] = &obj->btf_ext->func_info;
3391	ext_segs[1] = &obj->btf_ext->line_info;
3392	ext_segs[2] = &obj->btf_ext->core_relo_info;
3393	for (seg_num = 0; seg_num < ARRAY_SIZE(ext_segs); seg_num++) {
3394	struct btf_ext_info *seg = ext_segs[seg_num];
3395	const struct btf_ext_info_sec *sec;
3396	const char *sec_name;
3397	Elf_Scn *scn;
3398
3399	if (seg->sec_cnt == 0)
3400	continue;
3401
3402	seg->sec_idxs = calloc(seg->sec_cnt, sizeof(*seg->sec_idxs));
3403	if (!seg->sec_idxs) {
3404	err = -ENOMEM;
3405	goto out;
3406	}
3407
3408	sec_num = 0;
3409	for_each_btf_ext_sec(seg, sec) {
3410	/* preventively increment index to avoid doing
3411	* this before every continue below
3412	*/
3413	sec_num++;
3414
3415	sec_name = btf__name_by_offset(obj->btf, sec->sec_name_off);
3416	if (str_is_empty(sec_name))
3417	continue;
3418	scn = elf_sec_by_name(obj, sec_name);
3419	if (!scn)
3420	continue;
3421
3422	seg->sec_idxs[sec_num - 1] = elf_ndxscn(scn);
3423	}
3424	}
3425	}
3426	out:
3427	if (err && libbpf_needs_btf(obj)) {
3428	pr_warn("BTF is required, but is missing or corrupted.\n");
3429	return err;
3430	}
3431	return 0;
3432	}
3433
3434	static int compare_vsi_off(const void _a, const void _b)
3435	{
3436	const struct btf_var_secinfo *a = _a;
3437	const struct btf_var_secinfo *b = _b;
3438
3439	return a->offset - b->offset;
3440	}
3441
3442	static int btf_fixup_datasec(struct bpf_object obj, struct btf btf,
3443	struct btf_type *t)
3444	{
3445	__u32 size = 0, i, vars = btf_vlen(t);
3446	const char *sec_name = btf__name_by_offset(btf, t->name_off);
3447	struct btf_var_secinfo *vsi;
3448	bool fixup_offsets = false;
3449	int err;
3450
3451	if (!sec_name) {
3452	pr_debug("No name found in string section for DATASEC kind.\n");
3453	return -ENOENT;
3454	}
3455
3456	/* Extern-backing datasecs (.ksyms, .kconfig) have their size and
3457	* variable offsets set at the previous step. Further, not every
3458	* extern BTF VAR has corresponding ELF symbol preserved, so we skip
3459	* all fixups altogether for such sections and go straight to sorting
3460	* VARs within their DATASEC.
3461	*/
3462	if (strcmp(sec_name, KCONFIG_SEC) == 0 \|\| strcmp(sec_name, KSYMS_SEC) == 0)
3463	goto sort_vars;
3464
3465	/* Clang leaves DATASEC size and VAR offsets as zeroes, so we need to
3466	* fix this up. But BPF static linker already fixes this up and fills
3467	* all the sizes and offsets during static linking. So this step has
3468	* to be optional. But the STV_HIDDEN handling is non-optional for any
3469	* non-extern DATASEC, so the variable fixup loop below handles both
3470	* functions at the same time, paying the cost of BTF VAR <-> ELF
3471	* symbol matching just once.
3472	*/
3473	if (t->size == 0) {
3474	err = find_elf_sec_sz(obj, sec_name, &size);
3475	if (err \|\| !size) {
3476	pr_debug("sec '%s': failed to determine size from ELF: size %u, err %s\n",
3477	sec_name, size, errstr(err));
3478	return -ENOENT;
3479	}
3480
3481	t->size = size;
3482	fixup_offsets = true;
3483	}
3484
3485	for (i = 0, vsi = btf_var_secinfos(t); i < vars; i++, vsi++) {
3486	const struct btf_type *t_var;
3487	struct btf_var *var;
3488	const char *var_name;
3489	Elf64_Sym *sym;
3490
3491	t_var = btf__type_by_id(btf, vsi->type);
3492	if (!t_var \|\| !btf_is_var(t_var)) {
3493	pr_debug("sec '%s': unexpected non-VAR type found\n", sec_name);
3494	return -EINVAL;
3495	}
3496
3497	var = btf_var(t_var);
3498	if (var->linkage == BTF_VAR_STATIC \|\| var->linkage == BTF_VAR_GLOBAL_EXTERN)
3499	continue;
3500
3501	var_name = btf__name_by_offset(btf, t_var->name_off);
3502	if (!var_name) {
3503	pr_debug("sec '%s': failed to find name of DATASEC's member #%u\n",
3504	sec_name, i);
3505	return -ENOENT;
3506	}
3507
3508	sym = find_elf_var_sym(obj, var_name);
3509	if (IS_ERR(sym)) {
3510	pr_debug("sec '%s': failed to find ELF symbol for VAR '%s'\n",
3511	sec_name, var_name);
3512	return -ENOENT;
3513	}
3514
3515	if (fixup_offsets)
3516	vsi->offset = sym->st_value;
3517
3518	/* if variable is a global/weak symbol, but has restricted
3519	* (STV_HIDDEN or STV_INTERNAL) visibility, mark its BTF VAR
3520	* as static. This follows similar logic for functions (BPF
3521	* subprogs) and influences libbpf's further decisions about
3522	* whether to make global data BPF array maps as
3523	* BPF_F_MMAPABLE.
3524	*/
3525	if (ELF64_ST_VISIBILITY(sym->st_other) == STV_HIDDEN
3526	\|\| ELF64_ST_VISIBILITY(sym->st_other) == STV_INTERNAL)
3527	var->linkage = BTF_VAR_STATIC;
3528	}
3529
3530	sort_vars:
3531	qsort(btf_var_secinfos(t), vars, sizeof(*vsi), compare_vsi_off);
3532	return 0;
3533	}
3534
3535	static int bpf_object_fixup_btf(struct bpf_object *obj)
3536	{
3537	int i, n, err = 0;
3538
3539	if (!obj->btf)
3540	return 0;
3541
3542	n = btf__type_cnt(obj->btf);
3543	for (i = 1; i < n; i++) {
3544	struct btf_type *t = btf_type_by_id(obj->btf, i);
3545
3546	/* Loader needs to fix up some of the things compiler
3547	* couldn't get its hands on while emitting BTF. This
3548	* is section size and global variable offset. We use
3549	* the info from the ELF itself for this purpose.
3550	*/
3551	if (btf_is_datasec(t)) {
3552	err = btf_fixup_datasec(obj, obj->btf, t);
3553	if (err)
3554	return err;
3555	}
3556	}
3557
3558	return 0;
3559	}
3560
3561	static bool prog_needs_vmlinux_btf(struct bpf_program *prog)
3562	{
3563	if (prog->type == BPF_PROG_TYPE_STRUCT_OPS \|\|
3564	prog->type == BPF_PROG_TYPE_LSM)
3565	return true;
3566
3567	/* BPF_PROG_TYPE_TRACING programs which do not attach to other programs
3568	* also need vmlinux BTF
3569	*/
3570	if (prog->type == BPF_PROG_TYPE_TRACING && !prog->attach_prog_fd)
3571	return true;
3572
3573	return false;
3574	}
3575
3576	static bool map_needs_vmlinux_btf(struct bpf_map *map)
3577	{
3578	return bpf_map__is_struct_ops(map);
3579	}
3580
3581	static bool obj_needs_vmlinux_btf(const struct bpf_object *obj)
3582	{
3583	struct bpf_program *prog;
3584	struct bpf_map *map;
3585	int i;
3586
3587	/* CO-RE relocations need kernel BTF, only when btf_custom_path
3588	* is not specified
3589	*/
3590	if (obj->btf_ext && obj->btf_ext->core_relo_info.len && !obj->btf_custom_path)
3591	return true;
3592
3593	/* Support for typed ksyms needs kernel BTF */
3594	for (i = 0; i < obj->nr_extern; i++) {
3595	const struct extern_desc *ext;
3596
3597	ext = &obj->externs[i];
3598	if (ext->type == EXT_KSYM && ext->ksym.type_id)
3599	return true;
3600	}
3601
3602	bpf_object__for_each_program(prog, obj) {
3603	if (!prog->autoload)
3604	continue;
3605	if (prog_needs_vmlinux_btf(prog))
3606	return true;
3607	}
3608
3609	bpf_object__for_each_map(map, obj) {
3610	if (map_needs_vmlinux_btf(map))
3611	return true;
3612	}
3613
3614	return false;
3615	}
3616
3617	static int bpf_object__load_vmlinux_btf(struct bpf_object *obj, bool force)
3618	{
3619	int err;
3620
3621	/* btf_vmlinux could be loaded earlier */
3622	if (obj->btf_vmlinux \|\| obj->gen_loader)
3623	return 0;
3624
3625	if (!force && !obj_needs_vmlinux_btf(obj))
3626	return 0;
3627
3628	obj->btf_vmlinux = btf__load_vmlinux_btf();
3629	err = libbpf_get_error(obj->btf_vmlinux);
3630	if (err) {
3631	pr_warn("Error loading vmlinux BTF: %s\n", errstr(err));
3632	obj->btf_vmlinux = NULL;
3633	return err;
3634	}
3635	return 0;
3636	}
3637
3638	static int bpf_object__sanitize_and_load_btf(struct bpf_object *obj)
3639	{
3640	struct btf *kern_btf = obj->btf;
3641	bool btf_mandatory, sanitize;
3642	int i, err = 0;
3643
3644	if (!obj->btf)
3645	return 0;
3646
3647	if (!kernel_supports(obj, FEAT_BTF)) {
3648	if (kernel_needs_btf(obj)) {
3649	err = -EOPNOTSUPP;
3650	goto report;
3651	}
3652	pr_debug("Kernel doesn't support BTF, skipping uploading it.\n");
3653	return 0;
3654	}
3655
3656	/* Even though some subprogs are global/weak, user might prefer more
3657	* permissive BPF verification process that BPF verifier performs for
3658	* static functions, taking into account more context from the caller
3659	* functions. In such case, they need to mark such subprogs with
3660	* __attribute__((visibility("hidden"))) and libbpf will adjust
3661	* corresponding FUNC BTF type to be marked as static and trigger more
3662	* involved BPF verification process.
3663	*/
3664	for (i = 0; i < obj->nr_programs; i++) {
3665	struct bpf_program *prog = &obj->programs[i];
3666	struct btf_type *t;
3667	const char *name;
3668	int j, n;
3669
3670	if (!prog->mark_btf_static \|\| !prog_is_subprog(obj, prog))
3671	continue;
3672
3673	n = btf__type_cnt(obj->btf);
3674	for (j = 1; j < n; j++) {
3675	t = btf_type_by_id(obj->btf, j);
3676	if (!btf_is_func(t) \|\| btf_func_linkage(t) != BTF_FUNC_GLOBAL)
3677	continue;
3678
3679	name = btf__str_by_offset(obj->btf, t->name_off);
3680	if (strcmp(name, prog->name) != 0)
3681	continue;
3682
3683	t->info = btf_type_info(BTF_KIND_FUNC, BTF_FUNC_STATIC, 0);
3684	break;
3685	}
3686	}
3687
3688	sanitize = btf_needs_sanitization(obj);
3689	if (sanitize) {
3690	kern_btf = bpf_object__sanitize_btf(obj, obj->btf);
3691	if (IS_ERR(kern_btf))
3692	return PTR_ERR(kern_btf);
3693	}
3694
3695	if (obj->gen_loader) {
3696	__u32 raw_size = 0;
3697	const void *raw_data = btf__raw_data(kern_btf, &raw_size);
3698
3699	if (!raw_data)
3700	return -ENOMEM;
3701	bpf_gen__load_btf(obj->gen_loader, raw_data, raw_size);
3702	/* Pretend to have valid FD to pass various fd >= 0 checks.
3703	* This fd == 0 will not be used with any syscall and will be reset to -1 eventually.
3704	*/
3705	btf__set_fd(kern_btf, 0);
3706	} else {
3707	/* currently BPF_BTF_LOAD only supports log_level 1 */
3708	err = btf_load_into_kernel(kern_btf, obj->log_buf, obj->log_size,
3709	obj->log_level ? 1 : 0, obj->token_fd);
3710	}
3711	if (sanitize) {
3712	if (!err) {
3713	/* move fd to libbpf's BTF */
3714	btf__set_fd(obj->btf, btf__fd(kern_btf));
3715	btf__set_fd(kern_btf, -1);
3716	}
3717	btf__free(kern_btf);
3718	}
3719	report:
3720	if (err) {
3721	btf_mandatory = kernel_needs_btf(obj);
3722	if (btf_mandatory) {
3723	pr_warn("Error loading .BTF into kernel: %s. BTF is mandatory, can't proceed.\n",
3724	errstr(err));
3725	} else {
3726	pr_info("Error loading .BTF into kernel: %s. BTF is optional, ignoring.\n",
3727	errstr(err));
3728	err = 0;
3729	}
3730	}
3731	return err;
3732	}
3733
3734	static const char elf_sym_str(const struct bpf_object obj, size_t off)
3735	{
3736	const char *name;
3737
3738	name = elf_strptr(obj->efile.elf, obj->efile.strtabidx, off);
3739	if (!name) {
3740	pr_warn("elf: failed to get section name string at offset %zu from %s: %s\n",
3741	off, obj->path, elf_errmsg(-1));
3742	return NULL;
3743	}
3744
3745	return name;
3746	}
3747
3748	static const char elf_sec_str(const struct bpf_object obj, size_t off)
3749	{
3750	const char *name;
3751
3752	name = elf_strptr(obj->efile.elf, obj->efile.shstrndx, off);
3753	if (!name) {
3754	pr_warn("elf: failed to get section name string at offset %zu from %s: %s\n",
3755	off, obj->path, elf_errmsg(-1));
3756	return NULL;
3757	}
3758
3759	return name;
3760	}
3761
3762	static Elf_Scn elf_sec_by_idx(const struct bpf_object obj, size_t idx)
3763	{
3764	Elf_Scn *scn;
3765
3766	scn = elf_getscn(obj->efile.elf, idx);
3767	if (!scn) {
3768	pr_warn("elf: failed to get section(%zu) from %s: %s\n",
3769	idx, obj->path, elf_errmsg(-1));
3770	return NULL;
3771	}
3772	return scn;
3773	}
3774
3775	static Elf_Scn elf_sec_by_name(const struct bpf_object obj, const char *name)
3776	{
3777	Elf_Scn *scn = NULL;
3778	Elf *elf = obj->efile.elf;
3779	const char *sec_name;
3780
3781	while ((scn = elf_nextscn(elf, scn)) != NULL) {
3782	sec_name = elf_sec_name(obj, scn);
3783	if (!sec_name)
3784	return NULL;
3785
3786	if (strcmp(sec_name, name) != 0)
3787	continue;
3788
3789	return scn;
3790	}
3791	return NULL;
3792	}
3793
3794	static Elf64_Shdr elf_sec_hdr(const struct bpf_object obj, Elf_Scn *scn)
3795	{
3796	Elf64_Shdr *shdr;
3797
3798	if (!scn)
3799	return NULL;
3800
3801	shdr = elf64_getshdr(scn);
3802	if (!shdr) {
3803	pr_warn("elf: failed to get section(%zu) header from %s: %s\n",
3804	elf_ndxscn(scn), obj->path, elf_errmsg(-1));
3805	return NULL;
3806	}
3807
3808	return shdr;
3809	}
3810
3811	static const char elf_sec_name(const struct bpf_object obj, Elf_Scn *scn)
3812	{
3813	const char *name;
3814	Elf64_Shdr *sh;
3815
3816	if (!scn)
3817	return NULL;
3818
3819	sh = elf_sec_hdr(obj, scn);
3820	if (!sh)
3821	return NULL;
3822
3823	name = elf_sec_str(obj, sh->sh_name);
3824	if (!name) {
3825	pr_warn("elf: failed to get section(%zu) name from %s: %s\n",
3826	elf_ndxscn(scn), obj->path, elf_errmsg(-1));
3827	return NULL;
3828	}
3829
3830	return name;
3831	}
3832
3833	static Elf_Data elf_sec_data(const struct bpf_object obj, Elf_Scn *scn)
3834	{
3835	Elf_Data *data;
3836
3837	if (!scn)
3838	return NULL;
3839
3840	data = elf_getdata(scn, 0);
3841	if (!data) {
3842	pr_warn("elf: failed to get section(%zu) %s data from %s: %s\n",
3843	elf_ndxscn(scn), elf_sec_name(obj, scn) ?: "<?>",
3844	obj->path, elf_errmsg(-1));
3845	return NULL;
3846	}
3847
3848	return data;
3849	}
3850
3851	static Elf64_Sym elf_sym_by_idx(const struct bpf_object obj, size_t idx)
3852	{
3853	if (idx >= obj->efile.symbols->d_size / sizeof(Elf64_Sym))
3854	return NULL;
3855
3856	return (Elf64_Sym *)obj->efile.symbols->d_buf + idx;
3857	}
3858
3859	static Elf64_Rel elf_rel_by_idx(Elf_Data data, size_t idx)
3860	{
3861	if (idx >= data->d_size / sizeof(Elf64_Rel))
3862	return NULL;
3863
3864	return (Elf64_Rel *)data->d_buf + idx;
3865	}
3866
3867	static bool is_sec_name_dwarf(const char *name)
3868	{
3869	/* approximation, but the actual list is too long */
3870	return str_has_pfx(name, ".debug_");
3871	}
3872
3873	static bool ignore_elf_section(Elf64_Shdr hdr, const char name)
3874	{
3875	/* no special handling of .strtab */
3876	if (hdr->sh_type == SHT_STRTAB)
3877	return true;
3878
3879	/* ignore .llvm_addrsig section as well */
3880	if (hdr->sh_type == SHT_LLVM_ADDRSIG)
3881	return true;
3882
3883	/* no subprograms will lead to an empty .text section, ignore it */
3884	if (hdr->sh_type == SHT_PROGBITS && hdr->sh_size == 0 &&
3885	strcmp(name, ".text") == 0)
3886	return true;
3887
3888	/* DWARF sections */
3889	if (is_sec_name_dwarf(name))
3890	return true;
3891
3892	if (str_has_pfx(name, ".rel")) {
3893	name += sizeof(".rel") - 1;
3894	/* DWARF section relocations */
3895	if (is_sec_name_dwarf(name))
3896	return true;
3897
3898	/* .BTF and .BTF.ext don't need relocations */
3899	if (strcmp(name, BTF_ELF_SEC) == 0 \|\|
3900	strcmp(name, BTF_EXT_ELF_SEC) == 0)
3901	return true;
3902	}
3903
3904	return false;
3905	}
3906
3907	static int cmp_progs(const void _a, const void _b)
3908	{
3909	const struct bpf_program *a = _a;
3910	const struct bpf_program *b = _b;
3911
3912	if (a->sec_idx != b->sec_idx)
3913	return a->sec_idx < b->sec_idx ? -1 : 1;
3914
3915	/* sec_insn_off can't be the same within the section */
3916	return a->sec_insn_off < b->sec_insn_off ? -1 : 1;
3917	}
3918
3919	static int bpf_object__elf_collect(struct bpf_object *obj)
3920	{
3921	struct elf_sec_desc *sec_desc;
3922	Elf *elf = obj->efile.elf;
3923	Elf_Data *btf_ext_data = NULL;
3924	Elf_Data *btf_data = NULL;
3925	int idx = 0, err = 0;
3926	const char *name;
3927	Elf_Data *data;
3928	Elf_Scn *scn;
3929	Elf64_Shdr *sh;
3930
3931	/* ELF section indices are 0-based, but sec #0 is special "invalid"
3932	* section. Since section count retrieved by elf_getshdrnum() does
3933	* include sec #0, it is already the necessary size of an array to keep
3934	* all the sections.
3935	*/
3936	if (elf_getshdrnum(obj->efile.elf, &obj->efile.sec_cnt)) {
3937	pr_warn("elf: failed to get the number of sections for %s: %s\n",
3938	obj->path, elf_errmsg(-1));
3939	return -LIBBPF_ERRNO__FORMAT;
3940	}
3941	obj->efile.secs = calloc(obj->efile.sec_cnt, sizeof(*obj->efile.secs));
3942	if (!obj->efile.secs)
3943	return -ENOMEM;
3944
3945	/* a bunch of ELF parsing functionality depends on processing symbols,
3946	* so do the first pass and find the symbol table
3947	*/
3948	scn = NULL;
3949	while ((scn = elf_nextscn(elf, scn)) != NULL) {
3950	sh = elf_sec_hdr(obj, scn);
3951	if (!sh)
3952	return -LIBBPF_ERRNO__FORMAT;
3953
3954	if (sh->sh_type == SHT_SYMTAB) {
3955	if (obj->efile.symbols) {
3956	pr_warn("elf: multiple symbol tables in %s\n", obj->path);
3957	return -LIBBPF_ERRNO__FORMAT;
3958	}
3959
3960	data = elf_sec_data(obj, scn);
3961	if (!data)
3962	return -LIBBPF_ERRNO__FORMAT;
3963
3964	idx = elf_ndxscn(scn);
3965
3966	obj->efile.symbols = data;
3967	obj->efile.symbols_shndx = idx;
3968	obj->efile.strtabidx = sh->sh_link;
3969	}
3970	}
3971
3972	if (!obj->efile.symbols) {
3973	pr_warn("elf: couldn't find symbol table in %s, stripped object file?\n",
3974	obj->path);
3975	return -ENOENT;
3976	}
3977
3978	scn = NULL;
3979	while ((scn = elf_nextscn(elf, scn)) != NULL) {
3980	idx = elf_ndxscn(scn);
3981	sec_desc = &obj->efile.secs[idx];
3982
3983	sh = elf_sec_hdr(obj, scn);
3984	if (!sh)
3985	return -LIBBPF_ERRNO__FORMAT;
3986
3987	name = elf_sec_str(obj, sh->sh_name);
3988	if (!name)
3989	return -LIBBPF_ERRNO__FORMAT;
3990
3991	if (ignore_elf_section(sh, name))
3992	continue;
3993
3994	data = elf_sec_data(obj, scn);
3995	if (!data)
3996	return -LIBBPF_ERRNO__FORMAT;
3997
3998	pr_debug("elf: section(%d) %s, size %lu, link %d, flags %lx, type=%d\n",
3999	idx, name, (unsigned long)data->d_size,
4000	(int)sh->sh_link, (unsigned long)sh->sh_flags,
4001	(int)sh->sh_type);
4002
4003	if (strcmp(name, "license") == 0) {
4004	err = bpf_object__init_license(obj, data->d_buf, data->d_size);
4005	if (err)
4006	return err;
4007	} else if (strcmp(name, "version") == 0) {
4008	err = bpf_object__init_kversion(obj, data->d_buf, data->d_size);
4009	if (err)
4010	return err;
4011	} else if (strcmp(name, "maps") == 0) {
4012	pr_warn("elf: legacy map definitions in 'maps' section are not supported by libbpf v1.0+\n");
4013	return -ENOTSUP;
4014	} else if (strcmp(name, MAPS_ELF_SEC) == 0) {
4015	obj->efile.btf_maps_shndx = idx;
4016	} else if (strcmp(name, BTF_ELF_SEC) == 0) {
4017	if (sh->sh_type != SHT_PROGBITS)
4018	return -LIBBPF_ERRNO__FORMAT;
4019	btf_data = data;
4020	} else if (strcmp(name, BTF_EXT_ELF_SEC) == 0) {
4021	if (sh->sh_type != SHT_PROGBITS)
4022	return -LIBBPF_ERRNO__FORMAT;
4023	btf_ext_data = data;
4024	} else if (sh->sh_type == SHT_SYMTAB) {
4025	/* already processed during the first pass above */
4026	} else if (sh->sh_type == SHT_PROGBITS && data->d_size > 0) {
4027	if (sh->sh_flags & SHF_EXECINSTR) {
4028	if (strcmp(name, ".text") == 0)
4029	obj->efile.text_shndx = idx;
4030	err = bpf_object__add_programs(obj, data, name, idx);
4031	if (err)
4032	return err;
4033	} else if (strcmp(name, DATA_SEC) == 0 \|\|
4034	str_has_pfx(name, DATA_SEC ".")) {
4035	sec_desc->sec_type = SEC_DATA;
4036	sec_desc->shdr = sh;
4037	sec_desc->data = data;
4038	} else if (strcmp(name, RODATA_SEC) == 0 \|\|
4039	str_has_pfx(name, RODATA_SEC ".")) {
4040	sec_desc->sec_type = SEC_RODATA;
4041	sec_desc->shdr = sh;
4042	sec_desc->data = data;
4043	} else if (strcmp(name, PERCPU_SEC) == 0 \|\|
4044	str_has_pfx(name, PERCPU_SEC ".")) {
4045	sec_desc->sec_type = SEC_PERCPU;
4046	sec_desc->shdr = sh;
4047	sec_desc->data = data;
4048	} else if (strcmp(name, STRUCT_OPS_SEC) == 0 \|\|
4049	strcmp(name, STRUCT_OPS_LINK_SEC) == 0 \|\|
4050	strcmp(name, "?" STRUCT_OPS_SEC) == 0 \|\|
4051	strcmp(name, "?" STRUCT_OPS_LINK_SEC) == 0) {
4052	sec_desc->sec_type = SEC_ST_OPS;
4053	sec_desc->shdr = sh;
4054	sec_desc->data = data;
4055	obj->efile.has_st_ops = true;
4056	} else if (strcmp(name, ARENA_SEC) == 0) {
4057	obj->efile.arena_data = data;
4058	obj->efile.arena_data_shndx = idx;
4059	} else if (strcmp(name, JUMPTABLES_SEC) == 0) {
4060	obj->jumptables_data = malloc(data->d_size);
4061	if (!obj->jumptables_data)
4062	return -ENOMEM;
4063	memcpy(obj->jumptables_data, data->d_buf, data->d_size);
4064	obj->jumptables_data_sz = data->d_size;
4065	obj->efile.jumptables_data_shndx = idx;
4066	} else {
4067	pr_info("elf: skipping unrecognized data section(%d) %s\n",
4068	idx, name);
4069	}
4070	} else if (sh->sh_type == SHT_REL) {
4071	int targ_sec_idx = sh->sh_info; /* points to other section */
4072
4073	if (sh->sh_entsize != sizeof(Elf64_Rel) \|\|
4074	targ_sec_idx >= obj->efile.sec_cnt)
4075	return -LIBBPF_ERRNO__FORMAT;
4076
4077	/* Only do relo for section with exec instructions */
4078	if (!section_have_execinstr(obj, targ_sec_idx) &&
4079	strcmp(name, ".rel" STRUCT_OPS_SEC) &&
4080	strcmp(name, ".rel" STRUCT_OPS_LINK_SEC) &&
4081	strcmp(name, ".rel?" STRUCT_OPS_SEC) &&
4082	strcmp(name, ".rel?" STRUCT_OPS_LINK_SEC) &&
4083	strcmp(name, ".rel" MAPS_ELF_SEC)) {
4084	pr_info("elf: skipping relo section(%d) %s for section(%d) %s\n",
4085	idx, name, targ_sec_idx,
4086	elf_sec_name(obj, elf_sec_by_idx(obj, targ_sec_idx)) ?: "<?>");
4087	continue;
4088	}
4089
4090	sec_desc->sec_type = SEC_RELO;
4091	sec_desc->shdr = sh;
4092	sec_desc->data = data;
4093	} else if (sh->sh_type == SHT_NOBITS && (strcmp(name, BSS_SEC) == 0 \|\|
4094	str_has_pfx(name, BSS_SEC "."))) {
4095	sec_desc->sec_type = SEC_BSS;
4096	sec_desc->shdr = sh;
4097	sec_desc->data = data;
4098	} else {
4099	pr_info("elf: skipping section(%d) %s (size %zu)\n", idx, name,
4100	(size_t)sh->sh_size);
4101	}
4102	}
4103
4104	if (!obj->efile.strtabidx \|\| obj->efile.strtabidx > idx) {
4105	pr_warn("elf: symbol strings section missing or invalid in %s\n", obj->path);
4106	return -LIBBPF_ERRNO__FORMAT;
4107	}
4108
4109	/* change BPF program insns to native endianness for introspection */
4110	if (!is_native_endianness(obj))
4111	bpf_object_bswap_progs(obj);
4112
4113	/* sort BPF programs by section name and in-section instruction offset
4114	* for faster search
4115	*/
4116	if (obj->nr_programs)
4117	qsort(obj->programs, obj->nr_programs, sizeof(*obj->programs), cmp_progs);
4118
4119	return bpf_object__init_btf(obj, btf_data, btf_ext_data);
4120	}
4121
4122	static bool sym_is_extern(const Elf64_Sym *sym)
4123	{
4124	int bind = ELF64_ST_BIND(sym->st_info);
4125	/* externs are symbols w/ type=NOTYPE, bind=GLOBAL\|WEAK, section=UND */
4126	return sym->st_shndx == SHN_UNDEF &&
4127	(bind == STB_GLOBAL \|\| bind == STB_WEAK) &&
4128	ELF64_ST_TYPE(sym->st_info) == STT_NOTYPE;
4129	}
4130
4131	static bool sym_is_subprog(const Elf64_Sym *sym, int text_shndx)
4132	{
4133	int bind = ELF64_ST_BIND(sym->st_info);
4134	int type = ELF64_ST_TYPE(sym->st_info);
4135
4136	/* in .text section */
4137	if (sym->st_shndx != text_shndx)
4138	return false;
4139
4140	/* local function */
4141	if (bind == STB_LOCAL && type == STT_SECTION)
4142	return true;
4143
4144	/* global function */
4145	return (bind == STB_GLOBAL \|\| bind == STB_WEAK) && type == STT_FUNC;
4146	}
4147
4148	static int find_extern_btf_id(const struct btf btf, const char ext_name)
4149	{
4150	const struct btf_type *t;
4151	const char *tname;
4152	int i, n;
4153
4154	if (!btf)
4155	return -ESRCH;
4156
4157	n = btf__type_cnt(btf);
4158	for (i = 1; i < n; i++) {
4159	t = btf__type_by_id(btf, i);
4160
4161	if (!btf_is_var(t) && !btf_is_func(t))
4162	continue;
4163
4164	tname = btf__name_by_offset(btf, t->name_off);
4165	if (strcmp(tname, ext_name))
4166	continue;
4167
4168	if (btf_is_var(t) &&
4169	btf_var(t)->linkage != BTF_VAR_GLOBAL_EXTERN)
4170	return -EINVAL;
4171
4172	if (btf_is_func(t) && btf_func_linkage(t) != BTF_FUNC_EXTERN)
4173	return -EINVAL;
4174
4175	return i;
4176	}
4177
4178	return -ENOENT;
4179	}
4180
4181	static int find_extern_sec_btf_id(struct btf *btf, int ext_btf_id) {
4182	const struct btf_var_secinfo *vs;
4183	const struct btf_type *t;
4184	int i, j, n;
4185
4186	if (!btf)
4187	return -ESRCH;
4188
4189	n = btf__type_cnt(btf);
4190	for (i = 1; i < n; i++) {
4191	t = btf__type_by_id(btf, i);
4192
4193	if (!btf_is_datasec(t))
4194	continue;
4195
4196	vs = btf_var_secinfos(t);
4197	for (j = 0; j < btf_vlen(t); j++, vs++) {
4198	if (vs->type == ext_btf_id)
4199	return i;
4200	}
4201	}
4202
4203	return -ENOENT;
4204	}
4205
4206	static enum kcfg_type find_kcfg_type(const struct btf *btf, int id,
4207	bool *is_signed)
4208	{
4209	const struct btf_type *t;
4210	const char *name;
4211
4212	t = skip_mods_and_typedefs(btf, id, NULL);
4213	name = btf__name_by_offset(btf, t->name_off);
4214
4215	if (is_signed)
4216	*is_signed = false;
4217	switch (btf_kind(t)) {
4218	case BTF_KIND_INT: {
4219	int enc = btf_int_encoding(t);
4220
4221	if (enc & BTF_INT_BOOL)
4222	return t->size == 1 ? KCFG_BOOL : KCFG_UNKNOWN;
4223	if (is_signed)
4224	*is_signed = enc & BTF_INT_SIGNED;
4225	if (t->size == 1)
4226	return KCFG_CHAR;
4227	if (t->size < 1 \|\| t->size > 8 \|\| (t->size & (t->size - 1)))
4228	return KCFG_UNKNOWN;
4229	return KCFG_INT;
4230	}
4231	case BTF_KIND_ENUM:
4232	if (t->size != 4)
4233	return KCFG_UNKNOWN;
4234	if (strcmp(name, "libbpf_tristate"))
4235	return KCFG_UNKNOWN;
4236	return KCFG_TRISTATE;
4237	case BTF_KIND_ENUM64:
4238	if (strcmp(name, "libbpf_tristate"))
4239	return KCFG_UNKNOWN;
4240	return KCFG_TRISTATE;
4241	case BTF_KIND_ARRAY:
4242	if (btf_array(t)->nelems == 0)
4243	return KCFG_UNKNOWN;
4244	if (find_kcfg_type(btf, btf_array(t)->type, NULL) != KCFG_CHAR)
4245	return KCFG_UNKNOWN;
4246	return KCFG_CHAR_ARR;
4247	default:
4248	return KCFG_UNKNOWN;
4249	}
4250	}
4251
4252	static int cmp_externs(const void _a, const void _b)
4253	{
4254	const struct extern_desc *a = _a;
4255	const struct extern_desc *b = _b;
4256
4257	if (a->type != b->type)
4258	return a->type < b->type ? -1 : 1;
4259
4260	if (a->type == EXT_KCFG) {
4261	/* descending order by alignment requirements */
4262	if (a->kcfg.align != b->kcfg.align)
4263	return a->kcfg.align > b->kcfg.align ? -1 : 1;
4264	/* ascending order by size, within same alignment class */
4265	if (a->kcfg.sz != b->kcfg.sz)
4266	return a->kcfg.sz < b->kcfg.sz ? -1 : 1;
4267	}
4268
4269	/* resolve ties by name */
4270	return strcmp(a->name, b->name);
4271	}
4272
4273	static int find_int_btf_id(const struct btf *btf)
4274	{
4275	const struct btf_type *t;
4276	int i, n;
4277
4278	n = btf__type_cnt(btf);
4279	for (i = 1; i < n; i++) {
4280	t = btf__type_by_id(btf, i);
4281
4282	if (btf_is_int(t) && btf_int_bits(t) == 32)
4283	return i;
4284	}
4285
4286	return 0;
4287	}
4288
4289	static int add_dummy_ksym_var(struct btf *btf)
4290	{
4291	int i, int_btf_id, sec_btf_id, dummy_var_btf_id;
4292	const struct btf_var_secinfo *vs;
4293	const struct btf_type *sec;
4294
4295	if (!btf)
4296	return 0;
4297
4298	sec_btf_id = btf__find_by_name_kind(btf, KSYMS_SEC,
4299	BTF_KIND_DATASEC);
4300	if (sec_btf_id < 0)
4301	return 0;
4302
4303	sec = btf__type_by_id(btf, sec_btf_id);
4304	vs = btf_var_secinfos(sec);
4305	for (i = 0; i < btf_vlen(sec); i++, vs++) {
4306	const struct btf_type *vt;
4307
4308	vt = btf__type_by_id(btf, vs->type);
4309	if (btf_is_func(vt))
4310	break;
4311	}
4312
4313	/* No func in ksyms sec. No need to add dummy var. */
4314	if (i == btf_vlen(sec))
4315	return 0;
4316
4317	int_btf_id = find_int_btf_id(btf);
4318	dummy_var_btf_id = btf__add_var(btf,
4319	"dummy_ksym",
4320	BTF_VAR_GLOBAL_ALLOCATED,
4321	int_btf_id);
4322	if (dummy_var_btf_id < 0)
4323	pr_warn("cannot create a dummy_ksym var\n");
4324
4325	return dummy_var_btf_id;
4326	}
4327
4328	static int bpf_object__collect_externs(struct bpf_object *obj)
4329	{
4330	struct btf_type sec, kcfg_sec = NULL, *ksym_sec = NULL;
4331	const struct btf_type *t;
4332	struct extern_desc *ext;
4333	int i, n, off, dummy_var_btf_id;
4334	const char ext_name, sec_name;
4335	size_t ext_essent_len;
4336	Elf_Scn *scn;
4337	Elf64_Shdr *sh;
4338
4339	if (!obj->efile.symbols)
4340	return 0;
4341
4342	scn = elf_sec_by_idx(obj, obj->efile.symbols_shndx);
4343	sh = elf_sec_hdr(obj, scn);
4344	if (!sh \|\| sh->sh_entsize != sizeof(Elf64_Sym))
4345	return -LIBBPF_ERRNO__FORMAT;
4346
4347	dummy_var_btf_id = add_dummy_ksym_var(obj->btf);
4348	if (dummy_var_btf_id < 0)
4349	return dummy_var_btf_id;
4350
4351	n = sh->sh_size / sh->sh_entsize;
4352	pr_debug("looking for externs among %d symbols...\n", n);
4353
4354	for (i = 0; i < n; i++) {
4355	Elf64_Sym *sym = elf_sym_by_idx(obj, i);
4356
4357	if (!sym)
4358	return -LIBBPF_ERRNO__FORMAT;
4359	if (!sym_is_extern(sym))
4360	continue;
4361	ext_name = elf_sym_str(obj, sym->st_name);
4362	if (str_is_empty(ext_name))
4363	continue;
4364
4365	ext = obj->externs;
4366	ext = libbpf_reallocarray(ext, obj->nr_extern + 1, sizeof(*ext));
4367	if (!ext)
4368	return -ENOMEM;
4369	obj->externs = ext;
4370	ext = &ext[obj->nr_extern];
4371	memset(ext, 0, sizeof(*ext));
4372	obj->nr_extern++;
4373
4374	ext->btf_id = find_extern_btf_id(obj->btf, ext_name);
4375	if (ext->btf_id <= 0) {
4376	pr_warn("failed to find BTF for extern '%s': %d\n",
4377	ext_name, ext->btf_id);
4378	return ext->btf_id;
4379	}
4380	t = btf__type_by_id(obj->btf, ext->btf_id);
4381	ext->name = strdup(btf__name_by_offset(obj->btf, t->name_off));
4382	if (!ext->name)
4383	return -ENOMEM;
4384	ext->sym_idx = i;
4385	ext->is_weak = ELF64_ST_BIND(sym->st_info) == STB_WEAK;
4386
4387	ext_essent_len = bpf_core_essential_name_len(ext->name);
4388	ext->essent_name = NULL;
4389	if (ext_essent_len != strlen(ext->name)) {
4390	ext->essent_name = strndup(ext->name, ext_essent_len);
4391	if (!ext->essent_name)
4392	return -ENOMEM;
4393	}
4394
4395	ext->sec_btf_id = find_extern_sec_btf_id(obj->btf, ext->btf_id);
4396	if (ext->sec_btf_id <= 0) {
4397	pr_warn("failed to find BTF for extern '%s' [%d] section: %d\n",
4398	ext_name, ext->btf_id, ext->sec_btf_id);
4399	return ext->sec_btf_id;
4400	}
4401	sec = (void *)btf__type_by_id(obj->btf, ext->sec_btf_id);
4402	sec_name = btf__name_by_offset(obj->btf, sec->name_off);
4403
4404	if (strcmp(sec_name, KCONFIG_SEC) == 0) {
4405	if (btf_is_func(t)) {
4406	pr_warn("extern function %s is unsupported under %s section\n",
4407	ext->name, KCONFIG_SEC);
4408	return -ENOTSUP;
4409	}
4410	kcfg_sec = sec;
4411	ext->type = EXT_KCFG;
4412	ext->kcfg.sz = btf__resolve_size(obj->btf, t->type);
4413	if (ext->kcfg.sz <= 0) {
4414	pr_warn("failed to resolve size of extern (kcfg) '%s': %d\n",
4415	ext_name, ext->kcfg.sz);
4416	return ext->kcfg.sz;
4417	}
4418	ext->kcfg.align = btf__align_of(obj->btf, t->type);
4419	if (ext->kcfg.align <= 0) {
4420	pr_warn("failed to determine alignment of extern (kcfg) '%s': %d\n",
4421	ext_name, ext->kcfg.align);
4422	return -EINVAL;
4423	}
4424	ext->kcfg.type = find_kcfg_type(obj->btf, t->type,
4425	&ext->kcfg.is_signed);
4426	if (ext->kcfg.type == KCFG_UNKNOWN) {
4427	pr_warn("extern (kcfg) '%s': type is unsupported\n", ext_name);
4428	return -ENOTSUP;
4429	}
4430	} else if (strcmp(sec_name, KSYMS_SEC) == 0) {
4431	ksym_sec = sec;
4432	ext->type = EXT_KSYM;
4433	skip_mods_and_typedefs(obj->btf, t->type,
4434	&ext->ksym.type_id);
4435	} else {
4436	pr_warn("unrecognized extern section '%s'\n", sec_name);
4437	return -ENOTSUP;
4438	}
4439	}
4440	pr_debug("collected %d externs total\n", obj->nr_extern);
4441
4442	if (!obj->nr_extern)
4443	return 0;
4444
4445	/* sort externs by type, for kcfg ones also by (align, size, name) */
4446	qsort(obj->externs, obj->nr_extern, sizeof(*ext), cmp_externs);
4447
4448	/* for .ksyms section, we need to turn all externs into allocated
4449	* variables in BTF to pass kernel verification; we do this by
4450	* pretending that each extern is a 8-byte variable
4451	*/
4452	if (ksym_sec) {
4453	/* find existing 4-byte integer type in BTF to use for fake
4454	* extern variables in DATASEC
4455	*/
4456	int int_btf_id = find_int_btf_id(obj->btf);
4457	/* For extern function, a dummy_var added earlier
4458	* will be used to replace the vs->type and
4459	* its name string will be used to refill
4460	* the missing param's name.
4461	*/
4462	const struct btf_type *dummy_var;
4463
4464	dummy_var = btf__type_by_id(obj->btf, dummy_var_btf_id);
4465	for (i = 0; i < obj->nr_extern; i++) {
4466	ext = &obj->externs[i];
4467	if (ext->type != EXT_KSYM)
4468	continue;
4469	pr_debug("extern (ksym) #%d: symbol %d, name %s\n",
4470	i, ext->sym_idx, ext->name);
4471	}
4472
4473	sec = ksym_sec;
4474	n = btf_vlen(sec);
4475	for (i = 0, off = 0; i < n; i++, off += sizeof(int)) {
4476	struct btf_var_secinfo *vs = btf_var_secinfos(sec) + i;
4477	struct btf_type *vt;
4478
4479	vt = (void *)btf__type_by_id(obj->btf, vs->type);
4480	ext_name = btf__name_by_offset(obj->btf, vt->name_off);
4481	ext = find_extern_by_name(obj, ext_name);
4482	if (!ext) {
4483	pr_warn("failed to find extern definition for BTF %s '%s'\n",
4484	btf_kind_str(vt), ext_name);
4485	return -ESRCH;
4486	}
4487	if (btf_is_func(vt)) {
4488	const struct btf_type *func_proto;
4489	struct btf_param *param;
4490	int j;
4491
4492	func_proto = btf__type_by_id(obj->btf,
4493	vt->type);
4494	param = btf_params(func_proto);
4495	/* Reuse the dummy_var string if the
4496	* func proto does not have param name.
4497	*/
4498	for (j = 0; j < btf_vlen(func_proto); j++)
4499	if (param[j].type && !param[j].name_off)
4500	param[j].name_off =
4501	dummy_var->name_off;
4502	vs->type = dummy_var_btf_id;
4503	vt->info &= ~0xffff;
4504	vt->info \|= BTF_FUNC_GLOBAL;
4505	} else {
4506	btf_var(vt)->linkage = BTF_VAR_GLOBAL_ALLOCATED;
4507	vt->type = int_btf_id;
4508	}
4509	vs->offset = off;
4510	vs->size = sizeof(int);
4511	}
4512	sec->size = off;
4513	}
4514
4515	if (kcfg_sec) {
4516	sec = kcfg_sec;
4517	/* for kcfg externs calculate their offsets within a .kconfig map */
4518	off = 0;
4519	for (i = 0; i < obj->nr_extern; i++) {
4520	ext = &obj->externs[i];
4521	if (ext->type != EXT_KCFG)
4522	continue;
4523
4524	ext->kcfg.data_off = roundup(off, ext->kcfg.align);
4525	off = ext->kcfg.data_off + ext->kcfg.sz;
4526	pr_debug("extern (kcfg) #%d: symbol %d, off %d, name %s\n",
4527	i, ext->sym_idx, ext->kcfg.data_off, ext->name);
4528	}
4529	sec->size = off;
4530	n = btf_vlen(sec);
4531	for (i = 0; i < n; i++) {
4532	struct btf_var_secinfo *vs = btf_var_secinfos(sec) + i;
4533
4534	t = btf__type_by_id(obj->btf, vs->type);
4535	ext_name = btf__name_by_offset(obj->btf, t->name_off);
4536	ext = find_extern_by_name(obj, ext_name);
4537	if (!ext) {
4538	pr_warn("failed to find extern definition for BTF var '%s'\n",
4539	ext_name);
4540	return -ESRCH;
4541	}
4542	btf_var(t)->linkage = BTF_VAR_GLOBAL_ALLOCATED;
4543	vs->offset = ext->kcfg.data_off;
4544	}
4545	}
4546	return 0;
4547	}
4548
4549	static bool prog_is_subprog(const struct bpf_object obj, const struct bpf_program prog)
4550	{
4551	return prog->sec_idx == obj->efile.text_shndx;
4552	}
4553
4554	struct bpf_program *
4555	bpf_object__find_program_by_name(const struct bpf_object *obj,
4556	const char *name)
4557	{
4558	struct bpf_program *prog;
4559
4560	bpf_object__for_each_program(prog, obj) {
4561	if (prog_is_subprog(obj, prog))
4562	continue;
4563	if (!strcmp(prog->name, name))
4564	return prog;
4565	}
4566	return errno = ENOENT, NULL;
4567	}
4568
4569	static bool bpf_object__shndx_is_data(const struct bpf_object *obj,
4570	int shndx)
4571	{
4572	switch (obj->efile.secs[shndx].sec_type) {
4573	case SEC_BSS:
4574	case SEC_DATA:
4575	case SEC_RODATA:
4576	case SEC_PERCPU:
4577	return true;
4578	default:
4579	return false;
4580	}
4581	}
4582
4583	static bool bpf_object__shndx_is_maps(const struct bpf_object *obj,
4584	int shndx)
4585	{
4586	return shndx == obj->efile.btf_maps_shndx;
4587	}
4588
4589	static enum libbpf_map_type
4590	bpf_object__section_to_libbpf_map_type(const struct bpf_object *obj, int shndx)
4591	{
4592	if (shndx == obj->efile.symbols_shndx)
4593	return LIBBPF_MAP_KCONFIG;
4594
4595	switch (obj->efile.secs[shndx].sec_type) {
4596	case SEC_BSS:
4597	return LIBBPF_MAP_BSS;
4598	case SEC_DATA:
4599	return LIBBPF_MAP_DATA;
4600	case SEC_RODATA:
4601	return LIBBPF_MAP_RODATA;
4602	case SEC_PERCPU:
4603	return LIBBPF_MAP_PERCPU;
4604	default:
4605	return LIBBPF_MAP_UNSPEC;
4606	}
4607	}
4608
4609	static int bpf_prog_compute_hash(struct bpf_program *prog)
4610	{
4611	struct bpf_insn *purged;
4612	int i, err = 0;
4613
4614	purged = calloc(prog->insns_cnt, BPF_INSN_SZ);
4615	if (!purged)
4616	return -ENOMEM;
4617
4618	/* If relocations have been done, the map_fd needs to be
4619	* discarded for the digest calculation.
4620	*/
4621	for (i = 0; i < prog->insns_cnt; i++) {
4622	purged[i] = prog->insns[i];
4623	if (purged[i].code == (BPF_LD \| BPF_IMM \| BPF_DW) &&
4624	(purged[i].src_reg == BPF_PSEUDO_MAP_FD \|\|
4625	purged[i].src_reg == BPF_PSEUDO_MAP_VALUE)) {
4626	purged[i].imm = 0;
4627	i++;
4628	if (i >= prog->insns_cnt \|\|
4629	prog->insns[i].code != 0 \|\|
4630	prog->insns[i].dst_reg != 0 \|\|
4631	prog->insns[i].src_reg != 0 \|\|
4632	prog->insns[i].off != 0) {
4633	err = -EINVAL;
4634	goto out;
4635	}
4636	purged[i] = prog->insns[i];
4637	purged[i].imm = 0;
4638	}
4639	}
4640	libbpf_sha256(purged, prog->insns_cnt * sizeof(struct bpf_insn),
4641	prog->hash);
4642	out:
4643	free(purged);
4644	return err;
4645	}
4646
4647	static int bpf_program__record_reloc(struct bpf_program *prog,
4648	struct reloc_desc *reloc_desc,
4649	__u32 insn_idx, const char *sym_name,
4650	const Elf64_Sym sym, const Elf64_Rel rel)
4651	{
4652	struct bpf_insn *insn = &prog->insns[insn_idx];
4653	size_t map_idx, nr_maps = prog->obj->nr_maps;
4654	struct bpf_object *obj = prog->obj;
4655	__u32 shdr_idx = sym->st_shndx;
4656	enum libbpf_map_type type;
4657	const char *sym_sec_name;
4658	struct bpf_map *map;
4659
4660	if (!is_call_insn(insn) && !is_ldimm64_insn(insn)) {
4661	pr_warn("prog '%s': invalid relo against '%s' for insns[%u].code 0x%x\n",
4662	prog->name, sym_name, insn_idx, insn->code);
4663	return -LIBBPF_ERRNO__RELOC;
4664	}
4665
4666	if (sym_is_extern(sym)) {
4667	int sym_idx = ELF64_R_SYM(rel->r_info);
4668	int i, n = obj->nr_extern;
4669	struct extern_desc *ext;
4670
4671	for (i = 0; i < n; i++) {
4672	ext = &obj->externs[i];
4673	if (ext->sym_idx == sym_idx)
4674	break;
4675	}
4676	if (i >= n) {
4677	pr_warn("prog '%s': extern relo failed to find extern for '%s' (%d)\n",
4678	prog->name, sym_name, sym_idx);
4679	return -LIBBPF_ERRNO__RELOC;
4680	}
4681	pr_debug("prog '%s': found extern #%d '%s' (sym %d) for insn #%u\n",
4682	prog->name, i, ext->name, ext->sym_idx, insn_idx);
4683	if (insn->code == (BPF_JMP \| BPF_CALL))
4684	reloc_desc->type = RELO_EXTERN_CALL;
4685	else
4686	reloc_desc->type = RELO_EXTERN_LD64;
4687	reloc_desc->insn_idx = insn_idx;
4688	reloc_desc->ext_idx = i;
4689	return 0;
4690	}
4691
4692	/* sub-program call relocation */
4693	if (is_call_insn(insn)) {
4694	if (insn->src_reg != BPF_PSEUDO_CALL) {
4695	pr_warn("prog '%s': incorrect bpf_call opcode\n", prog->name);
4696	return -LIBBPF_ERRNO__RELOC;
4697	}
4698	/* text_shndx can be 0, if no default "main" program exists */
4699	if (!shdr_idx \|\| shdr_idx != obj->efile.text_shndx) {
4700	sym_sec_name = elf_sec_name(obj, elf_sec_by_idx(obj, shdr_idx));
4701	pr_warn("prog '%s': bad call relo against '%s' in section '%s'\n",
4702	prog->name, sym_name, sym_sec_name);
4703	return -LIBBPF_ERRNO__RELOC;
4704	}
4705	if (sym->st_value % BPF_INSN_SZ) {
4706	pr_warn("prog '%s': bad call relo against '%s' at offset %zu\n",
4707	prog->name, sym_name, (size_t)sym->st_value);
4708	return -LIBBPF_ERRNO__RELOC;
4709	}
4710	reloc_desc->type = RELO_CALL;
4711	reloc_desc->insn_idx = insn_idx;
4712	reloc_desc->sym_off = sym->st_value;
4713	return 0;
4714	}
4715
4716	if (!shdr_idx \|\| shdr_idx >= SHN_LORESERVE) {
4717	pr_warn("prog '%s': invalid relo against '%s' in special section 0x%x; forgot to initialize global var?..\n",
4718	prog->name, sym_name, shdr_idx);
4719	return -LIBBPF_ERRNO__RELOC;
4720	}
4721
4722	/* loading subprog addresses */
4723	if (sym_is_subprog(sym, obj->efile.text_shndx)) {
4724	/* global_func: sym->st_value = offset in the section, insn->imm = 0.
4725	* local_func: sym->st_value = 0, insn->imm = offset in the section.
4726	*/
4727	if ((sym->st_value % BPF_INSN_SZ) \|\| (insn->imm % BPF_INSN_SZ)) {
4728	pr_warn("prog '%s': bad subprog addr relo against '%s' at offset %zu+%d\n",
4729	prog->name, sym_name, (size_t)sym->st_value, insn->imm);
4730	return -LIBBPF_ERRNO__RELOC;
4731	}
4732
4733	reloc_desc->type = RELO_SUBPROG_ADDR;
4734	reloc_desc->insn_idx = insn_idx;
4735	reloc_desc->sym_off = sym->st_value;
4736	return 0;
4737	}
4738
4739	type = bpf_object__section_to_libbpf_map_type(obj, shdr_idx);
4740	sym_sec_name = elf_sec_name(obj, elf_sec_by_idx(obj, shdr_idx));
4741
4742	/* arena data relocation */
4743	if (shdr_idx == obj->efile.arena_data_shndx) {
4744	if (obj->arena_map_idx < 0) {
4745	pr_warn("prog '%s': bad arena data relocation at insn %u, no arena maps defined\n",
4746	prog->name, insn_idx);
4747	return -LIBBPF_ERRNO__RELOC;
4748	}
4749	reloc_desc->type = RELO_DATA;
4750	reloc_desc->insn_idx = insn_idx;
4751	reloc_desc->map_idx = obj->arena_map_idx;
4752	reloc_desc->sym_off = sym->st_value;
4753
4754	map = &obj->maps[obj->arena_map_idx];
4755	pr_debug("prog '%s': found arena map %d (%s, sec %d, off %zu) for insn %u\n",
4756	prog->name, obj->arena_map_idx, map->name, map->sec_idx,
4757	map->sec_offset, insn_idx);
4758	return 0;
4759	}
4760
4761	/* jump table data relocation */
4762	if (shdr_idx == obj->efile.jumptables_data_shndx) {
4763	reloc_desc->type = RELO_INSN_ARRAY;
4764	reloc_desc->insn_idx = insn_idx;
4765	reloc_desc->map_idx = -1;
4766	reloc_desc->sym_off = sym->st_value;
4767	reloc_desc->sym_size = sym->st_size;
4768	return 0;
4769	}
4770
4771	/* generic map reference relocation */
4772	if (type == LIBBPF_MAP_UNSPEC) {
4773	if (!bpf_object__shndx_is_maps(obj, shdr_idx)) {
4774	pr_warn("prog '%s': bad map relo against '%s' in section '%s'\n",
4775	prog->name, sym_name, sym_sec_name);
4776	return -LIBBPF_ERRNO__RELOC;
4777	}
4778	for (map_idx = 0; map_idx < nr_maps; map_idx++) {
4779	map = &obj->maps[map_idx];
4780	if (map->libbpf_type != type \|\|
4781	map->sec_idx != sym->st_shndx \|\|
4782	map->sec_offset != sym->st_value)
4783	continue;
4784	pr_debug("prog '%s': found map %zu (%s, sec %d, off %zu) for insn #%u\n",
4785	prog->name, map_idx, map->name, map->sec_idx,
4786	map->sec_offset, insn_idx);
4787	break;
4788	}
4789	if (map_idx >= nr_maps) {
4790	pr_warn("prog '%s': map relo failed to find map for section '%s', off %zu\n",
4791	prog->name, sym_sec_name, (size_t)sym->st_value);
4792	return -LIBBPF_ERRNO__RELOC;
4793	}
4794	reloc_desc->type = RELO_LD64;
4795	reloc_desc->insn_idx = insn_idx;
4796	reloc_desc->map_idx = map_idx;
4797	reloc_desc->sym_off = 0; /* sym->st_value determines map_idx */
4798	return 0;
4799	}
4800
4801	/* global data map relocation */
4802	if (!bpf_object__shndx_is_data(obj, shdr_idx)) {
4803	pr_warn("prog '%s': bad data relo against section '%s'\n",
4804	prog->name, sym_sec_name);
4805	return -LIBBPF_ERRNO__RELOC;
4806	}
4807	for (map_idx = 0; map_idx < nr_maps; map_idx++) {
4808	map = &obj->maps[map_idx];
4809	if (map->libbpf_type != type \|\| map->sec_idx != sym->st_shndx)
4810	continue;
4811	pr_debug("prog '%s': found data map %zu (%s, sec %d, off %zu) for insn %u\n",
4812	prog->name, map_idx, map->name, map->sec_idx,
4813	map->sec_offset, insn_idx);
4814	break;
4815	}
4816	if (map_idx >= nr_maps) {
4817	pr_warn("prog '%s': data relo failed to find map for section '%s'\n",
4818	prog->name, sym_sec_name);
4819	return -LIBBPF_ERRNO__RELOC;
4820	}
4821
4822	reloc_desc->type = RELO_DATA;
4823	reloc_desc->insn_idx = insn_idx;
4824	reloc_desc->map_idx = map_idx;
4825	reloc_desc->sym_off = sym->st_value;
4826	return 0;
4827	}
4828
4829	static bool prog_contains_insn(const struct bpf_program *prog, size_t insn_idx)
4830	{
4831	return insn_idx >= prog->sec_insn_off &&
4832	insn_idx < prog->sec_insn_off + prog->sec_insn_cnt;
4833	}
4834
4835	static struct bpf_program find_prog_by_sec_insn(const struct bpf_object obj,
4836	size_t sec_idx, size_t insn_idx)
4837	{
4838	int l = 0, r = obj->nr_programs - 1, m;
4839	struct bpf_program *prog;
4840
4841	if (!obj->nr_programs)
4842	return NULL;
4843
4844	while (l < r) {
4845	m = l + (r - l + 1) / 2;
4846	prog = &obj->programs[m];
4847
4848	if (prog->sec_idx < sec_idx \|\|
4849	(prog->sec_idx == sec_idx && prog->sec_insn_off <= insn_idx))
4850	l = m;
4851	else
4852	r = m - 1;
4853	}
4854	/* matching program could be at index l, but it still might be the
4855	* wrong one, so we need to double check conditions for the last time
4856	*/
4857	prog = &obj->programs[l];
4858	if (prog->sec_idx == sec_idx && prog_contains_insn(prog, insn_idx))
4859	return prog;
4860	return NULL;
4861	}
4862
4863	static int
4864	bpf_object__collect_prog_relos(struct bpf_object obj, Elf64_Shdr shdr, Elf_Data *data)
4865	{
4866	const char relo_sec_name, sec_name;
4867	size_t sec_idx = shdr->sh_info, sym_idx;
4868	struct bpf_program *prog;
4869	struct reloc_desc *relos;
4870	int err, i, nrels;
4871	const char *sym_name;
4872	__u32 insn_idx;
4873	Elf_Scn *scn;
4874	Elf_Data *scn_data;
4875	Elf64_Sym *sym;
4876	Elf64_Rel *rel;
4877
4878	if (sec_idx >= obj->efile.sec_cnt)
4879	return -EINVAL;
4880
4881	scn = elf_sec_by_idx(obj, sec_idx);
4882	scn_data = elf_sec_data(obj, scn);
4883	if (!scn_data)
4884	return -LIBBPF_ERRNO__FORMAT;
4885
4886	relo_sec_name = elf_sec_str(obj, shdr->sh_name);
4887	sec_name = elf_sec_name(obj, scn);
4888	if (!relo_sec_name \|\| !sec_name)
4889	return -EINVAL;
4890
4891	pr_debug("sec '%s': collecting relocation for section(%zu) '%s'\n",
4892	relo_sec_name, sec_idx, sec_name);
4893	nrels = shdr->sh_size / shdr->sh_entsize;
4894
4895	for (i = 0; i < nrels; i++) {
4896	rel = elf_rel_by_idx(data, i);
4897	if (!rel) {
4898	pr_warn("sec '%s': failed to get relo #%d\n", relo_sec_name, i);
4899	return -LIBBPF_ERRNO__FORMAT;
4900	}
4901
4902	sym_idx = ELF64_R_SYM(rel->r_info);
4903	sym = elf_sym_by_idx(obj, sym_idx);
4904	if (!sym) {
4905	pr_warn("sec '%s': symbol #%zu not found for relo #%d\n",
4906	relo_sec_name, sym_idx, i);
4907	return -LIBBPF_ERRNO__FORMAT;
4908	}
4909
4910	if (sym->st_shndx >= obj->efile.sec_cnt) {
4911	pr_warn("sec '%s': corrupted symbol #%zu pointing to invalid section #%zu for relo #%d\n",
4912	relo_sec_name, sym_idx, (size_t)sym->st_shndx, i);
4913	return -LIBBPF_ERRNO__FORMAT;
4914	}
4915
4916	if (rel->r_offset % BPF_INSN_SZ \|\| rel->r_offset >= scn_data->d_size) {
4917	pr_warn("sec '%s': invalid offset 0x%zx for relo #%d\n",
4918	relo_sec_name, (size_t)rel->r_offset, i);
4919	return -LIBBPF_ERRNO__FORMAT;
4920	}
4921
4922	insn_idx = rel->r_offset / BPF_INSN_SZ;
4923	/* relocations against static functions are recorded as
4924	* relocations against the section that contains a function;
4925	* in such case, symbol will be STT_SECTION and sym.st_name
4926	* will point to empty string (0), so fetch section name
4927	* instead
4928	*/
4929	if (ELF64_ST_TYPE(sym->st_info) == STT_SECTION && sym->st_name == 0)
4930	sym_name = elf_sec_name(obj, elf_sec_by_idx(obj, sym->st_shndx));
4931	else
4932	sym_name = elf_sym_str(obj, sym->st_name);
4933	sym_name = sym_name ?: "<?";
4934
4935	pr_debug("sec '%s': relo #%d: insn #%u against '%s'\n",
4936	relo_sec_name, i, insn_idx, sym_name);
4937
4938	prog = find_prog_by_sec_insn(obj, sec_idx, insn_idx);
4939	if (!prog) {
4940	pr_debug("sec '%s': relo #%d: couldn't find program in section '%s' for insn #%u, probably overridden weak function, skipping...\n",
4941	relo_sec_name, i, sec_name, insn_idx);
4942	continue;
4943	}
4944
4945	relos = libbpf_reallocarray(prog->reloc_desc,
4946	prog->nr_reloc + 1, sizeof(*relos));
4947	if (!relos)
4948	return -ENOMEM;
4949	prog->reloc_desc = relos;
4950
4951	/* adjust insn_idx to local BPF program frame of reference */
4952	insn_idx -= prog->sec_insn_off;
4953	err = bpf_program__record_reloc(prog, &relos[prog->nr_reloc],
4954	insn_idx, sym_name, sym, rel);
4955	if (err)
4956	return err;
4957
4958	prog->nr_reloc++;
4959	}
4960	return 0;
4961	}
4962
4963	static int map_fill_btf_type_info(struct bpf_object obj, struct bpf_map map)
4964	{
4965	int id;
4966
4967	if (!obj->btf)
4968	return -ENOENT;
4969
4970	/* if it's BTF-defined map, we don't need to search for type IDs.
4971	* For struct_ops map, it does not need btf_key_type_id and
4972	* btf_value_type_id.
4973	*/
4974	if (map->sec_idx == obj->efile.btf_maps_shndx \|\| bpf_map__is_struct_ops(map))
4975	return 0;
4976
4977	/*
4978	* LLVM annotates global data differently in BTF, that is,
4979	* only as '.data', '.bss', '.percpu' or '.rodata'.
4980	*/
4981	if (!bpf_map__is_internal(map))
4982	return -ENOENT;
4983
4984	id = btf__find_by_name(obj->btf, map->real_name);
4985	if (id < 0)
4986	return id;
4987
4988	map->btf_key_type_id = 0;
4989	map->btf_value_type_id = id;
4990	return 0;
4991	}
4992
4993	static int bpf_get_map_info_from_fdinfo(int fd, struct bpf_map_info *info)
4994	{
4995	char file[PATH_MAX], buff[4096];
4996	FILE *fp;
4997	__u32 val;
4998	int err;
4999
5000	snprintf(file, sizeof(file), "/proc/%d/fdinfo/%d", getpid(), fd);
5001	memset(info, 0, sizeof(*info));
5002
5003	fp = fopen(file, "re");
5004	if (!fp) {
5005	err = -errno;
5006	pr_warn("failed to open %s: %s. No procfs support?\n", file,
5007	errstr(err));
5008	return err;
5009	}
5010
5011	while (fgets(buff, sizeof(buff), fp)) {
5012	if (sscanf(buff, "map_type:\t%u", &val) == 1)
5013	info->type = val;
5014	else if (sscanf(buff, "key_size:\t%u", &val) == 1)
5015	info->key_size = val;
5016	else if (sscanf(buff, "value_size:\t%u", &val) == 1)
5017	info->value_size = val;
5018	else if (sscanf(buff, "max_entries:\t%u", &val) == 1)
5019	info->max_entries = val;
5020	else if (sscanf(buff, "map_flags:\t%x", &val) == 1)
5021	info->map_flags = val;
5022	}
5023
5024	fclose(fp);
5025
5026	return 0;
5027	}
5028
5029	static bool map_is_created(const struct bpf_map *map)
5030	{
5031	return map->obj->state >= OBJ_PREPARED \|\| map->reused;
5032	}
5033
5034	bool bpf_map__autocreate(const struct bpf_map *map)
5035	{
5036	return map->autocreate;
5037	}
5038
5039	int bpf_map__set_autocreate(struct bpf_map *map, bool autocreate)
5040	{
5041	if (map_is_created(map))
5042	return libbpf_err(-EBUSY);
5043
5044	map->autocreate = autocreate;
5045	return 0;
5046	}
5047
5048	int bpf_map__set_autoattach(struct bpf_map *map, bool autoattach)
5049	{
5050	if (!bpf_map__is_struct_ops(map))
5051	return libbpf_err(-EINVAL);
5052
5053	map->autoattach = autoattach;
5054	return 0;
5055	}
5056
5057	bool bpf_map__autoattach(const struct bpf_map *map)
5058	{
5059	return map->autoattach;
5060	}
5061
5062	int bpf_map__reuse_fd(struct bpf_map *map, int fd)
5063	{
5064	struct bpf_map_info info;
5065	__u32 len = sizeof(info), name_len;
5066	int new_fd, err;
5067	char *new_name;
5068
5069	memset(&info, 0, len);
5070	err = bpf_map_get_info_by_fd(fd, &info, &len);
5071	if (err && errno == EINVAL)
5072	err = bpf_get_map_info_from_fdinfo(fd, &info);
5073	if (err)
5074	return libbpf_err(err);
5075
5076	name_len = strlen(info.name);
5077	if (name_len == BPF_OBJ_NAME_LEN - 1 && strncmp(map->name, info.name, name_len) == 0)
5078	new_name = strdup(map->name);
5079	else
5080	new_name = strdup(info.name);
5081
5082	if (!new_name)
5083	return libbpf_err(-errno);
5084
5085	/*
5086	* Like dup(), but make sure new FD is >= 3 and has O_CLOEXEC set.
5087	* This is similar to what we do in ensure_good_fd(), but without
5088	* closing original FD.
5089	*/
5090	new_fd = fcntl(fd, F_DUPFD_CLOEXEC, 3);
5091	if (new_fd < 0) {
5092	err = -errno;
5093	goto err_free_new_name;
5094	}
5095
5096	err = reuse_fd(map->fd, new_fd);
5097	if (err)
5098	goto err_free_new_name;
5099
5100	free(map->name);
5101
5102	map->name = new_name;
5103	map->def.type = info.type;
5104	map->def.key_size = info.key_size;
5105	map->def.value_size = info.value_size;
5106	map->def.max_entries = info.max_entries;
5107	map->def.map_flags = info.map_flags;
5108	map->btf_key_type_id = info.btf_key_type_id;
5109	map->btf_value_type_id = info.btf_value_type_id;
5110	map->reused = true;
5111	map->map_extra = info.map_extra;
5112
5113	return 0;
5114
5115	err_free_new_name:
5116	free(new_name);
5117	return libbpf_err(err);
5118	}
5119
5120	__u32 bpf_map__max_entries(const struct bpf_map *map)
5121	{
5122	return map->def.max_entries;
5123	}
5124
5125	struct bpf_map bpf_map__inner_map(struct bpf_map map)
5126	{
5127	if (!bpf_map_type__is_map_in_map(map->def.type))
5128	return errno = EINVAL, NULL;
5129
5130	return map->inner_map;
5131	}
5132
5133	int bpf_map__set_max_entries(struct bpf_map *map, __u32 max_entries)
5134	{
5135	if (map_is_created(map))
5136	return libbpf_err(-EBUSY);
5137
5138	map->def.max_entries = max_entries;
5139
5140	/* auto-adjust BPF ringbuf map max_entries to be a multiple of page size */
5141	if (map_is_ringbuf(map))
5142	map->def.max_entries = adjust_ringbuf_sz(map->def.max_entries);
5143
5144	return 0;
5145	}
5146
5147	static int bpf_object_prepare_token(struct bpf_object *obj)
5148	{
5149	const char *bpffs_path;
5150	int bpffs_fd = -1, token_fd, err;
5151	bool mandatory;
5152	enum libbpf_print_level level;
5153
5154	/* token is explicitly prevented */
5155	if (obj->token_path && obj->token_path[0] == '\0') {
5156	pr_debug("object '%s': token is prevented, skipping...\n", obj->name);
5157	return 0;
5158	}
5159
5160	mandatory = obj->token_path != NULL;
5161	level = mandatory ? LIBBPF_WARN : LIBBPF_DEBUG;
5162
5163	bpffs_path = obj->token_path ?: BPF_FS_DEFAULT_PATH;
5164	bpffs_fd = open(bpffs_path, O_DIRECTORY, O_RDWR);
5165	if (bpffs_fd < 0) {
5166	err = -errno;
5167	__pr(level, "object '%s': failed (%s) to open BPF FS mount at '%s'%s\n",
5168	obj->name, errstr(err), bpffs_path,
5169	mandatory ? "" : ", skipping optional step...");
5170	return mandatory ? err : 0;
5171	}
5172
5173	token_fd = bpf_token_create(bpffs_fd, 0);
5174	close(bpffs_fd);
5175	if (token_fd < 0) {
5176	if (!mandatory && token_fd == -ENOENT) {
5177	pr_debug("object '%s': BPF FS at '%s' doesn't have BPF token delegation set up, skipping...\n",
5178	obj->name, bpffs_path);
5179	return 0;
5180	}
5181	__pr(level, "object '%s': failed (%d) to create BPF token from '%s'%s\n",
5182	obj->name, token_fd, bpffs_path,
5183	mandatory ? "" : ", skipping optional step...");
5184	return mandatory ? token_fd : 0;
5185	}
5186
5187	obj->feat_cache = calloc(1, sizeof(*obj->feat_cache));
5188	if (!obj->feat_cache) {
5189	close(token_fd);
5190	return -ENOMEM;
5191	}
5192
5193	obj->token_fd = token_fd;
5194	obj->feat_cache->token_fd = token_fd;
5195
5196	return 0;
5197	}
5198
5199	static int
5200	bpf_object__probe_loading(struct bpf_object *obj)
5201	{
5202	struct bpf_insn insns[] = {
5203	BPF_MOV64_IMM(BPF_REG_0, 0),
5204	BPF_EXIT_INSN(),
5205	};
5206	int ret, insn_cnt = ARRAY_SIZE(insns);
5207
5208	if (obj->gen_loader \|\| obj->token_fd)
5209	return 0;
5210
5211	ret = bump_rlimit_memlock();
5212	if (ret)
5213	pr_warn("Failed to bump RLIMIT_MEMLOCK (err = %s), you might need to do it explicitly!\n",
5214	errstr(ret));
5215
5216	/* make sure basic loading works */
5217	ret = bpf_prog_load(BPF_PROG_TYPE_SOCKET_FILTER, NULL, "GPL", insns, insn_cnt, NULL);
5218	if (ret < 0)
5219	ret = bpf_prog_load(BPF_PROG_TYPE_TRACEPOINT, NULL, "GPL", insns, insn_cnt, NULL);
5220	if (ret < 0) {
5221	ret = errno;
5222	pr_warn("Error in %s(): %s. Couldn't load trivial BPF program. Make sure your kernel supports BPF (CONFIG_BPF_SYSCALL=y) and/or that RLIMIT_MEMLOCK is set to big enough value.\n",
5223	__func__, errstr(ret));
5224	return -ret;
5225	}
5226	close(ret);
5227
5228	return 0;
5229	}
5230
5231	bool kernel_supports(const struct bpf_object *obj, enum kern_feature_id feat_id)
5232	{
5233	if (obj->gen_loader)
5234	/* To generate loader program assume the latest kernel
5235	* to avoid doing extra prog_load, map_create syscalls.
5236	*/
5237	return true;
5238
5239	if (obj->feat_cache)
5240	return feat_supported(obj->feat_cache, feat_id);
5241
5242	return feat_supported(NULL, feat_id);
5243	}
5244
5245	/* Used in testing to simulate missing features. */
5246	void bpf_object_set_feat_cache(struct bpf_object obj, struct kern_feature_cache cache)
5247	{
5248	if (obj->feat_cache)
5249	free(obj->feat_cache);
5250	obj->feat_cache = cache;
5251	}
5252
5253	static bool map_is_reuse_compat(const struct bpf_map *map, int map_fd)
5254	{
5255	struct bpf_map_info map_info;
5256	__u32 map_info_len = sizeof(map_info);
5257	int err;
5258
5259	memset(&map_info, 0, map_info_len);
5260	err = bpf_map_get_info_by_fd(map_fd, &map_info, &map_info_len);
5261	if (err && errno == EINVAL)
5262	err = bpf_get_map_info_from_fdinfo(map_fd, &map_info);
5263	if (err) {
5264	pr_warn("failed to get map info for map FD %d: %s\n", map_fd,
5265	errstr(err));
5266	return false;
5267	}
5268
5269	/*
5270	* bpf_get_map_info_by_fd() for DEVMAP will always return flags with
5271	* BPF_F_RDONLY_PROG set, but it generally is not set at map creation time.
5272	* Thus, ignore the BPF_F_RDONLY_PROG flag in the flags returned from
5273	* bpf_get_map_info_by_fd() when checking for compatibility with an
5274	* existing DEVMAP.
5275	*/
5276	if (map->def.type == BPF_MAP_TYPE_DEVMAP \|\| map->def.type == BPF_MAP_TYPE_DEVMAP_HASH)
5277	map_info.map_flags &= ~BPF_F_RDONLY_PROG;
5278
5279	return (map_info.type == map->def.type &&
5280	map_info.key_size == map->def.key_size &&
5281	map_info.value_size == map->def.value_size &&
5282	map_info.max_entries == map->def.max_entries &&
5283	map_info.map_flags == map->def.map_flags &&
5284	map_info.map_extra == map->map_extra);
5285	}
5286
5287	static int
5288	bpf_object__reuse_map(struct bpf_map *map)
5289	{
5290	int err, pin_fd;
5291
5292	pin_fd = bpf_obj_get(map->pin_path);
5293	if (pin_fd < 0) {
5294	err = -errno;
5295	if (err == -ENOENT) {
5296	pr_debug("found no pinned map to reuse at '%s'\n",
5297	map->pin_path);
5298	return 0;
5299	}
5300
5301	pr_warn("couldn't retrieve pinned map '%s': %s\n",
5302	map->pin_path, errstr(err));
5303	return err;
5304	}
5305
5306	if (!map_is_reuse_compat(map, pin_fd)) {
5307	pr_warn("couldn't reuse pinned map at '%s': parameter mismatch\n",
5308	map->pin_path);
5309	close(pin_fd);
5310	return -EINVAL;
5311	}
5312
5313	err = bpf_map__reuse_fd(map, pin_fd);
5314	close(pin_fd);
5315	if (err)
5316	return err;
5317
5318	map->pinned = true;
5319	pr_debug("reused pinned map at '%s'\n", map->pin_path);
5320
5321	return 0;
5322	}
5323
5324	static int
5325	bpf_object__populate_internal_map(struct bpf_object obj, struct bpf_map map)
5326	{
5327	enum libbpf_map_type map_type = map->libbpf_type;
5328	bool is_percpu = map_type == LIBBPF_MAP_PERCPU;
5329	const __u64 update_flags = is_percpu ? BPF_F_ALL_CPUS : 0;
5330	int err, zero = 0;
5331	size_t mmap_sz;
5332
5333	if (obj->gen_loader) {
5334	bpf_gen__map_update_elem(obj->gen_loader, map - obj->maps,
5335	map->mmaped, map->def.value_size, update_flags);
5336	if (map_type == LIBBPF_MAP_RODATA \|\| map_type == LIBBPF_MAP_KCONFIG)
5337	bpf_gen__map_freeze(obj->gen_loader, map - obj->maps);
5338	return 0;
5339	}
5340
5341	err = bpf_map_update_elem(map->fd, &zero, map->mmaped, update_flags);
5342	if (err) {
5343	err = -errno;
5344	pr_warn("map '%s': failed to set initial contents: %s\n",
5345	bpf_map__name(map), errstr(err));
5346	return err;
5347	}
5348
5349	/* Freeze .rodata and .kconfig map as read-only from syscall side. */
5350	if (map_type == LIBBPF_MAP_RODATA \|\| map_type == LIBBPF_MAP_KCONFIG) {
5351	err = bpf_map_freeze(map->fd);
5352	if (err) {
5353	err = -errno;
5354	pr_warn("map '%s': failed to freeze as read-only: %s\n",
5355	bpf_map__name(map), errstr(err));
5356	return err;
5357	}
5358	}
5359
5360	/* Remap anonymous mmap()-ed "map initialization image" as
5361	* a BPF map-backed mmap()-ed memory, but preserving the same
5362	* memory address. This will cause kernel to change process'
5363	* page table to point to a different piece of kernel memory,
5364	* but from userspace point of view memory address (and its
5365	* contents, being identical at this point) will stay the
5366	* same. This mapping will be released by bpf_object__close()
5367	* as per normal clean up procedure.
5368	*/
5369	mmap_sz = bpf_map_mmap_sz(map);
5370	if (map->def.map_flags & BPF_F_MMAPABLE) {
5371	void *mmaped;
5372	int prot;
5373
5374	if (map->def.map_flags & BPF_F_RDONLY_PROG)
5375	prot = PROT_READ;
5376	else
5377	prot = PROT_READ \| PROT_WRITE;
5378	mmaped = mmap(map->mmaped, mmap_sz, prot, MAP_SHARED \| MAP_FIXED, map->fd, 0);
5379	if (mmaped == MAP_FAILED) {
5380	err = -errno;
5381	pr_warn("map '%s': failed to re-mmap() contents: %s\n",
5382	bpf_map__name(map), errstr(err));
5383	return err;
5384	}
5385	map->mmaped = mmaped;
5386	} else if (is_percpu) {
5387	if (mprotect(map->mmaped, mmap_sz, PROT_READ)) {
5388	err = -errno;
5389	pr_warn("map '%s': failed to mprotect() contents: %s\n",
5390	bpf_map__name(map), errstr(err));
5391	return err;
5392	}
5393	} else if (map->mmaped) {
5394	munmap(map->mmaped, mmap_sz);
5395	map->mmaped = NULL;
5396	}
5397
5398	return 0;
5399	}
5400
5401	static void bpf_map__destroy(struct bpf_map *map);
5402
5403	static int bpf_object__create_map(struct bpf_object obj, struct bpf_map map, bool is_inner)
5404	{
5405	LIBBPF_OPTS(bpf_map_create_opts, create_attr);
5406	struct bpf_map_def *def = &map->def;
5407	const char *map_name = NULL;
5408	int err = 0, map_fd;
5409
5410	if (kernel_supports(obj, FEAT_PROG_NAME))
5411	map_name = map->name;
5412	create_attr.map_ifindex = map->map_ifindex;
5413	create_attr.map_flags = def->map_flags;
5414	create_attr.numa_node = map->numa_node;
5415	create_attr.map_extra = map->map_extra;
5416	create_attr.token_fd = obj->token_fd;
5417	if (obj->token_fd)
5418	create_attr.map_flags \|= BPF_F_TOKEN_FD;
5419	if (map->excl_prog) {
5420	err = bpf_prog_compute_hash(map->excl_prog);
5421	if (err)
5422	return err;
5423
5424	create_attr.excl_prog_hash = map->excl_prog->hash;
5425	create_attr.excl_prog_hash_size = SHA256_DIGEST_LENGTH;
5426	}
5427
5428	if (bpf_map__is_struct_ops(map)) {
5429	create_attr.btf_vmlinux_value_type_id = map->btf_vmlinux_value_type_id;
5430	if (map->mod_btf_fd >= 0) {
5431	create_attr.value_type_btf_obj_fd = map->mod_btf_fd;
5432	create_attr.map_flags \|= BPF_F_VTYPE_BTF_OBJ_FD;
5433	}
5434	}
5435
5436	if (obj->btf && btf__fd(obj->btf) >= 0) {
5437	create_attr.btf_fd = btf__fd(obj->btf);
5438	create_attr.btf_key_type_id = map->btf_key_type_id;
5439	create_attr.btf_value_type_id = map->btf_value_type_id;
5440	}
5441
5442	if (bpf_map_type__is_map_in_map(def->type)) {
5443	if (map->inner_map) {
5444	err = map_set_def_max_entries(map->inner_map);
5445	if (err)
5446	return err;
5447	err = bpf_object__create_map(obj, map->inner_map, true);
5448	if (err) {
5449	pr_warn("map '%s': failed to create inner map: %s\n",
5450	map->name, errstr(err));
5451	return err;
5452	}
5453	map->inner_map_fd = map->inner_map->fd;
5454	}
5455	if (map->inner_map_fd >= 0)
5456	create_attr.inner_map_fd = map->inner_map_fd;
5457	}
5458
5459	switch (def->type) {
5460	case BPF_MAP_TYPE_PERF_EVENT_ARRAY:
5461	case BPF_MAP_TYPE_CGROUP_ARRAY:
5462	case BPF_MAP_TYPE_STACK_TRACE:
5463	case BPF_MAP_TYPE_ARRAY_OF_MAPS:
5464	case BPF_MAP_TYPE_HASH_OF_MAPS:
5465	case BPF_MAP_TYPE_DEVMAP:
5466	case BPF_MAP_TYPE_DEVMAP_HASH:
5467	case BPF_MAP_TYPE_CPUMAP:
5468	case BPF_MAP_TYPE_XSKMAP:
5469	case BPF_MAP_TYPE_SOCKMAP:
5470	case BPF_MAP_TYPE_SOCKHASH:
5471	case BPF_MAP_TYPE_QUEUE:
5472	case BPF_MAP_TYPE_STACK:
5473	case BPF_MAP_TYPE_ARENA:
5474	create_attr.btf_fd = 0;
5475	create_attr.btf_key_type_id = 0;
5476	create_attr.btf_value_type_id = 0;
5477	map->btf_key_type_id = 0;
5478	map->btf_value_type_id = 0;
5479	break;
5480	case BPF_MAP_TYPE_STRUCT_OPS:
5481	create_attr.btf_value_type_id = 0;
5482	break;
5483	default:
5484	break;
5485	}
5486
5487	if (obj->gen_loader) {
5488	bpf_gen__map_create(obj->gen_loader, def->type, map_name,
5489	def->key_size, def->value_size, def->max_entries,
5490	&create_attr, is_inner ? -1 : map - obj->maps);
5491	/* We keep pretenting we have valid FD to pass various fd >= 0
5492	* checks by just keeping original placeholder FDs in place.
5493	* See bpf_object__add_map() comment.
5494	* This placeholder fd will not be used with any syscall and
5495	* will be reset to -1 eventually.
5496	*/
5497	map_fd = map->fd;
5498	} else {
5499	map_fd = bpf_map_create(def->type, map_name,
5500	def->key_size, def->value_size,
5501	def->max_entries, &create_attr);
5502	}
5503	if (map_fd < 0 && (create_attr.btf_key_type_id \|\| create_attr.btf_value_type_id)) {
5504	err = -errno;
5505	pr_warn("Error in bpf_create_map_xattr(%s): %s. Retrying without BTF.\n",
5506	map->name, errstr(err));
5507	create_attr.btf_fd = 0;
5508	create_attr.btf_key_type_id = 0;
5509	create_attr.btf_value_type_id = 0;
5510	map->btf_key_type_id = 0;
5511	map->btf_value_type_id = 0;
5512	map_fd = bpf_map_create(def->type, map_name,
5513	def->key_size, def->value_size,
5514	def->max_entries, &create_attr);
5515	}
5516
5517	if (bpf_map_type__is_map_in_map(def->type) && map->inner_map) {
5518	if (obj->gen_loader)
5519	map->inner_map->fd = -1;
5520	bpf_map__destroy(map->inner_map);
5521	zfree(&map->inner_map);
5522	}
5523
5524	if (map_fd < 0)
5525	return map_fd;
5526
5527	/* obj->gen_loader case, prevent reuse_fd() from closing map_fd */
5528	if (map->fd == map_fd)
5529	return 0;
5530
5531	/* Keep placeholder FD value but now point it to the BPF map object.
5532	* This way everything that relied on this map's FD (e.g., relocated
5533	* ldimm64 instructions) will stay valid and won't need adjustments.
5534	* map->fd stays valid but now point to what map_fd points to.
5535	*/
5536	return reuse_fd(map->fd, map_fd);
5537	}
5538
5539	static int init_map_in_map_slots(struct bpf_object obj, struct bpf_map map)
5540	{
5541	const struct bpf_map *targ_map;
5542	unsigned int i;
5543	int fd, err = 0;
5544
5545	for (i = 0; i < map->init_slots_sz; i++) {
5546	if (!map->init_slots[i])
5547	continue;
5548
5549	targ_map = map->init_slots[i];
5550	fd = targ_map->fd;
5551
5552	if (obj->gen_loader) {
5553	bpf_gen__populate_outer_map(obj->gen_loader,
5554	map - obj->maps, i,
5555	targ_map - obj->maps);
5556	} else {
5557	err = bpf_map_update_elem(map->fd, &i, &fd, 0);
5558	}
5559	if (err) {
5560	err = -errno;
5561	pr_warn("map '%s': failed to initialize slot [%u] to map '%s' fd=%d: %s\n",
5562	map->name, i, targ_map->name, fd, errstr(err));
5563	return err;
5564	}
5565	pr_debug("map '%s': slot [%u] set to map '%s' fd=%d\n",
5566	map->name, i, targ_map->name, fd);
5567	}
5568
5569	zfree(&map->init_slots);
5570	map->init_slots_sz = 0;
5571
5572	return 0;
5573	}
5574
5575	static int init_prog_array_slots(struct bpf_object obj, struct bpf_map map)
5576	{
5577	const struct bpf_program *targ_prog;
5578	unsigned int i;
5579	int fd, err;
5580
5581	if (obj->gen_loader)
5582	return -ENOTSUP;
5583
5584	for (i = 0; i < map->init_slots_sz; i++) {
5585	if (!map->init_slots[i])
5586	continue;
5587
5588	targ_prog = map->init_slots[i];
5589	fd = bpf_program__fd(targ_prog);
5590
5591	err = bpf_map_update_elem(map->fd, &i, &fd, 0);
5592	if (err) {
5593	err = -errno;
5594	pr_warn("map '%s': failed to initialize slot [%u] to prog '%s' fd=%d: %s\n",
5595	map->name, i, targ_prog->name, fd, errstr(err));
5596	return err;
5597	}
5598	pr_debug("map '%s': slot [%u] set to prog '%s' fd=%d\n",
5599	map->name, i, targ_prog->name, fd);
5600	}
5601
5602	zfree(&map->init_slots);
5603	map->init_slots_sz = 0;
5604
5605	return 0;
5606	}
5607
5608	static int bpf_object_init_prog_arrays(struct bpf_object *obj)
5609	{
5610	struct bpf_map *map;
5611	int i, err;
5612
5613	for (i = 0; i < obj->nr_maps; i++) {
5614	map = &obj->maps[i];
5615
5616	if (!map->init_slots_sz \|\| map->def.type != BPF_MAP_TYPE_PROG_ARRAY)
5617	continue;
5618
5619	err = init_prog_array_slots(obj, map);
5620	if (err < 0)
5621	return err;
5622	}
5623	return 0;
5624	}
5625
5626	static int map_set_def_max_entries(struct bpf_map *map)
5627	{
5628	if (map->def.type == BPF_MAP_TYPE_PERF_EVENT_ARRAY && !map->def.max_entries) {
5629	int nr_cpus;
5630
5631	nr_cpus = libbpf_num_possible_cpus();
5632	if (nr_cpus < 0) {
5633	pr_warn("map '%s': failed to determine number of system CPUs: %d\n",
5634	map->name, nr_cpus);
5635	return nr_cpus;
5636	}
5637	pr_debug("map '%s': setting size to %d\n", map->name, nr_cpus);
5638	map->def.max_entries = nr_cpus;
5639	}
5640
5641	return 0;
5642	}
5643
5644	static int
5645	bpf_object__create_maps(struct bpf_object *obj)
5646	{
5647	struct bpf_map *map;
5648	unsigned int i, j;
5649	int err;
5650	bool retried;
5651
5652	for (i = 0; i < obj->nr_maps; i++) {
5653	map = &obj->maps[i];
5654
5655	/* To support old kernels, we skip creating global data maps
5656	* (.rodata, .data, .kconfig, etc); later on, during program
5657	* loading, if we detect that at least one of the to-be-loaded
5658	* programs is referencing any global data map, we'll error
5659	* out with program name and relocation index logged.
5660	* This approach allows to accommodate Clang emitting
5661	* unnecessary .rodata.str1.1 sections for string literals,
5662	* but also it allows to have CO-RE applications that use
5663	* global variables in some of BPF programs, but not others.
5664	* If those global variable-using programs are not loaded at
5665	* runtime due to bpf_program__set_autoload(prog, false),
5666	* bpf_object loading will succeed just fine even on old
5667	* kernels.
5668	* Same skipping applies to percpu data.
5669	*/
5670	if (bpf_map__is_internal(map)) {
5671	bool is_percpu = map->libbpf_type == LIBBPF_MAP_PERCPU;
5672	enum kern_feature_id feat_id;
5673
5674	feat_id = is_percpu ? FEAT_PERCPU_DATA : FEAT_GLOBAL_DATA;
5675	if (!kernel_supports(obj, feat_id))
5676	map->autocreate = false;
5677	}
5678
5679	if (!map->autocreate) {
5680	pr_debug("map '%s': skipped auto-creating...\n", map->name);
5681	continue;
5682	}
5683
5684	err = map_set_def_max_entries(map);
5685	if (err)
5686	goto err_out;
5687
5688	retried = false;
5689	retry:
5690	if (map->pin_path) {
5691	err = bpf_object__reuse_map(map);
5692	if (err) {
5693	pr_warn("map '%s': error reusing pinned map\n",
5694	map->name);
5695	goto err_out;
5696	}
5697	if (retried && map->fd < 0) {
5698	pr_warn("map '%s': cannot find pinned map\n",
5699	map->name);
5700	err = -ENOENT;
5701	goto err_out;
5702	}
5703	}
5704
5705	if (map->reused) {
5706	pr_debug("map '%s': skipping creation (preset fd=%d)\n",
5707	map->name, map->fd);
5708	} else {
5709	err = bpf_object__create_map(obj, map, false);
5710	if (err)
5711	goto err_out;
5712
5713	pr_debug("map '%s': created successfully, fd=%d\n",
5714	map->name, map->fd);
5715
5716	if (bpf_map__is_internal(map)) {
5717	err = bpf_object__populate_internal_map(obj, map);
5718	if (err < 0)
5719	goto err_out;
5720	} else if (map->def.type == BPF_MAP_TYPE_ARENA) {
5721	map->mmaped = mmap((void *)(long)map->map_extra,
5722	bpf_map_mmap_sz(map), PROT_READ \| PROT_WRITE,
5723	map->map_extra ? MAP_SHARED \| MAP_FIXED : MAP_SHARED,
5724	map->fd, 0);
5725	if (map->mmaped == MAP_FAILED) {
5726	err = -errno;
5727	map->mmaped = NULL;
5728	pr_warn("map '%s': failed to mmap arena: %s\n",
5729	map->name, errstr(err));
5730	return err;
5731	}
5732	if (obj->arena_data) {
5733	memcpy(map->mmaped + obj->arena_data_off, obj->arena_data,
5734	obj->arena_data_sz);
5735	zfree(&obj->arena_data);
5736	}
5737	}
5738	if (map->init_slots_sz && map->def.type != BPF_MAP_TYPE_PROG_ARRAY) {
5739	err = init_map_in_map_slots(obj, map);
5740	if (err < 0)
5741	goto err_out;
5742	}
5743	}
5744
5745	if (map->pin_path && !map->pinned) {
5746	err = bpf_map__pin(map, NULL);
5747	if (err) {
5748	if (!retried && err == -EEXIST) {
5749	retried = true;
5750	goto retry;
5751	}
5752	pr_warn("map '%s': failed to auto-pin at '%s': %s\n",
5753	map->name, map->pin_path, errstr(err));
5754	goto err_out;
5755	}
5756	}
5757	}
5758
5759	return 0;
5760
5761	err_out:
5762	pr_warn("map '%s': failed to create: %s\n", map->name, errstr(err));
5763	pr_perm_msg(err);
5764	for (j = 0; j < i; j++)
5765	zclose(obj->maps[j].fd);
5766	return err;
5767	}
5768
5769	static bool bpf_core_is_flavor_sep(const char *s)
5770	{
5771	/* check X___Y name pattern, where X and Y are not underscores */
5772	return s[0] != '_' && /* X */
5773	s[1] == '_' && s[2] == '_' && s[3] == '_' && /* ___ */
5774	s[4] != '_'; /* Y */
5775	}
5776
5777	/* Given 'some_struct_name___with_flavor' return the length of a name prefix
5778	* before last triple underscore. Struct name part after last triple
5779	* underscore is ignored by BPF CO-RE relocation during relocation matching.
5780	*/
5781	size_t bpf_core_essential_name_len(const char *name)
5782	{
5783	size_t n = strlen(name);
5784	int i;
5785
5786	for (i = n - 5; i >= 0; i--) {
5787	if (bpf_core_is_flavor_sep(name + i))
5788	return i + 1;
5789	}
5790	return n;
5791	}
5792
5793	void bpf_core_free_cands(struct bpf_core_cand_list *cands)
5794	{
5795	if (!cands)
5796	return;
5797
5798	free(cands->cands);
5799	free(cands);
5800	}
5801
5802	int bpf_core_add_cands(struct bpf_core_cand *local_cand,
5803	size_t local_essent_len,
5804	const struct btf *targ_btf,
5805	const char *targ_btf_name,
5806	int targ_start_id,
5807	struct bpf_core_cand_list *cands)
5808	{
5809	struct bpf_core_cand new_cands, cand;
5810	const struct btf_type t, local_t;
5811	const char targ_name, local_name;
5812	size_t targ_essent_len;
5813	int n, i;
5814
5815	local_t = btf__type_by_id(local_cand->btf, local_cand->id);
5816	local_name = btf__str_by_offset(local_cand->btf, local_t->name_off);
5817
5818	n = btf__type_cnt(targ_btf);
5819	for (i = targ_start_id; i < n; i++) {
5820	t = btf__type_by_id(targ_btf, i);
5821	if (!btf_kind_core_compat(t, local_t))
5822	continue;
5823
5824	targ_name = btf__name_by_offset(targ_btf, t->name_off);
5825	if (str_is_empty(targ_name))
5826	continue;
5827
5828	targ_essent_len = bpf_core_essential_name_len(targ_name);
5829	if (targ_essent_len != local_essent_len)
5830	continue;
5831
5832	if (strncmp(local_name, targ_name, local_essent_len) != 0)
5833	continue;
5834
5835	pr_debug("CO-RE relocating [%u] %s %s: found target candidate [%d] %s %s in [%s]\n",
5836	local_cand->id, btf_kind_str(local_t),
5837	local_name, i, btf_kind_str(t), targ_name,
5838	targ_btf_name);
5839	new_cands = libbpf_reallocarray(cands->cands, cands->len + 1,
5840	sizeof(*cands->cands));
5841	if (!new_cands)
5842	return -ENOMEM;
5843
5844	cand = &new_cands[cands->len];
5845	cand->btf = targ_btf;
5846	cand->id = i;
5847
5848	cands->cands = new_cands;
5849	cands->len++;
5850	}
5851	return 0;
5852	}
5853
5854	static int load_module_btfs(struct bpf_object *obj)
5855	{
5856	struct bpf_btf_info info;
5857	struct module_btf *mod_btf;
5858	struct btf *btf;
5859	char name[64];
5860	__u32 id = 0, len;
5861	int err, fd;
5862
5863	if (obj->btf_modules_loaded)
5864	return 0;
5865
5866	if (obj->gen_loader)
5867	return 0;
5868
5869	/* don't do this again, even if we find no module BTFs */
5870	obj->btf_modules_loaded = true;
5871
5872	/* kernel too old to support module BTFs */
5873	if (!kernel_supports(obj, FEAT_MODULE_BTF))
5874	return 0;
5875
5876	while (true) {
5877	err = bpf_btf_get_next_id(id, &id);
5878	if (err && errno == ENOENT)
5879	return 0;
5880	if (err && errno == EPERM) {
5881	pr_debug("skipping module BTFs loading, missing privileges\n");
5882	return 0;
5883	}
5884	if (err) {
5885	err = -errno;
5886	pr_warn("failed to iterate BTF objects: %s\n", errstr(err));
5887	return err;
5888	}
5889
5890	fd = bpf_btf_get_fd_by_id(id);
5891	if (fd < 0) {
5892	if (errno == ENOENT)
5893	continue; /* expected race: BTF was unloaded */
5894	err = -errno;
5895	pr_warn("failed to get BTF object #%u FD: %s\n", id, errstr(err));
5896	return err;
5897	}
5898
5899	len = sizeof(info);
5900	memset(&info, 0, sizeof(info));
5901	info.name = ptr_to_u64(name);
5902	info.name_len = sizeof(name);
5903
5904	btf = NULL;
5905	err = bpf_btf_get_info_by_fd(fd, &info, &len);
5906	if (err) {
5907	err = -errno;
5908	pr_warn("failed to get BTF object #%u info: %s\n", id, errstr(err));
5909	break;
5910	}
5911
5912	/* ignore non-module BTFs */
5913	if (!info.kernel_btf \|\| strcmp(name, "vmlinux") == 0) {
5914	close(fd);
5915	continue;
5916	}
5917
5918	btf = btf_get_from_fd(fd, obj->btf_vmlinux);
5919	err = libbpf_get_error(btf);
5920	if (err) {
5921	pr_warn("failed to load module [%s]'s BTF object #%u: %s\n",
5922	name, id, errstr(err));
5923	break;
5924	}
5925
5926	err = libbpf_ensure_mem((void **)&obj->btf_modules, &obj->btf_module_cap,
5927	sizeof(*obj->btf_modules), obj->btf_module_cnt + 1);
5928	if (err)
5929	break;
5930
5931	mod_btf = &obj->btf_modules[obj->btf_module_cnt];
5932
5933	mod_btf->btf = btf;
5934	mod_btf->id = id;
5935	mod_btf->fd = fd;
5936	mod_btf->name = strdup(name);
5937	if (!mod_btf->name) {
5938	err = -ENOMEM;
5939	break;
5940	}
5941	obj->btf_module_cnt++;
5942	}
5943
5944	if (err) {
5945	btf__free(btf);
5946	close(fd);
5947	}
5948	return err;
5949	}
5950
5951	static struct bpf_core_cand_list *
5952	bpf_core_find_cands(struct bpf_object obj, const struct btf local_btf, __u32 local_type_id)
5953	{
5954	struct bpf_core_cand local_cand = {};
5955	struct bpf_core_cand_list *cands;
5956	const struct btf *main_btf;
5957	const struct btf_type *local_t;
5958	const char *local_name;
5959	size_t local_essent_len;
5960	int err, i;
5961
5962	local_cand.btf = local_btf;
5963	local_cand.id = local_type_id;
5964	local_t = btf__type_by_id(local_btf, local_type_id);
5965	if (!local_t)
5966	return ERR_PTR(-EINVAL);
5967
5968	local_name = btf__name_by_offset(local_btf, local_t->name_off);
5969	if (str_is_empty(local_name))
5970	return ERR_PTR(-EINVAL);
5971	local_essent_len = bpf_core_essential_name_len(local_name);
5972
5973	cands = calloc(1, sizeof(*cands));
5974	if (!cands)
5975	return ERR_PTR(-ENOMEM);
5976
5977	/* Attempt to find target candidates in vmlinux BTF first */
5978	main_btf = obj->btf_vmlinux_override ?: obj->btf_vmlinux;
5979	err = bpf_core_add_cands(&local_cand, local_essent_len, main_btf, "vmlinux", 1, cands);
5980	if (err)
5981	goto err_out;
5982
5983	/* if vmlinux BTF has any candidate, don't got for module BTFs */
5984	if (cands->len)
5985	return cands;
5986
5987	/* if vmlinux BTF was overridden, don't attempt to load module BTFs */
5988	if (obj->btf_vmlinux_override)
5989	return cands;
5990
5991	/* now look through module BTFs, trying to still find candidates */
5992	err = load_module_btfs(obj);
5993	if (err)
5994	goto err_out;
5995
5996	for (i = 0; i < obj->btf_module_cnt; i++) {
5997	err = bpf_core_add_cands(&local_cand, local_essent_len,
5998	obj->btf_modules[i].btf,
5999	obj->btf_modules[i].name,
6000	btf__type_cnt(obj->btf_vmlinux),
6001	cands);
6002	if (err)
6003	goto err_out;
6004	}
6005
6006	return cands;
6007	err_out:
6008	bpf_core_free_cands(cands);
6009	return ERR_PTR(err);
6010	}
6011
6012	/* Check local and target types for compatibility. This check is used for
6013	* type-based CO-RE relocations and follow slightly different rules than
6014	* field-based relocations. This function assumes that root types were already
6015	* checked for name match. Beyond that initial root-level name check, names
6016	* are completely ignored. Compatibility rules are as follows:
6017	* - any two STRUCTs/UNIONs/FWDs/ENUMs/INTs are considered compatible, but
6018	* kind should match for local and target types (i.e., STRUCT is not
6019	* compatible with UNION);
6020	* - for ENUMs, the size is ignored;
6021	* - for INT, size and signedness are ignored;
6022	* - for ARRAY, dimensionality is ignored, element types are checked for
6023	* compatibility recursively;
6024	* - CONST/VOLATILE/RESTRICT modifiers are ignored;
6025	* - TYPEDEFs/PTRs are compatible if types they pointing to are compatible;
6026	* - FUNC_PROTOs are compatible if they have compatible signature: same
6027	* number of input args and compatible return and argument types.
6028	* These rules are not set in stone and probably will be adjusted as we get
6029	* more experience with using BPF CO-RE relocations.
6030	*/
6031	int bpf_core_types_are_compat(const struct btf *local_btf, __u32 local_id,
6032	const struct btf *targ_btf, __u32 targ_id)
6033	{
6034	return __bpf_core_types_are_compat(local_btf, local_id, targ_btf, targ_id, 32);
6035	}
6036
6037	int bpf_core_types_match(const struct btf *local_btf, __u32 local_id,
6038	const struct btf *targ_btf, __u32 targ_id)
6039	{
6040	return __bpf_core_types_match(local_btf, local_id, targ_btf, targ_id, false, 32);
6041	}
6042
6043	static size_t bpf_core_hash_fn(const long key, void *ctx)
6044	{
6045	return key;
6046	}
6047
6048	static bool bpf_core_equal_fn(const long k1, const long k2, void *ctx)
6049	{
6050	return k1 == k2;
6051	}
6052
6053	static int record_relo_core(struct bpf_program *prog,
6054	const struct bpf_core_relo *core_relo, int insn_idx)
6055	{
6056	struct reloc_desc relos, relo;
6057
6058	relos = libbpf_reallocarray(prog->reloc_desc,
6059	prog->nr_reloc + 1, sizeof(*relos));
6060	if (!relos)
6061	return -ENOMEM;
6062	relo = &relos[prog->nr_reloc];
6063	relo->type = RELO_CORE;
6064	relo->insn_idx = insn_idx;
6065	relo->core_relo = core_relo;
6066	prog->reloc_desc = relos;
6067	prog->nr_reloc++;
6068	return 0;
6069	}
6070
6071	static const struct bpf_core_relo find_relo_core(struct bpf_program prog, int insn_idx)
6072	{
6073	struct reloc_desc *relo;
6074	int i;
6075
6076	for (i = 0; i < prog->nr_reloc; i++) {
6077	relo = &prog->reloc_desc[i];
6078	if (relo->type != RELO_CORE \|\| relo->insn_idx != insn_idx)
6079	continue;
6080
6081	return relo->core_relo;
6082	}
6083
6084	return NULL;
6085	}
6086
6087	static int bpf_core_resolve_relo(struct bpf_program *prog,
6088	const struct bpf_core_relo *relo,
6089	int relo_idx,
6090	const struct btf *local_btf,
6091	struct hashmap *cand_cache,
6092	struct bpf_core_relo_res *targ_res)
6093	{
6094	struct bpf_core_spec specs_scratch[3] = {};
6095	struct bpf_core_cand_list *cands = NULL;
6096	const char *prog_name = prog->name;
6097	const struct btf_type *local_type;
6098	const char *local_name;
6099	__u32 local_id = relo->type_id;
6100	int err;
6101
6102	local_type = btf__type_by_id(local_btf, local_id);
6103	if (!local_type)
6104	return -EINVAL;
6105
6106	local_name = btf__name_by_offset(local_btf, local_type->name_off);
6107	if (!local_name)
6108	return -EINVAL;
6109
6110	if (relo->kind != BPF_CORE_TYPE_ID_LOCAL &&
6111	!hashmap__find(cand_cache, local_id, &cands)) {
6112	cands = bpf_core_find_cands(prog->obj, local_btf, local_id);
6113	if (IS_ERR(cands)) {
6114	pr_warn("prog '%s': relo #%d: target candidate search failed for [%u] %s %s: %ld\n",
6115	prog_name, relo_idx, local_id, btf_kind_str(local_type),
6116	local_name, PTR_ERR(cands));
6117	return PTR_ERR(cands);
6118	}
6119	err = hashmap__set(cand_cache, local_id, cands, NULL, NULL);
6120	if (err) {
6121	bpf_core_free_cands(cands);
6122	return err;
6123	}
6124	}
6125
6126	return bpf_core_calc_relo_insn(prog_name, relo, relo_idx, local_btf, cands, specs_scratch,
6127	targ_res);
6128	}
6129
6130	static int
6131	bpf_object__relocate_core(struct bpf_object obj, const char targ_btf_path)
6132	{
6133	const struct btf_ext_info_sec *sec;
6134	struct bpf_core_relo_res targ_res;
6135	const struct bpf_core_relo *rec;
6136	const struct btf_ext_info *seg;
6137	struct hashmap_entry *entry;
6138	struct hashmap *cand_cache = NULL;
6139	struct bpf_program *prog;
6140	struct bpf_insn *insn;
6141	const char *sec_name;
6142	int i, err = 0, insn_idx, sec_idx, sec_num;
6143
6144	if (obj->btf_ext->core_relo_info.len == 0)
6145	return 0;
6146
6147	if (targ_btf_path) {
6148	obj->btf_vmlinux_override = btf__parse(targ_btf_path, NULL);
6149	err = libbpf_get_error(obj->btf_vmlinux_override);
6150	if (err) {
6151	pr_warn("failed to parse target BTF: %s\n", errstr(err));
6152	return err;
6153	}
6154	}
6155
6156	cand_cache = hashmap__new(bpf_core_hash_fn, bpf_core_equal_fn, NULL);
6157	if (IS_ERR(cand_cache)) {
6158	err = PTR_ERR(cand_cache);
6159	goto out;
6160	}
6161
6162	seg = &obj->btf_ext->core_relo_info;
6163	sec_num = 0;
6164	for_each_btf_ext_sec(seg, sec) {
6165	sec_idx = seg->sec_idxs[sec_num];
6166	sec_num++;
6167
6168	sec_name = btf__name_by_offset(obj->btf, sec->sec_name_off);
6169	if (str_is_empty(sec_name)) {
6170	err = -EINVAL;
6171	goto out;
6172	}
6173
6174	pr_debug("sec '%s': found %u CO-RE relocations\n", sec_name, sec->num_info);
6175
6176	for_each_btf_ext_rec(seg, sec, i, rec) {
6177	if (rec->insn_off % BPF_INSN_SZ)
6178	return -EINVAL;
6179	insn_idx = rec->insn_off / BPF_INSN_SZ;
6180	prog = find_prog_by_sec_insn(obj, sec_idx, insn_idx);
6181	if (!prog) {
6182	/* When __weak subprog is "overridden" by another instance
6183	* of the subprog from a different object file, linker still
6184	* appends all the .BTF.ext info that used to belong to that
6185	* eliminated subprogram.
6186	* This is similar to what x86-64 linker does for relocations.
6187	* So just ignore such relocations just like we ignore
6188	* subprog instructions when discovering subprograms.
6189	*/
6190	pr_debug("sec '%s': skipping CO-RE relocation #%d for insn #%d belonging to eliminated weak subprogram\n",
6191	sec_name, i, insn_idx);
6192	continue;
6193	}
6194	/* no need to apply CO-RE relocation if the program is
6195	* not going to be loaded
6196	*/
6197	if (!prog->autoload)
6198	continue;
6199
6200	/* adjust insn_idx from section frame of reference to the local
6201	* program's frame of reference; (sub-)program code is not yet
6202	* relocated, so it's enough to just subtract in-section offset
6203	*/
6204	insn_idx = insn_idx - prog->sec_insn_off;
6205	if (insn_idx >= prog->insns_cnt)
6206	return -EINVAL;
6207	insn = &prog->insns[insn_idx];
6208
6209	if (is_ldimm64_insn(insn) && (size_t)insn_idx + 1 >= prog->insns_cnt) {
6210	pr_warn("prog '%s': relo #%d: insn #%d (LDIMM64) is truncated\n",
6211	prog->name, i, insn_idx);
6212	err = -EINVAL;
6213	goto out;
6214	}
6215
6216	err = record_relo_core(prog, rec, insn_idx);
6217	if (err) {
6218	pr_warn("prog '%s': relo #%d: failed to record relocation: %s\n",
6219	prog->name, i, errstr(err));
6220	goto out;
6221	}
6222
6223	if (prog->obj->gen_loader)
6224	continue;
6225
6226	err = bpf_core_resolve_relo(prog, rec, i, obj->btf, cand_cache, &targ_res);
6227	if (err) {
6228	pr_warn("prog '%s': relo #%d: failed to relocate: %s\n",
6229	prog->name, i, errstr(err));
6230	goto out;
6231	}
6232
6233	err = bpf_core_patch_insn(prog->name, insn, insn_idx, rec, i, &targ_res);
6234	if (err) {
6235	pr_warn("prog '%s': relo #%d: failed to patch insn #%d: %s\n",
6236	prog->name, i, insn_idx, errstr(err));
6237	goto out;
6238	}
6239	}
6240	}
6241
6242	out:
6243	/* obj->btf_vmlinux and module BTFs are freed after object load */
6244	btf__free(obj->btf_vmlinux_override);
6245	obj->btf_vmlinux_override = NULL;
6246
6247	if (!IS_ERR_OR_NULL(cand_cache)) {
6248	hashmap__for_each_entry(cand_cache, entry, i) {
6249	bpf_core_free_cands(entry->pvalue);
6250	}
6251	hashmap__free(cand_cache);
6252	}
6253	return err;
6254	}
6255
6256	/* base map load ldimm64 special constant, used also for log fixup logic */
6257	#define POISON_LDIMM64_MAP_BASE 2001000000
6258	#define POISON_LDIMM64_MAP_PFX "200100"
6259
6260	static void poison_map_ldimm64(struct bpf_program *prog, int relo_idx,
6261	int insn_idx, struct bpf_insn *insn,
6262	int map_idx, const struct bpf_map *map)
6263	{
6264	int i;
6265
6266	pr_debug("prog '%s': relo #%d: poisoning insn #%d that loads map #%d '%s'\n",
6267	prog->name, relo_idx, insn_idx, map_idx, map->name);
6268
6269	/* we turn single ldimm64 into two identical invalid calls */
6270	for (i = 0; i < 2; i++) {
6271	insn->code = BPF_JMP \| BPF_CALL;
6272	insn->dst_reg = 0;
6273	insn->src_reg = 0;
6274	insn->off = 0;
6275	/* if this instruction is reachable (not a dead code),
6276	* verifier will complain with something like:
6277	* invalid func unknown#2001000123
6278	* where lower 123 is map index into obj->maps[] array
6279	*/
6280	insn->imm = POISON_LDIMM64_MAP_BASE + map_idx;
6281
6282	insn++;
6283	}
6284	}
6285
6286	/* unresolved kfunc call special constant, used also for log fixup logic */
6287	#define POISON_CALL_KFUNC_BASE 2002000000
6288	#define POISON_CALL_KFUNC_PFX "2002"
6289
6290	static void poison_kfunc_call(struct bpf_program *prog, int relo_idx,
6291	int insn_idx, struct bpf_insn *insn,
6292	int ext_idx, const struct extern_desc *ext)
6293	{
6294	pr_debug("prog '%s': relo #%d: poisoning insn #%d that calls kfunc '%s'\n",
6295	prog->name, relo_idx, insn_idx, ext->name);
6296
6297	/* we turn kfunc call into invalid helper call with identifiable constant */
6298	insn->code = BPF_JMP \| BPF_CALL;
6299	insn->dst_reg = 0;
6300	insn->src_reg = 0;
6301	insn->off = 0;
6302	/* if this instruction is reachable (not a dead code),
6303	* verifier will complain with something like:
6304	* invalid func unknown#2001000123
6305	* where lower 123 is extern index into obj->externs[] array
6306	*/
6307	insn->imm = POISON_CALL_KFUNC_BASE + ext_idx;
6308	}
6309
6310	static int find_jt_map(struct bpf_object obj, struct bpf_program prog, unsigned int sym_off)
6311	{
6312	size_t i;
6313
6314	for (i = 0; i < obj->jumptable_map_cnt; i++) {
6315	/*
6316	* This might happen that same offset is used for two different
6317	* programs (as jump tables can be the same). However, for
6318	* different programs different maps should be created.
6319	*/
6320	if (obj->jumptable_maps[i].sym_off == sym_off &&
6321	obj->jumptable_maps[i].prog == prog)
6322	return obj->jumptable_maps[i].fd;
6323	}
6324
6325	return -ENOENT;
6326	}
6327
6328	static int add_jt_map(struct bpf_object obj, struct bpf_program prog, unsigned int sym_off, int map_fd)
6329	{
6330	size_t cnt = obj->jumptable_map_cnt;
6331	size_t size = sizeof(obj->jumptable_maps[0]);
6332	void *tmp;
6333
6334	tmp = libbpf_reallocarray(obj->jumptable_maps, cnt + 1, size);
6335	if (!tmp)
6336	return -ENOMEM;
6337
6338	obj->jumptable_maps = tmp;
6339	obj->jumptable_maps[cnt].prog = prog;
6340	obj->jumptable_maps[cnt].sym_off = sym_off;
6341	obj->jumptable_maps[cnt].fd = map_fd;
6342	obj->jumptable_map_cnt++;
6343
6344	return 0;
6345	}
6346
6347	static int find_subprog_idx(struct bpf_program *prog, int insn_idx)
6348	{
6349	int i;
6350
6351	for (i = prog->subprog_cnt - 1; i >= 0; i--) {
6352	if (insn_idx >= prog->subprogs[i].sub_insn_off)
6353	return i;
6354	}
6355
6356	return -1;
6357	}
6358
6359	static int create_jt_map(struct bpf_object obj, struct bpf_program prog, struct reloc_desc *relo)
6360	{
6361	const __u32 jt_entry_size = 8;
6362	unsigned int sym_off = relo->sym_off;
6363	int jt_size = relo->sym_size;
6364	__u32 max_entries = jt_size / jt_entry_size;
6365	__u32 value_size = sizeof(struct bpf_insn_array_value);
6366	struct bpf_insn_array_value val = {};
6367	int subprog_idx;
6368	int map_fd, err;
6369	__u64 insn_off;
6370	__u64 *jt;
6371	__u32 i;
6372
6373	map_fd = find_jt_map(obj, prog, sym_off);
6374	if (map_fd >= 0)
6375	return map_fd;
6376
6377	if (sym_off % jt_entry_size) {
6378	pr_warn("map '.jumptables': jumptable start %u should be multiple of %u\n",
6379	sym_off, jt_entry_size);
6380	return -EINVAL;
6381	}
6382
6383	if (jt_size % jt_entry_size) {
6384	pr_warn("map '.jumptables': jumptable size %d should be multiple of %u\n",
6385	jt_size, jt_entry_size);
6386	return -EINVAL;
6387	}
6388
6389	map_fd = bpf_map_create(BPF_MAP_TYPE_INSN_ARRAY, ".jumptables",
6390	4, value_size, max_entries, NULL);
6391	if (map_fd < 0)
6392	return map_fd;
6393
6394	if (!obj->jumptables_data) {
6395	pr_warn("map '.jumptables': ELF file is missing jump table data\n");
6396	err = -EINVAL;
6397	goto err_close;
6398	}
6399	if (sym_off + jt_size > obj->jumptables_data_sz) {
6400	pr_warn("map '.jumptables': jumptables_data size is %zu, trying to access %u\n",
6401	obj->jumptables_data_sz, sym_off + jt_size);
6402	err = -EINVAL;
6403	goto err_close;
6404	}
6405
6406	subprog_idx = -1; /* main program */
6407	if (relo->insn_idx < 0 \|\| relo->insn_idx >= prog->insns_cnt) {
6408	pr_warn("map '.jumptables': invalid instruction index %d\n", relo->insn_idx);
6409	err = -EINVAL;
6410	goto err_close;
6411	}
6412	if (prog->subprogs)
6413	subprog_idx = find_subprog_idx(prog, relo->insn_idx);
6414
6415	jt = (__u64 *)(obj->jumptables_data + sym_off);
6416	for (i = 0; i < max_entries; i++) {
6417	/*
6418	* The offset should be made to be relative to the beginning of
6419	* the main function, not the subfunction.
6420	*/
6421	insn_off = jt[i]/sizeof(struct bpf_insn);
6422	if (subprog_idx >= 0) {
6423	insn_off -= prog->subprogs[subprog_idx].sec_insn_off;
6424	insn_off += prog->subprogs[subprog_idx].sub_insn_off;
6425	} else {
6426	insn_off -= prog->sec_insn_off;
6427	}
6428
6429	/*
6430	* LLVM-generated jump tables contain u64 records, however
6431	* should contain values that fit in u32.
6432	*/
6433	if (insn_off > UINT32_MAX) {
6434	pr_warn("map '.jumptables': invalid jump table value 0x%llx at offset %u\n",
6435	(unsigned long long)jt[i], sym_off + i * jt_entry_size);
6436	err = -EINVAL;
6437	goto err_close;
6438	}
6439
6440	val.orig_off = insn_off;
6441	err = bpf_map_update_elem(map_fd, &i, &val, 0);
6442	if (err)
6443	goto err_close;
6444	}
6445
6446	err = bpf_map_freeze(map_fd);
6447	if (err)
6448	goto err_close;
6449
6450	err = add_jt_map(obj, prog, sym_off, map_fd);
6451	if (err)
6452	goto err_close;
6453
6454	return map_fd;
6455
6456	err_close:
6457	close(map_fd);
6458	return err;
6459	}
6460
6461	/* Relocate data references within program code:
6462	* - map references;
6463	* - global variable references;
6464	* - extern references.
6465	*/
6466	static int
6467	bpf_object__relocate_data(struct bpf_object obj, struct bpf_program prog)
6468	{
6469	int i;
6470
6471	for (i = 0; i < prog->nr_reloc; i++) {
6472	struct reloc_desc *relo = &prog->reloc_desc[i];
6473	struct bpf_insn *insn = &prog->insns[relo->insn_idx];
6474	const struct bpf_map *map;
6475	struct extern_desc *ext;
6476
6477	switch (relo->type) {
6478	case RELO_LD64:
6479	map = &obj->maps[relo->map_idx];
6480	if (obj->gen_loader) {
6481	insn[0].src_reg = BPF_PSEUDO_MAP_IDX;
6482	insn[0].imm = relo->map_idx;
6483	} else if (map->autocreate) {
6484	insn[0].src_reg = BPF_PSEUDO_MAP_FD;
6485	insn[0].imm = map->fd;
6486	} else {
6487	poison_map_ldimm64(prog, i, relo->insn_idx, insn,
6488	relo->map_idx, map);
6489	}
6490	break;
6491	case RELO_DATA:
6492	map = &obj->maps[relo->map_idx];
6493	insn[1].imm = insn[0].imm + relo->sym_off;
6494
6495	if (relo->map_idx == obj->arena_map_idx)
6496	insn[1].imm += obj->arena_data_off;
6497
6498	if (obj->gen_loader) {
6499	insn[0].src_reg = BPF_PSEUDO_MAP_IDX_VALUE;
6500	insn[0].imm = relo->map_idx;
6501	} else if (map->autocreate) {
6502	insn[0].src_reg = BPF_PSEUDO_MAP_VALUE;
6503	insn[0].imm = map->fd;
6504	} else {
6505	poison_map_ldimm64(prog, i, relo->insn_idx, insn,
6506	relo->map_idx, map);
6507	}
6508	break;
6509	case RELO_EXTERN_LD64:
6510	ext = &obj->externs[relo->ext_idx];
6511	if (ext->type == EXT_KCFG) {
6512	if (obj->gen_loader) {
6513	insn[0].src_reg = BPF_PSEUDO_MAP_IDX_VALUE;
6514	insn[0].imm = obj->kconfig_map_idx;
6515	} else {
6516	insn[0].src_reg = BPF_PSEUDO_MAP_VALUE;
6517	insn[0].imm = obj->maps[obj->kconfig_map_idx].fd;
6518	}
6519	insn[1].imm = ext->kcfg.data_off;
6520	} else /* EXT_KSYM */ {
6521	if (ext->ksym.type_id && ext->is_set) { /* typed ksyms */
6522	insn[0].src_reg = BPF_PSEUDO_BTF_ID;
6523	insn[0].imm = ext->ksym.kernel_btf_id;
6524	insn[1].imm = ext->ksym.kernel_btf_obj_fd;
6525	} else { /* typeless ksyms or unresolved typed ksyms */
6526	insn[0].imm = (__u32)ext->ksym.addr;
6527	insn[1].imm = ext->ksym.addr >> 32;
6528	}
6529	}
6530	break;
6531	case RELO_EXTERN_CALL:
6532	ext = &obj->externs[relo->ext_idx];
6533	insn[0].src_reg = BPF_PSEUDO_KFUNC_CALL;
6534	if (ext->is_set) {
6535	insn[0].imm = ext->ksym.kernel_btf_id;
6536	insn[0].off = ext->ksym.btf_fd_idx;
6537	} else { /* unresolved weak kfunc call */
6538	poison_kfunc_call(prog, i, relo->insn_idx, insn,
6539	relo->ext_idx, ext);
6540	}
6541	break;
6542	case RELO_SUBPROG_ADDR:
6543	if (insn[0].src_reg != BPF_PSEUDO_FUNC) {
6544	pr_warn("prog '%s': relo #%d: bad insn\n",
6545	prog->name, i);
6546	return -EINVAL;
6547	}
6548	/* handled already */
6549	break;
6550	case RELO_CALL:
6551	/* handled already */
6552	break;
6553	case RELO_CORE:
6554	/* will be handled by bpf_program_record_relos() */
6555	break;
6556	case RELO_INSN_ARRAY: {
6557	int map_fd;
6558
6559	map_fd = create_jt_map(obj, prog, relo);
6560	if (map_fd < 0) {
6561	pr_warn("prog '%s': relo #%d: can't create jump table: sym_off %u\n",
6562	prog->name, i, relo->sym_off);
6563	return map_fd;
6564	}
6565	insn[0].src_reg = BPF_PSEUDO_MAP_VALUE;
6566	insn->imm = map_fd;
6567	insn->off = 0;
6568	}
6569	break;
6570	default:
6571	pr_warn("prog '%s': relo #%d: bad relo type %u\n",
6572	prog->name, i, relo->type);
6573	return -EINVAL;
6574	}
6575	}
6576
6577	return 0;
6578	}
6579
6580	static int adjust_prog_btf_ext_info(const struct bpf_object *obj,
6581	const struct bpf_program *prog,
6582	const struct btf_ext_info *ext_info,
6583	void *prog_info, __u32 prog_rec_cnt,
6584	__u32 *prog_rec_sz)
6585	{
6586	void copy_start = NULL, copy_end = NULL;
6587	void rec, rec_end, *new_prog_info;
6588	const struct btf_ext_info_sec *sec;
6589	size_t old_sz, new_sz;
6590	int i, sec_num, sec_idx, off_adj;
6591
6592	sec_num = 0;
6593	for_each_btf_ext_sec(ext_info, sec) {
6594	sec_idx = ext_info->sec_idxs[sec_num];
6595	sec_num++;
6596	if (prog->sec_idx != sec_idx)
6597	continue;
6598
6599	for_each_btf_ext_rec(ext_info, sec, i, rec) {
6600	__u32 insn_off = (__u32 )rec / BPF_INSN_SZ;
6601
6602	if (insn_off < prog->sec_insn_off)
6603	continue;
6604	if (insn_off >= prog->sec_insn_off + prog->sec_insn_cnt)
6605	break;
6606
6607	if (!copy_start)
6608	copy_start = rec;
6609	copy_end = rec + ext_info->rec_size;
6610	}
6611
6612	if (!copy_start)
6613	return -ENOENT;
6614
6615	/* append func/line info of a given (sub-)program to the main
6616	* program func/line info
6617	*/
6618	old_sz = (size_t)(prog_rec_cnt) ext_info->rec_size;
6619	new_sz = old_sz + (copy_end - copy_start);
6620	new_prog_info = realloc(*prog_info, new_sz);
6621	if (!new_prog_info)
6622	return -ENOMEM;
6623	*prog_info = new_prog_info;
6624	*prog_rec_cnt = new_sz / ext_info->rec_size;
6625	memcpy(new_prog_info + old_sz, copy_start, copy_end - copy_start);
6626
6627	/* Kernel instruction offsets are in units of 8-byte
6628	* instructions, while .BTF.ext instruction offsets generated
6629	* by Clang are in units of bytes. So convert Clang offsets
6630	* into kernel offsets and adjust offset according to program
6631	* relocated position.
6632	*/
6633	off_adj = prog->sub_insn_off - prog->sec_insn_off;
6634	rec = new_prog_info + old_sz;
6635	rec_end = new_prog_info + new_sz;
6636	for (; rec < rec_end; rec += ext_info->rec_size) {
6637	__u32 *insn_off = rec;
6638
6639	insn_off = insn_off / BPF_INSN_SZ + off_adj;
6640	}
6641	*prog_rec_sz = ext_info->rec_size;
6642	return 0;
6643	}
6644
6645	return -ENOENT;
6646	}
6647
6648	static int
6649	reloc_prog_func_and_line_info(const struct bpf_object *obj,
6650	struct bpf_program *main_prog,
6651	const struct bpf_program *prog)
6652	{
6653	int err;
6654
6655	/* no .BTF.ext relocation if .BTF.ext is missing or kernel doesn't
6656	* support func/line info
6657	*/
6658	if (!obj->btf_ext \|\| !kernel_supports(obj, FEAT_BTF_FUNC))
6659	return 0;
6660
6661	/* only attempt func info relocation if main program's func_info
6662	* relocation was successful
6663	*/
6664	if (main_prog != prog && !main_prog->func_info)
6665	goto line_info;
6666
6667	err = adjust_prog_btf_ext_info(obj, prog, &obj->btf_ext->func_info,
6668	&main_prog->func_info,
6669	&main_prog->func_info_cnt,
6670	&main_prog->func_info_rec_size);
6671	if (err) {
6672	if (err != -ENOENT) {
6673	pr_warn("prog '%s': error relocating .BTF.ext function info: %s\n",
6674	prog->name, errstr(err));
6675	return err;
6676	}
6677	if (main_prog->func_info) {
6678	/*
6679	* Some info has already been found but has problem
6680	* in the last btf_ext reloc. Must have to error out.
6681	*/
6682	pr_warn("prog '%s': missing .BTF.ext function info.\n", prog->name);
6683	return err;
6684	}
6685	/* Have problem loading the very first info. Ignore the rest. */
6686	pr_warn("prog '%s': missing .BTF.ext function info for the main program, skipping all of .BTF.ext func info.\n",
6687	prog->name);
6688	}
6689
6690	line_info:
6691	/* don't relocate line info if main program's relocation failed */
6692	if (main_prog != prog && !main_prog->line_info)
6693	return 0;
6694
6695	err = adjust_prog_btf_ext_info(obj, prog, &obj->btf_ext->line_info,
6696	&main_prog->line_info,
6697	&main_prog->line_info_cnt,
6698	&main_prog->line_info_rec_size);
6699	if (err) {
6700	if (err != -ENOENT) {
6701	pr_warn("prog '%s': error relocating .BTF.ext line info: %s\n",
6702	prog->name, errstr(err));
6703	return err;
6704	}
6705	if (main_prog->line_info) {
6706	/*
6707	* Some info has already been found but has problem
6708	* in the last btf_ext reloc. Must have to error out.
6709	*/
6710	pr_warn("prog '%s': missing .BTF.ext line info.\n", prog->name);
6711	return err;
6712	}
6713	/* Have problem loading the very first info. Ignore the rest. */
6714	pr_warn("prog '%s': missing .BTF.ext line info for the main program, skipping all of .BTF.ext line info.\n",
6715	prog->name);
6716	}
6717	return 0;
6718	}
6719
6720	static int cmp_relo_by_insn_idx(const void key, const void elem)
6721	{
6722	size_t insn_idx = (const size_t )key;
6723	const struct reloc_desc *relo = elem;
6724
6725	if (insn_idx == relo->insn_idx)
6726	return 0;
6727	return insn_idx < relo->insn_idx ? -1 : 1;
6728	}
6729
6730	static struct reloc_desc find_prog_insn_relo(const struct bpf_program prog, size_t insn_idx)
6731	{
6732	if (!prog->nr_reloc)
6733	return NULL;
6734	return bsearch(&insn_idx, prog->reloc_desc, prog->nr_reloc,
6735	sizeof(*prog->reloc_desc), cmp_relo_by_insn_idx);
6736	}
6737
6738	static int append_subprog_relos(struct bpf_program main_prog, struct bpf_program subprog)
6739	{
6740	int new_cnt = main_prog->nr_reloc + subprog->nr_reloc;
6741	struct reloc_desc *relos;
6742	int i;
6743
6744	if (main_prog == subprog)
6745	return 0;
6746	relos = libbpf_reallocarray(main_prog->reloc_desc, new_cnt, sizeof(*relos));
6747	/* if new count is zero, reallocarray can return a valid NULL result;
6748	* in this case the previous pointer will be freed, so we have to
6749	* reassign old pointer to the new value (even if it's NULL)
6750	*/
6751	if (!relos && new_cnt)
6752	return -ENOMEM;
6753	if (subprog->nr_reloc)
6754	memcpy(relos + main_prog->nr_reloc, subprog->reloc_desc,
6755	sizeof(relos) subprog->nr_reloc);
6756
6757	for (i = main_prog->nr_reloc; i < new_cnt; i++)
6758	relos[i].insn_idx += subprog->sub_insn_off;
6759	/* After insn_idx adjustment the 'relos' array is still sorted
6760	* by insn_idx and doesn't break bsearch.
6761	*/
6762	main_prog->reloc_desc = relos;
6763	main_prog->nr_reloc = new_cnt;
6764	return 0;
6765	}
6766
6767	static int save_subprog_offsets(struct bpf_program main_prog, struct bpf_program subprog)
6768	{
6769	size_t size = sizeof(main_prog->subprogs[0]);
6770	int cnt = main_prog->subprog_cnt;
6771	void *tmp;
6772
6773	tmp = libbpf_reallocarray(main_prog->subprogs, cnt + 1, size);
6774	if (!tmp)
6775	return -ENOMEM;
6776
6777	main_prog->subprogs = tmp;
6778	main_prog->subprogs[cnt].sec_insn_off = subprog->sec_insn_off;
6779	main_prog->subprogs[cnt].sub_insn_off = subprog->sub_insn_off;
6780	main_prog->subprog_cnt++;
6781
6782	return 0;
6783	}
6784
6785	static int
6786	bpf_object__append_subprog_code(struct bpf_object obj, struct bpf_program main_prog,
6787	struct bpf_program *subprog)
6788	{
6789	struct bpf_insn *insns;
6790	size_t new_cnt;
6791	int err;
6792
6793	subprog->sub_insn_off = main_prog->insns_cnt;
6794
6795	new_cnt = main_prog->insns_cnt + subprog->insns_cnt;
6796	insns = libbpf_reallocarray(main_prog->insns, new_cnt, sizeof(*insns));
6797	if (!insns) {
6798	pr_warn("prog '%s': failed to realloc prog code\n", main_prog->name);
6799	return -ENOMEM;
6800	}
6801	main_prog->insns = insns;
6802	main_prog->insns_cnt = new_cnt;
6803
6804	memcpy(main_prog->insns + subprog->sub_insn_off, subprog->insns,
6805	subprog->insns_cnt * sizeof(*insns));
6806
6807	pr_debug("prog '%s': added %zu insns from sub-prog '%s'\n",
6808	main_prog->name, subprog->insns_cnt, subprog->name);
6809
6810	/* The subprog insns are now appended. Append its relos too. */
6811	err = append_subprog_relos(main_prog, subprog);
6812	if (err)
6813	return err;
6814
6815	err = save_subprog_offsets(main_prog, subprog);
6816	if (err) {
6817	pr_warn("prog '%s': failed to add subprog offsets: %s\n",
6818	main_prog->name, errstr(err));
6819	return err;
6820	}
6821
6822	return 0;
6823	}
6824
6825	static int
6826	bpf_object__reloc_code(struct bpf_object obj, struct bpf_program main_prog,
6827	struct bpf_program *prog)
6828	{
6829	size_t sub_insn_idx, insn_idx;
6830	struct bpf_program *subprog;
6831	struct reloc_desc *relo;
6832	struct bpf_insn *insn;
6833	int err;
6834
6835	err = reloc_prog_func_and_line_info(obj, main_prog, prog);
6836	if (err)
6837	return err;
6838
6839	for (insn_idx = 0; insn_idx < prog->sec_insn_cnt; insn_idx++) {
6840	insn = &main_prog->insns[prog->sub_insn_off + insn_idx];
6841	if (!insn_is_subprog_call(insn) && !insn_is_pseudo_func(insn))
6842	continue;
6843
6844	relo = find_prog_insn_relo(prog, insn_idx);
6845	if (relo && relo->type == RELO_EXTERN_CALL)
6846	/* kfunc relocations will be handled later
6847	* in bpf_object__relocate_data()
6848	*/
6849	continue;
6850	if (relo && relo->type != RELO_CALL && relo->type != RELO_SUBPROG_ADDR) {
6851	pr_warn("prog '%s': unexpected relo for insn #%zu, type %u\n",
6852	prog->name, insn_idx, relo->type);
6853	return -LIBBPF_ERRNO__RELOC;
6854	}
6855	if (relo) {
6856	/* sub-program instruction index is a combination of
6857	* an offset of a symbol pointed to by relocation and
6858	* call instruction's imm field; for global functions,
6859	* call always has imm = -1, but for static functions
6860	* relocation is against STT_SECTION and insn->imm
6861	* points to a start of a static function
6862	*
6863	* for subprog addr relocation, the relo->sym_off + insn->imm is
6864	* the byte offset in the corresponding section.
6865	*/
6866	if (relo->type == RELO_CALL)
6867	sub_insn_idx = relo->sym_off / BPF_INSN_SZ + insn->imm + 1;
6868	else
6869	sub_insn_idx = (relo->sym_off + insn->imm) / BPF_INSN_SZ;
6870	} else if (insn_is_pseudo_func(insn)) {
6871	/*
6872	* RELO_SUBPROG_ADDR relo is always emitted even if both
6873	* functions are in the same section, so it shouldn't reach here.
6874	*/
6875	pr_warn("prog '%s': missing subprog addr relo for insn #%zu\n",
6876	prog->name, insn_idx);
6877	return -LIBBPF_ERRNO__RELOC;
6878	} else {
6879	/* if subprogram call is to a static function within
6880	* the same ELF section, there won't be any relocation
6881	* emitted, but it also means there is no additional
6882	* offset necessary, insns->imm is relative to
6883	* instruction's original position within the section
6884	*/
6885	sub_insn_idx = prog->sec_insn_off + insn_idx + insn->imm + 1;
6886	}
6887
6888	/* we enforce that sub-programs should be in .text section */
6889	subprog = find_prog_by_sec_insn(obj, obj->efile.text_shndx, sub_insn_idx);
6890	if (!subprog) {
6891	pr_warn("prog '%s': no .text section found yet sub-program call exists\n",
6892	prog->name);
6893	return -LIBBPF_ERRNO__RELOC;
6894	}
6895
6896	/* if it's the first call instruction calling into this
6897	* subprogram (meaning this subprog hasn't been processed
6898	* yet) within the context of current main program:
6899	* - append it at the end of main program's instructions blog;
6900	* - process is recursively, while current program is put on hold;
6901	* - if that subprogram calls some other not yet processes
6902	* subprogram, same thing will happen recursively until
6903	* there are no more unprocesses subprograms left to append
6904	* and relocate.
6905	*/
6906	if (subprog->sub_insn_off == 0) {
6907	err = bpf_object__append_subprog_code(obj, main_prog, subprog);
6908	if (err)
6909	return err;
6910	err = bpf_object__reloc_code(obj, main_prog, subprog);
6911	if (err)
6912	return err;
6913	}
6914
6915	/* main_prog->insns memory could have been re-allocated, so
6916	* calculate pointer again
6917	*/
6918	insn = &main_prog->insns[prog->sub_insn_off + insn_idx];
6919	/* calculate correct instruction position within current main
6920	* prog; each main prog can have a different set of
6921	* subprograms appended (potentially in different order as
6922	* well), so position of any subprog can be different for
6923	* different main programs
6924	*/
6925	insn->imm = subprog->sub_insn_off - (prog->sub_insn_off + insn_idx) - 1;
6926
6927	pr_debug("prog '%s': insn #%zu relocated, imm %d points to subprog '%s' (now at %zu offset)\n",
6928	prog->name, insn_idx, insn->imm, subprog->name, subprog->sub_insn_off);
6929	}
6930
6931	return 0;
6932	}
6933
6934	/*
6935	* Relocate sub-program calls.
6936	*
6937	* Algorithm operates as follows. Each entry-point BPF program (referred to as
6938	* main prog) is processed separately. For each subprog (non-entry functions,
6939	* that can be called from either entry progs or other subprogs) gets their
6940	* sub_insn_off reset to zero. This serves as indicator that this subprogram
6941	* hasn't been yet appended and relocated within current main prog. Once its
6942	* relocated, sub_insn_off will point at the position within current main prog
6943	* where given subprog was appended. This will further be used to relocate all
6944	* the call instructions jumping into this subprog.
6945	*
6946	* We start with main program and process all call instructions. If the call
6947	* is into a subprog that hasn't been processed (i.e., subprog->sub_insn_off
6948	* is zero), subprog instructions are appended at the end of main program's
6949	* instruction array. Then main program is "put on hold" while we recursively
6950	* process newly appended subprogram. If that subprogram calls into another
6951	* subprogram that hasn't been appended, new subprogram is appended again to
6952	* the main prog's instructions (subprog's instructions are always left
6953	* untouched, as they need to be in unmodified state for subsequent main progs
6954	* and subprog instructions are always sent only as part of a main prog) and
6955	* the process continues recursively. Once all the subprogs called from a main
6956	* prog or any of its subprogs are appended (and relocated), all their
6957	* positions within finalized instructions array are known, so it's easy to
6958	* rewrite call instructions with correct relative offsets, corresponding to
6959	* desired target subprog.
6960	*
6961	* Its important to realize that some subprogs might not be called from some
6962	* main prog and any of its called/used subprogs. Those will keep their
6963	* subprog->sub_insn_off as zero at all times and won't be appended to current
6964	* main prog and won't be relocated within the context of current main prog.
6965	* They might still be used from other main progs later.
6966	*
6967	* Visually this process can be shown as below. Suppose we have two main
6968	* programs mainA and mainB and BPF object contains three subprogs: subA,
6969	* subB, and subC. mainA calls only subA, mainB calls only subC, but subA and
6970	* subC both call subB:
6971	*
6972	* +--------+ +-------+
6973	* \| v v \|
6974	* +--+---+ +--+-+-+ +---+--+
6975	* \| subA \| \| subB \| \| subC \|
6976	* +--+---+ +------+ +---+--+
6977	* ^ ^
6978	* \| \|
6979	* +---+-------+ +------+----+
6980	* \| mainA \| \| mainB \|
6981	* +-----------+ +-----------+
6982	*
6983	* We'll start relocating mainA, will find subA, append it and start
6984	* processing sub A recursively:
6985	*
6986	* +-----------+------+
6987	* \| mainA \| subA \|
6988	* +-----------+------+
6989	*
6990	* At this point we notice that subB is used from subA, so we append it and
6991	* relocate (there are no further subcalls from subB):
6992	*
6993	* +-----------+------+------+
6994	* \| mainA \| subA \| subB \|
6995	* +-----------+------+------+
6996	*
6997	* At this point, we relocate subA calls, then go one level up and finish with
6998	* relocation mainA calls. mainA is done.
6999	*
7000	* For mainB process is similar but results in different order. We start with
7001	* mainB and skip subA and subB, as mainB never calls them (at least
7002	* directly), but we see subC is needed, so we append and start processing it:
7003	*
7004	* +-----------+------+
7005	* \| mainB \| subC \|
7006	* +-----------+------+
7007	* Now we see subC needs subB, so we go back to it, append and relocate it:
7008	*
7009	* +-----------+------+------+
7010	* \| mainB \| subC \| subB \|
7011	* +-----------+------+------+
7012	*
7013	* At this point we unwind recursion, relocate calls in subC, then in mainB.
7014	*/
7015	static int
7016	bpf_object__relocate_calls(struct bpf_object obj, struct bpf_program prog)
7017	{
7018	struct bpf_program *subprog;
7019	int i, err;
7020
7021	/* mark all subprogs as not relocated (yet) within the context of
7022	* current main program
7023	*/
7024	for (i = 0; i < obj->nr_programs; i++) {
7025	subprog = &obj->programs[i];
7026	if (!prog_is_subprog(obj, subprog))
7027	continue;
7028
7029	subprog->sub_insn_off = 0;
7030	}
7031
7032	err = bpf_object__reloc_code(obj, prog, prog);
7033	if (err)
7034	return err;
7035
7036	return 0;
7037	}
7038
7039	static void
7040	bpf_object__free_relocs(struct bpf_object *obj)
7041	{
7042	struct bpf_program *prog;
7043	int i;
7044
7045	/* free up relocation descriptors */
7046	for (i = 0; i < obj->nr_programs; i++) {
7047	prog = &obj->programs[i];
7048	zfree(&prog->reloc_desc);
7049	prog->nr_reloc = 0;
7050	}
7051	}
7052
7053	static int cmp_relocs(const void _a, const void _b)
7054	{
7055	const struct reloc_desc *a = _a;
7056	const struct reloc_desc *b = _b;
7057
7058	if (a->insn_idx != b->insn_idx)
7059	return a->insn_idx < b->insn_idx ? -1 : 1;
7060
7061	/* no two relocations should have the same insn_idx, but ... */
7062	if (a->type != b->type)
7063	return a->type < b->type ? -1 : 1;
7064
7065	return 0;
7066	}
7067
7068	static void bpf_object__sort_relos(struct bpf_object *obj)
7069	{
7070	int i;
7071
7072	for (i = 0; i < obj->nr_programs; i++) {
7073	struct bpf_program *p = &obj->programs[i];
7074
7075	if (!p->nr_reloc)
7076	continue;
7077
7078	qsort(p->reloc_desc, p->nr_reloc, sizeof(*p->reloc_desc), cmp_relocs);
7079	}
7080	}
7081
7082	static int bpf_prog_assign_exc_cb(struct bpf_object obj, struct bpf_program prog)
7083	{
7084	const char *str = "exception_callback:";
7085	size_t pfx_len = strlen(str);
7086	int i, j, n;
7087
7088	if (!obj->btf \|\| !kernel_supports(obj, FEAT_BTF_DECL_TAG))
7089	return 0;
7090
7091	n = btf__type_cnt(obj->btf);
7092	for (i = 1; i < n; i++) {
7093	const char *name;
7094	struct btf_type *t;
7095
7096	t = btf_type_by_id(obj->btf, i);
7097	if (!btf_is_decl_tag(t) \|\| btf_decl_tag(t)->component_idx != -1)
7098	continue;
7099
7100	name = btf__str_by_offset(obj->btf, t->name_off);
7101	if (strncmp(name, str, pfx_len) != 0)
7102	continue;
7103
7104	t = btf_type_by_id(obj->btf, t->type);
7105	if (!btf_is_func(t) \|\| btf_func_linkage(t) != BTF_FUNC_GLOBAL) {
7106	pr_warn("prog '%s': exception_callback:<value> decl tag not applied to the main program\n",
7107	prog->name);
7108	return -EINVAL;
7109	}
7110	if (strcmp(prog->name, btf__str_by_offset(obj->btf, t->name_off)) != 0)
7111	continue;
7112	/* Multiple callbacks are specified for the same prog,
7113	* the verifier will eventually return an error for this
7114	* case, hence simply skip appending a subprog.
7115	*/
7116	if (prog->exception_cb_idx >= 0) {
7117	prog->exception_cb_idx = -1;
7118	break;
7119	}
7120
7121	name += pfx_len;
7122	if (str_is_empty(name)) {
7123	pr_warn("prog '%s': exception_callback:<value> decl tag contains empty value\n",
7124	prog->name);
7125	return -EINVAL;
7126	}
7127
7128	for (j = 0; j < obj->nr_programs; j++) {
7129	struct bpf_program *subprog = &obj->programs[j];
7130
7131	if (!prog_is_subprog(obj, subprog))
7132	continue;
7133	if (strcmp(name, subprog->name) != 0)
7134	continue;
7135	/* Enforce non-hidden, as from verifier point of
7136	* view it expects global functions, whereas the
7137	* mark_btf_static fixes up linkage as static.
7138	*/
7139	if (!subprog->sym_global \|\| subprog->mark_btf_static) {
7140	pr_warn("prog '%s': exception callback %s must be a global non-hidden function\n",
7141	prog->name, subprog->name);
7142	return -EINVAL;
7143	}
7144	/* Let's see if we already saw a static exception callback with the same name */
7145	if (prog->exception_cb_idx >= 0) {
7146	pr_warn("prog '%s': multiple subprogs with same name as exception callback '%s'\n",
7147	prog->name, subprog->name);
7148	return -EINVAL;
7149	}
7150	prog->exception_cb_idx = j;
7151	break;
7152	}
7153
7154	if (prog->exception_cb_idx >= 0)
7155	continue;
7156
7157	pr_warn("prog '%s': cannot find exception callback '%s'\n", prog->name, name);
7158	return -ENOENT;
7159	}
7160
7161	return 0;
7162	}
7163
7164	static struct {
7165	enum bpf_prog_type prog_type;
7166	const char *ctx_name;
7167	} global_ctx_map[] = {
7168	{ BPF_PROG_TYPE_CGROUP_DEVICE, "bpf_cgroup_dev_ctx" },
7169	{ BPF_PROG_TYPE_CGROUP_SKB, "__sk_buff" },
7170	{ BPF_PROG_TYPE_CGROUP_SOCK, "bpf_sock" },
7171	{ BPF_PROG_TYPE_CGROUP_SOCK_ADDR, "bpf_sock_addr" },
7172	{ BPF_PROG_TYPE_CGROUP_SOCKOPT, "bpf_sockopt" },
7173	{ BPF_PROG_TYPE_CGROUP_SYSCTL, "bpf_sysctl" },
7174	{ BPF_PROG_TYPE_FLOW_DISSECTOR, "__sk_buff" },
7175	{ BPF_PROG_TYPE_KPROBE, "bpf_user_pt_regs_t" },
7176	{ BPF_PROG_TYPE_LWT_IN, "__sk_buff" },
7177	{ BPF_PROG_TYPE_LWT_OUT, "__sk_buff" },
7178	{ BPF_PROG_TYPE_LWT_SEG6LOCAL, "__sk_buff" },
7179	{ BPF_PROG_TYPE_LWT_XMIT, "__sk_buff" },
7180	{ BPF_PROG_TYPE_NETFILTER, "bpf_nf_ctx" },
7181	{ BPF_PROG_TYPE_PERF_EVENT, "bpf_perf_event_data" },
7182	{ BPF_PROG_TYPE_RAW_TRACEPOINT, "bpf_raw_tracepoint_args" },
7183	{ BPF_PROG_TYPE_RAW_TRACEPOINT_WRITABLE, "bpf_raw_tracepoint_args" },
7184	{ BPF_PROG_TYPE_SCHED_ACT, "__sk_buff" },
7185	{ BPF_PROG_TYPE_SCHED_CLS, "__sk_buff" },
7186	{ BPF_PROG_TYPE_SK_LOOKUP, "bpf_sk_lookup" },
7187	{ BPF_PROG_TYPE_SK_MSG, "sk_msg_md" },
7188	{ BPF_PROG_TYPE_SK_REUSEPORT, "sk_reuseport_md" },
7189	{ BPF_PROG_TYPE_SK_SKB, "__sk_buff" },
7190	{ BPF_PROG_TYPE_SOCK_OPS, "bpf_sock_ops" },
7191	{ BPF_PROG_TYPE_SOCKET_FILTER, "__sk_buff" },
7192	{ BPF_PROG_TYPE_XDP, "xdp_md" },
7193	/* all other program types don't have "named" context structs */
7194	};
7195
7196	/* forward declarations for arch-specific underlying types of bpf_user_pt_regs_t typedef,
7197	* for below __builtin_types_compatible_p() checks;
7198	* with this approach we don't need any extra arch-specific #ifdef guards
7199	*/
7200	struct pt_regs;
7201	struct user_pt_regs;
7202	struct user_regs_struct;
7203
7204	static bool need_func_arg_type_fixup(const struct btf btf, const struct bpf_program prog,
7205	const char *subprog_name, int arg_idx,
7206	int arg_type_id, const char *ctx_name)
7207	{
7208	const struct btf_type *t;
7209	const char *tname;
7210
7211	/* check if existing parameter already matches verifier expectations */
7212	t = skip_mods_and_typedefs(btf, arg_type_id, NULL);
7213	if (!btf_is_ptr(t))
7214	goto out_warn;
7215
7216	/* typedef bpf_user_pt_regs_t is a special PITA case, valid for kprobe
7217	* and perf_event programs, so check this case early on and forget
7218	* about it for subsequent checks
7219	*/
7220	while (btf_is_mod(t))
7221	t = btf__type_by_id(btf, t->type);
7222	if (btf_is_typedef(t) &&
7223	(prog->type == BPF_PROG_TYPE_KPROBE \|\| prog->type == BPF_PROG_TYPE_PERF_EVENT)) {
7224	tname = btf__str_by_offset(btf, t->name_off) ?: "<anon>";
7225	if (strcmp(tname, "bpf_user_pt_regs_t") == 0)
7226	return false; /* canonical type for kprobe/perf_event */
7227	}
7228
7229	/* now we can ignore typedefs moving forward */
7230	t = skip_mods_and_typedefs(btf, t->type, NULL);
7231
7232	/* if it's `void `, definitely fix up BTF info /
7233	if (btf_is_void(t))
7234	return true;
7235
7236	/* if it's already proper canonical type, no need to fix up */
7237	tname = btf__str_by_offset(btf, t->name_off) ?: "<anon>";
7238	if (btf_is_struct(t) && strcmp(tname, ctx_name) == 0)
7239	return false;
7240
7241	/* special cases */
7242	switch (prog->type) {
7243	case BPF_PROG_TYPE_KPROBE:
7244	/* `struct pt_regs ` is expected, but we need to fix up /
7245	if (btf_is_struct(t) && strcmp(tname, "pt_regs") == 0)
7246	return true;
7247	break;
7248	case BPF_PROG_TYPE_PERF_EVENT:
7249	if (__builtin_types_compatible_p(bpf_user_pt_regs_t, struct pt_regs) &&
7250	btf_is_struct(t) && strcmp(tname, "pt_regs") == 0)
7251	return true;
7252	if (__builtin_types_compatible_p(bpf_user_pt_regs_t, struct user_pt_regs) &&
7253	btf_is_struct(t) && strcmp(tname, "user_pt_regs") == 0)
7254	return true;
7255	if (__builtin_types_compatible_p(bpf_user_pt_regs_t, struct user_regs_struct) &&
7256	btf_is_struct(t) && strcmp(tname, "user_regs_struct") == 0)
7257	return true;
7258	break;
7259	case BPF_PROG_TYPE_RAW_TRACEPOINT:
7260	case BPF_PROG_TYPE_RAW_TRACEPOINT_WRITABLE:
7261	/* allow u64* as ctx */
7262	if (btf_is_int(t) && t->size == 8)
7263	return true;
7264	break;
7265	default:
7266	break;
7267	}
7268
7269	out_warn:
7270	pr_warn("prog '%s': subprog '%s' arg#%d is expected to be of `struct %s *` type\n",
7271	prog->name, subprog_name, arg_idx, ctx_name);
7272	return false;
7273	}
7274
7275	static int clone_func_btf_info(struct btf btf, int orig_fn_id, struct bpf_program prog)
7276	{
7277	int fn_id, fn_proto_id, ret_type_id, orig_proto_id;
7278	int i, err, arg_cnt, fn_name_off, linkage;
7279	struct btf_type fn_t, fn_proto_t, *t;
7280	struct btf_param *p;
7281
7282	/* caller already validated FUNC -> FUNC_PROTO validity */
7283	fn_t = btf_type_by_id(btf, orig_fn_id);
7284	fn_proto_t = btf_type_by_id(btf, fn_t->type);
7285
7286	/* Note that each btf__add_xxx() operation invalidates
7287	* all btf_type and string pointers, so we need to be
7288	* very careful when cloning BTF types. BTF type
7289	* pointers have to be always refetched. And to avoid
7290	* problems with invalidated string pointers, we
7291	* add empty strings initially, then just fix up
7292	* name_off offsets in place. Offsets are stable for
7293	* existing strings, so that works out.
7294	*/
7295	fn_name_off = fn_t->name_off; /* we are about to invalidate fn_t */
7296	linkage = btf_func_linkage(fn_t);
7297	orig_proto_id = fn_t->type; /* original FUNC_PROTO ID */
7298	ret_type_id = fn_proto_t->type; /* fn_proto_t will be invalidated */
7299	arg_cnt = btf_vlen(fn_proto_t);
7300
7301	/* clone FUNC_PROTO and its params */
7302	fn_proto_id = btf__add_func_proto(btf, ret_type_id);
7303	if (fn_proto_id < 0)
7304	return -EINVAL;
7305
7306	for (i = 0; i < arg_cnt; i++) {
7307	int name_off;
7308
7309	/* copy original parameter data */
7310	t = btf_type_by_id(btf, orig_proto_id);
7311	p = &btf_params(t)[i];
7312	name_off = p->name_off;
7313
7314	err = btf__add_func_param(btf, "", p->type);
7315	if (err)
7316	return err;
7317
7318	fn_proto_t = btf_type_by_id(btf, fn_proto_id);
7319	p = &btf_params(fn_proto_t)[i];
7320	p->name_off = name_off; /* use remembered str offset */
7321	}
7322
7323	/* clone FUNC now, btf__add_func() enforces non-empty name, so use
7324	* entry program's name as a placeholder, which we replace immediately
7325	* with original name_off
7326	*/
7327	fn_id = btf__add_func(btf, prog->name, linkage, fn_proto_id);
7328	if (fn_id < 0)
7329	return -EINVAL;
7330
7331	fn_t = btf_type_by_id(btf, fn_id);
7332	fn_t->name_off = fn_name_off; /* reuse original string */
7333
7334	return fn_id;
7335	}
7336
7337	/* Check if main program or global subprog's function prototype has `arg:ctx`
7338	* argument tags, and, if necessary, substitute correct type to match what BPF
7339	* verifier would expect, taking into account specific program type. This
7340	* allows to support __arg_ctx tag transparently on old kernels that don't yet
7341	* have a native support for it in the verifier, making user's life much
7342	* easier.
7343	*/
7344	static int bpf_program_fixup_func_info(struct bpf_object obj, struct bpf_program prog)
7345	{
7346	const char ctx_name = NULL, ctx_tag = "arg:ctx", *fn_name;
7347	struct bpf_func_info_min *func_rec;
7348	struct btf_type fn_t, fn_proto_t;
7349	struct btf *btf = obj->btf;
7350	const struct btf_type *t;
7351	struct btf_param *p;
7352	int ptr_id = 0, struct_id, tag_id, orig_fn_id;
7353	int i, n, arg_idx, arg_cnt, err, rec_idx;
7354	int *orig_ids;
7355
7356	/* no .BTF.ext, no problem */
7357	if (!obj->btf_ext \|\| !prog->func_info)
7358	return 0;
7359
7360	/* don't do any fix ups if kernel natively supports __arg_ctx */
7361	if (kernel_supports(obj, FEAT_ARG_CTX_TAG))
7362	return 0;
7363
7364	/* some BPF program types just don't have named context structs, so
7365	* this fallback mechanism doesn't work for them
7366	*/
7367	for (i = 0; i < ARRAY_SIZE(global_ctx_map); i++) {
7368	if (global_ctx_map[i].prog_type != prog->type)
7369	continue;
7370	ctx_name = global_ctx_map[i].ctx_name;
7371	break;
7372	}
7373	if (!ctx_name)
7374	return 0;
7375
7376	/* remember original func BTF IDs to detect if we already cloned them */
7377	orig_ids = calloc(prog->func_info_cnt, sizeof(*orig_ids));
7378	if (!orig_ids)
7379	return -ENOMEM;
7380	for (i = 0; i < prog->func_info_cnt; i++) {
7381	func_rec = prog->func_info + prog->func_info_rec_size * i;
7382	orig_ids[i] = func_rec->type_id;
7383	}
7384
7385	/* go through each DECL_TAG with "arg:ctx" and see if it points to one
7386	* of our subprogs; if yes and subprog is global and needs adjustment,
7387	* clone and adjust FUNC -> FUNC_PROTO combo
7388	*/
7389	for (i = 1, n = btf__type_cnt(btf); i < n; i++) {
7390	/* only DECL_TAG with "arg:ctx" value are interesting */
7391	t = btf__type_by_id(btf, i);
7392	if (!btf_is_decl_tag(t))
7393	continue;
7394	if (strcmp(btf__str_by_offset(btf, t->name_off), ctx_tag) != 0)
7395	continue;
7396
7397	/* only global funcs need adjustment, if at all */
7398	orig_fn_id = t->type;
7399	fn_t = btf_type_by_id(btf, orig_fn_id);
7400	if (!btf_is_func(fn_t) \|\| btf_func_linkage(fn_t) != BTF_FUNC_GLOBAL)
7401	continue;
7402
7403	/* sanity check FUNC -> FUNC_PROTO chain, just in case */
7404	fn_proto_t = btf_type_by_id(btf, fn_t->type);
7405	if (!fn_proto_t \|\| !btf_is_func_proto(fn_proto_t))
7406	continue;
7407
7408	/* find corresponding func_info record */
7409	func_rec = NULL;
7410	for (rec_idx = 0; rec_idx < prog->func_info_cnt; rec_idx++) {
7411	if (orig_ids[rec_idx] == t->type) {
7412	func_rec = prog->func_info + prog->func_info_rec_size * rec_idx;
7413	break;
7414	}
7415	}
7416	/* current main program doesn't call into this subprog */
7417	if (!func_rec)
7418	continue;
7419
7420	/* some more sanity checking of DECL_TAG */
7421	arg_cnt = btf_vlen(fn_proto_t);
7422	arg_idx = btf_decl_tag(t)->component_idx;
7423	if (arg_idx < 0 \|\| arg_idx >= arg_cnt)
7424	continue;
7425
7426	/* check if we should fix up argument type */
7427	p = &btf_params(fn_proto_t)[arg_idx];
7428	fn_name = btf__str_by_offset(btf, fn_t->name_off) ?: "<anon>";
7429	if (!need_func_arg_type_fixup(btf, prog, fn_name, arg_idx, p->type, ctx_name))
7430	continue;
7431
7432	/* clone fn/fn_proto, unless we already did it for another arg */
7433	if (func_rec->type_id == orig_fn_id) {
7434	int fn_id;
7435
7436	fn_id = clone_func_btf_info(btf, orig_fn_id, prog);
7437	if (fn_id < 0) {
7438	err = fn_id;
7439	goto err_out;
7440	}
7441
7442	/* point func_info record to a cloned FUNC type */
7443	func_rec->type_id = fn_id;
7444	}
7445
7446	/* create PTR -> STRUCT type chain to mark PTR_TO_CTX argument;
7447	* we do it just once per main BPF program, as all global
7448	* funcs share the same program type, so need only PTR ->
7449	* STRUCT type chain
7450	*/
7451	if (ptr_id == 0) {
7452	struct_id = btf__add_struct(btf, ctx_name, 0);
7453	ptr_id = btf__add_ptr(btf, struct_id);
7454	if (ptr_id < 0 \|\| struct_id < 0) {
7455	err = -EINVAL;
7456	goto err_out;
7457	}
7458	}
7459
7460	/* for completeness, clone DECL_TAG and point it to cloned param */
7461	tag_id = btf__add_decl_tag(btf, ctx_tag, func_rec->type_id, arg_idx);
7462	if (tag_id < 0) {
7463	err = -EINVAL;
7464	goto err_out;
7465	}
7466
7467	/* all the BTF manipulations invalidated pointers, refetch them */
7468	fn_t = btf_type_by_id(btf, func_rec->type_id);
7469	fn_proto_t = btf_type_by_id(btf, fn_t->type);
7470
7471	/* fix up type ID pointed to by param */
7472	p = &btf_params(fn_proto_t)[arg_idx];
7473	p->type = ptr_id;
7474	}
7475
7476	free(orig_ids);
7477	return 0;
7478	err_out:
7479	free(orig_ids);
7480	return err;
7481	}
7482
7483	static int bpf_object__relocate(struct bpf_object obj, const char targ_btf_path)
7484	{
7485	struct bpf_program *prog;
7486	size_t i, j;
7487	int err;
7488
7489	if (obj->btf_ext) {
7490	err = bpf_object__relocate_core(obj, targ_btf_path);
7491	if (err) {
7492	pr_warn("failed to perform CO-RE relocations: %s\n",
7493	errstr(err));
7494	return err;
7495	}
7496	bpf_object__sort_relos(obj);
7497	}
7498
7499	/* place globals at the end of the arena (if supported) */
7500	if (obj->arena_map_idx >= 0 && kernel_supports(obj, FEAT_LDIMM64_FULL_RANGE_OFF)) {
7501	struct bpf_map *arena_map = &obj->maps[obj->arena_map_idx];
7502
7503	obj->arena_data_off = bpf_map_mmap_sz(arena_map) -
7504	roundup(obj->arena_data_sz, sysconf(_SC_PAGE_SIZE));
7505	}
7506
7507	/* Before relocating calls pre-process relocations and mark
7508	* few ld_imm64 instructions that points to subprogs.
7509	* Otherwise bpf_object__reloc_code() later would have to consider
7510	* all ld_imm64 insns as relocation candidates. That would
7511	* reduce relocation speed, since amount of find_prog_insn_relo()
7512	* would increase and most of them will fail to find a relo.
7513	*/
7514	for (i = 0; i < obj->nr_programs; i++) {
7515	prog = &obj->programs[i];
7516	for (j = 0; j < prog->nr_reloc; j++) {
7517	struct reloc_desc *relo = &prog->reloc_desc[j];
7518	struct bpf_insn *insn = &prog->insns[relo->insn_idx];
7519
7520	/* mark the insn, so it's recognized by insn_is_pseudo_func() */
7521	if (relo->type == RELO_SUBPROG_ADDR)
7522	insn[0].src_reg = BPF_PSEUDO_FUNC;
7523	}
7524	}
7525
7526	/* relocate subprogram calls and append used subprograms to main
7527	* programs; each copy of subprogram code needs to be relocated
7528	* differently for each main program, because its code location might
7529	* have changed.
7530	* Append subprog relos to main programs to allow data relos to be
7531	* processed after text is completely relocated.
7532	*/
7533	for (i = 0; i < obj->nr_programs; i++) {
7534	prog = &obj->programs[i];
7535	/* sub-program's sub-calls are relocated within the context of
7536	* its main program only
7537	*/
7538	if (prog_is_subprog(obj, prog))
7539	continue;
7540	if (!prog->autoload)
7541	continue;
7542
7543	err = bpf_object__relocate_calls(obj, prog);
7544	if (err) {
7545	pr_warn("prog '%s': failed to relocate calls: %s\n",
7546	prog->name, errstr(err));
7547	return err;
7548	}
7549
7550	err = bpf_prog_assign_exc_cb(obj, prog);
7551	if (err)
7552	return err;
7553	/* Now, also append exception callback if it has not been done already. */
7554	if (prog->exception_cb_idx >= 0) {
7555	struct bpf_program *subprog = &obj->programs[prog->exception_cb_idx];
7556
7557	/* Calling exception callback directly is disallowed, which the
7558	* verifier will reject later. In case it was processed already,
7559	* we can skip this step, otherwise for all other valid cases we
7560	* have to append exception callback now.
7561	*/
7562	if (subprog->sub_insn_off == 0) {
7563	err = bpf_object__append_subprog_code(obj, prog, subprog);
7564	if (err)
7565	return err;
7566	err = bpf_object__reloc_code(obj, prog, subprog);
7567	if (err)
7568	return err;
7569	}
7570	}
7571	}
7572	for (i = 0; i < obj->nr_programs; i++) {
7573	prog = &obj->programs[i];
7574	if (prog_is_subprog(obj, prog))
7575	continue;
7576	if (!prog->autoload)
7577	continue;
7578
7579	/* Process data relos for main programs */
7580	err = bpf_object__relocate_data(obj, prog);
7581	if (err) {
7582	pr_warn("prog '%s': failed to relocate data references: %s\n",
7583	prog->name, errstr(err));
7584	return err;
7585	}
7586
7587	/* Fix up .BTF.ext information, if necessary */
7588	err = bpf_program_fixup_func_info(obj, prog);
7589	if (err) {
7590	pr_warn("prog '%s': failed to perform .BTF.ext fix ups: %s\n",
7591	prog->name, errstr(err));
7592	return err;
7593	}
7594	}
7595
7596	return 0;
7597	}
7598
7599	static int bpf_object__collect_st_ops_relos(struct bpf_object *obj,
7600	Elf64_Shdr shdr, Elf_Data data);
7601
7602	static int bpf_object__collect_map_relos(struct bpf_object *obj,
7603	Elf64_Shdr shdr, Elf_Data data)
7604	{
7605	const int bpf_ptr_sz = 8, host_ptr_sz = sizeof(void *);
7606	int i, j, nrels, new_sz;
7607	const struct btf_var_secinfo *vi = NULL;
7608	const struct btf_type sec, var, *def;
7609	struct bpf_map map = NULL, targ_map = NULL;
7610	struct bpf_program *targ_prog = NULL;
7611	bool is_prog_array, is_map_in_map;
7612	const struct btf_member *member;
7613	const char name, mname, *type;
7614	unsigned int moff;
7615	Elf64_Sym *sym;
7616	Elf64_Rel *rel;
7617	void *tmp;
7618
7619	if (!obj->efile.btf_maps_sec_btf_id \|\| !obj->btf)
7620	return -EINVAL;
7621	sec = btf__type_by_id(obj->btf, obj->efile.btf_maps_sec_btf_id);
7622	if (!sec)
7623	return -EINVAL;
7624
7625	nrels = shdr->sh_size / shdr->sh_entsize;
7626	for (i = 0; i < nrels; i++) {
7627	rel = elf_rel_by_idx(data, i);
7628	if (!rel) {
7629	pr_warn(".maps relo #%d: failed to get ELF relo\n", i);
7630	return -LIBBPF_ERRNO__FORMAT;
7631	}
7632
7633	sym = elf_sym_by_idx(obj, ELF64_R_SYM(rel->r_info));
7634	if (!sym) {
7635	pr_warn(".maps relo #%d: symbol %zx not found\n",
7636	i, (size_t)ELF64_R_SYM(rel->r_info));
7637	return -LIBBPF_ERRNO__FORMAT;
7638	}
7639	name = elf_sym_str(obj, sym->st_name) ?: "<?>";
7640
7641	pr_debug(".maps relo #%d: for %zd value %zu rel->r_offset %zu name %u ('%s')\n",
7642	i, (ssize_t)(rel->r_info >> 32), (size_t)sym->st_value,
7643	(size_t)rel->r_offset, sym->st_name, name);
7644
7645	for (j = 0; j < obj->nr_maps; j++) {
7646	map = &obj->maps[j];
7647	if (map->sec_idx != obj->efile.btf_maps_shndx)
7648	continue;
7649
7650	vi = btf_var_secinfos(sec) + map->btf_var_idx;
7651	if (vi->offset <= rel->r_offset &&
7652	rel->r_offset + bpf_ptr_sz <= vi->offset + vi->size)
7653	break;
7654	}
7655	if (j == obj->nr_maps) {
7656	pr_warn(".maps relo #%d: cannot find map '%s' at rel->r_offset %zu\n",
7657	i, name, (size_t)rel->r_offset);
7658	return -EINVAL;
7659	}
7660
7661	is_map_in_map = bpf_map_type__is_map_in_map(map->def.type);
7662	is_prog_array = map->def.type == BPF_MAP_TYPE_PROG_ARRAY;
7663	type = is_map_in_map ? "map" : "prog";
7664	if (is_map_in_map) {
7665	if (sym->st_shndx != obj->efile.btf_maps_shndx) {
7666	pr_warn(".maps relo #%d: '%s' isn't a BTF-defined map\n",
7667	i, name);
7668	return -LIBBPF_ERRNO__RELOC;
7669	}
7670	if (map->def.type == BPF_MAP_TYPE_HASH_OF_MAPS &&
7671	map->def.key_size != sizeof(int)) {
7672	pr_warn(".maps relo #%d: hash-of-maps '%s' should have key size %zu.\n",
7673	i, map->name, sizeof(int));
7674	return -EINVAL;
7675	}
7676	targ_map = bpf_object__find_map_by_name(obj, name);
7677	if (!targ_map) {
7678	pr_warn(".maps relo #%d: '%s' isn't a valid map reference\n",
7679	i, name);
7680	return -ESRCH;
7681	}
7682	} else if (is_prog_array) {
7683	targ_prog = bpf_object__find_program_by_name(obj, name);
7684	if (!targ_prog) {
7685	pr_warn(".maps relo #%d: '%s' isn't a valid program reference\n",
7686	i, name);
7687	return -ESRCH;
7688	}
7689	if (targ_prog->sec_idx != sym->st_shndx \|\|
7690	targ_prog->sec_insn_off * 8 != sym->st_value \|\|
7691	prog_is_subprog(obj, targ_prog)) {
7692	pr_warn(".maps relo #%d: '%s' isn't an entry-point program\n",
7693	i, name);
7694	return -LIBBPF_ERRNO__RELOC;
7695	}
7696	} else {
7697	return -EINVAL;
7698	}
7699
7700	var = btf__type_by_id(obj->btf, vi->type);
7701	def = skip_mods_and_typedefs(obj->btf, var->type, NULL);
7702	if (btf_vlen(def) == 0)
7703	return -EINVAL;
7704	member = btf_members(def) + btf_vlen(def) - 1;
7705	mname = btf__name_by_offset(obj->btf, member->name_off);
7706	if (strcmp(mname, "values"))
7707	return -EINVAL;
7708
7709	moff = btf_member_bit_offset(def, btf_vlen(def) - 1) / 8;
7710	if (rel->r_offset - vi->offset < moff)
7711	return -EINVAL;
7712
7713	moff = rel->r_offset - vi->offset - moff;
7714	/* here we use BPF pointer size, which is always 64 bit, as we
7715	* are parsing ELF that was built for BPF target
7716	*/
7717	if (moff % bpf_ptr_sz)
7718	return -EINVAL;
7719	moff /= bpf_ptr_sz;
7720	if (moff >= map->init_slots_sz) {
7721	new_sz = moff + 1;
7722	tmp = libbpf_reallocarray(map->init_slots, new_sz, host_ptr_sz);
7723	if (!tmp)
7724	return -ENOMEM;
7725	map->init_slots = tmp;
7726	memset(map->init_slots + map->init_slots_sz, 0,
7727	(new_sz - map->init_slots_sz) * host_ptr_sz);
7728	map->init_slots_sz = new_sz;
7729	}
7730	map->init_slots[moff] = is_map_in_map ? (void )targ_map : (void )targ_prog;
7731
7732	pr_debug(".maps relo #%d: map '%s' slot [%u] points to %s '%s'\n",
7733	i, map->name, moff, type, name);
7734	}
7735
7736	return 0;
7737	}
7738
7739	static int bpf_object__collect_relos(struct bpf_object *obj)
7740	{
7741	int i, err;
7742
7743	for (i = 0; i < obj->efile.sec_cnt; i++) {
7744	struct elf_sec_desc *sec_desc = &obj->efile.secs[i];
7745	Elf64_Shdr *shdr;
7746	Elf_Data *data;
7747	int idx;
7748
7749	if (sec_desc->sec_type != SEC_RELO)
7750	continue;
7751
7752	shdr = sec_desc->shdr;
7753	data = sec_desc->data;
7754	idx = shdr->sh_info;
7755
7756	if (shdr->sh_type != SHT_REL \|\| idx < 0 \|\| idx >= obj->efile.sec_cnt) {
7757	pr_warn("internal error at %d\n", __LINE__);
7758	return -LIBBPF_ERRNO__INTERNAL;
7759	}
7760
7761	if (obj->efile.secs[idx].sec_type == SEC_ST_OPS)
7762	err = bpf_object__collect_st_ops_relos(obj, shdr, data);
7763	else if (idx == obj->efile.btf_maps_shndx)
7764	err = bpf_object__collect_map_relos(obj, shdr, data);
7765	else
7766	err = bpf_object__collect_prog_relos(obj, shdr, data);
7767	if (err)
7768	return err;
7769	}
7770
7771	bpf_object__sort_relos(obj);
7772	return 0;
7773	}
7774
7775	static bool insn_is_helper_call(struct bpf_insn insn, enum bpf_func_id func_id)
7776	{
7777	if (BPF_CLASS(insn->code) == BPF_JMP &&
7778	BPF_OP(insn->code) == BPF_CALL &&
7779	BPF_SRC(insn->code) == BPF_K &&
7780	insn->src_reg == 0 &&
7781	insn->dst_reg == 0) {
7782	*func_id = insn->imm;
7783	return true;
7784	}
7785	return false;
7786	}
7787
7788	static int bpf_object__sanitize_prog(struct bpf_object obj, struct bpf_program prog)
7789	{
7790	struct bpf_insn *insn = prog->insns;
7791	enum bpf_func_id func_id;
7792	int i;
7793
7794	if (obj->gen_loader)
7795	return 0;
7796
7797	for (i = 0; i < prog->insns_cnt; i++, insn++) {
7798	if (!insn_is_helper_call(insn, &func_id))
7799	continue;
7800
7801	/* on kernels that don't yet support
7802	* bpf_probe_read_{kernel,user}[_str] helpers, fall back
7803	* to bpf_probe_read() which works well for old kernels
7804	*/
7805	switch (func_id) {
7806	case BPF_FUNC_probe_read_kernel:
7807	case BPF_FUNC_probe_read_user:
7808	if (!kernel_supports(obj, FEAT_PROBE_READ_KERN))
7809	insn->imm = BPF_FUNC_probe_read;
7810	break;
7811	case BPF_FUNC_probe_read_kernel_str:
7812	case BPF_FUNC_probe_read_user_str:
7813	if (!kernel_supports(obj, FEAT_PROBE_READ_KERN))
7814	insn->imm = BPF_FUNC_probe_read_str;
7815	break;
7816	default:
7817	break;
7818	}
7819	}
7820	return 0;
7821	}
7822
7823	static int libbpf_find_attach_btf_id(struct bpf_program prog, const char attach_name,
7824	int btf_obj_fd, int btf_type_id);
7825
7826	static inline bool is_tracing_multi(enum bpf_attach_type type)
7827	{
7828	return type == BPF_TRACE_FENTRY_MULTI \|\| type == BPF_TRACE_FEXIT_MULTI \|\|
7829	type == BPF_TRACE_FSESSION_MULTI;
7830	}
7831
7832	static const struct module_btf find_attach_module(struct bpf_object obj, const char *attach)
7833	{
7834	const char sep, mod_name = NULL;
7835	int i, mod_len, err;
7836
7837	/*
7838	* We expect attach string in the form of either
7839	* - function_pattern or
7840	* - <module>:function_pattern
7841	*/
7842	sep = strchr(attach, ':');
7843	if (sep) {
7844	mod_name = attach;
7845	mod_len = sep - mod_name;
7846	}
7847	if (!mod_name)
7848	return NULL;
7849
7850	err = load_module_btfs(obj);
7851	if (err)
7852	return NULL;
7853
7854	for (i = 0; i < obj->btf_module_cnt; i++) {
7855	const struct module_btf *mod = &obj->btf_modules[i];
7856
7857	if (strncmp(mod->name, mod_name, mod_len) == 0 && mod->name[mod_len] == '\0')
7858	return mod;
7859	}
7860	return NULL;
7861	}
7862
7863	static int tracing_multi_mod_fd(struct bpf_program prog, int btf_obj_fd)
7864	{
7865	const char attach_name, sep;
7866	const struct module_btf *mod;
7867
7868	*btf_obj_fd = 0;
7869	attach_name = strchr(prog->sec_name, '/');
7870
7871	/* Program with no details in spec, using kernel btf. */
7872	if (!attach_name)
7873	return 0;
7874
7875	/* Program with no module section, using kernel btf. */
7876	sep = strchr(++attach_name, ':');
7877	if (!sep)
7878	return 0;
7879
7880	/* Program with module specified, get its btf fd. */
7881	mod = find_attach_module(prog->obj, attach_name);
7882	if (!mod)
7883	return -EINVAL;
7884
7885	*btf_obj_fd = mod->fd;
7886	return 0;
7887	}
7888
7889	/* this is called as prog->sec_def->prog_prepare_load_fn for libbpf-supported sec_defs */
7890	static int libbpf_prepare_prog_load(struct bpf_program *prog,
7891	struct bpf_prog_load_opts *opts, long cookie)
7892	{
7893	enum sec_def_flags def = cookie;
7894
7895	/* old kernels might not support specifying expected_attach_type */
7896	if ((def & SEC_EXP_ATTACH_OPT) && !kernel_supports(prog->obj, FEAT_EXP_ATTACH_TYPE))
7897	opts->expected_attach_type = 0;
7898
7899	if (def & SEC_SLEEPABLE)
7900	opts->prog_flags \|= BPF_F_SLEEPABLE;
7901
7902	if (prog->type == BPF_PROG_TYPE_XDP && (def & SEC_XDP_FRAGS))
7903	opts->prog_flags \|= BPF_F_XDP_HAS_FRAGS;
7904
7905	/* special check for usdt to use uprobe_multi link */
7906	if ((def & SEC_USDT) && kernel_supports(prog->obj, FEAT_UPROBE_MULTI_LINK)) {
7907	/* for BPF_TRACE_UPROBE_MULTI, user might want to query expected_attach_type
7908	* in prog, and expected_attach_type we set in kernel is from opts, so we
7909	* update both.
7910	*/
7911	prog->expected_attach_type = BPF_TRACE_UPROBE_MULTI;
7912	opts->expected_attach_type = BPF_TRACE_UPROBE_MULTI;
7913	}
7914
7915	if ((def & SEC_ATTACH_BTF) && !prog->attach_btf_id) {
7916	int btf_obj_fd = 0, btf_type_id = 0, err;
7917	const char *attach_name;
7918
7919	attach_name = strchr(prog->sec_name, '/');
7920	if (!attach_name) {
7921	/* if BPF program is annotated with just SEC("fentry")
7922	* (or similar) without declaratively specifying
7923	* target, then it is expected that target will be
7924	* specified with bpf_program__set_attach_target() at
7925	* runtime before BPF object load step. If not, then
7926	* there is nothing to load into the kernel as BPF
7927	* verifier won't be able to validate BPF program
7928	* correctness anyways.
7929	*/
7930	pr_warn("prog '%s': no BTF-based attach target is specified, use bpf_program__set_attach_target()\n",
7931	prog->name);
7932	return -EINVAL;
7933	}
7934	attach_name++; /* skip over / */
7935
7936	err = libbpf_find_attach_btf_id(prog, attach_name, &btf_obj_fd, &btf_type_id);
7937	if (err)
7938	return err;
7939
7940	/* cache resolved BTF FD and BTF type ID in the prog */
7941	prog->attach_btf_obj_fd = btf_obj_fd;
7942	prog->attach_btf_id = btf_type_id;
7943
7944	/* but by now libbpf common logic is not utilizing
7945	* prog->attach_btf_obj_fd/prog->attach_btf_id anymore because
7946	* this callback is called after opts were populated by
7947	* libbpf, so this callback has to update opts explicitly here
7948	*/
7949	opts->attach_btf_obj_fd = btf_obj_fd;
7950	opts->attach_btf_id = btf_type_id;
7951	}
7952
7953	if (is_tracing_multi(prog->expected_attach_type)) {
7954	int err, btf_obj_fd = 0;
7955
7956	err = tracing_multi_mod_fd(prog, &btf_obj_fd);
7957	if (err < 0)
7958	return err;
7959
7960	prog->attach_btf_obj_fd = btf_obj_fd;
7961	opts->attach_btf_obj_fd = btf_obj_fd;
7962	}
7963
7964	return 0;
7965	}
7966
7967	static void fixup_verifier_log(struct bpf_program prog, char buf, size_t buf_sz);
7968
7969	static int bpf_object_load_prog(struct bpf_object obj, struct bpf_program prog,
7970	struct bpf_insn *insns, int insns_cnt,
7971	const char license, __u32 kern_version, int prog_fd)
7972	{
7973	LIBBPF_OPTS(bpf_prog_load_opts, load_attr);
7974	const char *prog_name = NULL;
7975	size_t log_buf_size = 0;
7976	char log_buf = NULL, tmp;
7977	bool own_log_buf = true;
7978	__u32 log_level = prog->log_level;
7979	int ret, err;
7980
7981	/* Be more helpful by rejecting programs that can't be validated early
7982	* with more meaningful and actionable error message.
7983	*/
7984	switch (prog->type) {
7985	case BPF_PROG_TYPE_UNSPEC:
7986	/*
7987	* The program type must be set. Most likely we couldn't find a proper
7988	* section definition at load time, and thus we didn't infer the type.
7989	*/
7990	pr_warn("prog '%s': missing BPF prog type, check ELF section name '%s'\n",
7991	prog->name, prog->sec_name);
7992	return -EINVAL;
7993	case BPF_PROG_TYPE_STRUCT_OPS:
7994	if (prog->attach_btf_id == 0) {
7995	pr_warn("prog '%s': SEC(\"struct_ops\") program isn't referenced anywhere, did you forget to use it?\n",
7996	prog->name);
7997	return -EINVAL;
7998	}
7999	break;
8000	default:
8001	break;
8002	}
8003
8004	if (!insns \|\| !insns_cnt)
8005	return -EINVAL;
8006
8007	if (kernel_supports(obj, FEAT_PROG_NAME))
8008	prog_name = prog->name;
8009	load_attr.attach_prog_fd = prog->attach_prog_fd;
8010	load_attr.attach_btf_obj_fd = prog->attach_btf_obj_fd;
8011	load_attr.attach_btf_id = prog->attach_btf_id;
8012	load_attr.kern_version = kern_version;
8013	load_attr.prog_ifindex = prog->prog_ifindex;
8014	load_attr.expected_attach_type = prog->expected_attach_type;
8015
8016	/* specify func_info/line_info only if kernel supports them */
8017	if (obj->btf && btf__fd(obj->btf) >= 0 && kernel_supports(obj, FEAT_BTF_FUNC)) {
8018	load_attr.prog_btf_fd = btf__fd(obj->btf);
8019	load_attr.func_info = prog->func_info;
8020	load_attr.func_info_rec_size = prog->func_info_rec_size;
8021	load_attr.func_info_cnt = prog->func_info_cnt;
8022	load_attr.line_info = prog->line_info;
8023	load_attr.line_info_rec_size = prog->line_info_rec_size;
8024	load_attr.line_info_cnt = prog->line_info_cnt;
8025	}
8026	load_attr.log_level = log_level;
8027	load_attr.prog_flags = prog->prog_flags;
8028	load_attr.fd_array = obj->fd_array;
8029
8030	load_attr.token_fd = obj->token_fd;
8031	if (obj->token_fd)
8032	load_attr.prog_flags \|= BPF_F_TOKEN_FD;
8033
8034	/* adjust load_attr if sec_def provides custom preload callback */
8035	if (prog->sec_def && prog->sec_def->prog_prepare_load_fn) {
8036	err = prog->sec_def->prog_prepare_load_fn(prog, &load_attr, prog->sec_def->cookie);
8037	if (err < 0) {
8038	pr_warn("prog '%s': failed to prepare load attributes: %s\n",
8039	prog->name, errstr(err));
8040	return err;
8041	}
8042	insns = prog->insns;
8043	insns_cnt = prog->insns_cnt;
8044	}
8045
8046	if (obj->gen_loader) {
8047	bpf_gen__prog_load(obj->gen_loader, prog->type, prog->name,
8048	license, insns, insns_cnt, &load_attr,
8049	prog - obj->programs);
8050	*prog_fd = -1;
8051	return 0;
8052	}
8053
8054	retry_load:
8055	/* if log_level is zero, we don't request logs initially even if
8056	* custom log_buf is specified; if the program load fails, then we'll
8057	* bump log_level to 1 and use either custom log_buf or we'll allocate
8058	* our own and retry the load to get details on what failed
8059	*/
8060	if (log_level) {
8061	if (prog->log_buf) {
8062	log_buf = prog->log_buf;
8063	log_buf_size = prog->log_size;
8064	own_log_buf = false;
8065	} else if (obj->log_buf) {
8066	log_buf = obj->log_buf;
8067	log_buf_size = obj->log_size;
8068	own_log_buf = false;
8069	} else {
8070	log_buf_size = max((size_t)BPF_LOG_BUF_SIZE, log_buf_size * 2);
8071	tmp = realloc(log_buf, log_buf_size);
8072	if (!tmp) {
8073	ret = -ENOMEM;
8074	goto out;
8075	}
8076	log_buf = tmp;
8077	log_buf[0] = '\0';
8078	own_log_buf = true;
8079	}
8080	}
8081
8082	load_attr.log_buf = log_buf;
8083	load_attr.log_size = log_buf_size;
8084	load_attr.log_level = log_level;
8085
8086	ret = bpf_prog_load(prog->type, prog_name, license, insns, insns_cnt, &load_attr);
8087	if (ret >= 0) {
8088	if (log_level && own_log_buf) {
8089	pr_debug("prog '%s': -- BEGIN PROG LOAD LOG --\n%s-- END PROG LOAD LOG --\n",
8090	prog->name, log_buf);
8091	}
8092
8093	if (obj->has_rodata && kernel_supports(obj, FEAT_PROG_BIND_MAP)) {
8094	struct bpf_map *map;
8095	int i;
8096
8097	for (i = 0; i < obj->nr_maps; i++) {
8098	map = &prog->obj->maps[i];
8099	if (map->libbpf_type != LIBBPF_MAP_RODATA)
8100	continue;
8101
8102	if (bpf_prog_bind_map(ret, map->fd, NULL)) {
8103	pr_warn("prog '%s': failed to bind map '%s': %s\n",
8104	prog->name, map->real_name, errstr(errno));
8105	/* Don't fail hard if can't bind rodata. */
8106	}
8107	}
8108	}
8109
8110	*prog_fd = ret;
8111	ret = 0;
8112	goto out;
8113	}
8114
8115	if (log_level == 0) {
8116	log_level = 1;
8117	goto retry_load;
8118	}
8119	/* On ENOSPC, increase log buffer size and retry, unless custom
8120	* log_buf is specified.
8121	* Be careful to not overflow u32, though. Kernel's log buf size limit
8122	* isn't part of UAPI so it can always be bumped to full 4GB. So don't
8123	* multiply by 2 unless we are sure we'll fit within 32 bits.
8124	* Currently, we'll get -EINVAL when we reach (UINT_MAX >> 2).
8125	*/
8126	if (own_log_buf && errno == ENOSPC && log_buf_size <= UINT_MAX / 2)
8127	goto retry_load;
8128
8129	ret = -errno;
8130
8131	/* post-process verifier log to improve error descriptions */
8132	fixup_verifier_log(prog, log_buf, log_buf_size);
8133
8134	pr_warn("prog '%s': BPF program load failed: %s\n", prog->name, errstr(errno));
8135	pr_perm_msg(ret);
8136
8137	if (own_log_buf && log_buf && log_buf[0] != '\0') {
8138	pr_warn("prog '%s': -- BEGIN PROG LOAD LOG --\n%s-- END PROG LOAD LOG --\n",
8139	prog->name, log_buf);
8140	}
8141
8142	out:
8143	if (own_log_buf)
8144	free(log_buf);
8145	return ret;
8146	}
8147
8148	static char find_prev_line(char buf, char *cur)
8149	{
8150	char *p;
8151
8152	if (cur == buf) /* end of a log buf */
8153	return NULL;
8154
8155	p = cur - 1;
8156	while (p - 1 >= buf && *(p - 1) != '\n')
8157	p--;
8158
8159	return p;
8160	}
8161
8162	static void patch_log(char *buf, size_t buf_sz, size_t log_sz,
8163	char orig, size_t orig_sz, const char patch)
8164	{
8165	/* size of the remaining log content to the right from the to-be-replaced part */
8166	size_t rem_sz = (buf + log_sz) - (orig + orig_sz);
8167	size_t patch_sz = strlen(patch);
8168
8169	if (patch_sz != orig_sz) {
8170	/* If patch line(s) are longer than original piece of verifier log,
8171	* shift log contents by (patch_sz - orig_sz) bytes to the right
8172	* starting from after to-be-replaced part of the log.
8173	*
8174	* If patch line(s) are shorter than original piece of verifier log,
8175	* shift log contents by (orig_sz - patch_sz) bytes to the left
8176	* starting from after to-be-replaced part of the log
8177	*
8178	* We need to be careful about not overflowing available
8179	* buf_sz capacity. If that's the case, we'll truncate the end
8180	* of the original log, as necessary.
8181	*/
8182	if (patch_sz > orig_sz) {
8183	if (orig + patch_sz >= buf + buf_sz) {
8184	/* patch is big enough to cover remaining space completely */
8185	patch_sz -= (orig + patch_sz) - (buf + buf_sz) + 1;
8186	rem_sz = 0;
8187	} else if (patch_sz - orig_sz > buf_sz - log_sz) {
8188	/* patch causes part of remaining log to be truncated */
8189	rem_sz -= (patch_sz - orig_sz) - (buf_sz - log_sz);
8190	}
8191	}
8192	/* shift remaining log to the right by calculated amount */
8193	memmove(orig + patch_sz, orig + orig_sz, rem_sz);
8194	}
8195
8196	memcpy(orig, patch, patch_sz);
8197	}
8198
8199	static void fixup_log_failed_core_relo(struct bpf_program *prog,
8200	char *buf, size_t buf_sz, size_t log_sz,
8201	char line1, char line2, char *line3)
8202	{
8203	/* Expected log for failed and not properly guarded CO-RE relocation:
8204	* line1 -> 123: (85) call unknown#195896080
8205	* line2 -> invalid func unknown#195896080
8206	* line3 -> <anything else or end of buffer>
8207	*
8208	* "123" is the index of the instruction that was poisoned. We extract
8209	* instruction index to find corresponding CO-RE relocation and
8210	* replace this part of the log with more relevant information about
8211	* failed CO-RE relocation.
8212	*/
8213	const struct bpf_core_relo *relo;
8214	struct bpf_core_spec spec;
8215	char patch[512], spec_buf[256];
8216	int insn_idx, err, spec_len;
8217
8218	if (sscanf(line1, "%d: (%*d) call unknown#195896080\n", &insn_idx) != 1)
8219	return;
8220
8221	relo = find_relo_core(prog, insn_idx);
8222	if (!relo)
8223	return;
8224
8225	err = bpf_core_parse_spec(prog->name, prog->obj->btf, relo, &spec);
8226	if (err)
8227	return;
8228
8229	spec_len = bpf_core_format_spec(spec_buf, sizeof(spec_buf), &spec);
8230	snprintf(patch, sizeof(patch),
8231	"%d: <invalid CO-RE relocation>\n"
8232	"failed to resolve CO-RE relocation %s%s\n",
8233	insn_idx, spec_buf, spec_len >= sizeof(spec_buf) ? "..." : "");
8234
8235	patch_log(buf, buf_sz, log_sz, line1, line3 - line1, patch);
8236	}
8237
8238	static void fixup_log_missing_map_load(struct bpf_program *prog,
8239	char *buf, size_t buf_sz, size_t log_sz,
8240	char line1, char line2, char *line3)
8241	{
8242	/* Expected log for failed and not properly guarded map reference:
8243	* line1 -> 123: (85) call unknown#2001000345
8244	* line2 -> invalid func unknown#2001000345
8245	* line3 -> <anything else or end of buffer>
8246	*
8247	* "123" is the index of the instruction that was poisoned.
8248	* "345" in "2001000345" is a map index in obj->maps to fetch map name.
8249	*/
8250	struct bpf_object *obj = prog->obj;
8251	const struct bpf_map *map;
8252	int insn_idx, map_idx;
8253	char patch[128];
8254
8255	if (sscanf(line1, "%d: (%*d) call unknown#%d\n", &insn_idx, &map_idx) != 2)
8256	return;
8257
8258	map_idx -= POISON_LDIMM64_MAP_BASE;
8259	if (map_idx < 0 \|\| map_idx >= obj->nr_maps)
8260	return;
8261	map = &obj->maps[map_idx];
8262
8263	snprintf(patch, sizeof(patch),
8264	"%d: <invalid BPF map reference>\n"
8265	"BPF map '%s' is referenced but wasn't created\n",
8266	insn_idx, map->name);
8267
8268	patch_log(buf, buf_sz, log_sz, line1, line3 - line1, patch);
8269	}
8270
8271	static void fixup_log_missing_kfunc_call(struct bpf_program *prog,
8272	char *buf, size_t buf_sz, size_t log_sz,
8273	char line1, char line2, char *line3)
8274	{
8275	/* Expected log for failed and not properly guarded kfunc call:
8276	* line1 -> 123: (85) call unknown#2002000345
8277	* line2 -> invalid func unknown#2002000345
8278	* line3 -> <anything else or end of buffer>
8279	*
8280	* "123" is the index of the instruction that was poisoned.
8281	* "345" in "2002000345" is an extern index in obj->externs to fetch kfunc name.
8282	*/
8283	struct bpf_object *obj = prog->obj;
8284	const struct extern_desc *ext;
8285	int insn_idx, ext_idx;
8286	char patch[128];
8287
8288	if (sscanf(line1, "%d: (%*d) call unknown#%d\n", &insn_idx, &ext_idx) != 2)
8289	return;
8290
8291	ext_idx -= POISON_CALL_KFUNC_BASE;
8292	if (ext_idx < 0 \|\| ext_idx >= obj->nr_extern)
8293	return;
8294	ext = &obj->externs[ext_idx];
8295
8296	snprintf(patch, sizeof(patch),
8297	"%d: <invalid kfunc call>\n"
8298	"kfunc '%s' is referenced but wasn't resolved\n",
8299	insn_idx, ext->name);
8300
8301	patch_log(buf, buf_sz, log_sz, line1, line3 - line1, patch);
8302	}
8303
8304	static void fixup_verifier_log(struct bpf_program prog, char buf, size_t buf_sz)
8305	{
8306	/* look for familiar error patterns in last N lines of the log */
8307	const size_t max_last_line_cnt = 10;
8308	char prev_line, cur_line, *next_line;
8309	size_t log_sz;
8310	int i;
8311
8312	if (!buf)
8313	return;
8314
8315	log_sz = strlen(buf) + 1;
8316	next_line = buf + log_sz - 1;
8317
8318	for (i = 0; i < max_last_line_cnt; i++, next_line = cur_line) {
8319	cur_line = find_prev_line(buf, next_line);
8320	if (!cur_line)
8321	return;
8322
8323	if (str_has_pfx(cur_line, "invalid func unknown#195896080\n")) {
8324	prev_line = find_prev_line(buf, cur_line);
8325	if (!prev_line)
8326	continue;
8327
8328	/* failed CO-RE relocation case */
8329	fixup_log_failed_core_relo(prog, buf, buf_sz, log_sz,
8330	prev_line, cur_line, next_line);
8331	return;
8332	} else if (str_has_pfx(cur_line, "invalid func unknown#"POISON_LDIMM64_MAP_PFX)) {
8333	prev_line = find_prev_line(buf, cur_line);
8334	if (!prev_line)
8335	continue;
8336
8337	/* reference to uncreated BPF map */
8338	fixup_log_missing_map_load(prog, buf, buf_sz, log_sz,
8339	prev_line, cur_line, next_line);
8340	return;
8341	} else if (str_has_pfx(cur_line, "invalid func unknown#"POISON_CALL_KFUNC_PFX)) {
8342	prev_line = find_prev_line(buf, cur_line);
8343	if (!prev_line)
8344	continue;
8345
8346	/* reference to unresolved kfunc */
8347	fixup_log_missing_kfunc_call(prog, buf, buf_sz, log_sz,
8348	prev_line, cur_line, next_line);
8349	return;
8350	}
8351	}
8352	}
8353
8354	static int bpf_program_record_relos(struct bpf_program *prog)
8355	{
8356	struct bpf_object *obj = prog->obj;
8357	int i;
8358
8359	for (i = 0; i < prog->nr_reloc; i++) {
8360	struct reloc_desc *relo = &prog->reloc_desc[i];
8361	struct extern_desc *ext = &obj->externs[relo->ext_idx];
8362	int kind;
8363
8364	switch (relo->type) {
8365	case RELO_EXTERN_LD64:
8366	if (ext->type != EXT_KSYM)
8367	continue;
8368	kind = btf_is_var(btf__type_by_id(obj->btf, ext->btf_id)) ?
8369	BTF_KIND_VAR : BTF_KIND_FUNC;
8370	bpf_gen__record_extern(obj->gen_loader, ext->name,
8371	ext->is_weak, !ext->ksym.type_id,
8372	true, kind, relo->insn_idx);
8373	break;
8374	case RELO_EXTERN_CALL:
8375	bpf_gen__record_extern(obj->gen_loader, ext->name,
8376	ext->is_weak, false, false, BTF_KIND_FUNC,
8377	relo->insn_idx);
8378	break;
8379	case RELO_CORE: {
8380	struct bpf_core_relo cr = {
8381	.insn_off = relo->insn_idx * 8,
8382	.type_id = relo->core_relo->type_id,
8383	.access_str_off = relo->core_relo->access_str_off,
8384	.kind = relo->core_relo->kind,
8385	};
8386
8387	bpf_gen__record_relo_core(obj->gen_loader, &cr);
8388	break;
8389	}
8390	default:
8391	continue;
8392	}
8393	}
8394	return 0;
8395	}
8396
8397	static int
8398	bpf_object__load_progs(struct bpf_object *obj, int log_level)
8399	{
8400	struct bpf_program *prog;
8401	size_t i;
8402	int err;
8403
8404	for (i = 0; i < obj->nr_programs; i++) {
8405	prog = &obj->programs[i];
8406	if (prog_is_subprog(obj, prog))
8407	continue;
8408	if (!prog->autoload) {
8409	pr_debug("prog '%s': skipped loading\n", prog->name);
8410	continue;
8411	}
8412	prog->log_level \|= log_level;
8413
8414	if (obj->gen_loader)
8415	bpf_program_record_relos(prog);
8416
8417	err = bpf_object_load_prog(obj, prog, prog->insns, prog->insns_cnt,
8418	obj->license, obj->kern_version, &prog->fd);
8419	if (err) {
8420	pr_warn("prog '%s': failed to load: %s\n", prog->name, errstr(err));
8421	return err;
8422	}
8423	}
8424
8425	bpf_object__free_relocs(obj);
8426	return 0;
8427	}
8428
8429	static int bpf_object_prepare_progs(struct bpf_object *obj)
8430	{
8431	struct bpf_program *prog;
8432	size_t i;
8433	int err;
8434
8435	for (i = 0; i < obj->nr_programs; i++) {
8436	prog = &obj->programs[i];
8437	err = bpf_object__sanitize_prog(obj, prog);
8438	if (err)
8439	return err;
8440	}
8441	return 0;
8442	}
8443
8444	static const struct bpf_sec_def find_sec_def(const char sec_name);
8445
8446	static int bpf_object_init_progs(struct bpf_object obj, const struct bpf_object_open_opts opts)
8447	{
8448	struct bpf_program *prog;
8449	int err;
8450
8451	bpf_object__for_each_program(prog, obj) {
8452	prog->sec_def = find_sec_def(prog->sec_name);
8453	if (!prog->sec_def) {
8454	/* couldn't guess, but user might manually specify */
8455	pr_debug("prog '%s': unrecognized ELF section name '%s'\n",
8456	prog->name, prog->sec_name);
8457	continue;
8458	}
8459
8460	prog->type = prog->sec_def->prog_type;
8461	prog->expected_attach_type = prog->sec_def->expected_attach_type;
8462
8463	/* sec_def can have custom callback which should be called
8464	* after bpf_program is initialized to adjust its properties
8465	*/
8466	if (prog->sec_def->prog_setup_fn) {
8467	err = prog->sec_def->prog_setup_fn(prog, prog->sec_def->cookie);
8468	if (err < 0) {
8469	pr_warn("prog '%s': failed to initialize: %s\n",
8470	prog->name, errstr(err));
8471	return err;
8472	}
8473	}
8474	}
8475
8476	return 0;
8477	}
8478
8479	static struct bpf_object bpf_object_open(const char path, const void *obj_buf, size_t obj_buf_sz,
8480	const char *obj_name,
8481	const struct bpf_object_open_opts *opts)
8482	{
8483	const char kconfig, btf_tmp_path, *token_path;
8484	struct bpf_object *obj;
8485	int err;
8486	char *log_buf;
8487	size_t log_size;
8488	__u32 log_level;
8489
8490	if (obj_buf && !obj_name)
8491	return ERR_PTR(-EINVAL);
8492
8493	if (elf_version(EV_CURRENT) == EV_NONE) {
8494	pr_warn("failed to init libelf for %s\n",
8495	path ? : "(mem buf)");
8496	return ERR_PTR(-LIBBPF_ERRNO__LIBELF);
8497	}
8498
8499	if (!OPTS_VALID(opts, bpf_object_open_opts))
8500	return ERR_PTR(-EINVAL);
8501
8502	obj_name = OPTS_GET(opts, object_name, NULL) ?: obj_name;
8503	if (obj_buf) {
8504	path = obj_name;
8505	pr_debug("loading object '%s' from buffer\n", obj_name);
8506	} else {
8507	pr_debug("loading object from %s\n", path);
8508	}
8509
8510	log_buf = OPTS_GET(opts, kernel_log_buf, NULL);
8511	log_size = OPTS_GET(opts, kernel_log_size, 0);
8512	log_level = OPTS_GET(opts, kernel_log_level, 0);
8513	if (log_size > UINT_MAX)
8514	return ERR_PTR(-EINVAL);
8515	if (log_size && !log_buf)
8516	return ERR_PTR(-EINVAL);
8517
8518	token_path = OPTS_GET(opts, bpf_token_path, NULL);
8519	/* if user didn't specify bpf_token_path explicitly, check if
8520	* LIBBPF_BPF_TOKEN_PATH envvar was set and treat it as bpf_token_path
8521	* option
8522	*/
8523	if (!token_path)
8524	token_path = getenv("LIBBPF_BPF_TOKEN_PATH");
8525	if (token_path && strlen(token_path) >= PATH_MAX)
8526	return ERR_PTR(-ENAMETOOLONG);
8527
8528	obj = bpf_object__new(path, obj_buf, obj_buf_sz, obj_name);
8529	if (IS_ERR(obj))
8530	return obj;
8531
8532	obj->log_buf = log_buf;
8533	obj->log_size = log_size;
8534	obj->log_level = log_level;
8535
8536	if (token_path) {
8537	obj->token_path = strdup(token_path);
8538	if (!obj->token_path) {
8539	err = -ENOMEM;
8540	goto out;
8541	}
8542	}
8543
8544	btf_tmp_path = OPTS_GET(opts, btf_custom_path, NULL);
8545	if (btf_tmp_path) {
8546	if (strlen(btf_tmp_path) >= PATH_MAX) {
8547	err = -ENAMETOOLONG;
8548	goto out;
8549	}
8550	obj->btf_custom_path = strdup(btf_tmp_path);
8551	if (!obj->btf_custom_path) {
8552	err = -ENOMEM;
8553	goto out;
8554	}
8555	}
8556
8557	kconfig = OPTS_GET(opts, kconfig, NULL);
8558	if (kconfig) {
8559	obj->kconfig = strdup(kconfig);
8560	if (!obj->kconfig) {
8561	err = -ENOMEM;
8562	goto out;
8563	}
8564	}
8565
8566	err = bpf_object__elf_init(obj);
8567	err = err ? : bpf_object__elf_collect(obj);
8568	err = err ? : bpf_object__collect_externs(obj);
8569	err = err ? : bpf_object_fixup_btf(obj);
8570	err = err ? : bpf_object__init_maps(obj, opts);
8571	err = err ? : bpf_object_init_progs(obj, opts);
8572	err = err ? : bpf_object__collect_relos(obj);
8573	if (err)
8574	goto out;
8575
8576	bpf_object__elf_finish(obj);
8577
8578	return obj;
8579	out:
8580	bpf_object__close(obj);
8581	return ERR_PTR(err);
8582	}
8583
8584	struct bpf_object *
8585	bpf_object__open_file(const char path, const struct bpf_object_open_opts opts)
8586	{
8587	if (!path)
8588	return libbpf_err_ptr(-EINVAL);
8589
8590	return libbpf_ptr(bpf_object_open(path, NULL, 0, NULL, opts));
8591	}
8592
8593	struct bpf_object bpf_object__open(const char path)
8594	{
8595	return bpf_object__open_file(path, NULL);
8596	}
8597
8598	struct bpf_object *
8599	bpf_object__open_mem(const void *obj_buf, size_t obj_buf_sz,
8600	const struct bpf_object_open_opts *opts)
8601	{
8602	char tmp_name[64];
8603
8604	if (!obj_buf \|\| obj_buf_sz == 0)
8605	return libbpf_err_ptr(-EINVAL);
8606
8607	/* create a (quite useless) default "name" for this memory buffer object */
8608	snprintf(tmp_name, sizeof(tmp_name), "%lx-%zx", (unsigned long)obj_buf, obj_buf_sz);
8609
8610	return libbpf_ptr(bpf_object_open(NULL, obj_buf, obj_buf_sz, tmp_name, opts));
8611	}
8612
8613	static int bpf_object_unload(struct bpf_object *obj)
8614	{
8615	size_t i;
8616
8617	if (!obj)
8618	return libbpf_err(-EINVAL);
8619
8620	for (i = 0; i < obj->nr_maps; i++) {
8621	zclose(obj->maps[i].fd);
8622	if (obj->maps[i].st_ops)
8623	zfree(&obj->maps[i].st_ops->kern_vdata);
8624	}
8625
8626	for (i = 0; i < obj->nr_programs; i++)
8627	bpf_program__unload(&obj->programs[i]);
8628
8629	return 0;
8630	}
8631
8632	static int bpf_object__sanitize_maps(struct bpf_object *obj)
8633	{
8634	struct bpf_map *m;
8635
8636	bpf_object__for_each_map(m, obj) {
8637	if (!bpf_map__is_internal(m))
8638	continue;
8639	if (!kernel_supports(obj, FEAT_ARRAY_MMAP))
8640	m->def.map_flags &= ~BPF_F_MMAPABLE;
8641	}
8642
8643	return 0;
8644	}
8645
8646	typedef int (*kallsyms_cb_t)(unsigned long long sym_addr, char sym_type,
8647	const char sym_name, void ctx);
8648
8649	static int libbpf_kallsyms_parse(kallsyms_cb_t cb, void *ctx)
8650	{
8651	char sym_type, sym_name[500];
8652	unsigned long long sym_addr;
8653	int ret, err = 0;
8654	FILE *f;
8655
8656	f = fopen("/proc/kallsyms", "re");
8657	if (!f) {
8658	err = -errno;
8659	pr_warn("failed to open /proc/kallsyms: %s\n", errstr(err));
8660	return err;
8661	}
8662
8663	while (true) {
8664	ret = fscanf(f, "%llx %c %499s%*[^\n]\n",
8665	&sym_addr, &sym_type, sym_name);
8666	if (ret == EOF && feof(f))
8667	break;
8668	if (ret != 3) {
8669	pr_warn("failed to read kallsyms entry: %d\n", ret);
8670	err = -EINVAL;
8671	break;
8672	}
8673
8674	err = cb(sym_addr, sym_type, sym_name, ctx);
8675	if (err)
8676	break;
8677	}
8678
8679	fclose(f);
8680	return err;
8681	}
8682
8683	static int kallsyms_cb(unsigned long long sym_addr, char sym_type,
8684	const char sym_name, void ctx)
8685	{
8686	struct bpf_object *obj = ctx;
8687	const struct btf_type *t;
8688	struct extern_desc *ext;
8689	const char *res;
8690
8691	res = strstr(sym_name, ".llvm.");
8692	if (sym_type == 'd' && res)
8693	ext = find_extern_by_name_with_len(obj, sym_name, res - sym_name);
8694	else
8695	ext = find_extern_by_name(obj, sym_name);
8696	if (!ext \|\| ext->type != EXT_KSYM)
8697	return 0;
8698
8699	t = btf__type_by_id(obj->btf, ext->btf_id);
8700	if (!btf_is_var(t))
8701	return 0;
8702
8703	if (ext->is_set && ext->ksym.addr != sym_addr) {
8704	pr_warn("extern (ksym) '%s': resolution is ambiguous: 0x%llx or 0x%llx\n",
8705	sym_name, ext->ksym.addr, sym_addr);
8706	return -EINVAL;
8707	}
8708	if (!ext->is_set) {
8709	ext->is_set = true;
8710	ext->ksym.addr = sym_addr;
8711	pr_debug("extern (ksym) '%s': set to 0x%llx\n", sym_name, sym_addr);
8712	}
8713	return 0;
8714	}
8715
8716	static int bpf_object__read_kallsyms_file(struct bpf_object *obj)
8717	{
8718	return libbpf_kallsyms_parse(kallsyms_cb, obj);
8719	}
8720
8721	static int find_ksym_btf_id(struct bpf_object obj, const char ksym_name,
8722	__u16 kind, struct btf **res_btf,
8723	struct module_btf **res_mod_btf)
8724	{
8725	struct module_btf *mod_btf;
8726	struct btf *btf;
8727	int i, id, err;
8728
8729	btf = obj->btf_vmlinux;
8730	mod_btf = NULL;
8731	id = btf__find_by_name_kind(btf, ksym_name, kind);
8732
8733	if (id == -ENOENT) {
8734	err = load_module_btfs(obj);
8735	if (err)
8736	return err;
8737
8738	for (i = 0; i < obj->btf_module_cnt; i++) {
8739	/* we assume module_btf's BTF FD is always >0 */
8740	mod_btf = &obj->btf_modules[i];
8741	btf = mod_btf->btf;
8742	id = btf__find_by_name_kind_own(btf, ksym_name, kind);
8743	if (id != -ENOENT)
8744	break;
8745	}
8746	}
8747	if (id <= 0)
8748	return -ESRCH;
8749
8750	*res_btf = btf;
8751	*res_mod_btf = mod_btf;
8752	return id;
8753	}
8754
8755	static int bpf_object__resolve_ksym_var_btf_id(struct bpf_object *obj,
8756	struct extern_desc *ext)
8757	{
8758	const struct btf_type targ_var, targ_type;
8759	__u32 targ_type_id, local_type_id;
8760	struct module_btf *mod_btf = NULL;
8761	const char *targ_var_name;
8762	struct btf *btf = NULL;
8763	int id, err;
8764
8765	id = find_ksym_btf_id(obj, ext->name, BTF_KIND_VAR, &btf, &mod_btf);
8766	if (id < 0) {
8767	if (id == -ESRCH && ext->is_weak)
8768	return 0;
8769	pr_warn("extern (var ksym) '%s': not found in kernel BTF\n",
8770	ext->name);
8771	return id;
8772	}
8773
8774	/* find local type_id */
8775	local_type_id = ext->ksym.type_id;
8776
8777	/* find target type_id */
8778	targ_var = btf__type_by_id(btf, id);
8779	targ_var_name = btf__name_by_offset(btf, targ_var->name_off);
8780	targ_type = skip_mods_and_typedefs(btf, targ_var->type, &targ_type_id);
8781
8782	err = bpf_core_types_are_compat(obj->btf, local_type_id,
8783	btf, targ_type_id);
8784	if (err <= 0) {
8785	const struct btf_type *local_type;
8786	const char targ_name, local_name;
8787
8788	local_type = btf__type_by_id(obj->btf, local_type_id);
8789	local_name = btf__name_by_offset(obj->btf, local_type->name_off);
8790	targ_name = btf__name_by_offset(btf, targ_type->name_off);
8791
8792	pr_warn("extern (var ksym) '%s': incompatible types, expected [%u] %s %s, but kernel has [%u] %s %s\n",
8793	ext->name, local_type_id,
8794	btf_kind_str(local_type), local_name, targ_type_id,
8795	btf_kind_str(targ_type), targ_name);
8796	return -EINVAL;
8797	}
8798
8799	ext->is_set = true;
8800	ext->ksym.kernel_btf_obj_fd = mod_btf ? mod_btf->fd : 0;
8801	ext->ksym.kernel_btf_id = id;
8802	pr_debug("extern (var ksym) '%s': resolved to [%d] %s %s\n",
8803	ext->name, id, btf_kind_str(targ_var), targ_var_name);
8804
8805	return 0;
8806	}
8807
8808	static int bpf_object__resolve_ksym_func_btf_id(struct bpf_object *obj,
8809	struct extern_desc *ext)
8810	{
8811	int local_func_proto_id, kfunc_proto_id, kfunc_id;
8812	struct module_btf *mod_btf = NULL;
8813	const struct btf_type *kern_func;
8814	struct btf *kern_btf = NULL;
8815	int ret;
8816
8817	local_func_proto_id = ext->ksym.type_id;
8818
8819	kfunc_id = find_ksym_btf_id(obj, ext->essent_name ?: ext->name, BTF_KIND_FUNC, &kern_btf,
8820	&mod_btf);
8821	if (kfunc_id < 0) {
8822	if (kfunc_id == -ESRCH && ext->is_weak)
8823	return 0;
8824	pr_warn("extern (func ksym) '%s': not found in kernel or module BTFs\n",
8825	ext->name);
8826	return kfunc_id;
8827	}
8828
8829	kern_func = btf__type_by_id(kern_btf, kfunc_id);
8830	kfunc_proto_id = kern_func->type;
8831
8832	ret = bpf_core_types_are_compat(obj->btf, local_func_proto_id,
8833	kern_btf, kfunc_proto_id);
8834	if (ret <= 0) {
8835	if (ext->is_weak)
8836	return 0;
8837
8838	pr_warn("extern (func ksym) '%s': func_proto [%d] incompatible with %s [%d]\n",
8839	ext->name, local_func_proto_id,
8840	mod_btf ? mod_btf->name : "vmlinux", kfunc_proto_id);
8841	return -EINVAL;
8842	}
8843
8844	/* set index for module BTF fd in fd_array, if unset */
8845	if (mod_btf && !mod_btf->fd_array_idx) {
8846	/* insn->off is s16 */
8847	if (obj->fd_array_cnt == INT16_MAX) {
8848	pr_warn("extern (func ksym) '%s': module BTF fd index %d too big to fit in bpf_insn offset\n",
8849	ext->name, mod_btf->fd_array_idx);
8850	return -E2BIG;
8851	}
8852	/* Cannot use index 0 for module BTF fd */
8853	if (!obj->fd_array_cnt)
8854	obj->fd_array_cnt = 1;
8855
8856	ret = libbpf_ensure_mem((void **)&obj->fd_array, &obj->fd_array_cap, sizeof(int),
8857	obj->fd_array_cnt + 1);
8858	if (ret)
8859	return ret;
8860	mod_btf->fd_array_idx = obj->fd_array_cnt;
8861	/* we assume module BTF FD is always >0 */
8862	obj->fd_array[obj->fd_array_cnt++] = mod_btf->fd;
8863	}
8864
8865	ext->is_set = true;
8866	ext->ksym.kernel_btf_id = kfunc_id;
8867	ext->ksym.btf_fd_idx = mod_btf ? mod_btf->fd_array_idx : 0;
8868	/* Also set kernel_btf_obj_fd to make sure that bpf_object__relocate_data()
8869	* populates FD into ld_imm64 insn when it's used to point to kfunc.
8870	* {kernel_btf_id, btf_fd_idx} -> fixup bpf_call.
8871	* {kernel_btf_id, kernel_btf_obj_fd} -> fixup ld_imm64.
8872	*/
8873	ext->ksym.kernel_btf_obj_fd = mod_btf ? mod_btf->fd : 0;
8874	pr_debug("extern (func ksym) '%s': resolved to %s [%d]\n",
8875	ext->name, mod_btf ? mod_btf->name : "vmlinux", kfunc_id);
8876
8877	return 0;
8878	}
8879
8880	static int bpf_object__resolve_ksyms_btf_id(struct bpf_object *obj)
8881	{
8882	const struct btf_type *t;
8883	struct extern_desc *ext;
8884	int i, err;
8885
8886	for (i = 0; i < obj->nr_extern; i++) {
8887	ext = &obj->externs[i];
8888	if (ext->type != EXT_KSYM \|\| !ext->ksym.type_id)
8889	continue;
8890
8891	if (obj->gen_loader) {
8892	ext->is_set = true;
8893	ext->ksym.kernel_btf_obj_fd = 0;
8894	ext->ksym.kernel_btf_id = 0;
8895	continue;
8896	}
8897	t = btf__type_by_id(obj->btf, ext->btf_id);
8898	if (btf_is_var(t))
8899	err = bpf_object__resolve_ksym_var_btf_id(obj, ext);
8900	else
8901	err = bpf_object__resolve_ksym_func_btf_id(obj, ext);
8902	if (err)
8903	return err;
8904	}
8905	return 0;
8906	}
8907
8908	static int bpf_object__resolve_externs(struct bpf_object *obj,
8909	const char *extra_kconfig)
8910	{
8911	bool need_config = false, need_kallsyms = false;
8912	bool need_vmlinux_btf = false;
8913	struct extern_desc *ext;
8914	void *kcfg_data = NULL;
8915	int err, i;
8916
8917	if (obj->nr_extern == 0)
8918	return 0;
8919
8920	if (obj->kconfig_map_idx >= 0)
8921	kcfg_data = obj->maps[obj->kconfig_map_idx].mmaped;
8922
8923	for (i = 0; i < obj->nr_extern; i++) {
8924	ext = &obj->externs[i];
8925
8926	if (ext->type == EXT_KSYM) {
8927	if (ext->ksym.type_id)
8928	need_vmlinux_btf = true;
8929	else
8930	need_kallsyms = true;
8931	continue;
8932	} else if (ext->type == EXT_KCFG) {
8933	void *ext_ptr = kcfg_data + ext->kcfg.data_off;
8934	__u64 value = 0;
8935
8936	/* Kconfig externs need actual /proc/config.gz */
8937	if (str_has_pfx(ext->name, "CONFIG_")) {
8938	need_config = true;
8939	continue;
8940	}
8941
8942	/* Virtual kcfg externs are customly handled by libbpf */
8943	if (strcmp(ext->name, "LINUX_KERNEL_VERSION") == 0) {
8944	value = get_kernel_version();
8945	if (!value) {
8946	pr_warn("extern (kcfg) '%s': failed to get kernel version\n", ext->name);
8947	return -EINVAL;
8948	}
8949	} else if (strcmp(ext->name, "LINUX_HAS_BPF_COOKIE") == 0) {
8950	value = kernel_supports(obj, FEAT_BPF_COOKIE);
8951	} else if (strcmp(ext->name, "LINUX_HAS_SYSCALL_WRAPPER") == 0) {
8952	value = kernel_supports(obj, FEAT_SYSCALL_WRAPPER);
8953	} else if (!str_has_pfx(ext->name, "LINUX_") \|\| !ext->is_weak) {
8954	/* Currently libbpf supports only CONFIG_ and LINUX_ prefixed
8955	* __kconfig externs, where LINUX_ ones are virtual and filled out
8956	* customly by libbpf (their values don't come from Kconfig).
8957	* If LINUX_xxx variable is not recognized by libbpf, but is marked
8958	* __weak, it defaults to zero value, just like for CONFIG_xxx
8959	* externs.
8960	*/
8961	pr_warn("extern (kcfg) '%s': unrecognized virtual extern\n", ext->name);
8962	return -EINVAL;
8963	}
8964
8965	err = set_kcfg_value_num(ext, ext_ptr, value);
8966	if (err)
8967	return err;
8968	pr_debug("extern (kcfg) '%s': set to 0x%llx\n",
8969	ext->name, (unsigned long long)value);
8970	} else {
8971	pr_warn("extern '%s': unrecognized extern kind\n", ext->name);
8972	return -EINVAL;
8973	}
8974	}
8975	if (need_config && extra_kconfig) {
8976	err = bpf_object__read_kconfig_mem(obj, extra_kconfig, kcfg_data);
8977	if (err)
8978	return -EINVAL;
8979	need_config = false;
8980	for (i = 0; i < obj->nr_extern; i++) {
8981	ext = &obj->externs[i];
8982	if (ext->type == EXT_KCFG && !ext->is_set) {
8983	need_config = true;
8984	break;
8985	}
8986	}
8987	}
8988	if (need_config) {
8989	err = bpf_object__read_kconfig_file(obj, kcfg_data);
8990	if (err)
8991	return -EINVAL;
8992	}
8993	if (need_kallsyms) {
8994	err = bpf_object__read_kallsyms_file(obj);
8995	if (err)
8996	return -EINVAL;
8997	}
8998	if (need_vmlinux_btf) {
8999	err = bpf_object__resolve_ksyms_btf_id(obj);
9000	if (err)
9001	return -EINVAL;
9002	}
9003	for (i = 0; i < obj->nr_extern; i++) {
9004	ext = &obj->externs[i];
9005
9006	if (!ext->is_set && !ext->is_weak) {
9007	pr_warn("extern '%s' (strong): not resolved\n", ext->name);
9008	return -ESRCH;
9009	} else if (!ext->is_set) {
9010	pr_debug("extern '%s' (weak): not resolved, defaulting to zero\n",
9011	ext->name);
9012	}
9013	}
9014
9015	return 0;
9016	}
9017
9018	static void bpf_map_prepare_vdata(const struct bpf_map *map)
9019	{
9020	const struct btf_type *type;
9021	struct bpf_struct_ops *st_ops;
9022	__u32 i;
9023
9024	st_ops = map->st_ops;
9025	type = btf__type_by_id(map->obj->btf, st_ops->type_id);
9026	for (i = 0; i < btf_vlen(type); i++) {
9027	struct bpf_program *prog = st_ops->progs[i];
9028	void *kern_data;
9029	int prog_fd;
9030
9031	if (!prog)
9032	continue;
9033
9034	prog_fd = bpf_program__fd(prog);
9035	kern_data = st_ops->kern_vdata + st_ops->kern_func_off[i];
9036	(unsigned long )kern_data = prog_fd;
9037	}
9038	}
9039
9040	static int bpf_object_prepare_struct_ops(struct bpf_object *obj)
9041	{
9042	struct bpf_map *map;
9043	int i;
9044
9045	for (i = 0; i < obj->nr_maps; i++) {
9046	map = &obj->maps[i];
9047
9048	if (!bpf_map__is_struct_ops(map))
9049	continue;
9050
9051	if (!map->autocreate)
9052	continue;
9053
9054	bpf_map_prepare_vdata(map);
9055	}
9056
9057	return 0;
9058	}
9059
9060	static void bpf_object_unpin(struct bpf_object *obj)
9061	{
9062	int i;
9063
9064	/* unpin any maps that were auto-pinned during load */
9065	for (i = 0; i < obj->nr_maps; i++)
9066	if (obj->maps[i].pinned && !obj->maps[i].reused)
9067	bpf_map__unpin(&obj->maps[i], NULL);
9068	}
9069
9070	static void bpf_object_cleanup_btf(struct bpf_object *obj)
9071	{
9072	int i;
9073
9074	/* clean up module BTFs */
9075	for (i = 0; i < obj->btf_module_cnt; i++) {
9076	close(obj->btf_modules[i].fd);
9077	btf__free(obj->btf_modules[i].btf);
9078	free(obj->btf_modules[i].name);
9079	}
9080	obj->btf_module_cnt = 0;
9081	obj->btf_module_cap = 0;
9082	obj->btf_modules_loaded = false;
9083	zfree(&obj->btf_modules);
9084
9085	/* clean up vmlinux BTF */
9086	btf__free(obj->btf_vmlinux);
9087	obj->btf_vmlinux = NULL;
9088	}
9089
9090	static void bpf_object_post_load_cleanup(struct bpf_object *obj)
9091	{
9092	/* clean up fd_array */
9093	zfree(&obj->fd_array);
9094
9095	/* clean up BTF */
9096	bpf_object_cleanup_btf(obj);
9097	}
9098
9099	static int bpf_object_prepare(struct bpf_object obj, const char target_btf_path)
9100	{
9101	int err;
9102
9103	if (obj->state >= OBJ_PREPARED) {
9104	pr_warn("object '%s': prepare loading can't be attempted twice\n", obj->name);
9105	return -EINVAL;
9106	}
9107
9108	err = bpf_object_prepare_token(obj);
9109	err = err ? : bpf_object__probe_loading(obj);
9110	err = err ? : bpf_object__load_vmlinux_btf(obj, false);
9111	err = err ? : bpf_object__resolve_externs(obj, obj->kconfig);
9112	err = err ? : bpf_object__sanitize_maps(obj);
9113	err = err ? : bpf_object__init_kern_struct_ops_maps(obj);
9114	err = err ? : bpf_object_adjust_struct_ops_autoload(obj);
9115	err = err ? : bpf_object__relocate(obj, obj->btf_custom_path ? : target_btf_path);
9116	err = err ? : bpf_object__sanitize_and_load_btf(obj);
9117	err = err ? : bpf_object__create_maps(obj);
9118	err = err ? : bpf_object_prepare_progs(obj);
9119
9120	if (err) {
9121	bpf_object_unpin(obj);
9122	bpf_object_unload(obj);
9123	obj->state = OBJ_LOADED;
9124	return err;
9125	}
9126
9127	obj->state = OBJ_PREPARED;
9128	return 0;
9129	}
9130
9131	static int bpf_object_load(struct bpf_object obj, int extra_log_level, const char target_btf_path)
9132	{
9133	int err;
9134
9135	if (!obj)
9136	return libbpf_err(-EINVAL);
9137
9138	if (obj->state >= OBJ_LOADED) {
9139	pr_warn("object '%s': load can't be attempted twice\n", obj->name);
9140	return libbpf_err(-EINVAL);
9141	}
9142
9143	/* Disallow kernel loading programs of non-native endianness but
9144	* permit cross-endian creation of "light skeleton".
9145	*/
9146	if (obj->gen_loader) {
9147	bpf_gen__init(obj->gen_loader, extra_log_level, obj->nr_programs, obj->nr_maps);
9148	} else if (!is_native_endianness(obj)) {
9149	pr_warn("object '%s': loading non-native endianness is unsupported\n", obj->name);
9150	return libbpf_err(-LIBBPF_ERRNO__ENDIAN);
9151	}
9152
9153	if (obj->state < OBJ_PREPARED) {
9154	err = bpf_object_prepare(obj, target_btf_path);
9155	if (err)
9156	return libbpf_err(err);
9157	}
9158	err = bpf_object__load_progs(obj, extra_log_level);
9159	err = err ? : bpf_object_init_prog_arrays(obj);
9160	err = err ? : bpf_object_prepare_struct_ops(obj);
9161
9162	if (obj->gen_loader) {
9163	/* reset FDs */
9164	if (obj->btf)
9165	btf__set_fd(obj->btf, -1);
9166	if (!err)
9167	err = bpf_gen__finish(obj->gen_loader, obj->nr_programs, obj->nr_maps);
9168	}
9169
9170	bpf_object_post_load_cleanup(obj);
9171	obj->state = OBJ_LOADED; /* doesn't matter if successfully or not */
9172
9173	if (err) {
9174	bpf_object_unpin(obj);
9175	bpf_object_unload(obj);
9176	pr_warn("failed to load object '%s'\n", obj->path);
9177	return libbpf_err(err);
9178	}
9179
9180	return 0;
9181	}
9182
9183	int bpf_object__prepare(struct bpf_object *obj)
9184	{
9185	return libbpf_err(bpf_object_prepare(obj, NULL));
9186	}
9187
9188	int bpf_object__load(struct bpf_object *obj)
9189	{
9190	return bpf_object_load(obj, 0, NULL);
9191	}
9192
9193	static int make_parent_dir(const char *path)
9194	{
9195	char dname, dir;
9196	int err = 0;
9197
9198	dname = strdup(path);
9199	if (dname == NULL)
9200	return -ENOMEM;
9201
9202	dir = dirname(dname);
9203	if (mkdir(dir, 0700) && errno != EEXIST)
9204	err = -errno;
9205
9206	free(dname);
9207	if (err) {
9208	pr_warn("failed to mkdir %s: %s\n", path, errstr(err));
9209	}
9210	return err;
9211	}
9212
9213	static int check_path(const char *path)
9214	{
9215	struct statfs st_fs;
9216	char dname, dir;
9217	int err = 0;
9218
9219	if (path == NULL)
9220	return -EINVAL;
9221
9222	dname = strdup(path);
9223	if (dname == NULL)
9224	return -ENOMEM;
9225
9226	dir = dirname(dname);
9227	if (statfs(dir, &st_fs)) {
9228	pr_warn("failed to statfs %s: %s\n", dir, errstr(errno));
9229	err = -errno;
9230	}
9231	free(dname);
9232
9233	if (!err && st_fs.f_type != BPF_FS_MAGIC) {
9234	pr_warn("specified path %s is not on BPF FS\n", path);
9235	err = -EINVAL;
9236	}
9237
9238	return err;
9239	}
9240
9241	int bpf_program__pin(struct bpf_program prog, const char path)
9242	{
9243	int err;
9244
9245	if (prog->fd < 0) {
9246	pr_warn("prog '%s': can't pin program that wasn't loaded\n", prog->name);
9247	return libbpf_err(-EINVAL);
9248	}
9249
9250	err = make_parent_dir(path);
9251	if (err)
9252	return libbpf_err(err);
9253
9254	err = check_path(path);
9255	if (err)
9256	return libbpf_err(err);
9257
9258	if (bpf_obj_pin(prog->fd, path)) {
9259	err = -errno;
9260	pr_warn("prog '%s': failed to pin at '%s': %s\n", prog->name, path, errstr(err));
9261	return libbpf_err(err);
9262	}
9263
9264	pr_debug("prog '%s': pinned at '%s'\n", prog->name, path);
9265	return 0;
9266	}
9267
9268	int bpf_program__unpin(struct bpf_program prog, const char path)
9269	{
9270	int err;
9271
9272	if (prog->fd < 0) {
9273	pr_warn("prog '%s': can't unpin program that wasn't loaded\n", prog->name);
9274	return libbpf_err(-EINVAL);
9275	}
9276
9277	err = check_path(path);
9278	if (err)
9279	return libbpf_err(err);
9280
9281	err = unlink(path);
9282	if (err)
9283	return libbpf_err(-errno);
9284
9285	pr_debug("prog '%s': unpinned from '%s'\n", prog->name, path);
9286	return 0;
9287	}
9288
9289	int bpf_map__pin(struct bpf_map map, const char path)
9290	{
9291	int err;
9292
9293	if (map == NULL) {
9294	pr_warn("invalid map pointer\n");
9295	return libbpf_err(-EINVAL);
9296	}
9297
9298	if (map->fd < 0) {
9299	pr_warn("map '%s': can't pin BPF map without FD (was it created?)\n", map->name);
9300	return libbpf_err(-EINVAL);
9301	}
9302
9303	if (map->pin_path) {
9304	if (path && strcmp(path, map->pin_path)) {
9305	pr_warn("map '%s' already has pin path '%s' different from '%s'\n",
9306	bpf_map__name(map), map->pin_path, path);
9307	return libbpf_err(-EINVAL);
9308	} else if (map->pinned) {
9309	pr_debug("map '%s' already pinned at '%s'; not re-pinning\n",
9310	bpf_map__name(map), map->pin_path);
9311	return 0;
9312	}
9313	} else {
9314	if (!path) {
9315	pr_warn("missing a path to pin map '%s' at\n",
9316	bpf_map__name(map));
9317	return libbpf_err(-EINVAL);
9318	} else if (map->pinned) {
9319	pr_warn("map '%s' already pinned\n", bpf_map__name(map));
9320	return libbpf_err(-EEXIST);
9321	}
9322
9323	map->pin_path = strdup(path);
9324	if (!map->pin_path) {
9325	err = -errno;
9326	goto out_err;
9327	}
9328	}
9329
9330	err = make_parent_dir(map->pin_path);
9331	if (err)
9332	return libbpf_err(err);
9333
9334	err = check_path(map->pin_path);
9335	if (err)
9336	return libbpf_err(err);
9337
9338	if (bpf_obj_pin(map->fd, map->pin_path)) {
9339	err = -errno;
9340	goto out_err;
9341	}
9342
9343	map->pinned = true;
9344	pr_debug("pinned map '%s'\n", map->pin_path);
9345
9346	return 0;
9347
9348	out_err:
9349	pr_warn("failed to pin map: %s\n", errstr(err));
9350	return libbpf_err(err);
9351	}
9352
9353	int bpf_map__unpin(struct bpf_map map, const char path)
9354	{
9355	int err;
9356
9357	if (map == NULL) {
9358	pr_warn("invalid map pointer\n");
9359	return libbpf_err(-EINVAL);
9360	}
9361
9362	if (map->pin_path) {
9363	if (path && strcmp(path, map->pin_path)) {
9364	pr_warn("map '%s' already has pin path '%s' different from '%s'\n",
9365	bpf_map__name(map), map->pin_path, path);
9366	return libbpf_err(-EINVAL);
9367	}
9368	path = map->pin_path;
9369	} else if (!path) {
9370	pr_warn("no path to unpin map '%s' from\n",
9371	bpf_map__name(map));
9372	return libbpf_err(-EINVAL);
9373	}
9374
9375	err = check_path(path);
9376	if (err)
9377	return libbpf_err(err);
9378
9379	err = unlink(path);
9380	if (err != 0)
9381	return libbpf_err(-errno);
9382
9383	map->pinned = false;
9384	pr_debug("unpinned map '%s' from '%s'\n", bpf_map__name(map), path);
9385
9386	return 0;
9387	}
9388
9389	int bpf_map__set_pin_path(struct bpf_map map, const char path)
9390	{
9391	char *new = NULL;
9392
9393	if (path) {
9394	new = strdup(path);
9395	if (!new)
9396	return libbpf_err(-errno);
9397	}
9398
9399	free(map->pin_path);
9400	map->pin_path = new;
9401	return 0;
9402	}
9403
9404	__alias(bpf_map__pin_path)
9405	const char bpf_map__get_pin_path(const struct bpf_map map);
9406
9407	const char bpf_map__pin_path(const struct bpf_map map)
9408	{
9409	return map->pin_path;
9410	}
9411
9412	bool bpf_map__is_pinned(const struct bpf_map *map)
9413	{
9414	return map->pinned;
9415	}
9416
9417	static void sanitize_pin_path(char *s)
9418	{
9419	/* bpffs disallows periods in path names */
9420	while (*s) {
9421	if (*s == '.')
9422	*s = '_';
9423	s++;
9424	}
9425	}
9426
9427	int bpf_object__pin_maps(struct bpf_object obj, const char path)
9428	{
9429	struct bpf_map *map;
9430	int err;
9431
9432	if (!obj)
9433	return libbpf_err(-ENOENT);
9434
9435	if (obj->state < OBJ_PREPARED) {
9436	pr_warn("object not yet loaded; load it first\n");
9437	return libbpf_err(-ENOENT);
9438	}
9439
9440	bpf_object__for_each_map(map, obj) {
9441	char *pin_path = NULL;
9442	char buf[PATH_MAX];
9443
9444	if (!map->autocreate)
9445	continue;
9446
9447	if (path) {
9448	err = pathname_concat(buf, sizeof(buf), path, bpf_map__name(map));
9449	if (err)
9450	goto err_unpin_maps;
9451	sanitize_pin_path(buf);
9452	pin_path = buf;
9453	} else if (!map->pin_path) {
9454	continue;
9455	}
9456
9457	err = bpf_map__pin(map, pin_path);
9458	if (err)
9459	goto err_unpin_maps;
9460	}
9461
9462	return 0;
9463
9464	err_unpin_maps:
9465	while ((map = bpf_object__prev_map(obj, map))) {
9466	if (!map->pin_path)
9467	continue;
9468
9469	bpf_map__unpin(map, NULL);
9470	}
9471
9472	return libbpf_err(err);
9473	}
9474
9475	int bpf_object__unpin_maps(struct bpf_object obj, const char path)
9476	{
9477	struct bpf_map *map;
9478	int err;
9479
9480	if (!obj)
9481	return libbpf_err(-ENOENT);
9482
9483	bpf_object__for_each_map(map, obj) {
9484	char *pin_path = NULL;
9485	char buf[PATH_MAX];
9486
9487	if (path) {
9488	err = pathname_concat(buf, sizeof(buf), path, bpf_map__name(map));
9489	if (err)
9490	return libbpf_err(err);
9491	sanitize_pin_path(buf);
9492	pin_path = buf;
9493	} else if (!map->pin_path) {
9494	continue;
9495	}
9496
9497	err = bpf_map__unpin(map, pin_path);
9498	if (err)
9499	return libbpf_err(err);
9500	}
9501
9502	return 0;
9503	}
9504
9505	int bpf_object__pin_programs(struct bpf_object obj, const char path)
9506	{
9507	struct bpf_program *prog;
9508	char buf[PATH_MAX];
9509	int err;
9510
9511	if (!obj)
9512	return libbpf_err(-ENOENT);
9513
9514	if (obj->state < OBJ_LOADED) {
9515	pr_warn("object not yet loaded; load it first\n");
9516	return libbpf_err(-ENOENT);
9517	}
9518
9519	bpf_object__for_each_program(prog, obj) {
9520	err = pathname_concat(buf, sizeof(buf), path, prog->name);
9521	if (err)
9522	goto err_unpin_programs;
9523
9524	err = bpf_program__pin(prog, buf);
9525	if (err)
9526	goto err_unpin_programs;
9527	}
9528
9529	return 0;
9530
9531	err_unpin_programs:
9532	while ((prog = bpf_object__prev_program(obj, prog))) {
9533	if (pathname_concat(buf, sizeof(buf), path, prog->name))
9534	continue;
9535
9536	bpf_program__unpin(prog, buf);
9537	}
9538
9539	return libbpf_err(err);
9540	}
9541
9542	int bpf_object__unpin_programs(struct bpf_object obj, const char path)
9543	{
9544	struct bpf_program *prog;
9545	int err;
9546
9547	if (!obj)
9548	return libbpf_err(-ENOENT);
9549
9550	bpf_object__for_each_program(prog, obj) {
9551	char buf[PATH_MAX];
9552
9553	err = pathname_concat(buf, sizeof(buf), path, prog->name);
9554	if (err)
9555	return libbpf_err(err);
9556
9557	err = bpf_program__unpin(prog, buf);
9558	if (err)
9559	return libbpf_err(err);
9560	}
9561
9562	return 0;
9563	}
9564
9565	int bpf_object__pin(struct bpf_object obj, const char path)
9566	{
9567	int err;
9568
9569	err = bpf_object__pin_maps(obj, path);
9570	if (err)
9571	return libbpf_err(err);
9572
9573	err = bpf_object__pin_programs(obj, path);
9574	if (err) {
9575	bpf_object__unpin_maps(obj, path);
9576	return libbpf_err(err);
9577	}
9578
9579	return 0;
9580	}
9581
9582	int bpf_object__unpin(struct bpf_object obj, const char path)
9583	{
9584	int err;
9585
9586	err = bpf_object__unpin_programs(obj, path);
9587	if (err)
9588	return libbpf_err(err);
9589
9590	err = bpf_object__unpin_maps(obj, path);
9591	if (err)
9592	return libbpf_err(err);
9593
9594	return 0;
9595	}
9596
9597	static void bpf_map__destroy(struct bpf_map *map)
9598	{
9599	if (map->inner_map) {
9600	bpf_map__destroy(map->inner_map);
9601	zfree(&map->inner_map);
9602	}
9603
9604	zfree(&map->init_slots);
9605	map->init_slots_sz = 0;
9606
9607	if (map->mmaped && map->mmaped != map->obj->arena_data)
9608	munmap(map->mmaped, bpf_map_mmap_sz(map));
9609	map->mmaped = NULL;
9610
9611	if (map->st_ops) {
9612	zfree(&map->st_ops->data);
9613	zfree(&map->st_ops->progs);
9614	zfree(&map->st_ops->kern_func_off);
9615	zfree(&map->st_ops);
9616	}
9617
9618	zfree(&map->name);
9619	zfree(&map->real_name);
9620	zfree(&map->pin_path);
9621
9622	if (map->fd >= 0)
9623	zclose(map->fd);
9624	}
9625
9626	void bpf_object__close(struct bpf_object *obj)
9627	{
9628	size_t i;
9629
9630	if (IS_ERR_OR_NULL(obj))
9631	return;
9632
9633	/*
9634	* if user called bpf_object__prepare() without ever getting to
9635	* bpf_object__load(), we need to clean up stuff that is normally
9636	* cleaned up at the end of loading step
9637	*/
9638	bpf_object_post_load_cleanup(obj);
9639
9640	usdt_manager_free(obj->usdt_man);
9641	obj->usdt_man = NULL;
9642
9643	bpf_gen__free(obj->gen_loader);
9644	bpf_object__elf_finish(obj);
9645	bpf_object_unload(obj);
9646	btf__free(obj->btf);
9647	btf__free(obj->btf_vmlinux);
9648	btf_ext__free(obj->btf_ext);
9649
9650	for (i = 0; i < obj->nr_maps; i++)
9651	bpf_map__destroy(&obj->maps[i]);
9652
9653	zfree(&obj->btf_custom_path);
9654	zfree(&obj->kconfig);
9655
9656	for (i = 0; i < obj->nr_extern; i++) {
9657	zfree(&obj->externs[i].name);
9658	zfree(&obj->externs[i].essent_name);
9659	}
9660
9661	zfree(&obj->externs);
9662	obj->nr_extern = 0;
9663
9664	zfree(&obj->maps);
9665	obj->nr_maps = 0;
9666
9667	if (obj->programs && obj->nr_programs) {
9668	for (i = 0; i < obj->nr_programs; i++)
9669	bpf_program__exit(&obj->programs[i]);
9670	}
9671	zfree(&obj->programs);
9672
9673	zfree(&obj->feat_cache);
9674	zfree(&obj->token_path);
9675	if (obj->token_fd > 0)
9676	close(obj->token_fd);
9677
9678	zfree(&obj->arena_data);
9679
9680	zfree(&obj->jumptables_data);
9681	obj->jumptables_data_sz = 0;
9682
9683	for (i = 0; i < obj->jumptable_map_cnt; i++)
9684	close(obj->jumptable_maps[i].fd);
9685	zfree(&obj->jumptable_maps);
9686
9687	free(obj);
9688	}
9689
9690	const char bpf_object__name(const struct bpf_object obj)
9691	{
9692	return obj ? obj->name : libbpf_err_ptr(-EINVAL);
9693	}
9694
9695	unsigned int bpf_object__kversion(const struct bpf_object *obj)
9696	{
9697	return obj ? obj->kern_version : 0;
9698	}
9699
9700	int bpf_object__token_fd(const struct bpf_object *obj)
9701	{
9702	return obj->token_fd ?: -1;
9703	}
9704
9705	struct btf bpf_object__btf(const struct bpf_object obj)
9706	{
9707	return obj ? obj->btf : NULL;
9708	}
9709
9710	int bpf_object__btf_fd(const struct bpf_object *obj)
9711	{
9712	return obj->btf ? btf__fd(obj->btf) : -1;
9713	}
9714
9715	int bpf_object__set_kversion(struct bpf_object *obj, __u32 kern_version)
9716	{
9717	if (obj->state >= OBJ_LOADED)
9718	return libbpf_err(-EINVAL);
9719
9720	obj->kern_version = kern_version;
9721
9722	return 0;
9723	}
9724
9725	int bpf_object__gen_loader(struct bpf_object obj, struct gen_loader_opts opts)
9726	{
9727	struct bpf_gen *gen;
9728
9729	if (!opts)
9730	return libbpf_err(-EFAULT);
9731	if (!OPTS_VALID(opts, gen_loader_opts))
9732	return libbpf_err(-EINVAL);
9733	gen = calloc(1, sizeof(*gen));
9734	if (!gen)
9735	return libbpf_err(-ENOMEM);
9736	gen->opts = opts;
9737	gen->swapped_endian = !is_native_endianness(obj);
9738	obj->gen_loader = gen;
9739	return 0;
9740	}
9741
9742	static struct bpf_program *
9743	__bpf_program__iter(const struct bpf_program p, const struct bpf_object obj,
9744	bool forward)
9745	{
9746	size_t nr_programs = obj->nr_programs;
9747	ssize_t idx;
9748
9749	if (!nr_programs)
9750	return NULL;
9751
9752	if (!p)
9753	/* Iter from the beginning */
9754	return forward ? &obj->programs[0] :
9755	&obj->programs[nr_programs - 1];
9756
9757	if (p->obj != obj) {
9758	pr_warn("error: program handler doesn't match object\n");
9759	return errno = EINVAL, NULL;
9760	}
9761
9762	idx = (p - obj->programs) + (forward ? 1 : -1);
9763	if (idx >= obj->nr_programs \|\| idx < 0)
9764	return NULL;
9765	return &obj->programs[idx];
9766	}
9767
9768	struct bpf_program *
9769	bpf_object__next_program(const struct bpf_object obj, struct bpf_program prev)
9770	{
9771	struct bpf_program *prog = prev;
9772
9773	do {
9774	prog = __bpf_program__iter(prog, obj, true);
9775	} while (prog && prog_is_subprog(obj, prog));
9776
9777	return prog;
9778	}
9779
9780	struct bpf_program *
9781	bpf_object__prev_program(const struct bpf_object obj, struct bpf_program next)
9782	{
9783	struct bpf_program *prog = next;
9784
9785	do {
9786	prog = __bpf_program__iter(prog, obj, false);
9787	} while (prog && prog_is_subprog(obj, prog));
9788
9789	return prog;
9790	}
9791
9792	void bpf_program__set_ifindex(struct bpf_program *prog, __u32 ifindex)
9793	{
9794	prog->prog_ifindex = ifindex;
9795	}
9796
9797	const char bpf_program__name(const struct bpf_program prog)
9798	{
9799	return prog->name;
9800	}
9801
9802	const char bpf_program__section_name(const struct bpf_program prog)
9803	{
9804	return prog->sec_name;
9805	}
9806
9807	bool bpf_program__autoload(const struct bpf_program *prog)
9808	{
9809	return prog->autoload;
9810	}
9811
9812	int bpf_program__set_autoload(struct bpf_program *prog, bool autoload)
9813	{
9814	if (prog->obj->state >= OBJ_LOADED)
9815	return libbpf_err(-EINVAL);
9816
9817	prog->autoload = autoload;
9818	return 0;
9819	}
9820
9821	bool bpf_program__autoattach(const struct bpf_program *prog)
9822	{
9823	return prog->autoattach;
9824	}
9825
9826	void bpf_program__set_autoattach(struct bpf_program *prog, bool autoattach)
9827	{
9828	prog->autoattach = autoattach;
9829	}
9830
9831	const struct bpf_insn bpf_program__insns(const struct bpf_program prog)
9832	{
9833	return prog->insns;
9834	}
9835
9836	size_t bpf_program__insn_cnt(const struct bpf_program *prog)
9837	{
9838	return prog->insns_cnt;
9839	}
9840
9841	int bpf_program__set_insns(struct bpf_program *prog,
9842	struct bpf_insn *new_insns, size_t new_insn_cnt)
9843	{
9844	struct bpf_insn *insns;
9845
9846	if (prog->obj->state >= OBJ_LOADED)
9847	return libbpf_err(-EBUSY);
9848
9849	insns = libbpf_reallocarray(prog->insns, new_insn_cnt, sizeof(*insns));
9850	/* NULL is a valid return from reallocarray if the new count is zero */
9851	if (!insns && new_insn_cnt) {
9852	pr_warn("prog '%s': failed to realloc prog code\n", prog->name);
9853	return libbpf_err(-ENOMEM);
9854	}
9855	memcpy(insns, new_insns, new_insn_cnt * sizeof(*insns));
9856
9857	prog->insns = insns;
9858	prog->insns_cnt = new_insn_cnt;
9859	return 0;
9860	}
9861
9862	int bpf_program__fd(const struct bpf_program *prog)
9863	{
9864	if (!prog)
9865	return libbpf_err(-EINVAL);
9866
9867	if (prog->fd < 0)
9868	return libbpf_err(-ENOENT);
9869
9870	return prog->fd;
9871	}
9872
9873	__alias(bpf_program__type)
9874	enum bpf_prog_type bpf_program__get_type(const struct bpf_program *prog);
9875
9876	enum bpf_prog_type bpf_program__type(const struct bpf_program *prog)
9877	{
9878	return prog->type;
9879	}
9880
9881	static size_t custom_sec_def_cnt;
9882	static struct bpf_sec_def *custom_sec_defs;
9883	static struct bpf_sec_def custom_fallback_def;
9884	static bool has_custom_fallback_def;
9885	static int last_custom_sec_def_handler_id;
9886
9887	int bpf_program__set_type(struct bpf_program *prog, enum bpf_prog_type type)
9888	{
9889	if (prog->obj->state >= OBJ_LOADED)
9890	return libbpf_err(-EBUSY);
9891
9892	/* if type is not changed, do nothing */
9893	if (prog->type == type)
9894	return 0;
9895
9896	prog->type = type;
9897
9898	/* If a program type was changed, we need to reset associated SEC()
9899	* handler, as it will be invalid now. The only exception is a generic
9900	* fallback handler, which by definition is program type-agnostic and
9901	* is a catch-all custom handler, optionally set by the application,
9902	* so should be able to handle any type of BPF program.
9903	*/
9904	if (prog->sec_def != &custom_fallback_def)
9905	prog->sec_def = NULL;
9906	return 0;
9907	}
9908
9909	__alias(bpf_program__expected_attach_type)
9910	enum bpf_attach_type bpf_program__get_expected_attach_type(const struct bpf_program *prog);
9911
9912	enum bpf_attach_type bpf_program__expected_attach_type(const struct bpf_program *prog)
9913	{
9914	return prog->expected_attach_type;
9915	}
9916
9917	int bpf_program__set_expected_attach_type(struct bpf_program *prog,
9918	enum bpf_attach_type type)
9919	{
9920	if (prog->obj->state >= OBJ_LOADED)
9921	return libbpf_err(-EBUSY);
9922
9923	prog->expected_attach_type = type;
9924	return 0;
9925	}
9926
9927	__u32 bpf_program__flags(const struct bpf_program *prog)
9928	{
9929	return prog->prog_flags;
9930	}
9931
9932	int bpf_program__set_flags(struct bpf_program *prog, __u32 flags)
9933	{
9934	if (prog->obj->state >= OBJ_LOADED)
9935	return libbpf_err(-EBUSY);
9936
9937	prog->prog_flags = flags;
9938	return 0;
9939	}
9940
9941	__u32 bpf_program__log_level(const struct bpf_program *prog)
9942	{
9943	return prog->log_level;
9944	}
9945
9946	int bpf_program__set_log_level(struct bpf_program *prog, __u32 log_level)
9947	{
9948	if (prog->obj->state >= OBJ_LOADED)
9949	return libbpf_err(-EBUSY);
9950
9951	prog->log_level = log_level;
9952	return 0;
9953	}
9954
9955	const char bpf_program__log_buf(const struct bpf_program prog, size_t *log_size)
9956	{
9957	*log_size = prog->log_size;
9958	return prog->log_buf;
9959	}
9960
9961	int bpf_program__set_log_buf(struct bpf_program prog, char log_buf, size_t log_size)
9962	{
9963	if (log_size && !log_buf)
9964	return libbpf_err(-EINVAL);
9965	if (prog->log_size > UINT_MAX)
9966	return libbpf_err(-EINVAL);
9967	if (prog->obj->state >= OBJ_LOADED)
9968	return libbpf_err(-EBUSY);
9969
9970	prog->log_buf = log_buf;
9971	prog->log_size = log_size;
9972	return 0;
9973	}
9974
9975	struct bpf_func_info bpf_program__func_info(const struct bpf_program prog)
9976	{
9977	if (prog->func_info_rec_size != sizeof(struct bpf_func_info))
9978	return libbpf_err_ptr(-EOPNOTSUPP);
9979	return prog->func_info;
9980	}
9981
9982	__u32 bpf_program__func_info_cnt(const struct bpf_program *prog)
9983	{
9984	return prog->func_info_cnt;
9985	}
9986
9987	struct bpf_line_info bpf_program__line_info(const struct bpf_program prog)
9988	{
9989	if (prog->line_info_rec_size != sizeof(struct bpf_line_info))
9990	return libbpf_err_ptr(-EOPNOTSUPP);
9991	return prog->line_info;
9992	}
9993
9994	__u32 bpf_program__line_info_cnt(const struct bpf_program *prog)
9995	{
9996	return prog->line_info_cnt;
9997	}
9998
9999	int bpf_program__clone(struct bpf_program prog, const struct bpf_prog_load_opts opts)
10000	{
10001	LIBBPF_OPTS(bpf_prog_load_opts, attr);
10002	struct bpf_object *obj;
10003	const void *info;
10004	__u32 info_cnt, info_rec_size;
10005	int err, fd, prog_btf_fd;
10006
10007	if (!prog)
10008	return libbpf_err(-EINVAL);
10009
10010	if (!OPTS_VALID(opts, bpf_prog_load_opts))
10011	return libbpf_err(-EINVAL);
10012
10013	obj = prog->obj;
10014	if (obj->state < OBJ_PREPARED)
10015	return libbpf_err(-EINVAL);
10016
10017	/*
10018	* Caller-provided opts take priority; fall back to
10019	* prog/object defaults when the caller leaves them zero.
10020	*/
10021	attr.attach_prog_fd = OPTS_GET(opts, attach_prog_fd, 0) ?: prog->attach_prog_fd;
10022	attr.prog_flags = OPTS_GET(opts, prog_flags, 0) ?: prog->prog_flags;
10023	attr.prog_ifindex = OPTS_GET(opts, prog_ifindex, 0) ?: prog->prog_ifindex;
10024	attr.kern_version = OPTS_GET(opts, kern_version, 0) ?: obj->kern_version;
10025	attr.fd_array = OPTS_GET(opts, fd_array, NULL) ?: obj->fd_array;
10026	attr.fd_array_cnt = OPTS_GET(opts, fd_array_cnt, 0) ?: obj->fd_array_cnt;
10027	attr.token_fd = OPTS_GET(opts, token_fd, 0) ?: obj->token_fd;
10028	if (attr.token_fd)
10029	attr.prog_flags \|= BPF_F_TOKEN_FD;
10030
10031	prog_btf_fd = OPTS_GET(opts, prog_btf_fd, 0);
10032	if (!prog_btf_fd && obj->btf)
10033	prog_btf_fd = btf__fd(obj->btf);
10034
10035	/* BTF func/line info: only pass if kernel supports it */
10036	if (kernel_supports(obj, FEAT_BTF_FUNC) && prog_btf_fd > 0) {
10037	attr.prog_btf_fd = prog_btf_fd;
10038
10039	/* func_info/line_info triples: all-or-nothing from caller */
10040	info = OPTS_GET(opts, func_info, NULL);
10041	info_cnt = OPTS_GET(opts, func_info_cnt, 0);
10042	info_rec_size = OPTS_GET(opts, func_info_rec_size, 0);
10043	if (!!info != !!info_cnt \|\| !!info != !!info_rec_size) {
10044	pr_warn("prog '%s': func_info, func_info_cnt, and func_info_rec_size must all be specified or all omitted\n",
10045	prog->name);
10046	return libbpf_err(-EINVAL);
10047	}
10048	attr.func_info = info ?: prog->func_info;
10049	attr.func_info_cnt = info ? info_cnt : prog->func_info_cnt;
10050	attr.func_info_rec_size = info ? info_rec_size : prog->func_info_rec_size;
10051
10052	info = OPTS_GET(opts, line_info, NULL);
10053	info_cnt = OPTS_GET(opts, line_info_cnt, 0);
10054	info_rec_size = OPTS_GET(opts, line_info_rec_size, 0);
10055	if (!!info != !!info_cnt \|\| !!info != !!info_rec_size) {
10056	pr_warn("prog '%s': line_info, line_info_cnt, and line_info_rec_size must all be specified or all omitted\n",
10057	prog->name);
10058	return libbpf_err(-EINVAL);
10059	}
10060	attr.line_info = info ?: prog->line_info;
10061	attr.line_info_cnt = info ? info_cnt : prog->line_info_cnt;
10062	attr.line_info_rec_size = info ? info_rec_size : prog->line_info_rec_size;
10063	}
10064
10065	/* Logging is caller-controlled; no fallback to prog/obj log settings */
10066	attr.log_buf = OPTS_GET(opts, log_buf, NULL);
10067	attr.log_size = OPTS_GET(opts, log_size, 0);
10068	attr.log_level = OPTS_GET(opts, log_level, 0);
10069
10070	/*
10071	* Fields below may be mutated by prog_prepare_load_fn:
10072	* Seed them from prog/obj defaults here;
10073	* Later override with caller-provided opts.
10074	*/
10075	attr.expected_attach_type = prog->expected_attach_type;
10076	attr.attach_btf_id = prog->attach_btf_id;
10077	attr.attach_btf_obj_fd = prog->attach_btf_obj_fd;
10078
10079	if (prog->sec_def && prog->sec_def->prog_prepare_load_fn) {
10080	err = prog->sec_def->prog_prepare_load_fn(prog, &attr, prog->sec_def->cookie);
10081	if (err)
10082	return libbpf_err(err);
10083	}
10084
10085	/* Re-apply caller overrides for output fields */
10086	if (OPTS_GET(opts, expected_attach_type, 0))
10087	attr.expected_attach_type = OPTS_GET(opts, expected_attach_type, 0);
10088	if (OPTS_GET(opts, attach_btf_id, 0))
10089	attr.attach_btf_id = OPTS_GET(opts, attach_btf_id, 0);
10090	if (OPTS_GET(opts, attach_btf_obj_fd, 0))
10091	attr.attach_btf_obj_fd = OPTS_GET(opts, attach_btf_obj_fd, 0);
10092
10093	/*
10094	* Unlike bpf_object_load_prog(), we intentionally do not call bpf_prog_bind_map()
10095	* for RODATA maps here to avoid mutating the object's state. Callers can bind the
10096	* required maps themselves using bpf_prog_bind_map().
10097	*/
10098	fd = bpf_prog_load(prog->type, prog->name, obj->license, prog->insns, prog->insns_cnt,
10099	&attr);
10100
10101	return libbpf_err(fd);
10102	}
10103
10104	#define SEC_DEF(sec_pfx, ptype, atype, flags, ...) { \
10105	.sec = (char *)sec_pfx, \
10106	.prog_type = BPF_PROG_TYPE_##ptype, \
10107	.expected_attach_type = atype, \
10108	.cookie = (long)(flags), \
10109	.prog_prepare_load_fn = libbpf_prepare_prog_load, \
10110	__VA_ARGS__ \
10111	}
10112
10113	static int attach_kprobe(const struct bpf_program prog, long cookie, struct bpf_link *link);
10114	static int attach_uprobe(const struct bpf_program prog, long cookie, struct bpf_link *link);
10115	static int attach_ksyscall(const struct bpf_program prog, long cookie, struct bpf_link *link);
10116	static int attach_usdt(const struct bpf_program prog, long cookie, struct bpf_link *link);
10117	static int attach_tp(const struct bpf_program prog, long cookie, struct bpf_link *link);
10118	static int attach_raw_tp(const struct bpf_program prog, long cookie, struct bpf_link *link);
10119	static int attach_trace(const struct bpf_program prog, long cookie, struct bpf_link *link);
10120	static int attach_kprobe_multi(const struct bpf_program prog, long cookie, struct bpf_link *link);
10121	static int attach_kprobe_session(const struct bpf_program prog, long cookie, struct bpf_link *link);
10122	static int attach_uprobe_multi(const struct bpf_program prog, long cookie, struct bpf_link *link);
10123	static int attach_lsm(const struct bpf_program prog, long cookie, struct bpf_link *link);
10124	static int attach_iter(const struct bpf_program prog, long cookie, struct bpf_link *link);
10125	static int attach_tracing_multi(const struct bpf_program prog, long cookie, struct bpf_link *link);
10126
10127	static const struct bpf_sec_def section_defs[] = {
10128	SEC_DEF("socket", SOCKET_FILTER, 0, SEC_NONE),
10129	SEC_DEF("sk_reuseport/migrate", SK_REUSEPORT, BPF_SK_REUSEPORT_SELECT_OR_MIGRATE, SEC_ATTACHABLE),
10130	SEC_DEF("sk_reuseport", SK_REUSEPORT, BPF_SK_REUSEPORT_SELECT, SEC_ATTACHABLE),
10131	SEC_DEF("kprobe+", KPROBE, 0, SEC_NONE, attach_kprobe),
10132	SEC_DEF("uprobe+", KPROBE, 0, SEC_NONE, attach_uprobe),
10133	SEC_DEF("uprobe.s+", KPROBE, 0, SEC_SLEEPABLE, attach_uprobe),
10134	SEC_DEF("kretprobe+", KPROBE, 0, SEC_NONE, attach_kprobe),
10135	SEC_DEF("uretprobe+", KPROBE, 0, SEC_NONE, attach_uprobe),
10136	SEC_DEF("uretprobe.s+", KPROBE, 0, SEC_SLEEPABLE, attach_uprobe),
10137	SEC_DEF("kprobe.multi+", KPROBE, BPF_TRACE_KPROBE_MULTI, SEC_NONE, attach_kprobe_multi),
10138	SEC_DEF("kretprobe.multi+", KPROBE, BPF_TRACE_KPROBE_MULTI, SEC_NONE, attach_kprobe_multi),
10139	SEC_DEF("kprobe.session+", KPROBE, BPF_TRACE_KPROBE_SESSION, SEC_NONE, attach_kprobe_session),
10140	SEC_DEF("uprobe.multi+", KPROBE, BPF_TRACE_UPROBE_MULTI, SEC_NONE, attach_uprobe_multi),
10141	SEC_DEF("uretprobe.multi+", KPROBE, BPF_TRACE_UPROBE_MULTI, SEC_NONE, attach_uprobe_multi),
10142	SEC_DEF("uprobe.session+", KPROBE, BPF_TRACE_UPROBE_SESSION, SEC_NONE, attach_uprobe_multi),
10143	SEC_DEF("uprobe.multi.s+", KPROBE, BPF_TRACE_UPROBE_MULTI, SEC_SLEEPABLE, attach_uprobe_multi),
10144	SEC_DEF("uretprobe.multi.s+", KPROBE, BPF_TRACE_UPROBE_MULTI, SEC_SLEEPABLE, attach_uprobe_multi),
10145	SEC_DEF("uprobe.session.s+", KPROBE, BPF_TRACE_UPROBE_SESSION, SEC_SLEEPABLE, attach_uprobe_multi),
10146	SEC_DEF("ksyscall+", KPROBE, 0, SEC_NONE, attach_ksyscall),
10147	SEC_DEF("kretsyscall+", KPROBE, 0, SEC_NONE, attach_ksyscall),
10148	SEC_DEF("usdt+", KPROBE, 0, SEC_USDT, attach_usdt),
10149	SEC_DEF("usdt.s+", KPROBE, 0, SEC_USDT \| SEC_SLEEPABLE, attach_usdt),
10150	SEC_DEF("tc/ingress", SCHED_CLS, BPF_TCX_INGRESS, SEC_NONE), /* alias for tcx */
10151	SEC_DEF("tc/egress", SCHED_CLS, BPF_TCX_EGRESS, SEC_NONE), /* alias for tcx */
10152	SEC_DEF("tcx/ingress", SCHED_CLS, BPF_TCX_INGRESS, SEC_NONE),
10153	SEC_DEF("tcx/egress", SCHED_CLS, BPF_TCX_EGRESS, SEC_NONE),
10154	SEC_DEF("tc", SCHED_CLS, 0, SEC_NONE), /* deprecated / legacy, use tcx */
10155	SEC_DEF("classifier", SCHED_CLS, 0, SEC_NONE), /* deprecated / legacy, use tcx */
10156	SEC_DEF("action", SCHED_ACT, 0, SEC_NONE), /* deprecated / legacy, use tcx */
10157	SEC_DEF("netkit/primary", SCHED_CLS, BPF_NETKIT_PRIMARY, SEC_NONE),
10158	SEC_DEF("netkit/peer", SCHED_CLS, BPF_NETKIT_PEER, SEC_NONE),
10159	SEC_DEF("tracepoint+", TRACEPOINT, 0, SEC_NONE, attach_tp),
10160	SEC_DEF("tp+", TRACEPOINT, 0, SEC_NONE, attach_tp),
10161	SEC_DEF("tracepoint.s+", TRACEPOINT, 0, SEC_SLEEPABLE, attach_tp),
10162	SEC_DEF("tp.s+", TRACEPOINT, 0, SEC_SLEEPABLE, attach_tp),
10163	SEC_DEF("raw_tracepoint+", RAW_TRACEPOINT, 0, SEC_NONE, attach_raw_tp),
10164	SEC_DEF("raw_tp+", RAW_TRACEPOINT, 0, SEC_NONE, attach_raw_tp),
10165	SEC_DEF("raw_tracepoint.s+", RAW_TRACEPOINT, 0, SEC_SLEEPABLE, attach_raw_tp),
10166	SEC_DEF("raw_tp.s+", RAW_TRACEPOINT, 0, SEC_SLEEPABLE, attach_raw_tp),
10167	SEC_DEF("raw_tracepoint.w+", RAW_TRACEPOINT_WRITABLE, 0, SEC_NONE, attach_raw_tp),
10168	SEC_DEF("raw_tp.w+", RAW_TRACEPOINT_WRITABLE, 0, SEC_NONE, attach_raw_tp),
10169	SEC_DEF("tp_btf+", TRACING, BPF_TRACE_RAW_TP, SEC_ATTACH_BTF, attach_trace),
10170	SEC_DEF("tp_btf.s+", TRACING, BPF_TRACE_RAW_TP, SEC_ATTACH_BTF \| SEC_SLEEPABLE, attach_trace),
10171	SEC_DEF("fentry+", TRACING, BPF_TRACE_FENTRY, SEC_ATTACH_BTF, attach_trace),
10172	SEC_DEF("fmod_ret+", TRACING, BPF_MODIFY_RETURN, SEC_ATTACH_BTF, attach_trace),
10173	SEC_DEF("fexit+", TRACING, BPF_TRACE_FEXIT, SEC_ATTACH_BTF, attach_trace),
10174	SEC_DEF("fentry.s+", TRACING, BPF_TRACE_FENTRY, SEC_ATTACH_BTF \| SEC_SLEEPABLE, attach_trace),
10175	SEC_DEF("fmod_ret.s+", TRACING, BPF_MODIFY_RETURN, SEC_ATTACH_BTF \| SEC_SLEEPABLE, attach_trace),
10176	SEC_DEF("fexit.s+", TRACING, BPF_TRACE_FEXIT, SEC_ATTACH_BTF \| SEC_SLEEPABLE, attach_trace),
10177	SEC_DEF("fsession+", TRACING, BPF_TRACE_FSESSION, SEC_ATTACH_BTF, attach_trace),
10178	SEC_DEF("fsession.s+", TRACING, BPF_TRACE_FSESSION, SEC_ATTACH_BTF \| SEC_SLEEPABLE, attach_trace),
10179	SEC_DEF("fsession.multi+", TRACING, BPF_TRACE_FSESSION_MULTI, 0, attach_tracing_multi),
10180	SEC_DEF("fsession.multi.s+", TRACING, BPF_TRACE_FSESSION_MULTI, SEC_SLEEPABLE, attach_tracing_multi),
10181	SEC_DEF("fentry.multi+", TRACING, BPF_TRACE_FENTRY_MULTI, 0, attach_tracing_multi),
10182	SEC_DEF("fexit.multi+", TRACING, BPF_TRACE_FEXIT_MULTI, 0, attach_tracing_multi),
10183	SEC_DEF("fentry.multi.s+", TRACING, BPF_TRACE_FENTRY_MULTI, SEC_SLEEPABLE, attach_tracing_multi),
10184	SEC_DEF("fexit.multi.s+", TRACING, BPF_TRACE_FEXIT_MULTI, SEC_SLEEPABLE, attach_tracing_multi),
10185	SEC_DEF("freplace+", EXT, 0, SEC_ATTACH_BTF, attach_trace),
10186	SEC_DEF("lsm+", LSM, BPF_LSM_MAC, SEC_ATTACH_BTF, attach_lsm),
10187	SEC_DEF("lsm.s+", LSM, BPF_LSM_MAC, SEC_ATTACH_BTF \| SEC_SLEEPABLE, attach_lsm),
10188	SEC_DEF("lsm_cgroup+", LSM, BPF_LSM_CGROUP, SEC_ATTACH_BTF),
10189	SEC_DEF("iter+", TRACING, BPF_TRACE_ITER, SEC_ATTACH_BTF, attach_iter),
10190	SEC_DEF("iter.s+", TRACING, BPF_TRACE_ITER, SEC_ATTACH_BTF \| SEC_SLEEPABLE, attach_iter),
10191	SEC_DEF("syscall", SYSCALL, 0, SEC_SLEEPABLE),
10192	SEC_DEF("xdp.frags/devmap", XDP, BPF_XDP_DEVMAP, SEC_XDP_FRAGS),
10193	SEC_DEF("xdp/devmap", XDP, BPF_XDP_DEVMAP, SEC_ATTACHABLE),
10194	SEC_DEF("xdp.frags/cpumap", XDP, BPF_XDP_CPUMAP, SEC_XDP_FRAGS),
10195	SEC_DEF("xdp/cpumap", XDP, BPF_XDP_CPUMAP, SEC_ATTACHABLE),
10196	SEC_DEF("xdp.frags", XDP, BPF_XDP, SEC_XDP_FRAGS),
10197	SEC_DEF("xdp", XDP, BPF_XDP, SEC_ATTACHABLE_OPT),
10198	SEC_DEF("perf_event", PERF_EVENT, 0, SEC_NONE),
10199	SEC_DEF("lwt_in", LWT_IN, 0, SEC_NONE),
10200	SEC_DEF("lwt_out", LWT_OUT, 0, SEC_NONE),
10201	SEC_DEF("lwt_xmit", LWT_XMIT, 0, SEC_NONE),
10202	SEC_DEF("lwt_seg6local", LWT_SEG6LOCAL, 0, SEC_NONE),
10203	SEC_DEF("sockops", SOCK_OPS, BPF_CGROUP_SOCK_OPS, SEC_ATTACHABLE_OPT),
10204	SEC_DEF("sk_skb/stream_parser", SK_SKB, BPF_SK_SKB_STREAM_PARSER, SEC_ATTACHABLE_OPT),
10205	SEC_DEF("sk_skb/stream_verdict",SK_SKB, BPF_SK_SKB_STREAM_VERDICT, SEC_ATTACHABLE_OPT),
10206	SEC_DEF("sk_skb/verdict", SK_SKB, BPF_SK_SKB_VERDICT, SEC_ATTACHABLE_OPT),
10207	SEC_DEF("sk_skb", SK_SKB, 0, SEC_NONE),
10208	SEC_DEF("sk_msg", SK_MSG, BPF_SK_MSG_VERDICT, SEC_ATTACHABLE_OPT),
10209	SEC_DEF("lirc_mode2", LIRC_MODE2, BPF_LIRC_MODE2, SEC_ATTACHABLE_OPT),
10210	SEC_DEF("flow_dissector", FLOW_DISSECTOR, BPF_FLOW_DISSECTOR, SEC_ATTACHABLE_OPT),
10211	SEC_DEF("cgroup_skb/ingress", CGROUP_SKB, BPF_CGROUP_INET_INGRESS, SEC_ATTACHABLE_OPT),
10212	SEC_DEF("cgroup_skb/egress", CGROUP_SKB, BPF_CGROUP_INET_EGRESS, SEC_ATTACHABLE_OPT),
10213	SEC_DEF("cgroup/skb", CGROUP_SKB, 0, SEC_NONE),
10214	SEC_DEF("cgroup/sock_create", CGROUP_SOCK, BPF_CGROUP_INET_SOCK_CREATE, SEC_ATTACHABLE),
10215	SEC_DEF("cgroup/sock_release", CGROUP_SOCK, BPF_CGROUP_INET_SOCK_RELEASE, SEC_ATTACHABLE),
10216	SEC_DEF("cgroup/sock", CGROUP_SOCK, BPF_CGROUP_INET_SOCK_CREATE, SEC_ATTACHABLE_OPT),
10217	SEC_DEF("cgroup/post_bind4", CGROUP_SOCK, BPF_CGROUP_INET4_POST_BIND, SEC_ATTACHABLE),
10218	SEC_DEF("cgroup/post_bind6", CGROUP_SOCK, BPF_CGROUP_INET6_POST_BIND, SEC_ATTACHABLE),
10219	SEC_DEF("cgroup/bind4", CGROUP_SOCK_ADDR, BPF_CGROUP_INET4_BIND, SEC_ATTACHABLE),
10220	SEC_DEF("cgroup/bind6", CGROUP_SOCK_ADDR, BPF_CGROUP_INET6_BIND, SEC_ATTACHABLE),
10221	SEC_DEF("cgroup/connect4", CGROUP_SOCK_ADDR, BPF_CGROUP_INET4_CONNECT, SEC_ATTACHABLE),
10222	SEC_DEF("cgroup/connect6", CGROUP_SOCK_ADDR, BPF_CGROUP_INET6_CONNECT, SEC_ATTACHABLE),
10223	SEC_DEF("cgroup/connect_unix", CGROUP_SOCK_ADDR, BPF_CGROUP_UNIX_CONNECT, SEC_ATTACHABLE),
10224	SEC_DEF("cgroup/sendmsg4", CGROUP_SOCK_ADDR, BPF_CGROUP_UDP4_SENDMSG, SEC_ATTACHABLE),
10225	SEC_DEF("cgroup/sendmsg6", CGROUP_SOCK_ADDR, BPF_CGROUP_UDP6_SENDMSG, SEC_ATTACHABLE),
10226	SEC_DEF("cgroup/sendmsg_unix", CGROUP_SOCK_ADDR, BPF_CGROUP_UNIX_SENDMSG, SEC_ATTACHABLE),
10227	SEC_DEF("cgroup/recvmsg4", CGROUP_SOCK_ADDR, BPF_CGROUP_UDP4_RECVMSG, SEC_ATTACHABLE),
10228	SEC_DEF("cgroup/recvmsg6", CGROUP_SOCK_ADDR, BPF_CGROUP_UDP6_RECVMSG, SEC_ATTACHABLE),
10229	SEC_DEF("cgroup/recvmsg_unix", CGROUP_SOCK_ADDR, BPF_CGROUP_UNIX_RECVMSG, SEC_ATTACHABLE),
10230	SEC_DEF("cgroup/getpeername4", CGROUP_SOCK_ADDR, BPF_CGROUP_INET4_GETPEERNAME, SEC_ATTACHABLE),
10231	SEC_DEF("cgroup/getpeername6", CGROUP_SOCK_ADDR, BPF_CGROUP_INET6_GETPEERNAME, SEC_ATTACHABLE),
10232	SEC_DEF("cgroup/getpeername_unix", CGROUP_SOCK_ADDR, BPF_CGROUP_UNIX_GETPEERNAME, SEC_ATTACHABLE),
10233	SEC_DEF("cgroup/getsockname4", CGROUP_SOCK_ADDR, BPF_CGROUP_INET4_GETSOCKNAME, SEC_ATTACHABLE),
10234	SEC_DEF("cgroup/getsockname6", CGROUP_SOCK_ADDR, BPF_CGROUP_INET6_GETSOCKNAME, SEC_ATTACHABLE),
10235	SEC_DEF("cgroup/getsockname_unix", CGROUP_SOCK_ADDR, BPF_CGROUP_UNIX_GETSOCKNAME, SEC_ATTACHABLE),
10236	SEC_DEF("cgroup/sysctl", CGROUP_SYSCTL, BPF_CGROUP_SYSCTL, SEC_ATTACHABLE),
10237	SEC_DEF("cgroup/getsockopt", CGROUP_SOCKOPT, BPF_CGROUP_GETSOCKOPT, SEC_ATTACHABLE),
10238	SEC_DEF("cgroup/setsockopt", CGROUP_SOCKOPT, BPF_CGROUP_SETSOCKOPT, SEC_ATTACHABLE),
10239	SEC_DEF("cgroup/dev", CGROUP_DEVICE, BPF_CGROUP_DEVICE, SEC_ATTACHABLE_OPT),
10240	SEC_DEF("struct_ops+", STRUCT_OPS, 0, SEC_NONE),
10241	SEC_DEF("struct_ops.s+", STRUCT_OPS, 0, SEC_SLEEPABLE),
10242	SEC_DEF("sk_lookup", SK_LOOKUP, BPF_SK_LOOKUP, SEC_ATTACHABLE),
10243	SEC_DEF("netfilter", NETFILTER, BPF_NETFILTER, SEC_NONE),
10244	};
10245
10246	int libbpf_register_prog_handler(const char *sec,
10247	enum bpf_prog_type prog_type,
10248	enum bpf_attach_type exp_attach_type,
10249	const struct libbpf_prog_handler_opts *opts)
10250	{
10251	struct bpf_sec_def *sec_def;
10252
10253	if (!OPTS_VALID(opts, libbpf_prog_handler_opts))
10254	return libbpf_err(-EINVAL);
10255
10256	if (last_custom_sec_def_handler_id == INT_MAX) /* prevent overflow */
10257	return libbpf_err(-E2BIG);
10258
10259	if (sec) {
10260	sec_def = libbpf_reallocarray(custom_sec_defs, custom_sec_def_cnt + 1,
10261	sizeof(*sec_def));
10262	if (!sec_def)
10263	return libbpf_err(-ENOMEM);
10264
10265	custom_sec_defs = sec_def;
10266	sec_def = &custom_sec_defs[custom_sec_def_cnt];
10267	} else {
10268	if (has_custom_fallback_def)
10269	return libbpf_err(-EBUSY);
10270
10271	sec_def = &custom_fallback_def;
10272	}
10273
10274	sec_def->sec = sec ? strdup(sec) : NULL;
10275	if (sec && !sec_def->sec)
10276	return libbpf_err(-ENOMEM);
10277
10278	sec_def->prog_type = prog_type;
10279	sec_def->expected_attach_type = exp_attach_type;
10280	sec_def->cookie = OPTS_GET(opts, cookie, 0);
10281
10282	sec_def->prog_setup_fn = OPTS_GET(opts, prog_setup_fn, NULL);
10283	sec_def->prog_prepare_load_fn = OPTS_GET(opts, prog_prepare_load_fn, NULL);
10284	sec_def->prog_attach_fn = OPTS_GET(opts, prog_attach_fn, NULL);
10285
10286	sec_def->handler_id = ++last_custom_sec_def_handler_id;
10287
10288	if (sec)
10289	custom_sec_def_cnt++;
10290	else
10291	has_custom_fallback_def = true;
10292
10293	return sec_def->handler_id;
10294	}
10295
10296	int libbpf_unregister_prog_handler(int handler_id)
10297	{
10298	struct bpf_sec_def *sec_defs;
10299	int i;
10300
10301	if (handler_id <= 0)
10302	return libbpf_err(-EINVAL);
10303
10304	if (has_custom_fallback_def && custom_fallback_def.handler_id == handler_id) {
10305	memset(&custom_fallback_def, 0, sizeof(custom_fallback_def));
10306	has_custom_fallback_def = false;
10307	return 0;
10308	}
10309
10310	for (i = 0; i < custom_sec_def_cnt; i++) {
10311	if (custom_sec_defs[i].handler_id == handler_id)
10312	break;
10313	}
10314
10315	if (i == custom_sec_def_cnt)
10316	return libbpf_err(-ENOENT);
10317
10318	free(custom_sec_defs[i].sec);
10319	for (i = i + 1; i < custom_sec_def_cnt; i++)
10320	custom_sec_defs[i - 1] = custom_sec_defs[i];
10321	custom_sec_def_cnt--;
10322
10323	/* try to shrink the array, but it's ok if we couldn't */
10324	sec_defs = libbpf_reallocarray(custom_sec_defs, custom_sec_def_cnt, sizeof(*sec_defs));
10325	/* if new count is zero, reallocarray can return a valid NULL result;
10326	* in this case the previous pointer will be freed, so we have to
10327	* reassign old pointer to the new value (even if it's NULL)
10328	*/
10329	if (sec_defs \|\| custom_sec_def_cnt == 0)
10330	custom_sec_defs = sec_defs;
10331
10332	return 0;
10333	}
10334
10335	static bool sec_def_matches(const struct bpf_sec_def sec_def, const char sec_name)
10336	{
10337	size_t len = strlen(sec_def->sec);
10338
10339	/* "type/" always has to have proper SEC("type/extras") form */
10340	if (sec_def->sec[len - 1] == '/') {
10341	if (str_has_pfx(sec_name, sec_def->sec))
10342	return true;
10343	return false;
10344	}
10345
10346	/* "type+" means it can be either exact SEC("type") or
10347	* well-formed SEC("type/extras") with proper '/' separator
10348	*/
10349	if (sec_def->sec[len - 1] == '+') {
10350	len--;
10351	/* not even a prefix */
10352	if (strncmp(sec_name, sec_def->sec, len) != 0)
10353	return false;
10354	/* exact match or has '/' separator */
10355	if (sec_name[len] == '\0' \|\| sec_name[len] == '/')
10356	return true;
10357	return false;
10358	}
10359
10360	return strcmp(sec_name, sec_def->sec) == 0;
10361	}
10362
10363	static const struct bpf_sec_def find_sec_def(const char sec_name)
10364	{
10365	const struct bpf_sec_def *sec_def;
10366	int i, n;
10367
10368	n = custom_sec_def_cnt;
10369	for (i = 0; i < n; i++) {
10370	sec_def = &custom_sec_defs[i];
10371	if (sec_def_matches(sec_def, sec_name))
10372	return sec_def;
10373	}
10374
10375	n = ARRAY_SIZE(section_defs);
10376	for (i = 0; i < n; i++) {
10377	sec_def = &section_defs[i];
10378	if (sec_def_matches(sec_def, sec_name))
10379	return sec_def;
10380	}
10381
10382	if (has_custom_fallback_def)
10383	return &custom_fallback_def;
10384
10385	return NULL;
10386	}
10387
10388	#define MAX_TYPE_NAME_SIZE 32
10389
10390	static char *libbpf_get_type_names(bool attach_type)
10391	{
10392	int i, len = ARRAY_SIZE(section_defs) * MAX_TYPE_NAME_SIZE;
10393	char *buf;
10394
10395	buf = malloc(len);
10396	if (!buf)
10397	return NULL;
10398
10399	buf[0] = '\0';
10400	/* Forge string buf with all available names */
10401	for (i = 0; i < ARRAY_SIZE(section_defs); i++) {
10402	const struct bpf_sec_def *sec_def = &section_defs[i];
10403
10404	if (attach_type) {
10405	if (sec_def->prog_prepare_load_fn != libbpf_prepare_prog_load)
10406	continue;
10407
10408	if (!(sec_def->cookie & SEC_ATTACHABLE))
10409	continue;
10410	}
10411
10412	if (strlen(buf) + strlen(section_defs[i].sec) + 2 > len) {
10413	free(buf);
10414	return NULL;
10415	}
10416	strcat(buf, " ");
10417	strcat(buf, section_defs[i].sec);
10418	}
10419
10420	return buf;
10421	}
10422
10423	int libbpf_prog_type_by_name(const char name, enum bpf_prog_type prog_type,
10424	enum bpf_attach_type *expected_attach_type)
10425	{
10426	const struct bpf_sec_def *sec_def;
10427	char *type_names;
10428
10429	if (!name)
10430	return libbpf_err(-EINVAL);
10431
10432	sec_def = find_sec_def(name);
10433	if (sec_def) {
10434	*prog_type = sec_def->prog_type;
10435	*expected_attach_type = sec_def->expected_attach_type;
10436	return 0;
10437	}
10438
10439	pr_debug("failed to guess program type from ELF section '%s'\n", name);
10440	type_names = libbpf_get_type_names(false);
10441	if (type_names != NULL) {
10442	pr_debug("supported section(type) names are:%s\n", type_names);
10443	free(type_names);
10444	}
10445
10446	return libbpf_err(-ESRCH);
10447	}
10448
10449	const char *libbpf_bpf_attach_type_str(enum bpf_attach_type t)
10450	{
10451	if (t < 0 \|\| t >= ARRAY_SIZE(attach_type_name))
10452	return NULL;
10453
10454	return attach_type_name[t];
10455	}
10456
10457	const char *libbpf_bpf_link_type_str(enum bpf_link_type t)
10458	{
10459	if (t < 0 \|\| t >= ARRAY_SIZE(link_type_name))
10460	return NULL;
10461
10462	return link_type_name[t];
10463	}
10464
10465	const char *libbpf_bpf_map_type_str(enum bpf_map_type t)
10466	{
10467	if (t < 0 \|\| t >= ARRAY_SIZE(map_type_name))
10468	return NULL;
10469
10470	return map_type_name[t];
10471	}
10472
10473	const char *libbpf_bpf_prog_type_str(enum bpf_prog_type t)
10474	{
10475	if (t < 0 \|\| t >= ARRAY_SIZE(prog_type_name))
10476	return NULL;
10477
10478	return prog_type_name[t];
10479	}
10480
10481	static struct bpf_map find_struct_ops_map_by_offset(struct bpf_object obj,
10482	int sec_idx,
10483	size_t offset)
10484	{
10485	struct bpf_map *map;
10486	size_t i;
10487
10488	for (i = 0; i < obj->nr_maps; i++) {
10489	map = &obj->maps[i];
10490	if (!bpf_map__is_struct_ops(map))
10491	continue;
10492	if (map->sec_idx == sec_idx &&
10493	map->sec_offset <= offset &&
10494	offset - map->sec_offset < map->def.value_size)
10495	return map;
10496	}
10497
10498	return NULL;
10499	}
10500
10501	/* Collect the reloc from ELF, populate the st_ops->progs[], and update
10502	* st_ops->data for shadow type.
10503	*/
10504	static int bpf_object__collect_st_ops_relos(struct bpf_object *obj,
10505	Elf64_Shdr shdr, Elf_Data data)
10506	{
10507	const struct btf_type *type;
10508	const struct btf_member *member;
10509	struct bpf_struct_ops *st_ops;
10510	struct bpf_program *prog;
10511	unsigned int shdr_idx;
10512	const struct btf *btf;
10513	struct bpf_map *map;
10514	unsigned int moff, insn_idx;
10515	const char *name;
10516	__u32 member_idx;
10517	Elf64_Sym *sym;
10518	Elf64_Rel *rel;
10519	int i, nrels;
10520
10521	btf = obj->btf;
10522	nrels = shdr->sh_size / shdr->sh_entsize;
10523	for (i = 0; i < nrels; i++) {
10524	rel = elf_rel_by_idx(data, i);
10525	if (!rel) {
10526	pr_warn("struct_ops reloc: failed to get %d reloc\n", i);
10527	return -LIBBPF_ERRNO__FORMAT;
10528	}
10529
10530	sym = elf_sym_by_idx(obj, ELF64_R_SYM(rel->r_info));
10531	if (!sym) {
10532	pr_warn("struct_ops reloc: symbol %zx not found\n",
10533	(size_t)ELF64_R_SYM(rel->r_info));
10534	return -LIBBPF_ERRNO__FORMAT;
10535	}
10536
10537	name = elf_sym_str(obj, sym->st_name) ?: "<?>";
10538	map = find_struct_ops_map_by_offset(obj, shdr->sh_info, rel->r_offset);
10539	if (!map) {
10540	pr_warn("struct_ops reloc: cannot find map at rel->r_offset %zu\n",
10541	(size_t)rel->r_offset);
10542	return -EINVAL;
10543	}
10544
10545	moff = rel->r_offset - map->sec_offset;
10546	shdr_idx = sym->st_shndx;
10547	st_ops = map->st_ops;
10548	pr_debug("struct_ops reloc %s: for %lld value %lld shdr_idx %u rel->r_offset %zu map->sec_offset %zu name %u (\'%s\')\n",
10549	map->name,
10550	(long long)(rel->r_info >> 32),
10551	(long long)sym->st_value,
10552	shdr_idx, (size_t)rel->r_offset,
10553	map->sec_offset, sym->st_name, name);
10554
10555	if (shdr_idx >= SHN_LORESERVE) {
10556	pr_warn("struct_ops reloc %s: rel->r_offset %zu shdr_idx %u unsupported non-static function\n",
10557	map->name, (size_t)rel->r_offset, shdr_idx);
10558	return -LIBBPF_ERRNO__RELOC;
10559	}
10560	if (sym->st_value % BPF_INSN_SZ) {
10561	pr_warn("struct_ops reloc %s: invalid target program offset %llu\n",
10562	map->name, (unsigned long long)sym->st_value);
10563	return -LIBBPF_ERRNO__FORMAT;
10564	}
10565	insn_idx = sym->st_value / BPF_INSN_SZ;
10566
10567	type = btf__type_by_id(btf, st_ops->type_id);
10568	member = find_member_by_offset(type, moff * 8);
10569	if (!member) {
10570	pr_warn("struct_ops reloc %s: cannot find member at moff %u\n",
10571	map->name, moff);
10572	return -EINVAL;
10573	}
10574	member_idx = member - btf_members(type);
10575	name = btf__name_by_offset(btf, member->name_off);
10576
10577	if (!resolve_func_ptr(btf, member->type, NULL)) {
10578	pr_warn("struct_ops reloc %s: cannot relocate non func ptr %s\n",
10579	map->name, name);
10580	return -EINVAL;
10581	}
10582
10583	prog = find_prog_by_sec_insn(obj, shdr_idx, insn_idx);
10584	if (!prog) {
10585	pr_warn("struct_ops reloc %s: cannot find prog at shdr_idx %u to relocate func ptr %s\n",
10586	map->name, shdr_idx, name);
10587	return -EINVAL;
10588	}
10589
10590	/* prevent the use of BPF prog with invalid type */
10591	if (prog->type != BPF_PROG_TYPE_STRUCT_OPS) {
10592	pr_warn("struct_ops reloc %s: prog %s is not struct_ops BPF program\n",
10593	map->name, prog->name);
10594	return -EINVAL;
10595	}
10596
10597	st_ops->progs[member_idx] = prog;
10598
10599	/* st_ops->data will be exposed to users, being returned by
10600	* bpf_map__initial_value() as a pointer to the shadow
10601	* type. All function pointers in the original struct type
10602	* should be converted to a pointer to struct bpf_program
10603	* in the shadow type.
10604	*/
10605	((struct bpf_program *)(st_ops->data + moff)) = prog;
10606	}
10607
10608	return 0;
10609	}
10610
10611	#define BTF_TRACE_PREFIX "btf_trace_"
10612	#define BTF_LSM_PREFIX "bpf_lsm_"
10613	#define BTF_ITER_PREFIX "bpf_iter_"
10614	#define BTF_MAX_NAME_SIZE 128
10615
10616	void btf_get_kernel_prefix_kind(enum bpf_attach_type attach_type,
10617	const char *prefix, int kind)
10618	{
10619	switch (attach_type) {
10620	case BPF_TRACE_RAW_TP:
10621	*prefix = BTF_TRACE_PREFIX;
10622	*kind = BTF_KIND_TYPEDEF;
10623	break;
10624	case BPF_LSM_MAC:
10625	case BPF_LSM_CGROUP:
10626	*prefix = BTF_LSM_PREFIX;
10627	*kind = BTF_KIND_FUNC;
10628	break;
10629	case BPF_TRACE_ITER:
10630	*prefix = BTF_ITER_PREFIX;
10631	*kind = BTF_KIND_FUNC;
10632	break;
10633	default:
10634	*prefix = "";
10635	*kind = BTF_KIND_FUNC;
10636	}
10637	}
10638
10639	static int find_btf_by_prefix_kind(const struct btf btf, const char prefix,
10640	const char *name, __u32 kind)
10641	{
10642	char btf_type_name[BTF_MAX_NAME_SIZE];
10643	int ret;
10644
10645	ret = snprintf(btf_type_name, sizeof(btf_type_name),
10646	"%s%s", prefix, name);
10647	/* snprintf returns the number of characters written excluding the
10648	* terminating null. So, if >= BTF_MAX_NAME_SIZE are written, it
10649	* indicates truncation.
10650	*/
10651	if (ret < 0 \|\| ret >= sizeof(btf_type_name))
10652	return -ENAMETOOLONG;
10653	return btf__find_by_name_kind(btf, btf_type_name, kind);
10654	}
10655
10656	static inline int find_attach_btf_id(struct btf btf, const char name,
10657	enum bpf_attach_type attach_type)
10658	{
10659	const char *prefix;
10660	int kind;
10661
10662	btf_get_kernel_prefix_kind(attach_type, &prefix, &kind);
10663	return find_btf_by_prefix_kind(btf, prefix, name, kind);
10664	}
10665
10666	int libbpf_find_vmlinux_btf_id(const char *name,
10667	enum bpf_attach_type attach_type)
10668	{
10669	struct btf *btf;
10670	int err;
10671
10672	btf = btf__load_vmlinux_btf();
10673	err = libbpf_get_error(btf);
10674	if (err) {
10675	pr_warn("vmlinux BTF is not found\n");
10676	return libbpf_err(err);
10677	}
10678
10679	err = find_attach_btf_id(btf, name, attach_type);
10680	if (err <= 0)
10681	pr_warn("%s is not found in vmlinux BTF\n", name);
10682
10683	btf__free(btf);
10684	return libbpf_err(err);
10685	}
10686
10687	static int libbpf_find_prog_btf_id(const char *name, __u32 attach_prog_fd, int token_fd)
10688	{
10689	struct bpf_prog_info info;
10690	__u32 info_len = sizeof(info);
10691	struct btf *btf;
10692	int err;
10693
10694	memset(&info, 0, info_len);
10695	err = bpf_prog_get_info_by_fd(attach_prog_fd, &info, &info_len);
10696	if (err) {
10697	pr_warn("failed bpf_prog_get_info_by_fd for FD %u: %s\n",
10698	attach_prog_fd, errstr(err));
10699	return err;
10700	}
10701
10702	err = -EINVAL;
10703	if (!info.btf_id) {
10704	pr_warn("The target program doesn't have BTF\n");
10705	goto out;
10706	}
10707	btf = btf_load_from_kernel(info.btf_id, NULL, token_fd);
10708	err = libbpf_get_error(btf);
10709	if (err) {
10710	pr_warn("Failed to get BTF %u of the program: %s\n", info.btf_id, errstr(err));
10711	goto out;
10712	}
10713	err = btf__find_by_name_kind(btf, name, BTF_KIND_FUNC);
10714	btf__free(btf);
10715	if (err <= 0) {
10716	pr_warn("%s is not found in prog's BTF\n", name);
10717	goto out;
10718	}
10719	out:
10720	return err;
10721	}
10722
10723	static int find_kernel_btf_id(struct bpf_object obj, const char attach_name,
10724	enum bpf_attach_type attach_type,
10725	int btf_obj_fd, int btf_type_id)
10726	{
10727	int ret, i, mod_len = 0;
10728	const char fn_name, mod_name = NULL;
10729
10730	fn_name = strchr(attach_name, ':');
10731	if (fn_name) {
10732	mod_name = attach_name;
10733	mod_len = fn_name - mod_name;
10734	fn_name++;
10735	}
10736
10737	if (!mod_name \|\| strncmp(mod_name, "vmlinux", mod_len) == 0) {
10738	ret = find_attach_btf_id(obj->btf_vmlinux,
10739	mod_name ? fn_name : attach_name,
10740	attach_type);
10741	if (ret > 0) {
10742	btf_obj_fd = 0; / vmlinux BTF */
10743	*btf_type_id = ret;
10744	return 0;
10745	}
10746	if (ret != -ENOENT)
10747	return ret;
10748	}
10749
10750	ret = load_module_btfs(obj);
10751	if (ret)
10752	return ret;
10753
10754	for (i = 0; i < obj->btf_module_cnt; i++) {
10755	const struct module_btf *mod = &obj->btf_modules[i];
10756
10757	if (mod_name && strncmp(mod->name, mod_name, mod_len) != 0)
10758	continue;
10759
10760	ret = find_attach_btf_id(mod->btf,
10761	mod_name ? fn_name : attach_name,
10762	attach_type);
10763	if (ret > 0) {
10764	*btf_obj_fd = mod->fd;
10765	*btf_type_id = ret;
10766	return 0;
10767	}
10768	if (ret == -ENOENT)
10769	continue;
10770
10771	return ret;
10772	}
10773
10774	return -ESRCH;
10775	}
10776
10777	static int libbpf_find_attach_btf_id(struct bpf_program prog, const char attach_name,
10778	int btf_obj_fd, int btf_type_id)
10779	{
10780	enum bpf_attach_type attach_type = prog->expected_attach_type;
10781	__u32 attach_prog_fd = prog->attach_prog_fd;
10782	int err = 0;
10783
10784	/* BPF program's BTF ID */
10785	if (prog->type == BPF_PROG_TYPE_EXT \|\| attach_prog_fd) {
10786	if (!attach_prog_fd) {
10787	pr_warn("prog '%s': attach program FD is not set\n", prog->name);
10788	return -EINVAL;
10789	}
10790	err = libbpf_find_prog_btf_id(attach_name, attach_prog_fd, prog->obj->token_fd);
10791	if (err < 0) {
10792	pr_warn("prog '%s': failed to find BPF program (FD %u) BTF ID for '%s': %s\n",
10793	prog->name, attach_prog_fd, attach_name, errstr(err));
10794	return err;
10795	}
10796	*btf_obj_fd = 0;
10797	*btf_type_id = err;
10798	return 0;
10799	}
10800
10801	/* kernel/module BTF ID */
10802	if (prog->obj->gen_loader) {
10803	bpf_gen__record_attach_target(prog->obj->gen_loader, attach_name, attach_type);
10804	*btf_obj_fd = 0;
10805	*btf_type_id = 1;
10806	} else {
10807	err = find_kernel_btf_id(prog->obj, attach_name,
10808	attach_type, btf_obj_fd,
10809	btf_type_id);
10810	}
10811	if (err) {
10812	pr_warn("prog '%s': failed to find kernel BTF type ID of '%s': %s\n",
10813	prog->name, attach_name, errstr(err));
10814	return err;
10815	}
10816	return 0;
10817	}
10818
10819	int libbpf_attach_type_by_name(const char *name,
10820	enum bpf_attach_type *attach_type)
10821	{
10822	char *type_names;
10823	const struct bpf_sec_def *sec_def;
10824
10825	if (!name)
10826	return libbpf_err(-EINVAL);
10827
10828	sec_def = find_sec_def(name);
10829	if (!sec_def) {
10830	pr_debug("failed to guess attach type based on ELF section name '%s'\n", name);
10831	type_names = libbpf_get_type_names(true);
10832	if (type_names != NULL) {
10833	pr_debug("attachable section(type) names are:%s\n", type_names);
10834	free(type_names);
10835	}
10836
10837	return libbpf_err(-EINVAL);
10838	}
10839
10840	if (sec_def->prog_prepare_load_fn != libbpf_prepare_prog_load)
10841	return libbpf_err(-EINVAL);
10842	if (!(sec_def->cookie & SEC_ATTACHABLE))
10843	return libbpf_err(-EINVAL);
10844
10845	*attach_type = sec_def->expected_attach_type;
10846	return 0;
10847	}
10848
10849	int bpf_map__fd(const struct bpf_map *map)
10850	{
10851	if (!map)
10852	return libbpf_err(-EINVAL);
10853	if (!map_is_created(map))
10854	return -1;
10855	return map->fd;
10856	}
10857
10858	static bool map_uses_real_name(const struct bpf_map *map)
10859	{
10860	/* Since libbpf started to support custom .data.* and .rodata.* maps,
10861	* their user-visible name differs from kernel-visible name. Users see
10862	* such map's corresponding ELF section name as a map name.
10863	* This check distinguishes .data/.rodata from .data.* and .rodata.*
10864	* maps to know which name has to be returned to the user.
10865	* Map name of the custom .percpu.* maps might be truncated to
10866	* BPF_OBJ_NAME_LEN-1 chars in internal_map_name(). Hence, percpu data
10867	* maps must use real name for their user-visible name.
10868	*/
10869	if (map->libbpf_type == LIBBPF_MAP_DATA && strcmp(map->real_name, DATA_SEC) != 0)
10870	return true;
10871	if (map->libbpf_type == LIBBPF_MAP_RODATA && strcmp(map->real_name, RODATA_SEC) != 0)
10872	return true;
10873	if (map->libbpf_type == LIBBPF_MAP_PERCPU)
10874	return true;
10875	return false;
10876	}
10877
10878	const char bpf_map__name(const struct bpf_map map)
10879	{
10880	if (!map)
10881	return NULL;
10882
10883	if (map_uses_real_name(map))
10884	return map->real_name;
10885
10886	return map->name;
10887	}
10888
10889	enum bpf_map_type bpf_map__type(const struct bpf_map *map)
10890	{
10891	return map->def.type;
10892	}
10893
10894	int bpf_map__set_type(struct bpf_map *map, enum bpf_map_type type)
10895	{
10896	if (map_is_created(map))
10897	return libbpf_err(-EBUSY);
10898	map->def.type = type;
10899	return 0;
10900	}
10901
10902	__u32 bpf_map__map_flags(const struct bpf_map *map)
10903	{
10904	return map->def.map_flags;
10905	}
10906
10907	int bpf_map__set_map_flags(struct bpf_map *map, __u32 flags)
10908	{
10909	if (map_is_created(map))
10910	return libbpf_err(-EBUSY);
10911	map->def.map_flags = flags;
10912	return 0;
10913	}
10914
10915	__u64 bpf_map__map_extra(const struct bpf_map *map)
10916	{
10917	return map->map_extra;
10918	}
10919
10920	int bpf_map__set_map_extra(struct bpf_map *map, __u64 map_extra)
10921	{
10922	if (map_is_created(map))
10923	return libbpf_err(-EBUSY);
10924	map->map_extra = map_extra;
10925	return 0;
10926	}
10927
10928	__u32 bpf_map__numa_node(const struct bpf_map *map)
10929	{
10930	return map->numa_node;
10931	}
10932
10933	int bpf_map__set_numa_node(struct bpf_map *map, __u32 numa_node)
10934	{
10935	if (map_is_created(map))
10936	return libbpf_err(-EBUSY);
10937	map->numa_node = numa_node;
10938	return 0;
10939	}
10940
10941	__u32 bpf_map__key_size(const struct bpf_map *map)
10942	{
10943	return map->def.key_size;
10944	}
10945
10946	int bpf_map__set_key_size(struct bpf_map *map, __u32 size)
10947	{
10948	if (map_is_created(map))
10949	return libbpf_err(-EBUSY);
10950	map->def.key_size = size;
10951	return 0;
10952	}
10953
10954	__u32 bpf_map__value_size(const struct bpf_map *map)
10955	{
10956	return map->def.value_size;
10957	}
10958
10959	static int map_btf_datasec_resize(struct bpf_map *map, __u32 size)
10960	{
10961	struct btf *btf;
10962	struct btf_type datasec_type, var_type;
10963	struct btf_var_secinfo *var;
10964	const struct btf_type *array_type;
10965	const struct btf_array *array;
10966	int vlen, element_sz, new_array_id;
10967	__u32 nr_elements;
10968
10969	/* check btf existence */
10970	btf = bpf_object__btf(map->obj);
10971	if (!btf)
10972	return -ENOENT;
10973
10974	/* verify map is datasec */
10975	datasec_type = btf_type_by_id(btf, bpf_map__btf_value_type_id(map));
10976	if (!btf_is_datasec(datasec_type)) {
10977	pr_warn("map '%s': cannot be resized, map value type is not a datasec\n",
10978	bpf_map__name(map));
10979	return -EINVAL;
10980	}
10981
10982	/* verify datasec has at least one var */
10983	vlen = btf_vlen(datasec_type);
10984	if (vlen == 0) {
10985	pr_warn("map '%s': cannot be resized, map value datasec is empty\n",
10986	bpf_map__name(map));
10987	return -EINVAL;
10988	}
10989
10990	/* verify last var in the datasec is an array */
10991	var = &btf_var_secinfos(datasec_type)[vlen - 1];
10992	var_type = btf_type_by_id(btf, var->type);
10993	array_type = skip_mods_and_typedefs(btf, var_type->type, NULL);
10994	if (!btf_is_array(array_type)) {
10995	pr_warn("map '%s': cannot be resized, last var must be an array\n",
10996	bpf_map__name(map));
10997	return -EINVAL;
10998	}
10999
11000	/* verify request size aligns with array */
11001	array = btf_array(array_type);
11002	element_sz = btf__resolve_size(btf, array->type);
11003	if (element_sz <= 0 \|\| (size - var->offset) % element_sz != 0) {
11004	pr_warn("map '%s': cannot be resized, element size (%d) doesn't align with new total size (%u)\n",
11005	bpf_map__name(map), element_sz, size);
11006	return -EINVAL;
11007	}
11008
11009	/* create a new array based on the existing array, but with new length */
11010	nr_elements = (size - var->offset) / element_sz;
11011	new_array_id = btf__add_array(btf, array->index_type, array->type, nr_elements);
11012	if (new_array_id < 0)
11013	return new_array_id;
11014
11015	/* adding a new btf type invalidates existing pointers to btf objects,
11016	* so refresh pointers before proceeding
11017	*/
11018	datasec_type = btf_type_by_id(btf, map->btf_value_type_id);
11019	var = &btf_var_secinfos(datasec_type)[vlen - 1];
11020	var_type = btf_type_by_id(btf, var->type);
11021
11022	/* finally update btf info */
11023	datasec_type->size = size;
11024	var->size = size - var->offset;
11025	var_type->type = new_array_id;
11026
11027	return 0;
11028	}
11029
11030	int bpf_map__set_value_size(struct bpf_map *map, __u32 size)
11031	{
11032	if (map_is_created(map))
11033	return libbpf_err(-EBUSY);
11034
11035	if (map->mmaped) {
11036	size_t mmap_old_sz, mmap_new_sz;
11037	int err;
11038
11039	if (map->def.type != BPF_MAP_TYPE_ARRAY &&
11040	map->def.type != BPF_MAP_TYPE_PERCPU_ARRAY)
11041	return libbpf_err(-EOPNOTSUPP);
11042
11043	mmap_old_sz = bpf_map_mmap_sz(map);
11044	mmap_new_sz = array_map_mmap_sz(size, map->def.max_entries);
11045	err = bpf_map_mmap_resize(map, mmap_old_sz, mmap_new_sz);
11046	if (err) {
11047	pr_warn("map '%s': failed to resize memory-mapped region: %s\n",
11048	bpf_map__name(map), errstr(err));
11049	return libbpf_err(err);
11050	}
11051	err = map_btf_datasec_resize(map, size);
11052	if (err && err != -ENOENT) {
11053	pr_warn("map '%s': failed to adjust resized BTF, clearing BTF key/value info: %s\n",
11054	bpf_map__name(map), errstr(err));
11055	map->btf_value_type_id = 0;
11056	map->btf_key_type_id = 0;
11057	}
11058	}
11059
11060	map->def.value_size = size;
11061	return 0;
11062	}
11063
11064	__u32 bpf_map__btf_key_type_id(const struct bpf_map *map)
11065	{
11066	return map ? map->btf_key_type_id : 0;
11067	}
11068
11069	__u32 bpf_map__btf_value_type_id(const struct bpf_map *map)
11070	{
11071	return map ? map->btf_value_type_id : 0;
11072	}
11073
11074	int bpf_map__set_initial_value(struct bpf_map *map,
11075	const void *data, size_t size)
11076	{
11077	size_t actual_sz;
11078
11079	if (map_is_created(map))
11080	return libbpf_err(-EBUSY);
11081
11082	if (!map->mmaped \|\| map->libbpf_type == LIBBPF_MAP_KCONFIG)
11083	return libbpf_err(-EINVAL);
11084
11085	if (map->def.type == BPF_MAP_TYPE_ARENA)
11086	actual_sz = map->obj->arena_data_sz;
11087	else
11088	actual_sz = map->def.value_size;
11089	if (size != actual_sz)
11090	return libbpf_err(-EINVAL);
11091
11092	memcpy(map->mmaped, data, size);
11093	return 0;
11094	}
11095
11096	void bpf_map__initial_value(const struct bpf_map map, size_t *psize)
11097	{
11098	if (bpf_map__is_struct_ops(map)) {
11099	if (psize)
11100	*psize = map->def.value_size;
11101	return map->st_ops->data;
11102	}
11103
11104	if (!map->mmaped)
11105	return NULL;
11106
11107	if (map->def.type == BPF_MAP_TYPE_ARENA)
11108	*psize = map->obj->arena_data_sz;
11109	else
11110	*psize = map->def.value_size;
11111
11112	return map->mmaped;
11113	}
11114
11115	bool bpf_map__is_internal(const struct bpf_map *map)
11116	{
11117	return map->libbpf_type != LIBBPF_MAP_UNSPEC;
11118	}
11119
11120	__u32 bpf_map__ifindex(const struct bpf_map *map)
11121	{
11122	return map->map_ifindex;
11123	}
11124
11125	int bpf_map__set_ifindex(struct bpf_map *map, __u32 ifindex)
11126	{
11127	if (map_is_created(map))
11128	return libbpf_err(-EBUSY);
11129	map->map_ifindex = ifindex;
11130	return 0;
11131	}
11132
11133	int bpf_map__set_inner_map_fd(struct bpf_map *map, int fd)
11134	{
11135	if (!bpf_map_type__is_map_in_map(map->def.type)) {
11136	pr_warn("error: unsupported map type\n");
11137	return libbpf_err(-EINVAL);
11138	}
11139	if (map->inner_map_fd != -1) {
11140	pr_warn("error: inner_map_fd already specified\n");
11141	return libbpf_err(-EINVAL);
11142	}
11143	if (map->inner_map) {
11144	bpf_map__destroy(map->inner_map);
11145	zfree(&map->inner_map);
11146	}
11147	map->inner_map_fd = fd;
11148	return 0;
11149	}
11150
11151	int bpf_map__set_exclusive_program(struct bpf_map map, struct bpf_program prog)
11152	{
11153	if (map_is_created(map)) {
11154	pr_warn("exclusive programs must be set before map creation\n");
11155	return libbpf_err(-EINVAL);
11156	}
11157
11158	if (map->obj != prog->obj) {
11159	pr_warn("excl_prog and map must be from the same bpf object\n");
11160	return libbpf_err(-EINVAL);
11161	}
11162
11163	map->excl_prog = prog;
11164	return 0;
11165	}
11166
11167	struct bpf_program bpf_map__exclusive_program(struct bpf_map map)
11168	{
11169	return map->excl_prog;
11170	}
11171
11172	static struct bpf_map *
11173	__bpf_map__iter(const struct bpf_map m, const struct bpf_object obj, int i)
11174	{
11175	ssize_t idx;
11176	struct bpf_map s, e;
11177
11178	if (!obj \|\| !obj->maps)
11179	return errno = EINVAL, NULL;
11180
11181	s = obj->maps;
11182	e = obj->maps + obj->nr_maps;
11183
11184	if ((m < s) \|\| (m >= e)) {
11185	pr_warn("error in %s: map handler doesn't belong to object\n",
11186	__func__);
11187	return errno = EINVAL, NULL;
11188	}
11189
11190	idx = (m - obj->maps) + i;
11191	if (idx >= obj->nr_maps \|\| idx < 0)
11192	return NULL;
11193	return &obj->maps[idx];
11194	}
11195
11196	struct bpf_map *
11197	bpf_object__next_map(const struct bpf_object obj, const struct bpf_map prev)
11198	{
11199	if (prev == NULL && obj != NULL)
11200	return obj->maps;
11201
11202	return __bpf_map__iter(prev, obj, 1);
11203	}
11204
11205	struct bpf_map *
11206	bpf_object__prev_map(const struct bpf_object obj, const struct bpf_map next)
11207	{
11208	if (next == NULL && obj != NULL) {
11209	if (!obj->nr_maps)
11210	return NULL;
11211	return obj->maps + obj->nr_maps - 1;
11212	}
11213
11214	return __bpf_map__iter(next, obj, -1);
11215	}
11216
11217	struct bpf_map *
11218	bpf_object__find_map_by_name(const struct bpf_object obj, const char name)
11219	{
11220	struct bpf_map *pos;
11221
11222	bpf_object__for_each_map(pos, obj) {
11223	/* if it's a special internal map name (which always starts
11224	* with dot) then check if that special name matches the
11225	* real map name (ELF section name)
11226	*/
11227	if (name[0] == '.') {
11228	if (pos->real_name && strcmp(pos->real_name, name) == 0)
11229	return pos;
11230	continue;
11231	}
11232	/* otherwise map name has to be an exact match */
11233	if (map_uses_real_name(pos)) {
11234	if (strcmp(pos->real_name, name) == 0)
11235	return pos;
11236	continue;
11237	}
11238	if (strcmp(pos->name, name) == 0)
11239	return pos;
11240	}
11241	return errno = ENOENT, NULL;
11242	}
11243
11244	int
11245	bpf_object__find_map_fd_by_name(const struct bpf_object obj, const char name)
11246	{
11247	return bpf_map__fd(bpf_object__find_map_by_name(obj, name));
11248	}
11249
11250	static int validate_map_op(const struct bpf_map *map, size_t key_sz,
11251	size_t value_sz, bool check_value_sz, __u64 flags)
11252	{
11253	if (!map_is_created(map)) /* map is not yet created */
11254	return -ENOENT;
11255
11256	if (map->def.key_size != key_sz) {
11257	pr_warn("map '%s': unexpected key size %zu provided, expected %u\n",
11258	map->name, key_sz, map->def.key_size);
11259	return -EINVAL;
11260	}
11261
11262	if (map->fd < 0) {
11263	pr_warn("map '%s': can't use BPF map without FD (was it created?)\n", map->name);
11264	return -EINVAL;
11265	}
11266
11267	if (!check_value_sz)
11268	return 0;
11269
11270	switch (map->def.type) {
11271	case BPF_MAP_TYPE_PERCPU_ARRAY:
11272	case BPF_MAP_TYPE_PERCPU_HASH:
11273	case BPF_MAP_TYPE_LRU_PERCPU_HASH:
11274	case BPF_MAP_TYPE_PERCPU_CGROUP_STORAGE: {
11275	int num_cpu = libbpf_num_possible_cpus();
11276	size_t elem_sz = roundup(map->def.value_size, 8);
11277
11278	if (flags & (BPF_F_CPU \| BPF_F_ALL_CPUS)) {
11279	if ((flags & BPF_F_CPU) && (flags & BPF_F_ALL_CPUS)) {
11280	pr_warn("map '%s': BPF_F_CPU and BPF_F_ALL_CPUS are mutually exclusive\n",
11281	map->name);
11282	return -EINVAL;
11283	}
11284	if (map->def.value_size != value_sz) {
11285	pr_warn("map '%s': unexpected value size %zu provided for either BPF_F_CPU or BPF_F_ALL_CPUS, expected %u\n",
11286	map->name, value_sz, map->def.value_size);
11287	return -EINVAL;
11288	}
11289	break;
11290	}
11291
11292	if (value_sz != num_cpu * elem_sz) {
11293	pr_warn("map '%s': unexpected value size %zu provided for per-CPU map, expected %d * %zu = %zu\n",
11294	map->name, value_sz, num_cpu, elem_sz, num_cpu * elem_sz);
11295	return -EINVAL;
11296	}
11297	break;
11298	}
11299	default:
11300	if (map->def.value_size != value_sz) {
11301	pr_warn("map '%s': unexpected value size %zu provided, expected %u\n",
11302	map->name, value_sz, map->def.value_size);
11303	return -EINVAL;
11304	}
11305	break;
11306	}
11307	return 0;
11308	}
11309
11310	int bpf_map__lookup_elem(const struct bpf_map *map,
11311	const void *key, size_t key_sz,
11312	void *value, size_t value_sz, __u64 flags)
11313	{
11314	int err;
11315
11316	err = validate_map_op(map, key_sz, value_sz, true, flags);
11317	if (err)
11318	return libbpf_err(err);
11319
11320	return bpf_map_lookup_elem_flags(map->fd, key, value, flags);
11321	}
11322
11323	int bpf_map__update_elem(const struct bpf_map *map,
11324	const void *key, size_t key_sz,
11325	const void *value, size_t value_sz, __u64 flags)
11326	{
11327	int err;
11328
11329	err = validate_map_op(map, key_sz, value_sz, true, flags);
11330	if (err)
11331	return libbpf_err(err);
11332
11333	return bpf_map_update_elem(map->fd, key, value, flags);
11334	}
11335
11336	int bpf_map__delete_elem(const struct bpf_map *map,
11337	const void *key, size_t key_sz, __u64 flags)
11338	{
11339	int err;
11340
11341	err = validate_map_op(map, key_sz, 0, false /* check_value_sz */, flags);
11342	if (err)
11343	return libbpf_err(err);
11344
11345	return bpf_map_delete_elem_flags(map->fd, key, flags);
11346	}
11347
11348	int bpf_map__lookup_and_delete_elem(const struct bpf_map *map,
11349	const void *key, size_t key_sz,
11350	void *value, size_t value_sz, __u64 flags)
11351	{
11352	int err;
11353
11354	err = validate_map_op(map, key_sz, value_sz, true, flags);
11355	if (err)
11356	return libbpf_err(err);
11357
11358	return bpf_map_lookup_and_delete_elem_flags(map->fd, key, value, flags);
11359	}
11360
11361	int bpf_map__get_next_key(const struct bpf_map *map,
11362	const void cur_key, void next_key, size_t key_sz)
11363	{
11364	int err;
11365
11366	err = validate_map_op(map, key_sz, 0, false /* check_value_sz */, 0);
11367	if (err)
11368	return libbpf_err(err);
11369
11370	return bpf_map_get_next_key(map->fd, cur_key, next_key);
11371	}
11372
11373	long libbpf_get_error(const void *ptr)
11374	{
11375	if (!IS_ERR_OR_NULL(ptr))
11376	return 0;
11377
11378	if (IS_ERR(ptr))
11379	errno = -PTR_ERR(ptr);
11380
11381	/* If ptr == NULL, then errno should be already set by the failing
11382	* API, because libbpf never returns NULL on success and it now always
11383	* sets errno on error. So no extra errno handling for ptr == NULL
11384	* case.
11385	*/
11386	return -errno;
11387	}
11388
11389	/* Replace link's underlying BPF program with the new one */
11390	int bpf_link__update_program(struct bpf_link link, struct bpf_program prog)
11391	{
11392	int ret;
11393	int prog_fd = bpf_program__fd(prog);
11394
11395	if (prog_fd < 0) {
11396	pr_warn("prog '%s': can't use BPF program without FD (was it loaded?)\n",
11397	prog->name);
11398	return libbpf_err(-EINVAL);
11399	}
11400
11401	ret = bpf_link_update(bpf_link__fd(link), prog_fd, NULL);
11402	return libbpf_err_errno(ret);
11403	}
11404
11405	/* Release "ownership" of underlying BPF resource (typically, BPF program
11406	* attached to some BPF hook, e.g., tracepoint, kprobe, etc). Disconnected
11407	* link, when destructed through bpf_link__destroy() call won't attempt to
11408	* detach/unregisted that BPF resource. This is useful in situations where,
11409	* say, attached BPF program has to outlive userspace program that attached it
11410	* in the system. Depending on type of BPF program, though, there might be
11411	* additional steps (like pinning BPF program in BPF FS) necessary to ensure
11412	* exit of userspace program doesn't trigger automatic detachment and clean up
11413	* inside the kernel.
11414	*/
11415	void bpf_link__disconnect(struct bpf_link *link)
11416	{
11417	link->disconnected = true;
11418	}
11419
11420	int bpf_link__destroy(struct bpf_link *link)
11421	{
11422	int err = 0;
11423
11424	if (IS_ERR_OR_NULL(link))
11425	return 0;
11426
11427	if (!link->disconnected && link->detach)
11428	err = link->detach(link);
11429	if (link->pin_path)
11430	free(link->pin_path);
11431	if (link->dealloc)
11432	link->dealloc(link);
11433	else
11434	free(link);
11435
11436	return libbpf_err(err);
11437	}
11438
11439	int bpf_link__fd(const struct bpf_link *link)
11440	{
11441	return link->fd;
11442	}
11443
11444	const char bpf_link__pin_path(const struct bpf_link link)
11445	{
11446	return link->pin_path;
11447	}
11448
11449	static int bpf_link__detach_fd(struct bpf_link *link)
11450	{
11451	return libbpf_err_errno(close(link->fd));
11452	}
11453
11454	struct bpf_link bpf_link__open(const char path)
11455	{
11456	struct bpf_link *link;
11457	int fd;
11458
11459	fd = bpf_obj_get(path);
11460	if (fd < 0) {
11461	fd = -errno;
11462	pr_warn("failed to open link at %s: %d\n", path, fd);
11463	return libbpf_err_ptr(fd);
11464	}
11465
11466	link = calloc(1, sizeof(*link));
11467	if (!link) {
11468	close(fd);
11469	return libbpf_err_ptr(-ENOMEM);
11470	}
11471	link->detach = &bpf_link__detach_fd;
11472	link->fd = fd;
11473
11474	link->pin_path = strdup(path);
11475	if (!link->pin_path) {
11476	bpf_link__destroy(link);
11477	return libbpf_err_ptr(-ENOMEM);
11478	}
11479
11480	return link;
11481	}
11482
11483	int bpf_link__detach(struct bpf_link *link)
11484	{
11485	return bpf_link_detach(link->fd) ? -errno : 0;
11486	}
11487
11488	int bpf_link__pin(struct bpf_link link, const char path)
11489	{
11490	int err;
11491
11492	if (link->pin_path)
11493	return libbpf_err(-EBUSY);
11494	err = make_parent_dir(path);
11495	if (err)
11496	return libbpf_err(err);
11497	err = check_path(path);
11498	if (err)
11499	return libbpf_err(err);
11500
11501	link->pin_path = strdup(path);
11502	if (!link->pin_path)
11503	return libbpf_err(-ENOMEM);
11504
11505	if (bpf_obj_pin(link->fd, link->pin_path)) {
11506	err = -errno;
11507	zfree(&link->pin_path);
11508	return libbpf_err(err);
11509	}
11510
11511	pr_debug("link fd=%d: pinned at %s\n", link->fd, link->pin_path);
11512	return 0;
11513	}
11514
11515	int bpf_link__unpin(struct bpf_link *link)
11516	{
11517	int err;
11518
11519	if (!link->pin_path)
11520	return libbpf_err(-EINVAL);
11521
11522	err = unlink(link->pin_path);
11523	if (err != 0)
11524	return -errno;
11525
11526	pr_debug("link fd=%d: unpinned from %s\n", link->fd, link->pin_path);
11527	zfree(&link->pin_path);
11528	return 0;
11529	}
11530
11531	struct bpf_link_perf {
11532	struct bpf_link link;
11533	int perf_event_fd;
11534	/* legacy kprobe support: keep track of probe identifier and type */
11535	char *legacy_probe_name;
11536	bool legacy_is_kprobe;
11537	bool legacy_is_retprobe;
11538	};
11539
11540	static int remove_kprobe_event_legacy(const char *probe_name, bool retprobe);
11541	static int remove_uprobe_event_legacy(const char *probe_name, bool retprobe);
11542
11543	static int bpf_link_perf_detach(struct bpf_link *link)
11544	{
11545	struct bpf_link_perf *perf_link = container_of(link, struct bpf_link_perf, link);
11546	int err = 0;
11547
11548	if (ioctl(perf_link->perf_event_fd, PERF_EVENT_IOC_DISABLE, 0) < 0)
11549	err = -errno;
11550
11551	if (perf_link->perf_event_fd != link->fd)
11552	close(perf_link->perf_event_fd);
11553	close(link->fd);
11554
11555	/* legacy uprobe/kprobe needs to be removed after perf event fd closure */
11556	if (perf_link->legacy_probe_name) {
11557	if (perf_link->legacy_is_kprobe) {
11558	err = remove_kprobe_event_legacy(perf_link->legacy_probe_name,
11559	perf_link->legacy_is_retprobe);
11560	} else {
11561	err = remove_uprobe_event_legacy(perf_link->legacy_probe_name,
11562	perf_link->legacy_is_retprobe);
11563	}
11564	}
11565
11566	return err;
11567	}
11568
11569	static void bpf_link_perf_dealloc(struct bpf_link *link)
11570	{
11571	struct bpf_link_perf *perf_link = container_of(link, struct bpf_link_perf, link);
11572
11573	free(perf_link->legacy_probe_name);
11574	free(perf_link);
11575	}
11576
11577	struct bpf_link bpf_program__attach_perf_event_opts(const struct bpf_program prog, int pfd,
11578	const struct bpf_perf_event_opts *opts)
11579	{
11580	struct bpf_link_perf *link;
11581	int prog_fd, link_fd = -1, err;
11582	bool force_ioctl_attach;
11583
11584	if (!OPTS_VALID(opts, bpf_perf_event_opts))
11585	return libbpf_err_ptr(-EINVAL);
11586
11587	if (pfd < 0) {
11588	pr_warn("prog '%s': invalid perf event FD %d\n",
11589	prog->name, pfd);
11590	return libbpf_err_ptr(-EINVAL);
11591	}
11592	prog_fd = bpf_program__fd(prog);
11593	if (prog_fd < 0) {
11594	pr_warn("prog '%s': can't attach BPF program without FD (was it loaded?)\n",
11595	prog->name);
11596	return libbpf_err_ptr(-EINVAL);
11597	}
11598
11599	link = calloc(1, sizeof(*link));
11600	if (!link)
11601	return libbpf_err_ptr(-ENOMEM);
11602	link->link.detach = &bpf_link_perf_detach;
11603	link->link.dealloc = &bpf_link_perf_dealloc;
11604	link->perf_event_fd = pfd;
11605
11606	force_ioctl_attach = OPTS_GET(opts, force_ioctl_attach, false);
11607	if (kernel_supports(prog->obj, FEAT_PERF_LINK) && !force_ioctl_attach) {
11608	DECLARE_LIBBPF_OPTS(bpf_link_create_opts, link_opts,
11609	.perf_event.bpf_cookie = OPTS_GET(opts, bpf_cookie, 0));
11610
11611	link_fd = bpf_link_create(prog_fd, pfd, BPF_PERF_EVENT, &link_opts);
11612	if (link_fd < 0) {
11613	err = -errno;
11614	pr_warn("prog '%s': failed to create BPF link for perf_event FD %d: %s\n",
11615	prog->name, pfd, errstr(err));
11616	goto err_out;
11617	}
11618	link->link.fd = link_fd;
11619	} else {
11620	if (OPTS_GET(opts, bpf_cookie, 0)) {
11621	pr_warn("prog '%s': user context value is not supported\n", prog->name);
11622	err = -EOPNOTSUPP;
11623	goto err_out;
11624	}
11625
11626	if (ioctl(pfd, PERF_EVENT_IOC_SET_BPF, prog_fd) < 0) {
11627	err = -errno;
11628	pr_warn("prog '%s': failed to attach to perf_event FD %d: %s\n",
11629	prog->name, pfd, errstr(err));
11630	if (err == -EPROTO)
11631	pr_warn("prog '%s': try add PERF_SAMPLE_CALLCHAIN to or remove exclude_callchain_[kernel\|user] from pfd %d\n",
11632	prog->name, pfd);
11633	goto err_out;
11634	}
11635	link->link.fd = pfd;
11636	}
11637
11638	if (!OPTS_GET(opts, dont_enable, false)) {
11639	if (ioctl(pfd, PERF_EVENT_IOC_ENABLE, 0) < 0) {
11640	err = -errno;
11641	pr_warn("prog '%s': failed to enable perf_event FD %d: %s\n",
11642	prog->name, pfd, errstr(err));
11643	goto err_out;
11644	}
11645	}
11646
11647	return &link->link;
11648	err_out:
11649	if (link_fd >= 0)
11650	close(link_fd);
11651	free(link);
11652	return libbpf_err_ptr(err);
11653	}
11654
11655	struct bpf_link bpf_program__attach_perf_event(const struct bpf_program prog, int pfd)
11656	{
11657	return bpf_program__attach_perf_event_opts(prog, pfd, NULL);
11658	}
11659
11660	/*
11661	* this function is expected to parse integer in the range of [0, 2^31-1] from
11662	* given file using scanf format string fmt. If actual parsed value is
11663	* negative, the result might be indistinguishable from error
11664	*/
11665	static int parse_uint_from_file(const char file, const char fmt)
11666	{
11667	int err, ret;
11668	FILE *f;
11669
11670	f = fopen(file, "re");
11671	if (!f) {
11672	err = -errno;
11673	pr_debug("failed to open '%s': %s\n", file, errstr(err));
11674	return err;
11675	}
11676	err = fscanf(f, fmt, &ret);
11677	if (err != 1) {
11678	err = err == EOF ? -EIO : -errno;
11679	pr_debug("failed to parse '%s': %s\n", file, errstr(err));
11680	fclose(f);
11681	return err;
11682	}
11683	fclose(f);
11684	return ret;
11685	}
11686
11687	static int determine_kprobe_perf_type(void)
11688	{
11689	const char *file = "/sys/bus/event_source/devices/kprobe/type";
11690
11691	return parse_uint_from_file(file, "%d\n");
11692	}
11693
11694	static int determine_uprobe_perf_type(void)
11695	{
11696	const char *file = "/sys/bus/event_source/devices/uprobe/type";
11697
11698	return parse_uint_from_file(file, "%d\n");
11699	}
11700
11701	static int determine_kprobe_retprobe_bit(void)
11702	{
11703	const char *file = "/sys/bus/event_source/devices/kprobe/format/retprobe";
11704
11705	return parse_uint_from_file(file, "config:%d\n");
11706	}
11707
11708	static int determine_uprobe_retprobe_bit(void)
11709	{
11710	const char *file = "/sys/bus/event_source/devices/uprobe/format/retprobe";
11711
11712	return parse_uint_from_file(file, "config:%d\n");
11713	}
11714
11715	#define PERF_UPROBE_REF_CTR_OFFSET_BITS 32
11716	#define PERF_UPROBE_REF_CTR_OFFSET_SHIFT 32
11717
11718	static int perf_event_open_probe(bool uprobe, bool retprobe, const char *name,
11719	uint64_t offset, int pid, size_t ref_ctr_off)
11720	{
11721	const size_t attr_sz = sizeof(struct perf_event_attr);
11722	struct perf_event_attr attr;
11723	int type, pfd;
11724
11725	if ((__u64)ref_ctr_off >= (1ULL << PERF_UPROBE_REF_CTR_OFFSET_BITS))
11726	return -EINVAL;
11727
11728	memset(&attr, 0, attr_sz);
11729
11730	type = uprobe ? determine_uprobe_perf_type()
11731	: determine_kprobe_perf_type();
11732	if (type < 0) {
11733	pr_warn("failed to determine %s perf type: %s\n",
11734	uprobe ? "uprobe" : "kprobe",
11735	errstr(type));
11736	return type;
11737	}
11738	if (retprobe) {
11739	int bit = uprobe ? determine_uprobe_retprobe_bit()
11740	: determine_kprobe_retprobe_bit();
11741
11742	if (bit < 0) {
11743	pr_warn("failed to determine %s retprobe bit: %s\n",
11744	uprobe ? "uprobe" : "kprobe",
11745	errstr(bit));
11746	return bit;
11747	}
11748	attr.config \|= 1 << bit;
11749	}
11750	attr.size = attr_sz;
11751	attr.type = type;
11752	attr.config \|= (__u64)ref_ctr_off << PERF_UPROBE_REF_CTR_OFFSET_SHIFT;
11753	attr.config1 = ptr_to_u64(name); /* kprobe_func or uprobe_path */
11754	attr.config2 = offset; /* kprobe_addr or probe_offset */
11755
11756	/* pid filter is meaningful only for uprobes */
11757	pfd = syscall(__NR_perf_event_open, &attr,
11758	pid < 0 ? -1 : pid /* pid */,
11759	pid == -1 ? 0 : -1 /* cpu */,
11760	-1 /* group_fd */, PERF_FLAG_FD_CLOEXEC);
11761	return pfd >= 0 ? pfd : -errno;
11762	}
11763
11764	static int append_to_file(const char file, const char fmt, ...)
11765	{
11766	int fd, n, err = 0;
11767	va_list ap;
11768	char buf[1024];
11769
11770	va_start(ap, fmt);
11771	n = vsnprintf(buf, sizeof(buf), fmt, ap);
11772	va_end(ap);
11773
11774	if (n < 0 \|\| n >= sizeof(buf))
11775	return -EINVAL;
11776
11777	fd = open(file, O_WRONLY \| O_APPEND \| O_CLOEXEC, 0);
11778	if (fd < 0)
11779	return -errno;
11780
11781	if (write(fd, buf, n) < 0)
11782	err = -errno;
11783
11784	close(fd);
11785	return err;
11786	}
11787
11788	#define DEBUGFS "/sys/kernel/debug/tracing"
11789	#define TRACEFS "/sys/kernel/tracing"
11790
11791	static bool use_debugfs(void)
11792	{
11793	static int has_debugfs = -1;
11794
11795	if (has_debugfs < 0)
11796	has_debugfs = faccessat(AT_FDCWD, DEBUGFS, F_OK, AT_EACCESS) == 0;
11797
11798	return has_debugfs == 1;
11799	}
11800
11801	static const char *tracefs_path(void)
11802	{
11803	return use_debugfs() ? DEBUGFS : TRACEFS;
11804	}
11805
11806	static const char *tracefs_kprobe_events(void)
11807	{
11808	return use_debugfs() ? DEBUGFS"/kprobe_events" : TRACEFS"/kprobe_events";
11809	}
11810
11811	static const char *tracefs_uprobe_events(void)
11812	{
11813	return use_debugfs() ? DEBUGFS"/uprobe_events" : TRACEFS"/uprobe_events";
11814	}
11815
11816	static const char *tracefs_available_filter_functions(void)
11817	{
11818	return use_debugfs() ? DEBUGFS"/available_filter_functions"
11819	: TRACEFS"/available_filter_functions";
11820	}
11821
11822	static const char *tracefs_available_filter_functions_addrs(void)
11823	{
11824	return use_debugfs() ? DEBUGFS"/available_filter_functions_addrs"
11825	: TRACEFS"/available_filter_functions_addrs";
11826	}
11827
11828	static void gen_probe_legacy_event_name(char *buf, size_t buf_sz,
11829	const char *name, size_t offset)
11830	{
11831	static int index = 0;
11832	int i;
11833
11834	snprintf(buf, buf_sz, "libbpf_%d_%d_%s_0x%zx", getpid(),
11835	__sync_fetch_and_add(&index, 1), name, offset);
11836
11837	/* sanitize name in the probe name */
11838	for (i = 0; buf[i]; i++) {
11839	if (!isalnum(buf[i]))
11840	buf[i] = '_';
11841	}
11842	}
11843
11844	static int add_kprobe_event_legacy(const char *probe_name, bool retprobe,
11845	const char *kfunc_name, size_t offset)
11846	{
11847	return append_to_file(tracefs_kprobe_events(), "%c:%s/%s %s+0x%zx",
11848	retprobe ? 'r' : 'p',
11849	retprobe ? "kretprobes" : "kprobes",
11850	probe_name, kfunc_name, offset);
11851	}
11852
11853	static int remove_kprobe_event_legacy(const char *probe_name, bool retprobe)
11854	{
11855	return append_to_file(tracefs_kprobe_events(), "-:%s/%s",
11856	retprobe ? "kretprobes" : "kprobes", probe_name);
11857	}
11858
11859	static int determine_kprobe_perf_type_legacy(const char *probe_name, bool retprobe)
11860	{
11861	char file[256];
11862
11863	snprintf(file, sizeof(file), "%s/events/%s/%s/id",
11864	tracefs_path(), retprobe ? "kretprobes" : "kprobes", probe_name);
11865
11866	return parse_uint_from_file(file, "%d\n");
11867	}
11868
11869	static int perf_event_kprobe_open_legacy(const char *probe_name, bool retprobe,
11870	const char *kfunc_name, size_t offset, int pid)
11871	{
11872	const size_t attr_sz = sizeof(struct perf_event_attr);
11873	struct perf_event_attr attr;
11874	int type, pfd, err;
11875
11876	err = add_kprobe_event_legacy(probe_name, retprobe, kfunc_name, offset);
11877	if (err < 0) {
11878	pr_warn("failed to add legacy kprobe event for '%s+0x%zx': %s\n",
11879	kfunc_name, offset,
11880	errstr(err));
11881	return err;
11882	}
11883	type = determine_kprobe_perf_type_legacy(probe_name, retprobe);
11884	if (type < 0) {
11885	err = type;
11886	pr_warn("failed to determine legacy kprobe event id for '%s+0x%zx': %s\n",
11887	kfunc_name, offset,
11888	errstr(err));
11889	goto err_clean_legacy;
11890	}
11891
11892	memset(&attr, 0, attr_sz);
11893	attr.size = attr_sz;
11894	attr.config = type;
11895	attr.type = PERF_TYPE_TRACEPOINT;
11896
11897	pfd = syscall(__NR_perf_event_open, &attr,
11898	pid < 0 ? -1 : pid, /* pid */
11899	pid == -1 ? 0 : -1, /* cpu */
11900	-1 /* group_fd */, PERF_FLAG_FD_CLOEXEC);
11901	if (pfd < 0) {
11902	err = -errno;
11903	pr_warn("legacy kprobe perf_event_open() failed: %s\n",
11904	errstr(err));
11905	goto err_clean_legacy;
11906	}
11907	return pfd;
11908
11909	err_clean_legacy:
11910	/* Clear the newly added legacy kprobe_event */
11911	remove_kprobe_event_legacy(probe_name, retprobe);
11912	return err;
11913	}
11914
11915	static const char *arch_specific_syscall_pfx(void)
11916	{
11917	#if defined(__x86_64__)
11918	return "x64";
11919	#elif defined(__i386__)
11920	return "ia32";
11921	#elif defined(__s390x__)
11922	return "s390x";
11923	#elif defined(__arm__)
11924	return "arm";
11925	#elif defined(__aarch64__)
11926	return "arm64";
11927	#elif defined(__mips__)
11928	return "mips";
11929	#elif defined(__riscv)
11930	return "riscv";
11931	#elif defined(__powerpc__)
11932	return "powerpc";
11933	#elif defined(__powerpc64__)
11934	return "powerpc64";
11935	#else
11936	return NULL;
11937	#endif
11938	}
11939
11940	int probe_kern_syscall_wrapper(int token_fd)
11941	{
11942	char syscall_name[64];
11943	const char *ksys_pfx;
11944
11945	ksys_pfx = arch_specific_syscall_pfx();
11946	if (!ksys_pfx)
11947	return 0;
11948
11949	snprintf(syscall_name, sizeof(syscall_name), "__%s_sys_bpf", ksys_pfx);
11950
11951	if (determine_kprobe_perf_type() >= 0) {
11952	int pfd;
11953
11954	pfd = perf_event_open_probe(false, false, syscall_name, 0, getpid(), 0);
11955	if (pfd >= 0)
11956	close(pfd);
11957
11958	return pfd >= 0 ? 1 : 0;
11959	} else { /* legacy mode */
11960	char probe_name[MAX_EVENT_NAME_LEN];
11961
11962	gen_probe_legacy_event_name(probe_name, sizeof(probe_name), syscall_name, 0);
11963	if (add_kprobe_event_legacy(probe_name, false, syscall_name, 0) < 0)
11964	return 0;
11965
11966	(void)remove_kprobe_event_legacy(probe_name, false);
11967	return 1;
11968	}
11969	}
11970
11971	struct bpf_link *
11972	bpf_program__attach_kprobe_opts(const struct bpf_program *prog,
11973	const char *func_name,
11974	const struct bpf_kprobe_opts *opts)
11975	{
11976	DECLARE_LIBBPF_OPTS(bpf_perf_event_opts, pe_opts);
11977	enum probe_attach_mode attach_mode;
11978	char *legacy_probe = NULL;
11979	struct bpf_link *link;
11980	size_t offset;
11981	bool retprobe, legacy;
11982	int pfd, err;
11983
11984	if (!OPTS_VALID(opts, bpf_kprobe_opts))
11985	return libbpf_err_ptr(-EINVAL);
11986
11987	attach_mode = OPTS_GET(opts, attach_mode, PROBE_ATTACH_MODE_DEFAULT);
11988	retprobe = OPTS_GET(opts, retprobe, false);
11989	offset = OPTS_GET(opts, offset, 0);
11990	pe_opts.bpf_cookie = OPTS_GET(opts, bpf_cookie, 0);
11991
11992	legacy = determine_kprobe_perf_type() < 0;
11993	switch (attach_mode) {
11994	case PROBE_ATTACH_MODE_LEGACY:
11995	legacy = true;
11996	pe_opts.force_ioctl_attach = true;
11997	break;
11998	case PROBE_ATTACH_MODE_PERF:
11999	if (legacy)
12000	return libbpf_err_ptr(-ENOTSUP);
12001	pe_opts.force_ioctl_attach = true;
12002	break;
12003	case PROBE_ATTACH_MODE_LINK:
12004	if (legacy \|\| !kernel_supports(prog->obj, FEAT_PERF_LINK))
12005	return libbpf_err_ptr(-ENOTSUP);
12006	break;
12007	case PROBE_ATTACH_MODE_DEFAULT:
12008	break;
12009	default:
12010	return libbpf_err_ptr(-EINVAL);
12011	}
12012	if (!func_name && legacy)
12013	return libbpf_err_ptr(-EOPNOTSUPP);
12014
12015	if (!legacy) {
12016	pfd = perf_event_open_probe(false /* uprobe */, retprobe,
12017	func_name, offset,
12018	-1 /* pid /, 0 / ref_ctr_off */);
12019	} else {
12020	char probe_name[MAX_EVENT_NAME_LEN];
12021
12022	gen_probe_legacy_event_name(probe_name, sizeof(probe_name),
12023	func_name, offset);
12024
12025	legacy_probe = strdup(probe_name);
12026	if (!legacy_probe)
12027	return libbpf_err_ptr(-ENOMEM);
12028
12029	pfd = perf_event_kprobe_open_legacy(legacy_probe, retprobe, func_name,
12030	offset, -1 /* pid */);
12031	}
12032	if (pfd < 0) {
12033	err = pfd;
12034	pr_warn("prog '%s': failed to create %s '%s%s0x%zx' perf event: %s\n",
12035	prog->name, retprobe ? "kretprobe" : "kprobe",
12036	func_name ?: "", func_name ? "+" : "",
12037	offset, errstr(err));
12038	goto err_out;
12039	}
12040	link = bpf_program__attach_perf_event_opts(prog, pfd, &pe_opts);
12041	err = libbpf_get_error(link);
12042	if (err) {
12043	close(pfd);
12044	pr_warn("prog '%s': failed to attach to %s '%s%s0x%zx': %s\n",
12045	prog->name, retprobe ? "kretprobe" : "kprobe",
12046	func_name ?: "", func_name ? "+" : "",
12047	offset, errstr(err));
12048	goto err_clean_legacy;
12049	}
12050	if (legacy) {
12051	struct bpf_link_perf *perf_link = container_of(link, struct bpf_link_perf, link);
12052
12053	perf_link->legacy_probe_name = legacy_probe;
12054	perf_link->legacy_is_kprobe = true;
12055	perf_link->legacy_is_retprobe = retprobe;
12056	}
12057
12058	return link;
12059
12060	err_clean_legacy:
12061	if (legacy)
12062	remove_kprobe_event_legacy(legacy_probe, retprobe);
12063	err_out:
12064	free(legacy_probe);
12065	return libbpf_err_ptr(err);
12066	}
12067
12068	struct bpf_link bpf_program__attach_kprobe(const struct bpf_program prog,
12069	bool retprobe,
12070	const char *func_name)
12071	{
12072	DECLARE_LIBBPF_OPTS(bpf_kprobe_opts, opts,
12073	.retprobe = retprobe,
12074	);
12075
12076	return bpf_program__attach_kprobe_opts(prog, func_name, &opts);
12077	}
12078
12079	struct bpf_link bpf_program__attach_ksyscall(const struct bpf_program prog,
12080	const char *syscall_name,
12081	const struct bpf_ksyscall_opts *opts)
12082	{
12083	LIBBPF_OPTS(bpf_kprobe_opts, kprobe_opts);
12084	char func_name[128];
12085
12086	if (!OPTS_VALID(opts, bpf_ksyscall_opts))
12087	return libbpf_err_ptr(-EINVAL);
12088
12089	if (kernel_supports(prog->obj, FEAT_SYSCALL_WRAPPER)) {
12090	/* arch_specific_syscall_pfx() should never return NULL here
12091	* because it is guarded by kernel_supports(). However, since
12092	* compiler does not know that we have an explicit conditional
12093	* as well.
12094	*/
12095	snprintf(func_name, sizeof(func_name), "__%s_sys_%s",
12096	arch_specific_syscall_pfx() ? : "", syscall_name);
12097	} else {
12098	snprintf(func_name, sizeof(func_name), "__se_sys_%s", syscall_name);
12099	}
12100
12101	kprobe_opts.retprobe = OPTS_GET(opts, retprobe, false);
12102	kprobe_opts.bpf_cookie = OPTS_GET(opts, bpf_cookie, 0);
12103
12104	return bpf_program__attach_kprobe_opts(prog, func_name, &kprobe_opts);
12105	}
12106
12107	/* Adapted from perf/util/string.c */
12108	bool glob_match(const char str, const char pat)
12109	{
12110	while (str && pat && pat != '') {
12111	if (pat == '?') { / Matches any single character */
12112	str++;
12113	pat++;
12114	continue;
12115	}
12116	if (str != pat)
12117	return false;
12118	str++;
12119	pat++;
12120	}
12121	/* Check wild card */
12122	if (pat == '') {
12123	while (pat == '')
12124	pat++;
12125	if (!pat) / Tail wild card matches all */
12126	return true;
12127	while (*str)
12128	if (glob_match(str++, pat))
12129	return true;
12130	}
12131	return !str && !pat;
12132	}
12133
12134	struct kprobe_multi_resolve {
12135	const char *pattern;
12136	unsigned long *addrs;
12137	size_t cap;
12138	size_t cnt;
12139	};
12140
12141	struct avail_kallsyms_data {
12142	char **syms;
12143	size_t cnt;
12144	struct kprobe_multi_resolve *res;
12145	};
12146
12147	static int avail_func_cmp(const void a, const void b)
12148	{
12149	return strcmp((const char )a, (const char **)b);
12150	}
12151
12152	static int avail_kallsyms_cb(unsigned long long sym_addr, char sym_type,
12153	const char sym_name, void ctx)
12154	{
12155	struct avail_kallsyms_data *data = ctx;
12156	struct kprobe_multi_resolve *res = data->res;
12157	int err;
12158
12159	if (!glob_match(sym_name, res->pattern))
12160	return 0;
12161
12162	if (!bsearch(&sym_name, data->syms, data->cnt, sizeof(*data->syms), avail_func_cmp)) {
12163	/* Some versions of kernel strip out .llvm.<hash> suffix from
12164	* function names reported in available_filter_functions, but
12165	* don't do so for kallsyms. While this is clearly a kernel
12166	* bug (fixed by [0]) we try to accommodate that in libbpf to
12167	* make multi-kprobe usability a bit better: if no match is
12168	* found, we will strip .llvm. suffix and try one more time.
12169	*
12170	* [0] fb6a421fb615 ("kallsyms: Match symbols exactly with CONFIG_LTO_CLANG")
12171	*/
12172	char sym_trim[256], *psym_trim = sym_trim;
12173	const char *sym_sfx;
12174
12175	if (!(sym_sfx = strstr(sym_name, ".llvm.")))
12176	return 0;
12177
12178	/* psym_trim vs sym_trim dance is done to avoid pointer vs array
12179	* coercion differences and get proper `const char **` pointer
12180	* which avail_func_cmp() expects
12181	*/
12182	snprintf(sym_trim, sizeof(sym_trim), "%.*s", (int)(sym_sfx - sym_name), sym_name);
12183	if (!bsearch(&psym_trim, data->syms, data->cnt, sizeof(*data->syms), avail_func_cmp))
12184	return 0;
12185	}
12186
12187	err = libbpf_ensure_mem((void *)&res->addrs, &res->cap, sizeof(res->addrs), res->cnt + 1);
12188	if (err)
12189	return err;
12190
12191	res->addrs[res->cnt++] = (unsigned long)sym_addr;
12192	return 0;
12193	}
12194
12195	static int libbpf_available_kallsyms_parse(struct kprobe_multi_resolve *res)
12196	{
12197	const char *available_functions_file = tracefs_available_filter_functions();
12198	struct avail_kallsyms_data data;
12199	char sym_name[500];
12200	FILE *f;
12201	int err = 0, ret, i;
12202	char **syms = NULL;
12203	size_t cap = 0, cnt = 0;
12204
12205	f = fopen(available_functions_file, "re");
12206	if (!f) {
12207	err = -errno;
12208	pr_warn("failed to open %s: %s\n", available_functions_file, errstr(err));
12209	return err;
12210	}
12211
12212	while (true) {
12213	char *name;
12214
12215	ret = fscanf(f, "%499s%*[^\n]\n", sym_name);
12216	if (ret == EOF && feof(f))
12217	break;
12218
12219	if (ret != 1) {
12220	pr_warn("failed to parse available_filter_functions entry: %d\n", ret);
12221	err = -EINVAL;
12222	goto cleanup;
12223	}
12224
12225	if (!glob_match(sym_name, res->pattern))
12226	continue;
12227
12228	err = libbpf_ensure_mem((void *)&syms, &cap, sizeof(syms), cnt + 1);
12229	if (err)
12230	goto cleanup;
12231
12232	name = strdup(sym_name);
12233	if (!name) {
12234	err = -errno;
12235	goto cleanup;
12236	}
12237
12238	syms[cnt++] = name;
12239	}
12240
12241	/* no entries found, bail out */
12242	if (cnt == 0) {
12243	err = -ENOENT;
12244	goto cleanup;
12245	}
12246
12247	/* sort available functions */
12248	qsort(syms, cnt, sizeof(*syms), avail_func_cmp);
12249
12250	data.syms = syms;
12251	data.res = res;
12252	data.cnt = cnt;
12253	libbpf_kallsyms_parse(avail_kallsyms_cb, &data);
12254
12255	if (res->cnt == 0)
12256	err = -ENOENT;
12257
12258	cleanup:
12259	for (i = 0; i < cnt; i++)
12260	free((char *)syms[i]);
12261	free(syms);
12262
12263	fclose(f);
12264	return err;
12265	}
12266
12267	static bool has_available_filter_functions_addrs(void)
12268	{
12269	return access(tracefs_available_filter_functions_addrs(), R_OK) != -1;
12270	}
12271
12272	static int libbpf_available_kprobes_parse(struct kprobe_multi_resolve *res)
12273	{
12274	const char *available_path = tracefs_available_filter_functions_addrs();
12275	char sym_name[500];
12276	FILE *f;
12277	int ret, err = 0;
12278	unsigned long long sym_addr;
12279
12280	f = fopen(available_path, "re");
12281	if (!f) {
12282	err = -errno;
12283	pr_warn("failed to open %s: %s\n", available_path, errstr(err));
12284	return err;
12285	}
12286
12287	while (true) {
12288	ret = fscanf(f, "%llx %499s%*[^\n]\n", &sym_addr, sym_name);
12289	if (ret == EOF && feof(f))
12290	break;
12291
12292	if (ret != 2) {
12293	pr_warn("failed to parse available_filter_functions_addrs entry: %d\n",
12294	ret);
12295	err = -EINVAL;
12296	goto cleanup;
12297	}
12298
12299	if (!glob_match(sym_name, res->pattern))
12300	continue;
12301
12302	err = libbpf_ensure_mem((void **)&res->addrs, &res->cap,
12303	sizeof(*res->addrs), res->cnt + 1);
12304	if (err)
12305	goto cleanup;
12306
12307	res->addrs[res->cnt++] = (unsigned long)sym_addr;
12308	}
12309
12310	if (res->cnt == 0)
12311	err = -ENOENT;
12312
12313	cleanup:
12314	fclose(f);
12315	return err;
12316	}
12317
12318	struct bpf_link *
12319	bpf_program__attach_kprobe_multi_opts(const struct bpf_program *prog,
12320	const char *pattern,
12321	const struct bpf_kprobe_multi_opts *opts)
12322	{
12323	LIBBPF_OPTS(bpf_link_create_opts, lopts);
12324	struct kprobe_multi_resolve res = {
12325	.pattern = pattern,
12326	};
12327	enum bpf_attach_type attach_type;
12328	struct bpf_link *link = NULL;
12329	const unsigned long *addrs;
12330	int err, link_fd, prog_fd;
12331	bool retprobe, session, unique_match;
12332	const __u64 *cookies;
12333	const char **syms;
12334	size_t cnt;
12335
12336	if (!OPTS_VALID(opts, bpf_kprobe_multi_opts))
12337	return libbpf_err_ptr(-EINVAL);
12338
12339	prog_fd = bpf_program__fd(prog);
12340	if (prog_fd < 0) {
12341	pr_warn("prog '%s': can't attach BPF program without FD (was it loaded?)\n",
12342	prog->name);
12343	return libbpf_err_ptr(-EINVAL);
12344	}
12345
12346	syms = OPTS_GET(opts, syms, false);
12347	addrs = OPTS_GET(opts, addrs, false);
12348	cnt = OPTS_GET(opts, cnt, false);
12349	cookies = OPTS_GET(opts, cookies, false);
12350	unique_match = OPTS_GET(opts, unique_match, false);
12351
12352	if (!pattern && !addrs && !syms)
12353	return libbpf_err_ptr(-EINVAL);
12354	if (pattern && (addrs \|\| syms \|\| cookies \|\| cnt))
12355	return libbpf_err_ptr(-EINVAL);
12356	if (!pattern && !cnt)
12357	return libbpf_err_ptr(-EINVAL);
12358	if (!pattern && unique_match)
12359	return libbpf_err_ptr(-EINVAL);
12360	if (addrs && syms)
12361	return libbpf_err_ptr(-EINVAL);
12362
12363	/*
12364	* Exact function name (no wildcards) without unique_match:
12365	* bypass kallsyms parsing and pass the symbol directly to the
12366	* kernel via syms[] array. When unique_match is set, fall
12367	* through to the slow path which detects duplicate symbols.
12368	*/
12369	if (pattern && !strpbrk(pattern, "*?") && !unique_match) {
12370	syms = &pattern;
12371	cnt = 1;
12372	} else if (pattern) {
12373	if (has_available_filter_functions_addrs())
12374	err = libbpf_available_kprobes_parse(&res);
12375	else
12376	err = libbpf_available_kallsyms_parse(&res);
12377	if (err)
12378	goto error;
12379
12380	if (unique_match && res.cnt != 1) {
12381	pr_warn("prog '%s': failed to find a unique match for '%s' (%zu matches)\n",
12382	prog->name, pattern, res.cnt);
12383	err = -EINVAL;
12384	goto error;
12385	}
12386
12387	addrs = res.addrs;
12388	cnt = res.cnt;
12389	}
12390
12391	retprobe = OPTS_GET(opts, retprobe, false);
12392	session = OPTS_GET(opts, session, false);
12393
12394	if (retprobe && session)
12395	return libbpf_err_ptr(-EINVAL);
12396
12397	attach_type = session ? BPF_TRACE_KPROBE_SESSION : BPF_TRACE_KPROBE_MULTI;
12398
12399	lopts.kprobe_multi.syms = syms;
12400	lopts.kprobe_multi.addrs = addrs;
12401	lopts.kprobe_multi.cookies = cookies;
12402	lopts.kprobe_multi.cnt = cnt;
12403	lopts.kprobe_multi.flags = retprobe ? BPF_F_KPROBE_MULTI_RETURN : 0;
12404
12405	link = calloc(1, sizeof(*link));
12406	if (!link) {
12407	err = -ENOMEM;
12408	goto error;
12409	}
12410	link->detach = &bpf_link__detach_fd;
12411
12412	link_fd = bpf_link_create(prog_fd, 0, attach_type, &lopts);
12413	if (link_fd < 0) {
12414	err = -errno;
12415	/*
12416	* Normalize error code: when exact name bypasses kallsyms
12417	* parsing, kernel returns ESRCH from ftrace_lookup_symbols().
12418	* Convert to ENOENT for API consistency with the pattern
12419	* matching path which returns ENOENT from userspace.
12420	*/
12421	if (err == -ESRCH)
12422	err = -ENOENT;
12423	pr_warn("prog '%s': failed to attach: %s\n",
12424	prog->name, errstr(err));
12425	goto error;
12426	}
12427	link->fd = link_fd;
12428	free(res.addrs);
12429	return link;
12430
12431	error:
12432	free(link);
12433	free(res.addrs);
12434	return libbpf_err_ptr(err);
12435	}
12436
12437	static int attach_kprobe(const struct bpf_program prog, long cookie, struct bpf_link *link)
12438	{
12439	DECLARE_LIBBPF_OPTS(bpf_kprobe_opts, opts);
12440	long offset = 0;
12441	const char *func_name;
12442	char *func;
12443	int n;
12444
12445	*link = NULL;
12446
12447	/* no auto-attach for SEC("kprobe") and SEC("kretprobe") */
12448	if (strcmp(prog->sec_name, "kprobe") == 0 \|\| strcmp(prog->sec_name, "kretprobe") == 0)
12449	return 0;
12450
12451	opts.retprobe = str_has_pfx(prog->sec_name, "kretprobe/");
12452	if (opts.retprobe)
12453	func_name = prog->sec_name + sizeof("kretprobe/") - 1;
12454	else
12455	func_name = prog->sec_name + sizeof("kprobe/") - 1;
12456
12457	n = sscanf(func_name, "%m[a-zA-Z0-9_.]+%li", &func, &offset);
12458	if (n < 1) {
12459	pr_warn("kprobe name is invalid: %s\n", func_name);
12460	return -EINVAL;
12461	}
12462
12463	if (offset < 0) {
12464	free(func);
12465	pr_warn("kprobe offset must be a non-negative integer: %li\n", offset);
12466	return -EINVAL;
12467	}
12468
12469	if (opts.retprobe && offset != 0) {
12470	free(func);
12471	pr_warn("kretprobes do not support offset specification\n");
12472	return -EINVAL;
12473	}
12474
12475	opts.offset = offset;
12476	*link = bpf_program__attach_kprobe_opts(prog, func, &opts);
12477	free(func);
12478	return libbpf_get_error(*link);
12479	}
12480
12481	static int attach_ksyscall(const struct bpf_program prog, long cookie, struct bpf_link *link)
12482	{
12483	LIBBPF_OPTS(bpf_ksyscall_opts, opts);
12484	const char *syscall_name;
12485
12486	*link = NULL;
12487
12488	/* no auto-attach for SEC("ksyscall") and SEC("kretsyscall") */
12489	if (strcmp(prog->sec_name, "ksyscall") == 0 \|\| strcmp(prog->sec_name, "kretsyscall") == 0)
12490	return 0;
12491
12492	opts.retprobe = str_has_pfx(prog->sec_name, "kretsyscall/");
12493	if (opts.retprobe)
12494	syscall_name = prog->sec_name + sizeof("kretsyscall/") - 1;
12495	else
12496	syscall_name = prog->sec_name + sizeof("ksyscall/") - 1;
12497
12498	*link = bpf_program__attach_ksyscall(prog, syscall_name, &opts);
12499	return *link ? 0 : -errno;
12500	}
12501
12502	static int attach_kprobe_multi(const struct bpf_program prog, long cookie, struct bpf_link *link)
12503	{
12504	LIBBPF_OPTS(bpf_kprobe_multi_opts, opts);
12505	const char *spec;
12506	char *pattern;
12507	int n;
12508
12509	*link = NULL;
12510
12511	/* no auto-attach for SEC("kprobe.multi") and SEC("kretprobe.multi") */
12512	if (strcmp(prog->sec_name, "kprobe.multi") == 0 \|\|
12513	strcmp(prog->sec_name, "kretprobe.multi") == 0)
12514	return 0;
12515
12516	opts.retprobe = str_has_pfx(prog->sec_name, "kretprobe.multi/");
12517	if (opts.retprobe)
12518	spec = prog->sec_name + sizeof("kretprobe.multi/") - 1;
12519	else
12520	spec = prog->sec_name + sizeof("kprobe.multi/") - 1;
12521
12522	n = sscanf(spec, "%m[a-zA-Z0-9_.*?]", &pattern);
12523	if (n < 1) {
12524	pr_warn("kprobe multi pattern is invalid: %s\n", spec);
12525	return -EINVAL;
12526	}
12527
12528	*link = bpf_program__attach_kprobe_multi_opts(prog, pattern, &opts);
12529	free(pattern);
12530	return libbpf_get_error(*link);
12531	}
12532
12533	static int attach_kprobe_session(const struct bpf_program *prog, long cookie,
12534	struct bpf_link **link)
12535	{
12536	LIBBPF_OPTS(bpf_kprobe_multi_opts, opts, .session = true);
12537	const char *spec;
12538	char *pattern;
12539	int n;
12540
12541	*link = NULL;
12542
12543	/* no auto-attach for SEC("kprobe.session") */
12544	if (strcmp(prog->sec_name, "kprobe.session") == 0)
12545	return 0;
12546
12547	spec = prog->sec_name + sizeof("kprobe.session/") - 1;
12548	n = sscanf(spec, "%m[a-zA-Z0-9_.*?]", &pattern);
12549	if (n < 1) {
12550	pr_warn("kprobe session pattern is invalid: %s\n", spec);
12551	return -EINVAL;
12552	}
12553
12554	*link = bpf_program__attach_kprobe_multi_opts(prog, pattern, &opts);
12555	free(pattern);
12556	return *link ? 0 : -errno;
12557	}
12558
12559	static int attach_uprobe_multi(const struct bpf_program prog, long cookie, struct bpf_link *link)
12560	{
12561	char probe_type = NULL, binary_path = NULL, *func_name = NULL;
12562	LIBBPF_OPTS(bpf_uprobe_multi_opts, opts);
12563	int n, ret = -EINVAL;
12564
12565	*link = NULL;
12566
12567	n = sscanf(prog->sec_name, "%m[^/]/%m[^:]:%m[^\n]",
12568	&probe_type, &binary_path, &func_name);
12569	switch (n) {
12570	case 1:
12571	/* handle SEC("u[ret]probe") - format is valid, but auto-attach is impossible. */
12572	ret = 0;
12573	break;
12574	case 3:
12575	opts.session = str_has_pfx(probe_type, "uprobe.session");
12576	opts.retprobe = str_has_pfx(probe_type, "uretprobe.multi");
12577
12578	*link = bpf_program__attach_uprobe_multi(prog, -1, binary_path, func_name, &opts);
12579	ret = libbpf_get_error(*link);
12580	break;
12581	default:
12582	pr_warn("prog '%s': invalid format of section definition '%s'\n", prog->name,
12583	prog->sec_name);
12584	break;
12585	}
12586	free(probe_type);
12587	free(binary_path);
12588	free(func_name);
12589	return ret;
12590	}
12591
12592	#define MAX_BPF_FUNC_ARGS 12
12593
12594	static bool btf_type_is_modifier(const struct btf_type *t)
12595	{
12596	switch (BTF_INFO_KIND(t->info)) {
12597	case BTF_KIND_TYPEDEF:
12598	case BTF_KIND_VOLATILE:
12599	case BTF_KIND_CONST:
12600	case BTF_KIND_RESTRICT:
12601	case BTF_KIND_TYPE_TAG:
12602	return true;
12603	default:
12604	return false;
12605	}
12606	}
12607
12608	#define MAX_RESOLVE_DEPTH 32
12609
12610	static int btf_get_type_size(const struct btf *btf, __u32 type_id,
12611	const struct btf_type **ret_type)
12612	{
12613	const struct btf_type *t;
12614	int i;
12615
12616	*ret_type = btf__type_by_id(btf, 0);
12617	if (!type_id)
12618	return 0;
12619	t = btf__type_by_id(btf, type_id);
12620	for (i = 0; i < MAX_RESOLVE_DEPTH && t && btf_type_is_modifier(t); i++)
12621	t = btf__type_by_id(btf, t->type);
12622	if (!t \|\| i == MAX_RESOLVE_DEPTH)
12623	return -EINVAL;
12624	*ret_type = t;
12625	if (btf_is_ptr(t))
12626	return btf__pointer_size(btf);
12627	if (btf_is_int(t) \|\| btf_is_any_enum(t) \|\| btf_is_struct(t) \|\| btf_is_union(t))
12628	return t->size;
12629	return -EINVAL;
12630	}
12631
12632	bool btf_type_is_traceable_func(const struct btf btf, const struct btf_type t)
12633	{
12634	const struct btf_param *args;
12635	const struct btf_type *proto;
12636	__u32 i, nargs;
12637	int ret;
12638
12639	if (!btf_is_func(t))
12640	return false;
12641	proto = btf__type_by_id(btf, t->type);
12642	if (!proto \|\| !btf_is_func_proto(proto))
12643	return false;
12644
12645	args = (const struct btf_param *)(proto + 1);
12646	nargs = btf_vlen(proto);
12647	if (nargs > MAX_BPF_FUNC_ARGS)
12648	return false;
12649
12650	/* No support for struct return type. */
12651	ret = btf_get_type_size(btf, proto->type, &t);
12652	if (ret < 0 \|\| btf_is_struct(t) \|\| btf_is_union(t))
12653	return false;
12654
12655	for (i = 0; i < nargs; i++) {
12656	/* No support for variable args. */
12657	if (i == nargs - 1 && args[i].type == 0)
12658	return false;
12659	ret = btf_get_type_size(btf, args[i].type, &t);
12660	/* No support of struct argument size greater than 16 bytes. */
12661	if (ret < 0 \|\| ret > 16)
12662	return false;
12663	/* No support for void argument. */
12664	if (ret == 0)
12665	return false;
12666	}
12667
12668	return true;
12669	}
12670
12671	static int
12672	collect_btf_func_ids_by_glob(const struct btf btf, const char pattern, __u32 **ids)
12673	{
12674	__u32 type_id, nr_types = btf__type_cnt(btf);
12675	size_t cap = 0, cnt = 0;
12676
12677	if (!pattern)
12678	return -EINVAL;
12679
12680	for (type_id = 1; type_id < nr_types; type_id++) {
12681	const struct btf_type *t = btf__type_by_id(btf, type_id);
12682	const char *name;
12683	int err;
12684
12685	if (btf_kind(t) != BTF_KIND_FUNC)
12686	continue;
12687	name = btf__name_by_offset(btf, t->name_off);
12688	if (!name)
12689	continue;
12690
12691	if (!glob_match(name, pattern))
12692	continue;
12693	if (!btf_type_is_traceable_func(btf, t))
12694	continue;
12695
12696	err = libbpf_ensure_mem((void ) ids, &cap, sizeof(ids), cnt + 1);
12697	if (err) {
12698	free(*ids);
12699	return -ENOMEM;
12700	}
12701	(*ids)[cnt++] = type_id;
12702	}
12703
12704	return cnt;
12705	}
12706
12707	static int collect_func_ids_by_glob(const struct bpf_program prog, const char pattern, __u32 **ids)
12708	{
12709	struct bpf_object *obj = prog->obj;
12710	const struct module_btf *mod;
12711	struct btf *btf = NULL;
12712	const char *sep;
12713	int err;
12714
12715	err = bpf_object__load_vmlinux_btf(obj, true);
12716	if (err)
12717	return err;
12718
12719	/* In case we have module specified, we will find its btf and use that. */
12720	sep = strchr(pattern, ':');
12721	if (sep) {
12722	mod = find_attach_module(obj, pattern);
12723	if (!mod) {
12724	err = -EINVAL;
12725	goto cleanup;
12726	}
12727	btf = mod->btf;
12728	pattern = sep + 1;
12729	} else {
12730	/* Program is loaded for kernel module. */
12731	if (prog->attach_btf_obj_fd) {
12732	err = -EINVAL;
12733	goto cleanup;
12734	}
12735	btf = obj->btf_vmlinux;
12736	}
12737
12738	err = collect_btf_func_ids_by_glob(btf, pattern, ids);
12739
12740	cleanup:
12741	bpf_object_cleanup_btf(obj);
12742	return err;
12743	}
12744
12745	struct bpf_link *
12746	bpf_program__attach_tracing_multi(const struct bpf_program prog, const char pattern,
12747	const struct bpf_tracing_multi_opts *opts)
12748	{
12749	LIBBPF_OPTS(bpf_link_create_opts, lopts);
12750	int prog_fd, link_fd, err, cnt;
12751	__u32 *free_ids = NULL;
12752	struct bpf_link *link;
12753	const __u64 *cookies;
12754	const __u32 *ids;
12755
12756	if (!OPTS_VALID(opts, bpf_tracing_multi_opts))
12757	return libbpf_err_ptr(-EINVAL);
12758
12759	prog_fd = bpf_program__fd(prog);
12760	if (prog_fd < 0) {
12761	pr_warn("prog '%s': can't attach BPF program without FD (was it loaded?)\n",
12762	prog->name);
12763	return libbpf_err_ptr(-EINVAL);
12764	}
12765
12766	cnt = OPTS_GET(opts, cnt, 0);
12767	ids = OPTS_GET(opts, ids, NULL);
12768	cookies = OPTS_GET(opts, cookies, NULL);
12769
12770	if (!!ids != !!cnt)
12771	return libbpf_err_ptr(-EINVAL);
12772	if (pattern && (ids \|\| cookies))
12773	return libbpf_err_ptr(-EINVAL);
12774	if (!pattern && !ids)
12775	return libbpf_err_ptr(-EINVAL);
12776
12777	if (pattern) {
12778	cnt = collect_func_ids_by_glob(prog, pattern, &free_ids);
12779	if (cnt < 0)
12780	return libbpf_err_ptr(cnt);
12781	if (cnt == 0)
12782	return libbpf_err_ptr(-EINVAL);
12783	ids = (const __u32 *) free_ids;
12784	}
12785
12786	lopts.tracing_multi.ids = ids;
12787	lopts.tracing_multi.cookies = cookies;
12788	lopts.tracing_multi.cnt = cnt;
12789
12790	link = calloc(1, sizeof(*link));
12791	if (!link) {
12792	err = -ENOMEM;
12793	goto error;
12794	}
12795	link->detach = &bpf_link__detach_fd;
12796
12797	link_fd = bpf_link_create(prog_fd, 0, prog->expected_attach_type, &lopts);
12798	if (link_fd < 0) {
12799	err = -errno;
12800	pr_warn("prog '%s': failed to attach: %s\n", prog->name, errstr(err));
12801	goto error;
12802	}
12803	link->fd = link_fd;
12804	free(free_ids);
12805	return link;
12806
12807	error:
12808	free(link);
12809	free(free_ids);
12810	return libbpf_err_ptr(err);
12811	}
12812
12813	static int attach_tracing_multi(const struct bpf_program prog, long cookie, struct bpf_link *link)
12814	{
12815	static const char *const prefixes[] = {
12816	"fentry.multi",
12817	"fexit.multi",
12818	"fsession.multi",
12819	"fentry.multi.s",
12820	"fexit.multi.s",
12821	"fsession.multi.s",
12822	};
12823	const char *spec = NULL;
12824	char *pattern;
12825	size_t i;
12826	int n;
12827
12828	*link = NULL;
12829
12830	for (i = 0; i < ARRAY_SIZE(prefixes); i++) {
12831	size_t pfx_len;
12832
12833	if (!str_has_pfx(prog->sec_name, prefixes[i]))
12834	continue;
12835
12836	pfx_len = strlen(prefixes[i]);
12837	/* no auto-attach case of, e.g., SEC("fentry.multi") */
12838	if (prog->sec_name[pfx_len] == '\0')
12839	return 0;
12840
12841	if (prog->sec_name[pfx_len] != '/')
12842	continue;
12843
12844	spec = prog->sec_name + pfx_len + 1;
12845	break;
12846	}
12847
12848	if (!spec) {
12849	pr_warn("prog '%s': invalid section name '%s'\n",
12850	prog->name, prog->sec_name);
12851	return -EINVAL;
12852	}
12853
12854	n = sscanf(spec, "%m[a-zA-Z0-9_.*?:]", &pattern);
12855	if (n < 1) {
12856	pr_warn("tracing multi pattern is invalid: %s\n", spec);
12857	return -EINVAL;
12858	}
12859
12860	*link = bpf_program__attach_tracing_multi(prog, pattern, NULL);
12861	free(pattern);
12862	return libbpf_get_error(*link);
12863	}
12864
12865	static inline int add_uprobe_event_legacy(const char *probe_name, bool retprobe,
12866	const char *binary_path, size_t offset)
12867	{
12868	return append_to_file(tracefs_uprobe_events(), "%c:%s/%s %s:0x%zx",
12869	retprobe ? 'r' : 'p',
12870	retprobe ? "uretprobes" : "uprobes",
12871	probe_name, binary_path, offset);
12872	}
12873
12874	static inline int remove_uprobe_event_legacy(const char *probe_name, bool retprobe)
12875	{
12876	return append_to_file(tracefs_uprobe_events(), "-:%s/%s",
12877	retprobe ? "uretprobes" : "uprobes", probe_name);
12878	}
12879
12880	static int determine_uprobe_perf_type_legacy(const char *probe_name, bool retprobe)
12881	{
12882	char file[512];
12883
12884	snprintf(file, sizeof(file), "%s/events/%s/%s/id",
12885	tracefs_path(), retprobe ? "uretprobes" : "uprobes", probe_name);
12886
12887	return parse_uint_from_file(file, "%d\n");
12888	}
12889
12890	static int perf_event_uprobe_open_legacy(const char *probe_name, bool retprobe,
12891	const char *binary_path, size_t offset, int pid)
12892	{
12893	const size_t attr_sz = sizeof(struct perf_event_attr);
12894	struct perf_event_attr attr;
12895	int type, pfd, err;
12896
12897	err = add_uprobe_event_legacy(probe_name, retprobe, binary_path, offset);
12898	if (err < 0) {
12899	pr_warn("failed to add legacy uprobe event for %s:0x%zx: %s\n",
12900	binary_path, (size_t)offset, errstr(err));
12901	return err;
12902	}
12903	type = determine_uprobe_perf_type_legacy(probe_name, retprobe);
12904	if (type < 0) {
12905	err = type;
12906	pr_warn("failed to determine legacy uprobe event id for %s:0x%zx: %s\n",
12907	binary_path, offset, errstr(err));
12908	goto err_clean_legacy;
12909	}
12910
12911	memset(&attr, 0, attr_sz);
12912	attr.size = attr_sz;
12913	attr.config = type;
12914	attr.type = PERF_TYPE_TRACEPOINT;
12915
12916	pfd = syscall(__NR_perf_event_open, &attr,
12917	pid < 0 ? -1 : pid, /* pid */
12918	pid == -1 ? 0 : -1, /* cpu */
12919	-1 /* group_fd */, PERF_FLAG_FD_CLOEXEC);
12920	if (pfd < 0) {
12921	err = -errno;
12922	pr_warn("legacy uprobe perf_event_open() failed: %s\n", errstr(err));
12923	goto err_clean_legacy;
12924	}
12925	return pfd;
12926
12927	err_clean_legacy:
12928	/* Clear the newly added legacy uprobe_event */
12929	remove_uprobe_event_legacy(probe_name, retprobe);
12930	return err;
12931	}
12932
12933	/* Find offset of function name in archive specified by path. Currently
12934	* supported are .zip files that do not compress their contents, as used on
12935	* Android in the form of APKs, for example. "file_name" is the name of the ELF
12936	* file inside the archive. "func_name" matches symbol name or name@@LIB for
12937	* library functions.
12938	*
12939	* An overview of the APK format specifically provided here:
12940	* https://en.wikipedia.org/w/index.php?title=Apk_(file_format)&oldid=1139099120#Package_contents
12941	*/
12942	static long elf_find_func_offset_from_archive(const char archive_path, const char file_name,
12943	const char *func_name)
12944	{
12945	struct zip_archive *archive;
12946	struct zip_entry entry;
12947	long ret;
12948	Elf *elf;
12949
12950	archive = zip_archive_open(archive_path);
12951	if (IS_ERR(archive)) {
12952	ret = PTR_ERR(archive);
12953	pr_warn("zip: failed to open %s: %ld\n", archive_path, ret);
12954	return ret;
12955	}
12956
12957	ret = zip_archive_find_entry(archive, file_name, &entry);
12958	if (ret) {
12959	pr_warn("zip: could not find archive member %s in %s: %ld\n", file_name,
12960	archive_path, ret);
12961	goto out;
12962	}
12963	pr_debug("zip: found entry for %s in %s at 0x%lx\n", file_name, archive_path,
12964	(unsigned long)entry.data_offset);
12965
12966	if (entry.compression) {
12967	pr_warn("zip: entry %s of %s is compressed and cannot be handled\n", file_name,
12968	archive_path);
12969	ret = -LIBBPF_ERRNO__FORMAT;
12970	goto out;
12971	}
12972
12973	elf = elf_memory((void *)entry.data, entry.data_length);
12974	if (!elf) {
12975	pr_warn("elf: could not read elf file %s from %s: %s\n", file_name, archive_path,
12976	elf_errmsg(-1));
12977	ret = -LIBBPF_ERRNO__LIBELF;
12978	goto out;
12979	}
12980
12981	ret = elf_find_func_offset(elf, file_name, func_name);
12982	if (ret > 0) {
12983	pr_debug("elf: symbol address match for %s of %s in %s: 0x%x + 0x%lx = 0x%lx\n",
12984	func_name, file_name, archive_path, entry.data_offset, (unsigned long)ret,
12985	(unsigned long)(ret + entry.data_offset));
12986	ret += entry.data_offset;
12987	}
12988	elf_end(elf);
12989
12990	out:
12991	zip_archive_close(archive);
12992	return ret;
12993	}
12994
12995	static const char *arch_specific_lib_paths(void)
12996	{
12997	/*
12998	* Based on https://packages.debian.org/sid/libc6.
12999	*
13000	* Assume that the traced program is built for the same architecture
13001	* as libbpf, which should cover the vast majority of cases.
13002	*/
13003	#if defined(__x86_64__)
13004	return "/lib/x86_64-linux-gnu";
13005	#elif defined(__i386__)
13006	return "/lib/i386-linux-gnu";
13007	#elif defined(__s390x__)
13008	return "/lib/s390x-linux-gnu";
13009	#elif defined(__arm__) && defined(__SOFTFP__)
13010	return "/lib/arm-linux-gnueabi";
13011	#elif defined(__arm__) && !defined(__SOFTFP__)
13012	return "/lib/arm-linux-gnueabihf";
13013	#elif defined(__aarch64__)
13014	return "/lib/aarch64-linux-gnu";
13015	#elif defined(__mips__) && defined(__MIPSEL__) && _MIPS_SZLONG == 64
13016	return "/lib/mips64el-linux-gnuabi64";
13017	#elif defined(__mips__) && defined(__MIPSEL__) && _MIPS_SZLONG == 32
13018	return "/lib/mipsel-linux-gnu";
13019	#elif defined(__powerpc64__) && __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__
13020	return "/lib/powerpc64le-linux-gnu";
13021	#elif defined(__sparc__) && defined(__arch64__)
13022	return "/lib/sparc64-linux-gnu";
13023	#elif defined(__riscv) && __riscv_xlen == 64
13024	return "/lib/riscv64-linux-gnu";
13025	#else
13026	return NULL;
13027	#endif
13028	}
13029
13030	/* Get full path to program/shared library. */
13031	static int resolve_full_path(const char file, char result, size_t result_sz)
13032	{
13033	const char *search_paths[4] = {};
13034	int i, perm;
13035
13036	if (str_has_sfx(file, ".so") \|\| strstr(file, ".so.")) {
13037	search_paths[0] = getenv("LD_LIBRARY_PATH");
13038	search_paths[1] = "/usr/lib64:/usr/lib";
13039	search_paths[2] = arch_specific_lib_paths();
13040	search_paths[3] = "/lib64:/lib";
13041	perm = R_OK;
13042	} else {
13043	search_paths[0] = getenv("PATH");
13044	search_paths[1] = "/usr/bin:/usr/sbin";
13045	perm = R_OK \| X_OK;
13046	}
13047
13048	for (i = 0; i < ARRAY_SIZE(search_paths); i++) {
13049	const char *s;
13050
13051	if (!search_paths[i])
13052	continue;
13053	for (s = search_paths[i]; s != NULL; s = strchr(s, ':')) {
13054	const char *next_path;
13055	int seg_len;
13056
13057	if (s[0] == ':')
13058	s++;
13059	next_path = strchr(s, ':');
13060	seg_len = next_path ? next_path - s : strlen(s);
13061	if (!seg_len)
13062	continue;
13063	snprintf(result, result_sz, "%.*s/%s", seg_len, s, file);
13064	/* ensure it has required permissions */
13065	if (faccessat(AT_FDCWD, result, perm, AT_EACCESS) < 0)
13066	continue;
13067	pr_debug("resolved '%s' to '%s'\n", file, result);
13068	return 0;
13069	}
13070	}
13071	return -ENOENT;
13072	}
13073
13074	struct bpf_link *
13075	bpf_program__attach_uprobe_multi(const struct bpf_program *prog,
13076	pid_t pid,
13077	const char *path,
13078	const char *func_pattern,
13079	const struct bpf_uprobe_multi_opts *opts)
13080	{
13081	const unsigned long ref_ctr_offsets = NULL, offsets = NULL;
13082	LIBBPF_OPTS(bpf_link_create_opts, lopts);
13083	unsigned long *resolved_offsets = NULL;
13084	enum bpf_attach_type attach_type;
13085	int err = 0, link_fd, prog_fd;
13086	struct bpf_link *link = NULL;
13087	char full_path[PATH_MAX];
13088	bool retprobe, session;
13089	const __u64 *cookies;
13090	const char **syms;
13091	size_t cnt;
13092
13093	if (!OPTS_VALID(opts, bpf_uprobe_multi_opts))
13094	return libbpf_err_ptr(-EINVAL);
13095
13096	prog_fd = bpf_program__fd(prog);
13097	if (prog_fd < 0) {
13098	pr_warn("prog '%s': can't attach BPF program without FD (was it loaded?)\n",
13099	prog->name);
13100	return libbpf_err_ptr(-EINVAL);
13101	}
13102
13103	syms = OPTS_GET(opts, syms, NULL);
13104	offsets = OPTS_GET(opts, offsets, NULL);
13105	ref_ctr_offsets = OPTS_GET(opts, ref_ctr_offsets, NULL);
13106	cookies = OPTS_GET(opts, cookies, NULL);
13107	cnt = OPTS_GET(opts, cnt, 0);
13108	retprobe = OPTS_GET(opts, retprobe, false);
13109	session = OPTS_GET(opts, session, false);
13110
13111	/*
13112	* User can specify 2 mutually exclusive set of inputs:
13113	*
13114	* 1) use only path/func_pattern/pid arguments
13115	*
13116	* 2) use path/pid with allowed combinations of:
13117	* syms/offsets/ref_ctr_offsets/cookies/cnt
13118	*
13119	* - syms and offsets are mutually exclusive
13120	* - ref_ctr_offsets and cookies are optional
13121	*
13122	* Any other usage results in error.
13123	*/
13124
13125	if (!path)
13126	return libbpf_err_ptr(-EINVAL);
13127	if (!func_pattern && cnt == 0)
13128	return libbpf_err_ptr(-EINVAL);
13129
13130	if (func_pattern) {
13131	if (syms \|\| offsets \|\| ref_ctr_offsets \|\| cookies \|\| cnt)
13132	return libbpf_err_ptr(-EINVAL);
13133	} else {
13134	if (!!syms == !!offsets)
13135	return libbpf_err_ptr(-EINVAL);
13136	}
13137
13138	if (retprobe && session)
13139	return libbpf_err_ptr(-EINVAL);
13140
13141	if (func_pattern) {
13142	if (!strchr(path, '/')) {
13143	err = resolve_full_path(path, full_path, sizeof(full_path));
13144	if (err) {
13145	pr_warn("prog '%s': failed to resolve full path for '%s': %s\n",
13146	prog->name, path, errstr(err));
13147	return libbpf_err_ptr(err);
13148	}
13149	path = full_path;
13150	}
13151
13152	err = elf_resolve_pattern_offsets(path, func_pattern,
13153	&resolved_offsets, &cnt);
13154	if (err < 0)
13155	return libbpf_err_ptr(err);
13156	offsets = resolved_offsets;
13157	} else if (syms) {
13158	err = elf_resolve_syms_offsets(path, cnt, syms, &resolved_offsets, STT_FUNC);
13159	if (err < 0)
13160	return libbpf_err_ptr(err);
13161	offsets = resolved_offsets;
13162	}
13163
13164	attach_type = session ? BPF_TRACE_UPROBE_SESSION : BPF_TRACE_UPROBE_MULTI;
13165
13166	lopts.uprobe_multi.path = path;
13167	lopts.uprobe_multi.offsets = offsets;
13168	lopts.uprobe_multi.ref_ctr_offsets = ref_ctr_offsets;
13169	lopts.uprobe_multi.cookies = cookies;
13170	lopts.uprobe_multi.cnt = cnt;
13171	lopts.uprobe_multi.flags = retprobe ? BPF_F_UPROBE_MULTI_RETURN : 0;
13172
13173	if (pid == 0)
13174	pid = getpid();
13175	if (pid > 0)
13176	lopts.uprobe_multi.pid = pid;
13177
13178	link = calloc(1, sizeof(*link));
13179	if (!link) {
13180	err = -ENOMEM;
13181	goto error;
13182	}
13183	link->detach = &bpf_link__detach_fd;
13184
13185	link_fd = bpf_link_create(prog_fd, 0, attach_type, &lopts);
13186	if (link_fd < 0) {
13187	err = -errno;
13188	pr_warn("prog '%s': failed to attach multi-uprobe: %s\n",
13189	prog->name, errstr(err));
13190	goto error;
13191	}
13192	link->fd = link_fd;
13193	free(resolved_offsets);
13194	return link;
13195
13196	error:
13197	free(resolved_offsets);
13198	free(link);
13199	return libbpf_err_ptr(err);
13200	}
13201
13202	LIBBPF_API struct bpf_link *
13203	bpf_program__attach_uprobe_opts(const struct bpf_program *prog, pid_t pid,
13204	const char *binary_path, size_t func_offset,
13205	const struct bpf_uprobe_opts *opts)
13206	{
13207	const char archive_path = NULL, archive_sep = NULL;
13208	char *legacy_probe = NULL;
13209	DECLARE_LIBBPF_OPTS(bpf_perf_event_opts, pe_opts);
13210	enum probe_attach_mode attach_mode;
13211	char full_path[PATH_MAX];
13212	struct bpf_link *link;
13213	size_t ref_ctr_off;
13214	int pfd, err;
13215	bool retprobe, legacy;
13216	const char *func_name;
13217
13218	if (!OPTS_VALID(opts, bpf_uprobe_opts))
13219	return libbpf_err_ptr(-EINVAL);
13220
13221	attach_mode = OPTS_GET(opts, attach_mode, PROBE_ATTACH_MODE_DEFAULT);
13222	retprobe = OPTS_GET(opts, retprobe, false);
13223	ref_ctr_off = OPTS_GET(opts, ref_ctr_offset, 0);
13224	pe_opts.bpf_cookie = OPTS_GET(opts, bpf_cookie, 0);
13225
13226	if (!binary_path)
13227	return libbpf_err_ptr(-EINVAL);
13228
13229	/* Check if "binary_path" refers to an archive. */
13230	archive_sep = strstr(binary_path, "!/");
13231	if (archive_sep) {
13232	full_path[0] = '\0';
13233	libbpf_strlcpy(full_path, binary_path,
13234	min(sizeof(full_path), (size_t)(archive_sep - binary_path + 1)));
13235	archive_path = full_path;
13236	binary_path = archive_sep + 2;
13237	} else if (!strchr(binary_path, '/')) {
13238	err = resolve_full_path(binary_path, full_path, sizeof(full_path));
13239	if (err) {
13240	pr_warn("prog '%s': failed to resolve full path for '%s': %s\n",
13241	prog->name, binary_path, errstr(err));
13242	return libbpf_err_ptr(err);
13243	}
13244	binary_path = full_path;
13245	}
13246	func_name = OPTS_GET(opts, func_name, NULL);
13247	if (func_name) {
13248	long sym_off;
13249
13250	if (archive_path) {
13251	sym_off = elf_find_func_offset_from_archive(archive_path, binary_path,
13252	func_name);
13253	binary_path = archive_path;
13254	} else {
13255	sym_off = elf_find_func_offset_from_file(binary_path, func_name);
13256	}
13257	if (sym_off < 0)
13258	return libbpf_err_ptr(sym_off);
13259	func_offset += sym_off;
13260	}
13261
13262	legacy = determine_uprobe_perf_type() < 0;
13263	switch (attach_mode) {
13264	case PROBE_ATTACH_MODE_LEGACY:
13265	legacy = true;
13266	pe_opts.force_ioctl_attach = true;
13267	break;
13268	case PROBE_ATTACH_MODE_PERF:
13269	if (legacy)
13270	return libbpf_err_ptr(-ENOTSUP);
13271	pe_opts.force_ioctl_attach = true;
13272	break;
13273	case PROBE_ATTACH_MODE_LINK:
13274	if (legacy \|\| !kernel_supports(prog->obj, FEAT_PERF_LINK))
13275	return libbpf_err_ptr(-ENOTSUP);
13276	break;
13277	case PROBE_ATTACH_MODE_DEFAULT:
13278	break;
13279	default:
13280	return libbpf_err_ptr(-EINVAL);
13281	}
13282
13283	if (!legacy) {
13284	pfd = perf_event_open_probe(true /* uprobe */, retprobe, binary_path,
13285	func_offset, pid, ref_ctr_off);
13286	} else {
13287	char probe_name[MAX_EVENT_NAME_LEN];
13288
13289	if (ref_ctr_off)
13290	return libbpf_err_ptr(-EINVAL);
13291
13292	gen_probe_legacy_event_name(probe_name, sizeof(probe_name),
13293	strrchr(binary_path, '/') ? : binary_path,
13294	func_offset);
13295
13296	legacy_probe = strdup(probe_name);
13297	if (!legacy_probe)
13298	return libbpf_err_ptr(-ENOMEM);
13299
13300	pfd = perf_event_uprobe_open_legacy(legacy_probe, retprobe,
13301	binary_path, func_offset, pid);
13302	}
13303	if (pfd < 0) {
13304	err = pfd;
13305	pr_warn("prog '%s': failed to create %s '%s:0x%zx' perf event: %s\n",
13306	prog->name, retprobe ? "uretprobe" : "uprobe",
13307	binary_path, func_offset,
13308	errstr(err));
13309	goto err_out;
13310	}
13311
13312	link = bpf_program__attach_perf_event_opts(prog, pfd, &pe_opts);
13313	err = libbpf_get_error(link);
13314	if (err) {
13315	close(pfd);
13316	pr_warn("prog '%s': failed to attach to %s '%s:0x%zx': %s\n",
13317	prog->name, retprobe ? "uretprobe" : "uprobe",
13318	binary_path, func_offset,
13319	errstr(err));
13320	goto err_clean_legacy;
13321	}
13322	if (legacy) {
13323	struct bpf_link_perf *perf_link = container_of(link, struct bpf_link_perf, link);
13324
13325	perf_link->legacy_probe_name = legacy_probe;
13326	perf_link->legacy_is_kprobe = false;
13327	perf_link->legacy_is_retprobe = retprobe;
13328	}
13329	return link;
13330
13331	err_clean_legacy:
13332	if (legacy)
13333	remove_uprobe_event_legacy(legacy_probe, retprobe);
13334	err_out:
13335	free(legacy_probe);
13336	return libbpf_err_ptr(err);
13337	}
13338
13339	/* Format of u[ret]probe section definition supporting auto-attach:
13340	* u[ret]probe/binary:function[+offset]
13341	*
13342	* binary can be an absolute/relative path or a filename; the latter is resolved to a
13343	* full binary path via bpf_program__attach_uprobe_opts.
13344	*
13345	* Specifying uprobe+ ensures we carry out strict matching; either "uprobe" must be
13346	* specified (and auto-attach is not possible) or the above format is specified for
13347	* auto-attach.
13348	*/
13349	static int attach_uprobe(const struct bpf_program prog, long cookie, struct bpf_link *link)
13350	{
13351	DECLARE_LIBBPF_OPTS(bpf_uprobe_opts, opts);
13352	char probe_type = NULL, binary_path = NULL, func_name = NULL, func_off;
13353	int n, c, ret = -EINVAL;
13354	long offset = 0;
13355
13356	*link = NULL;
13357
13358	n = sscanf(prog->sec_name, "%m[^/]/%m[^:]:%m[^\n]",
13359	&probe_type, &binary_path, &func_name);
13360	switch (n) {
13361	case 1:
13362	/* handle SEC("u[ret]probe") - format is valid, but auto-attach is impossible. */
13363	ret = 0;
13364	break;
13365	case 2:
13366	pr_warn("prog '%s': section '%s' missing ':function[+offset]' specification\n",
13367	prog->name, prog->sec_name);
13368	break;
13369	case 3:
13370	/* check if user specifies `+offset`, if yes, this should be
13371	* the last part of the string, make sure sscanf read to EOL
13372	*/
13373	func_off = strrchr(func_name, '+');
13374	if (func_off) {
13375	n = sscanf(func_off, "+%li%n", &offset, &c);
13376	if (n == 1 && *(func_off + c) == '\0')
13377	func_off[0] = '\0';
13378	else
13379	offset = 0;
13380	}
13381	opts.retprobe = strcmp(probe_type, "uretprobe") == 0 \|\|
13382	strcmp(probe_type, "uretprobe.s") == 0;
13383	if (opts.retprobe && offset != 0) {
13384	pr_warn("prog '%s': uretprobes do not support offset specification\n",
13385	prog->name);
13386	break;
13387	}
13388	opts.func_name = func_name;
13389	*link = bpf_program__attach_uprobe_opts(prog, -1, binary_path, offset, &opts);
13390	ret = libbpf_get_error(*link);
13391	break;
13392	default:
13393	pr_warn("prog '%s': invalid format of section definition '%s'\n", prog->name,
13394	prog->sec_name);
13395	break;
13396	}
13397	free(probe_type);
13398	free(binary_path);
13399	free(func_name);
13400
13401	return ret;
13402	}
13403
13404	struct bpf_link bpf_program__attach_uprobe(const struct bpf_program prog,
13405	bool retprobe, pid_t pid,
13406	const char *binary_path,
13407	size_t func_offset)
13408	{
13409	DECLARE_LIBBPF_OPTS(bpf_uprobe_opts, opts, .retprobe = retprobe);
13410
13411	return bpf_program__attach_uprobe_opts(prog, pid, binary_path, func_offset, &opts);
13412	}
13413
13414	struct bpf_link bpf_program__attach_usdt(const struct bpf_program prog,
13415	pid_t pid, const char *binary_path,
13416	const char usdt_provider, const char usdt_name,
13417	const struct bpf_usdt_opts *opts)
13418	{
13419	char resolved_path[512];
13420	struct bpf_object *obj = prog->obj;
13421	struct bpf_link *link;
13422	__u64 usdt_cookie;
13423	int err;
13424
13425	if (!OPTS_VALID(opts, bpf_uprobe_opts))
13426	return libbpf_err_ptr(-EINVAL);
13427
13428	if (bpf_program__fd(prog) < 0) {
13429	pr_warn("prog '%s': can't attach BPF program without FD (was it loaded?)\n",
13430	prog->name);
13431	return libbpf_err_ptr(-EINVAL);
13432	}
13433
13434	if (!binary_path)
13435	return libbpf_err_ptr(-EINVAL);
13436
13437	if (!strchr(binary_path, '/')) {
13438	err = resolve_full_path(binary_path, resolved_path, sizeof(resolved_path));
13439	if (err) {
13440	pr_warn("prog '%s': failed to resolve full path for '%s': %s\n",
13441	prog->name, binary_path, errstr(err));
13442	return libbpf_err_ptr(err);
13443	}
13444	binary_path = resolved_path;
13445	}
13446
13447	/* USDT manager is instantiated lazily on first USDT attach. It will
13448	* be destroyed together with BPF object in bpf_object__close().
13449	*/
13450	if (IS_ERR(obj->usdt_man))
13451	return libbpf_ptr(obj->usdt_man);
13452	if (!obj->usdt_man) {
13453	obj->usdt_man = usdt_manager_new(obj);
13454	if (IS_ERR(obj->usdt_man))
13455	return libbpf_ptr(obj->usdt_man);
13456	}
13457
13458	usdt_cookie = OPTS_GET(opts, usdt_cookie, 0);
13459	link = usdt_manager_attach_usdt(obj->usdt_man, prog, pid, binary_path,
13460	usdt_provider, usdt_name, usdt_cookie);
13461	err = libbpf_get_error(link);
13462	if (err)
13463	return libbpf_err_ptr(err);
13464	return link;
13465	}
13466
13467	static int attach_usdt(const struct bpf_program prog, long cookie, struct bpf_link *link)
13468	{
13469	char path = NULL, provider = NULL, *name = NULL;
13470	const char *sec_name;
13471	int n, err;
13472
13473	sec_name = bpf_program__section_name(prog);
13474	if (strcmp(sec_name, "usdt") == 0) {
13475	/* no auto-attach for just SEC("usdt") */
13476	*link = NULL;
13477	return 0;
13478	}
13479
13480	n = sscanf(sec_name, "usdt/%m[^:]:%m[^:]:%m[^:]", &path, &provider, &name);
13481	if (n != 3) {
13482	pr_warn("invalid section '%s', expected SEC(\"usdt/<path>:<provider>:<name>\")\n",
13483	sec_name);
13484	err = -EINVAL;
13485	} else {
13486	link = bpf_program__attach_usdt(prog, -1 / any process */, path,
13487	provider, name, NULL);
13488	err = libbpf_get_error(*link);
13489	}
13490	free(path);
13491	free(provider);
13492	free(name);
13493	return err;
13494	}
13495
13496	static int determine_tracepoint_id(const char *tp_category,
13497	const char *tp_name)
13498	{
13499	char file[PATH_MAX];
13500	int ret;
13501
13502	ret = snprintf(file, sizeof(file), "%s/events/%s/%s/id",
13503	tracefs_path(), tp_category, tp_name);
13504	if (ret < 0)
13505	return -errno;
13506	if (ret >= sizeof(file)) {
13507	pr_debug("tracepoint %s/%s path is too long\n",
13508	tp_category, tp_name);
13509	return -E2BIG;
13510	}
13511	return parse_uint_from_file(file, "%d\n");
13512	}
13513
13514	static int perf_event_open_tracepoint(const char *tp_category,
13515	const char *tp_name)
13516	{
13517	const size_t attr_sz = sizeof(struct perf_event_attr);
13518	struct perf_event_attr attr;
13519	int tp_id, pfd, err;
13520
13521	tp_id = determine_tracepoint_id(tp_category, tp_name);
13522	if (tp_id < 0) {
13523	pr_warn("failed to determine tracepoint '%s/%s' perf event ID: %s\n",
13524	tp_category, tp_name,
13525	errstr(tp_id));
13526	return tp_id;
13527	}
13528
13529	memset(&attr, 0, attr_sz);
13530	attr.type = PERF_TYPE_TRACEPOINT;
13531	attr.size = attr_sz;
13532	attr.config = tp_id;
13533
13534	pfd = syscall(__NR_perf_event_open, &attr, -1 /* pid /, 0 / cpu */,
13535	-1 /* group_fd */, PERF_FLAG_FD_CLOEXEC);
13536	if (pfd < 0) {
13537	err = -errno;
13538	pr_warn("tracepoint '%s/%s' perf_event_open() failed: %s\n",
13539	tp_category, tp_name,
13540	errstr(err));
13541	return err;
13542	}
13543	return pfd;
13544	}
13545
13546	struct bpf_link bpf_program__attach_tracepoint_opts(const struct bpf_program prog,
13547	const char *tp_category,
13548	const char *tp_name,
13549	const struct bpf_tracepoint_opts *opts)
13550	{
13551	DECLARE_LIBBPF_OPTS(bpf_perf_event_opts, pe_opts);
13552	struct bpf_link *link;
13553	int pfd, err;
13554
13555	if (!OPTS_VALID(opts, bpf_tracepoint_opts))
13556	return libbpf_err_ptr(-EINVAL);
13557
13558	pe_opts.bpf_cookie = OPTS_GET(opts, bpf_cookie, 0);
13559
13560	pfd = perf_event_open_tracepoint(tp_category, tp_name);
13561	if (pfd < 0) {
13562	pr_warn("prog '%s': failed to create tracepoint '%s/%s' perf event: %s\n",
13563	prog->name, tp_category, tp_name,
13564	errstr(pfd));
13565	return libbpf_err_ptr(pfd);
13566	}
13567	link = bpf_program__attach_perf_event_opts(prog, pfd, &pe_opts);
13568	err = libbpf_get_error(link);
13569	if (err) {
13570	close(pfd);
13571	pr_warn("prog '%s': failed to attach to tracepoint '%s/%s': %s\n",
13572	prog->name, tp_category, tp_name,
13573	errstr(err));
13574	return libbpf_err_ptr(err);
13575	}
13576	return link;
13577	}
13578
13579	struct bpf_link bpf_program__attach_tracepoint(const struct bpf_program prog,
13580	const char *tp_category,
13581	const char *tp_name)
13582	{
13583	return bpf_program__attach_tracepoint_opts(prog, tp_category, tp_name, NULL);
13584	}
13585
13586	/*
13587	* Match section name against a prefix array. Returns pointer past
13588	* "prefix/" on match, empty string for bare sections (exact prefix
13589	* match), or NULL if no prefix matches.
13590	*/
13591	static const char sec_name_match_prefix(const char sec_name,
13592	const char const prefixes,
13593	size_t n)
13594	{
13595	size_t i;
13596
13597	for (i = 0; i < n; i++) {
13598	size_t pfx_len;
13599
13600	if (!str_has_pfx(sec_name, prefixes[i]))
13601	continue;
13602
13603	pfx_len = strlen(prefixes[i]);
13604	if (sec_name[pfx_len] == '\0')
13605	return sec_name + pfx_len;
13606
13607	if (sec_name[pfx_len] != '/' \|\| sec_name[pfx_len + 1] == '\0')
13608	continue;
13609
13610	return sec_name + pfx_len + 1;
13611	}
13612	return NULL;
13613	}
13614
13615	static int attach_tp(const struct bpf_program prog, long cookie, struct bpf_link *link)
13616	{
13617	static const char *const prefixes[] = {
13618	"tp.s",
13619	"tp",
13620	"tracepoint.s",
13621	"tracepoint",
13622	};
13623	char sec_name, tp_cat, *tp_name;
13624	const char *match;
13625
13626	*link = NULL;
13627
13628	match = sec_name_match_prefix(prog->sec_name, prefixes, ARRAY_SIZE(prefixes));
13629	if (!match) {
13630	pr_warn("prog '%s': invalid section name '%s'\n", prog->name, prog->sec_name);
13631	return -EINVAL;
13632	}
13633	if (!match[0]) /* bare section name no autoattach */
13634	return 0;
13635
13636	sec_name = strdup(prog->sec_name);
13637	if (!sec_name)
13638	return -ENOMEM;
13639
13640	tp_cat = sec_name + (match - prog->sec_name);
13641	tp_name = strchr(tp_cat, '/');
13642	if (!tp_name) {
13643	free(sec_name);
13644	return -EINVAL;
13645	}
13646	*tp_name = '\0';
13647	tp_name++;
13648
13649	*link = bpf_program__attach_tracepoint(prog, tp_cat, tp_name);
13650	free(sec_name);
13651	return libbpf_get_error(*link);
13652	}
13653
13654	struct bpf_link *
13655	bpf_program__attach_raw_tracepoint_opts(const struct bpf_program *prog,
13656	const char *tp_name,
13657	struct bpf_raw_tracepoint_opts *opts)
13658	{
13659	LIBBPF_OPTS(bpf_raw_tp_opts, raw_opts);
13660	struct bpf_link *link;
13661	int prog_fd, pfd;
13662
13663	if (!OPTS_VALID(opts, bpf_raw_tracepoint_opts))
13664	return libbpf_err_ptr(-EINVAL);
13665
13666	prog_fd = bpf_program__fd(prog);
13667	if (prog_fd < 0) {
13668	pr_warn("prog '%s': can't attach before loaded\n", prog->name);
13669	return libbpf_err_ptr(-EINVAL);
13670	}
13671
13672	link = calloc(1, sizeof(*link));
13673	if (!link)
13674	return libbpf_err_ptr(-ENOMEM);
13675	link->detach = &bpf_link__detach_fd;
13676
13677	raw_opts.tp_name = tp_name;
13678	raw_opts.cookie = OPTS_GET(opts, cookie, 0);
13679	pfd = bpf_raw_tracepoint_open_opts(prog_fd, &raw_opts);
13680	if (pfd < 0) {
13681	pfd = -errno;
13682	free(link);
13683	pr_warn("prog '%s': failed to attach to raw tracepoint '%s': %s\n",
13684	prog->name, tp_name, errstr(pfd));
13685	return libbpf_err_ptr(pfd);
13686	}
13687	link->fd = pfd;
13688	return link;
13689	}
13690
13691	struct bpf_link bpf_program__attach_raw_tracepoint(const struct bpf_program prog,
13692	const char *tp_name)
13693	{
13694	return bpf_program__attach_raw_tracepoint_opts(prog, tp_name, NULL);
13695	}
13696
13697	static int attach_raw_tp(const struct bpf_program prog, long cookie, struct bpf_link *link)
13698	{
13699	static const char *const prefixes[] = {
13700	"raw_tp",
13701	"raw_tracepoint",
13702	"raw_tp.w",
13703	"raw_tracepoint.w",
13704	"raw_tp.s",
13705	"raw_tracepoint.s",
13706	};
13707	const char *match;
13708
13709	*link = NULL;
13710
13711	match = sec_name_match_prefix(prog->sec_name, prefixes, ARRAY_SIZE(prefixes));
13712	if (!match) {
13713	pr_warn("prog '%s': invalid section name '%s'\n", prog->name, prog->sec_name);
13714	return -EINVAL;
13715	}
13716	if (!match[0])
13717	return 0;
13718
13719	*link = bpf_program__attach_raw_tracepoint(prog, match);
13720	return libbpf_get_error(*link);
13721	}
13722
13723	/* Common logic for all BPF program types that attach to a btf_id */
13724	static struct bpf_link bpf_program__attach_btf_id(const struct bpf_program prog,
13725	const struct bpf_trace_opts *opts)
13726	{
13727	LIBBPF_OPTS(bpf_link_create_opts, link_opts);
13728	struct bpf_link *link;
13729	int prog_fd, pfd;
13730
13731	if (!OPTS_VALID(opts, bpf_trace_opts))
13732	return libbpf_err_ptr(-EINVAL);
13733
13734	prog_fd = bpf_program__fd(prog);
13735	if (prog_fd < 0) {
13736	pr_warn("prog '%s': can't attach before loaded\n", prog->name);
13737	return libbpf_err_ptr(-EINVAL);
13738	}
13739
13740	link = calloc(1, sizeof(*link));
13741	if (!link)
13742	return libbpf_err_ptr(-ENOMEM);
13743	link->detach = &bpf_link__detach_fd;
13744
13745	/* libbpf is smart enough to redirect to BPF_RAW_TRACEPOINT_OPEN on old kernels */
13746	link_opts.tracing.cookie = OPTS_GET(opts, cookie, 0);
13747	pfd = bpf_link_create(prog_fd, 0, bpf_program__expected_attach_type(prog), &link_opts);
13748	if (pfd < 0) {
13749	pfd = -errno;
13750	free(link);
13751	pr_warn("prog '%s': failed to attach: %s\n",
13752	prog->name, errstr(pfd));
13753	return libbpf_err_ptr(pfd);
13754	}
13755	link->fd = pfd;
13756	return link;
13757	}
13758
13759	struct bpf_link bpf_program__attach_trace(const struct bpf_program prog)
13760	{
13761	return bpf_program__attach_btf_id(prog, NULL);
13762	}
13763
13764	struct bpf_link bpf_program__attach_trace_opts(const struct bpf_program prog,
13765	const struct bpf_trace_opts *opts)
13766	{
13767	return bpf_program__attach_btf_id(prog, opts);
13768	}
13769
13770	struct bpf_link bpf_program__attach_lsm(const struct bpf_program prog)
13771	{
13772	return bpf_program__attach_btf_id(prog, NULL);
13773	}
13774
13775	static int attach_trace(const struct bpf_program prog, long cookie, struct bpf_link *link)
13776	{
13777	*link = bpf_program__attach_trace(prog);
13778	return libbpf_get_error(*link);
13779	}
13780
13781	static int attach_lsm(const struct bpf_program prog, long cookie, struct bpf_link *link)
13782	{
13783	*link = bpf_program__attach_lsm(prog);
13784	return libbpf_get_error(*link);
13785	}
13786
13787	static struct bpf_link *
13788	bpf_program_attach_fd(const struct bpf_program *prog,
13789	int target_fd, const char *target_name,
13790	const struct bpf_link_create_opts *opts)
13791	{
13792	enum bpf_attach_type attach_type;
13793	struct bpf_link *link;
13794	int prog_fd, link_fd;
13795
13796	prog_fd = bpf_program__fd(prog);
13797	if (prog_fd < 0) {
13798	pr_warn("prog '%s': can't attach before loaded\n", prog->name);
13799	return libbpf_err_ptr(-EINVAL);
13800	}
13801
13802	link = calloc(1, sizeof(*link));
13803	if (!link)
13804	return libbpf_err_ptr(-ENOMEM);
13805	link->detach = &bpf_link__detach_fd;
13806
13807	attach_type = bpf_program__expected_attach_type(prog);
13808	link_fd = bpf_link_create(prog_fd, target_fd, attach_type, opts);
13809	if (link_fd < 0) {
13810	link_fd = -errno;
13811	free(link);
13812	pr_warn("prog '%s': failed to attach to %s: %s\n",
13813	prog->name, target_name,
13814	errstr(link_fd));
13815	return libbpf_err_ptr(link_fd);
13816	}
13817	link->fd = link_fd;
13818	return link;
13819	}
13820
13821	struct bpf_link *
13822	bpf_program__attach_cgroup(const struct bpf_program *prog, int cgroup_fd)
13823	{
13824	return bpf_program_attach_fd(prog, cgroup_fd, "cgroup", NULL);
13825	}
13826
13827	struct bpf_link *
13828	bpf_program__attach_netns(const struct bpf_program *prog, int netns_fd)
13829	{
13830	return bpf_program_attach_fd(prog, netns_fd, "netns", NULL);
13831	}
13832
13833	struct bpf_link *
13834	bpf_program__attach_sockmap(const struct bpf_program *prog, int map_fd)
13835	{
13836	return bpf_program_attach_fd(prog, map_fd, "sockmap", NULL);
13837	}
13838
13839	struct bpf_link bpf_program__attach_xdp(const struct bpf_program prog, int ifindex)
13840	{
13841	/* target_fd/target_ifindex use the same field in LINK_CREATE */
13842	return bpf_program_attach_fd(prog, ifindex, "xdp", NULL);
13843	}
13844
13845	struct bpf_link *
13846	bpf_program__attach_cgroup_opts(const struct bpf_program *prog, int cgroup_fd,
13847	const struct bpf_cgroup_opts *opts)
13848	{
13849	LIBBPF_OPTS(bpf_link_create_opts, link_create_opts);
13850	__u32 relative_id;
13851	int relative_fd;
13852
13853	if (!OPTS_VALID(opts, bpf_cgroup_opts))
13854	return libbpf_err_ptr(-EINVAL);
13855
13856	relative_id = OPTS_GET(opts, relative_id, 0);
13857	relative_fd = OPTS_GET(opts, relative_fd, 0);
13858
13859	if (relative_fd && relative_id) {
13860	pr_warn("prog '%s': relative_fd and relative_id cannot be set at the same time\n",
13861	prog->name);
13862	return libbpf_err_ptr(-EINVAL);
13863	}
13864
13865	link_create_opts.cgroup.expected_revision = OPTS_GET(opts, expected_revision, 0);
13866	link_create_opts.cgroup.relative_fd = relative_fd;
13867	link_create_opts.cgroup.relative_id = relative_id;
13868	link_create_opts.flags = OPTS_GET(opts, flags, 0);
13869
13870	return bpf_program_attach_fd(prog, cgroup_fd, "cgroup", &link_create_opts);
13871	}
13872
13873	struct bpf_link *
13874	bpf_program__attach_tcx(const struct bpf_program *prog, int ifindex,
13875	const struct bpf_tcx_opts *opts)
13876	{
13877	LIBBPF_OPTS(bpf_link_create_opts, link_create_opts);
13878	__u32 relative_id;
13879	int relative_fd;
13880
13881	if (!OPTS_VALID(opts, bpf_tcx_opts))
13882	return libbpf_err_ptr(-EINVAL);
13883
13884	relative_id = OPTS_GET(opts, relative_id, 0);
13885	relative_fd = OPTS_GET(opts, relative_fd, 0);
13886
13887	/* validate we don't have unexpected combinations of non-zero fields */
13888	if (!ifindex) {
13889	pr_warn("prog '%s': target netdevice ifindex cannot be zero\n",
13890	prog->name);
13891	return libbpf_err_ptr(-EINVAL);
13892	}
13893	if (relative_fd && relative_id) {
13894	pr_warn("prog '%s': relative_fd and relative_id cannot be set at the same time\n",
13895	prog->name);
13896	return libbpf_err_ptr(-EINVAL);
13897	}
13898
13899	link_create_opts.tcx.expected_revision = OPTS_GET(opts, expected_revision, 0);
13900	link_create_opts.tcx.relative_fd = relative_fd;
13901	link_create_opts.tcx.relative_id = relative_id;
13902	link_create_opts.flags = OPTS_GET(opts, flags, 0);
13903
13904	/* target_fd/target_ifindex use the same field in LINK_CREATE */
13905	return bpf_program_attach_fd(prog, ifindex, "tcx", &link_create_opts);
13906	}
13907
13908	struct bpf_link *
13909	bpf_program__attach_netkit(const struct bpf_program *prog, int ifindex,
13910	const struct bpf_netkit_opts *opts)
13911	{
13912	LIBBPF_OPTS(bpf_link_create_opts, link_create_opts);
13913	__u32 relative_id;
13914	int relative_fd;
13915
13916	if (!OPTS_VALID(opts, bpf_netkit_opts))
13917	return libbpf_err_ptr(-EINVAL);
13918
13919	relative_id = OPTS_GET(opts, relative_id, 0);
13920	relative_fd = OPTS_GET(opts, relative_fd, 0);
13921
13922	/* validate we don't have unexpected combinations of non-zero fields */
13923	if (!ifindex) {
13924	pr_warn("prog '%s': target netdevice ifindex cannot be zero\n",
13925	prog->name);
13926	return libbpf_err_ptr(-EINVAL);
13927	}
13928	if (relative_fd && relative_id) {
13929	pr_warn("prog '%s': relative_fd and relative_id cannot be set at the same time\n",
13930	prog->name);
13931	return libbpf_err_ptr(-EINVAL);
13932	}
13933
13934	link_create_opts.netkit.expected_revision = OPTS_GET(opts, expected_revision, 0);
13935	link_create_opts.netkit.relative_fd = relative_fd;
13936	link_create_opts.netkit.relative_id = relative_id;
13937	link_create_opts.flags = OPTS_GET(opts, flags, 0);
13938
13939	return bpf_program_attach_fd(prog, ifindex, "netkit", &link_create_opts);
13940	}
13941
13942	struct bpf_link bpf_program__attach_freplace(const struct bpf_program prog,
13943	int target_fd,
13944	const char *attach_func_name)
13945	{
13946	int btf_id;
13947
13948	if (!!target_fd != !!attach_func_name) {
13949	pr_warn("prog '%s': supply none or both of target_fd and attach_func_name\n",
13950	prog->name);
13951	return libbpf_err_ptr(-EINVAL);
13952	}
13953
13954	if (prog->type != BPF_PROG_TYPE_EXT) {
13955	pr_warn("prog '%s': only BPF_PROG_TYPE_EXT can attach as freplace\n",
13956	prog->name);
13957	return libbpf_err_ptr(-EINVAL);
13958	}
13959
13960	if (target_fd) {
13961	LIBBPF_OPTS(bpf_link_create_opts, target_opts);
13962
13963	btf_id = libbpf_find_prog_btf_id(attach_func_name, target_fd, prog->obj->token_fd);
13964	if (btf_id < 0)
13965	return libbpf_err_ptr(btf_id);
13966
13967	target_opts.target_btf_id = btf_id;
13968
13969	return bpf_program_attach_fd(prog, target_fd, "freplace",
13970	&target_opts);
13971	} else {
13972	/* no target, so use raw_tracepoint_open for compatibility
13973	* with old kernels
13974	*/
13975	return bpf_program__attach_trace(prog);
13976	}
13977	}
13978
13979	struct bpf_link *
13980	bpf_program__attach_iter(const struct bpf_program *prog,
13981	const struct bpf_iter_attach_opts *opts)
13982	{
13983	DECLARE_LIBBPF_OPTS(bpf_link_create_opts, link_create_opts);
13984	struct bpf_link *link;
13985	int prog_fd, link_fd;
13986	__u32 target_fd = 0;
13987
13988	if (!OPTS_VALID(opts, bpf_iter_attach_opts))
13989	return libbpf_err_ptr(-EINVAL);
13990
13991	link_create_opts.iter_info = OPTS_GET(opts, link_info, (void *)0);
13992	link_create_opts.iter_info_len = OPTS_GET(opts, link_info_len, 0);
13993
13994	prog_fd = bpf_program__fd(prog);
13995	if (prog_fd < 0) {
13996	pr_warn("prog '%s': can't attach before loaded\n", prog->name);
13997	return libbpf_err_ptr(-EINVAL);
13998	}
13999
14000	link = calloc(1, sizeof(*link));
14001	if (!link)
14002	return libbpf_err_ptr(-ENOMEM);
14003	link->detach = &bpf_link__detach_fd;
14004
14005	link_fd = bpf_link_create(prog_fd, target_fd, BPF_TRACE_ITER,
14006	&link_create_opts);
14007	if (link_fd < 0) {
14008	link_fd = -errno;
14009	free(link);
14010	pr_warn("prog '%s': failed to attach to iterator: %s\n",
14011	prog->name, errstr(link_fd));
14012	return libbpf_err_ptr(link_fd);
14013	}
14014	link->fd = link_fd;
14015	return link;
14016	}
14017
14018	static int attach_iter(const struct bpf_program prog, long cookie, struct bpf_link *link)
14019	{
14020	*link = bpf_program__attach_iter(prog, NULL);
14021	return libbpf_get_error(*link);
14022	}
14023
14024	struct bpf_link bpf_program__attach_netfilter(const struct bpf_program prog,
14025	const struct bpf_netfilter_opts *opts)
14026	{
14027	LIBBPF_OPTS(bpf_link_create_opts, lopts);
14028	struct bpf_link *link;
14029	int prog_fd, link_fd;
14030
14031	if (!OPTS_VALID(opts, bpf_netfilter_opts))
14032	return libbpf_err_ptr(-EINVAL);
14033
14034	prog_fd = bpf_program__fd(prog);
14035	if (prog_fd < 0) {
14036	pr_warn("prog '%s': can't attach before loaded\n", prog->name);
14037	return libbpf_err_ptr(-EINVAL);
14038	}
14039
14040	link = calloc(1, sizeof(*link));
14041	if (!link)
14042	return libbpf_err_ptr(-ENOMEM);
14043
14044	link->detach = &bpf_link__detach_fd;
14045
14046	lopts.netfilter.pf = OPTS_GET(opts, pf, 0);
14047	lopts.netfilter.hooknum = OPTS_GET(opts, hooknum, 0);
14048	lopts.netfilter.priority = OPTS_GET(opts, priority, 0);
14049	lopts.netfilter.flags = OPTS_GET(opts, flags, 0);
14050
14051	link_fd = bpf_link_create(prog_fd, 0, BPF_NETFILTER, &lopts);
14052	if (link_fd < 0) {
14053	link_fd = -errno;
14054	free(link);
14055	pr_warn("prog '%s': failed to attach to netfilter: %s\n",
14056	prog->name, errstr(link_fd));
14057	return libbpf_err_ptr(link_fd);
14058	}
14059	link->fd = link_fd;
14060
14061	return link;
14062	}
14063
14064	struct bpf_link bpf_program__attach(const struct bpf_program prog)
14065	{
14066	struct bpf_link *link = NULL;
14067	int err;
14068
14069	if (!prog->sec_def \|\| !prog->sec_def->prog_attach_fn)
14070	return libbpf_err_ptr(-EOPNOTSUPP);
14071
14072	if (bpf_program__fd(prog) < 0) {
14073	pr_warn("prog '%s': can't attach BPF program without FD (was it loaded?)\n",
14074	prog->name);
14075	return libbpf_err_ptr(-EINVAL);
14076	}
14077
14078	err = prog->sec_def->prog_attach_fn(prog, prog->sec_def->cookie, &link);
14079	if (err)
14080	return libbpf_err_ptr(err);
14081
14082	/* When calling bpf_program__attach() explicitly, auto-attach support
14083	* is expected to work, so NULL returned link is considered an error.
14084	* This is different for skeleton's attach, see comment in
14085	* bpf_object__attach_skeleton().
14086	*/
14087	if (!link)
14088	return libbpf_err_ptr(-EOPNOTSUPP);
14089
14090	return link;
14091	}
14092
14093	struct bpf_link_struct_ops {
14094	struct bpf_link link;
14095	int map_fd;
14096	};
14097
14098	static int bpf_link__detach_struct_ops(struct bpf_link *link)
14099	{
14100	struct bpf_link_struct_ops *st_link;
14101	__u32 zero = 0;
14102
14103	st_link = container_of(link, struct bpf_link_struct_ops, link);
14104
14105	if (st_link->map_fd < 0)
14106	/* w/o a real link */
14107	return bpf_map_delete_elem(link->fd, &zero);
14108
14109	return close(link->fd);
14110	}
14111
14112	struct bpf_link bpf_map__attach_struct_ops(const struct bpf_map map)
14113	{
14114	struct bpf_link_struct_ops *link;
14115	__u32 zero = 0;
14116	int err, fd;
14117
14118	if (!bpf_map__is_struct_ops(map)) {
14119	pr_warn("map '%s': can't attach non-struct_ops map\n", map->name);
14120	return libbpf_err_ptr(-EINVAL);
14121	}
14122
14123	if (map->fd < 0) {
14124	pr_warn("map '%s': can't attach BPF map without FD (was it created?)\n", map->name);
14125	return libbpf_err_ptr(-EINVAL);
14126	}
14127
14128	link = calloc(1, sizeof(*link));
14129	if (!link)
14130	return libbpf_err_ptr(-EINVAL);
14131
14132	/* kern_vdata should be prepared during the loading phase. */
14133	err = bpf_map_update_elem(map->fd, &zero, map->st_ops->kern_vdata, 0);
14134	/* It can be EBUSY if the map has been used to create or
14135	* update a link before. We don't allow updating the value of
14136	* a struct_ops once it is set. That ensures that the value
14137	* never changed. So, it is safe to skip EBUSY.
14138	*/
14139	if (err && (!(map->def.map_flags & BPF_F_LINK) \|\| err != -EBUSY)) {
14140	free(link);
14141	return libbpf_err_ptr(err);
14142	}
14143
14144	link->link.detach = bpf_link__detach_struct_ops;
14145
14146	if (!(map->def.map_flags & BPF_F_LINK)) {
14147	/* w/o a real link */
14148	link->link.fd = map->fd;
14149	link->map_fd = -1;
14150	return &link->link;
14151	}
14152
14153	fd = bpf_link_create(map->fd, 0, BPF_STRUCT_OPS, NULL);
14154	if (fd < 0) {
14155	free(link);
14156	return libbpf_err_ptr(fd);
14157	}
14158
14159	link->link.fd = fd;
14160	link->map_fd = map->fd;
14161
14162	return &link->link;
14163	}
14164
14165	/*
14166	* Swap the back struct_ops of a link with a new struct_ops map.
14167	*/
14168	int bpf_link__update_map(struct bpf_link link, const struct bpf_map map)
14169	{
14170	struct bpf_link_struct_ops *st_ops_link;
14171	__u32 zero = 0;
14172	int err;
14173
14174	if (!bpf_map__is_struct_ops(map))
14175	return libbpf_err(-EINVAL);
14176
14177	if (map->fd < 0) {
14178	pr_warn("map '%s': can't use BPF map without FD (was it created?)\n", map->name);
14179	return libbpf_err(-EINVAL);
14180	}
14181
14182	st_ops_link = container_of(link, struct bpf_link_struct_ops, link);
14183	/* Ensure the type of a link is correct */
14184	if (st_ops_link->map_fd < 0)
14185	return libbpf_err(-EINVAL);
14186
14187	err = bpf_map_update_elem(map->fd, &zero, map->st_ops->kern_vdata, 0);
14188	/* It can be EBUSY if the map has been used to create or
14189	* update a link before. We don't allow updating the value of
14190	* a struct_ops once it is set. That ensures that the value
14191	* never changed. So, it is safe to skip EBUSY.
14192	*/
14193	if (err && err != -EBUSY)
14194	return err;
14195
14196	err = bpf_link_update(link->fd, map->fd, NULL);
14197	if (err < 0)
14198	return err;
14199
14200	st_ops_link->map_fd = map->fd;
14201
14202	return 0;
14203	}
14204
14205	typedef enum bpf_perf_event_ret (bpf_perf_event_print_t)(struct perf_event_header hdr,
14206	void *private_data);
14207
14208	static enum bpf_perf_event_ret
14209	perf_event_read_simple(void *mmap_mem, size_t mmap_size, size_t page_size,
14210	void *copy_mem, size_t copy_size,
14211	bpf_perf_event_print_t fn, void *private_data)
14212	{
14213	struct perf_event_mmap_page *header = mmap_mem;
14214	__u64 data_head = ring_buffer_read_head(header);
14215	__u64 data_tail = header->data_tail;
14216	void base = ((__u8 )header) + page_size;
14217	int ret = LIBBPF_PERF_EVENT_CONT;
14218	struct perf_event_header *ehdr;
14219	size_t ehdr_size;
14220
14221	while (data_head != data_tail) {
14222	ehdr = base + (data_tail & (mmap_size - 1));
14223	ehdr_size = ehdr->size;
14224
14225	if (((void *)ehdr) + ehdr_size > base + mmap_size) {
14226	void *copy_start = ehdr;
14227	size_t len_first = base + mmap_size - copy_start;
14228	size_t len_second = ehdr_size - len_first;
14229
14230	if (*copy_size < ehdr_size) {
14231	free(*copy_mem);
14232	*copy_mem = malloc(ehdr_size);
14233	if (!*copy_mem) {
14234	*copy_size = 0;
14235	ret = LIBBPF_PERF_EVENT_ERROR;
14236	break;
14237	}
14238	*copy_size = ehdr_size;
14239	}
14240
14241	memcpy(*copy_mem, copy_start, len_first);
14242	memcpy(*copy_mem + len_first, base, len_second);
14243	ehdr = *copy_mem;
14244	}
14245
14246	ret = fn(ehdr, private_data);
14247	data_tail += ehdr_size;
14248	if (ret != LIBBPF_PERF_EVENT_CONT)
14249	break;
14250	}
14251
14252	ring_buffer_write_tail(header, data_tail);
14253	return libbpf_err(ret);
14254	}
14255
14256	struct perf_buffer;
14257
14258	struct perf_buffer_params {
14259	struct perf_event_attr *attr;
14260	/* if event_cb is specified, it takes precedence */
14261	perf_buffer_event_fn event_cb;
14262	/* sample_cb and lost_cb are higher-level common-case callbacks */
14263	perf_buffer_sample_fn sample_cb;
14264	perf_buffer_lost_fn lost_cb;
14265	void *ctx;
14266	int cpu_cnt;
14267	int *cpus;
14268	int *map_keys;
14269	};
14270
14271	struct perf_cpu_buf {
14272	struct perf_buffer *pb;
14273	void base; / mmap()'ed memory */
14274	void buf; / for reconstructing segmented data */
14275	size_t buf_size;
14276	int fd;
14277	int cpu;
14278	int map_key;
14279	};
14280
14281	struct perf_buffer {
14282	perf_buffer_event_fn event_cb;
14283	perf_buffer_sample_fn sample_cb;
14284	perf_buffer_lost_fn lost_cb;
14285	void ctx; / passed into callbacks */
14286
14287	size_t page_size;
14288	size_t mmap_size;
14289	struct perf_cpu_buf **cpu_bufs;
14290	struct epoll_event *events;
14291	int cpu_cnt; /* number of allocated CPU buffers */
14292	int epoll_fd; /* perf event FD */
14293	int map_fd; /* BPF_MAP_TYPE_PERF_EVENT_ARRAY BPF map FD */
14294	};
14295
14296	static void perf_buffer__free_cpu_buf(struct perf_buffer *pb,
14297	struct perf_cpu_buf *cpu_buf)
14298	{
14299	if (!cpu_buf)
14300	return;
14301	if (cpu_buf->base &&
14302	munmap(cpu_buf->base, pb->mmap_size + pb->page_size))
14303	pr_warn("failed to munmap cpu_buf #%d\n", cpu_buf->cpu);
14304	if (cpu_buf->fd >= 0) {
14305	ioctl(cpu_buf->fd, PERF_EVENT_IOC_DISABLE, 0);
14306	close(cpu_buf->fd);
14307	}
14308	free(cpu_buf->buf);
14309	free(cpu_buf);
14310	}
14311
14312	void perf_buffer__free(struct perf_buffer *pb)
14313	{
14314	int i;
14315
14316	if (IS_ERR_OR_NULL(pb))
14317	return;
14318	if (pb->cpu_bufs) {
14319	for (i = 0; i < pb->cpu_cnt; i++) {
14320	struct perf_cpu_buf *cpu_buf = pb->cpu_bufs[i];
14321
14322	if (!cpu_buf)
14323	continue;
14324
14325	bpf_map_delete_elem(pb->map_fd, &cpu_buf->map_key);
14326	perf_buffer__free_cpu_buf(pb, cpu_buf);
14327	}
14328	free(pb->cpu_bufs);
14329	}
14330	if (pb->epoll_fd >= 0)
14331	close(pb->epoll_fd);
14332	free(pb->events);
14333	free(pb);
14334	}
14335
14336	static struct perf_cpu_buf *
14337	perf_buffer__open_cpu_buf(struct perf_buffer pb, struct perf_event_attr attr,
14338	int cpu, int map_key)
14339	{
14340	struct perf_cpu_buf *cpu_buf;
14341	int err;
14342
14343	cpu_buf = calloc(1, sizeof(*cpu_buf));
14344	if (!cpu_buf)
14345	return ERR_PTR(-ENOMEM);
14346
14347	cpu_buf->pb = pb;
14348	cpu_buf->cpu = cpu;
14349	cpu_buf->map_key = map_key;
14350
14351	cpu_buf->fd = syscall(__NR_perf_event_open, attr, -1 /* pid */, cpu,
14352	-1, PERF_FLAG_FD_CLOEXEC);
14353	if (cpu_buf->fd < 0) {
14354	err = -errno;
14355	pr_warn("failed to open perf buffer event on cpu #%d: %s\n",
14356	cpu, errstr(err));
14357	goto error;
14358	}
14359
14360	cpu_buf->base = mmap(NULL, pb->mmap_size + pb->page_size,
14361	PROT_READ \| PROT_WRITE, MAP_SHARED,
14362	cpu_buf->fd, 0);
14363	if (cpu_buf->base == MAP_FAILED) {
14364	cpu_buf->base = NULL;
14365	err = -errno;
14366	pr_warn("failed to mmap perf buffer on cpu #%d: %s\n",
14367	cpu, errstr(err));
14368	goto error;
14369	}
14370
14371	if (ioctl(cpu_buf->fd, PERF_EVENT_IOC_ENABLE, 0) < 0) {
14372	err = -errno;
14373	pr_warn("failed to enable perf buffer event on cpu #%d: %s\n",
14374	cpu, errstr(err));
14375	goto error;
14376	}
14377
14378	return cpu_buf;
14379
14380	error:
14381	perf_buffer__free_cpu_buf(pb, cpu_buf);
14382	return (struct perf_cpu_buf *)ERR_PTR(err);
14383	}
14384
14385	static struct perf_buffer *__perf_buffer__new(int map_fd, size_t page_cnt,
14386	struct perf_buffer_params *p);
14387
14388	struct perf_buffer *perf_buffer__new(int map_fd, size_t page_cnt,
14389	perf_buffer_sample_fn sample_cb,
14390	perf_buffer_lost_fn lost_cb,
14391	void *ctx,
14392	const struct perf_buffer_opts *opts)
14393	{
14394	const size_t attr_sz = sizeof(struct perf_event_attr);
14395	struct perf_buffer_params p = {};
14396	struct perf_event_attr attr;
14397	__u32 sample_period;
14398
14399	if (!OPTS_VALID(opts, perf_buffer_opts))
14400	return libbpf_err_ptr(-EINVAL);
14401
14402	sample_period = OPTS_GET(opts, sample_period, 1);
14403	if (!sample_period)
14404	sample_period = 1;
14405
14406	memset(&attr, 0, attr_sz);
14407	attr.size = attr_sz;
14408	attr.config = PERF_COUNT_SW_BPF_OUTPUT;
14409	attr.type = PERF_TYPE_SOFTWARE;
14410	attr.sample_type = PERF_SAMPLE_RAW;
14411	attr.wakeup_events = sample_period;
14412
14413	p.attr = &attr;
14414	p.sample_cb = sample_cb;
14415	p.lost_cb = lost_cb;
14416	p.ctx = ctx;
14417
14418	return libbpf_ptr(__perf_buffer__new(map_fd, page_cnt, &p));
14419	}
14420
14421	struct perf_buffer *perf_buffer__new_raw(int map_fd, size_t page_cnt,
14422	struct perf_event_attr *attr,
14423	perf_buffer_event_fn event_cb, void *ctx,
14424	const struct perf_buffer_raw_opts *opts)
14425	{
14426	struct perf_buffer_params p = {};
14427
14428	if (!attr)
14429	return libbpf_err_ptr(-EINVAL);
14430
14431	if (!OPTS_VALID(opts, perf_buffer_raw_opts))
14432	return libbpf_err_ptr(-EINVAL);
14433
14434	p.attr = attr;
14435	p.event_cb = event_cb;
14436	p.ctx = ctx;
14437	p.cpu_cnt = OPTS_GET(opts, cpu_cnt, 0);
14438	p.cpus = OPTS_GET(opts, cpus, NULL);
14439	p.map_keys = OPTS_GET(opts, map_keys, NULL);
14440
14441	return libbpf_ptr(__perf_buffer__new(map_fd, page_cnt, &p));
14442	}
14443
14444	static struct perf_buffer *__perf_buffer__new(int map_fd, size_t page_cnt,
14445	struct perf_buffer_params *p)
14446	{
14447	const char *online_cpus_file = "/sys/devices/system/cpu/online";
14448	struct bpf_map_info map;
14449	struct perf_buffer *pb;
14450	bool *online = NULL;
14451	__u32 map_info_len;
14452	int err, i, j, n;
14453
14454	if (page_cnt == 0 \|\| (page_cnt & (page_cnt - 1))) {
14455	pr_warn("page count should be power of two, but is %zu\n",
14456	page_cnt);
14457	return ERR_PTR(-EINVAL);
14458	}
14459
14460	/* best-effort sanity checks */
14461	memset(&map, 0, sizeof(map));
14462	map_info_len = sizeof(map);
14463	err = bpf_map_get_info_by_fd(map_fd, &map, &map_info_len);
14464	if (err) {
14465	err = -errno;
14466	/* if BPF_OBJ_GET_INFO_BY_FD is supported, will return
14467	* -EBADFD, -EFAULT, or -E2BIG on real error
14468	*/
14469	if (err != -EINVAL) {
14470	pr_warn("failed to get map info for map FD %d: %s\n",
14471	map_fd, errstr(err));
14472	return ERR_PTR(err);
14473	}
14474	pr_debug("failed to get map info for FD %d; API not supported? Ignoring...\n",
14475	map_fd);
14476	} else {
14477	if (map.type != BPF_MAP_TYPE_PERF_EVENT_ARRAY) {
14478	pr_warn("map '%s' should be BPF_MAP_TYPE_PERF_EVENT_ARRAY\n",
14479	map.name);
14480	return ERR_PTR(-EINVAL);
14481	}
14482	}
14483
14484	pb = calloc(1, sizeof(*pb));
14485	if (!pb)
14486	return ERR_PTR(-ENOMEM);
14487
14488	pb->event_cb = p->event_cb;
14489	pb->sample_cb = p->sample_cb;
14490	pb->lost_cb = p->lost_cb;
14491	pb->ctx = p->ctx;
14492
14493	pb->page_size = getpagesize();
14494	pb->mmap_size = pb->page_size * page_cnt;
14495	pb->map_fd = map_fd;
14496
14497	pb->epoll_fd = epoll_create1(EPOLL_CLOEXEC);
14498	if (pb->epoll_fd < 0) {
14499	err = -errno;
14500	pr_warn("failed to create epoll instance: %s\n",
14501	errstr(err));
14502	goto error;
14503	}
14504
14505	if (p->cpu_cnt > 0) {
14506	pb->cpu_cnt = p->cpu_cnt;
14507	} else {
14508	pb->cpu_cnt = libbpf_num_possible_cpus();
14509	if (pb->cpu_cnt < 0) {
14510	err = pb->cpu_cnt;
14511	goto error;
14512	}
14513	if (map.max_entries && map.max_entries < pb->cpu_cnt)
14514	pb->cpu_cnt = map.max_entries;
14515	}
14516
14517	pb->events = calloc(pb->cpu_cnt, sizeof(*pb->events));
14518	if (!pb->events) {
14519	err = -ENOMEM;
14520	pr_warn("failed to allocate events: out of memory\n");
14521	goto error;
14522	}
14523	pb->cpu_bufs = calloc(pb->cpu_cnt, sizeof(*pb->cpu_bufs));
14524	if (!pb->cpu_bufs) {
14525	err = -ENOMEM;
14526	pr_warn("failed to allocate buffers: out of memory\n");
14527	goto error;
14528	}
14529
14530	err = parse_cpu_mask_file(online_cpus_file, &online, &n);
14531	if (err) {
14532	pr_warn("failed to get online CPU mask: %s\n", errstr(err));
14533	goto error;
14534	}
14535
14536	for (i = 0, j = 0; i < pb->cpu_cnt; i++) {
14537	struct perf_cpu_buf *cpu_buf;
14538	int cpu, map_key;
14539
14540	cpu = p->cpu_cnt > 0 ? p->cpus[i] : i;
14541	map_key = p->cpu_cnt > 0 ? p->map_keys[i] : i;
14542
14543	/* in case user didn't explicitly requested particular CPUs to
14544	* be attached to, skip offline/not present CPUs
14545	*/
14546	if (p->cpu_cnt <= 0 && (cpu >= n \|\| !online[cpu]))
14547	continue;
14548
14549	cpu_buf = perf_buffer__open_cpu_buf(pb, p->attr, cpu, map_key);
14550	if (IS_ERR(cpu_buf)) {
14551	err = PTR_ERR(cpu_buf);
14552	goto error;
14553	}
14554
14555	pb->cpu_bufs[j] = cpu_buf;
14556
14557	err = bpf_map_update_elem(pb->map_fd, &map_key,
14558	&cpu_buf->fd, 0);
14559	if (err) {
14560	err = -errno;
14561	pr_warn("failed to set cpu #%d, key %d -> perf FD %d: %s\n",
14562	cpu, map_key, cpu_buf->fd,
14563	errstr(err));
14564	goto error;
14565	}
14566
14567	pb->events[j].events = EPOLLIN;
14568	pb->events[j].data.ptr = cpu_buf;
14569	if (epoll_ctl(pb->epoll_fd, EPOLL_CTL_ADD, cpu_buf->fd,
14570	&pb->events[j]) < 0) {
14571	err = -errno;
14572	pr_warn("failed to epoll_ctl cpu #%d perf FD %d: %s\n",
14573	cpu, cpu_buf->fd,
14574	errstr(err));
14575	goto error;
14576	}
14577	j++;
14578	}
14579	pb->cpu_cnt = j;
14580	free(online);
14581
14582	return pb;
14583
14584	error:
14585	free(online);
14586	if (pb)
14587	perf_buffer__free(pb);
14588	return ERR_PTR(err);
14589	}
14590
14591	struct perf_sample_raw {
14592	struct perf_event_header header;
14593	uint32_t size;
14594	char data[];
14595	};
14596
14597	struct perf_sample_lost {
14598	struct perf_event_header header;
14599	uint64_t id;
14600	uint64_t lost;
14601	uint64_t sample_id;
14602	};
14603
14604	static enum bpf_perf_event_ret
14605	perf_buffer__process_record(struct perf_event_header e, void ctx)
14606	{
14607	struct perf_cpu_buf *cpu_buf = ctx;
14608	struct perf_buffer *pb = cpu_buf->pb;
14609	void *data = e;
14610
14611	/* user wants full control over parsing perf event */
14612	if (pb->event_cb)
14613	return pb->event_cb(pb->ctx, cpu_buf->cpu, e);
14614
14615	switch (e->type) {
14616	case PERF_RECORD_SAMPLE: {
14617	struct perf_sample_raw *s = data;
14618
14619	if (pb->sample_cb)
14620	pb->sample_cb(pb->ctx, cpu_buf->cpu, s->data, s->size);
14621	break;
14622	}
14623	case PERF_RECORD_LOST: {
14624	struct perf_sample_lost *s = data;
14625
14626	if (pb->lost_cb)
14627	pb->lost_cb(pb->ctx, cpu_buf->cpu, s->lost);
14628	break;
14629	}
14630	default:
14631	pr_warn("unknown perf sample type %u\n", e->type);
14632	return LIBBPF_PERF_EVENT_ERROR;
14633	}
14634	return LIBBPF_PERF_EVENT_CONT;
14635	}
14636
14637	static int perf_buffer__process_records(struct perf_buffer *pb,
14638	struct perf_cpu_buf *cpu_buf)
14639	{
14640	enum bpf_perf_event_ret ret;
14641
14642	ret = perf_event_read_simple(cpu_buf->base, pb->mmap_size,
14643	pb->page_size, &cpu_buf->buf,
14644	&cpu_buf->buf_size,
14645	perf_buffer__process_record, cpu_buf);
14646	if (ret != LIBBPF_PERF_EVENT_CONT)
14647	return ret;
14648	return 0;
14649	}
14650
14651	int perf_buffer__epoll_fd(const struct perf_buffer *pb)
14652	{
14653	return pb->epoll_fd;
14654	}
14655
14656	int perf_buffer__poll(struct perf_buffer *pb, int timeout_ms)
14657	{
14658	int i, cnt, err;
14659
14660	cnt = epoll_wait(pb->epoll_fd, pb->events, pb->cpu_cnt, timeout_ms);
14661	if (cnt < 0)
14662	return -errno;
14663
14664	for (i = 0; i < cnt; i++) {
14665	struct perf_cpu_buf *cpu_buf = pb->events[i].data.ptr;
14666
14667	err = perf_buffer__process_records(pb, cpu_buf);
14668	if (err) {
14669	pr_warn("error while processing records: %s\n", errstr(err));
14670	return libbpf_err(err);
14671	}
14672	}
14673	return cnt;
14674	}
14675
14676	/* Return number of PERF_EVENT_ARRAY map slots set up by this perf_buffer
14677	* manager.
14678	*/
14679	size_t perf_buffer__buffer_cnt(const struct perf_buffer *pb)
14680	{
14681	return pb->cpu_cnt;
14682	}
14683
14684	/*
14685	* Return perf_event FD of a ring buffer in buf_idx slot of
14686	* PERF_EVENT_ARRAY BPF map. This FD can be polled for new data using
14687	* select()/poll()/epoll() Linux syscalls.
14688	*/
14689	int perf_buffer__buffer_fd(const struct perf_buffer *pb, size_t buf_idx)
14690	{
14691	struct perf_cpu_buf *cpu_buf;
14692
14693	if (buf_idx >= pb->cpu_cnt)
14694	return libbpf_err(-EINVAL);
14695
14696	cpu_buf = pb->cpu_bufs[buf_idx];
14697	if (!cpu_buf)
14698	return libbpf_err(-ENOENT);
14699
14700	return cpu_buf->fd;
14701	}
14702
14703	int perf_buffer__buffer(struct perf_buffer pb, int buf_idx, void buf, size_t buf_size)
14704	{
14705	struct perf_cpu_buf *cpu_buf;
14706
14707	if (buf_idx >= pb->cpu_cnt)
14708	return libbpf_err(-EINVAL);
14709
14710	cpu_buf = pb->cpu_bufs[buf_idx];
14711	if (!cpu_buf)
14712	return libbpf_err(-ENOENT);
14713
14714	*buf = cpu_buf->base;
14715	*buf_size = pb->mmap_size;
14716	return 0;
14717	}
14718
14719	/*
14720	* Consume data from perf ring buffer corresponding to slot buf_idx in
14721	* PERF_EVENT_ARRAY BPF map without waiting/polling. If there is no data to
14722	* consume, do nothing and return success.
14723	* Returns:
14724	* - 0 on success;
14725	* - <0 on failure.
14726	*/
14727	int perf_buffer__consume_buffer(struct perf_buffer *pb, size_t buf_idx)
14728	{
14729	struct perf_cpu_buf *cpu_buf;
14730
14731	if (buf_idx >= pb->cpu_cnt)
14732	return libbpf_err(-EINVAL);
14733
14734	cpu_buf = pb->cpu_bufs[buf_idx];
14735	if (!cpu_buf)
14736	return libbpf_err(-ENOENT);
14737
14738	return perf_buffer__process_records(pb, cpu_buf);
14739	}
14740
14741	int perf_buffer__consume(struct perf_buffer *pb)
14742	{
14743	int i, err;
14744
14745	for (i = 0; i < pb->cpu_cnt; i++) {
14746	struct perf_cpu_buf *cpu_buf = pb->cpu_bufs[i];
14747
14748	if (!cpu_buf)
14749	continue;
14750
14751	err = perf_buffer__process_records(pb, cpu_buf);
14752	if (err) {
14753	pr_warn("perf_buffer: failed to process records in buffer #%d: %s\n",
14754	i, errstr(err));
14755	return libbpf_err(err);
14756	}
14757	}
14758	return 0;
14759	}
14760
14761	int bpf_program__set_attach_target(struct bpf_program *prog,
14762	int attach_prog_fd,
14763	const char *attach_func_name)
14764	{
14765	int btf_obj_fd = 0, btf_id = 0, err;
14766
14767	if (!prog \|\| attach_prog_fd < 0)
14768	return libbpf_err(-EINVAL);
14769
14770	if (prog->obj->state >= OBJ_LOADED)
14771	return libbpf_err(-EINVAL);
14772
14773	if (attach_prog_fd && !attach_func_name) {
14774	/* Store attach_prog_fd. The BTF ID will be resolved later during
14775	* the normal object/program load phase.
14776	*/
14777	prog->attach_prog_fd = attach_prog_fd;
14778	return 0;
14779	}
14780
14781	if (attach_prog_fd) {
14782	btf_id = libbpf_find_prog_btf_id(attach_func_name,
14783	attach_prog_fd, prog->obj->token_fd);
14784	if (btf_id < 0)
14785	return libbpf_err(btf_id);
14786	} else {
14787	if (!attach_func_name)
14788	return libbpf_err(-EINVAL);
14789
14790	/* load btf_vmlinux, if not yet */
14791	err = bpf_object__load_vmlinux_btf(prog->obj, true);
14792	if (err)
14793	return libbpf_err(err);
14794	err = find_kernel_btf_id(prog->obj, attach_func_name,
14795	prog->expected_attach_type,
14796	&btf_obj_fd, &btf_id);
14797	if (err)
14798	return libbpf_err(err);
14799	}
14800
14801	prog->attach_btf_id = btf_id;
14802	prog->attach_btf_obj_fd = btf_obj_fd;
14803	prog->attach_prog_fd = attach_prog_fd;
14804	return 0;
14805	}
14806
14807	int bpf_program__assoc_struct_ops(struct bpf_program prog, struct bpf_map map,
14808	struct bpf_prog_assoc_struct_ops_opts *opts)
14809	{
14810	int prog_fd, map_fd;
14811
14812	prog_fd = bpf_program__fd(prog);
14813	if (prog_fd < 0) {
14814	pr_warn("prog '%s': can't associate BPF program without FD (was it loaded?)\n",
14815	prog->name);
14816	return libbpf_err(-EINVAL);
14817	}
14818
14819	if (prog->type == BPF_PROG_TYPE_STRUCT_OPS) {
14820	pr_warn("prog '%s': can't associate struct_ops program\n", prog->name);
14821	return libbpf_err(-EINVAL);
14822	}
14823
14824	map_fd = bpf_map__fd(map);
14825	if (map_fd < 0) {
14826	pr_warn("map '%s': can't associate BPF map without FD (was it created?)\n", map->name);
14827	return libbpf_err(-EINVAL);
14828	}
14829
14830	if (!bpf_map__is_struct_ops(map)) {
14831	pr_warn("map '%s': can't associate non-struct_ops map\n", map->name);
14832	return libbpf_err(-EINVAL);
14833	}
14834
14835	return bpf_prog_assoc_struct_ops(prog_fd, map_fd, opts);
14836	}
14837
14838	int parse_cpu_mask_str(const char s, bool mask, int mask_sz)
14839	{
14840	int err = 0, n, len, start, end = -1;
14841	bool *tmp;
14842
14843	*mask = NULL;
14844	*mask_sz = 0;
14845
14846	/* Each sub string separated by ',' has format \d+-\d+ or \d+ */
14847	while (*s) {
14848	if (s == ',' \|\| s == '\n') {
14849	s++;
14850	continue;
14851	}
14852	n = sscanf(s, "%d%n-%d%n", &start, &len, &end, &len);
14853	if (n <= 0 \|\| n > 2) {
14854	pr_warn("Failed to get CPU range %s: %d\n", s, n);
14855	err = -EINVAL;
14856	goto cleanup;
14857	} else if (n == 1) {
14858	end = start;
14859	}
14860	if (start < 0 \|\| start > end) {
14861	pr_warn("Invalid CPU range [%d,%d] in %s\n",
14862	start, end, s);
14863	err = -EINVAL;
14864	goto cleanup;
14865	}
14866	tmp = realloc(*mask, end + 1);
14867	if (!tmp) {
14868	err = -ENOMEM;
14869	goto cleanup;
14870	}
14871	*mask = tmp;
14872	memset(tmp + mask_sz, 0, start - mask_sz);
14873	memset(tmp + start, 1, end - start + 1);
14874	*mask_sz = end + 1;
14875	s += len;
14876	}
14877	if (!*mask_sz) {
14878	pr_warn("Empty CPU range\n");
14879	return -EINVAL;
14880	}
14881	return 0;
14882	cleanup:
14883	free(*mask);
14884	*mask = NULL;
14885	return err;
14886	}
14887
14888	int parse_cpu_mask_file(const char fcpu, bool mask, int mask_sz)
14889	{
14890	int fd, err = 0, len;
14891	char buf[128];
14892
14893	fd = open(fcpu, O_RDONLY \| O_CLOEXEC);
14894	if (fd < 0) {
14895	err = -errno;
14896	pr_warn("Failed to open cpu mask file %s: %s\n", fcpu, errstr(err));
14897	return err;
14898	}
14899	len = read(fd, buf, sizeof(buf));
14900	close(fd);
14901	if (len <= 0) {
14902	err = len ? -errno : -EINVAL;
14903	pr_warn("Failed to read cpu mask from %s: %s\n", fcpu, errstr(err));
14904	return err;
14905	}
14906	if (len >= sizeof(buf)) {
14907	pr_warn("CPU mask is too big in file %s\n", fcpu);
14908	return -E2BIG;
14909	}
14910	buf[len] = '\0';
14911
14912	return parse_cpu_mask_str(buf, mask, mask_sz);
14913	}
14914
14915	int libbpf_num_possible_cpus(void)
14916	{
14917	static const char *fcpu = "/sys/devices/system/cpu/possible";
14918	static int cpus;
14919	int err, n, i, tmp_cpus;
14920	bool *mask;
14921
14922	tmp_cpus = READ_ONCE(cpus);
14923	if (tmp_cpus > 0)
14924	return tmp_cpus;
14925
14926	err = parse_cpu_mask_file(fcpu, &mask, &n);
14927	if (err)
14928	return libbpf_err(err);
14929
14930	tmp_cpus = 0;
14931	for (i = 0; i < n; i++) {
14932	if (mask[i])
14933	tmp_cpus++;
14934	}
14935	free(mask);
14936
14937	WRITE_ONCE(cpus, tmp_cpus);
14938	return tmp_cpus;
14939	}
14940
14941	static int populate_skeleton_maps(const struct bpf_object *obj,
14942	struct bpf_map_skeleton *maps,
14943	size_t map_cnt, size_t map_skel_sz)
14944	{
14945	int i;
14946
14947	for (i = 0; i < map_cnt; i++) {
14948	struct bpf_map_skeleton map_skel = (void )maps + i * map_skel_sz;
14949	struct bpf_map **map = map_skel->map;
14950	const char *name = map_skel->name;
14951	void **mmaped = map_skel->mmaped;
14952
14953	*map = bpf_object__find_map_by_name(obj, name);
14954	if (!*map) {
14955	pr_warn("failed to find skeleton map '%s'\n", name);
14956	return -ESRCH;
14957	}
14958
14959	/* externs shouldn't be pre-setup from user code */
14960	if (mmaped && (*map)->libbpf_type != LIBBPF_MAP_KCONFIG)
14961	mmaped = (map)->mmaped;
14962	}
14963	return 0;
14964	}
14965
14966	static int populate_skeleton_progs(const struct bpf_object *obj,
14967	struct bpf_prog_skeleton *progs,
14968	size_t prog_cnt, size_t prog_skel_sz)
14969	{
14970	int i;
14971
14972	for (i = 0; i < prog_cnt; i++) {
14973	struct bpf_prog_skeleton prog_skel = (void )progs + i * prog_skel_sz;
14974	struct bpf_program **prog = prog_skel->prog;
14975	const char *name = prog_skel->name;
14976
14977	*prog = bpf_object__find_program_by_name(obj, name);
14978	if (!*prog) {
14979	pr_warn("failed to find skeleton program '%s'\n", name);
14980	return -ESRCH;
14981	}
14982	}
14983	return 0;
14984	}
14985
14986	int bpf_object__open_skeleton(struct bpf_object_skeleton *s,
14987	const struct bpf_object_open_opts *opts)
14988	{
14989	struct bpf_object *obj;
14990	int err;
14991
14992	obj = bpf_object_open(NULL, s->data, s->data_sz, s->name, opts);
14993	if (IS_ERR(obj)) {
14994	err = PTR_ERR(obj);
14995	pr_warn("failed to initialize skeleton BPF object '%s': %s\n",
14996	s->name, errstr(err));
14997	return libbpf_err(err);
14998	}
14999
15000	*s->obj = obj;
15001	err = populate_skeleton_maps(obj, s->maps, s->map_cnt, s->map_skel_sz);
15002	if (err) {
15003	pr_warn("failed to populate skeleton maps for '%s': %s\n", s->name, errstr(err));
15004	return libbpf_err(err);
15005	}
15006
15007	err = populate_skeleton_progs(obj, s->progs, s->prog_cnt, s->prog_skel_sz);
15008	if (err) {
15009	pr_warn("failed to populate skeleton progs for '%s': %s\n", s->name, errstr(err));
15010	return libbpf_err(err);
15011	}
15012
15013	return 0;
15014	}
15015
15016	int bpf_object__open_subskeleton(struct bpf_object_subskeleton *s)
15017	{
15018	int err, len, var_idx, i;
15019	const char *var_name;
15020	const struct bpf_map *map;
15021	struct btf *btf;
15022	__u32 map_type_id;
15023	const struct btf_type map_type, var_type;
15024	const struct bpf_var_skeleton *var_skel;
15025	struct btf_var_secinfo *var;
15026
15027	if (!s->obj)
15028	return libbpf_err(-EINVAL);
15029
15030	btf = bpf_object__btf(s->obj);
15031	if (!btf) {
15032	pr_warn("subskeletons require BTF at runtime (object %s)\n",
15033	bpf_object__name(s->obj));
15034	return libbpf_err(-errno);
15035	}
15036
15037	err = populate_skeleton_maps(s->obj, s->maps, s->map_cnt, s->map_skel_sz);
15038	if (err) {
15039	pr_warn("failed to populate subskeleton maps: %s\n", errstr(err));
15040	return libbpf_err(err);
15041	}
15042
15043	err = populate_skeleton_progs(s->obj, s->progs, s->prog_cnt, s->prog_skel_sz);
15044	if (err) {
15045	pr_warn("failed to populate subskeleton maps: %s\n", errstr(err));
15046	return libbpf_err(err);
15047	}
15048
15049	for (var_idx = 0; var_idx < s->var_cnt; var_idx++) {
15050	var_skel = (void )s->vars + var_idx s->var_skel_sz;
15051	map = *var_skel->map;
15052	map_type_id = bpf_map__btf_value_type_id(map);
15053	map_type = btf__type_by_id(btf, map_type_id);
15054
15055	if (!btf_is_datasec(map_type)) {
15056	pr_warn("type for map '%1$s' is not a datasec: %2$s\n",
15057	bpf_map__name(map),
15058	__btf_kind_str(btf_kind(map_type)));
15059	return libbpf_err(-EINVAL);
15060	}
15061
15062	len = btf_vlen(map_type);
15063	var = btf_var_secinfos(map_type);
15064	for (i = 0; i < len; i++, var++) {
15065	var_type = btf__type_by_id(btf, var->type);
15066	var_name = btf__name_by_offset(btf, var_type->name_off);
15067	if (strcmp(var_name, var_skel->name) == 0) {
15068	*var_skel->addr = map->mmaped + var->offset;
15069	break;
15070	}
15071	}
15072	}
15073	return 0;
15074	}
15075
15076	void bpf_object__destroy_subskeleton(struct bpf_object_subskeleton *s)
15077	{
15078	if (!s)
15079	return;
15080	free(s->maps);
15081	free(s->progs);
15082	free(s->vars);
15083	free(s);
15084	}
15085
15086	int bpf_object__load_skeleton(struct bpf_object_skeleton *s)
15087	{
15088	int i, err;
15089
15090	err = bpf_object__load(*s->obj);
15091	if (err) {
15092	pr_warn("failed to load BPF skeleton '%s': %s\n", s->name, errstr(err));
15093	return libbpf_err(err);
15094	}
15095
15096	for (i = 0; i < s->map_cnt; i++) {
15097	struct bpf_map_skeleton map_skel = (void )s->maps + i * s->map_skel_sz;
15098	struct bpf_map map = map_skel->map;
15099
15100	if (!map_skel->mmaped)
15101	continue;
15102
15103	if (map->def.type == BPF_MAP_TYPE_ARENA)
15104	*map_skel->mmaped = map->mmaped + map->obj->arena_data_off;
15105	else
15106	*map_skel->mmaped = map->mmaped;
15107	}
15108
15109	return 0;
15110	}
15111
15112	int bpf_object__attach_skeleton(struct bpf_object_skeleton *s)
15113	{
15114	int i, err;
15115
15116	for (i = 0; i < s->prog_cnt; i++) {
15117	struct bpf_prog_skeleton prog_skel = (void )s->progs + i * s->prog_skel_sz;
15118	struct bpf_program prog = prog_skel->prog;
15119	struct bpf_link **link = prog_skel->link;
15120
15121	if (!prog->autoload \|\| !prog->autoattach)
15122	continue;
15123
15124	/* auto-attaching not supported for this program */
15125	if (!prog->sec_def \|\| !prog->sec_def->prog_attach_fn)
15126	continue;
15127
15128	/* if user already set the link manually, don't attempt auto-attach */
15129	if (*link)
15130	continue;
15131
15132	err = prog->sec_def->prog_attach_fn(prog, prog->sec_def->cookie, link);
15133	if (err) {
15134	pr_warn("prog '%s': failed to auto-attach: %s\n",
15135	bpf_program__name(prog), errstr(err));
15136	return libbpf_err(err);
15137	}
15138
15139	/* It's possible that for some SEC() definitions auto-attach
15140	* is supported in some cases (e.g., if definition completely
15141	* specifies target information), but is not in other cases.
15142	* SEC("uprobe") is one such case. If user specified target
15143	* binary and function name, such BPF program can be
15144	* auto-attached. But if not, it shouldn't trigger skeleton's
15145	* attach to fail. It should just be skipped.
15146	* attach_fn signals such case with returning 0 (no error) and
15147	* setting link to NULL.
15148	*/
15149	}
15150
15151
15152	for (i = 0; i < s->map_cnt; i++) {
15153	struct bpf_map_skeleton map_skel = (void )s->maps + i * s->map_skel_sz;
15154	struct bpf_map map = map_skel->map;
15155	struct bpf_link **link;
15156
15157	if (!map->autocreate \|\| !map->autoattach)
15158	continue;
15159
15160	/* only struct_ops maps can be attached */
15161	if (!bpf_map__is_struct_ops(map))
15162	continue;
15163
15164	/* skeleton is created with earlier version of bpftool, notify user */
15165	if (s->map_skel_sz < offsetofend(struct bpf_map_skeleton, link)) {
15166	pr_warn("map '%s': BPF skeleton version is old, skipping map auto-attachment...\n",
15167	bpf_map__name(map));
15168	continue;
15169	}
15170
15171	link = map_skel->link;
15172	if (!link) {
15173	pr_warn("map '%s': BPF map skeleton link is uninitialized\n",
15174	bpf_map__name(map));
15175	continue;
15176	}
15177
15178	if (*link)
15179	continue;
15180
15181	*link = bpf_map__attach_struct_ops(map);
15182	if (!*link) {
15183	err = -errno;
15184	pr_warn("map '%s': failed to auto-attach: %s\n",
15185	bpf_map__name(map), errstr(err));
15186	return libbpf_err(err);
15187	}
15188	}
15189
15190	return 0;
15191	}
15192
15193	void bpf_object__detach_skeleton(struct bpf_object_skeleton *s)
15194	{
15195	int i;
15196
15197	for (i = 0; i < s->prog_cnt; i++) {
15198	struct bpf_prog_skeleton prog_skel = (void )s->progs + i * s->prog_skel_sz;
15199	struct bpf_link **link = prog_skel->link;
15200
15201	bpf_link__destroy(*link);
15202	*link = NULL;
15203	}
15204
15205	if (s->map_skel_sz < sizeof(struct bpf_map_skeleton))
15206	return;
15207
15208	for (i = 0; i < s->map_cnt; i++) {
15209	struct bpf_map_skeleton map_skel = (void )s->maps + i * s->map_skel_sz;
15210	struct bpf_link **link = map_skel->link;
15211
15212	if (link) {
15213	bpf_link__destroy(*link);
15214	*link = NULL;
15215	}
15216	}
15217	}
15218
15219	void bpf_object__destroy_skeleton(struct bpf_object_skeleton *s)
15220	{
15221	if (!s)
15222	return;
15223
15224	bpf_object__detach_skeleton(s);
15225	if (s->obj)
15226	bpf_object__close(*s->obj);
15227	free(s->maps);
15228	free(s->progs);
15229	free(s);
15230	}