linux - kernel/bpf/btf.c

10084 lines · 258.9 KiB

1	// SPDX-License-Identifier: GPL-2.0
2	/* Copyright (c) 2018 Facebook */
3
4	#include <uapi/linux/btf.h>
5	#include <uapi/linux/bpf.h>
6	#include <uapi/linux/bpf_perf_event.h>
7	#include <uapi/linux/types.h>
8	#include <linux/seq_file.h>
9	#include <linux/compiler.h>
10	#include <linux/ctype.h>
11	#include <linux/errno.h>
12	#include <linux/slab.h>
13	#include <linux/anon_inodes.h>
14	#include <linux/file.h>
15	#include <linux/uaccess.h>
16	#include <linux/kernel.h>
17	#include <linux/idr.h>
18	#include <linux/sort.h>
19	#include <linux/bpf_verifier.h>
20	#include <linux/btf.h>
21	#include <linux/btf_ids.h>
22	#include <linux/bpf.h>
23	#include <linux/bpf_lsm.h>
24	#include <linux/skmsg.h>
25	#include <linux/perf_event.h>
26	#include <linux/bsearch.h>
27	#include <linux/kobject.h>
28	#include <linux/string.h>
29	#include <linux/sysfs.h>
30	#include <linux/overflow.h>
31	#include <linux/bitops.h>
32
33	#include <net/netfilter/nf_bpf_link.h>
34
35	#include <net/sock.h>
36	#include <net/xdp.h>
37	#include "../tools/lib/bpf/relo_core.h"
38
39	/* BTF (BPF Type Format) is the meta data format which describes
40	* the data types of BPF program/map. Hence, it basically focus
41	* on the C programming language which the modern BPF is primary
42	* using.
43	*
44	* ELF Section:
45	* ~~~~~~~~~~~
46	* The BTF data is stored under the ".BTF" ELF section
47	*
48	* struct btf_type:
49	* ~~~~~~~~~~~~~~~
50	* Each 'struct btf_type' object describes a C data type.
51	* Depending on the type it is describing, a 'struct btf_type'
52	* object may be followed by more data. F.e.
53	* To describe an array, 'struct btf_type' is followed by
54	* 'struct btf_array'.
55	*
56	* 'struct btf_type' and any extra data following it are
57	* 4 bytes aligned.
58	*
59	* Type section:
60	* ~~~~~~~~~~~~~
61	* The BTF type section contains a list of 'struct btf_type' objects.
62	* Each one describes a C type. Recall from the above section
63	* that a 'struct btf_type' object could be immediately followed by extra
64	* data in order to describe some particular C types.
65	*
66	* type_id:
67	* ~~~~~~~
68	* Each btf_type object is identified by a type_id. The type_id
69	* is implicitly implied by the location of the btf_type object in
70	* the BTF type section. The first one has type_id 1. The second
71	* one has type_id 2...etc. Hence, an earlier btf_type has
72	* a smaller type_id.
73	*
74	* A btf_type object may refer to another btf_type object by using
75	* type_id (i.e. the "type" in the "struct btf_type").
76	*
77	* NOTE that we cannot assume any reference-order.
78	* A btf_type object can refer to an earlier btf_type object
79	* but it can also refer to a later btf_type object.
80	*
81	* For example, to describe "const void *". A btf_type
82	* object describing "const" may refer to another btf_type
83	* object describing "void *". This type-reference is done
84	* by specifying type_id:
85	*
86	* [1] CONST (anon) type_id=2
87	* [2] PTR (anon) type_id=0
88	*
89	* The above is the btf_verifier debug log:
90	* - Each line started with "[?]" is a btf_type object
91	* - [?] is the type_id of the btf_type object.
92	* - CONST/PTR is the BTF_KIND_XXX
93	* - "(anon)" is the name of the type. It just
94	* happens that CONST and PTR has no name.
95	* - type_id=XXX is the 'u32 type' in btf_type
96	*
97	* NOTE: "void" has type_id 0
98	*
99	* String section:
100	* ~~~~~~~~~~~~~~
101	* The BTF string section contains the names used by the type section.
102	* Each string is referred by an "offset" from the beginning of the
103	* string section.
104	*
105	* Each string is '\0' terminated.
106	*
107	* The first character in the string section must be '\0'
108	* which is used to mean 'anonymous'. Some btf_type may not
109	* have a name.
110	*/
111
112	/* BTF verification:
113	*
114	* To verify BTF data, two passes are needed.
115	*
116	* Pass #1
117	* ~~~~~~~
118	* The first pass is to collect all btf_type objects to
119	* an array: "btf->types".
120	*
121	* Depending on the C type that a btf_type is describing,
122	* a btf_type may be followed by extra data. We don't know
123	* how many btf_type is there, and more importantly we don't
124	* know where each btf_type is located in the type section.
125	*
126	* Without knowing the location of each type_id, most verifications
127	* cannot be done. e.g. an earlier btf_type may refer to a later
128	* btf_type (recall the "const void *" above), so we cannot
129	* check this type-reference in the first pass.
130	*
131	* In the first pass, it still does some verifications (e.g.
132	* checking the name is a valid offset to the string section).
133	*
134	* Pass #2
135	* ~~~~~~~
136	* The main focus is to resolve a btf_type that is referring
137	* to another type.
138	*
139	* We have to ensure the referring type:
140	* 1) does exist in the BTF (i.e. in btf->types[])
141	* 2) does not cause a loop:
142	* struct A {
143	* struct B b;
144	* };
145	*
146	* struct B {
147	* struct A a;
148	* };
149	*
150	* btf_type_needs_resolve() decides if a btf_type needs
151	* to be resolved.
152	*
153	* The needs_resolve type implements the "resolve()" ops which
154	* essentially does a DFS and detects backedge.
155	*
156	* During resolve (or DFS), different C types have different
157	* "RESOLVED" conditions.
158	*
159	* When resolving a BTF_KIND_STRUCT, we need to resolve all its
160	* members because a member is always referring to another
161	* type. A struct's member can be treated as "RESOLVED" if
162	* it is referring to a BTF_KIND_PTR. Otherwise, the
163	* following valid C struct would be rejected:
164	*
165	* struct A {
166	* int m;
167	* struct A *a;
168	* };
169	*
170	* When resolving a BTF_KIND_PTR, it needs to keep resolving if
171	* it is referring to another BTF_KIND_PTR. Otherwise, we cannot
172	* detect a pointer loop, e.g.:
173	* BTF_KIND_CONST -> BTF_KIND_PTR -> BTF_KIND_CONST -> BTF_KIND_PTR +
174	* ^ \|
175	* +-----------------------------------------+
176	*
177	*/
178
179	#define BITS_PER_U128 (sizeof(u64) * BITS_PER_BYTE * 2)
180	#define BITS_PER_BYTE_MASK (BITS_PER_BYTE - 1)
181	#define BITS_PER_BYTE_MASKED(bits) ((bits) & BITS_PER_BYTE_MASK)
182	#define BITS_ROUNDDOWN_BYTES(bits) ((bits) >> 3)
183	#define BITS_ROUNDUP_BYTES(bits) \
184	(BITS_ROUNDDOWN_BYTES(bits) + !!BITS_PER_BYTE_MASKED(bits))
185
186	#define BTF_INT_MASK 0x0fffffff
187	#define BTF_TYPE_ID_VALID(type_id) ((type_id) <= BTF_MAX_TYPE)
188	#define BTF_STR_OFFSET_VALID(name_off) ((name_off) <= BTF_MAX_NAME_OFFSET)
189
190	/* 16MB for 64k structs and each has 16 members and
191	* a few MB spaces for the string section.
192	* The hard limit is S32_MAX.
193	*/
194	#define BTF_MAX_SIZE (16 * 1024 * 1024)
195
196	#define for_each_member_from(i, from, struct_type, member) \
197	for (i = from, member = btf_type_member(struct_type) + from; \
198	i < btf_type_vlen(struct_type); \
199	i++, member++)
200
201	#define for_each_vsi_from(i, from, struct_type, member) \
202	for (i = from, member = btf_type_var_secinfo(struct_type) + from; \
203	i < btf_type_vlen(struct_type); \
204	i++, member++)
205
206	DEFINE_IDR(btf_idr);
207	DEFINE_SPINLOCK(btf_idr_lock);
208
209	enum btf_kfunc_hook {
210	BTF_KFUNC_HOOK_COMMON,
211	BTF_KFUNC_HOOK_XDP,
212	BTF_KFUNC_HOOK_TC,
213	BTF_KFUNC_HOOK_STRUCT_OPS,
214	BTF_KFUNC_HOOK_TRACING,
215	BTF_KFUNC_HOOK_SYSCALL,
216	BTF_KFUNC_HOOK_FMODRET,
217	BTF_KFUNC_HOOK_CGROUP,
218	BTF_KFUNC_HOOK_SCHED_ACT,
219	BTF_KFUNC_HOOK_SK_SKB,
220	BTF_KFUNC_HOOK_SOCKET_FILTER,
221	BTF_KFUNC_HOOK_LWT,
222	BTF_KFUNC_HOOK_NETFILTER,
223	BTF_KFUNC_HOOK_KPROBE,
224	BTF_KFUNC_HOOK_MAX,
225	};
226
227	enum {
228	BTF_KFUNC_SET_MAX_CNT = 256,
229	BTF_DTOR_KFUNC_MAX_CNT = 256,
230	BTF_KFUNC_FILTER_MAX_CNT = 16,
231	};
232
233	struct btf_kfunc_hook_filter {
234	btf_kfunc_filter_t filters[BTF_KFUNC_FILTER_MAX_CNT];
235	u32 nr_filters;
236	};
237
238	struct btf_kfunc_set_tab {
239	struct btf_id_set8 *sets[BTF_KFUNC_HOOK_MAX];
240	struct btf_kfunc_hook_filter hook_filters[BTF_KFUNC_HOOK_MAX];
241	};
242
243	struct btf_id_dtor_kfunc_tab {
244	u32 cnt;
245	struct btf_id_dtor_kfunc dtors[];
246	};
247
248	struct btf_struct_ops_tab {
249	u32 cnt;
250	u32 capacity;
251	struct bpf_struct_ops_desc ops[];
252	};
253
254	struct btf {
255	void *data;
256	struct btf_type **types;
257	u32 *resolved_ids;
258	u32 *resolved_sizes;
259	const char *strings;
260	void *nohdr_data;
261	struct btf_header hdr;
262	u32 nr_types; /* includes VOID for base BTF */
263	u32 named_start_id;
264	u32 types_size;
265	u32 data_size;
266	refcount_t refcnt;
267	u32 id;
268	struct rcu_head rcu;
269	struct btf_kfunc_set_tab *kfunc_set_tab;
270	struct btf_id_dtor_kfunc_tab *dtor_kfunc_tab;
271	struct btf_struct_metas *struct_meta_tab;
272	struct btf_struct_ops_tab *struct_ops_tab;
273	struct btf_layout *layout;
274
275	/* split BTF support */
276	struct btf *base_btf;
277	u32 start_id; /* first type ID in this BTF (0 for base BTF) */
278	u32 start_str_off; /* first string offset (0 for base BTF) */
279	char name[MODULE_NAME_LEN];
280	bool kernel_btf;
281	__u32 base_id_map; / map from distilled base BTF -> vmlinux BTF ids */
282	};
283
284	enum verifier_phase {
285	CHECK_META,
286	CHECK_TYPE,
287	};
288
289	struct resolve_vertex {
290	const struct btf_type *t;
291	u32 type_id;
292	u32 next_member;
293	};
294
295	enum visit_state {
296	NOT_VISITED,
297	VISITED,
298	RESOLVED,
299	};
300
301	enum resolve_mode {
302	RESOLVE_TBD, /* To Be Determined */
303	RESOLVE_PTR, /* Resolving for Pointer */
304	RESOLVE_STRUCT_OR_ARRAY, /* Resolving for struct/union
305	* or array
306	*/
307	};
308
309	#define MAX_RESOLVE_DEPTH 32
310
311	struct btf_sec_info {
312	u32 off;
313	u32 len;
314	};
315
316	struct btf_verifier_env {
317	struct btf *btf;
318	u8 *visit_states;
319	struct resolve_vertex stack[MAX_RESOLVE_DEPTH];
320	struct bpf_verifier_log log;
321	u32 log_type_id;
322	u32 top_stack;
323	enum verifier_phase phase;
324	enum resolve_mode resolve_mode;
325	};
326
327	static const char * const btf_kind_str[NR_BTF_KINDS] = {
328	[BTF_KIND_UNKN] = "UNKNOWN",
329	[BTF_KIND_INT] = "INT",
330	[BTF_KIND_PTR] = "PTR",
331	[BTF_KIND_ARRAY] = "ARRAY",
332	[BTF_KIND_STRUCT] = "STRUCT",
333	[BTF_KIND_UNION] = "UNION",
334	[BTF_KIND_ENUM] = "ENUM",
335	[BTF_KIND_FWD] = "FWD",
336	[BTF_KIND_TYPEDEF] = "TYPEDEF",
337	[BTF_KIND_VOLATILE] = "VOLATILE",
338	[BTF_KIND_CONST] = "CONST",
339	[BTF_KIND_RESTRICT] = "RESTRICT",
340	[BTF_KIND_FUNC] = "FUNC",
341	[BTF_KIND_FUNC_PROTO] = "FUNC_PROTO",
342	[BTF_KIND_VAR] = "VAR",
343	[BTF_KIND_DATASEC] = "DATASEC",
344	[BTF_KIND_FLOAT] = "FLOAT",
345	[BTF_KIND_DECL_TAG] = "DECL_TAG",
346	[BTF_KIND_TYPE_TAG] = "TYPE_TAG",
347	[BTF_KIND_ENUM64] = "ENUM64",
348	};
349
350	const char btf_type_str(const struct btf_type t)
351	{
352	return btf_kind_str[BTF_INFO_KIND(t->info)];
353	}
354
355	/* Chunk size we use in safe copy of data to be shown. */
356	#define BTF_SHOW_OBJ_SAFE_SIZE 32
357
358	/*
359	* This is the maximum size of a base type value (equivalent to a
360	* 128-bit int); if we are at the end of our safe buffer and have
361	* less than 16 bytes space we can't be assured of being able
362	* to copy the next type safely, so in such cases we will initiate
363	* a new copy.
364	*/
365	#define BTF_SHOW_OBJ_BASE_TYPE_SIZE 16
366
367	/* Type name size */
368	#define BTF_SHOW_NAME_SIZE 80
369
370	/*
371	* The suffix of a type that indicates it cannot alias another type when
372	* comparing BTF IDs for kfunc invocations.
373	*/
374	#define NOCAST_ALIAS_SUFFIX "___init"
375
376	/*
377	* Common data to all BTF show operations. Private show functions can add
378	* their own data to a structure containing a struct btf_show and consult it
379	* in the show callback. See btf_type_show() below.
380	*
381	* One challenge with showing nested data is we want to skip 0-valued
382	* data, but in order to figure out whether a nested object is all zeros
383	* we need to walk through it. As a result, we need to make two passes
384	* when handling structs, unions and arrays; the first path simply looks
385	* for nonzero data, while the second actually does the display. The first
386	* pass is signalled by show->state.depth_check being set, and if we
387	* encounter a non-zero value we set show->state.depth_to_show to
388	* the depth at which we encountered it. When we have completed the
389	* first pass, we will know if anything needs to be displayed if
390	* depth_to_show > depth. See btf_[struct,array]_show() for the
391	* implementation of this.
392	*
393	* Another problem is we want to ensure the data for display is safe to
394	* access. To support this, the anonymous "struct {} obj" tracks the data
395	* object and our safe copy of it. We copy portions of the data needed
396	* to the object "copy" buffer, but because its size is limited to
397	* BTF_SHOW_OBJ_COPY_LEN bytes, multiple copies may be required as we
398	* traverse larger objects for display.
399	*
400	* The various data type show functions all start with a call to
401	* btf_show_start_type() which returns a pointer to the safe copy
402	* of the data needed (or if BTF_SHOW_UNSAFE is specified, to the
403	* raw data itself). btf_show_obj_safe() is responsible for
404	* using copy_from_kernel_nofault() to update the safe data if necessary
405	* as we traverse the object's data. skbuff-like semantics are
406	* used:
407	*
408	* - obj.head points to the start of the toplevel object for display
409	* - obj.size is the size of the toplevel object
410	* - obj.data points to the current point in the original data at
411	* which our safe data starts. obj.data will advance as we copy
412	* portions of the data.
413	*
414	* In most cases a single copy will suffice, but larger data structures
415	* such as "struct task_struct" will require many copies. The logic in
416	* btf_show_obj_safe() handles the logic that determines if a new
417	* copy_from_kernel_nofault() is needed.
418	*/
419	struct btf_show {
420	u64 flags;
421	void target; / target of show operation (seq file, buffer) */
422	__printf(2, 0) void (showfn)(struct btf_show show, const char *fmt, va_list args);
423	const struct btf *btf;
424	/* below are used during iteration */
425	struct {
426	u8 depth;
427	u8 depth_to_show;
428	u8 depth_check;
429	u8 array_member:1,
430	array_terminated:1;
431	u16 array_encoding;
432	u32 type_id;
433	int status; /* non-zero for error */
434	const struct btf_type *type;
435	const struct btf_member *member;
436	char name[BTF_SHOW_NAME_SIZE]; /* space for member name/type */
437	} state;
438	struct {
439	u32 size;
440	void *head;
441	void *data;
442	u8 safe[BTF_SHOW_OBJ_SAFE_SIZE];
443	} obj;
444	};
445
446	struct btf_kind_operations {
447	s32 (check_meta)(struct btf_verifier_env env,
448	const struct btf_type *t,
449	u32 meta_left);
450	int (resolve)(struct btf_verifier_env env,
451	const struct resolve_vertex *v);
452	int (check_member)(struct btf_verifier_env env,
453	const struct btf_type *struct_type,
454	const struct btf_member *member,
455	const struct btf_type *member_type);
456	int (check_kflag_member)(struct btf_verifier_env env,
457	const struct btf_type *struct_type,
458	const struct btf_member *member,
459	const struct btf_type *member_type);
460	void (log_details)(struct btf_verifier_env env,
461	const struct btf_type *t);
462	void (show)(const struct btf btf, const struct btf_type *t,
463	u32 type_id, void *data, u8 bits_offsets,
464	struct btf_show *show);
465	};
466
467	static const struct btf_kind_operations * const kind_ops[NR_BTF_KINDS];
468	static struct btf_type btf_void;
469
470	static int btf_resolve(struct btf_verifier_env *env,
471	const struct btf_type *t, u32 type_id);
472
473	static int btf_func_check(struct btf_verifier_env *env,
474	const struct btf_type *t);
475
476	static bool btf_type_is_modifier(const struct btf_type *t)
477	{
478	/* Some of them is not strictly a C modifier
479	* but they are grouped into the same bucket
480	* for BTF concern:
481	* A type (t) that refers to another
482	* type through t->type AND its size cannot
483	* be determined without following the t->type.
484	*
485	* ptr does not fall into this bucket
486	* because its size is always sizeof(void *).
487	*/
488	switch (BTF_INFO_KIND(t->info)) {
489	case BTF_KIND_TYPEDEF:
490	case BTF_KIND_VOLATILE:
491	case BTF_KIND_CONST:
492	case BTF_KIND_RESTRICT:
493	case BTF_KIND_TYPE_TAG:
494	return true;
495	}
496
497	return false;
498	}
499
500	static int btf_start_id(const struct btf *btf)
501	{
502	return btf->start_id + (btf->base_btf ? 0 : 1);
503	}
504
505	bool btf_type_is_void(const struct btf_type *t)
506	{
507	return t == &btf_void;
508	}
509
510	static bool btf_type_is_datasec(const struct btf_type *t)
511	{
512	return BTF_INFO_KIND(t->info) == BTF_KIND_DATASEC;
513	}
514
515	static bool btf_type_is_decl_tag(const struct btf_type *t)
516	{
517	return BTF_INFO_KIND(t->info) == BTF_KIND_DECL_TAG;
518	}
519
520	static bool btf_type_nosize(const struct btf_type *t)
521	{
522	return btf_type_is_void(t) \|\| btf_type_is_fwd(t) \|\|
523	btf_type_is_func(t) \|\| btf_type_is_func_proto(t) \|\|
524	btf_type_is_decl_tag(t);
525	}
526
527	static bool btf_type_nosize_or_null(const struct btf_type *t)
528	{
529	return !t \|\| btf_type_nosize(t);
530	}
531
532	static bool btf_type_is_decl_tag_target(const struct btf_type *t)
533	{
534	return btf_type_is_func(t) \|\| btf_type_is_struct(t) \|\|
535	btf_type_is_var(t) \|\| btf_type_is_typedef(t);
536	}
537
538	bool btf_is_vmlinux(const struct btf *btf)
539	{
540	return btf->kernel_btf && !btf->base_btf;
541	}
542
543	u32 btf_nr_types(const struct btf *btf)
544	{
545	u32 total = 0;
546
547	while (btf) {
548	total += btf->nr_types;
549	btf = btf->base_btf;
550	}
551
552	return total;
553	}
554
555	/*
556	* Note that vmlinux and kernel module BTFs are always sorted
557	* during the building phase.
558	*/
559	static void btf_check_sorted(struct btf *btf)
560	{
561	u32 i, n, named_start_id = 0;
562
563	n = btf_nr_types(btf);
564	if (btf_is_vmlinux(btf)) {
565	for (i = btf_start_id(btf); i < n; i++) {
566	const struct btf_type *t = btf_type_by_id(btf, i);
567	const char *n = btf_name_by_offset(btf, t->name_off);
568
569	if (n[0] != '\0') {
570	btf->named_start_id = i;
571	return;
572	}
573	}
574	return;
575	}
576
577	for (i = btf_start_id(btf) + 1; i < n; i++) {
578	const struct btf_type *ta = btf_type_by_id(btf, i - 1);
579	const struct btf_type *tb = btf_type_by_id(btf, i);
580	const char *na = btf_name_by_offset(btf, ta->name_off);
581	const char *nb = btf_name_by_offset(btf, tb->name_off);
582
583	if (strcmp(na, nb) > 0)
584	return;
585
586	if (named_start_id == 0 && na[0] != '\0')
587	named_start_id = i - 1;
588	if (named_start_id == 0 && nb[0] != '\0')
589	named_start_id = i;
590	}
591
592	if (named_start_id)
593	btf->named_start_id = named_start_id;
594	}
595
596	/*
597	* btf_named_start_id - Get the named starting ID for the BTF
598	* @btf: Pointer to the target BTF object
599	* @own: Flag indicating whether to query only the current BTF (true = current BTF only,
600	* false = recursively traverse the base BTF chain)
601	*
602	* Return value rules:
603	* 1. For a sorted btf, return its named_start_id
604	* 2. Else for a split BTF, return its start_id
605	* 3. Else for a base BTF, return 1
606	*/
607	u32 btf_named_start_id(const struct btf *btf, bool own)
608	{
609	const struct btf *base_btf = btf;
610
611	while (!own && base_btf->base_btf)
612	base_btf = base_btf->base_btf;
613
614	return base_btf->named_start_id ?: (base_btf->start_id ?: 1);
615	}
616
617	static s32 btf_find_by_name_kind_bsearch(const struct btf btf, const char name)
618	{
619	const struct btf_type *t;
620	const char *tname;
621	s32 l, r, m;
622
623	l = btf_named_start_id(btf, true);
624	r = btf_nr_types(btf) - 1;
625	while (l <= r) {
626	m = l + (r - l) / 2;
627	t = btf_type_by_id(btf, m);
628	tname = btf_name_by_offset(btf, t->name_off);
629	if (strcmp(tname, name) >= 0) {
630	if (l == r)
631	return r;
632	r = m;
633	} else {
634	l = m + 1;
635	}
636	}
637
638	return btf_nr_types(btf);
639	}
640
641	s32 btf_find_by_name_kind(const struct btf btf, const char name, u8 kind)
642	{
643	const struct btf *base_btf = btf_base_btf(btf);
644	const struct btf_type *t;
645	const char *tname;
646	s32 id, total;
647
648	if (base_btf) {
649	id = btf_find_by_name_kind(base_btf, name, kind);
650	if (id > 0)
651	return id;
652	}
653
654	total = btf_nr_types(btf);
655	if (btf->named_start_id > 0 && name[0]) {
656	id = btf_find_by_name_kind_bsearch(btf, name);
657	for (; id < total; id++) {
658	t = btf_type_by_id(btf, id);
659	tname = btf_name_by_offset(btf, t->name_off);
660	if (strcmp(tname, name) != 0)
661	return -ENOENT;
662	if (BTF_INFO_KIND(t->info) == kind)
663	return id;
664	}
665	} else {
666	for (id = btf_start_id(btf); id < total; id++) {
667	t = btf_type_by_id(btf, id);
668	if (BTF_INFO_KIND(t->info) != kind)
669	continue;
670	tname = btf_name_by_offset(btf, t->name_off);
671	if (strcmp(tname, name) == 0)
672	return id;
673	}
674	}
675
676	return -ENOENT;
677	}
678
679	s32 bpf_find_btf_id(const char name, u32 kind, struct btf *btf_p)
680	{
681	struct btf *btf;
682	s32 ret;
683	int id;
684
685	btf = bpf_get_btf_vmlinux();
686	if (IS_ERR(btf))
687	return PTR_ERR(btf);
688	if (!btf)
689	return -EINVAL;
690
691	ret = btf_find_by_name_kind(btf, name, kind);
692	/* ret is never zero, since btf_find_by_name_kind returns
693	* positive btf_id or negative error.
694	*/
695	if (ret > 0) {
696	btf_get(btf);
697	*btf_p = btf;
698	return ret;
699	}
700
701	/* If name is not found in vmlinux's BTF then search in module's BTFs */
702	spin_lock_bh(&btf_idr_lock);
703	idr_for_each_entry(&btf_idr, btf, id) {
704	if (!btf_is_module(btf))
705	continue;
706	/* linear search could be slow hence unlock/lock
707	* the IDR to avoiding holding it for too long
708	*/
709	btf_get(btf);
710	spin_unlock_bh(&btf_idr_lock);
711	ret = btf_find_by_name_kind(btf, name, kind);
712	if (ret > 0) {
713	*btf_p = btf;
714	return ret;
715	}
716	btf_put(btf);
717	spin_lock_bh(&btf_idr_lock);
718	}
719	spin_unlock_bh(&btf_idr_lock);
720	return ret;
721	}
722	EXPORT_SYMBOL_GPL(bpf_find_btf_id);
723
724	const struct btf_type btf_type_skip_modifiers(const struct btf btf,
725	u32 id, u32 *res_id)
726	{
727	const struct btf_type *t = btf_type_by_id(btf, id);
728
729	while (btf_type_is_modifier(t)) {
730	id = t->type;
731	t = btf_type_by_id(btf, t->type);
732	}
733
734	if (res_id)
735	*res_id = id;
736
737	return t;
738	}
739
740	const struct btf_type btf_type_resolve_ptr(const struct btf btf,
741	u32 id, u32 *res_id)
742	{
743	const struct btf_type *t;
744
745	t = btf_type_skip_modifiers(btf, id, NULL);
746	if (!btf_type_is_ptr(t))
747	return NULL;
748
749	return btf_type_skip_modifiers(btf, t->type, res_id);
750	}
751
752	const struct btf_type btf_type_resolve_func_ptr(const struct btf btf,
753	u32 id, u32 *res_id)
754	{
755	const struct btf_type *ptype;
756
757	ptype = btf_type_resolve_ptr(btf, id, res_id);
758	if (ptype && btf_type_is_func_proto(ptype))
759	return ptype;
760
761	return NULL;
762	}
763
764	/* Types that act only as a source, not sink or intermediate
765	* type when resolving.
766	*/
767	static bool btf_type_is_resolve_source_only(const struct btf_type *t)
768	{
769	return btf_type_is_var(t) \|\|
770	btf_type_is_decl_tag(t) \|\|
771	btf_type_is_datasec(t);
772	}
773
774	/* What types need to be resolved?
775	*
776	* btf_type_is_modifier() is an obvious one.
777	*
778	* btf_type_is_struct() because its member refers to
779	* another type (through member->type).
780	*
781	* btf_type_is_var() because the variable refers to
782	* another type. btf_type_is_datasec() holds multiple
783	* btf_type_is_var() types that need resolving.
784	*
785	* btf_type_is_array() because its element (array->type)
786	* refers to another type. Array can be thought of a
787	* special case of struct while array just has the same
788	* member-type repeated by array->nelems of times.
789	*/
790	static bool btf_type_needs_resolve(const struct btf_type *t)
791	{
792	return btf_type_is_modifier(t) \|\|
793	btf_type_is_ptr(t) \|\|
794	btf_type_is_struct(t) \|\|
795	btf_type_is_array(t) \|\|
796	btf_type_is_var(t) \|\|
797	btf_type_is_func(t) \|\|
798	btf_type_is_decl_tag(t) \|\|
799	btf_type_is_datasec(t);
800	}
801
802	/* t->size can be used */
803	static bool btf_type_has_size(const struct btf_type *t)
804	{
805	switch (BTF_INFO_KIND(t->info)) {
806	case BTF_KIND_INT:
807	case BTF_KIND_STRUCT:
808	case BTF_KIND_UNION:
809	case BTF_KIND_ENUM:
810	case BTF_KIND_DATASEC:
811	case BTF_KIND_FLOAT:
812	case BTF_KIND_ENUM64:
813	return true;
814	}
815
816	return false;
817	}
818
819	static const char *btf_int_encoding_str(u8 encoding)
820	{
821	if (encoding == 0)
822	return "(none)";
823	else if (encoding == BTF_INT_SIGNED)
824	return "SIGNED";
825	else if (encoding == BTF_INT_CHAR)
826	return "CHAR";
827	else if (encoding == BTF_INT_BOOL)
828	return "BOOL";
829	else
830	return "UNKN";
831	}
832
833	static u32 btf_type_int(const struct btf_type *t)
834	{
835	return (u32 )(t + 1);
836	}
837
838	static const struct btf_array btf_type_array(const struct btf_type t)
839	{
840	return (const struct btf_array *)(t + 1);
841	}
842
843	static const struct btf_enum btf_type_enum(const struct btf_type t)
844	{
845	return (const struct btf_enum *)(t + 1);
846	}
847
848	static const struct btf_var btf_type_var(const struct btf_type t)
849	{
850	return (const struct btf_var *)(t + 1);
851	}
852
853	static const struct btf_decl_tag btf_type_decl_tag(const struct btf_type t)
854	{
855	return (const struct btf_decl_tag *)(t + 1);
856	}
857
858	static const struct btf_enum64 btf_type_enum64(const struct btf_type t)
859	{
860	return (const struct btf_enum64 *)(t + 1);
861	}
862
863	static const struct btf_kind_operations btf_type_ops(const struct btf_type t)
864	{
865	return kind_ops[BTF_INFO_KIND(t->info)];
866	}
867
868	static bool btf_name_offset_valid(const struct btf *btf, u32 offset)
869	{
870	if (!BTF_STR_OFFSET_VALID(offset))
871	return false;
872
873	while (offset < btf->start_str_off)
874	btf = btf->base_btf;
875
876	offset -= btf->start_str_off;
877	return offset < btf->hdr.str_len;
878	}
879
880	static bool __btf_name_char_ok(char c, bool first)
881	{
882	if ((first ? !isalpha(c) :
883	!isalnum(c)) &&
884	c != '_' &&
885	c != '.')
886	return false;
887	return true;
888	}
889
890	const char btf_str_by_offset(const struct btf btf, u32 offset)
891	{
892	while (offset < btf->start_str_off)
893	btf = btf->base_btf;
894
895	offset -= btf->start_str_off;
896	if (offset < btf->hdr.str_len)
897	return &btf->strings[offset];
898
899	return NULL;
900	}
901
902	static bool btf_name_valid_identifier(const struct btf *btf, u32 offset)
903	{
904	/* offset must be valid */
905	const char *src = btf_str_by_offset(btf, offset);
906	const char *src_limit;
907
908	if (!__btf_name_char_ok(*src, true))
909	return false;
910
911	/* set a limit on identifier length */
912	src_limit = src + KSYM_NAME_LEN;
913	src++;
914	while (*src && src < src_limit) {
915	if (!__btf_name_char_ok(*src, false))
916	return false;
917	src++;
918	}
919
920	return !*src;
921	}
922
923	/* Allow any printable character in DATASEC names */
924	static bool btf_name_valid_section(const struct btf *btf, u32 offset)
925	{
926	/* offset must be valid */
927	const char *src = btf_str_by_offset(btf, offset);
928	const char *src_limit;
929
930	if (!*src)
931	return false;
932
933	/* set a limit on identifier length */
934	src_limit = src + KSYM_NAME_LEN;
935	while (*src && src < src_limit) {
936	if (!isprint(*src))
937	return false;
938	src++;
939	}
940
941	return !*src;
942	}
943
944	static const char __btf_name_by_offset(const struct btf btf, u32 offset)
945	{
946	const char *name;
947
948	if (!offset)
949	return "(anon)";
950
951	name = btf_str_by_offset(btf, offset);
952	return name ?: "(invalid-name-offset)";
953	}
954
955	const char btf_name_by_offset(const struct btf btf, u32 offset)
956	{
957	return btf_str_by_offset(btf, offset);
958	}
959
960	const struct btf_type btf_type_by_id(const struct btf btf, u32 type_id)
961	{
962	while (type_id < btf->start_id)
963	btf = btf->base_btf;
964
965	type_id -= btf->start_id;
966	if (type_id >= btf->nr_types)
967	return NULL;
968	return btf->types[type_id];
969	}
970	EXPORT_SYMBOL_GPL(btf_type_by_id);
971
972	/*
973	* Check that the type @t is a regular int. This means that @t is not
974	* a bit field and it has the same size as either of u8/u16/u32/u64
975	* or __int128. If @expected_size is not zero, then size of @t should
976	* be the same. A caller should already have checked that the type @t
977	* is an integer.
978	*/
979	static bool __btf_type_int_is_regular(const struct btf_type *t, size_t expected_size)
980	{
981	u32 int_data = btf_type_int(t);
982	u8 nr_bits = BTF_INT_BITS(int_data);
983	u8 nr_bytes = BITS_ROUNDUP_BYTES(nr_bits);
984
985	return BITS_PER_BYTE_MASKED(nr_bits) == 0 &&
986	BTF_INT_OFFSET(int_data) == 0 &&
987	(nr_bytes <= 16 && is_power_of_2(nr_bytes)) &&
988	(expected_size == 0 \|\| nr_bytes == expected_size);
989	}
990
991	static bool btf_type_int_is_regular(const struct btf_type *t)
992	{
993	return __btf_type_int_is_regular(t, 0);
994	}
995
996	bool btf_type_is_i32(const struct btf_type *t)
997	{
998	return btf_type_is_int(t) && __btf_type_int_is_regular(t, 4);
999	}
1000
1001	bool btf_type_is_i64(const struct btf_type *t)
1002	{
1003	return btf_type_is_int(t) && __btf_type_int_is_regular(t, 8);
1004	}
1005
1006	bool btf_type_is_primitive(const struct btf_type *t)
1007	{
1008	return (btf_type_is_int(t) && btf_type_int_is_regular(t)) \|\|
1009	btf_is_any_enum(t);
1010	}
1011
1012	/*
1013	* Check that given struct member is a regular int with expected
1014	* offset and size.
1015	*/
1016	bool btf_member_is_reg_int(const struct btf btf, const struct btf_type s,
1017	const struct btf_member *m,
1018	u32 expected_offset, u32 expected_size)
1019	{
1020	const struct btf_type *t;
1021	u32 id, int_data;
1022	u8 nr_bits;
1023
1024	id = m->type;
1025	t = btf_type_id_size(btf, &id, NULL);
1026	if (!t \|\| !btf_type_is_int(t))
1027	return false;
1028
1029	int_data = btf_type_int(t);
1030	nr_bits = BTF_INT_BITS(int_data);
1031	if (btf_type_kflag(s)) {
1032	u32 bitfield_size = BTF_MEMBER_BITFIELD_SIZE(m->offset);
1033	u32 bit_offset = BTF_MEMBER_BIT_OFFSET(m->offset);
1034
1035	/* if kflag set, int should be a regular int and
1036	* bit offset should be at byte boundary.
1037	*/
1038	return !bitfield_size &&
1039	BITS_ROUNDUP_BYTES(bit_offset) == expected_offset &&
1040	BITS_ROUNDUP_BYTES(nr_bits) == expected_size;
1041	}
1042
1043	if (BTF_INT_OFFSET(int_data) \|\|
1044	BITS_PER_BYTE_MASKED(m->offset) \|\|
1045	BITS_ROUNDUP_BYTES(m->offset) != expected_offset \|\|
1046	BITS_PER_BYTE_MASKED(nr_bits) \|\|
1047	BITS_ROUNDUP_BYTES(nr_bits) != expected_size)
1048	return false;
1049
1050	return true;
1051	}
1052
1053	/* Similar to btf_type_skip_modifiers() but does not skip typedefs. */
1054	static const struct btf_type btf_type_skip_qualifiers(const struct btf btf,
1055	u32 id)
1056	{
1057	const struct btf_type *t = btf_type_by_id(btf, id);
1058
1059	while (btf_type_is_modifier(t) &&
1060	BTF_INFO_KIND(t->info) != BTF_KIND_TYPEDEF) {
1061	t = btf_type_by_id(btf, t->type);
1062	}
1063
1064	return t;
1065	}
1066
1067	#define BTF_SHOW_MAX_ITER 10
1068
1069	#define BTF_KIND_BIT(kind) (1ULL << kind)
1070
1071	/*
1072	* Populate show->state.name with type name information.
1073	* Format of type name is
1074	*
1075	* [.member_name = ] (type_name)
1076	*/
1077	static const char btf_show_name(struct btf_show show)
1078	{
1079	/* BTF_MAX_ITER array suffixes "[]" */
1080	const char *array_suffixes = "[][][][][][][][][][]";
1081	const char *array_suffix = &array_suffixes[strlen(array_suffixes)];
1082	/* BTF_MAX_ITER pointer suffixes "" /
1083	const char ptr_suffixes = "*********";
1084	const char *ptr_suffix = &ptr_suffixes[strlen(ptr_suffixes)];
1085	const char name = NULL, prefix = "", *parens = "";
1086	const struct btf_member *m = show->state.member;
1087	const struct btf_type *t;
1088	const struct btf_array *array;
1089	u32 id = show->state.type_id;
1090	const char *member = NULL;
1091	bool show_member = false;
1092	u64 kinds = 0;
1093	int i;
1094
1095	show->state.name[0] = '\0';
1096
1097	/*
1098	* Don't show type name if we're showing an array member;
1099	* in that case we show the array type so don't need to repeat
1100	* ourselves for each member.
1101	*/
1102	if (show->state.array_member)
1103	return "";
1104
1105	/* Retrieve member name, if any. */
1106	if (m) {
1107	member = btf_name_by_offset(show->btf, m->name_off);
1108	show_member = strlen(member) > 0;
1109	id = m->type;
1110	}
1111
1112	/*
1113	* Start with type_id, as we have resolved the struct btf_type *
1114	* via btf_modifier_show() past the parent typedef to the child
1115	* struct, int etc it is defined as. In such cases, the type_id
1116	* still represents the starting type while the struct btf_type *
1117	* in our show->state points at the resolved type of the typedef.
1118	*/
1119	t = btf_type_by_id(show->btf, id);
1120	if (!t)
1121	return "";
1122
1123	/*
1124	* The goal here is to build up the right number of pointer and
1125	* array suffixes while ensuring the type name for a typedef
1126	* is represented. Along the way we accumulate a list of
1127	* BTF kinds we have encountered, since these will inform later
1128	* display; for example, pointer types will not require an
1129	* opening "{" for struct, we will just display the pointer value.
1130	*
1131	* We also want to accumulate the right number of pointer or array
1132	* indices in the format string while iterating until we get to
1133	* the typedef/pointee/array member target type.
1134	*
1135	* We start by pointing at the end of pointer and array suffix
1136	* strings; as we accumulate pointers and arrays we move the pointer
1137	* or array string backwards so it will show the expected number of
1138	* '*' or '[]' for the type. BTF_SHOW_MAX_ITER of nesting of pointers
1139	* and/or arrays and typedefs are supported as a precaution.
1140	*
1141	* We also want to get typedef name while proceeding to resolve
1142	* type it points to so that we can add parentheses if it is a
1143	* "typedef struct" etc.
1144	*/
1145	for (i = 0; i < BTF_SHOW_MAX_ITER; i++) {
1146
1147	switch (BTF_INFO_KIND(t->info)) {
1148	case BTF_KIND_TYPEDEF:
1149	if (!name)
1150	name = btf_name_by_offset(show->btf,
1151	t->name_off);
1152	kinds \|= BTF_KIND_BIT(BTF_KIND_TYPEDEF);
1153	id = t->type;
1154	break;
1155	case BTF_KIND_ARRAY:
1156	kinds \|= BTF_KIND_BIT(BTF_KIND_ARRAY);
1157	parens = "[";
1158	if (!t)
1159	return "";
1160	array = btf_type_array(t);
1161	if (array_suffix > array_suffixes)
1162	array_suffix -= 2;
1163	id = array->type;
1164	break;
1165	case BTF_KIND_PTR:
1166	kinds \|= BTF_KIND_BIT(BTF_KIND_PTR);
1167	if (ptr_suffix > ptr_suffixes)
1168	ptr_suffix -= 1;
1169	id = t->type;
1170	break;
1171	default:
1172	goto resolved;
1173	}
1174	t = btf_type_skip_qualifiers(show->btf, id);
1175	if (!id)
1176	break;
1177	}
1178	/* We may not be able to represent this type; bail to be safe */
1179	if (i == BTF_SHOW_MAX_ITER)
1180	return "";
1181
1182	resolved:
1183	if (!name)
1184	name = btf_type_is_void(t) ? "void" : btf_name_by_offset(show->btf, t->name_off);
1185
1186	switch (BTF_INFO_KIND(t->info)) {
1187	case BTF_KIND_STRUCT:
1188	case BTF_KIND_UNION:
1189	prefix = BTF_INFO_KIND(t->info) == BTF_KIND_STRUCT ?
1190	"struct" : "union";
1191	/* if it's an array of struct/union, parens is already set */
1192	if (!(kinds & (BTF_KIND_BIT(BTF_KIND_ARRAY))))
1193	parens = "{";
1194	break;
1195	case BTF_KIND_ENUM:
1196	case BTF_KIND_ENUM64:
1197	prefix = "enum";
1198	break;
1199	default:
1200	break;
1201	}
1202
1203	/* pointer does not require parens */
1204	if (kinds & BTF_KIND_BIT(BTF_KIND_PTR))
1205	parens = "";
1206	/* typedef does not require struct/union/enum prefix */
1207	if (kinds & BTF_KIND_BIT(BTF_KIND_TYPEDEF))
1208	prefix = "";
1209
1210	if (!name)
1211	name = "";
1212
1213	/* Even if we don't want type name info, we want parentheses etc */
1214	if (show->flags & BTF_SHOW_NONAME)
1215	snprintf(show->state.name, sizeof(show->state.name), "%s",
1216	parens);
1217	else
1218	snprintf(show->state.name, sizeof(show->state.name),
1219	"%s%s%s(%s%s%s%s%s%s)%s",
1220	/* first 3 strings comprise ".member = " */
1221	show_member ? "." : "",
1222	show_member ? member : "",
1223	show_member ? " = " : "",
1224	/* ...next is our prefix (struct, enum, etc) */
1225	prefix,
1226	strlen(prefix) > 0 && strlen(name) > 0 ? " " : "",
1227	/* ...this is the type name itself */
1228	name,
1229	/* ...suffixed by the appropriate '', '[]' suffixes /
1230	strlen(ptr_suffix) > 0 ? " " : "", ptr_suffix,
1231	array_suffix, parens);
1232
1233	return show->state.name;
1234	}
1235
1236	static const char __btf_show_indent(struct btf_show show)
1237	{
1238	const char *indents = " ";
1239	const char *indent = &indents[strlen(indents)];
1240
1241	if ((indent - show->state.depth) >= indents)
1242	return indent - show->state.depth;
1243	return indents;
1244	}
1245
1246	static const char btf_show_indent(struct btf_show show)
1247	{
1248	return show->flags & BTF_SHOW_COMPACT ? "" : __btf_show_indent(show);
1249	}
1250
1251	static const char btf_show_newline(struct btf_show show)
1252	{
1253	return show->flags & BTF_SHOW_COMPACT ? "" : "\n";
1254	}
1255
1256	static const char btf_show_delim(struct btf_show show)
1257	{
1258	if (show->state.depth == 0)
1259	return "";
1260
1261	if ((show->flags & BTF_SHOW_COMPACT) && show->state.type &&
1262	BTF_INFO_KIND(show->state.type->info) == BTF_KIND_UNION)
1263	return "\|";
1264
1265	return ",";
1266	}
1267
1268	__printf(2, 3) static void btf_show(struct btf_show show, const char fmt, ...)
1269	{
1270	va_list args;
1271
1272	if (!show->state.depth_check) {
1273	va_start(args, fmt);
1274	show->showfn(show, fmt, args);
1275	va_end(args);
1276	}
1277	}
1278
1279	/* Macros are used here as btf_show_type_value[s]() prepends and appends
1280	* format specifiers to the format specifier passed in; these do the work of
1281	* adding indentation, delimiters etc while the caller simply has to specify
1282	* the type value(s) in the format specifier + value(s).
1283	*/
1284	#define btf_show_type_value(show, fmt, value) \
1285	do { \
1286	if ((value) != (__typeof__(value))0 \|\| \
1287	(show->flags & BTF_SHOW_ZERO) \|\| \
1288	show->state.depth == 0) { \
1289	btf_show(show, "%s%s" fmt "%s%s", \
1290	btf_show_indent(show), \
1291	btf_show_name(show), \
1292	value, btf_show_delim(show), \
1293	btf_show_newline(show)); \
1294	if (show->state.depth > show->state.depth_to_show) \
1295	show->state.depth_to_show = show->state.depth; \
1296	} \
1297	} while (0)
1298
1299	#define btf_show_type_values(show, fmt, ...) \
1300	do { \
1301	btf_show(show, "%s%s" fmt "%s%s", btf_show_indent(show), \
1302	btf_show_name(show), \
1303	__VA_ARGS__, btf_show_delim(show), \
1304	btf_show_newline(show)); \
1305	if (show->state.depth > show->state.depth_to_show) \
1306	show->state.depth_to_show = show->state.depth; \
1307	} while (0)
1308
1309	/* How much is left to copy to safe buffer after @data? */
1310	static int btf_show_obj_size_left(struct btf_show show, void data)
1311	{
1312	return show->obj.head + show->obj.size - data;
1313	}
1314
1315	/* Is object pointed to by @data of @size already copied to our safe buffer? */
1316	static bool btf_show_obj_is_safe(struct btf_show show, void data, int size)
1317	{
1318	return data >= show->obj.data &&
1319	(data + size) < (show->obj.data + BTF_SHOW_OBJ_SAFE_SIZE);
1320	}
1321
1322	/*
1323	* If object pointed to by @data of @size falls within our safe buffer, return
1324	* the equivalent pointer to the same safe data. Assumes
1325	* copy_from_kernel_nofault() has already happened and our safe buffer is
1326	* populated.
1327	*/
1328	static void __btf_show_obj_safe(struct btf_show show, void *data, int size)
1329	{
1330	if (btf_show_obj_is_safe(show, data, size))
1331	return show->obj.safe + (data - show->obj.data);
1332	return NULL;
1333	}
1334
1335	/*
1336	* Return a safe-to-access version of data pointed to by @data.
1337	* We do this by copying the relevant amount of information
1338	* to the struct btf_show obj.safe buffer using copy_from_kernel_nofault().
1339	*
1340	* If BTF_SHOW_UNSAFE is specified, just return data as-is; no
1341	* safe copy is needed.
1342	*
1343	* Otherwise we need to determine if we have the required amount
1344	* of data (determined by the @data pointer and the size of the
1345	* largest base type we can encounter (represented by
1346	* BTF_SHOW_OBJ_BASE_TYPE_SIZE). Having that much data ensures
1347	* that we will be able to print some of the current object,
1348	* and if more is needed a copy will be triggered.
1349	* Some objects such as structs will not fit into the buffer;
1350	* in such cases additional copies when we iterate over their
1351	* members may be needed.
1352	*
1353	* btf_show_obj_safe() is used to return a safe buffer for
1354	* btf_show_start_type(); this ensures that as we recurse into
1355	* nested types we always have safe data for the given type.
1356	* This approach is somewhat wasteful; it's possible for example
1357	* that when iterating over a large union we'll end up copying the
1358	* same data repeatedly, but the goal is safety not performance.
1359	* We use stack data as opposed to per-CPU buffers because the
1360	* iteration over a type can take some time, and preemption handling
1361	* would greatly complicate use of the safe buffer.
1362	*/
1363	static void btf_show_obj_safe(struct btf_show show,
1364	const struct btf_type *t,
1365	void *data)
1366	{
1367	const struct btf_type *rt;
1368	int size_left, size;
1369	void *safe = NULL;
1370
1371	if (show->flags & BTF_SHOW_UNSAFE)
1372	return data;
1373
1374	rt = btf_resolve_size(show->btf, t, &size);
1375	if (IS_ERR(rt)) {
1376	show->state.status = PTR_ERR(rt);
1377	return NULL;
1378	}
1379
1380	/*
1381	* Is this toplevel object? If so, set total object size and
1382	* initialize pointers. Otherwise check if we still fall within
1383	* our safe object data.
1384	*/
1385	if (show->state.depth == 0) {
1386	show->obj.size = size;
1387	show->obj.head = data;
1388	} else {
1389	/*
1390	* If the size of the current object is > our remaining
1391	* safe buffer we _may_ need to do a new copy. However
1392	* consider the case of a nested struct; it's size pushes
1393	* us over the safe buffer limit, but showing any individual
1394	* struct members does not. In such cases, we don't need
1395	* to initiate a fresh copy yet; however we definitely need
1396	* at least BTF_SHOW_OBJ_BASE_TYPE_SIZE bytes left
1397	* in our buffer, regardless of the current object size.
1398	* The logic here is that as we resolve types we will
1399	* hit a base type at some point, and we need to be sure
1400	* the next chunk of data is safely available to display
1401	* that type info safely. We cannot rely on the size of
1402	* the current object here because it may be much larger
1403	* than our current buffer (e.g. task_struct is 8k).
1404	* All we want to do here is ensure that we can print the
1405	* next basic type, which we can if either
1406	* - the current type size is within the safe buffer; or
1407	* - at least BTF_SHOW_OBJ_BASE_TYPE_SIZE bytes are left in
1408	* the safe buffer.
1409	*/
1410	safe = __btf_show_obj_safe(show, data,
1411	min(size,
1412	BTF_SHOW_OBJ_BASE_TYPE_SIZE));
1413	}
1414
1415	/*
1416	* We need a new copy to our safe object, either because we haven't
1417	* yet copied and are initializing safe data, or because the data
1418	* we want falls outside the boundaries of the safe object.
1419	*/
1420	if (!safe) {
1421	size_left = btf_show_obj_size_left(show, data);
1422	if (size_left > BTF_SHOW_OBJ_SAFE_SIZE)
1423	size_left = BTF_SHOW_OBJ_SAFE_SIZE;
1424	show->state.status = copy_from_kernel_nofault(show->obj.safe,
1425	data, size_left);
1426	if (!show->state.status) {
1427	show->obj.data = data;
1428	safe = show->obj.safe;
1429	}
1430	}
1431
1432	return safe;
1433	}
1434
1435	/*
1436	* Set the type we are starting to show and return a safe data pointer
1437	* to be used for showing the associated data.
1438	*/
1439	static void btf_show_start_type(struct btf_show show,
1440	const struct btf_type *t,
1441	u32 type_id, void *data)
1442	{
1443	show->state.type = t;
1444	show->state.type_id = type_id;
1445	show->state.name[0] = '\0';
1446
1447	return btf_show_obj_safe(show, t, data);
1448	}
1449
1450	static void btf_show_end_type(struct btf_show *show)
1451	{
1452	show->state.type = NULL;
1453	show->state.type_id = 0;
1454	show->state.name[0] = '\0';
1455	}
1456
1457	static void btf_show_start_aggr_type(struct btf_show show,
1458	const struct btf_type *t,
1459	u32 type_id, void *data)
1460	{
1461	void *safe_data = btf_show_start_type(show, t, type_id, data);
1462
1463	if (!safe_data)
1464	return safe_data;
1465
1466	btf_show(show, "%s%s%s", btf_show_indent(show),
1467	btf_show_name(show),
1468	btf_show_newline(show));
1469	show->state.depth++;
1470	return safe_data;
1471	}
1472
1473	static void btf_show_end_aggr_type(struct btf_show *show,
1474	const char *suffix)
1475	{
1476	show->state.depth--;
1477	btf_show(show, "%s%s%s%s", btf_show_indent(show), suffix,
1478	btf_show_delim(show), btf_show_newline(show));
1479	btf_show_end_type(show);
1480	}
1481
1482	static void btf_show_start_member(struct btf_show *show,
1483	const struct btf_member *m)
1484	{
1485	show->state.member = m;
1486	}
1487
1488	static void btf_show_start_array_member(struct btf_show *show)
1489	{
1490	show->state.array_member = 1;
1491	btf_show_start_member(show, NULL);
1492	}
1493
1494	static void btf_show_end_member(struct btf_show *show)
1495	{
1496	show->state.member = NULL;
1497	}
1498
1499	static void btf_show_end_array_member(struct btf_show *show)
1500	{
1501	show->state.array_member = 0;
1502	btf_show_end_member(show);
1503	}
1504
1505	static void btf_show_start_array_type(struct btf_show show,
1506	const struct btf_type *t,
1507	u32 type_id,
1508	u16 array_encoding,
1509	void *data)
1510	{
1511	show->state.array_encoding = array_encoding;
1512	show->state.array_terminated = 0;
1513	return btf_show_start_aggr_type(show, t, type_id, data);
1514	}
1515
1516	static void btf_show_end_array_type(struct btf_show *show)
1517	{
1518	show->state.array_encoding = 0;
1519	show->state.array_terminated = 0;
1520	btf_show_end_aggr_type(show, "]");
1521	}
1522
1523	static void btf_show_start_struct_type(struct btf_show show,
1524	const struct btf_type *t,
1525	u32 type_id,
1526	void *data)
1527	{
1528	return btf_show_start_aggr_type(show, t, type_id, data);
1529	}
1530
1531	static void btf_show_end_struct_type(struct btf_show *show)
1532	{
1533	btf_show_end_aggr_type(show, "}");
1534	}
1535
1536	__printf(2, 3) static void __btf_verifier_log(struct bpf_verifier_log *log,
1537	const char *fmt, ...)
1538	{
1539	va_list args;
1540
1541	va_start(args, fmt);
1542	bpf_verifier_vlog(log, fmt, args);
1543	va_end(args);
1544	}
1545
1546	__printf(2, 3) static void btf_verifier_log(struct btf_verifier_env *env,
1547	const char *fmt, ...)
1548	{
1549	struct bpf_verifier_log *log = &env->log;
1550	va_list args;
1551
1552	if (!bpf_verifier_log_needed(log))
1553	return;
1554
1555	va_start(args, fmt);
1556	bpf_verifier_vlog(log, fmt, args);
1557	va_end(args);
1558	}
1559
1560	__printf(4, 5) static void __btf_verifier_log_type(struct btf_verifier_env *env,
1561	const struct btf_type *t,
1562	bool log_details,
1563	const char *fmt, ...)
1564	{
1565	struct bpf_verifier_log *log = &env->log;
1566	struct btf *btf = env->btf;
1567	va_list args;
1568
1569	if (!bpf_verifier_log_needed(log))
1570	return;
1571
1572	if (log->level == BPF_LOG_KERNEL) {
1573	/* btf verifier prints all types it is processing via
1574	* btf_verifier_log_type(..., fmt = NULL).
1575	* Skip those prints for in-kernel BTF verification.
1576	*/
1577	if (!fmt)
1578	return;
1579
1580	/* Skip logging when loading module BTF with mismatches permitted */
1581	if (env->btf->base_btf && IS_ENABLED(CONFIG_MODULE_ALLOW_BTF_MISMATCH))
1582	return;
1583	}
1584
1585	__btf_verifier_log(log, "[%u] %s %s%s",
1586	env->log_type_id,
1587	btf_type_str(t),
1588	__btf_name_by_offset(btf, t->name_off),
1589	log_details ? " " : "");
1590
1591	if (log_details)
1592	btf_type_ops(t)->log_details(env, t);
1593
1594	if (fmt && *fmt) {
1595	__btf_verifier_log(log, " ");
1596	va_start(args, fmt);
1597	bpf_verifier_vlog(log, fmt, args);
1598	va_end(args);
1599	}
1600
1601	__btf_verifier_log(log, "\n");
1602	}
1603
1604	#define btf_verifier_log_type(env, t, ...) \
1605	__btf_verifier_log_type((env), (t), true, __VA_ARGS__)
1606	#define btf_verifier_log_basic(env, t, ...) \
1607	__btf_verifier_log_type((env), (t), false, __VA_ARGS__)
1608
1609	__printf(4, 5)
1610	static void btf_verifier_log_member(struct btf_verifier_env *env,
1611	const struct btf_type *struct_type,
1612	const struct btf_member *member,
1613	const char *fmt, ...)
1614	{
1615	struct bpf_verifier_log *log = &env->log;
1616	struct btf *btf = env->btf;
1617	va_list args;
1618
1619	if (!bpf_verifier_log_needed(log))
1620	return;
1621
1622	if (log->level == BPF_LOG_KERNEL) {
1623	if (!fmt)
1624	return;
1625
1626	/* Skip logging when loading module BTF with mismatches permitted */
1627	if (env->btf->base_btf && IS_ENABLED(CONFIG_MODULE_ALLOW_BTF_MISMATCH))
1628	return;
1629	}
1630
1631	/* The CHECK_META phase already did a btf dump.
1632	*
1633	* If member is logged again, it must hit an error in
1634	* parsing this member. It is useful to print out which
1635	* struct this member belongs to.
1636	*/
1637	if (env->phase != CHECK_META)
1638	btf_verifier_log_type(env, struct_type, NULL);
1639
1640	if (btf_type_kflag(struct_type))
1641	__btf_verifier_log(log,
1642	"\t%s type_id=%u bitfield_size=%u bits_offset=%u",
1643	__btf_name_by_offset(btf, member->name_off),
1644	member->type,
1645	BTF_MEMBER_BITFIELD_SIZE(member->offset),
1646	BTF_MEMBER_BIT_OFFSET(member->offset));
1647	else
1648	__btf_verifier_log(log, "\t%s type_id=%u bits_offset=%u",
1649	__btf_name_by_offset(btf, member->name_off),
1650	member->type, member->offset);
1651
1652	if (fmt && *fmt) {
1653	__btf_verifier_log(log, " ");
1654	va_start(args, fmt);
1655	bpf_verifier_vlog(log, fmt, args);
1656	va_end(args);
1657	}
1658
1659	__btf_verifier_log(log, "\n");
1660	}
1661
1662	__printf(4, 5)
1663	static void btf_verifier_log_vsi(struct btf_verifier_env *env,
1664	const struct btf_type *datasec_type,
1665	const struct btf_var_secinfo *vsi,
1666	const char *fmt, ...)
1667	{
1668	struct bpf_verifier_log *log = &env->log;
1669	va_list args;
1670
1671	if (!bpf_verifier_log_needed(log))
1672	return;
1673	if (log->level == BPF_LOG_KERNEL && !fmt)
1674	return;
1675	if (env->phase != CHECK_META)
1676	btf_verifier_log_type(env, datasec_type, NULL);
1677
1678	__btf_verifier_log(log, "\t type_id=%u offset=%u size=%u",
1679	vsi->type, vsi->offset, vsi->size);
1680	if (fmt && *fmt) {
1681	__btf_verifier_log(log, " ");
1682	va_start(args, fmt);
1683	bpf_verifier_vlog(log, fmt, args);
1684	va_end(args);
1685	}
1686
1687	__btf_verifier_log(log, "\n");
1688	}
1689
1690	static void btf_verifier_log_hdr(struct btf_verifier_env *env,
1691	u32 btf_data_size)
1692	{
1693	struct bpf_verifier_log *log = &env->log;
1694	const struct btf *btf = env->btf;
1695	const struct btf_header *hdr;
1696
1697	if (!bpf_verifier_log_needed(log))
1698	return;
1699
1700	if (log->level == BPF_LOG_KERNEL)
1701	return;
1702	hdr = &btf->hdr;
1703	__btf_verifier_log(log, "magic: 0x%x\n", hdr->magic);
1704	__btf_verifier_log(log, "version: %u\n", hdr->version);
1705	__btf_verifier_log(log, "flags: 0x%x\n", hdr->flags);
1706	__btf_verifier_log(log, "hdr_len: %u\n", hdr->hdr_len);
1707	__btf_verifier_log(log, "type_off: %u\n", hdr->type_off);
1708	__btf_verifier_log(log, "type_len: %u\n", hdr->type_len);
1709	__btf_verifier_log(log, "str_off: %u\n", hdr->str_off);
1710	__btf_verifier_log(log, "str_len: %u\n", hdr->str_len);
1711	if (hdr->hdr_len >= sizeof(struct btf_header) &&
1712	btf_data_size >= hdr->hdr_len) {
1713	__btf_verifier_log(log, "layout_off: %u\n", hdr->layout_off);
1714	__btf_verifier_log(log, "layout_len: %u\n", hdr->layout_len);
1715	}
1716	__btf_verifier_log(log, "btf_total_size: %u\n", btf_data_size);
1717	}
1718
1719	static int btf_add_type(struct btf_verifier_env env, struct btf_type t)
1720	{
1721	struct btf *btf = env->btf;
1722
1723	if (btf->types_size == btf->nr_types) {
1724	/* Expand 'types' array */
1725
1726	struct btf_type **new_types;
1727	u32 expand_by, new_size;
1728
1729	if (btf->start_id + btf->types_size == BTF_MAX_TYPE) {
1730	btf_verifier_log(env, "Exceeded max num of types");
1731	return -E2BIG;
1732	}
1733
1734	expand_by = max_t(u32, btf->types_size >> 2, 16);
1735	new_size = min_t(u32, BTF_MAX_TYPE,
1736	btf->types_size + expand_by);
1737
1738	new_types = kvzalloc_objs(*new_types, new_size,
1739	GFP_KERNEL \| __GFP_NOWARN);
1740	if (!new_types)
1741	return -ENOMEM;
1742
1743	if (btf->nr_types == 0) {
1744	if (!btf->base_btf) {
1745	/* lazily init VOID type */
1746	new_types[0] = &btf_void;
1747	btf->nr_types++;
1748	}
1749	} else {
1750	memcpy(new_types, btf->types,
1751	sizeof(btf->types) btf->nr_types);
1752	}
1753
1754	kvfree(btf->types);
1755	btf->types = new_types;
1756	btf->types_size = new_size;
1757	}
1758
1759	btf->types[btf->nr_types++] = t;
1760
1761	return 0;
1762	}
1763
1764	static int btf_alloc_id(struct btf *btf)
1765	{
1766	int id;
1767
1768	idr_preload(GFP_KERNEL);
1769	spin_lock_bh(&btf_idr_lock);
1770	id = idr_alloc_cyclic(&btf_idr, btf, 1, INT_MAX, GFP_ATOMIC);
1771	if (id > 0)
1772	btf->id = id;
1773	spin_unlock_bh(&btf_idr_lock);
1774	idr_preload_end();
1775
1776	if (WARN_ON_ONCE(!id))
1777	return -ENOSPC;
1778
1779	return id > 0 ? 0 : id;
1780	}
1781
1782	static void btf_free_id(struct btf *btf)
1783	{
1784	unsigned long flags;
1785
1786	/*
1787	* In map-in-map, calling map_delete_elem() on outer
1788	* map will call bpf_map_put on the inner map.
1789	* It will then eventually call btf_free_id()
1790	* on the inner map. Some of the map_delete_elem()
1791	* implementation may have irq disabled, so
1792	* we need to use the _irqsave() version instead
1793	* of the _bh() version.
1794	*/
1795	spin_lock_irqsave(&btf_idr_lock, flags);
1796	if (btf->id) {
1797	idr_remove(&btf_idr, btf->id);
1798	/*
1799	* Clear the id here to make this function idempotent, since it will get
1800	* called a couple of times for module BTFs: on module unload, and then
1801	* the final btf_put(). btf_alloc_id() starts IDs with 1, so we can use
1802	* 0 as sentinel value.
1803	*/
1804	WRITE_ONCE(btf->id, 0);
1805	}
1806	spin_unlock_irqrestore(&btf_idr_lock, flags);
1807	}
1808
1809	static void btf_free_kfunc_set_tab(struct btf *btf)
1810	{
1811	struct btf_kfunc_set_tab *tab = btf->kfunc_set_tab;
1812	int hook;
1813
1814	if (!tab)
1815	return;
1816	for (hook = 0; hook < ARRAY_SIZE(tab->sets); hook++)
1817	kfree(tab->sets[hook]);
1818	kfree(tab);
1819	btf->kfunc_set_tab = NULL;
1820	}
1821
1822	static void btf_free_dtor_kfunc_tab(struct btf *btf)
1823	{
1824	struct btf_id_dtor_kfunc_tab *tab = btf->dtor_kfunc_tab;
1825
1826	if (!tab)
1827	return;
1828	kfree(tab);
1829	btf->dtor_kfunc_tab = NULL;
1830	}
1831
1832	static void btf_struct_metas_free(struct btf_struct_metas *tab)
1833	{
1834	int i;
1835
1836	if (!tab)
1837	return;
1838	for (i = 0; i < tab->cnt; i++)
1839	btf_record_free(tab->types[i].record);
1840	kfree(tab);
1841	}
1842
1843	static void btf_free_struct_meta_tab(struct btf *btf)
1844	{
1845	struct btf_struct_metas *tab = btf->struct_meta_tab;
1846
1847	btf_struct_metas_free(tab);
1848	btf->struct_meta_tab = NULL;
1849	}
1850
1851	static void btf_free_struct_ops_tab(struct btf *btf)
1852	{
1853	struct btf_struct_ops_tab *tab = btf->struct_ops_tab;
1854	u32 i;
1855
1856	if (!tab)
1857	return;
1858
1859	for (i = 0; i < tab->cnt; i++)
1860	bpf_struct_ops_desc_release(&tab->ops[i]);
1861
1862	kfree(tab);
1863	btf->struct_ops_tab = NULL;
1864	}
1865
1866	static void btf_free(struct btf *btf)
1867	{
1868	btf_free_struct_meta_tab(btf);
1869	btf_free_dtor_kfunc_tab(btf);
1870	btf_free_kfunc_set_tab(btf);
1871	btf_free_struct_ops_tab(btf);
1872	kvfree(btf->types);
1873	kvfree(btf->resolved_sizes);
1874	kvfree(btf->resolved_ids);
1875	/* vmlinux does not allocate btf->data, it simply points it at
1876	* __start_BTF.
1877	*/
1878	if (!btf_is_vmlinux(btf))
1879	kvfree(btf->data);
1880	kvfree(btf->base_id_map);
1881	kfree(btf);
1882	}
1883
1884	static void btf_free_rcu(struct rcu_head *rcu)
1885	{
1886	struct btf *btf = container_of(rcu, struct btf, rcu);
1887
1888	btf_free(btf);
1889	}
1890
1891	const char btf_get_name(const struct btf btf)
1892	{
1893	return btf->name;
1894	}
1895
1896	void btf_get(struct btf *btf)
1897	{
1898	refcount_inc(&btf->refcnt);
1899	}
1900
1901	void btf_put(struct btf *btf)
1902	{
1903	if (btf && refcount_dec_and_test(&btf->refcnt)) {
1904	btf_free_id(btf);
1905	call_rcu(&btf->rcu, btf_free_rcu);
1906	}
1907	}
1908
1909	struct btf btf_base_btf(const struct btf btf)
1910	{
1911	return btf->base_btf;
1912	}
1913
1914	const struct btf_header btf_header(const struct btf btf)
1915	{
1916	return &btf->hdr;
1917	}
1918
1919	void btf_set_base_btf(struct btf btf, const struct btf base_btf)
1920	{
1921	btf->base_btf = (struct btf *)base_btf;
1922	btf->start_id = btf_nr_types(base_btf);
1923	btf->start_str_off = base_btf->hdr.str_len;
1924	}
1925
1926	static int env_resolve_init(struct btf_verifier_env *env)
1927	{
1928	struct btf *btf = env->btf;
1929	u32 nr_types = btf->nr_types;
1930	u32 *resolved_sizes = NULL;
1931	u32 *resolved_ids = NULL;
1932	u8 *visit_states = NULL;
1933
1934	resolved_sizes = kvcalloc(nr_types, sizeof(*resolved_sizes),
1935	GFP_KERNEL \| __GFP_NOWARN);
1936	if (!resolved_sizes)
1937	goto nomem;
1938
1939	resolved_ids = kvcalloc(nr_types, sizeof(*resolved_ids),
1940	GFP_KERNEL \| __GFP_NOWARN);
1941	if (!resolved_ids)
1942	goto nomem;
1943
1944	visit_states = kvcalloc(nr_types, sizeof(*visit_states),
1945	GFP_KERNEL \| __GFP_NOWARN);
1946	if (!visit_states)
1947	goto nomem;
1948
1949	btf->resolved_sizes = resolved_sizes;
1950	btf->resolved_ids = resolved_ids;
1951	env->visit_states = visit_states;
1952
1953	return 0;
1954
1955	nomem:
1956	kvfree(resolved_sizes);
1957	kvfree(resolved_ids);
1958	kvfree(visit_states);
1959	return -ENOMEM;
1960	}
1961
1962	static void btf_verifier_env_free(struct btf_verifier_env *env)
1963	{
1964	kvfree(env->visit_states);
1965	kfree(env);
1966	}
1967
1968	static bool env_type_is_resolve_sink(const struct btf_verifier_env *env,
1969	const struct btf_type *next_type)
1970	{
1971	switch (env->resolve_mode) {
1972	case RESOLVE_TBD:
1973	/* int, enum or void is a sink */
1974	return !btf_type_needs_resolve(next_type);
1975	case RESOLVE_PTR:
1976	/* int, enum, void, struct, array, func or func_proto is a sink
1977	* for ptr
1978	*/
1979	return !btf_type_is_modifier(next_type) &&
1980	!btf_type_is_ptr(next_type);
1981	case RESOLVE_STRUCT_OR_ARRAY:
1982	/* int, enum, void, ptr, func or func_proto is a sink
1983	* for struct and array
1984	*/
1985	return !btf_type_is_modifier(next_type) &&
1986	!btf_type_is_array(next_type) &&
1987	!btf_type_is_struct(next_type);
1988	default:
1989	BUG();
1990	}
1991	}
1992
1993	static bool env_type_is_resolved(const struct btf_verifier_env *env,
1994	u32 type_id)
1995	{
1996	/* base BTF types should be resolved by now */
1997	if (type_id < env->btf->start_id)
1998	return true;
1999
2000	return env->visit_states[type_id - env->btf->start_id] == RESOLVED;
2001	}
2002
2003	static int env_stack_push(struct btf_verifier_env *env,
2004	const struct btf_type *t, u32 type_id)
2005	{
2006	const struct btf *btf = env->btf;
2007	struct resolve_vertex *v;
2008
2009	if (env->top_stack == MAX_RESOLVE_DEPTH)
2010	return -E2BIG;
2011
2012	if (type_id < btf->start_id
2013	\|\| env->visit_states[type_id - btf->start_id] != NOT_VISITED)
2014	return -EEXIST;
2015
2016	env->visit_states[type_id - btf->start_id] = VISITED;
2017
2018	v = &env->stack[env->top_stack++];
2019	v->t = t;
2020	v->type_id = type_id;
2021	v->next_member = 0;
2022
2023	if (env->resolve_mode == RESOLVE_TBD) {
2024	if (btf_type_is_ptr(t))
2025	env->resolve_mode = RESOLVE_PTR;
2026	else if (btf_type_is_struct(t) \|\| btf_type_is_array(t))
2027	env->resolve_mode = RESOLVE_STRUCT_OR_ARRAY;
2028	}
2029
2030	return 0;
2031	}
2032
2033	static void env_stack_set_next_member(struct btf_verifier_env *env,
2034	u32 next_member)
2035	{
2036	env->stack[env->top_stack - 1].next_member = next_member;
2037	}
2038
2039	static void env_stack_pop_resolved(struct btf_verifier_env *env,
2040	u32 resolved_type_id,
2041	u32 resolved_size)
2042	{
2043	u32 type_id = env->stack[--(env->top_stack)].type_id;
2044	struct btf *btf = env->btf;
2045
2046	type_id -= btf->start_id; /* adjust to local type id */
2047	btf->resolved_sizes[type_id] = resolved_size;
2048	btf->resolved_ids[type_id] = resolved_type_id;
2049	env->visit_states[type_id] = RESOLVED;
2050	}
2051
2052	static const struct resolve_vertex env_stack_peak(struct btf_verifier_env env)
2053	{
2054	return env->top_stack ? &env->stack[env->top_stack - 1] : NULL;
2055	}
2056
2057	/* Resolve the size of a passed-in "type"
2058	*
2059	* type: is an array (e.g. u32 array[x][y])
2060	* return type: type "u32[x][y]", i.e. BTF_KIND_ARRAY,
2061	* type_size: (x y * sizeof(u32)). Hence, *type_size always
2062	* corresponds to the return type.
2063	* *elem_type: u32
2064	* *elem_id: id of u32
2065	* total_nelems: (x y). Hence, individual elem size is
2066	* (type_size / total_nelems)
2067	* *type_id: id of type if it's changed within the function, 0 if not
2068	*
2069	* type: is not an array (e.g. const struct X)
2070	* return type: type "struct X"
2071	* *type_size: sizeof(struct X)
2072	* *elem_type: same as return type ("struct X")
2073	* *elem_id: 0
2074	* *total_nelems: 1
2075	* *type_id: id of type if it's changed within the function, 0 if not
2076	*/
2077	static const struct btf_type *
2078	__btf_resolve_size(const struct btf btf, const struct btf_type type,
2079	u32 type_size, const struct btf_type *elem_type,
2080	u32 elem_id, u32 total_nelems, u32 *type_id)
2081	{
2082	const struct btf_type *array_type = NULL;
2083	const struct btf_array *array = NULL;
2084	u32 i, size, nelems = 1, id = 0;
2085
2086	for (i = 0; i < MAX_RESOLVE_DEPTH; i++) {
2087	switch (BTF_INFO_KIND(type->info)) {
2088	/* type->size can be used */
2089	case BTF_KIND_INT:
2090	case BTF_KIND_STRUCT:
2091	case BTF_KIND_UNION:
2092	case BTF_KIND_ENUM:
2093	case BTF_KIND_FLOAT:
2094	case BTF_KIND_ENUM64:
2095	size = type->size;
2096	goto resolved;
2097
2098	case BTF_KIND_PTR:
2099	size = sizeof(void *);
2100	goto resolved;
2101
2102	/* Modifiers */
2103	case BTF_KIND_TYPEDEF:
2104	case BTF_KIND_VOLATILE:
2105	case BTF_KIND_CONST:
2106	case BTF_KIND_RESTRICT:
2107	case BTF_KIND_TYPE_TAG:
2108	id = type->type;
2109	type = btf_type_by_id(btf, type->type);
2110	break;
2111
2112	case BTF_KIND_ARRAY:
2113	if (!array_type)
2114	array_type = type;
2115	array = btf_type_array(type);
2116	if (nelems && array->nelems > U32_MAX / nelems)
2117	return ERR_PTR(-EINVAL);
2118	nelems *= array->nelems;
2119	type = btf_type_by_id(btf, array->type);
2120	break;
2121
2122	/* type without size */
2123	default:
2124	return ERR_PTR(-EINVAL);
2125	}
2126	}
2127
2128	return ERR_PTR(-EINVAL);
2129
2130	resolved:
2131	if (nelems && size > U32_MAX / nelems)
2132	return ERR_PTR(-EINVAL);
2133
2134	type_size = nelems size;
2135	if (total_nelems)
2136	*total_nelems = nelems;
2137	if (elem_type)
2138	*elem_type = type;
2139	if (elem_id)
2140	*elem_id = array ? array->type : 0;
2141	if (type_id && id)
2142	*type_id = id;
2143
2144	return array_type ? : type;
2145	}
2146
2147	const struct btf_type *
2148	btf_resolve_size(const struct btf btf, const struct btf_type type,
2149	u32 *type_size)
2150	{
2151	return __btf_resolve_size(btf, type, type_size, NULL, NULL, NULL, NULL);
2152	}
2153
2154	static u32 btf_resolved_type_id(const struct btf *btf, u32 type_id)
2155	{
2156	while (type_id < btf->start_id)
2157	btf = btf->base_btf;
2158
2159	return btf->resolved_ids[type_id - btf->start_id];
2160	}
2161
2162	/* The input param "type_id" must point to a needs_resolve type */
2163	static const struct btf_type btf_type_id_resolve(const struct btf btf,
2164	u32 *type_id)
2165	{
2166	type_id = btf_resolved_type_id(btf, type_id);
2167	return btf_type_by_id(btf, *type_id);
2168	}
2169
2170	static u32 btf_resolved_type_size(const struct btf *btf, u32 type_id)
2171	{
2172	while (type_id < btf->start_id)
2173	btf = btf->base_btf;
2174
2175	return btf->resolved_sizes[type_id - btf->start_id];
2176	}
2177
2178	const struct btf_type btf_type_id_size(const struct btf btf,
2179	u32 type_id, u32 ret_size)
2180	{
2181	const struct btf_type *size_type;
2182	u32 size_type_id = *type_id;
2183	u32 size = 0;
2184
2185	size_type = btf_type_by_id(btf, size_type_id);
2186	if (btf_type_nosize_or_null(size_type))
2187	return NULL;
2188
2189	if (btf_type_has_size(size_type)) {
2190	size = size_type->size;
2191	} else if (btf_type_is_array(size_type)) {
2192	size = btf_resolved_type_size(btf, size_type_id);
2193	} else if (btf_type_is_ptr(size_type)) {
2194	size = sizeof(void *);
2195	} else {
2196	if (WARN_ON_ONCE(!btf_type_is_modifier(size_type) &&
2197	!btf_type_is_var(size_type)))
2198	return NULL;
2199
2200	size_type_id = btf_resolved_type_id(btf, size_type_id);
2201	size_type = btf_type_by_id(btf, size_type_id);
2202	if (btf_type_nosize_or_null(size_type))
2203	return NULL;
2204	else if (btf_type_has_size(size_type))
2205	size = size_type->size;
2206	else if (btf_type_is_array(size_type))
2207	size = btf_resolved_type_size(btf, size_type_id);
2208	else if (btf_type_is_ptr(size_type))
2209	size = sizeof(void *);
2210	else
2211	return NULL;
2212	}
2213
2214	*type_id = size_type_id;
2215	if (ret_size)
2216	*ret_size = size;
2217
2218	return size_type;
2219	}
2220
2221	static int btf_df_check_member(struct btf_verifier_env *env,
2222	const struct btf_type *struct_type,
2223	const struct btf_member *member,
2224	const struct btf_type *member_type)
2225	{
2226	btf_verifier_log_basic(env, struct_type,
2227	"Unsupported check_member");
2228	return -EINVAL;
2229	}
2230
2231	static int btf_df_check_kflag_member(struct btf_verifier_env *env,
2232	const struct btf_type *struct_type,
2233	const struct btf_member *member,
2234	const struct btf_type *member_type)
2235	{
2236	btf_verifier_log_basic(env, struct_type,
2237	"Unsupported check_kflag_member");
2238	return -EINVAL;
2239	}
2240
2241	/* Used for ptr, array struct/union and float type members.
2242	* int, enum and modifier types have their specific callback functions.
2243	*/
2244	static int btf_generic_check_kflag_member(struct btf_verifier_env *env,
2245	const struct btf_type *struct_type,
2246	const struct btf_member *member,
2247	const struct btf_type *member_type)
2248	{
2249	if (BTF_MEMBER_BITFIELD_SIZE(member->offset)) {
2250	btf_verifier_log_member(env, struct_type, member,
2251	"Invalid member bitfield_size");
2252	return -EINVAL;
2253	}
2254
2255	/* bitfield size is 0, so member->offset represents bit offset only.
2256	* It is safe to call non kflag check_member variants.
2257	*/
2258	return btf_type_ops(member_type)->check_member(env, struct_type,
2259	member,
2260	member_type);
2261	}
2262
2263	static int btf_df_resolve(struct btf_verifier_env *env,
2264	const struct resolve_vertex *v)
2265	{
2266	btf_verifier_log_basic(env, v->t, "Unsupported resolve");
2267	return -EINVAL;
2268	}
2269
2270	static void btf_df_show(const struct btf btf, const struct btf_type t,
2271	u32 type_id, void *data, u8 bits_offsets,
2272	struct btf_show *show)
2273	{
2274	btf_show(show, "<unsupported kind:%u>", BTF_INFO_KIND(t->info));
2275	}
2276
2277	static int btf_int_check_member(struct btf_verifier_env *env,
2278	const struct btf_type *struct_type,
2279	const struct btf_member *member,
2280	const struct btf_type *member_type)
2281	{
2282	u32 int_data = btf_type_int(member_type);
2283	u32 struct_bits_off = member->offset;
2284	u32 struct_size = struct_type->size;
2285	u32 nr_copy_bits;
2286	u32 bytes_offset;
2287
2288	if (U32_MAX - struct_bits_off < BTF_INT_OFFSET(int_data)) {
2289	btf_verifier_log_member(env, struct_type, member,
2290	"bits_offset exceeds U32_MAX");
2291	return -EINVAL;
2292	}
2293
2294	struct_bits_off += BTF_INT_OFFSET(int_data);
2295	bytes_offset = BITS_ROUNDDOWN_BYTES(struct_bits_off);
2296	nr_copy_bits = BTF_INT_BITS(int_data) +
2297	BITS_PER_BYTE_MASKED(struct_bits_off);
2298
2299	if (nr_copy_bits > BITS_PER_U128) {
2300	btf_verifier_log_member(env, struct_type, member,
2301	"nr_copy_bits exceeds 128");
2302	return -EINVAL;
2303	}
2304
2305	if (struct_size < bytes_offset \|\|
2306	struct_size - bytes_offset < BITS_ROUNDUP_BYTES(nr_copy_bits)) {
2307	btf_verifier_log_member(env, struct_type, member,
2308	"Member exceeds struct_size");
2309	return -EINVAL;
2310	}
2311
2312	return 0;
2313	}
2314
2315	static int btf_int_check_kflag_member(struct btf_verifier_env *env,
2316	const struct btf_type *struct_type,
2317	const struct btf_member *member,
2318	const struct btf_type *member_type)
2319	{
2320	u32 struct_bits_off, nr_bits, nr_int_data_bits, bytes_offset;
2321	u32 int_data = btf_type_int(member_type);
2322	u32 struct_size = struct_type->size;
2323	u32 nr_copy_bits;
2324
2325	/* a regular int type is required for the kflag int member */
2326	if (!btf_type_int_is_regular(member_type)) {
2327	btf_verifier_log_member(env, struct_type, member,
2328	"Invalid member base type");
2329	return -EINVAL;
2330	}
2331
2332	/* check sanity of bitfield size */
2333	nr_bits = BTF_MEMBER_BITFIELD_SIZE(member->offset);
2334	struct_bits_off = BTF_MEMBER_BIT_OFFSET(member->offset);
2335	nr_int_data_bits = BTF_INT_BITS(int_data);
2336	if (!nr_bits) {
2337	/* Not a bitfield member, member offset must be at byte
2338	* boundary.
2339	*/
2340	if (BITS_PER_BYTE_MASKED(struct_bits_off)) {
2341	btf_verifier_log_member(env, struct_type, member,
2342	"Invalid member offset");
2343	return -EINVAL;
2344	}
2345
2346	nr_bits = nr_int_data_bits;
2347	} else if (nr_bits > nr_int_data_bits) {
2348	btf_verifier_log_member(env, struct_type, member,
2349	"Invalid member bitfield_size");
2350	return -EINVAL;
2351	}
2352
2353	bytes_offset = BITS_ROUNDDOWN_BYTES(struct_bits_off);
2354	nr_copy_bits = nr_bits + BITS_PER_BYTE_MASKED(struct_bits_off);
2355	if (nr_copy_bits > BITS_PER_U128) {
2356	btf_verifier_log_member(env, struct_type, member,
2357	"nr_copy_bits exceeds 128");
2358	return -EINVAL;
2359	}
2360
2361	if (struct_size < bytes_offset \|\|
2362	struct_size - bytes_offset < BITS_ROUNDUP_BYTES(nr_copy_bits)) {
2363	btf_verifier_log_member(env, struct_type, member,
2364	"Member exceeds struct_size");
2365	return -EINVAL;
2366	}
2367
2368	return 0;
2369	}
2370
2371	static s32 btf_int_check_meta(struct btf_verifier_env *env,
2372	const struct btf_type *t,
2373	u32 meta_left)
2374	{
2375	u32 int_data, nr_bits, meta_needed = sizeof(int_data);
2376	u16 encoding;
2377
2378	if (meta_left < meta_needed) {
2379	btf_verifier_log_basic(env, t,
2380	"meta_left:%u meta_needed:%u",
2381	meta_left, meta_needed);
2382	return -EINVAL;
2383	}
2384
2385	if (btf_type_vlen(t)) {
2386	btf_verifier_log_type(env, t, "vlen != 0");
2387	return -EINVAL;
2388	}
2389
2390	if (btf_type_kflag(t)) {
2391	btf_verifier_log_type(env, t, "Invalid btf_info kind_flag");
2392	return -EINVAL;
2393	}
2394
2395	int_data = btf_type_int(t);
2396	if (int_data & ~BTF_INT_MASK) {
2397	btf_verifier_log_basic(env, t, "Invalid int_data:%x",
2398	int_data);
2399	return -EINVAL;
2400	}
2401
2402	nr_bits = BTF_INT_BITS(int_data) + BTF_INT_OFFSET(int_data);
2403
2404	if (nr_bits > BITS_PER_U128) {
2405	btf_verifier_log_type(env, t, "nr_bits exceeds %zu",
2406	BITS_PER_U128);
2407	return -EINVAL;
2408	}
2409
2410	if (BITS_ROUNDUP_BYTES(nr_bits) > t->size) {
2411	btf_verifier_log_type(env, t, "nr_bits exceeds type_size");
2412	return -EINVAL;
2413	}
2414
2415	/*
2416	* Only one of the encoding bits is allowed and it
2417	* should be sufficient for the pretty print purpose (i.e. decoding).
2418	* Multiple bits can be allowed later if it is found
2419	* to be insufficient.
2420	*/
2421	encoding = BTF_INT_ENCODING(int_data);
2422	if (encoding &&
2423	encoding != BTF_INT_SIGNED &&
2424	encoding != BTF_INT_CHAR &&
2425	encoding != BTF_INT_BOOL) {
2426	btf_verifier_log_type(env, t, "Unsupported encoding");
2427	return -ENOTSUPP;
2428	}
2429
2430	btf_verifier_log_type(env, t, NULL);
2431
2432	return meta_needed;
2433	}
2434
2435	static void btf_int_log(struct btf_verifier_env *env,
2436	const struct btf_type *t)
2437	{
2438	int int_data = btf_type_int(t);
2439
2440	btf_verifier_log(env,
2441	"size=%u bits_offset=%u nr_bits=%u encoding=%s",
2442	t->size, BTF_INT_OFFSET(int_data),
2443	BTF_INT_BITS(int_data),
2444	btf_int_encoding_str(BTF_INT_ENCODING(int_data)));
2445	}
2446
2447	static void btf_int128_print(struct btf_show show, void data)
2448	{
2449	/* data points to a __int128 number.
2450	* Suppose
2451	* int128_num = (__int128 )data;
2452	* The below formulas shows what upper_num and lower_num represents:
2453	* upper_num = int128_num >> 64;
2454	* lower_num = int128_num & 0xffffffffFFFFFFFFULL;
2455	*/
2456	u64 upper_num, lower_num;
2457
2458	#ifdef __BIG_ENDIAN_BITFIELD
2459	upper_num = (u64 )data;
2460	lower_num = (u64 )(data + 8);
2461	#else
2462	upper_num = (u64 )(data + 8);
2463	lower_num = (u64 )data;
2464	#endif
2465	if (upper_num == 0)
2466	btf_show_type_value(show, "0x%llx", lower_num);
2467	else
2468	btf_show_type_values(show, "0x%llx%016llx", upper_num,
2469	lower_num);
2470	}
2471
2472	static void btf_int128_shift(u64 *print_num, u16 left_shift_bits,
2473	u16 right_shift_bits)
2474	{
2475	u64 upper_num, lower_num;
2476
2477	#ifdef __BIG_ENDIAN_BITFIELD
2478	upper_num = print_num[0];
2479	lower_num = print_num[1];
2480	#else
2481	upper_num = print_num[1];
2482	lower_num = print_num[0];
2483	#endif
2484
2485	/* shake out un-needed bits by shift/or operations */
2486	if (left_shift_bits >= 64) {
2487	upper_num = lower_num << (left_shift_bits - 64);
2488	lower_num = 0;
2489	} else {
2490	upper_num = (upper_num << left_shift_bits) \|
2491	(lower_num >> (64 - left_shift_bits));
2492	lower_num = lower_num << left_shift_bits;
2493	}
2494
2495	if (right_shift_bits >= 64) {
2496	lower_num = upper_num >> (right_shift_bits - 64);
2497	upper_num = 0;
2498	} else {
2499	lower_num = (lower_num >> right_shift_bits) \|
2500	(upper_num << (64 - right_shift_bits));
2501	upper_num = upper_num >> right_shift_bits;
2502	}
2503
2504	#ifdef __BIG_ENDIAN_BITFIELD
2505	print_num[0] = upper_num;
2506	print_num[1] = lower_num;
2507	#else
2508	print_num[0] = lower_num;
2509	print_num[1] = upper_num;
2510	#endif
2511	}
2512
2513	static void btf_bitfield_show(void *data, u8 bits_offset,
2514	u8 nr_bits, struct btf_show *show)
2515	{
2516	u16 left_shift_bits, right_shift_bits;
2517	u8 nr_copy_bytes;
2518	u8 nr_copy_bits;
2519	u64 print_num[2] = {};
2520
2521	nr_copy_bits = nr_bits + bits_offset;
2522	nr_copy_bytes = BITS_ROUNDUP_BYTES(nr_copy_bits);
2523
2524	memcpy(print_num, data, nr_copy_bytes);
2525
2526	#ifdef __BIG_ENDIAN_BITFIELD
2527	left_shift_bits = bits_offset;
2528	#else
2529	left_shift_bits = BITS_PER_U128 - nr_copy_bits;
2530	#endif
2531	right_shift_bits = BITS_PER_U128 - nr_bits;
2532
2533	btf_int128_shift(print_num, left_shift_bits, right_shift_bits);
2534	btf_int128_print(show, print_num);
2535	}
2536
2537	static void btf_int_bits_show(const struct btf *btf,
2538	const struct btf_type *t,
2539	void *data, u8 bits_offset,
2540	struct btf_show *show)
2541	{
2542	u32 int_data = btf_type_int(t);
2543	u8 nr_bits = BTF_INT_BITS(int_data);
2544	u8 total_bits_offset;
2545
2546	/*
2547	* bits_offset is at most 7.
2548	* BTF_INT_OFFSET() cannot exceed 128 bits.
2549	*/
2550	total_bits_offset = bits_offset + BTF_INT_OFFSET(int_data);
2551	data += BITS_ROUNDDOWN_BYTES(total_bits_offset);
2552	bits_offset = BITS_PER_BYTE_MASKED(total_bits_offset);
2553	btf_bitfield_show(data, bits_offset, nr_bits, show);
2554	}
2555
2556	static void btf_int_show(const struct btf btf, const struct btf_type t,
2557	u32 type_id, void *data, u8 bits_offset,
2558	struct btf_show *show)
2559	{
2560	u32 int_data = btf_type_int(t);
2561	u8 encoding = BTF_INT_ENCODING(int_data);
2562	bool sign = encoding & BTF_INT_SIGNED;
2563	u8 nr_bits = BTF_INT_BITS(int_data);
2564	void *safe_data;
2565
2566	safe_data = btf_show_start_type(show, t, type_id, data);
2567	if (!safe_data)
2568	return;
2569
2570	if (bits_offset \|\| BTF_INT_OFFSET(int_data) \|\|
2571	BITS_PER_BYTE_MASKED(nr_bits)) {
2572	btf_int_bits_show(btf, t, safe_data, bits_offset, show);
2573	goto out;
2574	}
2575
2576	switch (nr_bits) {
2577	case 128:
2578	btf_int128_print(show, safe_data);
2579	break;
2580	case 64:
2581	if (sign)
2582	btf_show_type_value(show, "%lld", (s64 )safe_data);
2583	else
2584	btf_show_type_value(show, "%llu", (u64 )safe_data);
2585	break;
2586	case 32:
2587	if (sign)
2588	btf_show_type_value(show, "%d", (s32 )safe_data);
2589	else
2590	btf_show_type_value(show, "%u", (u32 )safe_data);
2591	break;
2592	case 16:
2593	if (sign)
2594	btf_show_type_value(show, "%d", (s16 )safe_data);
2595	else
2596	btf_show_type_value(show, "%u", (u16 )safe_data);
2597	break;
2598	case 8:
2599	if (show->state.array_encoding == BTF_INT_CHAR) {
2600	/* check for null terminator */
2601	if (show->state.array_terminated)
2602	break;
2603	if ((char )data == '\0') {
2604	show->state.array_terminated = 1;
2605	break;
2606	}
2607	if (isprint((char )data)) {
2608	btf_show_type_value(show, "'%c'",
2609	(char )safe_data);
2610	break;
2611	}
2612	}
2613	if (sign)
2614	btf_show_type_value(show, "%d", (s8 )safe_data);
2615	else
2616	btf_show_type_value(show, "%u", (u8 )safe_data);
2617	break;
2618	default:
2619	btf_int_bits_show(btf, t, safe_data, bits_offset, show);
2620	break;
2621	}
2622	out:
2623	btf_show_end_type(show);
2624	}
2625
2626	static const struct btf_kind_operations int_ops = {
2627	.check_meta = btf_int_check_meta,
2628	.resolve = btf_df_resolve,
2629	.check_member = btf_int_check_member,
2630	.check_kflag_member = btf_int_check_kflag_member,
2631	.log_details = btf_int_log,
2632	.show = btf_int_show,
2633	};
2634
2635	static int btf_modifier_check_member(struct btf_verifier_env *env,
2636	const struct btf_type *struct_type,
2637	const struct btf_member *member,
2638	const struct btf_type *member_type)
2639	{
2640	const struct btf_type *resolved_type;
2641	u32 resolved_type_id = member->type;
2642	struct btf_member resolved_member;
2643	struct btf *btf = env->btf;
2644
2645	resolved_type = btf_type_id_size(btf, &resolved_type_id, NULL);
2646	if (!resolved_type) {
2647	btf_verifier_log_member(env, struct_type, member,
2648	"Invalid member");
2649	return -EINVAL;
2650	}
2651
2652	resolved_member = *member;
2653	resolved_member.type = resolved_type_id;
2654
2655	return btf_type_ops(resolved_type)->check_member(env, struct_type,
2656	&resolved_member,
2657	resolved_type);
2658	}
2659
2660	static int btf_modifier_check_kflag_member(struct btf_verifier_env *env,
2661	const struct btf_type *struct_type,
2662	const struct btf_member *member,
2663	const struct btf_type *member_type)
2664	{
2665	const struct btf_type *resolved_type;
2666	u32 resolved_type_id = member->type;
2667	struct btf_member resolved_member;
2668	struct btf *btf = env->btf;
2669
2670	resolved_type = btf_type_id_size(btf, &resolved_type_id, NULL);
2671	if (!resolved_type) {
2672	btf_verifier_log_member(env, struct_type, member,
2673	"Invalid member");
2674	return -EINVAL;
2675	}
2676
2677	resolved_member = *member;
2678	resolved_member.type = resolved_type_id;
2679
2680	return btf_type_ops(resolved_type)->check_kflag_member(env, struct_type,
2681	&resolved_member,
2682	resolved_type);
2683	}
2684
2685	static int btf_ptr_check_member(struct btf_verifier_env *env,
2686	const struct btf_type *struct_type,
2687	const struct btf_member *member,
2688	const struct btf_type *member_type)
2689	{
2690	u32 struct_size, struct_bits_off, bytes_offset;
2691
2692	struct_size = struct_type->size;
2693	struct_bits_off = member->offset;
2694	bytes_offset = BITS_ROUNDDOWN_BYTES(struct_bits_off);
2695
2696	if (BITS_PER_BYTE_MASKED(struct_bits_off)) {
2697	btf_verifier_log_member(env, struct_type, member,
2698	"Member is not byte aligned");
2699	return -EINVAL;
2700	}
2701
2702	if (struct_size - bytes_offset < sizeof(void *)) {
2703	btf_verifier_log_member(env, struct_type, member,
2704	"Member exceeds struct_size");
2705	return -EINVAL;
2706	}
2707
2708	return 0;
2709	}
2710
2711	static int btf_ref_type_check_meta(struct btf_verifier_env *env,
2712	const struct btf_type *t,
2713	u32 meta_left)
2714	{
2715	const char *value;
2716
2717	if (btf_type_vlen(t)) {
2718	btf_verifier_log_type(env, t, "vlen != 0");
2719	return -EINVAL;
2720	}
2721
2722	if (btf_type_kflag(t) && !btf_type_is_type_tag(t)) {
2723	btf_verifier_log_type(env, t, "Invalid btf_info kind_flag");
2724	return -EINVAL;
2725	}
2726
2727	if (!BTF_TYPE_ID_VALID(t->type)) {
2728	btf_verifier_log_type(env, t, "Invalid type_id");
2729	return -EINVAL;
2730	}
2731
2732	/* typedef/type_tag type must have a valid name, and other ref types,
2733	* volatile, const, restrict, should have a null name.
2734	*/
2735	if (BTF_INFO_KIND(t->info) == BTF_KIND_TYPEDEF) {
2736	if (!t->name_off \|\|
2737	!btf_name_valid_identifier(env->btf, t->name_off)) {
2738	btf_verifier_log_type(env, t, "Invalid name");
2739	return -EINVAL;
2740	}
2741	} else if (BTF_INFO_KIND(t->info) == BTF_KIND_TYPE_TAG) {
2742	value = btf_name_by_offset(env->btf, t->name_off);
2743	if (!value \|\| !value[0]) {
2744	btf_verifier_log_type(env, t, "Invalid name");
2745	return -EINVAL;
2746	}
2747	} else {
2748	if (t->name_off) {
2749	btf_verifier_log_type(env, t, "Invalid name");
2750	return -EINVAL;
2751	}
2752	}
2753
2754	btf_verifier_log_type(env, t, NULL);
2755
2756	return 0;
2757	}
2758
2759	static int btf_modifier_resolve(struct btf_verifier_env *env,
2760	const struct resolve_vertex *v)
2761	{
2762	const struct btf_type *t = v->t;
2763	const struct btf_type *next_type;
2764	u32 next_type_id = t->type;
2765	struct btf *btf = env->btf;
2766
2767	next_type = btf_type_by_id(btf, next_type_id);
2768	if (!next_type \|\| btf_type_is_resolve_source_only(next_type)) {
2769	btf_verifier_log_type(env, v->t, "Invalid type_id");
2770	return -EINVAL;
2771	}
2772
2773	if (!env_type_is_resolve_sink(env, next_type) &&
2774	!env_type_is_resolved(env, next_type_id))
2775	return env_stack_push(env, next_type, next_type_id);
2776
2777	/* Figure out the resolved next_type_id with size.
2778	* They will be stored in the current modifier's
2779	* resolved_ids and resolved_sizes such that it can
2780	* save us a few type-following when we use it later (e.g. in
2781	* pretty print).
2782	*/
2783	if (!btf_type_id_size(btf, &next_type_id, NULL)) {
2784	if (env_type_is_resolved(env, next_type_id))
2785	next_type = btf_type_id_resolve(btf, &next_type_id);
2786
2787	/* "typedef void new_void", "const void"...etc */
2788	if (!btf_type_is_void(next_type) &&
2789	!btf_type_is_fwd(next_type) &&
2790	!btf_type_is_func_proto(next_type)) {
2791	btf_verifier_log_type(env, v->t, "Invalid type_id");
2792	return -EINVAL;
2793	}
2794	}
2795
2796	env_stack_pop_resolved(env, next_type_id, 0);
2797
2798	return 0;
2799	}
2800
2801	static int btf_var_resolve(struct btf_verifier_env *env,
2802	const struct resolve_vertex *v)
2803	{
2804	const struct btf_type *next_type;
2805	const struct btf_type *t = v->t;
2806	u32 next_type_id = t->type;
2807	struct btf *btf = env->btf;
2808
2809	next_type = btf_type_by_id(btf, next_type_id);
2810	if (!next_type \|\| btf_type_is_resolve_source_only(next_type)) {
2811	btf_verifier_log_type(env, v->t, "Invalid type_id");
2812	return -EINVAL;
2813	}
2814
2815	if (!env_type_is_resolve_sink(env, next_type) &&
2816	!env_type_is_resolved(env, next_type_id))
2817	return env_stack_push(env, next_type, next_type_id);
2818
2819	if (btf_type_is_modifier(next_type)) {
2820	const struct btf_type *resolved_type;
2821	u32 resolved_type_id;
2822
2823	resolved_type_id = next_type_id;
2824	resolved_type = btf_type_id_resolve(btf, &resolved_type_id);
2825
2826	if (btf_type_is_ptr(resolved_type) &&
2827	!env_type_is_resolve_sink(env, resolved_type) &&
2828	!env_type_is_resolved(env, resolved_type_id))
2829	return env_stack_push(env, resolved_type,
2830	resolved_type_id);
2831	}
2832
2833	/* We must resolve to something concrete at this point, no
2834	* forward types or similar that would resolve to size of
2835	* zero is allowed.
2836	*/
2837	if (!btf_type_id_size(btf, &next_type_id, NULL)) {
2838	btf_verifier_log_type(env, v->t, "Invalid type_id");
2839	return -EINVAL;
2840	}
2841
2842	env_stack_pop_resolved(env, next_type_id, 0);
2843
2844	return 0;
2845	}
2846
2847	static int btf_ptr_resolve(struct btf_verifier_env *env,
2848	const struct resolve_vertex *v)
2849	{
2850	const struct btf_type *next_type;
2851	const struct btf_type *t = v->t;
2852	u32 next_type_id = t->type;
2853	struct btf *btf = env->btf;
2854
2855	next_type = btf_type_by_id(btf, next_type_id);
2856	if (!next_type \|\| btf_type_is_resolve_source_only(next_type)) {
2857	btf_verifier_log_type(env, v->t, "Invalid type_id");
2858	return -EINVAL;
2859	}
2860
2861	if (!env_type_is_resolve_sink(env, next_type) &&
2862	!env_type_is_resolved(env, next_type_id))
2863	return env_stack_push(env, next_type, next_type_id);
2864
2865	/* If the modifier was RESOLVED during RESOLVE_STRUCT_OR_ARRAY,
2866	* the modifier may have stopped resolving when it was resolved
2867	* to a ptr (last-resolved-ptr).
2868	*
2869	* We now need to continue from the last-resolved-ptr to
2870	* ensure the last-resolved-ptr will not referring back to
2871	* the current ptr (t).
2872	*/
2873	if (btf_type_is_modifier(next_type)) {
2874	const struct btf_type *resolved_type;
2875	u32 resolved_type_id;
2876
2877	resolved_type_id = next_type_id;
2878	resolved_type = btf_type_id_resolve(btf, &resolved_type_id);
2879
2880	if (btf_type_is_ptr(resolved_type) &&
2881	!env_type_is_resolve_sink(env, resolved_type) &&
2882	!env_type_is_resolved(env, resolved_type_id))
2883	return env_stack_push(env, resolved_type,
2884	resolved_type_id);
2885	}
2886
2887	if (!btf_type_id_size(btf, &next_type_id, NULL)) {
2888	if (env_type_is_resolved(env, next_type_id))
2889	next_type = btf_type_id_resolve(btf, &next_type_id);
2890
2891	if (!btf_type_is_void(next_type) &&
2892	!btf_type_is_fwd(next_type) &&
2893	!btf_type_is_func_proto(next_type)) {
2894	btf_verifier_log_type(env, v->t, "Invalid type_id");
2895	return -EINVAL;
2896	}
2897	}
2898
2899	env_stack_pop_resolved(env, next_type_id, 0);
2900
2901	return 0;
2902	}
2903
2904	static void btf_modifier_show(const struct btf *btf,
2905	const struct btf_type *t,
2906	u32 type_id, void *data,
2907	u8 bits_offset, struct btf_show *show)
2908	{
2909	if (btf->resolved_ids)
2910	t = btf_type_id_resolve(btf, &type_id);
2911	else
2912	t = btf_type_skip_modifiers(btf, type_id, NULL);
2913
2914	/*
2915	* A modifier can resolve to void, which has no show op; print a
2916	* placeholder rather than dereferencing NULL.
2917	*/
2918	if (!btf_type_ops(t))
2919	btf_df_show(btf, t, type_id, data, bits_offset, show);
2920	else
2921	btf_type_ops(t)->show(btf, t, type_id, data, bits_offset, show);
2922	}
2923
2924	static void btf_var_show(const struct btf btf, const struct btf_type t,
2925	u32 type_id, void *data, u8 bits_offset,
2926	struct btf_show *show)
2927	{
2928	/*
2929	* btf_type_id_resolve() dereferences btf->resolved_ids, which is NULL
2930	* for a base BTF (e.g. the vmlinux BTF that bpf_snprintf_btf() uses).
2931	* Resolve the var's type directly in that case.
2932	*/
2933	if (btf->resolved_ids)
2934	t = btf_type_id_resolve(btf, &type_id);
2935	else
2936	t = btf_type_skip_modifiers(btf, t->type, &type_id);
2937
2938	btf_type_ops(t)->show(btf, t, type_id, data, bits_offset, show);
2939	}
2940
2941	static void btf_ptr_show(const struct btf btf, const struct btf_type t,
2942	u32 type_id, void *data, u8 bits_offset,
2943	struct btf_show *show)
2944	{
2945	void *safe_data;
2946
2947	safe_data = btf_show_start_type(show, t, type_id, data);
2948	if (!safe_data)
2949	return;
2950
2951	/* It is a hashed value unless BTF_SHOW_PTR_RAW is specified */
2952	if (show->flags & BTF_SHOW_PTR_RAW)
2953	btf_show_type_value(show, "0x%px", (void *)safe_data);
2954	else
2955	btf_show_type_value(show, "0x%p", (void *)safe_data);
2956	btf_show_end_type(show);
2957	}
2958
2959	static void btf_ref_type_log(struct btf_verifier_env *env,
2960	const struct btf_type *t)
2961	{
2962	btf_verifier_log(env, "type_id=%u", t->type);
2963	}
2964
2965	static const struct btf_kind_operations modifier_ops = {
2966	.check_meta = btf_ref_type_check_meta,
2967	.resolve = btf_modifier_resolve,
2968	.check_member = btf_modifier_check_member,
2969	.check_kflag_member = btf_modifier_check_kflag_member,
2970	.log_details = btf_ref_type_log,
2971	.show = btf_modifier_show,
2972	};
2973
2974	static const struct btf_kind_operations ptr_ops = {
2975	.check_meta = btf_ref_type_check_meta,
2976	.resolve = btf_ptr_resolve,
2977	.check_member = btf_ptr_check_member,
2978	.check_kflag_member = btf_generic_check_kflag_member,
2979	.log_details = btf_ref_type_log,
2980	.show = btf_ptr_show,
2981	};
2982
2983	static s32 btf_fwd_check_meta(struct btf_verifier_env *env,
2984	const struct btf_type *t,
2985	u32 meta_left)
2986	{
2987	if (btf_type_vlen(t)) {
2988	btf_verifier_log_type(env, t, "vlen != 0");
2989	return -EINVAL;
2990	}
2991
2992	if (t->type) {
2993	btf_verifier_log_type(env, t, "type != 0");
2994	return -EINVAL;
2995	}
2996
2997	/* fwd type must have a valid name */
2998	if (!t->name_off \|\|
2999	!btf_name_valid_identifier(env->btf, t->name_off)) {
3000	btf_verifier_log_type(env, t, "Invalid name");
3001	return -EINVAL;
3002	}
3003
3004	btf_verifier_log_type(env, t, NULL);
3005
3006	return 0;
3007	}
3008
3009	static void btf_fwd_type_log(struct btf_verifier_env *env,
3010	const struct btf_type *t)
3011	{
3012	btf_verifier_log(env, "%s", btf_type_kflag(t) ? "union" : "struct");
3013	}
3014
3015	static const struct btf_kind_operations fwd_ops = {
3016	.check_meta = btf_fwd_check_meta,
3017	.resolve = btf_df_resolve,
3018	.check_member = btf_df_check_member,
3019	.check_kflag_member = btf_df_check_kflag_member,
3020	.log_details = btf_fwd_type_log,
3021	.show = btf_df_show,
3022	};
3023
3024	static int btf_array_check_member(struct btf_verifier_env *env,
3025	const struct btf_type *struct_type,
3026	const struct btf_member *member,
3027	const struct btf_type *member_type)
3028	{
3029	u32 struct_bits_off = member->offset;
3030	u32 struct_size, bytes_offset;
3031	u32 array_type_id, array_size;
3032	struct btf *btf = env->btf;
3033
3034	if (BITS_PER_BYTE_MASKED(struct_bits_off)) {
3035	btf_verifier_log_member(env, struct_type, member,
3036	"Member is not byte aligned");
3037	return -EINVAL;
3038	}
3039
3040	array_type_id = member->type;
3041	btf_type_id_size(btf, &array_type_id, &array_size);
3042	struct_size = struct_type->size;
3043	bytes_offset = BITS_ROUNDDOWN_BYTES(struct_bits_off);
3044	if (struct_size - bytes_offset < array_size) {
3045	btf_verifier_log_member(env, struct_type, member,
3046	"Member exceeds struct_size");
3047	return -EINVAL;
3048	}
3049
3050	return 0;
3051	}
3052
3053	static s32 btf_array_check_meta(struct btf_verifier_env *env,
3054	const struct btf_type *t,
3055	u32 meta_left)
3056	{
3057	const struct btf_array *array = btf_type_array(t);
3058	u32 meta_needed = sizeof(*array);
3059
3060	if (meta_left < meta_needed) {
3061	btf_verifier_log_basic(env, t,
3062	"meta_left:%u meta_needed:%u",
3063	meta_left, meta_needed);
3064	return -EINVAL;
3065	}
3066
3067	/* array type should not have a name */
3068	if (t->name_off) {
3069	btf_verifier_log_type(env, t, "Invalid name");
3070	return -EINVAL;
3071	}
3072
3073	if (btf_type_vlen(t)) {
3074	btf_verifier_log_type(env, t, "vlen != 0");
3075	return -EINVAL;
3076	}
3077
3078	if (btf_type_kflag(t)) {
3079	btf_verifier_log_type(env, t, "Invalid btf_info kind_flag");
3080	return -EINVAL;
3081	}
3082
3083	if (t->size) {
3084	btf_verifier_log_type(env, t, "size != 0");
3085	return -EINVAL;
3086	}
3087
3088	/* Array elem type and index type cannot be in type void,
3089	* so !array->type and !array->index_type are not allowed.
3090	*/
3091	if (!array->type \|\| !BTF_TYPE_ID_VALID(array->type)) {
3092	btf_verifier_log_type(env, t, "Invalid elem");
3093	return -EINVAL;
3094	}
3095
3096	if (!array->index_type \|\| !BTF_TYPE_ID_VALID(array->index_type)) {
3097	btf_verifier_log_type(env, t, "Invalid index");
3098	return -EINVAL;
3099	}
3100
3101	btf_verifier_log_type(env, t, NULL);
3102
3103	return meta_needed;
3104	}
3105
3106	static int btf_array_resolve(struct btf_verifier_env *env,
3107	const struct resolve_vertex *v)
3108	{
3109	const struct btf_array *array = btf_type_array(v->t);
3110	const struct btf_type elem_type, index_type;
3111	u32 elem_type_id, index_type_id;
3112	struct btf *btf = env->btf;
3113	u32 elem_size;
3114
3115	/* Check array->index_type */
3116	index_type_id = array->index_type;
3117	index_type = btf_type_by_id(btf, index_type_id);
3118	if (btf_type_nosize_or_null(index_type) \|\|
3119	btf_type_is_resolve_source_only(index_type)) {
3120	btf_verifier_log_type(env, v->t, "Invalid index");
3121	return -EINVAL;
3122	}
3123
3124	if (!env_type_is_resolve_sink(env, index_type) &&
3125	!env_type_is_resolved(env, index_type_id))
3126	return env_stack_push(env, index_type, index_type_id);
3127
3128	index_type = btf_type_id_size(btf, &index_type_id, NULL);
3129	if (!index_type \|\| !btf_type_is_int(index_type) \|\|
3130	!btf_type_int_is_regular(index_type)) {
3131	btf_verifier_log_type(env, v->t, "Invalid index");
3132	return -EINVAL;
3133	}
3134
3135	/* Check array->type */
3136	elem_type_id = array->type;
3137	elem_type = btf_type_by_id(btf, elem_type_id);
3138	if (btf_type_nosize_or_null(elem_type) \|\|
3139	btf_type_is_resolve_source_only(elem_type)) {
3140	btf_verifier_log_type(env, v->t,
3141	"Invalid elem");
3142	return -EINVAL;
3143	}
3144
3145	if (!env_type_is_resolve_sink(env, elem_type) &&
3146	!env_type_is_resolved(env, elem_type_id))
3147	return env_stack_push(env, elem_type, elem_type_id);
3148
3149	elem_type = btf_type_id_size(btf, &elem_type_id, &elem_size);
3150	if (!elem_type) {
3151	btf_verifier_log_type(env, v->t, "Invalid elem");
3152	return -EINVAL;
3153	}
3154
3155	if (btf_type_is_int(elem_type) && !btf_type_int_is_regular(elem_type)) {
3156	btf_verifier_log_type(env, v->t, "Invalid array of int");
3157	return -EINVAL;
3158	}
3159
3160	if (array->nelems && elem_size > U32_MAX / array->nelems) {
3161	btf_verifier_log_type(env, v->t,
3162	"Array size overflows U32_MAX");
3163	return -EINVAL;
3164	}
3165
3166	env_stack_pop_resolved(env, elem_type_id, elem_size * array->nelems);
3167
3168	return 0;
3169	}
3170
3171	static void btf_array_log(struct btf_verifier_env *env,
3172	const struct btf_type *t)
3173	{
3174	const struct btf_array *array = btf_type_array(t);
3175
3176	btf_verifier_log(env, "type_id=%u index_type_id=%u nr_elems=%u",
3177	array->type, array->index_type, array->nelems);
3178	}
3179
3180	static void __btf_array_show(const struct btf btf, const struct btf_type t,
3181	u32 type_id, void *data, u8 bits_offset,
3182	struct btf_show *show)
3183	{
3184	const struct btf_array *array = btf_type_array(t);
3185	const struct btf_kind_operations *elem_ops;
3186	const struct btf_type *elem_type;
3187	u32 i, elem_size = 0, elem_type_id;
3188	u16 encoding = 0;
3189
3190	elem_type_id = array->type;
3191	elem_type = btf_type_skip_modifiers(btf, elem_type_id, NULL);
3192	if (elem_type && btf_type_has_size(elem_type))
3193	elem_size = elem_type->size;
3194
3195	if (elem_type && btf_type_is_int(elem_type)) {
3196	u32 int_type = btf_type_int(elem_type);
3197
3198	encoding = BTF_INT_ENCODING(int_type);
3199
3200	/*
3201	* BTF_INT_CHAR encoding never seems to be set for
3202	* char arrays, so if size is 1 and element is
3203	* printable as a char, we'll do that.
3204	*/
3205	if (elem_size == 1)
3206	encoding = BTF_INT_CHAR;
3207	}
3208
3209	if (!btf_show_start_array_type(show, t, type_id, encoding, data))
3210	return;
3211
3212	if (!elem_type)
3213	goto out;
3214	elem_ops = btf_type_ops(elem_type);
3215
3216	for (i = 0; i < array->nelems; i++) {
3217
3218	btf_show_start_array_member(show);
3219
3220	elem_ops->show(btf, elem_type, elem_type_id, data,
3221	bits_offset, show);
3222	data += elem_size;
3223
3224	btf_show_end_array_member(show);
3225
3226	if (show->state.array_terminated)
3227	break;
3228	}
3229	out:
3230	btf_show_end_array_type(show);
3231	}
3232
3233	static void btf_array_show(const struct btf btf, const struct btf_type t,
3234	u32 type_id, void *data, u8 bits_offset,
3235	struct btf_show *show)
3236	{
3237	const struct btf_member *m = show->state.member;
3238
3239	/*
3240	* First check if any members would be shown (are non-zero).
3241	* See comments above "struct btf_show" definition for more
3242	* details on how this works at a high-level.
3243	*/
3244	if (show->state.depth > 0 && !(show->flags & BTF_SHOW_ZERO)) {
3245	if (!show->state.depth_check) {
3246	show->state.depth_check = show->state.depth + 1;
3247	show->state.depth_to_show = 0;
3248	}
3249	__btf_array_show(btf, t, type_id, data, bits_offset, show);
3250	show->state.member = m;
3251
3252	if (show->state.depth_check != show->state.depth + 1)
3253	return;
3254	show->state.depth_check = 0;
3255
3256	if (show->state.depth_to_show <= show->state.depth)
3257	return;
3258	/*
3259	* Reaching here indicates we have recursed and found
3260	* non-zero array member(s).
3261	*/
3262	}
3263	__btf_array_show(btf, t, type_id, data, bits_offset, show);
3264	}
3265
3266	static const struct btf_kind_operations array_ops = {
3267	.check_meta = btf_array_check_meta,
3268	.resolve = btf_array_resolve,
3269	.check_member = btf_array_check_member,
3270	.check_kflag_member = btf_generic_check_kflag_member,
3271	.log_details = btf_array_log,
3272	.show = btf_array_show,
3273	};
3274
3275	static int btf_struct_check_member(struct btf_verifier_env *env,
3276	const struct btf_type *struct_type,
3277	const struct btf_member *member,
3278	const struct btf_type *member_type)
3279	{
3280	u32 struct_bits_off = member->offset;
3281	u32 struct_size, bytes_offset;
3282
3283	if (BITS_PER_BYTE_MASKED(struct_bits_off)) {
3284	btf_verifier_log_member(env, struct_type, member,
3285	"Member is not byte aligned");
3286	return -EINVAL;
3287	}
3288
3289	struct_size = struct_type->size;
3290	bytes_offset = BITS_ROUNDDOWN_BYTES(struct_bits_off);
3291	if (struct_size - bytes_offset < member_type->size) {
3292	btf_verifier_log_member(env, struct_type, member,
3293	"Member exceeds struct_size");
3294	return -EINVAL;
3295	}
3296
3297	return 0;
3298	}
3299
3300	static s32 btf_struct_check_meta(struct btf_verifier_env *env,
3301	const struct btf_type *t,
3302	u32 meta_left)
3303	{
3304	bool is_union = BTF_INFO_KIND(t->info) == BTF_KIND_UNION;
3305	const struct btf_member *member;
3306	u32 meta_needed, last_offset;
3307	struct btf *btf = env->btf;
3308	u32 struct_size = t->size;
3309	u32 offset;
3310	u32 i;
3311
3312	meta_needed = btf_type_vlen(t) * sizeof(*member);
3313	if (meta_left < meta_needed) {
3314	btf_verifier_log_basic(env, t,
3315	"meta_left:%u meta_needed:%u",
3316	meta_left, meta_needed);
3317	return -EINVAL;
3318	}
3319
3320	/* struct type either no name or a valid one */
3321	if (t->name_off &&
3322	!btf_name_valid_identifier(env->btf, t->name_off)) {
3323	btf_verifier_log_type(env, t, "Invalid name");
3324	return -EINVAL;
3325	}
3326
3327	btf_verifier_log_type(env, t, NULL);
3328
3329	last_offset = 0;
3330	for_each_member(i, t, member) {
3331	if (!btf_name_offset_valid(btf, member->name_off)) {
3332	btf_verifier_log_member(env, t, member,
3333	"Invalid member name_offset:%u",
3334	member->name_off);
3335	return -EINVAL;
3336	}
3337
3338	/* struct member either no name or a valid one */
3339	if (member->name_off &&
3340	!btf_name_valid_identifier(btf, member->name_off)) {
3341	btf_verifier_log_member(env, t, member, "Invalid name");
3342	return -EINVAL;
3343	}
3344	/* A member cannot be in type void */
3345	if (!member->type \|\| !BTF_TYPE_ID_VALID(member->type)) {
3346	btf_verifier_log_member(env, t, member,
3347	"Invalid type_id");
3348	return -EINVAL;
3349	}
3350
3351	offset = __btf_member_bit_offset(t, member);
3352	if (is_union && offset) {
3353	btf_verifier_log_member(env, t, member,
3354	"Invalid member bits_offset");
3355	return -EINVAL;
3356	}
3357
3358	/*
3359	* ">" instead of ">=" because the last member could be
3360	* "char a[0];"
3361	*/
3362	if (last_offset > offset) {
3363	btf_verifier_log_member(env, t, member,
3364	"Invalid member bits_offset");
3365	return -EINVAL;
3366	}
3367
3368	if (BITS_ROUNDUP_BYTES(offset) > struct_size) {
3369	btf_verifier_log_member(env, t, member,
3370	"Member bits_offset exceeds its struct size");
3371	return -EINVAL;
3372	}
3373
3374	btf_verifier_log_member(env, t, member, NULL);
3375	last_offset = offset;
3376	}
3377
3378	return meta_needed;
3379	}
3380
3381	static int btf_struct_resolve(struct btf_verifier_env *env,
3382	const struct resolve_vertex *v)
3383	{
3384	const struct btf_member *member;
3385	int err;
3386	u32 i;
3387
3388	/* Before continue resolving the next_member,
3389	* ensure the last member is indeed resolved to a
3390	* type with size info.
3391	*/
3392	if (v->next_member) {
3393	const struct btf_type *last_member_type;
3394	const struct btf_member *last_member;
3395	u32 last_member_type_id;
3396
3397	last_member = btf_type_member(v->t) + v->next_member - 1;
3398	last_member_type_id = last_member->type;
3399	if (WARN_ON_ONCE(!env_type_is_resolved(env,
3400	last_member_type_id)))
3401	return -EINVAL;
3402
3403	last_member_type = btf_type_by_id(env->btf,
3404	last_member_type_id);
3405	if (btf_type_kflag(v->t))
3406	err = btf_type_ops(last_member_type)->check_kflag_member(env, v->t,
3407	last_member,
3408	last_member_type);
3409	else
3410	err = btf_type_ops(last_member_type)->check_member(env, v->t,
3411	last_member,
3412	last_member_type);
3413	if (err)
3414	return err;
3415	}
3416
3417	for_each_member_from(i, v->next_member, v->t, member) {
3418	u32 member_type_id = member->type;
3419	const struct btf_type *member_type = btf_type_by_id(env->btf,
3420	member_type_id);
3421
3422	if (btf_type_nosize_or_null(member_type) \|\|
3423	btf_type_is_resolve_source_only(member_type)) {
3424	btf_verifier_log_member(env, v->t, member,
3425	"Invalid member");
3426	return -EINVAL;
3427	}
3428
3429	if (!env_type_is_resolve_sink(env, member_type) &&
3430	!env_type_is_resolved(env, member_type_id)) {
3431	env_stack_set_next_member(env, i + 1);
3432	return env_stack_push(env, member_type, member_type_id);
3433	}
3434
3435	if (btf_type_kflag(v->t))
3436	err = btf_type_ops(member_type)->check_kflag_member(env, v->t,
3437	member,
3438	member_type);
3439	else
3440	err = btf_type_ops(member_type)->check_member(env, v->t,
3441	member,
3442	member_type);
3443	if (err)
3444	return err;
3445	}
3446
3447	env_stack_pop_resolved(env, 0, 0);
3448
3449	return 0;
3450	}
3451
3452	static void btf_struct_log(struct btf_verifier_env *env,
3453	const struct btf_type *t)
3454	{
3455	btf_verifier_log(env, "size=%u vlen=%u", t->size, btf_type_vlen(t));
3456	}
3457
3458	enum {
3459	BTF_FIELD_IGNORE = 0,
3460	BTF_FIELD_FOUND = 1,
3461	};
3462
3463	struct btf_field_info {
3464	enum btf_field_type type;
3465	u32 off;
3466	union {
3467	struct {
3468	u32 type_id;
3469	} kptr;
3470	struct {
3471	const char *node_name;
3472	u32 value_btf_id;
3473	} graph_root;
3474	};
3475	};
3476
3477	static int btf_find_struct(const struct btf btf, const struct btf_type t,
3478	u32 off, int sz, enum btf_field_type field_type,
3479	struct btf_field_info *info)
3480	{
3481	if (!__btf_type_is_struct(t))
3482	return BTF_FIELD_IGNORE;
3483	if (t->size != sz)
3484	return BTF_FIELD_IGNORE;
3485	info->type = field_type;
3486	info->off = off;
3487	return BTF_FIELD_FOUND;
3488	}
3489
3490	struct btf_type_tag_match {
3491	const char *name;
3492	u32 flag;
3493	};
3494
3495	struct btf_type_tag_walk_ctx {
3496	const struct btf_type t; / Input/Output */
3497	u32 id; /* Output */
3498	u32 res; /* Output */
3499	};
3500
3501	static int btf_type_tag_walk(const struct btf *btf,
3502	struct btf_type_tag_walk_ctx *ctx,
3503	const struct btf_type_tag_match *matches,
3504	u32 match_cnt)
3505	{
3506	const struct btf_type *t = ctx->t;
3507	u32 res = 0;
3508	const char *tag;
3509	u32 id, i;
3510
3511	do {
3512	id = t->type;
3513	t = btf_type_by_id(btf, id);
3514
3515	if (!btf_type_is_modifier(t))
3516	break;
3517
3518	if (!btf_type_is_type_tag(t) \|\| btf_type_kflag(t))
3519	continue;
3520
3521	tag = __btf_name_by_offset(btf, t->name_off);
3522	for (i = 0; i < match_cnt; i++) {
3523	if (strcmp(tag, matches[i].name))
3524	continue;
3525	res \|= matches[i].flag;
3526	break;
3527	}
3528	} while (true);
3529
3530	/* We only support a single tag. */
3531	if (hweight32(res) > 1)
3532	return -EINVAL;
3533
3534	ctx->t = t;
3535	ctx->id = id;
3536	ctx->res = res;
3537
3538	return 0;
3539	}
3540
3541	static int btf_find_kptr(const struct btf btf, const struct btf_type t,
3542	u32 off, int sz, struct btf_field_info *info, u32 field_mask)
3543	{
3544	static const struct btf_type_tag_match kptr_type_tags[] = {
3545	{ "kptr_untrusted", BPF_KPTR_UNREF },
3546	{ "kptr", BPF_KPTR_REF },
3547	{ "percpu_kptr", BPF_KPTR_PERCPU },
3548	{ "uptr", BPF_UPTR },
3549	};
3550	struct btf_type_tag_walk_ctx ctx;
3551	enum btf_field_type type = 0;
3552	int err;
3553	u32 res_id;
3554
3555	/* Permit modifiers on the pointer itself */
3556	if (btf_type_is_volatile(t))
3557	t = btf_type_by_id(btf, t->type);
3558	/* For PTR, sz is always == 8 */
3559	if (!btf_type_is_ptr(t))
3560	return BTF_FIELD_IGNORE;
3561
3562	ctx.t = t;
3563	err = btf_type_tag_walk(btf, &ctx, kptr_type_tags,
3564	ARRAY_SIZE(kptr_type_tags));
3565	if (err)
3566	return err;
3567
3568	t = ctx.t;
3569	res_id = ctx.id;
3570	type = ctx.res;
3571
3572	if (!(type & field_mask))
3573	return BTF_FIELD_IGNORE;
3574
3575	/* Only pointer to struct is allowed */
3576	if (!__btf_type_is_struct(t))
3577	return -EINVAL;
3578
3579	info->type = type;
3580	info->off = off;
3581	info->kptr.type_id = res_id;
3582	return BTF_FIELD_FOUND;
3583	}
3584
3585	int btf_find_next_decl_tag(const struct btf btf, const struct btf_type pt,
3586	int comp_idx, const char *tag_key, int last_id)
3587	{
3588	int len = strlen(tag_key);
3589	int i, n;
3590
3591	for (i = last_id + 1, n = btf_nr_types(btf); i < n; i++) {
3592	const struct btf_type *t = btf_type_by_id(btf, i);
3593
3594	if (!btf_type_is_decl_tag(t))
3595	continue;
3596	if (pt != btf_type_by_id(btf, t->type))
3597	continue;
3598	if (btf_type_decl_tag(t)->component_idx != comp_idx)
3599	continue;
3600	if (strncmp(__btf_name_by_offset(btf, t->name_off), tag_key, len))
3601	continue;
3602	return i;
3603	}
3604	return -ENOENT;
3605	}
3606
3607	const char btf_find_decl_tag_value(const struct btf btf, const struct btf_type *pt,
3608	int comp_idx, const char *tag_key)
3609	{
3610	const char *value = NULL;
3611	const struct btf_type *t;
3612	int len, id;
3613
3614	id = btf_find_next_decl_tag(btf, pt, comp_idx, tag_key,
3615	btf_named_start_id(btf, false) - 1);
3616	if (id < 0)
3617	return ERR_PTR(id);
3618
3619	t = btf_type_by_id(btf, id);
3620	len = strlen(tag_key);
3621	value = __btf_name_by_offset(btf, t->name_off) + len;
3622
3623	/* Prevent duplicate entries for same type */
3624	id = btf_find_next_decl_tag(btf, pt, comp_idx, tag_key, id);
3625	if (id >= 0)
3626	return ERR_PTR(-EEXIST);
3627
3628	return value;
3629	}
3630
3631	static int
3632	btf_find_graph_root(const struct btf btf, const struct btf_type pt,
3633	const struct btf_type *t, int comp_idx, u32 off,
3634	int sz, struct btf_field_info *info,
3635	enum btf_field_type head_type)
3636	{
3637	const char *node_field_name;
3638	const char *value_type;
3639	s32 id;
3640
3641	if (!__btf_type_is_struct(t))
3642	return BTF_FIELD_IGNORE;
3643	if (t->size != sz)
3644	return BTF_FIELD_IGNORE;
3645	value_type = btf_find_decl_tag_value(btf, pt, comp_idx, "contains:");
3646	if (IS_ERR(value_type))
3647	return -EINVAL;
3648	node_field_name = strstr(value_type, ":");
3649	if (!node_field_name)
3650	return -EINVAL;
3651	value_type = kstrndup(value_type, node_field_name - value_type,
3652	GFP_KERNEL_ACCOUNT \| __GFP_NOWARN);
3653	if (!value_type)
3654	return -ENOMEM;
3655	id = btf_find_by_name_kind(btf, value_type, BTF_KIND_STRUCT);
3656	kfree(value_type);
3657	if (id < 0)
3658	return id;
3659	node_field_name++;
3660	if (str_is_empty(node_field_name))
3661	return -EINVAL;
3662	info->type = head_type;
3663	info->off = off;
3664	info->graph_root.value_btf_id = id;
3665	info->graph_root.node_name = node_field_name;
3666	return BTF_FIELD_FOUND;
3667	}
3668
3669	static int btf_get_field_type(const struct btf btf, const struct btf_type var_type,
3670	u32 field_mask, u32 seen_mask, int align, int *sz)
3671	{
3672	const struct {
3673	enum btf_field_type type;
3674	const char *const name;
3675	const bool is_unique;
3676	} field_types[] = {
3677	{ BPF_SPIN_LOCK, "bpf_spin_lock", true },
3678	{ BPF_RES_SPIN_LOCK, "bpf_res_spin_lock", true },
3679	{ BPF_TIMER, "bpf_timer", true },
3680	{ BPF_WORKQUEUE, "bpf_wq", true },
3681	{ BPF_TASK_WORK, "bpf_task_work", true },
3682	{ BPF_LIST_HEAD, "bpf_list_head", false },
3683	{ BPF_LIST_NODE, "bpf_list_node", false },
3684	{ BPF_RB_ROOT, "bpf_rb_root", false },
3685	{ BPF_RB_NODE, "bpf_rb_node", false },
3686	{ BPF_REFCOUNT, "bpf_refcount", true },
3687	};
3688	int type = 0, i;
3689	const char *name = __btf_name_by_offset(btf, var_type->name_off);
3690	const char *field_type_name;
3691	enum btf_field_type field_type;
3692	bool is_unique;
3693
3694	for (i = 0; i < ARRAY_SIZE(field_types); ++i) {
3695	field_type = field_types[i].type;
3696	field_type_name = field_types[i].name;
3697	is_unique = field_types[i].is_unique;
3698	if (!(field_mask & field_type) \|\| strcmp(name, field_type_name))
3699	continue;
3700	if (is_unique) {
3701	if (*seen_mask & field_type)
3702	return -E2BIG;
3703	*seen_mask \|= field_type;
3704	}
3705	type = field_type;
3706	goto end;
3707	}
3708
3709	/* Only return BPF_KPTR when all other types with matchable names fail */
3710	if (field_mask & (BPF_KPTR \| BPF_UPTR) && !__btf_type_is_struct(var_type)) {
3711	type = BPF_KPTR_REF;
3712	goto end;
3713	}
3714	return 0;
3715	end:
3716	*sz = btf_field_type_size(type);
3717	*align = btf_field_type_align(type);
3718	return type;
3719	}
3720
3721	/* Repeat a number of fields for a specified number of times.
3722	*
3723	* Copy the fields starting from the first field and repeat them for
3724	* repeat_cnt times. The fields are repeated by adding the offset of each
3725	* field with
3726	* (i + 1) * elem_size
3727	* where i is the repeat index and elem_size is the size of an element.
3728	*/
3729	static int btf_repeat_fields(struct btf_field_info *info, int info_cnt,
3730	u32 field_cnt, u32 repeat_cnt, u32 elem_size)
3731	{
3732	u32 i, j, total_cnt, total_repeats;
3733	u32 cur;
3734
3735	/* Ensure not repeating fields that should not be repeated. */
3736	for (i = 0; i < field_cnt; i++) {
3737	switch (info[i].type) {
3738	case BPF_KPTR_UNREF:
3739	case BPF_KPTR_REF:
3740	case BPF_KPTR_PERCPU:
3741	case BPF_UPTR:
3742	case BPF_LIST_HEAD:
3743	case BPF_RB_ROOT:
3744	break;
3745	default:
3746	return -EINVAL;
3747	}
3748	}
3749
3750	if (check_add_overflow(repeat_cnt, 1, &total_repeats) \|\|
3751	check_mul_overflow(field_cnt, total_repeats, &total_cnt) \|\|
3752	total_cnt > (u32)info_cnt)
3753	return -E2BIG;
3754
3755	cur = field_cnt;
3756	for (i = 0; i < repeat_cnt; i++) {
3757	memcpy(&info[cur], &info[0], field_cnt * sizeof(info[0]));
3758	for (j = 0; j < field_cnt; j++)
3759	info[cur++].off += (i + 1) * elem_size;
3760	}
3761
3762	return 0;
3763	}
3764
3765	static int btf_find_struct_field(const struct btf *btf,
3766	const struct btf_type *t, u32 field_mask,
3767	struct btf_field_info *info, int info_cnt,
3768	u32 level, u32 *seen_mask);
3769
3770	/* Find special fields in the struct type of a field.
3771	*
3772	* This function is used to find fields of special types that is not a
3773	* global variable or a direct field of a struct type. It also handles the
3774	* repetition if it is the element type of an array.
3775	*/
3776	static int btf_find_nested_struct(const struct btf btf, const struct btf_type t,
3777	u32 off, u32 nelems,
3778	u32 field_mask, struct btf_field_info *info,
3779	int info_cnt, u32 level, u32 *seen_mask)
3780	{
3781	int ret, err, i;
3782
3783	level++;
3784	if (level >= MAX_RESOLVE_DEPTH)
3785	return -E2BIG;
3786
3787	ret = btf_find_struct_field(btf, t, field_mask, info, info_cnt, level, seen_mask);
3788
3789	if (ret <= 0)
3790	return ret;
3791
3792	/* Shift the offsets of the nested struct fields to the offsets
3793	* related to the container.
3794	*/
3795	for (i = 0; i < ret; i++)
3796	info[i].off += off;
3797
3798	if (nelems > 1) {
3799	err = btf_repeat_fields(info, info_cnt, ret, nelems - 1, t->size);
3800	if (err == 0)
3801	ret *= nelems;
3802	else
3803	ret = err;
3804	}
3805
3806	return ret;
3807	}
3808
3809	static int btf_find_field_one(const struct btf *btf,
3810	const struct btf_type *var,
3811	const struct btf_type *var_type,
3812	int var_idx,
3813	u32 off, u32 expected_size,
3814	u32 field_mask, u32 *seen_mask,
3815	struct btf_field_info *info, int info_cnt,
3816	u32 level)
3817	{
3818	int ret, align, sz, field_type;
3819	struct btf_field_info tmp;
3820	const struct btf_array *array;
3821	u32 i, nelems = 1;
3822
3823	/* Walk into array types to find the element type and the number of
3824	* elements in the (flattened) array.
3825	*/
3826	for (i = 0; i < MAX_RESOLVE_DEPTH && btf_type_is_array(var_type); i++) {
3827	array = btf_array(var_type);
3828	nelems *= array->nelems;
3829	var_type = btf_type_by_id(btf, array->type);
3830	}
3831	if (i == MAX_RESOLVE_DEPTH)
3832	return -E2BIG;
3833	if (nelems == 0)
3834	return 0;
3835
3836	field_type = btf_get_field_type(btf, var_type,
3837	field_mask, seen_mask, &align, &sz);
3838	/* Look into variables of struct types */
3839	if (!field_type && __btf_type_is_struct(var_type)) {
3840	sz = var_type->size;
3841	if (expected_size && expected_size != sz * nelems)
3842	return 0;
3843	ret = btf_find_nested_struct(btf, var_type, off, nelems, field_mask,
3844	&info[0], info_cnt, level, seen_mask);
3845	return ret;
3846	}
3847
3848	if (field_type == 0)
3849	return 0;
3850	if (field_type < 0)
3851	return field_type;
3852
3853	if (expected_size && expected_size != sz * nelems)
3854	return 0;
3855	if (off % align)
3856	return 0;
3857
3858	switch (field_type) {
3859	case BPF_SPIN_LOCK:
3860	case BPF_RES_SPIN_LOCK:
3861	case BPF_TIMER:
3862	case BPF_WORKQUEUE:
3863	case BPF_LIST_NODE:
3864	case BPF_RB_NODE:
3865	case BPF_REFCOUNT:
3866	case BPF_TASK_WORK:
3867	ret = btf_find_struct(btf, var_type, off, sz, field_type,
3868	info_cnt ? &info[0] : &tmp);
3869	if (ret < 0)
3870	return ret;
3871	break;
3872	case BPF_KPTR_UNREF:
3873	case BPF_KPTR_REF:
3874	case BPF_KPTR_PERCPU:
3875	case BPF_UPTR:
3876	ret = btf_find_kptr(btf, var_type, off, sz,
3877	info_cnt ? &info[0] : &tmp, field_mask);
3878	if (ret < 0)
3879	return ret;
3880	break;
3881	case BPF_LIST_HEAD:
3882	case BPF_RB_ROOT:
3883	ret = btf_find_graph_root(btf, var, var_type,
3884	var_idx, off, sz,
3885	info_cnt ? &info[0] : &tmp,
3886	field_type);
3887	if (ret < 0)
3888	return ret;
3889	break;
3890	default:
3891	return -EFAULT;
3892	}
3893
3894	if (ret == BTF_FIELD_IGNORE)
3895	return 0;
3896	if (!info_cnt)
3897	return -E2BIG;
3898	if (nelems > 1) {
3899	ret = btf_repeat_fields(info, info_cnt, 1, nelems - 1, sz);
3900	if (ret < 0)
3901	return ret;
3902	}
3903	return nelems;
3904	}
3905
3906	static int btf_find_struct_field(const struct btf *btf,
3907	const struct btf_type *t, u32 field_mask,
3908	struct btf_field_info *info, int info_cnt,
3909	u32 level, u32 *seen_mask)
3910	{
3911	int ret, idx = 0;
3912	const struct btf_member *member;
3913	u32 i, off;
3914
3915	for_each_member(i, t, member) {
3916	const struct btf_type *member_type = btf_type_by_id(btf,
3917	member->type);
3918
3919	off = __btf_member_bit_offset(t, member);
3920	if (off % 8)
3921	/* valid C code cannot generate such BTF */
3922	return -EINVAL;
3923	off /= 8;
3924
3925	ret = btf_find_field_one(btf, t, member_type, i,
3926	off, 0,
3927	field_mask, seen_mask,
3928	&info[idx], info_cnt - idx, level);
3929	if (ret < 0)
3930	return ret;
3931	idx += ret;
3932	}
3933	return idx;
3934	}
3935
3936	static int btf_find_datasec_var(const struct btf btf, const struct btf_type t,
3937	u32 field_mask, struct btf_field_info *info,
3938	int info_cnt, u32 level, u32 *seen_mask)
3939	{
3940	int ret, idx = 0;
3941	const struct btf_var_secinfo *vsi;
3942	u32 i, off;
3943
3944	for_each_vsi(i, t, vsi) {
3945	const struct btf_type *var = btf_type_by_id(btf, vsi->type);
3946	const struct btf_type *var_type = btf_type_by_id(btf, var->type);
3947
3948	off = vsi->offset;
3949	ret = btf_find_field_one(btf, var, var_type, -1, off, vsi->size,
3950	field_mask, seen_mask,
3951	&info[idx], info_cnt - idx,
3952	level);
3953	if (ret < 0)
3954	return ret;
3955	idx += ret;
3956	}
3957	return idx;
3958	}
3959
3960	static int btf_find_field(const struct btf btf, const struct btf_type t,
3961	u32 field_mask, struct btf_field_info *info,
3962	int info_cnt)
3963	{
3964	u32 seen_mask = 0;
3965
3966	if (__btf_type_is_struct(t))
3967	return btf_find_struct_field(btf, t, field_mask, info, info_cnt, 0, &seen_mask);
3968	else if (btf_type_is_datasec(t))
3969	return btf_find_datasec_var(btf, t, field_mask, info, info_cnt, 0, &seen_mask);
3970	return -EINVAL;
3971	}
3972
3973	/* Callers have to ensure the life cycle of btf if it is program BTF */
3974	static int btf_parse_kptr(const struct btf btf, struct btf_field field,
3975	struct btf_field_info *info)
3976	{
3977	struct module *mod = NULL;
3978	const struct btf_type *t;
3979	/* If a matching btf type is found in kernel or module BTFs, kptr_ref
3980	* is that BTF, otherwise it's program BTF
3981	*/
3982	struct btf *kptr_btf;
3983	int ret;
3984	s32 id;
3985
3986	/* Find type in map BTF, and use it to look up the matching type
3987	* in vmlinux or module BTFs, by name and kind.
3988	*/
3989	t = btf_type_by_id(btf, info->kptr.type_id);
3990	id = bpf_find_btf_id(__btf_name_by_offset(btf, t->name_off), BTF_INFO_KIND(t->info),
3991	&kptr_btf);
3992	if (id == -ENOENT) {
3993	/* btf_parse_kptr should only be called w/ btf = program BTF */
3994	WARN_ON_ONCE(btf_is_kernel(btf));
3995
3996	/* Type exists only in program BTF. Assume that it's a MEM_ALLOC
3997	* kptr allocated via bpf_obj_new
3998	*/
3999	field->kptr.dtor = NULL;
4000	id = info->kptr.type_id;
4001	kptr_btf = (struct btf *)btf;
4002	goto found_dtor;
4003	}
4004	if (id < 0)
4005	return id;
4006
4007	/* Find and stash the function pointer for the destruction function that
4008	* needs to be eventually invoked from the map free path.
4009	*/
4010	if (info->type == BPF_KPTR_REF) {
4011	const struct btf_type *dtor_func;
4012	const char *dtor_func_name;
4013	unsigned long addr;
4014	s32 dtor_btf_id;
4015
4016	/* This call also serves as a whitelist of allowed objects that
4017	* can be used as a referenced pointer and be stored in a map at
4018	* the same time.
4019	*/
4020	dtor_btf_id = btf_find_dtor_kfunc(kptr_btf, id);
4021	if (dtor_btf_id < 0) {
4022	ret = dtor_btf_id;
4023	goto end_btf;
4024	}
4025
4026	dtor_func = btf_type_by_id(kptr_btf, dtor_btf_id);
4027	if (!dtor_func) {
4028	ret = -ENOENT;
4029	goto end_btf;
4030	}
4031
4032	if (btf_is_module(kptr_btf)) {
4033	mod = btf_try_get_module(kptr_btf);
4034	if (!mod) {
4035	ret = -ENXIO;
4036	goto end_btf;
4037	}
4038	}
4039
4040	/* We already verified dtor_func to be btf_type_is_func
4041	* in register_btf_id_dtor_kfuncs.
4042	*/
4043	dtor_func_name = __btf_name_by_offset(kptr_btf, dtor_func->name_off);
4044	addr = kallsyms_lookup_name(dtor_func_name);
4045	if (!addr) {
4046	ret = -EINVAL;
4047	goto end_mod;
4048	}
4049	field->kptr.dtor = (void *)addr;
4050	}
4051
4052	found_dtor:
4053	field->kptr.btf_id = id;
4054	field->kptr.btf = kptr_btf;
4055	field->kptr.module = mod;
4056	return 0;
4057	end_mod:
4058	module_put(mod);
4059	end_btf:
4060	btf_put(kptr_btf);
4061	return ret;
4062	}
4063
4064	static int btf_parse_graph_root(const struct btf *btf,
4065	struct btf_field *field,
4066	struct btf_field_info *info,
4067	const char *node_type_name,
4068	size_t node_type_align)
4069	{
4070	const struct btf_type t, n = NULL;
4071	const struct btf_member *member;
4072	u32 offset;
4073	int i;
4074
4075	t = btf_type_by_id(btf, info->graph_root.value_btf_id);
4076	/* We've already checked that value_btf_id is a struct type. We
4077	* just need to figure out the offset of the list_node, and
4078	* verify its type.
4079	*/
4080	for_each_member(i, t, member) {
4081	if (strcmp(info->graph_root.node_name,
4082	__btf_name_by_offset(btf, member->name_off)))
4083	continue;
4084	/* Invalid BTF, two members with same name */
4085	if (n)
4086	return -EINVAL;
4087	n = btf_type_by_id(btf, member->type);
4088	if (!__btf_type_is_struct(n))
4089	return -EINVAL;
4090	if (strcmp(node_type_name, __btf_name_by_offset(btf, n->name_off)))
4091	return -EINVAL;
4092	offset = __btf_member_bit_offset(n, member);
4093	if (offset % 8)
4094	return -EINVAL;
4095	offset /= 8;
4096	if (offset % node_type_align)
4097	return -EINVAL;
4098
4099	field->graph_root.btf = (struct btf *)btf;
4100	field->graph_root.value_btf_id = info->graph_root.value_btf_id;
4101	field->graph_root.node_offset = offset;
4102	}
4103	if (!n)
4104	return -ENOENT;
4105	return 0;
4106	}
4107
4108	static int btf_parse_list_head(const struct btf btf, struct btf_field field,
4109	struct btf_field_info *info)
4110	{
4111	return btf_parse_graph_root(btf, field, info, "bpf_list_node",
4112	__alignof__(struct bpf_list_node));
4113	}
4114
4115	static int btf_parse_rb_root(const struct btf btf, struct btf_field field,
4116	struct btf_field_info *info)
4117	{
4118	return btf_parse_graph_root(btf, field, info, "bpf_rb_node",
4119	__alignof__(struct bpf_rb_node));
4120	}
4121
4122	static int btf_field_cmp(const void _a, const void _b, const void *priv)
4123	{
4124	const struct btf_field a = (const struct btf_field )_a;
4125	const struct btf_field b = (const struct btf_field )_b;
4126
4127	if (a->offset < b->offset)
4128	return -1;
4129	else if (a->offset > b->offset)
4130	return 1;
4131	return 0;
4132	}
4133
4134	struct btf_record btf_parse_fields(const struct btf btf, const struct btf_type *t,
4135	u32 field_mask, u32 value_size)
4136	{
4137	struct btf_field_info info_arr[BTF_FIELDS_MAX];
4138	u32 next_off = 0, field_type_size;
4139	struct btf_record *rec;
4140	int ret, i, cnt;
4141
4142	ret = btf_find_field(btf, t, field_mask, info_arr, ARRAY_SIZE(info_arr));
4143	if (ret < 0)
4144	return ERR_PTR(ret);
4145	if (!ret)
4146	return NULL;
4147
4148	cnt = ret;
4149	/* This needs to be kzalloc to zero out padding and unused fields, see
4150	* comment in btf_record_equal.
4151	*/
4152	rec = kzalloc_flex(*rec, fields, cnt, GFP_KERNEL_ACCOUNT \| __GFP_NOWARN);
4153	if (!rec)
4154	return ERR_PTR(-ENOMEM);
4155
4156	rec->spin_lock_off = -EINVAL;
4157	rec->res_spin_lock_off = -EINVAL;
4158	rec->timer_off = -EINVAL;
4159	rec->wq_off = -EINVAL;
4160	rec->refcount_off = -EINVAL;
4161	rec->task_work_off = -EINVAL;
4162	for (i = 0; i < cnt; i++) {
4163	field_type_size = btf_field_type_size(info_arr[i].type);
4164	if (info_arr[i].off + field_type_size > value_size) {
4165	WARN_ONCE(1, "verifier bug off %d size %d", info_arr[i].off, value_size);
4166	ret = -EFAULT;
4167	goto end;
4168	}
4169	if (info_arr[i].off < next_off) {
4170	ret = -EEXIST;
4171	goto end;
4172	}
4173	next_off = info_arr[i].off + field_type_size;
4174
4175	rec->field_mask \|= info_arr[i].type;
4176	rec->fields[i].offset = info_arr[i].off;
4177	rec->fields[i].type = info_arr[i].type;
4178	rec->fields[i].size = field_type_size;
4179
4180	switch (info_arr[i].type) {
4181	case BPF_SPIN_LOCK:
4182	WARN_ON_ONCE(rec->spin_lock_off >= 0);
4183	/* Cache offset for faster lookup at runtime */
4184	rec->spin_lock_off = rec->fields[i].offset;
4185	break;
4186	case BPF_RES_SPIN_LOCK:
4187	WARN_ON_ONCE(rec->res_spin_lock_off >= 0);
4188	/* Cache offset for faster lookup at runtime */
4189	rec->res_spin_lock_off = rec->fields[i].offset;
4190	break;
4191	case BPF_TIMER:
4192	WARN_ON_ONCE(rec->timer_off >= 0);
4193	/* Cache offset for faster lookup at runtime */
4194	rec->timer_off = rec->fields[i].offset;
4195	break;
4196	case BPF_WORKQUEUE:
4197	WARN_ON_ONCE(rec->wq_off >= 0);
4198	/* Cache offset for faster lookup at runtime */
4199	rec->wq_off = rec->fields[i].offset;
4200	break;
4201	case BPF_TASK_WORK:
4202	WARN_ON_ONCE(rec->task_work_off >= 0);
4203	rec->task_work_off = rec->fields[i].offset;
4204	break;
4205	case BPF_REFCOUNT:
4206	WARN_ON_ONCE(rec->refcount_off >= 0);
4207	/* Cache offset for faster lookup at runtime */
4208	rec->refcount_off = rec->fields[i].offset;
4209	break;
4210	case BPF_KPTR_UNREF:
4211	case BPF_KPTR_REF:
4212	case BPF_KPTR_PERCPU:
4213	case BPF_UPTR:
4214	ret = btf_parse_kptr(btf, &rec->fields[i], &info_arr[i]);
4215	if (ret < 0)
4216	goto end;
4217	break;
4218	case BPF_LIST_HEAD:
4219	ret = btf_parse_list_head(btf, &rec->fields[i], &info_arr[i]);
4220	if (ret < 0)
4221	goto end;
4222	break;
4223	case BPF_RB_ROOT:
4224	ret = btf_parse_rb_root(btf, &rec->fields[i], &info_arr[i]);
4225	if (ret < 0)
4226	goto end;
4227	break;
4228	case BPF_LIST_NODE:
4229	case BPF_RB_NODE:
4230	break;
4231	default:
4232	ret = -EFAULT;
4233	goto end;
4234	}
4235	rec->cnt++;
4236	}
4237
4238	if (rec->spin_lock_off >= 0 && rec->res_spin_lock_off >= 0) {
4239	ret = -EINVAL;
4240	goto end;
4241	}
4242
4243	/* bpf_{list_head, rb_node} require bpf_spin_lock */
4244	if ((btf_record_has_field(rec, BPF_LIST_HEAD) \|\|
4245	btf_record_has_field(rec, BPF_RB_ROOT)) &&
4246	(rec->spin_lock_off < 0 && rec->res_spin_lock_off < 0)) {
4247	ret = -EINVAL;
4248	goto end;
4249	}
4250
4251	if (rec->refcount_off < 0 &&
4252	btf_record_has_field(rec, BPF_LIST_NODE) &&
4253	btf_record_has_field(rec, BPF_RB_NODE)) {
4254	ret = -EINVAL;
4255	goto end;
4256	}
4257
4258	sort_r(rec->fields, rec->cnt, sizeof(struct btf_field), btf_field_cmp,
4259	NULL, rec);
4260
4261	return rec;
4262	end:
4263	btf_record_free(rec);
4264	return ERR_PTR(ret);
4265	}
4266
4267	int btf_check_and_fixup_fields(const struct btf btf, struct btf_record rec)
4268	{
4269	int i;
4270
4271	/*
4272	* Check fields which require the complete BTF and initialize runtime
4273	* metadata. Ownership relationships are validated after every record has
4274	* been fixed up.
4275	*/
4276	if (IS_ERR_OR_NULL(rec) \|\| !(rec->field_mask & (BPF_GRAPH_ROOT \| BPF_UPTR)))
4277	return 0;
4278	for (i = 0; i < rec->cnt; i++) {
4279	struct btf_struct_meta *meta;
4280	const struct btf_type *t;
4281	u32 btf_id;
4282
4283	if (rec->fields[i].type == BPF_UPTR) {
4284	/* The uptr only supports pinning one page and cannot
4285	* point to a kernel struct
4286	*/
4287	if (btf_is_kernel(rec->fields[i].kptr.btf))
4288	return -EINVAL;
4289	t = btf_type_by_id(rec->fields[i].kptr.btf,
4290	rec->fields[i].kptr.btf_id);
4291	if (!t->size)
4292	return -EINVAL;
4293	if (t->size > PAGE_SIZE)
4294	return -E2BIG;
4295	continue;
4296	}
4297
4298	if (!(rec->fields[i].type & BPF_GRAPH_ROOT))
4299	continue;
4300	btf_id = rec->fields[i].graph_root.value_btf_id;
4301	meta = btf_find_struct_meta(btf, btf_id);
4302	if (!meta)
4303	return -EFAULT;
4304	rec->fields[i].graph_root.value_rec = meta->record;
4305	}
4306	return 0;
4307	}
4308
4309	static int btf_owned_type_idx(const struct btf btf, struct btf_struct_metas tab,
4310	const struct btf_field *field)
4311	{
4312	struct btf_struct_meta *meta;
4313	u32 btf_id;
4314
4315	if (field->type & BPF_GRAPH_ROOT) {
4316	btf_id = field->graph_root.value_btf_id;
4317	} else if (field->type == BPF_KPTR_REF \|\| field->type == BPF_KPTR_PERCPU) {
4318	if (btf_is_kernel(field->kptr.btf))
4319	return -ENOENT;
4320	btf_id = field->kptr.btf_id;
4321	} else {
4322	return -ENOENT;
4323	}
4324
4325	meta = btf_find_struct_meta(btf, btf_id);
4326	if (!meta)
4327	return field->type & BPF_GRAPH_ROOT ? -EFAULT : -ENOENT;
4328	return meta - tab->types;
4329	}
4330
4331	/*
4332	* Each ownership edge adds kernel frames through bpf_obj_free_fields() and
4333	* __bpf_obj_drop_impl(). Keep the bound deliberately small because object
4334	* destruction can itself run below a BPF call chain. A final pointee without
4335	* special fields is not present in the struct metadata table and adds only a
4336	* non-recursing drop.
4337	*/
4338	#define BTF_MAX_OWNERSHIP_DEPTH 8
4339
4340	static int btf_ownership_depth(const struct btf *btf,
4341	struct btf_struct_metas tab, u8 depth,
4342	int idx, int depth_left)
4343	{
4344	const struct btf_record *rec = tab->types[idx].record;
4345	int i, ret, max_depth = 0;
4346
4347	if (!depth_left)
4348	return -ELOOP;
4349	if (depth[idx])
4350	goto done;
4351
4352	for (i = 0; i < rec->cnt; i++) {
4353	ret = btf_owned_type_idx(btf, tab, &rec->fields[i]);
4354	if (ret == -ENOENT)
4355	continue;
4356	if (ret < 0)
4357	return ret;
4358	ret = btf_ownership_depth(btf, tab, depth, ret, depth_left - 1);
4359	if (ret < 0)
4360	return ret;
4361	max_depth = max(max_depth, ret);
4362	}
4363	depth[idx] = max_depth + 1;
4364	done:
4365	return depth[idx] > depth_left ? -ELOOP : depth[idx];
4366	}
4367
4368	static int btf_check_ownership_depth(const struct btf *btf,
4369	struct btf_struct_metas *tab)
4370	{
4371	u8 *depth;
4372	int i, ret = 0;
4373
4374	depth = kvcalloc(tab->cnt, sizeof(*depth), GFP_KERNEL \| __GFP_NOWARN);
4375	if (!depth)
4376	return -ENOMEM;
4377
4378	for (i = 0; i < tab->cnt; i++) {
4379	ret = btf_ownership_depth(btf, tab, depth, i,
4380	BTF_MAX_OWNERSHIP_DEPTH);
4381	if (ret < 0)
4382	break;
4383	ret = 0;
4384	}
4385	kvfree(depth);
4386	return ret;
4387	}
4388
4389	static void __btf_struct_show(const struct btf btf, const struct btf_type t,
4390	u32 type_id, void *data, u8 bits_offset,
4391	struct btf_show *show)
4392	{
4393	const struct btf_member *member;
4394	void *safe_data;
4395	u32 i;
4396
4397	safe_data = btf_show_start_struct_type(show, t, type_id, data);
4398	if (!safe_data)
4399	return;
4400
4401	for_each_member(i, t, member) {
4402	const struct btf_type *member_type = btf_type_by_id(btf,
4403	member->type);
4404	const struct btf_kind_operations *ops;
4405	u32 member_offset, bitfield_size;
4406	u32 bytes_offset;
4407	u8 bits8_offset;
4408
4409	btf_show_start_member(show, member);
4410
4411	member_offset = __btf_member_bit_offset(t, member);
4412	bitfield_size = __btf_member_bitfield_size(t, member);
4413	bytes_offset = BITS_ROUNDDOWN_BYTES(member_offset);
4414	bits8_offset = BITS_PER_BYTE_MASKED(member_offset);
4415	if (bitfield_size) {
4416	safe_data = btf_show_start_type(show, member_type,
4417	member->type,
4418	data + bytes_offset);
4419	if (safe_data)
4420	btf_bitfield_show(safe_data,
4421	bits8_offset,
4422	bitfield_size, show);
4423	btf_show_end_type(show);
4424	} else {
4425	ops = btf_type_ops(member_type);
4426	ops->show(btf, member_type, member->type,
4427	data + bytes_offset, bits8_offset, show);
4428	}
4429
4430	btf_show_end_member(show);
4431	}
4432
4433	btf_show_end_struct_type(show);
4434	}
4435
4436	static void btf_struct_show(const struct btf btf, const struct btf_type t,
4437	u32 type_id, void *data, u8 bits_offset,
4438	struct btf_show *show)
4439	{
4440	const struct btf_member *m = show->state.member;
4441
4442	/*
4443	* First check if any members would be shown (are non-zero).
4444	* See comments above "struct btf_show" definition for more
4445	* details on how this works at a high-level.
4446	*/
4447	if (show->state.depth > 0 && !(show->flags & BTF_SHOW_ZERO)) {
4448	if (!show->state.depth_check) {
4449	show->state.depth_check = show->state.depth + 1;
4450	show->state.depth_to_show = 0;
4451	}
4452	__btf_struct_show(btf, t, type_id, data, bits_offset, show);
4453	/* Restore saved member data here */
4454	show->state.member = m;
4455	if (show->state.depth_check != show->state.depth + 1)
4456	return;
4457	show->state.depth_check = 0;
4458
4459	if (show->state.depth_to_show <= show->state.depth)
4460	return;
4461	/*
4462	* Reaching here indicates we have recursed and found
4463	* non-zero child values.
4464	*/
4465	}
4466
4467	__btf_struct_show(btf, t, type_id, data, bits_offset, show);
4468	}
4469
4470	static const struct btf_kind_operations struct_ops = {
4471	.check_meta = btf_struct_check_meta,
4472	.resolve = btf_struct_resolve,
4473	.check_member = btf_struct_check_member,
4474	.check_kflag_member = btf_generic_check_kflag_member,
4475	.log_details = btf_struct_log,
4476	.show = btf_struct_show,
4477	};
4478
4479	static int btf_enum_check_member(struct btf_verifier_env *env,
4480	const struct btf_type *struct_type,
4481	const struct btf_member *member,
4482	const struct btf_type *member_type)
4483	{
4484	u32 struct_bits_off = member->offset;
4485	u32 struct_size, bytes_offset;
4486
4487	if (BITS_PER_BYTE_MASKED(struct_bits_off)) {
4488	btf_verifier_log_member(env, struct_type, member,
4489	"Member is not byte aligned");
4490	return -EINVAL;
4491	}
4492
4493	struct_size = struct_type->size;
4494	bytes_offset = BITS_ROUNDDOWN_BYTES(struct_bits_off);
4495	if (struct_size - bytes_offset < member_type->size) {
4496	btf_verifier_log_member(env, struct_type, member,
4497	"Member exceeds struct_size");
4498	return -EINVAL;
4499	}
4500
4501	return 0;
4502	}
4503
4504	static int btf_enum_check_kflag_member(struct btf_verifier_env *env,
4505	const struct btf_type *struct_type,
4506	const struct btf_member *member,
4507	const struct btf_type *member_type)
4508	{
4509	u32 struct_bits_off, nr_bits, bytes_end, struct_size;
4510	u32 int_bitsize = sizeof(int) * BITS_PER_BYTE;
4511
4512	struct_bits_off = BTF_MEMBER_BIT_OFFSET(member->offset);
4513	nr_bits = BTF_MEMBER_BITFIELD_SIZE(member->offset);
4514	if (!nr_bits) {
4515	if (BITS_PER_BYTE_MASKED(struct_bits_off)) {
4516	btf_verifier_log_member(env, struct_type, member,
4517	"Member is not byte aligned");
4518	return -EINVAL;
4519	}
4520
4521	nr_bits = int_bitsize;
4522	} else if (nr_bits > int_bitsize) {
4523	btf_verifier_log_member(env, struct_type, member,
4524	"Invalid member bitfield_size");
4525	return -EINVAL;
4526	}
4527
4528	struct_size = struct_type->size;
4529	bytes_end = BITS_ROUNDUP_BYTES(struct_bits_off + nr_bits);
4530	if (struct_size < bytes_end) {
4531	btf_verifier_log_member(env, struct_type, member,
4532	"Member exceeds struct_size");
4533	return -EINVAL;
4534	}
4535
4536	return 0;
4537	}
4538
4539	static s32 btf_enum_check_meta(struct btf_verifier_env *env,
4540	const struct btf_type *t,
4541	u32 meta_left)
4542	{
4543	const struct btf_enum *enums = btf_type_enum(t);
4544	struct btf *btf = env->btf;
4545	const char *fmt_str;
4546	u32 i, nr_enums;
4547	u32 meta_needed;
4548
4549	nr_enums = btf_type_vlen(t);
4550	meta_needed = nr_enums * sizeof(*enums);
4551
4552	if (meta_left < meta_needed) {
4553	btf_verifier_log_basic(env, t,
4554	"meta_left:%u meta_needed:%u",
4555	meta_left, meta_needed);
4556	return -EINVAL;
4557	}
4558
4559	if (t->size > 8 \|\| !is_power_of_2(t->size)) {
4560	btf_verifier_log_type(env, t, "Unexpected size");
4561	return -EINVAL;
4562	}
4563
4564	/* enum type either no name or a valid one */
4565	if (t->name_off &&
4566	!btf_name_valid_identifier(env->btf, t->name_off)) {
4567	btf_verifier_log_type(env, t, "Invalid name");
4568	return -EINVAL;
4569	}
4570
4571	btf_verifier_log_type(env, t, NULL);
4572
4573	for (i = 0; i < nr_enums; i++) {
4574	if (!btf_name_offset_valid(btf, enums[i].name_off)) {
4575	btf_verifier_log(env, "\tInvalid name_offset:%u",
4576	enums[i].name_off);
4577	return -EINVAL;
4578	}
4579
4580	/* enum member must have a valid name */
4581	if (!enums[i].name_off \|\|
4582	!btf_name_valid_identifier(btf, enums[i].name_off)) {
4583	btf_verifier_log_type(env, t, "Invalid name");
4584	return -EINVAL;
4585	}
4586
4587	if (env->log.level == BPF_LOG_KERNEL)
4588	continue;
4589	fmt_str = btf_type_kflag(t) ? "\t%s val=%d\n" : "\t%s val=%u\n";
4590	btf_verifier_log(env, fmt_str,
4591	__btf_name_by_offset(btf, enums[i].name_off),
4592	enums[i].val);
4593	}
4594
4595	return meta_needed;
4596	}
4597
4598	static void btf_enum_log(struct btf_verifier_env *env,
4599	const struct btf_type *t)
4600	{
4601	btf_verifier_log(env, "size=%u vlen=%u", t->size, btf_type_vlen(t));
4602	}
4603
4604	static void btf_enum_show(const struct btf btf, const struct btf_type t,
4605	u32 type_id, void *data, u8 bits_offset,
4606	struct btf_show *show)
4607	{
4608	const struct btf_enum *enums = btf_type_enum(t);
4609	u32 i, nr_enums = btf_type_vlen(t);
4610	void *safe_data;
4611	int v;
4612
4613	safe_data = btf_show_start_type(show, t, type_id, data);
4614	if (!safe_data)
4615	return;
4616
4617	v = (int )safe_data;
4618
4619	for (i = 0; i < nr_enums; i++) {
4620	if (v != enums[i].val)
4621	continue;
4622
4623	btf_show_type_value(show, "%s",
4624	__btf_name_by_offset(btf,
4625	enums[i].name_off));
4626
4627	btf_show_end_type(show);
4628	return;
4629	}
4630
4631	if (btf_type_kflag(t))
4632	btf_show_type_value(show, "%d", v);
4633	else
4634	btf_show_type_value(show, "%u", v);
4635	btf_show_end_type(show);
4636	}
4637
4638	static const struct btf_kind_operations enum_ops = {
4639	.check_meta = btf_enum_check_meta,
4640	.resolve = btf_df_resolve,
4641	.check_member = btf_enum_check_member,
4642	.check_kflag_member = btf_enum_check_kflag_member,
4643	.log_details = btf_enum_log,
4644	.show = btf_enum_show,
4645	};
4646
4647	static s32 btf_enum64_check_meta(struct btf_verifier_env *env,
4648	const struct btf_type *t,
4649	u32 meta_left)
4650	{
4651	const struct btf_enum64 *enums = btf_type_enum64(t);
4652	struct btf *btf = env->btf;
4653	const char *fmt_str;
4654	u32 i, nr_enums;
4655	u32 meta_needed;
4656
4657	nr_enums = btf_type_vlen(t);
4658	meta_needed = nr_enums * sizeof(*enums);
4659
4660	if (meta_left < meta_needed) {
4661	btf_verifier_log_basic(env, t,
4662	"meta_left:%u meta_needed:%u",
4663	meta_left, meta_needed);
4664	return -EINVAL;
4665	}
4666
4667	if (t->size > 8 \|\| !is_power_of_2(t->size)) {
4668	btf_verifier_log_type(env, t, "Unexpected size");
4669	return -EINVAL;
4670	}
4671
4672	/* enum type either no name or a valid one */
4673	if (t->name_off &&
4674	!btf_name_valid_identifier(env->btf, t->name_off)) {
4675	btf_verifier_log_type(env, t, "Invalid name");
4676	return -EINVAL;
4677	}
4678
4679	btf_verifier_log_type(env, t, NULL);
4680
4681	for (i = 0; i < nr_enums; i++) {
4682	if (!btf_name_offset_valid(btf, enums[i].name_off)) {
4683	btf_verifier_log(env, "\tInvalid name_offset:%u",
4684	enums[i].name_off);
4685	return -EINVAL;
4686	}
4687
4688	/* enum member must have a valid name */
4689	if (!enums[i].name_off \|\|
4690	!btf_name_valid_identifier(btf, enums[i].name_off)) {
4691	btf_verifier_log_type(env, t, "Invalid name");
4692	return -EINVAL;
4693	}
4694
4695	if (env->log.level == BPF_LOG_KERNEL)
4696	continue;
4697
4698	fmt_str = btf_type_kflag(t) ? "\t%s val=%lld\n" : "\t%s val=%llu\n";
4699	btf_verifier_log(env, fmt_str,
4700	__btf_name_by_offset(btf, enums[i].name_off),
4701	btf_enum64_value(enums + i));
4702	}
4703
4704	return meta_needed;
4705	}
4706
4707	static void btf_enum64_show(const struct btf btf, const struct btf_type t,
4708	u32 type_id, void *data, u8 bits_offset,
4709	struct btf_show *show)
4710	{
4711	const struct btf_enum64 *enums = btf_type_enum64(t);
4712	u32 i, nr_enums = btf_type_vlen(t);
4713	void *safe_data;
4714	s64 v;
4715
4716	safe_data = btf_show_start_type(show, t, type_id, data);
4717	if (!safe_data)
4718	return;
4719
4720	v = (u64 )safe_data;
4721
4722	for (i = 0; i < nr_enums; i++) {
4723	if (v != btf_enum64_value(enums + i))
4724	continue;
4725
4726	btf_show_type_value(show, "%s",
4727	__btf_name_by_offset(btf,
4728	enums[i].name_off));
4729
4730	btf_show_end_type(show);
4731	return;
4732	}
4733
4734	if (btf_type_kflag(t))
4735	btf_show_type_value(show, "%lld", v);
4736	else
4737	btf_show_type_value(show, "%llu", v);
4738	btf_show_end_type(show);
4739	}
4740
4741	static const struct btf_kind_operations enum64_ops = {
4742	.check_meta = btf_enum64_check_meta,
4743	.resolve = btf_df_resolve,
4744	.check_member = btf_enum_check_member,
4745	.check_kflag_member = btf_enum_check_kflag_member,
4746	.log_details = btf_enum_log,
4747	.show = btf_enum64_show,
4748	};
4749
4750	static s32 btf_func_proto_check_meta(struct btf_verifier_env *env,
4751	const struct btf_type *t,
4752	u32 meta_left)
4753	{
4754	u32 meta_needed = btf_type_vlen(t) * sizeof(struct btf_param);
4755
4756	if (meta_left < meta_needed) {
4757	btf_verifier_log_basic(env, t,
4758	"meta_left:%u meta_needed:%u",
4759	meta_left, meta_needed);
4760	return -EINVAL;
4761	}
4762
4763	if (t->name_off) {
4764	btf_verifier_log_type(env, t, "Invalid name");
4765	return -EINVAL;
4766	}
4767
4768	if (btf_type_kflag(t)) {
4769	btf_verifier_log_type(env, t, "Invalid btf_info kind_flag");
4770	return -EINVAL;
4771	}
4772
4773	btf_verifier_log_type(env, t, NULL);
4774
4775	return meta_needed;
4776	}
4777
4778	static void btf_func_proto_log(struct btf_verifier_env *env,
4779	const struct btf_type *t)
4780	{
4781	const struct btf_param args = (const struct btf_param )(t + 1);
4782	u32 nr_args = btf_type_vlen(t), i;
4783
4784	btf_verifier_log(env, "return=%u args=(", t->type);
4785	if (!nr_args) {
4786	btf_verifier_log(env, "void");
4787	goto done;
4788	}
4789
4790	if (nr_args == 1 && !args[0].type) {
4791	/* Only one vararg */
4792	btf_verifier_log(env, "vararg");
4793	goto done;
4794	}
4795
4796	btf_verifier_log(env, "%u %s", args[0].type,
4797	__btf_name_by_offset(env->btf,
4798	args[0].name_off));
4799	for (i = 1; i < nr_args - 1; i++)
4800	btf_verifier_log(env, ", %u %s", args[i].type,
4801	__btf_name_by_offset(env->btf,
4802	args[i].name_off));
4803
4804	if (nr_args > 1) {
4805	const struct btf_param *last_arg = &args[nr_args - 1];
4806
4807	if (last_arg->type)
4808	btf_verifier_log(env, ", %u %s", last_arg->type,
4809	__btf_name_by_offset(env->btf,
4810	last_arg->name_off));
4811	else
4812	btf_verifier_log(env, ", vararg");
4813	}
4814
4815	done:
4816	btf_verifier_log(env, ")");
4817	}
4818
4819	static const struct btf_kind_operations func_proto_ops = {
4820	.check_meta = btf_func_proto_check_meta,
4821	.resolve = btf_df_resolve,
4822	/*
4823	* BTF_KIND_FUNC_PROTO cannot be directly referred by
4824	* a struct's member.
4825	*
4826	* It should be a function pointer instead.
4827	* (i.e. struct's member -> BTF_KIND_PTR -> BTF_KIND_FUNC_PROTO)
4828	*
4829	* Hence, there is no btf_func_check_member().
4830	*/
4831	.check_member = btf_df_check_member,
4832	.check_kflag_member = btf_df_check_kflag_member,
4833	.log_details = btf_func_proto_log,
4834	.show = btf_df_show,
4835	};
4836
4837	static s32 btf_func_check_meta(struct btf_verifier_env *env,
4838	const struct btf_type *t,
4839	u32 meta_left)
4840	{
4841	if (!t->name_off \|\|
4842	!btf_name_valid_identifier(env->btf, t->name_off)) {
4843	btf_verifier_log_type(env, t, "Invalid name");
4844	return -EINVAL;
4845	}
4846
4847	if (btf_type_vlen(t) > BTF_FUNC_GLOBAL) {
4848	btf_verifier_log_type(env, t, "Invalid func linkage");
4849	return -EINVAL;
4850	}
4851
4852	if (btf_type_kflag(t)) {
4853	btf_verifier_log_type(env, t, "Invalid btf_info kind_flag");
4854	return -EINVAL;
4855	}
4856
4857	btf_verifier_log_type(env, t, NULL);
4858
4859	return 0;
4860	}
4861
4862	static int btf_func_resolve(struct btf_verifier_env *env,
4863	const struct resolve_vertex *v)
4864	{
4865	const struct btf_type *t = v->t;
4866	u32 next_type_id = t->type;
4867	int err;
4868
4869	err = btf_func_check(env, t);
4870	if (err)
4871	return err;
4872
4873	env_stack_pop_resolved(env, next_type_id, 0);
4874	return 0;
4875	}
4876
4877	static const struct btf_kind_operations func_ops = {
4878	.check_meta = btf_func_check_meta,
4879	.resolve = btf_func_resolve,
4880	.check_member = btf_df_check_member,
4881	.check_kflag_member = btf_df_check_kflag_member,
4882	.log_details = btf_ref_type_log,
4883	.show = btf_df_show,
4884	};
4885
4886	static s32 btf_var_check_meta(struct btf_verifier_env *env,
4887	const struct btf_type *t,
4888	u32 meta_left)
4889	{
4890	const struct btf_var *var;
4891	u32 meta_needed = sizeof(*var);
4892
4893	if (meta_left < meta_needed) {
4894	btf_verifier_log_basic(env, t,
4895	"meta_left:%u meta_needed:%u",
4896	meta_left, meta_needed);
4897	return -EINVAL;
4898	}
4899
4900	if (btf_type_vlen(t)) {
4901	btf_verifier_log_type(env, t, "vlen != 0");
4902	return -EINVAL;
4903	}
4904
4905	if (btf_type_kflag(t)) {
4906	btf_verifier_log_type(env, t, "Invalid btf_info kind_flag");
4907	return -EINVAL;
4908	}
4909
4910	if (!t->name_off \|\|
4911	!btf_name_valid_identifier(env->btf, t->name_off)) {
4912	btf_verifier_log_type(env, t, "Invalid name");
4913	return -EINVAL;
4914	}
4915
4916	/* A var cannot be in type void */
4917	if (!t->type \|\| !BTF_TYPE_ID_VALID(t->type)) {
4918	btf_verifier_log_type(env, t, "Invalid type_id");
4919	return -EINVAL;
4920	}
4921
4922	var = btf_type_var(t);
4923	if (var->linkage != BTF_VAR_STATIC &&
4924	var->linkage != BTF_VAR_GLOBAL_ALLOCATED) {
4925	btf_verifier_log_type(env, t, "Linkage not supported");
4926	return -EINVAL;
4927	}
4928
4929	btf_verifier_log_type(env, t, NULL);
4930
4931	return meta_needed;
4932	}
4933
4934	static void btf_var_log(struct btf_verifier_env env, const struct btf_type t)
4935	{
4936	const struct btf_var *var = btf_type_var(t);
4937
4938	btf_verifier_log(env, "type_id=%u linkage=%u", t->type, var->linkage);
4939	}
4940
4941	static const struct btf_kind_operations var_ops = {
4942	.check_meta = btf_var_check_meta,
4943	.resolve = btf_var_resolve,
4944	.check_member = btf_df_check_member,
4945	.check_kflag_member = btf_df_check_kflag_member,
4946	.log_details = btf_var_log,
4947	.show = btf_var_show,
4948	};
4949
4950	static s32 btf_datasec_check_meta(struct btf_verifier_env *env,
4951	const struct btf_type *t,
4952	u32 meta_left)
4953	{
4954	const struct btf_var_secinfo *vsi;
4955	u64 last_vsi_end_off = 0, sum = 0;
4956	u32 i, meta_needed;
4957
4958	meta_needed = btf_type_vlen(t) * sizeof(*vsi);
4959	if (meta_left < meta_needed) {
4960	btf_verifier_log_basic(env, t,
4961	"meta_left:%u meta_needed:%u",
4962	meta_left, meta_needed);
4963	return -EINVAL;
4964	}
4965
4966	if (!t->size) {
4967	btf_verifier_log_type(env, t, "size == 0");
4968	return -EINVAL;
4969	}
4970
4971	if (btf_type_kflag(t)) {
4972	btf_verifier_log_type(env, t, "Invalid btf_info kind_flag");
4973	return -EINVAL;
4974	}
4975
4976	if (!t->name_off \|\|
4977	!btf_name_valid_section(env->btf, t->name_off)) {
4978	btf_verifier_log_type(env, t, "Invalid name");
4979	return -EINVAL;
4980	}
4981
4982	btf_verifier_log_type(env, t, NULL);
4983
4984	for_each_vsi(i, t, vsi) {
4985	/* A var cannot be in type void */
4986	if (!vsi->type \|\| !BTF_TYPE_ID_VALID(vsi->type)) {
4987	btf_verifier_log_vsi(env, t, vsi,
4988	"Invalid type_id");
4989	return -EINVAL;
4990	}
4991
4992	if (vsi->offset < last_vsi_end_off \|\| vsi->offset >= t->size) {
4993	btf_verifier_log_vsi(env, t, vsi,
4994	"Invalid offset");
4995	return -EINVAL;
4996	}
4997
4998	if (!vsi->size \|\| vsi->size > t->size) {
4999	btf_verifier_log_vsi(env, t, vsi,
5000	"Invalid size");
5001	return -EINVAL;
5002	}
5003
5004	last_vsi_end_off = vsi->offset + vsi->size;
5005	if (last_vsi_end_off > t->size) {
5006	btf_verifier_log_vsi(env, t, vsi,
5007	"Invalid offset+size");
5008	return -EINVAL;
5009	}
5010
5011	btf_verifier_log_vsi(env, t, vsi, NULL);
5012	sum += vsi->size;
5013	}
5014
5015	if (t->size < sum) {
5016	btf_verifier_log_type(env, t, "Invalid btf_info size");
5017	return -EINVAL;
5018	}
5019
5020	return meta_needed;
5021	}
5022
5023	static int btf_datasec_resolve(struct btf_verifier_env *env,
5024	const struct resolve_vertex *v)
5025	{
5026	const struct btf_var_secinfo *vsi;
5027	struct btf *btf = env->btf;
5028	u32 i;
5029
5030	env->resolve_mode = RESOLVE_TBD;
5031	for_each_vsi_from(i, v->next_member, v->t, vsi) {
5032	u32 var_type_id = vsi->type, type_id, type_size = 0;
5033	const struct btf_type *var_type = btf_type_by_id(env->btf,
5034	var_type_id);
5035	if (!var_type \|\| !btf_type_is_var(var_type)) {
5036	btf_verifier_log_vsi(env, v->t, vsi,
5037	"Not a VAR kind member");
5038	return -EINVAL;
5039	}
5040
5041	if (!env_type_is_resolve_sink(env, var_type) &&
5042	!env_type_is_resolved(env, var_type_id)) {
5043	env_stack_set_next_member(env, i + 1);
5044	return env_stack_push(env, var_type, var_type_id);
5045	}
5046
5047	type_id = var_type->type;
5048	if (!btf_type_id_size(btf, &type_id, &type_size)) {
5049	btf_verifier_log_vsi(env, v->t, vsi, "Invalid type");
5050	return -EINVAL;
5051	}
5052
5053	if (vsi->size < type_size) {
5054	btf_verifier_log_vsi(env, v->t, vsi, "Invalid size");
5055	return -EINVAL;
5056	}
5057	}
5058
5059	env_stack_pop_resolved(env, 0, 0);
5060	return 0;
5061	}
5062
5063	static void btf_datasec_log(struct btf_verifier_env *env,
5064	const struct btf_type *t)
5065	{
5066	btf_verifier_log(env, "size=%u vlen=%u", t->size, btf_type_vlen(t));
5067	}
5068
5069	static void btf_datasec_show(const struct btf *btf,
5070	const struct btf_type *t, u32 type_id,
5071	void *data, u8 bits_offset,
5072	struct btf_show *show)
5073	{
5074	const struct btf_var_secinfo *vsi;
5075	const struct btf_type *var;
5076	u32 i;
5077
5078	if (!btf_show_start_type(show, t, type_id, data))
5079	return;
5080
5081	btf_show_type_value(show, "section (\"%s\") = {",
5082	__btf_name_by_offset(btf, t->name_off));
5083	for_each_vsi(i, t, vsi) {
5084	var = btf_type_by_id(btf, vsi->type);
5085	if (i)
5086	btf_show(show, ",");
5087	btf_type_ops(var)->show(btf, var, vsi->type,
5088	data + vsi->offset, bits_offset, show);
5089	}
5090	btf_show_end_type(show);
5091	}
5092
5093	static const struct btf_kind_operations datasec_ops = {
5094	.check_meta = btf_datasec_check_meta,
5095	.resolve = btf_datasec_resolve,
5096	.check_member = btf_df_check_member,
5097	.check_kflag_member = btf_df_check_kflag_member,
5098	.log_details = btf_datasec_log,
5099	.show = btf_datasec_show,
5100	};
5101
5102	static s32 btf_float_check_meta(struct btf_verifier_env *env,
5103	const struct btf_type *t,
5104	u32 meta_left)
5105	{
5106	if (btf_type_vlen(t)) {
5107	btf_verifier_log_type(env, t, "vlen != 0");
5108	return -EINVAL;
5109	}
5110
5111	if (btf_type_kflag(t)) {
5112	btf_verifier_log_type(env, t, "Invalid btf_info kind_flag");
5113	return -EINVAL;
5114	}
5115
5116	if (t->size != 2 && t->size != 4 && t->size != 8 && t->size != 12 &&
5117	t->size != 16) {
5118	btf_verifier_log_type(env, t, "Invalid type_size");
5119	return -EINVAL;
5120	}
5121
5122	btf_verifier_log_type(env, t, NULL);
5123
5124	return 0;
5125	}
5126
5127	static int btf_float_check_member(struct btf_verifier_env *env,
5128	const struct btf_type *struct_type,
5129	const struct btf_member *member,
5130	const struct btf_type *member_type)
5131	{
5132	u64 start_offset_bytes;
5133	u64 end_offset_bytes;
5134	u64 misalign_bits;
5135	u64 align_bytes;
5136	u64 align_bits;
5137
5138	/* Different architectures have different alignment requirements, so
5139	* here we check only for the reasonable minimum. This way we ensure
5140	* that types after CO-RE can pass the kernel BTF verifier.
5141	*/
5142	align_bytes = min_t(u64, sizeof(void *), member_type->size);
5143	align_bits = align_bytes * BITS_PER_BYTE;
5144	div64_u64_rem(member->offset, align_bits, &misalign_bits);
5145	if (misalign_bits) {
5146	btf_verifier_log_member(env, struct_type, member,
5147	"Member is not properly aligned");
5148	return -EINVAL;
5149	}
5150
5151	start_offset_bytes = member->offset / BITS_PER_BYTE;
5152	end_offset_bytes = start_offset_bytes + member_type->size;
5153	if (end_offset_bytes > struct_type->size) {
5154	btf_verifier_log_member(env, struct_type, member,
5155	"Member exceeds struct_size");
5156	return -EINVAL;
5157	}
5158
5159	return 0;
5160	}
5161
5162	static void btf_float_log(struct btf_verifier_env *env,
5163	const struct btf_type *t)
5164	{
5165	btf_verifier_log(env, "size=%u", t->size);
5166	}
5167
5168	static const struct btf_kind_operations float_ops = {
5169	.check_meta = btf_float_check_meta,
5170	.resolve = btf_df_resolve,
5171	.check_member = btf_float_check_member,
5172	.check_kflag_member = btf_generic_check_kflag_member,
5173	.log_details = btf_float_log,
5174	.show = btf_df_show,
5175	};
5176
5177	static s32 btf_decl_tag_check_meta(struct btf_verifier_env *env,
5178	const struct btf_type *t,
5179	u32 meta_left)
5180	{
5181	const struct btf_decl_tag *tag;
5182	u32 meta_needed = sizeof(*tag);
5183	s32 component_idx;
5184	const char *value;
5185
5186	if (meta_left < meta_needed) {
5187	btf_verifier_log_basic(env, t,
5188	"meta_left:%u meta_needed:%u",
5189	meta_left, meta_needed);
5190	return -EINVAL;
5191	}
5192
5193	value = btf_name_by_offset(env->btf, t->name_off);
5194	if (!value \|\| !value[0]) {
5195	btf_verifier_log_type(env, t, "Invalid value");
5196	return -EINVAL;
5197	}
5198
5199	if (btf_type_vlen(t)) {
5200	btf_verifier_log_type(env, t, "vlen != 0");
5201	return -EINVAL;
5202	}
5203
5204	component_idx = btf_type_decl_tag(t)->component_idx;
5205	if (component_idx < -1) {
5206	btf_verifier_log_type(env, t, "Invalid component_idx");
5207	return -EINVAL;
5208	}
5209
5210	btf_verifier_log_type(env, t, NULL);
5211
5212	return meta_needed;
5213	}
5214
5215	static int btf_decl_tag_resolve(struct btf_verifier_env *env,
5216	const struct resolve_vertex *v)
5217	{
5218	const struct btf_type *next_type;
5219	const struct btf_type *t = v->t;
5220	u32 next_type_id = t->type;
5221	struct btf *btf = env->btf;
5222	s32 component_idx;
5223	u32 vlen;
5224
5225	next_type = btf_type_by_id(btf, next_type_id);
5226	if (!next_type \|\| !btf_type_is_decl_tag_target(next_type)) {
5227	btf_verifier_log_type(env, v->t, "Invalid type_id");
5228	return -EINVAL;
5229	}
5230
5231	if (!env_type_is_resolve_sink(env, next_type) &&
5232	!env_type_is_resolved(env, next_type_id))
5233	return env_stack_push(env, next_type, next_type_id);
5234
5235	component_idx = btf_type_decl_tag(t)->component_idx;
5236	if (component_idx != -1) {
5237	if (btf_type_is_var(next_type) \|\| btf_type_is_typedef(next_type)) {
5238	btf_verifier_log_type(env, v->t, "Invalid component_idx");
5239	return -EINVAL;
5240	}
5241
5242	if (btf_type_is_struct(next_type)) {
5243	vlen = btf_type_vlen(next_type);
5244	} else {
5245	/* next_type should be a function */
5246	next_type = btf_type_by_id(btf, next_type->type);
5247	vlen = btf_type_vlen(next_type);
5248	}
5249
5250	if ((u32)component_idx >= vlen) {
5251	btf_verifier_log_type(env, v->t, "Invalid component_idx");
5252	return -EINVAL;
5253	}
5254	}
5255
5256	env_stack_pop_resolved(env, next_type_id, 0);
5257
5258	return 0;
5259	}
5260
5261	static void btf_decl_tag_log(struct btf_verifier_env env, const struct btf_type t)
5262	{
5263	btf_verifier_log(env, "type=%u component_idx=%d", t->type,
5264	btf_type_decl_tag(t)->component_idx);
5265	}
5266
5267	static const struct btf_kind_operations decl_tag_ops = {
5268	.check_meta = btf_decl_tag_check_meta,
5269	.resolve = btf_decl_tag_resolve,
5270	.check_member = btf_df_check_member,
5271	.check_kflag_member = btf_df_check_kflag_member,
5272	.log_details = btf_decl_tag_log,
5273	.show = btf_df_show,
5274	};
5275
5276	static int btf_func_proto_check(struct btf_verifier_env *env,
5277	const struct btf_type *t)
5278	{
5279	const struct btf_type *ret_type;
5280	const struct btf_param *args;
5281	const struct btf *btf;
5282	u32 nr_args, i;
5283	int err;
5284
5285	btf = env->btf;
5286	args = (const struct btf_param *)(t + 1);
5287	nr_args = btf_type_vlen(t);
5288
5289	/* Check func return type which could be "void" (t->type == 0) */
5290	if (t->type) {
5291	u32 ret_type_id = t->type;
5292
5293	ret_type = btf_type_by_id(btf, ret_type_id);
5294	if (!ret_type) {
5295	btf_verifier_log_type(env, t, "Invalid return type");
5296	return -EINVAL;
5297	}
5298
5299	if (btf_type_is_resolve_source_only(ret_type)) {
5300	btf_verifier_log_type(env, t, "Invalid return type");
5301	return -EINVAL;
5302	}
5303
5304	if (btf_type_needs_resolve(ret_type) &&
5305	!env_type_is_resolved(env, ret_type_id)) {
5306	err = btf_resolve(env, ret_type, ret_type_id);
5307	if (err)
5308	return err;
5309	}
5310
5311	/* Ensure the return type is a type that has a size */
5312	if (!btf_type_id_size(btf, &ret_type_id, NULL)) {
5313	btf_verifier_log_type(env, t, "Invalid return type");
5314	return -EINVAL;
5315	}
5316	}
5317
5318	if (!nr_args)
5319	return 0;
5320
5321	/* Last func arg type_id could be 0 if it is a vararg */
5322	if (!args[nr_args - 1].type) {
5323	if (args[nr_args - 1].name_off) {
5324	btf_verifier_log_type(env, t, "Invalid arg#%u",
5325	nr_args);
5326	return -EINVAL;
5327	}
5328	nr_args--;
5329	}
5330
5331	for (i = 0; i < nr_args; i++) {
5332	const struct btf_type *arg_type;
5333	u32 arg_type_id;
5334
5335	arg_type_id = args[i].type;
5336	arg_type = btf_type_by_id(btf, arg_type_id);
5337	if (!arg_type) {
5338	btf_verifier_log_type(env, t, "Invalid arg#%u", i + 1);
5339	return -EINVAL;
5340	}
5341
5342	if (btf_type_is_resolve_source_only(arg_type)) {
5343	btf_verifier_log_type(env, t, "Invalid arg#%u", i + 1);
5344	return -EINVAL;
5345	}
5346
5347	if (args[i].name_off &&
5348	(!btf_name_offset_valid(btf, args[i].name_off) \|\|
5349	!btf_name_valid_identifier(btf, args[i].name_off))) {
5350	btf_verifier_log_type(env, t,
5351	"Invalid arg#%u", i + 1);
5352	return -EINVAL;
5353	}
5354
5355	if (btf_type_needs_resolve(arg_type) &&
5356	!env_type_is_resolved(env, arg_type_id)) {
5357	err = btf_resolve(env, arg_type, arg_type_id);
5358	if (err)
5359	return err;
5360	}
5361
5362	if (!btf_type_id_size(btf, &arg_type_id, NULL)) {
5363	btf_verifier_log_type(env, t, "Invalid arg#%u", i + 1);
5364	return -EINVAL;
5365	}
5366	}
5367
5368	return 0;
5369	}
5370
5371	static int btf_func_check(struct btf_verifier_env *env,
5372	const struct btf_type *t)
5373	{
5374	const struct btf_type *proto_type;
5375	const struct btf_param *args;
5376	const struct btf *btf;
5377	u32 nr_args, i;
5378
5379	btf = env->btf;
5380	proto_type = btf_type_by_id(btf, t->type);
5381
5382	if (!proto_type \|\| !btf_type_is_func_proto(proto_type)) {
5383	btf_verifier_log_type(env, t, "Invalid type_id");
5384	return -EINVAL;
5385	}
5386
5387	args = (const struct btf_param *)(proto_type + 1);
5388	nr_args = btf_type_vlen(proto_type);
5389	for (i = 0; i < nr_args; i++) {
5390	if (!args[i].name_off && args[i].type) {
5391	btf_verifier_log_type(env, t, "Invalid arg#%u", i + 1);
5392	return -EINVAL;
5393	}
5394	}
5395
5396	return 0;
5397	}
5398
5399	static const struct btf_kind_operations * const kind_ops[NR_BTF_KINDS] = {
5400	[BTF_KIND_INT] = &int_ops,
5401	[BTF_KIND_PTR] = &ptr_ops,
5402	[BTF_KIND_ARRAY] = &array_ops,
5403	[BTF_KIND_STRUCT] = &struct_ops,
5404	[BTF_KIND_UNION] = &struct_ops,
5405	[BTF_KIND_ENUM] = &enum_ops,
5406	[BTF_KIND_FWD] = &fwd_ops,
5407	[BTF_KIND_TYPEDEF] = &modifier_ops,
5408	[BTF_KIND_VOLATILE] = &modifier_ops,
5409	[BTF_KIND_CONST] = &modifier_ops,
5410	[BTF_KIND_RESTRICT] = &modifier_ops,
5411	[BTF_KIND_FUNC] = &func_ops,
5412	[BTF_KIND_FUNC_PROTO] = &func_proto_ops,
5413	[BTF_KIND_VAR] = &var_ops,
5414	[BTF_KIND_DATASEC] = &datasec_ops,
5415	[BTF_KIND_FLOAT] = &float_ops,
5416	[BTF_KIND_DECL_TAG] = &decl_tag_ops,
5417	[BTF_KIND_TYPE_TAG] = &modifier_ops,
5418	[BTF_KIND_ENUM64] = &enum64_ops,
5419	};
5420
5421	static s32 btf_check_meta(struct btf_verifier_env *env,
5422	const struct btf_type *t,
5423	u32 meta_left)
5424	{
5425	u32 saved_meta_left = meta_left;
5426	s32 var_meta_size;
5427
5428	if (meta_left < sizeof(*t)) {
5429	btf_verifier_log(env, "[%u] meta_left:%u meta_needed:%zu",
5430	env->log_type_id, meta_left, sizeof(*t));
5431	return -EINVAL;
5432	}
5433	meta_left -= sizeof(*t);
5434
5435	if (BTF_INFO_KIND(t->info) > BTF_KIND_MAX \|\|
5436	BTF_INFO_KIND(t->info) == BTF_KIND_UNKN) {
5437	btf_verifier_log(env, "[%u] Invalid kind:%u",
5438	env->log_type_id, BTF_INFO_KIND(t->info));
5439	return -EINVAL;
5440	}
5441
5442	if (!btf_name_offset_valid(env->btf, t->name_off)) {
5443	btf_verifier_log(env, "[%u] Invalid name_offset:%u",
5444	env->log_type_id, t->name_off);
5445	return -EINVAL;
5446	}
5447
5448	var_meta_size = btf_type_ops(t)->check_meta(env, t, meta_left);
5449	if (var_meta_size < 0)
5450	return var_meta_size;
5451
5452	meta_left -= var_meta_size;
5453
5454	return saved_meta_left - meta_left;
5455	}
5456
5457	static int btf_check_all_metas(struct btf_verifier_env *env)
5458	{
5459	struct btf *btf = env->btf;
5460	struct btf_header *hdr;
5461	void cur, end;
5462
5463	hdr = &btf->hdr;
5464	cur = btf->nohdr_data + hdr->type_off;
5465	end = cur + hdr->type_len;
5466
5467	env->log_type_id = btf->base_btf ? btf->start_id : 1;
5468	while (cur < end) {
5469	struct btf_type *t = cur;
5470	s32 meta_size;
5471
5472	meta_size = btf_check_meta(env, t, end - cur);
5473	if (meta_size < 0)
5474	return meta_size;
5475
5476	btf_add_type(env, t);
5477	cur += meta_size;
5478	env->log_type_id++;
5479	}
5480
5481	return 0;
5482	}
5483
5484	static bool btf_resolve_valid(struct btf_verifier_env *env,
5485	const struct btf_type *t,
5486	u32 type_id)
5487	{
5488	struct btf *btf = env->btf;
5489
5490	if (!env_type_is_resolved(env, type_id))
5491	return false;
5492
5493	if (btf_type_is_struct(t) \|\| btf_type_is_datasec(t))
5494	return !btf_resolved_type_id(btf, type_id) &&
5495	!btf_resolved_type_size(btf, type_id);
5496
5497	if (btf_type_is_decl_tag(t) \|\| btf_type_is_func(t))
5498	return btf_resolved_type_id(btf, type_id) &&
5499	!btf_resolved_type_size(btf, type_id);
5500
5501	if (btf_type_is_modifier(t) \|\| btf_type_is_ptr(t) \|\|
5502	btf_type_is_var(t)) {
5503	t = btf_type_id_resolve(btf, &type_id);
5504	return t &&
5505	!btf_type_is_modifier(t) &&
5506	!btf_type_is_var(t) &&
5507	!btf_type_is_datasec(t);
5508	}
5509
5510	if (btf_type_is_array(t)) {
5511	const struct btf_array *array = btf_type_array(t);
5512	const struct btf_type *elem_type;
5513	u32 elem_type_id = array->type;
5514	u32 elem_size;
5515
5516	elem_type = btf_type_id_size(btf, &elem_type_id, &elem_size);
5517	return elem_type && !btf_type_is_modifier(elem_type) &&
5518	(array->nelems * elem_size ==
5519	btf_resolved_type_size(btf, type_id));
5520	}
5521
5522	return false;
5523	}
5524
5525	static int btf_resolve(struct btf_verifier_env *env,
5526	const struct btf_type *t, u32 type_id)
5527	{
5528	u32 save_log_type_id = env->log_type_id;
5529	const struct resolve_vertex *v;
5530	int err = 0;
5531
5532	env->resolve_mode = RESOLVE_TBD;
5533	env_stack_push(env, t, type_id);
5534	while (!err && (v = env_stack_peak(env))) {
5535	env->log_type_id = v->type_id;
5536	err = btf_type_ops(v->t)->resolve(env, v);
5537	}
5538
5539	env->log_type_id = type_id;
5540	if (err == -E2BIG) {
5541	btf_verifier_log_type(env, t,
5542	"Exceeded max resolving depth:%u",
5543	MAX_RESOLVE_DEPTH);
5544	} else if (err == -EEXIST) {
5545	btf_verifier_log_type(env, t, "Loop detected");
5546	}
5547
5548	/* Final sanity check */
5549	if (!err && !btf_resolve_valid(env, t, type_id)) {
5550	btf_verifier_log_type(env, t, "Invalid resolve state");
5551	err = -EINVAL;
5552	}
5553
5554	env->log_type_id = save_log_type_id;
5555	return err;
5556	}
5557
5558	static int btf_check_all_types(struct btf_verifier_env *env)
5559	{
5560	struct btf *btf = env->btf;
5561	const struct btf_type *t;
5562	u32 type_id, i;
5563	int err;
5564
5565	err = env_resolve_init(env);
5566	if (err)
5567	return err;
5568
5569	env->phase++;
5570	for (i = btf->base_btf ? 0 : 1; i < btf->nr_types; i++) {
5571	type_id = btf->start_id + i;
5572	t = btf_type_by_id(btf, type_id);
5573
5574	env->log_type_id = type_id;
5575	if (btf_type_needs_resolve(t) &&
5576	!env_type_is_resolved(env, type_id)) {
5577	err = btf_resolve(env, t, type_id);
5578	if (err)
5579	return err;
5580	}
5581
5582	if (btf_type_is_func_proto(t)) {
5583	err = btf_func_proto_check(env, t);
5584	if (err)
5585	return err;
5586	}
5587	}
5588
5589	return 0;
5590	}
5591
5592	static int btf_parse_type_sec(struct btf_verifier_env *env)
5593	{
5594	const struct btf_header *hdr = &env->btf->hdr;
5595	int err;
5596
5597	/* Type section must align to 4 bytes */
5598	if (hdr->type_off & (sizeof(u32) - 1)) {
5599	btf_verifier_log(env, "Unaligned type_off");
5600	return -EINVAL;
5601	}
5602
5603	if (!env->btf->base_btf && !hdr->type_len) {
5604	btf_verifier_log(env, "No type found");
5605	return -EINVAL;
5606	}
5607
5608	err = btf_check_all_metas(env);
5609	if (err)
5610	return err;
5611
5612	return btf_check_all_types(env);
5613	}
5614
5615	static int btf_parse_str_sec(struct btf_verifier_env *env)
5616	{
5617	const struct btf_header *hdr;
5618	struct btf *btf = env->btf;
5619	const char start, end;
5620
5621	hdr = &btf->hdr;
5622	start = btf->nohdr_data + hdr->str_off;
5623	end = start + hdr->str_len;
5624
5625	if (hdr->hdr_len < sizeof(struct btf_header) &&
5626	end != btf->data + btf->data_size) {
5627	btf_verifier_log(env, "String section is not at the end");
5628	return -EINVAL;
5629	}
5630
5631	btf->strings = start;
5632
5633	if (btf->base_btf && !hdr->str_len)
5634	return 0;
5635	if (!hdr->str_len \|\| hdr->str_len - 1 > BTF_MAX_NAME_OFFSET \|\| end[-1]) {
5636	btf_verifier_log(env, "Invalid string section");
5637	return -EINVAL;
5638	}
5639	if (!btf->base_btf && start[0]) {
5640	btf_verifier_log(env, "Invalid string section");
5641	return -EINVAL;
5642	}
5643
5644	return 0;
5645	}
5646
5647	static int btf_parse_layout_sec(struct btf_verifier_env *env)
5648	{
5649	const struct btf_header *hdr = &env->btf->hdr;
5650	struct btf *btf = env->btf;
5651	void start, end;
5652
5653	if (hdr->hdr_len < sizeof(struct btf_header) \|\|
5654	hdr->layout_len == 0)
5655	return 0;
5656
5657	/* Layout section must align to 4 bytes */
5658	if (hdr->layout_off & (sizeof(u32) - 1)) {
5659	btf_verifier_log(env, "Unaligned layout_off");
5660	return -EINVAL;
5661	}
5662	start = btf->nohdr_data + hdr->layout_off;
5663	end = start + hdr->layout_len;
5664
5665	if (hdr->layout_len < sizeof(struct btf_layout)) {
5666	btf_verifier_log(env, "Layout section is too small");
5667	return -EINVAL;
5668	}
5669	if (hdr->layout_len % sizeof(struct btf_layout) != 0) {
5670	btf_verifier_log(env, "layout_len is not multiple of %zu",
5671	sizeof(struct btf_layout));
5672	return -EINVAL;
5673	}
5674	if (end > btf->data + btf->data_size) {
5675	btf_verifier_log(env, "Layout section is too big");
5676	return -EINVAL;
5677	}
5678	btf->layout = start;
5679
5680	return 0;
5681	}
5682
5683	static const size_t btf_sec_info_offset[] = {
5684	offsetof(struct btf_header, type_off),
5685	offsetof(struct btf_header, str_off),
5686	offsetof(struct btf_header, layout_off)
5687	};
5688
5689	static int btf_sec_info_cmp(const void a, const void b)
5690	{
5691	const struct btf_sec_info *x = a;
5692	const struct btf_sec_info *y = b;
5693
5694	return (int)(x->off - y->off) ? : (int)(x->len - y->len);
5695	}
5696
5697	static int btf_check_sec_info(struct btf_verifier_env *env,
5698	u32 btf_data_size)
5699	{
5700	struct btf_sec_info secs[ARRAY_SIZE(btf_sec_info_offset)];
5701	u32 total, expected_total, i;
5702	u32 nr_secs = ARRAY_SIZE(btf_sec_info_offset);
5703	const struct btf_header *hdr;
5704	const struct btf *btf;
5705
5706	btf = env->btf;
5707	hdr = &btf->hdr;
5708
5709	if (hdr->hdr_len < sizeof(struct btf_header) \|\| hdr->layout_len == 0)
5710	nr_secs--;
5711
5712	/* Populate the secs from hdr */
5713	for (i = 0; i < nr_secs; i++)
5714	secs[i] = (struct btf_sec_info )((void *)hdr +
5715	btf_sec_info_offset[i]);
5716
5717	sort(secs, nr_secs,
5718	sizeof(struct btf_sec_info), btf_sec_info_cmp, NULL);
5719
5720	/* Check for gaps and overlap among sections */
5721	total = 0;
5722	expected_total = btf_data_size - hdr->hdr_len;
5723	for (i = 0; i < nr_secs; i++) {
5724	if (expected_total < secs[i].off) {
5725	btf_verifier_log(env, "Invalid section offset");
5726	return -EINVAL;
5727	}
5728	if (total < secs[i].off) {
5729	/* gap */
5730	btf_verifier_log(env, "Unsupported section found");
5731	return -EINVAL;
5732	}
5733	if (total > secs[i].off) {
5734	btf_verifier_log(env, "Section overlap found");
5735	return -EINVAL;
5736	}
5737	if (expected_total - total < secs[i].len) {
5738	btf_verifier_log(env,
5739	"Total section length too long");
5740	return -EINVAL;
5741	}
5742	total += secs[i].len;
5743	}
5744
5745	/* There is data other than hdr and known sections */
5746	if (expected_total != total) {
5747	btf_verifier_log(env, "Unsupported section found");
5748	return -EINVAL;
5749	}
5750
5751	return 0;
5752	}
5753
5754	static int btf_parse_hdr(struct btf_verifier_env *env)
5755	{
5756	u32 hdr_len, hdr_copy, btf_data_size;
5757	const struct btf_header *hdr;
5758	struct btf *btf;
5759
5760	btf = env->btf;
5761	btf_data_size = btf->data_size;
5762
5763	if (btf_data_size < offsetofend(struct btf_header, hdr_len)) {
5764	btf_verifier_log(env, "hdr_len not found");
5765	return -EINVAL;
5766	}
5767
5768	hdr = btf->data;
5769	hdr_len = hdr->hdr_len;
5770	if (btf_data_size < hdr_len) {
5771	btf_verifier_log(env, "btf_header not found");
5772	return -EINVAL;
5773	}
5774
5775	/* Ensure the unsupported header fields are zero */
5776	if (hdr_len > sizeof(btf->hdr)) {
5777	u8 *expected_zero = btf->data + sizeof(btf->hdr);
5778	u8 *end = btf->data + hdr_len;
5779
5780	for (; expected_zero < end; expected_zero++) {
5781	if (*expected_zero) {
5782	btf_verifier_log(env, "Unsupported btf_header");
5783	return -E2BIG;
5784	}
5785	}
5786	}
5787
5788	hdr_copy = min_t(u32, hdr_len, sizeof(btf->hdr));
5789	memcpy(&btf->hdr, btf->data, hdr_copy);
5790
5791	hdr = &btf->hdr;
5792
5793	btf_verifier_log_hdr(env, btf_data_size);
5794
5795	if (hdr->magic != BTF_MAGIC) {
5796	btf_verifier_log(env, "Invalid magic");
5797	return -EINVAL;
5798	}
5799
5800	if (hdr->version != BTF_VERSION) {
5801	btf_verifier_log(env, "Unsupported version");
5802	return -ENOTSUPP;
5803	}
5804
5805	if (hdr->flags) {
5806	btf_verifier_log(env, "Unsupported flags");
5807	return -ENOTSUPP;
5808	}
5809
5810	if (!btf->base_btf && btf_data_size == hdr->hdr_len) {
5811	btf_verifier_log(env, "No data");
5812	return -EINVAL;
5813	}
5814
5815	return btf_check_sec_info(env, btf_data_size);
5816	}
5817
5818	static const char *alloc_obj_fields[] = {
5819	"bpf_spin_lock",
5820	"bpf_list_head",
5821	"bpf_list_node",
5822	"bpf_rb_root",
5823	"bpf_rb_node",
5824	"bpf_refcount",
5825	};
5826
5827	static struct btf_struct_metas *
5828	btf_parse_struct_metas(struct bpf_verifier_log log, struct btf btf)
5829	{
5830	struct btf_struct_metas *tab = NULL;
5831	struct btf_id_set *aof;
5832	int i, n, id, ret;
5833
5834	BUILD_BUG_ON(offsetof(struct btf_id_set, cnt) != 0);
5835	BUILD_BUG_ON(sizeof(struct btf_id_set) != sizeof(u32));
5836
5837	aof = kmalloc_obj(*aof, GFP_KERNEL \| __GFP_NOWARN);
5838	if (!aof)
5839	return ERR_PTR(-ENOMEM);
5840	aof->cnt = 0;
5841
5842	for (i = 0; i < ARRAY_SIZE(alloc_obj_fields); i++) {
5843	/* Try to find whether this special type exists in user BTF, and
5844	* if so remember its ID so we can easily find it among members
5845	* of structs that we iterate in the next loop.
5846	*/
5847	struct btf_id_set *new_aof;
5848
5849	id = btf_find_by_name_kind(btf, alloc_obj_fields[i], BTF_KIND_STRUCT);
5850	if (id < 0)
5851	continue;
5852
5853	new_aof = krealloc(aof, struct_size(new_aof, ids, aof->cnt + 1),
5854	GFP_KERNEL \| __GFP_NOWARN);
5855	if (!new_aof) {
5856	ret = -ENOMEM;
5857	goto free_aof;
5858	}
5859	aof = new_aof;
5860	aof->ids[aof->cnt++] = id;
5861	}
5862
5863	n = btf_nr_types(btf);
5864	for (i = 1; i < n; i++) {
5865	/* Try to find if there are kptrs in user BTF and remember their ID */
5866	struct btf_id_set *new_aof;
5867	struct btf_field_info tmp;
5868	const struct btf_type *t;
5869
5870	t = btf_type_by_id(btf, i);
5871	if (!t) {
5872	ret = -EINVAL;
5873	goto free_aof;
5874	}
5875
5876	ret = btf_find_kptr(btf, t, 0, 0, &tmp, BPF_KPTR);
5877	if (ret != BTF_FIELD_FOUND)
5878	continue;
5879
5880	new_aof = krealloc(aof, struct_size(new_aof, ids, aof->cnt + 1),
5881	GFP_KERNEL \| __GFP_NOWARN);
5882	if (!new_aof) {
5883	ret = -ENOMEM;
5884	goto free_aof;
5885	}
5886	aof = new_aof;
5887	aof->ids[aof->cnt++] = i;
5888	}
5889
5890	if (!aof->cnt) {
5891	kfree(aof);
5892	return NULL;
5893	}
5894	sort(&aof->ids, aof->cnt, sizeof(aof->ids[0]), btf_id_cmp_func, NULL);
5895
5896	for (i = 1; i < n; i++) {
5897	struct btf_struct_metas *new_tab;
5898	const struct btf_member *member;
5899	struct btf_struct_meta *type;
5900	struct btf_record *record;
5901	const struct btf_type *t;
5902	int j, tab_cnt;
5903
5904	t = btf_type_by_id(btf, i);
5905	if (!__btf_type_is_struct(t))
5906	continue;
5907
5908	cond_resched();
5909
5910	for_each_member(j, t, member) {
5911	if (btf_id_set_contains(aof, member->type))
5912	goto parse;
5913	}
5914	continue;
5915	parse:
5916	tab_cnt = tab ? tab->cnt : 0;
5917	new_tab = krealloc(tab, struct_size(new_tab, types, tab_cnt + 1),
5918	GFP_KERNEL \| __GFP_NOWARN);
5919	if (!new_tab) {
5920	ret = -ENOMEM;
5921	goto free;
5922	}
5923	if (!tab)
5924	new_tab->cnt = 0;
5925	tab = new_tab;
5926
5927	type = &tab->types[tab->cnt];
5928	type->btf_id = i;
5929	record = btf_parse_fields(btf, t, BPF_SPIN_LOCK \| BPF_RES_SPIN_LOCK \| BPF_LIST_HEAD \| BPF_LIST_NODE \|
5930	BPF_RB_ROOT \| BPF_RB_NODE \| BPF_REFCOUNT \|
5931	BPF_KPTR, t->size);
5932	/* The record cannot be unset, treat it as an error if so */
5933	if (IS_ERR_OR_NULL(record)) {
5934	ret = PTR_ERR_OR_ZERO(record) ?: -EFAULT;
5935	goto free;
5936	}
5937	type->record = record;
5938	tab->cnt++;
5939	}
5940	kfree(aof);
5941	return tab;
5942	free:
5943	btf_struct_metas_free(tab);
5944	free_aof:
5945	kfree(aof);
5946	return ERR_PTR(ret);
5947	}
5948
5949	struct btf_struct_meta btf_find_struct_meta(const struct btf btf, u32 btf_id)
5950	{
5951	struct btf_struct_metas *tab;
5952
5953	BUILD_BUG_ON(offsetof(struct btf_struct_meta, btf_id) != 0);
5954	tab = btf->struct_meta_tab;
5955	if (!tab)
5956	return NULL;
5957	return bsearch(&btf_id, tab->types, tab->cnt, sizeof(tab->types[0]), btf_id_cmp_func);
5958	}
5959
5960	static int btf_check_modifier_chain_length(struct btf_verifier_env *env,
5961	struct btf *btf, int start_id)
5962	{
5963	int i, n, good_id = start_id - 1;
5964
5965	n = btf_nr_types(btf);
5966	for (i = start_id; i < n; i++) {
5967	const struct btf_type *t;
5968	int chain_limit = 32;
5969	u32 cur_id = i;
5970
5971	t = btf_type_by_id(btf, i);
5972	if (!t)
5973	return -EINVAL;
5974	if (!btf_type_is_modifier(t))
5975	continue;
5976
5977	cond_resched();
5978
5979	while (btf_type_is_modifier(t)) {
5980	if (!chain_limit--) {
5981	btf_verifier_log(env, "Max chain length or cycle detected");
5982	return -ELOOP;
5983	}
5984
5985	if (cur_id <= good_id)
5986	break;
5987	/* Move to next type */
5988	cur_id = t->type;
5989	t = btf_type_by_id(btf, cur_id);
5990	if (!t)
5991	return -EINVAL;
5992	}
5993	good_id = i;
5994	}
5995	return 0;
5996	}
5997
5998	static struct btf btf_parse(const union bpf_attr attr, bpfptr_t uattr,
5999	struct bpf_log_attr *attr_log)
6000	{
6001	bpfptr_t btf_data = make_bpfptr(attr->btf, uattr.is_kernel);
6002	struct btf_struct_metas *struct_meta_tab;
6003	struct btf_verifier_env *env = NULL;
6004	struct btf *btf = NULL;
6005	u8 *data;
6006	int err, ret;
6007
6008	if (attr->btf_size > BTF_MAX_SIZE)
6009	return ERR_PTR(-E2BIG);
6010
6011	env = kzalloc_obj(*env, GFP_KERNEL \| __GFP_NOWARN);
6012	if (!env)
6013	return ERR_PTR(-ENOMEM);
6014
6015	/* user could have requested verbose verifier output
6016	* and supplied buffer to store the verification trace
6017	*/
6018	err = bpf_vlog_init(&env->log, attr_log->level, attr_log->ubuf, attr_log->size);
6019	if (err)
6020	goto errout_free;
6021
6022	btf = kzalloc_obj(*btf, GFP_KERNEL \| __GFP_NOWARN);
6023	if (!btf) {
6024	err = -ENOMEM;
6025	goto errout;
6026	}
6027	env->btf = btf;
6028	btf->named_start_id = 0;
6029
6030	data = kvmalloc(attr->btf_size, GFP_KERNEL \| __GFP_NOWARN);
6031	if (!data) {
6032	err = -ENOMEM;
6033	goto errout;
6034	}
6035
6036	btf->data = data;
6037	btf->data_size = attr->btf_size;
6038
6039	if (copy_from_bpfptr(data, btf_data, attr->btf_size)) {
6040	err = -EFAULT;
6041	goto errout;
6042	}
6043
6044	err = btf_parse_hdr(env);
6045	if (err)
6046	goto errout;
6047
6048	btf->nohdr_data = btf->data + btf->hdr.hdr_len;
6049
6050	err = btf_parse_str_sec(env);
6051	if (err)
6052	goto errout;
6053
6054	err = btf_parse_layout_sec(env);
6055	if (err)
6056	goto errout;
6057
6058	err = btf_parse_type_sec(env);
6059	if (err)
6060	goto errout;
6061
6062	err = btf_check_modifier_chain_length(env, btf, 1);
6063	if (err)
6064	goto errout;
6065
6066	struct_meta_tab = btf_parse_struct_metas(&env->log, btf);
6067	if (IS_ERR(struct_meta_tab)) {
6068	err = PTR_ERR(struct_meta_tab);
6069	goto errout;
6070	}
6071	btf->struct_meta_tab = struct_meta_tab;
6072
6073	if (struct_meta_tab) {
6074	int i;
6075
6076	for (i = 0; i < struct_meta_tab->cnt; i++) {
6077	err = btf_check_and_fixup_fields(btf, struct_meta_tab->types[i].record);
6078	if (err < 0)
6079	goto errout_meta;
6080	}
6081
6082	err = btf_check_ownership_depth(btf, struct_meta_tab);
6083	if (err < 0)
6084	goto errout_meta;
6085	}
6086
6087	err = bpf_log_attr_finalize(attr_log, &env->log);
6088	if (err)
6089	goto errout_free;
6090
6091	btf_verifier_env_free(env);
6092	refcount_set(&btf->refcnt, 1);
6093	return btf;
6094
6095	errout_meta:
6096	btf_free_struct_meta_tab(btf);
6097	errout:
6098	/* overwrite err with -ENOSPC or -EFAULT */
6099	ret = bpf_log_attr_finalize(attr_log, &env->log);
6100	if (ret)
6101	err = ret;
6102	errout_free:
6103	btf_verifier_env_free(env);
6104	if (btf)
6105	btf_free(btf);
6106	return ERR_PTR(err);
6107	}
6108
6109	extern char __start_BTF[];
6110	extern char __stop_BTF[];
6111	extern struct btf *btf_vmlinux;
6112
6113	#define BPF_MAP_TYPE(_id, _ops)
6114	#define BPF_LINK_TYPE(_id, _name)
6115	static union {
6116	struct bpf_ctx_convert {
6117	#define BPF_PROG_TYPE(_id, _name, prog_ctx_type, kern_ctx_type) \
6118	prog_ctx_type _id##_prog; \
6119	kern_ctx_type _id##_kern;
6120	#include <linux/bpf_types.h>
6121	#undef BPF_PROG_TYPE
6122	} *__t;
6123	/* 't' is written once under lock. Read many times. */
6124	const struct btf_type *t;
6125	} bpf_ctx_convert;
6126	enum {
6127	#define BPF_PROG_TYPE(_id, _name, prog_ctx_type, kern_ctx_type) \
6128	__ctx_convert##_id,
6129	#include <linux/bpf_types.h>
6130	#undef BPF_PROG_TYPE
6131	__ctx_convert_unused, /* to avoid empty enum in extreme .config */
6132	};
6133	static u8 bpf_ctx_convert_map[] = {
6134	#define BPF_PROG_TYPE(_id, _name, prog_ctx_type, kern_ctx_type) \
6135	[_id] = __ctx_convert##_id,
6136	#include <linux/bpf_types.h>
6137	#undef BPF_PROG_TYPE
6138	0, /* avoid empty array */
6139	};
6140	#undef BPF_MAP_TYPE
6141	#undef BPF_LINK_TYPE
6142
6143	static const struct btf_type *find_canonical_prog_ctx_type(enum bpf_prog_type prog_type)
6144	{
6145	const struct btf_type *conv_struct;
6146	const struct btf_member *ctx_type;
6147
6148	conv_struct = bpf_ctx_convert.t;
6149	if (!conv_struct)
6150	return NULL;
6151	/* prog_type is valid bpf program type. No need for bounds check. */
6152	ctx_type = btf_type_member(conv_struct) + bpf_ctx_convert_map[prog_type] * 2;
6153	/* ctx_type is a pointer to prog_ctx_type in vmlinux.
6154	* Like 'struct __sk_buff'
6155	*/
6156	return btf_type_by_id(btf_vmlinux, ctx_type->type);
6157	}
6158
6159	static int find_kern_ctx_type_id(enum bpf_prog_type prog_type)
6160	{
6161	const struct btf_type *conv_struct;
6162	const struct btf_member *ctx_type;
6163
6164	conv_struct = bpf_ctx_convert.t;
6165	if (!conv_struct)
6166	return -EFAULT;
6167	/* prog_type is valid bpf program type. No need for bounds check. */
6168	ctx_type = btf_type_member(conv_struct) + bpf_ctx_convert_map[prog_type] * 2 + 1;
6169	/* ctx_type is a pointer to prog_ctx_type in vmlinux.
6170	* Like 'struct sk_buff'
6171	*/
6172	return ctx_type->type;
6173	}
6174
6175	bool btf_is_projection_of(const char pname, const char tname)
6176	{
6177	if (strcmp(pname, "__sk_buff") == 0 && strcmp(tname, "sk_buff") == 0)
6178	return true;
6179	if (strcmp(pname, "xdp_md") == 0 && strcmp(tname, "xdp_buff") == 0)
6180	return true;
6181	return false;
6182	}
6183
6184	bool btf_is_prog_ctx_type(struct bpf_verifier_log log, const struct btf btf,
6185	const struct btf_type *t, enum bpf_prog_type prog_type,
6186	int arg)
6187	{
6188	const struct btf_type *ctx_type;
6189	const char tname, ctx_tname;
6190
6191	t = btf_type_by_id(btf, t->type);
6192
6193	/* KPROBE programs allow bpf_user_pt_regs_t typedef, which we need to
6194	* check before we skip all the typedef below.
6195	*/
6196	if (prog_type == BPF_PROG_TYPE_KPROBE) {
6197	while (btf_type_is_modifier(t) && !btf_type_is_typedef(t))
6198	t = btf_type_by_id(btf, t->type);
6199
6200	if (btf_type_is_typedef(t)) {
6201	tname = btf_name_by_offset(btf, t->name_off);
6202	if (tname && strcmp(tname, "bpf_user_pt_regs_t") == 0)
6203	return true;
6204	}
6205	}
6206
6207	while (btf_type_is_modifier(t))
6208	t = btf_type_by_id(btf, t->type);
6209	if (!btf_type_is_struct(t)) {
6210	/* Only pointer to struct is supported for now.
6211	* That means that BPF_PROG_TYPE_TRACEPOINT with BTF
6212	* is not supported yet.
6213	* BPF_PROG_TYPE_RAW_TRACEPOINT is fine.
6214	*/
6215	return false;
6216	}
6217	tname = btf_name_by_offset(btf, t->name_off);
6218	if (!tname) {
6219	bpf_log(log, "arg#%d struct doesn't have a name\n", arg);
6220	return false;
6221	}
6222
6223	ctx_type = find_canonical_prog_ctx_type(prog_type);
6224	if (!ctx_type) {
6225	bpf_log(log, "btf_vmlinux is malformed\n");
6226	/* should not happen */
6227	return false;
6228	}
6229	again:
6230	ctx_tname = btf_name_by_offset(btf_vmlinux, ctx_type->name_off);
6231	if (!ctx_tname) {
6232	/* should not happen */
6233	bpf_log(log, "Please fix kernel include/linux/bpf_types.h\n");
6234	return false;
6235	}
6236	/* program types without named context types work only with arg:ctx tag */
6237	if (ctx_tname[0] == '\0')
6238	return false;
6239	/* only compare that prog's ctx type name is the same as
6240	* kernel expects. No need to compare field by field.
6241	* It's ok for bpf prog to do:
6242	* struct __sk_buff {};
6243	* int socket_filter_bpf_prog(struct __sk_buff *skb)
6244	* { // no fields of skb are ever used }
6245	*/
6246	if (btf_is_projection_of(ctx_tname, tname))
6247	return true;
6248	if (strcmp(ctx_tname, tname)) {
6249	/* bpf_user_pt_regs_t is a typedef, so resolve it to
6250	* underlying struct and check name again
6251	*/
6252	if (!btf_type_is_modifier(ctx_type))
6253	return false;
6254	while (btf_type_is_modifier(ctx_type))
6255	ctx_type = btf_type_by_id(btf_vmlinux, ctx_type->type);
6256	goto again;
6257	}
6258	return true;
6259	}
6260
6261	/* forward declarations for arch-specific underlying types of
6262	* bpf_user_pt_regs_t; this avoids the need for arch-specific #ifdef
6263	* compilation guards below for BPF_PROG_TYPE_PERF_EVENT checks, but still
6264	* works correctly with __builtin_types_compatible_p() on respective
6265	* architectures
6266	*/
6267	struct user_regs_struct;
6268	struct user_pt_regs;
6269
6270	static int btf_validate_prog_ctx_type(struct bpf_verifier_log log, const struct btf btf,
6271	const struct btf_type *t, int arg,
6272	enum bpf_prog_type prog_type,
6273	enum bpf_attach_type attach_type)
6274	{
6275	const struct btf_type *ctx_type;
6276	const char tname, ctx_tname;
6277
6278	if (!btf_is_ptr(t)) {
6279	bpf_log(log, "arg#%d type isn't a pointer\n", arg);
6280	return -EINVAL;
6281	}
6282	t = btf_type_by_id(btf, t->type);
6283
6284	/* KPROBE and PERF_EVENT programs allow bpf_user_pt_regs_t typedef */
6285	if (prog_type == BPF_PROG_TYPE_KPROBE \|\| prog_type == BPF_PROG_TYPE_PERF_EVENT) {
6286	while (btf_type_is_modifier(t) && !btf_type_is_typedef(t))
6287	t = btf_type_by_id(btf, t->type);
6288
6289	if (btf_type_is_typedef(t)) {
6290	tname = btf_name_by_offset(btf, t->name_off);
6291	if (tname && strcmp(tname, "bpf_user_pt_regs_t") == 0)
6292	return 0;
6293	}
6294	}
6295
6296	/* all other program types don't use typedefs for context type */
6297	while (btf_type_is_modifier(t))
6298	t = btf_type_by_id(btf, t->type);
6299
6300	/* `void ctx __arg_ctx` is always valid /
6301	if (btf_type_is_void(t))
6302	return 0;
6303
6304	tname = btf_name_by_offset(btf, t->name_off);
6305	if (str_is_empty(tname)) {
6306	bpf_log(log, "arg#%d type doesn't have a name\n", arg);
6307	return -EINVAL;
6308	}
6309
6310	/* special cases */
6311	switch (prog_type) {
6312	case BPF_PROG_TYPE_KPROBE:
6313	if (__btf_type_is_struct(t) && strcmp(tname, "pt_regs") == 0)
6314	return 0;
6315	break;
6316	case BPF_PROG_TYPE_PERF_EVENT:
6317	if (__builtin_types_compatible_p(bpf_user_pt_regs_t, struct pt_regs) &&
6318	__btf_type_is_struct(t) && strcmp(tname, "pt_regs") == 0)
6319	return 0;
6320	if (__builtin_types_compatible_p(bpf_user_pt_regs_t, struct user_pt_regs) &&
6321	__btf_type_is_struct(t) && strcmp(tname, "user_pt_regs") == 0)
6322	return 0;
6323	if (__builtin_types_compatible_p(bpf_user_pt_regs_t, struct user_regs_struct) &&
6324	__btf_type_is_struct(t) && strcmp(tname, "user_regs_struct") == 0)
6325	return 0;
6326	break;
6327	case BPF_PROG_TYPE_RAW_TRACEPOINT:
6328	case BPF_PROG_TYPE_RAW_TRACEPOINT_WRITABLE:
6329	/* allow u64* as ctx */
6330	if (btf_is_int(t) && t->size == 8)
6331	return 0;
6332	break;
6333	case BPF_PROG_TYPE_TRACING:
6334	switch (attach_type) {
6335	case BPF_TRACE_RAW_TP:
6336	/* tp_btf program is TRACING, so need special case here */
6337	if (__btf_type_is_struct(t) &&
6338	strcmp(tname, "bpf_raw_tracepoint_args") == 0)
6339	return 0;
6340	/* allow u64* as ctx */
6341	if (btf_is_int(t) && t->size == 8)
6342	return 0;
6343	break;
6344	case BPF_TRACE_ITER:
6345	/* allow struct bpf_iter__xxx types only */
6346	if (__btf_type_is_struct(t) &&
6347	strncmp(tname, "bpf_iter__", sizeof("bpf_iter__") - 1) == 0)
6348	return 0;
6349	break;
6350	case BPF_TRACE_FENTRY:
6351	case BPF_TRACE_FEXIT:
6352	case BPF_MODIFY_RETURN:
6353	case BPF_TRACE_FSESSION:
6354	/* allow u64* as ctx */
6355	if (btf_is_int(t) && t->size == 8)
6356	return 0;
6357	break;
6358	default:
6359	break;
6360	}
6361	break;
6362	case BPF_PROG_TYPE_LSM:
6363	case BPF_PROG_TYPE_STRUCT_OPS:
6364	/* allow u64* as ctx */
6365	if (btf_is_int(t) && t->size == 8)
6366	return 0;
6367	break;
6368	case BPF_PROG_TYPE_TRACEPOINT:
6369	case BPF_PROG_TYPE_SYSCALL:
6370	case BPF_PROG_TYPE_EXT:
6371	return 0; /* anything goes */
6372	default:
6373	break;
6374	}
6375
6376	ctx_type = find_canonical_prog_ctx_type(prog_type);
6377	if (!ctx_type) {
6378	/* should not happen */
6379	bpf_log(log, "btf_vmlinux is malformed\n");
6380	return -EINVAL;
6381	}
6382
6383	/* resolve typedefs and check that underlying structs are matching as well */
6384	while (btf_type_is_modifier(ctx_type))
6385	ctx_type = btf_type_by_id(btf_vmlinux, ctx_type->type);
6386
6387	/* if program type doesn't have distinctly named struct type for
6388	* context, then __arg_ctx argument can only be `void *`, which we
6389	* already checked above
6390	*/
6391	if (!__btf_type_is_struct(ctx_type)) {
6392	bpf_log(log, "arg#%d should be void pointer\n", arg);
6393	return -EINVAL;
6394	}
6395
6396	ctx_tname = btf_name_by_offset(btf_vmlinux, ctx_type->name_off);
6397	if (!__btf_type_is_struct(t) \|\| strcmp(ctx_tname, tname) != 0) {
6398	bpf_log(log, "arg#%d should be `struct %s *`\n", arg, ctx_tname);
6399	return -EINVAL;
6400	}
6401
6402	return 0;
6403	}
6404
6405	static int btf_translate_to_vmlinux(struct bpf_verifier_log *log,
6406	struct btf *btf,
6407	const struct btf_type *t,
6408	enum bpf_prog_type prog_type,
6409	int arg)
6410	{
6411	if (!btf_is_prog_ctx_type(log, btf, t, prog_type, arg))
6412	return -ENOENT;
6413	return find_kern_ctx_type_id(prog_type);
6414	}
6415
6416	int get_kern_ctx_btf_id(struct bpf_verifier_log *log, enum bpf_prog_type prog_type)
6417	{
6418	const struct btf_member *kctx_member;
6419	const struct btf_type *conv_struct;
6420	const struct btf_type *kctx_type;
6421	u32 kctx_type_id;
6422
6423	conv_struct = bpf_ctx_convert.t;
6424	/* get member for kernel ctx type */
6425	kctx_member = btf_type_member(conv_struct) + bpf_ctx_convert_map[prog_type] * 2 + 1;
6426	kctx_type_id = kctx_member->type;
6427	kctx_type = btf_type_by_id(btf_vmlinux, kctx_type_id);
6428	if (!btf_type_is_struct(kctx_type)) {
6429	bpf_log(log, "kern ctx type id %u is not a struct\n", kctx_type_id);
6430	return -EINVAL;
6431	}
6432
6433	return kctx_type_id;
6434	}
6435
6436	BTF_ID_LIST_SINGLE(bpf_ctx_convert_btf_id, struct, bpf_ctx_convert)
6437
6438	static struct btf btf_parse_base(struct btf_verifier_env env, const char *name,
6439	void *data, unsigned int data_size)
6440	{
6441	struct btf *btf = NULL;
6442	int err;
6443
6444	if (!IS_ENABLED(CONFIG_DEBUG_INFO_BTF))
6445	return ERR_PTR(-ENOENT);
6446
6447	btf = kzalloc_obj(*btf, GFP_KERNEL \| __GFP_NOWARN);
6448	if (!btf) {
6449	err = -ENOMEM;
6450	goto errout;
6451	}
6452	env->btf = btf;
6453
6454	btf->data = data;
6455	btf->data_size = data_size;
6456	btf->kernel_btf = true;
6457	btf->named_start_id = 0;
6458	strscpy(btf->name, name);
6459
6460	err = btf_parse_hdr(env);
6461	if (err)
6462	goto errout;
6463
6464	btf->nohdr_data = btf->data + btf->hdr.hdr_len;
6465
6466	err = btf_parse_str_sec(env);
6467	if (err)
6468	goto errout;
6469
6470	err = btf_check_all_metas(env);
6471	if (err)
6472	goto errout;
6473
6474	err = btf_check_modifier_chain_length(env, btf, 1);
6475	if (err)
6476	goto errout;
6477
6478	btf_check_sorted(btf);
6479	refcount_set(&btf->refcnt, 1);
6480
6481	return btf;
6482
6483	errout:
6484	if (btf) {
6485	kvfree(btf->types);
6486	kfree(btf);
6487	}
6488	return ERR_PTR(err);
6489	}
6490
6491	struct btf *btf_parse_vmlinux(void)
6492	{
6493	struct btf_verifier_env *env = NULL;
6494	struct bpf_verifier_log *log;
6495	struct btf *btf;
6496	int err;
6497
6498	env = kzalloc_obj(*env, GFP_KERNEL \| __GFP_NOWARN);
6499	if (!env)
6500	return ERR_PTR(-ENOMEM);
6501
6502	log = &env->log;
6503	log->level = BPF_LOG_KERNEL;
6504	btf = btf_parse_base(env, "vmlinux", __start_BTF, __stop_BTF - __start_BTF);
6505	if (IS_ERR(btf))
6506	goto err_out;
6507
6508	/* btf_parse_vmlinux() runs under btf_vmlinux_lock */
6509	bpf_ctx_convert.t = btf_type_by_id(btf, bpf_ctx_convert_btf_id[0]);
6510	err = btf_alloc_id(btf);
6511	if (err) {
6512	btf_free(btf);
6513	btf = ERR_PTR(err);
6514	}
6515	err_out:
6516	btf_verifier_env_free(env);
6517	return btf;
6518	}
6519
6520	/* If .BTF_ids section was created with distilled base BTF, both base and
6521	* split BTF ids will need to be mapped to actual base/split ids for
6522	* BTF now that it has been relocated.
6523	*/
6524	__u32 btf_relocate_id(const struct btf *btf, __u32 id)
6525	{
6526	if (!btf->base_btf \|\| !btf->base_id_map)
6527	return id;
6528	return btf->base_id_map[id];
6529	}
6530
6531	#ifdef CONFIG_DEBUG_INFO_BTF_MODULES
6532
6533	static struct btf btf_parse_module(const char module_name, const void *data,
6534	unsigned int data_size, void *base_data,
6535	unsigned int base_data_size)
6536	{
6537	struct btf btf = NULL, vmlinux_btf, *base_btf = NULL;
6538	struct btf_verifier_env *env = NULL;
6539	struct bpf_verifier_log *log;
6540	int err = 0;
6541
6542	vmlinux_btf = bpf_get_btf_vmlinux();
6543	if (IS_ERR(vmlinux_btf))
6544	return vmlinux_btf;
6545	if (!vmlinux_btf)
6546	return ERR_PTR(-EINVAL);
6547
6548	env = kzalloc_obj(*env, GFP_KERNEL \| __GFP_NOWARN);
6549	if (!env)
6550	return ERR_PTR(-ENOMEM);
6551
6552	log = &env->log;
6553	log->level = BPF_LOG_KERNEL;
6554
6555	if (base_data) {
6556	base_btf = btf_parse_base(env, ".BTF.base", base_data, base_data_size);
6557	if (IS_ERR(base_btf)) {
6558	err = PTR_ERR(base_btf);
6559	goto errout;
6560	}
6561	} else {
6562	base_btf = vmlinux_btf;
6563	}
6564
6565	btf = kzalloc_obj(*btf, GFP_KERNEL \| __GFP_NOWARN);
6566	if (!btf) {
6567	err = -ENOMEM;
6568	goto errout;
6569	}
6570	env->btf = btf;
6571
6572	btf->base_btf = base_btf;
6573	btf->start_id = base_btf->nr_types;
6574	btf->start_str_off = base_btf->hdr.str_len;
6575	btf->kernel_btf = true;
6576	btf->named_start_id = 0;
6577	strscpy(btf->name, module_name);
6578
6579	btf->data = kvmemdup(data, data_size, GFP_KERNEL \| __GFP_NOWARN);
6580	if (!btf->data) {
6581	err = -ENOMEM;
6582	goto errout;
6583	}
6584	btf->data_size = data_size;
6585
6586	err = btf_parse_hdr(env);
6587	if (err)
6588	goto errout;
6589
6590	btf->nohdr_data = btf->data + btf->hdr.hdr_len;
6591
6592	err = btf_parse_str_sec(env);
6593	if (err)
6594	goto errout;
6595
6596	err = btf_check_all_metas(env);
6597	if (err)
6598	goto errout;
6599
6600	err = btf_check_modifier_chain_length(env, btf, btf_nr_types(base_btf));
6601	if (err)
6602	goto errout;
6603
6604	if (base_btf != vmlinux_btf) {
6605	err = btf_relocate(btf, vmlinux_btf, &btf->base_id_map);
6606	if (err)
6607	goto errout;
6608	btf_free(base_btf);
6609	base_btf = vmlinux_btf;
6610	}
6611
6612	btf_verifier_env_free(env);
6613	btf_check_sorted(btf);
6614	refcount_set(&btf->refcnt, 1);
6615	return btf;
6616
6617	errout:
6618	btf_verifier_env_free(env);
6619	if (!IS_ERR(base_btf) && base_btf != vmlinux_btf)
6620	btf_free(base_btf);
6621	if (btf) {
6622	kvfree(btf->data);
6623	kvfree(btf->types);
6624	kfree(btf);
6625	}
6626	return ERR_PTR(err);
6627	}
6628
6629	#endif /* CONFIG_DEBUG_INFO_BTF_MODULES */
6630
6631	struct btf bpf_prog_get_target_btf(const struct bpf_prog prog)
6632	{
6633	struct bpf_prog *tgt_prog = prog->aux->dst_prog;
6634
6635	if (tgt_prog)
6636	return tgt_prog->aux->btf;
6637	else
6638	return prog->aux->attach_btf;
6639	}
6640
6641	u32 btf_ctx_arg_idx(struct btf btf, const struct btf_type func_proto,
6642	int off)
6643	{
6644	const struct btf_param *args;
6645	const struct btf_type *t;
6646	u32 offset = 0, nr_args;
6647	int i;
6648
6649	if (!func_proto)
6650	return off / 8;
6651
6652	nr_args = btf_type_vlen(func_proto);
6653	args = (const struct btf_param *)(func_proto + 1);
6654	for (i = 0; i < nr_args; i++) {
6655	t = btf_type_skip_modifiers(btf, args[i].type, NULL);
6656	offset += btf_type_is_ptr(t) ? 8 : roundup(t->size, 8);
6657	if (off < offset)
6658	return i;
6659	}
6660
6661	t = btf_type_skip_modifiers(btf, func_proto->type, NULL);
6662	offset += btf_type_is_ptr(t) ? 8 : roundup(t->size, 8);
6663	if (off < offset)
6664	return nr_args;
6665
6666	return nr_args + 1;
6667	}
6668
6669	static bool prog_args_trusted(const struct bpf_prog *prog)
6670	{
6671	enum bpf_attach_type atype = prog->expected_attach_type;
6672
6673	switch (prog->type) {
6674	case BPF_PROG_TYPE_TRACING:
6675	return atype == BPF_TRACE_RAW_TP \|\| atype == BPF_TRACE_ITER;
6676	case BPF_PROG_TYPE_LSM:
6677	return bpf_lsm_is_trusted(prog);
6678	case BPF_PROG_TYPE_STRUCT_OPS:
6679	return true;
6680	default:
6681	return false;
6682	}
6683	}
6684
6685	int btf_ctx_arg_offset(const struct btf btf, const struct btf_type func_proto,
6686	u32 arg_no)
6687	{
6688	const struct btf_param *args;
6689	const struct btf_type *t;
6690	int off = 0, i;
6691	u32 sz;
6692
6693	args = btf_params(func_proto);
6694	for (i = 0; i < arg_no; i++) {
6695	t = btf_type_by_id(btf, args[i].type);
6696	t = btf_resolve_size(btf, t, &sz);
6697	if (IS_ERR(t))
6698	return PTR_ERR(t);
6699	off += roundup(sz, 8);
6700	}
6701
6702	return off;
6703	}
6704
6705	struct bpf_raw_tp_null_args {
6706	const char *func;
6707	u64 mask;
6708	};
6709
6710	static const struct bpf_raw_tp_null_args raw_tp_null_args[] = {
6711	/* sched */
6712	{ "sched_pi_setprio", 0x10 },
6713	/*
6714	* do_wait() passes NULL for wait4(-1) and waitid(P_ALL).
6715	*/
6716	{ "sched_process_wait", 0x1 },
6717	/* ... from sched_numa_pair_template event class */
6718	{ "sched_stick_numa", 0x100 },
6719	{ "sched_swap_numa", 0x100 },
6720	/* afs */
6721	{ "afs_make_fs_call", 0x10 },
6722	{ "afs_make_fs_calli", 0x10 },
6723	{ "afs_make_fs_call1", 0x10 },
6724	{ "afs_make_fs_call2", 0x10 },
6725	{ "afs_protocol_error", 0x1 },
6726	{ "afs_flock_ev", 0x10 },
6727	/* cachefiles */
6728	{ "cachefiles_lookup", 0x1 \| 0x200 },
6729	{ "cachefiles_unlink", 0x1 },
6730	{ "cachefiles_rename", 0x1 },
6731	{ "cachefiles_prep_read", 0x1 },
6732	{ "cachefiles_mark_active", 0x1 },
6733	{ "cachefiles_mark_failed", 0x1 },
6734	{ "cachefiles_mark_inactive", 0x1 },
6735	{ "cachefiles_vfs_error", 0x1 },
6736	{ "cachefiles_io_error", 0x1 },
6737	/* ext4, from ext4__mballoc event class */
6738	{ "ext4_mballoc_discard", 0x10 },
6739	{ "ext4_mballoc_free", 0x10 },
6740	/* fib */
6741	{ "fib_table_lookup", 0x100 },
6742	/* filelock */
6743	/* ... from filelock_lock event class */
6744	{ "posix_lock_inode", 0x10 },
6745	{ "fcntl_setlk", 0x10 },
6746	{ "locks_remove_posix", 0x10 },
6747	{ "flock_lock_inode", 0x10 },
6748	/* ... from filelock_lease event class */
6749	{ "break_lease_noblock", 0x10 },
6750	{ "break_lease_block", 0x10 },
6751	{ "break_lease_unblock", 0x10 },
6752	{ "generic_delete_lease", 0x10 },
6753	{ "time_out_leases", 0x10 },
6754	/* host1x */
6755	{ "host1x_cdma_push_gather", 0x10000 },
6756	/* huge_memory */
6757	{ "mm_khugepaged_scan_pmd", 0x10 },
6758	{ "mm_collapse_huge_page_isolate", 0x1 },
6759	{ "mm_khugepaged_scan_file", 0x10 },
6760	{ "mm_khugepaged_collapse_file", 0x10 },
6761	/* kmem */
6762	{ "mm_page_alloc", 0x1 },
6763	{ "mm_page_pcpu_drain", 0x1 },
6764	/* .. from mm_page event class */
6765	{ "mm_page_alloc_zone_locked", 0x1 },
6766	/* netfs */
6767	{ "netfs_failure", 0x10 },
6768	/* power */
6769	{ "device_pm_callback_start", 0x10 },
6770	/* qdisc */
6771	{ "qdisc_dequeue", 0x1000 },
6772	/* rxrpc */
6773	{ "rxrpc_recvdata", 0x1 },
6774	{ "rxrpc_resend", 0x10 },
6775	{ "rxrpc_tq", 0x10 },
6776	{ "rxrpc_client", 0x1 },
6777	/* signal */
6778	{ "signal_generate", 0x20 },
6779	{ "signal_deliver", 0x20 },
6780	/* skb */
6781	{"kfree_skb", 0x1000},
6782	/* sunrpc */
6783	{ "xs_stream_read_data", 0x1 },
6784	/* ... from xprt_cong_event event class */
6785	{ "xprt_reserve_cong", 0x10 },
6786	{ "xprt_release_cong", 0x10 },
6787	{ "xprt_get_cong", 0x10 },
6788	{ "xprt_put_cong", 0x10 },
6789	/* tcp */
6790	{ "tcp_send_reset", 0x11 },
6791	{ "tcp_sendmsg_locked", 0x100 },
6792	/* tegra_apb_dma */
6793	{ "tegra_dma_tx_status", 0x100 },
6794	/* timer_migration */
6795	{ "tmigr_update_events", 0x1 },
6796	/* writeback, from writeback_folio_template event class */
6797	{ "writeback_dirty_folio", 0x10 },
6798	{ "folio_wait_writeback", 0x10 },
6799	/* rdma */
6800	{ "mr_integ_alloc", 0x2000 },
6801	/* bpf_testmod */
6802	{ "bpf_testmod_test_read", 0x0 },
6803	/* amdgpu */
6804	{ "amdgpu_vm_bo_map", 0x1 },
6805	{ "amdgpu_vm_bo_unmap", 0x1 },
6806	/* netfs */
6807	{ "netfs_folioq", 0x1 },
6808	/* xfs from xfs_defer_pending_class */
6809	{ "xfs_defer_create_intent", 0x1 },
6810	{ "xfs_defer_cancel_list", 0x1 },
6811	{ "xfs_defer_pending_finish", 0x1 },
6812	{ "xfs_defer_pending_abort", 0x1 },
6813	{ "xfs_defer_relog_intent", 0x1 },
6814	{ "xfs_defer_isolate_paused", 0x1 },
6815	{ "xfs_defer_item_pause", 0x1 },
6816	{ "xfs_defer_item_unpause", 0x1 },
6817	/* xfs from xfs_defer_pending_item_class */
6818	{ "xfs_defer_add_item", 0x1 },
6819	{ "xfs_defer_cancel_item", 0x1 },
6820	{ "xfs_defer_finish_item", 0x1 },
6821	/* xfs from xfs_icwalk_class */
6822	{ "xfs_ioc_free_eofblocks", 0x10 },
6823	{ "xfs_blockgc_free_space", 0x10 },
6824	/* xfs from xfs_btree_cur_class */
6825	{ "xfs_btree_updkeys", 0x100 },
6826	{ "xfs_btree_overlapped_query_range", 0x100 },
6827	/* xfs from xfs_imap_class*/
6828	{ "xfs_map_blocks_found", 0x10000 },
6829	{ "xfs_map_blocks_alloc", 0x10000 },
6830	{ "xfs_iomap_alloc", 0x1000 },
6831	{ "xfs_iomap_found", 0x1000 },
6832	/* xfs from xfs_fs_class */
6833	{ "xfs_inodegc_flush", 0x1 },
6834	{ "xfs_inodegc_push", 0x1 },
6835	{ "xfs_inodegc_start", 0x1 },
6836	{ "xfs_inodegc_stop", 0x1 },
6837	{ "xfs_inodegc_queue", 0x1 },
6838	{ "xfs_inodegc_throttle", 0x1 },
6839	{ "xfs_fs_sync_fs", 0x1 },
6840	{ "xfs_blockgc_start", 0x1 },
6841	{ "xfs_blockgc_stop", 0x1 },
6842	{ "xfs_blockgc_worker", 0x1 },
6843	{ "xfs_blockgc_flush_all", 0x1 },
6844	/* xfs_scrub */
6845	{ "xchk_nlinks_live_update", 0x10 },
6846	/* xfs_scrub from xchk_metapath_class */
6847	{ "xchk_metapath_lookup", 0x100 },
6848	/* nfsd */
6849	{ "nfsd_dirent", 0x1 },
6850	{ "nfsd_file_acquire", 0x1001 },
6851	{ "nfsd_file_insert_err", 0x1 },
6852	{ "nfsd_file_cons_err", 0x1 },
6853	/* nfs4 */
6854	{ "nfs4_setup_sequence", 0x1 },
6855	{ "pnfs_update_layout", 0x10000 },
6856	{ "nfs4_inode_callback_event", 0x200 },
6857	{ "nfs4_inode_stateid_callback_event", 0x200 },
6858	/* nfs from pnfs_layout_event */
6859	{ "pnfs_mds_fallback_pg_init_read", 0x10000 },
6860	{ "pnfs_mds_fallback_pg_init_write", 0x10000 },
6861	{ "pnfs_mds_fallback_pg_get_mirror_count", 0x10000 },
6862	{ "pnfs_mds_fallback_read_done", 0x10000 },
6863	{ "pnfs_mds_fallback_write_done", 0x10000 },
6864	{ "pnfs_mds_fallback_read_pagelist", 0x10000 },
6865	{ "pnfs_mds_fallback_write_pagelist", 0x10000 },
6866	/* coda */
6867	{ "coda_dec_pic_run", 0x10 },
6868	{ "coda_dec_pic_done", 0x10 },
6869	/* cfg80211 */
6870	{ "cfg80211_scan_done", 0x11 },
6871	{ "rdev_set_coalesce", 0x10 },
6872	{ "cfg80211_report_wowlan_wakeup", 0x100 },
6873	{ "cfg80211_inform_bss_frame", 0x100 },
6874	{ "cfg80211_michael_mic_failure", 0x10000 },
6875	/* cfg80211 from wiphy_work_event */
6876	{ "wiphy_work_queue", 0x10 },
6877	{ "wiphy_work_run", 0x10 },
6878	{ "wiphy_work_cancel", 0x10 },
6879	{ "wiphy_work_flush", 0x10 },
6880	/* hugetlbfs */
6881	{ "hugetlbfs_alloc_inode", 0x10 },
6882	/* spufs */
6883	{ "spufs_context", 0x10 },
6884	/* kvm_hv */
6885	{ "kvm_page_fault_enter", 0x100 },
6886	/* dpu */
6887	{ "dpu_crtc_setup_mixer", 0x100 },
6888	/* binder */
6889	{ "binder_transaction", 0x100 },
6890	/* bcachefs */
6891	{ "btree_path_free", 0x100 },
6892	/* hfi1_tx */
6893	{ "hfi1_sdma_progress", 0x1000 },
6894	/* iptfs */
6895	{ "iptfs_ingress_postq_event", 0x1000 },
6896	/* neigh */
6897	{ "neigh_update", 0x10 },
6898	/* snd_firewire_lib */
6899	{ "amdtp_packet", 0x100 },
6900	};
6901
6902	bool btf_ctx_access(int off, int size, enum bpf_access_type type,
6903	const struct bpf_prog *prog,
6904	struct bpf_insn_access_aux *info)
6905	{
6906	static const struct btf_type_tag_match ctx_type_tags[] = {
6907	{ "user", MEM_USER },
6908	{ "percpu", MEM_PERCPU },
6909	};
6910	const struct btf_type *t = prog->aux->attach_func_proto;
6911	struct bpf_prog *tgt_prog = prog->aux->dst_prog;
6912	struct btf *btf = bpf_prog_get_target_btf(prog);
6913	const char *tname = prog->aux->attach_func_name;
6914	struct bpf_verifier_log *log = info->log;
6915	struct btf_type_tag_walk_ctx ctx;
6916	const struct btf_param *args;
6917	bool ptr_err_raw_tp = false;
6918	u32 nr_args, arg;
6919	int i, ret;
6920
6921	if (off % 8) {
6922	bpf_log(log, "func '%s' offset %d is not multiple of 8\n",
6923	tname, off);
6924	return false;
6925	}
6926	arg = btf_ctx_arg_idx(btf, t, off);
6927	args = (const struct btf_param *)(t + 1);
6928	/* if (t == NULL) Fall back to default BPF prog with
6929	* MAX_BPF_FUNC_REG_ARGS u64 arguments.
6930	*/
6931	nr_args = t ? btf_type_vlen(t) : MAX_BPF_FUNC_REG_ARGS;
6932	if (prog->aux->attach_btf_trace) {
6933	/* skip first 'void __data' argument in btf_trace_##name typedef /
6934	args++;
6935	nr_args--;
6936	}
6937
6938	if (arg > nr_args) {
6939	bpf_log(log, "func '%s' doesn't have %d-th argument\n",
6940	tname, arg + 1);
6941	return false;
6942	}
6943
6944	if (arg == nr_args) {
6945	switch (prog->expected_attach_type) {
6946	case BPF_LSM_MAC:
6947	/* mark we are accessing the return value */
6948	info->is_retval = true;
6949	fallthrough;
6950	case BPF_LSM_CGROUP:
6951	case BPF_TRACE_FEXIT:
6952	case BPF_TRACE_FSESSION:
6953	/* When LSM programs are attached to void LSM hooks
6954	* they use FEXIT trampolines and when attached to
6955	* int LSM hooks, they use MODIFY_RETURN trampolines.
6956	*
6957	* While the LSM programs are BPF_MODIFY_RETURN-like
6958	* the check:
6959	*
6960	* if (ret_type != 'int')
6961	* return -EINVAL;
6962	*
6963	* is _not_ done here. This is still safe as LSM hooks
6964	* have only void and int return types.
6965	*/
6966	if (!t)
6967	return true;
6968	t = btf_type_by_id(btf, t->type);
6969	break;
6970	case BPF_MODIFY_RETURN:
6971	/* For now the BPF_MODIFY_RETURN can only be attached to
6972	* functions that return an int.
6973	*/
6974	if (!t)
6975	return false;
6976
6977	t = btf_type_skip_modifiers(btf, t->type, NULL);
6978	if (!btf_type_is_small_int(t)) {
6979	bpf_log(log,
6980	"ret type %s not allowed for fmod_ret\n",
6981	btf_type_str(t));
6982	return false;
6983	}
6984	break;
6985	default:
6986	bpf_log(log, "func '%s' doesn't have %d-th argument\n",
6987	tname, arg + 1);
6988	return false;
6989	}
6990	} else {
6991	if (!t)
6992	/* Default prog with MAX_BPF_FUNC_REG_ARGS args */
6993	return true;
6994	t = btf_type_by_id(btf, args[arg].type);
6995	}
6996
6997	/* skip modifiers */
6998	while (btf_type_is_modifier(t))
6999	t = btf_type_by_id(btf, t->type);
7000	if (btf_type_is_small_int(t) \|\| btf_is_any_enum(t) \|\| btf_type_is_struct(t))
7001	/* accessing a scalar */
7002	return true;
7003	if (!btf_type_is_ptr(t)) {
7004	bpf_log(log,
7005	"func '%s' arg%d '%s' has type %s. Only pointer access is allowed\n",
7006	tname, arg,
7007	__btf_name_by_offset(btf, t->name_off),
7008	btf_type_str(t));
7009	return false;
7010	}
7011
7012	if (size != sizeof(u64)) {
7013	bpf_log(log, "func '%s' size %d must be 8\n",
7014	tname, size);
7015	return false;
7016	}
7017
7018	/*
7019	* Check for PTR_TO_RDONLY_BUF_OR_NULL, PTR_TO_RDWR_BUF_OR_NULL or
7020	* PTR_TO_ARENA (both nullable and non-nullable cases).
7021	*/
7022	for (i = 0; i < prog->aux->ctx_arg_info_size; i++) {
7023	const struct bpf_ctx_arg_aux *ctx_arg_info = &prog->aux->ctx_arg_info[i];
7024	u32 type, flag;
7025
7026	type = base_type(ctx_arg_info->reg_type);
7027	flag = type_flag(ctx_arg_info->reg_type);
7028	if (ctx_arg_info->offset == off &&
7029	(type == PTR_TO_ARENA \|\|
7030	(type == PTR_TO_BUF && (flag & PTR_MAYBE_NULL)))) {
7031	info->reg_type = ctx_arg_info->reg_type;
7032	return true;
7033	}
7034	}
7035
7036	/*
7037	* If it's a single or multilevel pointer, except a pointer
7038	* to a structure, it's the same as scalar from the verifier
7039	* safety POV. Multilevel pointers to structures are treated as
7040	* scalars. The verifier lacks the context to infer the size of
7041	* their target memory regions. Either way, no further pointer
7042	* walking is allowed.
7043	*/
7044	if (!btf_type_is_struct_ptr(btf, t))
7045	return true;
7046
7047	/* this is a pointer to another type */
7048	for (i = 0; i < prog->aux->ctx_arg_info_size; i++) {
7049	const struct bpf_ctx_arg_aux *ctx_arg_info = &prog->aux->ctx_arg_info[i];
7050
7051	if (ctx_arg_info->offset == off) {
7052	if (!ctx_arg_info->btf_id) {
7053	bpf_log(log,"invalid btf_id for context argument offset %u\n", off);
7054	return false;
7055	}
7056
7057	info->reg_type = ctx_arg_info->reg_type;
7058	info->btf = ctx_arg_info->btf ? : btf_vmlinux;
7059	info->btf_id = ctx_arg_info->btf_id;
7060	info->ref_id = ctx_arg_info->ref_id;
7061	return true;
7062	}
7063	}
7064
7065	info->reg_type = PTR_TO_BTF_ID;
7066	if (prog_args_trusted(prog))
7067	info->reg_type \|= PTR_TRUSTED;
7068
7069	if (btf_param_match_suffix(btf, &args[arg], "__nullable"))
7070	info->reg_type \|= PTR_MAYBE_NULL;
7071
7072	if (prog->expected_attach_type == BPF_TRACE_RAW_TP) {
7073	struct btf *btf = prog->aux->attach_btf;
7074	const struct btf_type *t;
7075	const char *tname;
7076
7077	/* BTF lookups cannot fail, return false on error */
7078	t = btf_type_by_id(btf, prog->aux->attach_btf_id);
7079	if (!t)
7080	return false;
7081	tname = btf_name_by_offset(btf, t->name_off);
7082	if (!tname)
7083	return false;
7084	/* Checked by bpf_check_attach_target */
7085	tname += sizeof("btf_trace_") - 1;
7086	for (i = 0; i < ARRAY_SIZE(raw_tp_null_args); i++) {
7087	/* Is this a func with potential NULL args? */
7088	if (strcmp(tname, raw_tp_null_args[i].func))
7089	continue;
7090	if (raw_tp_null_args[i].mask & (0x1ULL << (arg * 4)))
7091	info->reg_type \|= PTR_MAYBE_NULL;
7092	/* Is the current arg IS_ERR? */
7093	if (raw_tp_null_args[i].mask & (0x2ULL << (arg * 4)))
7094	ptr_err_raw_tp = true;
7095	break;
7096	}
7097	/* If we don't know NULL-ness specification and the tracepoint
7098	* is coming from a loadable module, be conservative and mark
7099	* argument as PTR_MAYBE_NULL.
7100	*/
7101	if (i == ARRAY_SIZE(raw_tp_null_args) && btf_is_module(btf))
7102	info->reg_type \|= PTR_MAYBE_NULL;
7103	}
7104
7105	if (tgt_prog) {
7106	enum bpf_prog_type tgt_type;
7107
7108	if (tgt_prog->type == BPF_PROG_TYPE_EXT)
7109	tgt_type = tgt_prog->aux->saved_dst_prog_type;
7110	else
7111	tgt_type = tgt_prog->type;
7112
7113	ret = btf_translate_to_vmlinux(log, btf, t, tgt_type, arg);
7114	if (ret > 0) {
7115	info->btf = btf_vmlinux;
7116	info->btf_id = ret;
7117	return true;
7118	} else {
7119	return false;
7120	}
7121	}
7122
7123	info->btf = btf;
7124	ctx.t = t;
7125	ret = btf_type_tag_walk(btf, &ctx, ctx_type_tags,
7126	ARRAY_SIZE(ctx_type_tags));
7127	if (ret) {
7128	bpf_log(log, "func '%s' arg%d type %s has multiple type tags\n",
7129	tname, arg, btf_type_str(t));
7130	return false;
7131	}
7132	info->reg_type \|= ctx.res;
7133	info->btf_id = ctx.id;
7134	t = ctx.t;
7135
7136	if (!btf_type_is_struct(t)) {
7137	bpf_log(log,
7138	"func '%s' arg%d type %s is not a struct\n",
7139	tname, arg, btf_type_str(t));
7140	return false;
7141	}
7142	bpf_log(log, "func '%s' arg%d has btf_id %d type %s '%s'\n",
7143	tname, arg, info->btf_id, btf_type_str(t),
7144	__btf_name_by_offset(btf, t->name_off));
7145
7146	/* Perform all checks on the validity of type for this argument, but if
7147	* we know it can be IS_ERR at runtime, scrub pointer type and mark as
7148	* scalar.
7149	*/
7150	if (ptr_err_raw_tp) {
7151	bpf_log(log, "marking pointer arg%d as scalar as it may encode error", arg);
7152	info->reg_type = SCALAR_VALUE;
7153	}
7154	return true;
7155	}
7156	EXPORT_SYMBOL_GPL(btf_ctx_access);
7157
7158	enum bpf_struct_walk_result {
7159	/* < 0 error */
7160	WALK_SCALAR = 0,
7161	WALK_PTR,
7162	WALK_PTR_UNTRUSTED,
7163	WALK_STRUCT,
7164	};
7165
7166	static int btf_struct_walk(struct bpf_verifier_log log, const struct btf btf,
7167	const struct btf_type *t, int off, int size,
7168	u32 next_btf_id, enum bpf_type_flag flag,
7169	const char **field_name, bool walk_flex_arrays)
7170	{
7171	u32 i, moff, mtrue_end, msize = 0, total_nelems = 0;
7172	const struct btf_type mtype, elem_type = NULL;
7173	const struct btf_member *member;
7174	const char tname, mname;
7175	u32 vlen, elem_id, mid;
7176
7177	again:
7178	if (btf_type_is_modifier(t))
7179	t = btf_type_skip_modifiers(btf, t->type, NULL);
7180	tname = __btf_name_by_offset(btf, t->name_off);
7181	if (!btf_type_is_struct(t)) {
7182	bpf_log(log, "Type '%s' is not a struct\n", tname);
7183	return -EINVAL;
7184	}
7185
7186	vlen = btf_type_vlen(t);
7187	if (BTF_INFO_KIND(t->info) == BTF_KIND_UNION && vlen != 1 && !(*flag & PTR_UNTRUSTED))
7188	/*
7189	* walking unions yields untrusted pointers
7190	* with exception of __bpf_md_ptr and other
7191	* unions with a single member
7192	*/
7193	*flag \|= PTR_UNTRUSTED;
7194
7195	if (off + size > t->size) {
7196	struct btf_array *array_elem;
7197
7198	if (!walk_flex_arrays)
7199	goto error;
7200
7201	/* If the last element is a variable size array, we may
7202	* need to relax the rule.
7203	*/
7204	if (vlen == 0)
7205	goto error;
7206
7207	member = btf_type_member(t) + vlen - 1;
7208	mtype = btf_type_skip_modifiers(btf, member->type,
7209	NULL);
7210	if (!btf_type_is_array(mtype))
7211	goto error;
7212
7213	array_elem = (struct btf_array *)(mtype + 1);
7214	if (array_elem->nelems != 0)
7215	goto error;
7216
7217	moff = __btf_member_bit_offset(t, member) / 8;
7218	if (off < moff)
7219	goto error;
7220
7221	/* allow structure and integer */
7222	t = btf_type_skip_modifiers(btf, array_elem->type,
7223	NULL);
7224
7225	if (btf_type_is_int(t))
7226	return WALK_SCALAR;
7227
7228	if (!btf_type_is_struct(t) \|\| !t->size)
7229	goto error;
7230
7231	off = (off - moff) % t->size;
7232	goto again;
7233
7234	error:
7235	bpf_log(log, "access beyond struct %s at off %u size %u\n",
7236	tname, off, size);
7237	return -EACCES;
7238	}
7239
7240	for_each_member(i, t, member) {
7241	/* offset of the field in bytes */
7242	moff = __btf_member_bit_offset(t, member) / 8;
7243	if (off + size <= moff)
7244	/* won't find anything, field is already too far */
7245	break;
7246
7247	if (__btf_member_bitfield_size(t, member)) {
7248	u32 end_bit = __btf_member_bit_offset(t, member) +
7249	__btf_member_bitfield_size(t, member);
7250
7251	/* off <= moff instead of off == moff because clang
7252	* does not generate a BTF member for anonymous
7253	* bitfield like the ":16" here:
7254	* struct {
7255	* int :16;
7256	* int x:8;
7257	* };
7258	*/
7259	if (off <= moff &&
7260	BITS_ROUNDUP_BYTES(end_bit) <= off + size)
7261	return WALK_SCALAR;
7262
7263	/* off may be accessing a following member
7264	*
7265	* or
7266	*
7267	* Doing partial access at either end of this
7268	* bitfield. Continue on this case also to
7269	* treat it as not accessing this bitfield
7270	* and eventually error out as field not
7271	* found to keep it simple.
7272	* It could be relaxed if there was a legit
7273	* partial access case later.
7274	*/
7275	continue;
7276	}
7277
7278	/* In case of "off" is pointing to holes of a struct */
7279	if (off < moff)
7280	break;
7281
7282	/* type of the field */
7283	mid = member->type;
7284	mtype = btf_type_by_id(btf, member->type);
7285	mname = __btf_name_by_offset(btf, member->name_off);
7286
7287	mtype = __btf_resolve_size(btf, mtype, &msize,
7288	&elem_type, &elem_id, &total_nelems,
7289	&mid);
7290	if (IS_ERR(mtype)) {
7291	bpf_log(log, "field %s doesn't have size\n", mname);
7292	return -EFAULT;
7293	}
7294
7295	mtrue_end = moff + msize;
7296	if (off >= mtrue_end)
7297	/* no overlap with member, keep iterating */
7298	continue;
7299
7300	if (btf_type_is_array(mtype)) {
7301	u32 elem_idx;
7302
7303	/* __btf_resolve_size() above helps to
7304	* linearize a multi-dimensional array.
7305	*
7306	* The logic here is treating an array
7307	* in a struct as the following way:
7308	*
7309	* struct outer {
7310	* struct inner array[2][2];
7311	* };
7312	*
7313	* looks like:
7314	*
7315	* struct outer {
7316	* struct inner array_elem0;
7317	* struct inner array_elem1;
7318	* struct inner array_elem2;
7319	* struct inner array_elem3;
7320	* };
7321	*
7322	* When accessing outer->array[1][0], it moves
7323	* moff to "array_elem2", set mtype to
7324	* "struct inner", and msize also becomes
7325	* sizeof(struct inner). Then most of the
7326	* remaining logic will fall through without
7327	* caring the current member is an array or
7328	* not.
7329	*
7330	* Unlike mtype/msize/moff, mtrue_end does not
7331	* change. The naming difference ("_true") tells
7332	* that it is not always corresponding to
7333	* the current mtype/msize/moff.
7334	* It is the true end of the current
7335	* member (i.e. array in this case). That
7336	* will allow an int array to be accessed like
7337	* a scratch space,
7338	* i.e. allow access beyond the size of
7339	* the array's element as long as it is
7340	* within the mtrue_end boundary.
7341	*/
7342
7343	/* skip empty array */
7344	if (moff == mtrue_end)
7345	continue;
7346
7347	msize /= total_nelems;
7348	elem_idx = (off - moff) / msize;
7349	moff += elem_idx * msize;
7350	mtype = elem_type;
7351	mid = elem_id;
7352	}
7353
7354	/* the 'off' we're looking for is either equal to start
7355	* of this field or inside of this struct
7356	*/
7357	if (btf_type_is_struct(mtype)) {
7358	/* our field must be inside that union or struct */
7359	t = mtype;
7360
7361	/* return if the offset matches the member offset */
7362	if (off == moff) {
7363	*next_btf_id = mid;
7364	return WALK_STRUCT;
7365	}
7366
7367	/* adjust offset we're looking for */
7368	off -= moff;
7369	goto again;
7370	}
7371
7372	if (btf_type_is_ptr(mtype)) {
7373	static const struct btf_type_tag_match walk_type_tags[] = {
7374	{ "user", MEM_USER },
7375	{ "percpu", MEM_PERCPU },
7376	{ "rcu", MEM_RCU },
7377	};
7378	enum bpf_type_flag tmp_flag = 0;
7379	struct btf_type_tag_walk_ctx ctx = { .t = mtype };
7380	const struct btf_type *stype;
7381	int err;
7382	u32 id;
7383
7384	if (msize != size \|\| off != moff) {
7385	bpf_log(log,
7386	"cannot access ptr member %s with moff %u in struct %s with off %u size %u\n",
7387	mname, moff, tname, off, size);
7388	return -EACCES;
7389	}
7390
7391	err = btf_type_tag_walk(btf, &ctx, walk_type_tags,
7392	ARRAY_SIZE(walk_type_tags));
7393	if (err) {
7394	bpf_log(log, "type '%s' has multiple type tags\n",
7395	btf_type_str(mtype));
7396	return err;
7397	}
7398	tmp_flag = ctx.res;
7399	id = ctx.id;
7400	stype = ctx.t;
7401
7402	if (btf_type_is_struct(stype)) {
7403	*next_btf_id = id;
7404	*flag \|= tmp_flag;
7405	if (field_name)
7406	*field_name = mname;
7407	return WALK_PTR;
7408	}
7409
7410	return WALK_PTR_UNTRUSTED;
7411	}
7412
7413	/* Allow more flexible access within an int as long as
7414	* it is within mtrue_end.
7415	* Since mtrue_end could be the end of an array,
7416	* that also allows using an array of int as a scratch
7417	* space. e.g. skb->cb[].
7418	*/
7419	if (off + size > mtrue_end && !(*flag & PTR_UNTRUSTED)) {
7420	bpf_log(log,
7421	"access beyond the end of member %s (mend:%u) in struct %s with off %u size %u\n",
7422	mname, mtrue_end, tname, off, size);
7423	return -EACCES;
7424	}
7425
7426	return WALK_SCALAR;
7427	}
7428	bpf_log(log, "struct %s doesn't have field at offset %d\n", tname, off);
7429	return -EINVAL;
7430	}
7431
7432	int btf_struct_access(struct bpf_verifier_log *log,
7433	const struct bpf_reg_state *reg,
7434	int off, int size, enum bpf_access_type atype __maybe_unused,
7435	u32 next_btf_id, enum bpf_type_flag flag,
7436	const char **field_name)
7437	{
7438	const struct btf *btf = reg->btf;
7439	enum bpf_type_flag tmp_flag = 0;
7440	const struct btf_type *t;
7441	u32 id = reg->btf_id;
7442	int err;
7443
7444	while (type_is_alloc(reg->type)) {
7445	struct btf_struct_meta *meta;
7446	struct btf_record *rec;
7447	int i;
7448
7449	meta = btf_find_struct_meta(btf, id);
7450	if (!meta)
7451	break;
7452	rec = meta->record;
7453	for (i = 0; i < rec->cnt; i++) {
7454	struct btf_field *field = &rec->fields[i];
7455	u32 offset = field->offset;
7456	if (off < offset + field->size && offset < off + size) {
7457	bpf_log(log,
7458	"direct access to %s is disallowed\n",
7459	btf_field_type_name(field->type));
7460	return -EACCES;
7461	}
7462	}
7463	break;
7464	}
7465
7466	t = btf_type_by_id(btf, id);
7467	do {
7468	err = btf_struct_walk(log, btf, t, off, size, &id, &tmp_flag,
7469	field_name, !type_is_alloc(reg->type));
7470
7471	switch (err) {
7472	case WALK_PTR:
7473	/* For local types, the destination register cannot
7474	* become a pointer again.
7475	*/
7476	if (type_is_alloc(reg->type))
7477	return SCALAR_VALUE;
7478	/* If we found the pointer or scalar on t+off,
7479	* we're done.
7480	*/
7481	*next_btf_id = id;
7482	*flag = tmp_flag;
7483	return PTR_TO_BTF_ID;
7484	case WALK_PTR_UNTRUSTED:
7485	*flag = MEM_RDONLY \| PTR_UNTRUSTED;
7486	return PTR_TO_MEM;
7487	case WALK_SCALAR:
7488	return SCALAR_VALUE;
7489	case WALK_STRUCT:
7490	/* We found nested struct, so continue the search
7491	* by diving in it. At this point the offset is
7492	* aligned with the new type, so set it to 0.
7493	*/
7494	t = btf_type_by_id(btf, id);
7495	off = 0;
7496	break;
7497	default:
7498	/* It's either error or unknown return value..
7499	* scream and leave.
7500	*/
7501	if (WARN_ONCE(err > 0, "unknown btf_struct_walk return value"))
7502	return -EINVAL;
7503	return err;
7504	}
7505	} while (t);
7506
7507	return -EINVAL;
7508	}
7509
7510	/* Check that two BTF types, each specified as an BTF object + id, are exactly
7511	* the same. Trivial ID check is not enough due to module BTFs, because we can
7512	* end up with two different module BTFs, but IDs point to the common type in
7513	* vmlinux BTF.
7514	*/
7515	bool btf_types_are_same(const struct btf *btf1, u32 id1,
7516	const struct btf *btf2, u32 id2)
7517	{
7518	if (id1 != id2)
7519	return false;
7520	if (btf1 == btf2)
7521	return true;
7522	return btf_type_by_id(btf1, id1) == btf_type_by_id(btf2, id2);
7523	}
7524
7525	bool btf_struct_ids_match(struct bpf_verifier_log *log,
7526	const struct btf *btf, u32 id, int off,
7527	const struct btf *need_btf, u32 need_type_id,
7528	bool strict, bool walk_flex_arrays)
7529	{
7530	const struct btf_type *type;
7531	enum bpf_type_flag flag = 0;
7532	int err;
7533
7534	/* Are we already done? */
7535	if (off == 0 && btf_types_are_same(btf, id, need_btf, need_type_id))
7536	return true;
7537	/* In case of strict type match, we do not walk struct, the top level
7538	* type match must succeed. When strict is true, off should have already
7539	* been 0.
7540	*/
7541	if (strict)
7542	return false;
7543	again:
7544	type = btf_type_by_id(btf, id);
7545	if (!type)
7546	return false;
7547	err = btf_struct_walk(log, btf, type, off, 1, &id, &flag, NULL,
7548	walk_flex_arrays);
7549	if (err != WALK_STRUCT)
7550	return false;
7551
7552	/* We found nested struct object. If it matches
7553	* the requested ID, we're done. Otherwise let's
7554	* continue the search with offset 0 in the new
7555	* type.
7556	*/
7557	if (!btf_types_are_same(btf, id, need_btf, need_type_id)) {
7558	off = 0;
7559	goto again;
7560	}
7561
7562	return true;
7563	}
7564
7565	static int __get_type_size(struct btf *btf, u32 btf_id,
7566	const struct btf_type **ret_type)
7567	{
7568	const struct btf_type *t;
7569
7570	*ret_type = btf_type_by_id(btf, 0);
7571	if (!btf_id)
7572	/* void */
7573	return 0;
7574	t = btf_type_by_id(btf, btf_id);
7575	while (t && btf_type_is_modifier(t))
7576	t = btf_type_by_id(btf, t->type);
7577	if (!t)
7578	return -EINVAL;
7579	*ret_type = t;
7580	if (btf_type_is_ptr(t))
7581	/* kernel size of pointer. Not BPF's size of pointer*/
7582	return sizeof(void *);
7583	if (btf_type_is_int(t) \|\| btf_is_any_enum(t) \|\| btf_type_is_struct(t))
7584	return t->size;
7585	return -EINVAL;
7586	}
7587
7588	static u8 __get_type_fmodel_flags(const struct btf_type *t)
7589	{
7590	u8 flags = 0;
7591
7592	if (btf_type_is_signed_int(t))
7593	flags \|= BTF_FMODEL_SIGNED_ARG;
7594
7595	return flags;
7596	}
7597
7598	static u8 __get_arg_fmodel_flags(const struct btf *btf,
7599	const struct btf_param *arg,
7600	const struct btf_type *t)
7601	{
7602	u8 flags = __get_type_fmodel_flags(t);
7603
7604	if (btf_param_match_suffix(btf, arg, "__arena__nullable"))
7605	flags \|= BTF_FMODEL_ARENA_ARG \| BTF_FMODEL_NULLABLE_ARG;
7606	else if (btf_param_match_suffix(btf, arg, "__arena"))
7607	flags \|= BTF_FMODEL_ARENA_ARG;
7608	else if (btf_param_match_suffix(btf, arg, "__nullable"))
7609	flags \|= BTF_FMODEL_NULLABLE_ARG;
7610
7611	return flags;
7612	}
7613
7614	int btf_distill_func_proto(struct bpf_verifier_log *log,
7615	struct btf *btf,
7616	const struct btf_type *func,
7617	const char *tname,
7618	struct btf_func_model *m)
7619	{
7620	const struct btf_param *args;
7621	const struct btf_type *t;
7622	u32 i, nargs;
7623	int ret;
7624
7625	if (!func) {
7626	/* BTF function prototype doesn't match the verifier types.
7627	* Fall back to MAX_BPF_FUNC_REG_ARGS u64 args.
7628	*/
7629	for (i = 0; i < MAX_BPF_FUNC_REG_ARGS; i++) {
7630	m->arg_size[i] = 8;
7631	m->arg_flags[i] = 0;
7632	}
7633	m->ret_size = 8;
7634	m->ret_flags = 0;
7635	m->nr_args = MAX_BPF_FUNC_REG_ARGS;
7636	return 0;
7637	}
7638	args = (const struct btf_param *)(func + 1);
7639	nargs = btf_type_vlen(func);
7640	if (nargs > MAX_BPF_FUNC_ARGS) {
7641	bpf_log(log,
7642	"The function %s has %d arguments. Too many.\n",
7643	tname, nargs);
7644	return -EINVAL;
7645	}
7646	ret = __get_type_size(btf, func->type, &t);
7647	if (ret < 0 \|\| btf_type_is_struct(t)) {
7648	bpf_log(log,
7649	"The function %s return type %s is unsupported.\n",
7650	tname, btf_type_str(t));
7651	return -EINVAL;
7652	}
7653	m->ret_size = ret;
7654	m->ret_flags = __get_type_fmodel_flags(t);
7655
7656	for (i = 0; i < nargs; i++) {
7657	if (i == nargs - 1 && args[i].type == 0) {
7658	bpf_log(log,
7659	"The function %s with variable args is unsupported.\n",
7660	tname);
7661	return -EINVAL;
7662	}
7663	ret = __get_type_size(btf, args[i].type, &t);
7664
7665	/* No support of struct argument size greater than 16 bytes */
7666	if (ret < 0 \|\| ret > 16) {
7667	bpf_log(log,
7668	"The function %s arg%d type %s is unsupported.\n",
7669	tname, i, btf_type_str(t));
7670	return -EINVAL;
7671	}
7672	if (ret == 0) {
7673	bpf_log(log,
7674	"The function %s has malformed void argument.\n",
7675	tname);
7676	return -EINVAL;
7677	}
7678	m->arg_size[i] = ret;
7679	m->arg_flags[i] = __get_arg_fmodel_flags(btf, &args[i], t);
7680	}
7681	m->nr_args = nargs;
7682	return 0;
7683	}
7684
7685	/* Compare BTFs of two functions assuming only scalars and pointers to context.
7686	* t1 points to BTF_KIND_FUNC in btf1
7687	* t2 points to BTF_KIND_FUNC in btf2
7688	* Returns:
7689	* EINVAL - function prototype mismatch
7690	* EFAULT - verifier bug
7691	* 0 - 99% match. The last 1% is validated by the verifier.
7692	*/
7693	static int btf_check_func_type_match(struct bpf_verifier_log *log,
7694	struct btf btf1, const struct btf_type t1,
7695	struct btf btf2, const struct btf_type t2)
7696	{
7697	const struct btf_param args1, args2;
7698	const char fn1, fn2, s1, s2;
7699	u32 nargs1, nargs2, i;
7700
7701	fn1 = btf_name_by_offset(btf1, t1->name_off);
7702	fn2 = btf_name_by_offset(btf2, t2->name_off);
7703
7704	if (btf_func_linkage(t1) != BTF_FUNC_GLOBAL) {
7705	bpf_log(log, "%s() is not a global function\n", fn1);
7706	return -EINVAL;
7707	}
7708	if (btf_func_linkage(t2) != BTF_FUNC_GLOBAL) {
7709	bpf_log(log, "%s() is not a global function\n", fn2);
7710	return -EINVAL;
7711	}
7712
7713	t1 = btf_type_by_id(btf1, t1->type);
7714	if (!t1 \|\| !btf_type_is_func_proto(t1))
7715	return -EFAULT;
7716	t2 = btf_type_by_id(btf2, t2->type);
7717	if (!t2 \|\| !btf_type_is_func_proto(t2))
7718	return -EFAULT;
7719
7720	args1 = (const struct btf_param *)(t1 + 1);
7721	nargs1 = btf_type_vlen(t1);
7722	args2 = (const struct btf_param *)(t2 + 1);
7723	nargs2 = btf_type_vlen(t2);
7724
7725	if (nargs1 != nargs2) {
7726	bpf_log(log, "%s() has %d args while %s() has %d args\n",
7727	fn1, nargs1, fn2, nargs2);
7728	return -EINVAL;
7729	}
7730
7731	t1 = btf_type_skip_modifiers(btf1, t1->type, NULL);
7732	t2 = btf_type_skip_modifiers(btf2, t2->type, NULL);
7733	if (t1->info != t2->info) {
7734	bpf_log(log,
7735	"Return type %s of %s() doesn't match type %s of %s()\n",
7736	btf_type_str(t1), fn1,
7737	btf_type_str(t2), fn2);
7738	return -EINVAL;
7739	}
7740
7741	for (i = 0; i < nargs1; i++) {
7742	t1 = btf_type_skip_modifiers(btf1, args1[i].type, NULL);
7743	t2 = btf_type_skip_modifiers(btf2, args2[i].type, NULL);
7744
7745	if (t1->info != t2->info) {
7746	bpf_log(log, "arg%d in %s() is %s while %s() has %s\n",
7747	i, fn1, btf_type_str(t1),
7748	fn2, btf_type_str(t2));
7749	return -EINVAL;
7750	}
7751	if (btf_type_has_size(t1) && t1->size != t2->size) {
7752	bpf_log(log,
7753	"arg%d in %s() has size %d while %s() has %d\n",
7754	i, fn1, t1->size,
7755	fn2, t2->size);
7756	return -EINVAL;
7757	}
7758
7759	/* global functions are validated with scalars and pointers
7760	* to context only. And only global functions can be replaced.
7761	* Hence type check only those types.
7762	*/
7763	if (btf_type_is_int(t1) \|\| btf_is_any_enum(t1))
7764	continue;
7765	if (!btf_type_is_ptr(t1)) {
7766	bpf_log(log,
7767	"arg%d in %s() has unrecognized type\n",
7768	i, fn1);
7769	return -EINVAL;
7770	}
7771	t1 = btf_type_skip_modifiers(btf1, t1->type, NULL);
7772	t2 = btf_type_skip_modifiers(btf2, t2->type, NULL);
7773	if (!btf_type_is_struct(t1)) {
7774	bpf_log(log,
7775	"arg%d in %s() is not a pointer to context\n",
7776	i, fn1);
7777	return -EINVAL;
7778	}
7779	if (!btf_type_is_struct(t2)) {
7780	bpf_log(log,
7781	"arg%d in %s() is not a pointer to context\n",
7782	i, fn2);
7783	return -EINVAL;
7784	}
7785	/* This is an optional check to make program writing easier.
7786	* Compare names of structs and report an error to the user.
7787	* btf_prepare_func_args() already checked that t2 struct
7788	* is a context type. btf_prepare_func_args() will check
7789	* later that t1 struct is a context type as well.
7790	*/
7791	s1 = btf_name_by_offset(btf1, t1->name_off);
7792	s2 = btf_name_by_offset(btf2, t2->name_off);
7793	if (strcmp(s1, s2)) {
7794	bpf_log(log,
7795	"arg%d %s(struct %s ) doesn't match %s(struct %s )\n",
7796	i, fn1, s1, fn2, s2);
7797	return -EINVAL;
7798	}
7799	}
7800	return 0;
7801	}
7802
7803	/* Compare BTFs of given program with BTF of target program */
7804	int btf_check_type_match(struct bpf_verifier_log log, const struct bpf_prog prog,
7805	struct btf btf2, const struct btf_type t2)
7806	{
7807	struct btf *btf1 = prog->aux->btf;
7808	const struct btf_type *t1;
7809	u32 btf_id = 0;
7810
7811	if (!prog->aux->func_info) {
7812	bpf_log(log, "Program extension requires BTF\n");
7813	return -EINVAL;
7814	}
7815
7816	btf_id = prog->aux->func_info[0].type_id;
7817	if (!btf_id)
7818	return -EFAULT;
7819
7820	t1 = btf_type_by_id(btf1, btf_id);
7821	if (!t1 \|\| !btf_type_is_func(t1))
7822	return -EFAULT;
7823
7824	return btf_check_func_type_match(log, btf1, t1, btf2, t2);
7825	}
7826
7827	static bool btf_is_dynptr_ptr(const struct btf btf, const struct btf_type t)
7828	{
7829	const char *name;
7830
7831	t = btf_type_by_id(btf, t->type); /* skip PTR */
7832
7833	while (btf_type_is_modifier(t))
7834	t = btf_type_by_id(btf, t->type);
7835
7836	/* allow either struct or struct forward declaration */
7837	if (btf_type_is_struct(t) \|\|
7838	(btf_type_is_fwd(t) && btf_type_kflag(t) == 0)) {
7839	name = btf_str_by_offset(btf, t->name_off);
7840	return name && strcmp(name, "bpf_dynptr") == 0;
7841	}
7842
7843	return false;
7844	}
7845
7846	struct bpf_cand_cache {
7847	const char *name;
7848	u32 name_len;
7849	u16 kind;
7850	u16 cnt;
7851	struct {
7852	const struct btf *btf;
7853	u32 id;
7854	} cands[];
7855	};
7856
7857	static DEFINE_MUTEX(cand_cache_mutex);
7858
7859	static struct bpf_cand_cache *
7860	bpf_core_find_cands(struct bpf_core_ctx *ctx, u32 local_type_id);
7861
7862	static int btf_get_ptr_to_btf_id(struct bpf_verifier_log *log, int arg_idx,
7863	const struct btf btf, const struct btf_type t)
7864	{
7865	struct bpf_cand_cache *cc;
7866	struct bpf_core_ctx ctx = {
7867	.btf = btf,
7868	.log = log,
7869	};
7870	u32 kern_type_id, type_id;
7871	int err = 0;
7872
7873	/* skip PTR and modifiers */
7874	type_id = t->type;
7875	t = btf_type_by_id(btf, t->type);
7876	while (btf_type_is_modifier(t)) {
7877	type_id = t->type;
7878	t = btf_type_by_id(btf, t->type);
7879	}
7880
7881	mutex_lock(&cand_cache_mutex);
7882	cc = bpf_core_find_cands(&ctx, type_id);
7883	if (IS_ERR(cc)) {
7884	err = PTR_ERR(cc);
7885	bpf_log(log, "arg#%d reference type('%s %s') candidate matching error: %d\n",
7886	arg_idx, btf_type_str(t), __btf_name_by_offset(btf, t->name_off),
7887	err);
7888	goto cand_cache_unlock;
7889	}
7890	if (cc->cnt != 1) {
7891	bpf_log(log, "arg#%d reference type('%s %s') %s\n",
7892	arg_idx, btf_type_str(t), __btf_name_by_offset(btf, t->name_off),
7893	cc->cnt == 0 ? "has no matches" : "is ambiguous");
7894	err = cc->cnt == 0 ? -ENOENT : -ESRCH;
7895	goto cand_cache_unlock;
7896	}
7897	if (btf_is_module(cc->cands[0].btf)) {
7898	bpf_log(log, "arg#%d reference type('%s %s') points to kernel module type (unsupported)\n",
7899	arg_idx, btf_type_str(t), __btf_name_by_offset(btf, t->name_off));
7900	err = -EOPNOTSUPP;
7901	goto cand_cache_unlock;
7902	}
7903	kern_type_id = cc->cands[0].id;
7904
7905	cand_cache_unlock:
7906	mutex_unlock(&cand_cache_mutex);
7907	if (err)
7908	return err;
7909
7910	return kern_type_id;
7911	}
7912
7913	enum btf_arg_tag {
7914	ARG_TAG_CTX = BIT_ULL(0),
7915	ARG_TAG_NONNULL = BIT_ULL(1),
7916	ARG_TAG_TRUSTED = BIT_ULL(2),
7917	ARG_TAG_UNTRUSTED = BIT_ULL(3),
7918	ARG_TAG_NULLABLE = BIT_ULL(4),
7919	ARG_TAG_ARENA = BIT_ULL(5),
7920	};
7921
7922	static int btf_scan_decl_tags(struct bpf_verifier_env *env,
7923	const struct btf *btf,
7924	const struct btf_type *fn_t,
7925	u32 arg_idx, bool is_global, u32 *tags)
7926	{
7927	int id = btf_named_start_id(btf, false) - 1;
7928	const char tag_key[] = "arg:";
7929	static const struct {
7930	const char *tag_value;
7931	enum btf_arg_tag arg_tag;
7932	} tag_values[] = {
7933	{ "ctx", ARG_TAG_CTX },
7934	{ "trusted", ARG_TAG_TRUSTED },
7935	{ "untrusted", ARG_TAG_UNTRUSTED },
7936	{ "nonnull", ARG_TAG_NONNULL },
7937	{ "nullable", ARG_TAG_NULLABLE },
7938	{ "arena", ARG_TAG_ARENA },
7939	};
7940
7941	/*
7942	* The 'arg:<tag>' decl_tag takes precedence over the derivation
7943	* of the register type from the BTF type itself.
7944	*/
7945	while ((id = btf_find_next_decl_tag(btf, fn_t, arg_idx, tag_key, id)) > 0) {
7946	const struct btf_type *tag_t;
7947	const char *tag;
7948	int i;
7949	bool found;
7950
7951	/* disallow arg tags in static subprogs */
7952	if (!is_global) {
7953	bpf_log(&env->log,
7954	"arg#%d type tag is not supported in static functions\n",
7955	arg_idx);
7956	return -EOPNOTSUPP;
7957	}
7958
7959	tag_t = btf_type_by_id(btf, id);
7960	tag = __btf_name_by_offset(btf, tag_t->name_off) + (sizeof(tag_key) - 1);
7961
7962	found = false;
7963	for (i = 0; i < ARRAY_SIZE(tag_values); ++i) {
7964	if (!strcmp(tag, tag_values[i].tag_value)) {
7965	*tags \|= tag_values[i].arg_tag;
7966	found = true;
7967	break;
7968	}
7969	}
7970
7971	if (!found) {
7972	bpf_log(&env->log, "arg#%d has unsupported set of tags\n", arg_idx);
7973	return -EOPNOTSUPP;
7974	}
7975	}
7976	if (id != -ENOENT) {
7977	bpf_log(&env->log, "arg#%d type tag fetching failure: %d\n", arg_idx, id);
7978	return id;
7979	}
7980
7981	return 0;
7982	}
7983
7984	static int btf_scan_type_tags(struct bpf_verifier_env *env,
7985	const struct btf *btf, u32 type_id,
7986	u32 *tags)
7987	{
7988	static const struct btf_type_tag_match func_type_tags[] = {
7989	{ "arena", ARG_TAG_ARENA },
7990	};
7991	struct btf_type_tag_walk_ctx ctx;
7992	const struct btf_type *t;
7993	int err;
7994
7995	/* Find the first pointer type in the chain. */
7996	t = btf_type_skip_modifiers(btf, type_id, NULL);
7997
7998	/*
7999	* We currently reject type tags on non-pointer types,
8000	* which neither LLVM nor GCC support anyway.
8001	*/
8002	if (!t \|\| !btf_type_is_ptr(t))
8003	return 0;
8004
8005	ctx.t = t;
8006	err = btf_type_tag_walk(btf, &ctx, func_type_tags,
8007	ARRAY_SIZE(func_type_tags));
8008	if (err) {
8009	bpf_log(&env->log,
8010	"function signature member has multiple type tags\n");
8011	return err;
8012	}
8013	*tags \|= ctx.res;
8014
8015	return 0;
8016	}
8017
8018	/* Check whether the type is a valid return type. */
8019	static int btf_validate_return_type(struct bpf_verifier_env env, struct btf btf,
8020	const struct btf_type *t, int subprog)
8021	{
8022	u32 tags = 0;
8023	int err;
8024
8025	err = btf_scan_type_tags(env, btf, t->type, &tags);
8026	if (err)
8027	return err;
8028
8029	t = btf_type_skip_modifiers(btf, t->type, NULL);
8030
8031	/*
8032	* We allow all subprogs except for the main one to return any kind of arena pointer.
8033	* General arena variables are not allowed, since it makes no sense to return by value
8034	* a variable that's on the heap in the first place.
8035	*/
8036	if (subprog && (tags & ARG_TAG_ARENA) && btf_type_is_ptr(t))
8037	return 0;
8038
8039	/* We always accept void or scalars. */
8040	if (btf_type_is_void(t) \|\| btf_type_is_int(t) \|\| btf_is_any_enum(t))
8041	return 0;
8042
8043	return -EOPNOTSUPP;
8044	}
8045
8046	/* Process BTF of a function to produce high-level expectation of function
8047	* arguments (like ARG_PTR_TO_CTX, or ARG_PTR_TO_MEM, etc). This information
8048	* is cached in subprog info for reuse.
8049	* Returns:
8050	* EFAULT - there is a verifier bug. Abort verification.
8051	* EINVAL - cannot convert BTF.
8052	* 0 - Successfully processed BTF and constructed argument expectations.
8053	*/
8054	int btf_prepare_func_args(struct bpf_verifier_env *env, int subprog)
8055	{
8056	bool is_global = subprog_aux(env, subprog)->linkage == BTF_FUNC_GLOBAL;
8057	struct bpf_subprog_info *sub = subprog_info(env, subprog);
8058	struct bpf_verifier_log *log = &env->log;
8059	struct bpf_prog *prog = env->prog;
8060	enum bpf_prog_type prog_type = prog->type;
8061	struct btf *btf = prog->aux->btf;
8062	const struct btf_param *args;
8063	const struct btf_type t, ref_t, *fn_t;
8064	int err;
8065	u32 i, nargs, btf_id;
8066	const char *tname;
8067
8068	if (sub->args_cached)
8069	return 0;
8070
8071	if (!prog->aux->func_info) {
8072	verifier_bug(env, "func_info undefined");
8073	return -EFAULT;
8074	}
8075
8076	btf_id = prog->aux->func_info[subprog].type_id;
8077	if (!btf_id) {
8078	if (!is_global) /* not fatal for static funcs */
8079	return -EINVAL;
8080	bpf_log(log, "Global functions need valid BTF\n");
8081	return -EFAULT;
8082	}
8083
8084	fn_t = btf_type_by_id(btf, btf_id);
8085	if (!fn_t \|\| !btf_type_is_func(fn_t)) {
8086	/* These checks were already done by the verifier while loading
8087	* struct bpf_func_info
8088	*/
8089	bpf_log(log, "BTF of func#%d doesn't point to KIND_FUNC\n",
8090	subprog);
8091	return -EFAULT;
8092	}
8093	tname = btf_name_by_offset(btf, fn_t->name_off);
8094
8095	if (prog->aux->func_info_aux[subprog].unreliable) {
8096	verifier_bug(env, "unreliable BTF for function %s()", tname);
8097	return -EFAULT;
8098	}
8099	if (prog_type == BPF_PROG_TYPE_EXT)
8100	prog_type = prog->aux->dst_prog->type;
8101
8102	t = btf_type_by_id(btf, fn_t->type);
8103	if (!t \|\| !btf_type_is_func_proto(t)) {
8104	bpf_log(log, "Invalid type of function %s()\n", tname);
8105	return -EFAULT;
8106	}
8107	args = (const struct btf_param *)(t + 1);
8108	nargs = btf_type_vlen(t);
8109	sub->arg_cnt = nargs;
8110	if (nargs > MAX_BPF_FUNC_ARGS) {
8111	bpf_log(log, "kernel supports at most %d parameters, function %s has %d\n",
8112	MAX_BPF_FUNC_ARGS, tname, nargs);
8113	return -EFAULT;
8114	}
8115	if (nargs > MAX_BPF_FUNC_REG_ARGS) {
8116	if (!bpf_jit_supports_stack_args()) {
8117	bpf_log(log, "JIT does not support function %s() with %d args\n",
8118	tname, nargs);
8119	return -EFAULT;
8120	}
8121	sub->stack_arg_cnt = nargs - MAX_BPF_FUNC_REG_ARGS;
8122	}
8123
8124	if (is_global && nargs > MAX_BPF_FUNC_REG_ARGS) {
8125	bpf_log(log, "global function %s has %d > %d args, stack args not supported\n",
8126	tname, nargs, MAX_BPF_FUNC_REG_ARGS);
8127	return -EINVAL;
8128	}
8129
8130	err = btf_validate_return_type(env, btf, t, subprog);
8131	if (err) {
8132	if (is_global) {
8133	bpf_log(log,
8134	"Global function %s() return value not void or scalar. "
8135	"Only those are supported.\n",
8136	tname);
8137	}
8138	return err;
8139	}
8140
8141	/* Convert BTF function arguments into verifier types.
8142	* Only PTR_TO_CTX and SCALAR are supported atm.
8143	*/
8144	for (i = 0; i < nargs; i++) {
8145	u32 tags = 0;
8146	err = btf_scan_decl_tags(env, btf, fn_t, i, is_global, &tags);
8147	if (err)
8148	return err;
8149
8150	err = btf_scan_type_tags(env, btf, args[i].type, &tags);
8151	if (err)
8152	return err;
8153
8154	t = btf_type_by_id(btf, args[i].type);
8155	while (btf_type_is_modifier(t))
8156	t = btf_type_by_id(btf, t->type);
8157	if (!btf_type_is_ptr(t))
8158	goto skip_pointer;
8159
8160	if ((tags & ARG_TAG_CTX) \|\| btf_is_prog_ctx_type(log, btf, t, prog_type, i)) {
8161	if (tags & ~ARG_TAG_CTX) {
8162	bpf_log(log, "arg#%d has invalid combination of tags\n", i);
8163	return -EINVAL;
8164	}
8165	if ((tags & ARG_TAG_CTX) &&
8166	btf_validate_prog_ctx_type(log, btf, t, i, prog_type,
8167	prog->expected_attach_type))
8168	return -EINVAL;
8169	sub->args[i].arg_type = ARG_PTR_TO_CTX;
8170	continue;
8171	}
8172	if (btf_is_dynptr_ptr(btf, t)) {
8173	if (tags) {
8174	bpf_log(log, "arg#%d has invalid combination of tags\n", i);
8175	return -EINVAL;
8176	}
8177	sub->args[i].arg_type = ARG_PTR_TO_DYNPTR;
8178	continue;
8179	}
8180	if (tags & ARG_TAG_TRUSTED) {
8181	int kern_type_id;
8182
8183	if (tags & ARG_TAG_NONNULL) {
8184	bpf_log(log, "arg#%d has invalid combination of tags\n", i);
8185	return -EINVAL;
8186	}
8187
8188	kern_type_id = btf_get_ptr_to_btf_id(log, i, btf, t);
8189	if (kern_type_id < 0)
8190	return kern_type_id;
8191
8192	sub->args[i].arg_type = ARG_PTR_TO_BTF_ID \| PTR_TRUSTED;
8193	if (tags & ARG_TAG_NULLABLE)
8194	sub->args[i].arg_type \|= PTR_MAYBE_NULL;
8195	sub->args[i].btf_id = kern_type_id;
8196	continue;
8197	}
8198	if (tags & ARG_TAG_UNTRUSTED) {
8199	struct btf *vmlinux_btf;
8200	int kern_type_id;
8201
8202	if (tags & ~ARG_TAG_UNTRUSTED) {
8203	bpf_log(log, "arg#%d untrusted cannot be combined with any other tags\n", i);
8204	return -EINVAL;
8205	}
8206
8207	ref_t = btf_type_skip_modifiers(btf, t->type, NULL);
8208	if (btf_type_is_void(ref_t) \|\| btf_type_is_primitive(ref_t)) {
8209	sub->args[i].arg_type = ARG_PTR_TO_MEM \| MEM_RDONLY \| PTR_UNTRUSTED;
8210	sub->args[i].mem_size = 0;
8211	continue;
8212	}
8213
8214	kern_type_id = btf_get_ptr_to_btf_id(log, i, btf, t);
8215	if (kern_type_id < 0)
8216	return kern_type_id;
8217
8218	vmlinux_btf = bpf_get_btf_vmlinux();
8219	ref_t = btf_type_by_id(vmlinux_btf, kern_type_id);
8220	if (!btf_type_is_struct(ref_t)) {
8221	tname = __btf_name_by_offset(vmlinux_btf, t->name_off);
8222	bpf_log(log, "arg#%d has type %s '%s', but only struct or primitive types are allowed\n",
8223	i, btf_type_str(ref_t), tname);
8224	return -EINVAL;
8225	}
8226	sub->args[i].arg_type = ARG_PTR_TO_BTF_ID \| PTR_UNTRUSTED;
8227	sub->args[i].btf_id = kern_type_id;
8228	continue;
8229	}
8230	if (tags & ARG_TAG_ARENA) {
8231	if (tags & ~ARG_TAG_ARENA) {
8232	bpf_log(log, "arg#%d arena cannot be combined with any other tags\n", i);
8233	return -EINVAL;
8234	}
8235	sub->args[i].arg_type = ARG_PTR_TO_ARENA;
8236	continue;
8237	}
8238	if (is_global) { /* generic user data pointer */
8239	u32 mem_size;
8240
8241	if (tags & ARG_TAG_NULLABLE) {
8242	bpf_log(log, "arg#%d has invalid combination of tags\n", i);
8243	return -EINVAL;
8244	}
8245
8246	t = btf_type_skip_modifiers(btf, t->type, NULL);
8247	ref_t = btf_resolve_size(btf, t, &mem_size);
8248	if (IS_ERR(ref_t)) {
8249	bpf_log(log, "arg#%d reference type('%s %s') size cannot be determined: %ld\n",
8250	i, btf_type_str(t), btf_name_by_offset(btf, t->name_off),
8251	PTR_ERR(ref_t));
8252	return -EINVAL;
8253	}
8254
8255	sub->args[i].arg_type = ARG_PTR_TO_MEM \| PTR_MAYBE_NULL;
8256	if (tags & ARG_TAG_NONNULL)
8257	sub->args[i].arg_type &= ~PTR_MAYBE_NULL;
8258	sub->args[i].mem_size = mem_size;
8259	continue;
8260	}
8261
8262	skip_pointer:
8263	if (tags) {
8264	bpf_log(log, "arg#%d has pointer tag, but is not a pointer type\n", i);
8265	return -EINVAL;
8266	}
8267	if (btf_type_is_int(t) \|\| btf_is_any_enum(t)) {
8268	sub->args[i].arg_type = ARG_ANYTHING;
8269	continue;
8270	}
8271	if (!is_global)
8272	return -EINVAL;
8273	bpf_log(log, "Arg#%d type %s in %s() is not supported yet.\n",
8274	i, btf_type_str(t), tname);
8275	return -EINVAL;
8276	}
8277
8278	sub->args_cached = true;
8279
8280	return 0;
8281	}
8282
8283	static void btf_type_show(const struct btf btf, u32 type_id, void obj,
8284	struct btf_show *show)
8285	{
8286	const struct btf_type *t = btf_type_by_id(btf, type_id);
8287
8288	show->btf = btf;
8289	memset(&show->state, 0, sizeof(show->state));
8290	memset(&show->obj, 0, sizeof(show->obj));
8291
8292	btf_type_ops(t)->show(btf, t, type_id, obj, 0, show);
8293	}
8294
8295	__printf(2, 0) static void btf_seq_show(struct btf_show show, const char fmt,
8296	va_list args)
8297	{
8298	seq_vprintf((struct seq_file *)show->target, fmt, args);
8299	}
8300
8301	int btf_type_seq_show_flags(const struct btf *btf, u32 type_id,
8302	void obj, struct seq_file m, u64 flags)
8303	{
8304	struct btf_show sseq;
8305
8306	sseq.target = m;
8307	sseq.showfn = btf_seq_show;
8308	sseq.flags = flags;
8309
8310	btf_type_show(btf, type_id, obj, &sseq);
8311
8312	return sseq.state.status;
8313	}
8314
8315	void btf_type_seq_show(const struct btf btf, u32 type_id, void obj,
8316	struct seq_file *m)
8317	{
8318	(void) btf_type_seq_show_flags(btf, type_id, obj, m,
8319	BTF_SHOW_NONAME \| BTF_SHOW_COMPACT \|
8320	BTF_SHOW_ZERO \| BTF_SHOW_UNSAFE);
8321	}
8322
8323	struct btf_show_snprintf {
8324	struct btf_show show;
8325	int len_left; /* space left in string */
8326	int len; /* length we would have written */
8327	};
8328
8329	__printf(2, 0) static void btf_snprintf_show(struct btf_show show, const char fmt,
8330	va_list args)
8331	{
8332	struct btf_show_snprintf ssnprintf = (struct btf_show_snprintf )show;
8333	int len;
8334
8335	len = vsnprintf(show->target, ssnprintf->len_left, fmt, args);
8336
8337	if (len < 0) {
8338	ssnprintf->len_left = 0;
8339	ssnprintf->len = len;
8340	} else if (len >= ssnprintf->len_left) {
8341	/* no space, drive on to get length we would have written */
8342	ssnprintf->len_left = 0;
8343	ssnprintf->len += len;
8344	} else {
8345	ssnprintf->len_left -= len;
8346	ssnprintf->len += len;
8347	show->target += len;
8348	}
8349	}
8350
8351	int btf_type_snprintf_show(const struct btf btf, u32 type_id, void obj,
8352	char *buf, int len, u64 flags)
8353	{
8354	struct btf_show_snprintf ssnprintf;
8355
8356	ssnprintf.show.target = buf;
8357	ssnprintf.show.flags = flags;
8358	ssnprintf.show.showfn = btf_snprintf_show;
8359	ssnprintf.len_left = len;
8360	ssnprintf.len = 0;
8361
8362	btf_type_show(btf, type_id, obj, (struct btf_show *)&ssnprintf);
8363
8364	/* If we encountered an error, return it. */
8365	if (ssnprintf.show.state.status)
8366	return ssnprintf.show.state.status;
8367
8368	/* Otherwise return length we would have written */
8369	return ssnprintf.len;
8370	}
8371
8372	int btf_type_name_to_buf(const struct btf btf, u32 type_id, char buf, int len)
8373	{
8374	struct btf_show show = {
8375	.btf = btf,
8376	.state.type_id = type_id,
8377	};
8378
8379	return snprintf(buf, len, "%s", btf_show_name(&show));
8380	}
8381
8382	#ifdef CONFIG_PROC_FS
8383	static void bpf_btf_show_fdinfo(struct seq_file m, struct file filp)
8384	{
8385	const struct btf *btf = filp->private_data;
8386
8387	seq_printf(m, "btf_id:\t%u\n", READ_ONCE(btf->id));
8388	}
8389	#endif
8390
8391	static int btf_release(struct inode inode, struct file filp)
8392	{
8393	btf_put(filp->private_data);
8394	return 0;
8395	}
8396
8397	const struct file_operations btf_fops = {
8398	#ifdef CONFIG_PROC_FS
8399	.show_fdinfo = bpf_btf_show_fdinfo,
8400	#endif
8401	.release = btf_release,
8402	};
8403
8404	static int __btf_new_fd(struct btf *btf)
8405	{
8406	return anon_inode_getfd("btf", &btf_fops, btf, O_RDONLY \| O_CLOEXEC);
8407	}
8408
8409	int btf_new_fd(const union bpf_attr attr, bpfptr_t uattr, struct bpf_log_attr attr_log)
8410	{
8411	struct btf *btf;
8412	int ret;
8413
8414	btf = btf_parse(attr, uattr, attr_log);
8415	if (IS_ERR(btf))
8416	return PTR_ERR(btf);
8417
8418	ret = btf_alloc_id(btf);
8419	if (ret) {
8420	btf_free(btf);
8421	return ret;
8422	}
8423
8424	/*
8425	* The BTF ID is published to the userspace.
8426	* All BTF free must go through call_rcu() from
8427	* now on (i.e. free by calling btf_put()).
8428	*/
8429
8430	ret = __btf_new_fd(btf);
8431	if (ret < 0)
8432	btf_put(btf);
8433
8434	return ret;
8435	}
8436
8437	struct btf *btf_get_by_fd(int fd)
8438	{
8439	struct btf *btf;
8440	CLASS(fd, f)(fd);
8441
8442	btf = __btf_get_by_fd(f);
8443	if (!IS_ERR(btf))
8444	refcount_inc(&btf->refcnt);
8445
8446	return btf;
8447	}
8448
8449	int btf_get_info_by_fd(const struct btf *btf,
8450	const union bpf_attr *attr,
8451	union bpf_attr __user *uattr)
8452	{
8453	struct bpf_btf_info __user *uinfo;
8454	struct bpf_btf_info info;
8455	u32 info_copy, btf_copy;
8456	void __user *ubtf;
8457	char __user *uname;
8458	u32 uinfo_len, uname_len, name_len;
8459	int ret = 0;
8460
8461	uinfo = u64_to_user_ptr(attr->info.info);
8462	uinfo_len = attr->info.info_len;
8463
8464	info_copy = min_t(u32, uinfo_len, sizeof(info));
8465	memset(&info, 0, sizeof(info));
8466	if (copy_from_user(&info, uinfo, info_copy))
8467	return -EFAULT;
8468
8469	info.id = READ_ONCE(btf->id);
8470	ubtf = u64_to_user_ptr(info.btf);
8471	btf_copy = min_t(u32, btf->data_size, info.btf_size);
8472	if (copy_to_user(ubtf, btf->data, btf_copy))
8473	return -EFAULT;
8474	info.btf_size = btf->data_size;
8475
8476	info.kernel_btf = btf->kernel_btf;
8477
8478	uname = u64_to_user_ptr(info.name);
8479	uname_len = info.name_len;
8480	if (!uname ^ !uname_len)
8481	return -EINVAL;
8482
8483	name_len = strlen(btf->name);
8484	info.name_len = name_len;
8485
8486	if (uname) {
8487	if (uname_len >= name_len + 1) {
8488	if (copy_to_user(uname, btf->name, name_len + 1))
8489	return -EFAULT;
8490	} else {
8491	char zero = '\0';
8492
8493	if (copy_to_user(uname, btf->name, uname_len - 1))
8494	return -EFAULT;
8495	if (put_user(zero, uname + uname_len - 1))
8496	return -EFAULT;
8497	/* let user-space know about too short buffer */
8498	ret = -ENOSPC;
8499	}
8500	}
8501
8502	if (copy_to_user(uinfo, &info, info_copy) \|\|
8503	put_user(info_copy, &uattr->info.info_len))
8504	return -EFAULT;
8505
8506	return ret;
8507	}
8508
8509	int btf_get_fd_by_id(u32 id)
8510	{
8511	struct btf *btf;
8512	int fd;
8513
8514	rcu_read_lock();
8515	btf = idr_find(&btf_idr, id);
8516	if (!btf \|\| !refcount_inc_not_zero(&btf->refcnt))
8517	btf = ERR_PTR(-ENOENT);
8518	rcu_read_unlock();
8519
8520	if (IS_ERR(btf))
8521	return PTR_ERR(btf);
8522
8523	fd = __btf_new_fd(btf);
8524	if (fd < 0)
8525	btf_put(btf);
8526
8527	return fd;
8528	}
8529
8530	u32 btf_obj_id(const struct btf *btf)
8531	{
8532	return READ_ONCE(btf->id);
8533	}
8534
8535	bool btf_is_kernel(const struct btf *btf)
8536	{
8537	return btf->kernel_btf;
8538	}
8539
8540	bool btf_is_module(const struct btf *btf)
8541	{
8542	return btf->kernel_btf && strcmp(btf->name, "vmlinux") != 0;
8543	}
8544
8545	enum {
8546	BTF_MODULE_F_LIVE = (1 << 0),
8547	};
8548
8549	#ifdef CONFIG_DEBUG_INFO_BTF_MODULES
8550	struct btf_module {
8551	struct list_head list;
8552	struct module *module;
8553	struct btf *btf;
8554	struct bin_attribute *sysfs_attr;
8555	int flags;
8556	};
8557
8558	static LIST_HEAD(btf_modules);
8559	static DEFINE_MUTEX(btf_module_mutex);
8560
8561	static void purge_cand_cache(struct btf *btf);
8562
8563	static int btf_module_notify(struct notifier_block *nb, unsigned long op,
8564	void *module)
8565	{
8566	struct btf_module btf_mod, tmp;
8567	struct module *mod = module;
8568	struct btf *btf;
8569	int err = 0;
8570
8571	if (mod->btf_data_size == 0 \|\|
8572	(op != MODULE_STATE_COMING && op != MODULE_STATE_LIVE &&
8573	op != MODULE_STATE_GOING))
8574	goto out;
8575
8576	switch (op) {
8577	case MODULE_STATE_COMING:
8578	btf_mod = kzalloc_obj(*btf_mod);
8579	if (!btf_mod) {
8580	err = -ENOMEM;
8581	goto out;
8582	}
8583	btf = btf_parse_module(mod->name, mod->btf_data, mod->btf_data_size,
8584	mod->btf_base_data, mod->btf_base_data_size);
8585	if (IS_ERR(btf)) {
8586	kfree(btf_mod);
8587	if (!IS_ENABLED(CONFIG_MODULE_ALLOW_BTF_MISMATCH)) {
8588	pr_warn("failed to validate module [%s] BTF: %ld\n",
8589	mod->name, PTR_ERR(btf));
8590	err = PTR_ERR(btf);
8591	} else {
8592	pr_warn_once("Kernel module BTF mismatch detected, BTF debug info may be unavailable for some modules\n");
8593	}
8594	goto out;
8595	}
8596	err = btf_alloc_id(btf);
8597	if (err) {
8598	btf_free(btf);
8599	kfree(btf_mod);
8600	goto out;
8601	}
8602
8603	purge_cand_cache(NULL);
8604	mutex_lock(&btf_module_mutex);
8605	btf_mod->module = module;
8606	btf_mod->btf = btf;
8607	list_add(&btf_mod->list, &btf_modules);
8608	mutex_unlock(&btf_module_mutex);
8609
8610	if (IS_ENABLED(CONFIG_SYSFS)) {
8611	struct bin_attribute *attr;
8612
8613	attr = kzalloc_obj(*attr);
8614	if (!attr)
8615	goto out;
8616
8617	sysfs_bin_attr_init(attr);
8618	attr->attr.name = btf->name;
8619	attr->attr.mode = 0444;
8620	attr->size = btf->data_size;
8621	attr->private = btf->data;
8622	attr->read = sysfs_bin_attr_simple_read;
8623
8624	err = sysfs_create_bin_file(btf_kobj, attr);
8625	if (err) {
8626	pr_warn("failed to register module [%s] BTF in sysfs: %d\n",
8627	mod->name, err);
8628	kfree(attr);
8629	err = 0;
8630	goto out;
8631	}
8632
8633	btf_mod->sysfs_attr = attr;
8634	}
8635
8636	break;
8637	case MODULE_STATE_LIVE:
8638	mutex_lock(&btf_module_mutex);
8639	list_for_each_entry_safe(btf_mod, tmp, &btf_modules, list) {
8640	if (btf_mod->module != module)
8641	continue;
8642
8643	btf_mod->flags \|= BTF_MODULE_F_LIVE;
8644	break;
8645	}
8646	mutex_unlock(&btf_module_mutex);
8647	break;
8648	case MODULE_STATE_GOING:
8649	mutex_lock(&btf_module_mutex);
8650	list_for_each_entry_safe(btf_mod, tmp, &btf_modules, list) {
8651	if (btf_mod->module != module)
8652	continue;
8653
8654	/*
8655	* For modules, we do the freeing of BTF IDR as soon as
8656	* module goes away to disable BTF discovery, since the
8657	* btf_try_get_module() on such BTFs will fail. This may
8658	* be called again on btf_put(), but it's ok to do so.
8659	*/
8660	btf_free_id(btf_mod->btf);
8661	list_del(&btf_mod->list);
8662	if (btf_mod->sysfs_attr)
8663	sysfs_remove_bin_file(btf_kobj, btf_mod->sysfs_attr);
8664	purge_cand_cache(btf_mod->btf);
8665	btf_put(btf_mod->btf);
8666	kfree(btf_mod->sysfs_attr);
8667	kfree(btf_mod);
8668	break;
8669	}
8670	mutex_unlock(&btf_module_mutex);
8671	break;
8672	}
8673	out:
8674	return notifier_from_errno(err);
8675	}
8676
8677	static struct notifier_block btf_module_nb = {
8678	.notifier_call = btf_module_notify,
8679	};
8680
8681	static int __init btf_module_init(void)
8682	{
8683	register_module_notifier(&btf_module_nb);
8684	return 0;
8685	}
8686
8687	fs_initcall(btf_module_init);
8688	#endif /* CONFIG_DEBUG_INFO_BTF_MODULES */
8689
8690	struct module btf_try_get_module(const struct btf btf)
8691	{
8692	struct module *res = NULL;
8693	#ifdef CONFIG_DEBUG_INFO_BTF_MODULES
8694	struct btf_module btf_mod, tmp;
8695
8696	mutex_lock(&btf_module_mutex);
8697	list_for_each_entry_safe(btf_mod, tmp, &btf_modules, list) {
8698	if (btf_mod->btf != btf)
8699	continue;
8700
8701	/* We must only consider module whose __init routine has
8702	* finished, hence we must check for BTF_MODULE_F_LIVE flag,
8703	* which is set from the notifier callback for
8704	* MODULE_STATE_LIVE.
8705	*/
8706	if ((btf_mod->flags & BTF_MODULE_F_LIVE) && try_module_get(btf_mod->module))
8707	res = btf_mod->module;
8708
8709	break;
8710	}
8711	mutex_unlock(&btf_module_mutex);
8712	#endif
8713
8714	return res;
8715	}
8716
8717	/* Returns struct btf corresponding to the struct module.
8718	* This function can return NULL or ERR_PTR.
8719	*/
8720	struct btf btf_get_module_btf(const struct module module)
8721	{
8722	#ifdef CONFIG_DEBUG_INFO_BTF_MODULES
8723	struct btf_module btf_mod, tmp;
8724	#endif
8725	struct btf *btf = NULL;
8726
8727	if (!module) {
8728	btf = bpf_get_btf_vmlinux();
8729	if (!IS_ERR_OR_NULL(btf))
8730	btf_get(btf);
8731	return btf;
8732	}
8733
8734	#ifdef CONFIG_DEBUG_INFO_BTF_MODULES
8735	mutex_lock(&btf_module_mutex);
8736	list_for_each_entry_safe(btf_mod, tmp, &btf_modules, list) {
8737	if (btf_mod->module != module)
8738	continue;
8739
8740	btf_get(btf_mod->btf);
8741	btf = btf_mod->btf;
8742	break;
8743	}
8744	mutex_unlock(&btf_module_mutex);
8745	#endif
8746
8747	return btf;
8748	}
8749
8750	static int check_btf_kconfigs(const struct module module, const char feature)
8751	{
8752	if (!module && IS_ENABLED(CONFIG_DEBUG_INFO_BTF)) {
8753	pr_err("missing vmlinux BTF, cannot register %s\n", feature);
8754	return -ENOENT;
8755	}
8756	if (module && IS_ENABLED(CONFIG_DEBUG_INFO_BTF_MODULES))
8757	pr_warn("missing module BTF, cannot register %s\n", feature);
8758	return 0;
8759	}
8760
8761	BPF_CALL_4(bpf_btf_find_by_name_kind, char *, name, int, name_sz, u32, kind, int, flags)
8762	{
8763	struct btf *btf = NULL;
8764	int btf_obj_fd = 0;
8765	long ret;
8766
8767	if (flags)
8768	return -EINVAL;
8769
8770	if (name_sz <= 1 \|\| name[name_sz - 1])
8771	return -EINVAL;
8772
8773	ret = bpf_find_btf_id(name, kind, &btf);
8774	if (ret > 0 && btf_is_module(btf)) {
8775	btf_obj_fd = __btf_new_fd(btf);
8776	if (btf_obj_fd < 0) {
8777	btf_put(btf);
8778	return btf_obj_fd;
8779	}
8780	return ret \| (((u64)btf_obj_fd) << 32);
8781	}
8782	if (ret > 0)
8783	btf_put(btf);
8784	return ret;
8785	}
8786
8787	const struct bpf_func_proto bpf_btf_find_by_name_kind_proto = {
8788	.func = bpf_btf_find_by_name_kind,
8789	.gpl_only = false,
8790	.might_sleep = true,
8791	.ret_type = RET_INTEGER,
8792	.arg1_type = ARG_PTR_TO_MEM \| MEM_RDONLY,
8793	.arg2_type = ARG_MEM_SIZE,
8794	.arg3_type = ARG_ANYTHING,
8795	.arg4_type = ARG_ANYTHING,
8796	};
8797
8798	BTF_ID_LIST_GLOBAL(btf_tracing_ids, MAX_BTF_TRACING_TYPE)
8799	#define BTF_TRACING_TYPE(name, type) BTF_ID(struct, type)
8800	BTF_TRACING_TYPE_xxx
8801	#undef BTF_TRACING_TYPE
8802
8803	/* Validate well-formedness of iter argument type.
8804	* On success, return positive BTF ID of iter state's STRUCT type.
8805	* On error, negative error is returned.
8806	*/
8807	int btf_check_iter_arg(struct btf btf, const struct btf_type func, int arg_idx)
8808	{
8809	const struct btf_param *arg;
8810	const struct btf_type *t;
8811	const char *name;
8812	int btf_id;
8813
8814	if (btf_type_vlen(func) <= arg_idx)
8815	return -EINVAL;
8816
8817	arg = &btf_params(func)[arg_idx];
8818	t = btf_type_skip_modifiers(btf, arg->type, NULL);
8819	if (!t \|\| !btf_type_is_ptr(t))
8820	return -EINVAL;
8821	t = btf_type_skip_modifiers(btf, t->type, &btf_id);
8822	if (!t \|\| !__btf_type_is_struct(t))
8823	return -EINVAL;
8824
8825	name = btf_name_by_offset(btf, t->name_off);
8826	if (!name \|\| strncmp(name, ITER_PREFIX, sizeof(ITER_PREFIX) - 1))
8827	return -EINVAL;
8828
8829	return btf_id;
8830	}
8831
8832	static int btf_check_iter_kfuncs(struct btf btf, const char func_name,
8833	const struct btf_type *func, u32 func_flags)
8834	{
8835	u32 flags = func_flags & (KF_ITER_NEW \| KF_ITER_NEXT \| KF_ITER_DESTROY);
8836	const char sfx, iter_name;
8837	const struct btf_type *t;
8838	char exp_name[128];
8839	u32 nr_args;
8840	int btf_id;
8841
8842	/* exactly one of KF_ITER_{NEW,NEXT,DESTROY} can be set */
8843	if (!flags \|\| (flags & (flags - 1)))
8844	return -EINVAL;
8845
8846	/* any BPF iter kfunc should have `struct bpf_iter_<type> ` first arg /
8847	nr_args = btf_type_vlen(func);
8848	if (nr_args < 1)
8849	return -EINVAL;
8850
8851	btf_id = btf_check_iter_arg(btf, func, 0);
8852	if (btf_id < 0)
8853	return btf_id;
8854
8855	/* sizeof(struct bpf_iter_<type>) should be a multiple of 8 to
8856	* fit nicely in stack slots
8857	*/
8858	t = btf_type_by_id(btf, btf_id);
8859	if (t->size == 0 \|\| (t->size % 8))
8860	return -EINVAL;
8861
8862	/* validate bpf_iter_<type>_{new,next,destroy}(struct bpf_iter_<type> *)
8863	* naming pattern
8864	*/
8865	iter_name = btf_name_by_offset(btf, t->name_off) + sizeof(ITER_PREFIX) - 1;
8866	if (flags & KF_ITER_NEW)
8867	sfx = "new";
8868	else if (flags & KF_ITER_NEXT)
8869	sfx = "next";
8870	else /* (flags & KF_ITER_DESTROY) */
8871	sfx = "destroy";
8872
8873	snprintf(exp_name, sizeof(exp_name), "bpf_iter_%s_%s", iter_name, sfx);
8874	if (strcmp(func_name, exp_name))
8875	return -EINVAL;
8876
8877	/* only iter constructor should have extra arguments */
8878	if (!(flags & KF_ITER_NEW) && nr_args != 1)
8879	return -EINVAL;
8880
8881	if (flags & KF_ITER_NEXT) {
8882	/* bpf_iter_<type>_next() should return pointer */
8883	t = btf_type_skip_modifiers(btf, func->type, NULL);
8884	if (!t \|\| !btf_type_is_ptr(t))
8885	return -EINVAL;
8886	}
8887
8888	if (flags & KF_ITER_DESTROY) {
8889	/* bpf_iter_<type>_destroy() should return void */
8890	t = btf_type_by_id(btf, func->type);
8891	if (!t \|\| !btf_type_is_void(t))
8892	return -EINVAL;
8893	}
8894
8895	return 0;
8896	}
8897
8898	static int btf_check_kfunc_name(struct btf btf, const char func_name, u32 kind)
8899	{
8900	#ifdef CONFIG_DEBUG_INFO_BTF_MODULES
8901	struct btf_module btf_mod, tmp;
8902	#endif
8903	s32 id;
8904
8905	if (!btf_is_module(btf))
8906	return 0;
8907
8908	id = btf_find_by_name_kind(bpf_get_btf_vmlinux(), func_name, kind);
8909	if (id >= 0) {
8910	pr_err("kfunc %s (id: %d) is already present in vmlinux.\n",
8911	func_name, id);
8912	return -EINVAL;
8913	}
8914
8915	#ifdef CONFIG_DEBUG_INFO_BTF_MODULES
8916	guard(mutex)(&btf_module_mutex);
8917	list_for_each_entry_safe(btf_mod, tmp, &btf_modules, list) {
8918	if (btf_mod->btf == btf)
8919	continue;
8920	id = btf_find_by_name_kind(btf_mod->btf, func_name, kind);
8921	if (id >= 0) {
8922	pr_err("kfunc %s (id: %d) is already present in module %s.\n",
8923	func_name, id, btf_mod->module->name);
8924	return -EINVAL;
8925	}
8926	}
8927	#endif
8928	return 0;
8929	}
8930
8931	static int btf_check_kfunc_protos(struct btf *btf, u32 func_id, u32 func_flags)
8932	{
8933	const struct btf_type *func;
8934	const char *func_name;
8935	int err;
8936
8937	/* any kfunc should be FUNC -> FUNC_PROTO */
8938	func = btf_type_by_id(btf, func_id);
8939	if (!func \|\| !btf_type_is_func(func))
8940	return -EINVAL;
8941
8942	/* sanity check kfunc name */
8943	func_name = btf_name_by_offset(btf, func->name_off);
8944	if (!func_name \|\| !func_name[0] \|\|
8945	btf_check_kfunc_name(btf, func_name, BTF_INFO_KIND(func->info)))
8946	return -EINVAL;
8947
8948	func = btf_type_by_id(btf, func->type);
8949	if (!func \|\| !btf_type_is_func_proto(func))
8950	return -EINVAL;
8951
8952	if (func_flags & (KF_ITER_NEW \| KF_ITER_NEXT \| KF_ITER_DESTROY)) {
8953	err = btf_check_iter_kfuncs(btf, func_name, func, func_flags);
8954	if (err)
8955	return err;
8956	}
8957
8958	return 0;
8959	}
8960
8961	/* Kernel Function (kfunc) BTF ID set registration API */
8962
8963	static int btf_populate_kfunc_set(struct btf *btf, enum btf_kfunc_hook hook,
8964	const struct btf_kfunc_id_set *kset)
8965	{
8966	struct btf_kfunc_hook_filter *hook_filter;
8967	struct btf_id_set8 *add_set = kset->set;
8968	bool vmlinux_set = !btf_is_module(btf);
8969	bool add_filter = !!kset->filter;
8970	struct btf_kfunc_set_tab *tab;
8971	struct btf_id_set8 *set;
8972	u32 set_cnt, i;
8973	int ret;
8974
8975	if (hook >= BTF_KFUNC_HOOK_MAX) {
8976	ret = -EINVAL;
8977	goto end;
8978	}
8979
8980	if (!add_set->cnt)
8981	return 0;
8982
8983	tab = btf->kfunc_set_tab;
8984
8985	if (tab && add_filter) {
8986	u32 i;
8987
8988	hook_filter = &tab->hook_filters[hook];
8989	for (i = 0; i < hook_filter->nr_filters; i++) {
8990	if (hook_filter->filters[i] == kset->filter) {
8991	add_filter = false;
8992	break;
8993	}
8994	}
8995
8996	if (add_filter && hook_filter->nr_filters == BTF_KFUNC_FILTER_MAX_CNT) {
8997	ret = -E2BIG;
8998	goto end;
8999	}
9000	}
9001
9002	if (!tab) {
9003	tab = kzalloc_obj(*tab, GFP_KERNEL \| __GFP_NOWARN);
9004	if (!tab)
9005	return -ENOMEM;
9006	btf->kfunc_set_tab = tab;
9007	}
9008
9009	set = tab->sets[hook];
9010	/* Warn when register_btf_kfunc_id_set is called twice for the same hook
9011	* for module sets.
9012	*/
9013	if (WARN_ON_ONCE(set && !vmlinux_set)) {
9014	ret = -EINVAL;
9015	goto end;
9016	}
9017
9018	/* In case of vmlinux sets, there may be more than one set being
9019	* registered per hook. To create a unified set, we allocate a new set
9020	* and concatenate all individual sets being registered. While each set
9021	* is individually sorted, they may become unsorted when concatenated,
9022	* hence re-sorting the final set again is required to make binary
9023	* searching the set using btf_id_set8_contains function work.
9024	*
9025	* For module sets, we need to allocate as we may need to relocate
9026	* BTF ids.
9027	*/
9028	set_cnt = set ? set->cnt : 0;
9029
9030	if (set_cnt > U32_MAX - add_set->cnt) {
9031	ret = -EOVERFLOW;
9032	goto end;
9033	}
9034
9035	if (set_cnt + add_set->cnt > BTF_KFUNC_SET_MAX_CNT) {
9036	ret = -E2BIG;
9037	goto end;
9038	}
9039
9040	/* Grow set */
9041	set = krealloc(tab->sets[hook],
9042	struct_size(set, pairs, set_cnt + add_set->cnt),
9043	GFP_KERNEL \| __GFP_NOWARN);
9044	if (!set) {
9045	ret = -ENOMEM;
9046	goto end;
9047	}
9048
9049	/* For newly allocated set, initialize set->cnt to 0 */
9050	if (!tab->sets[hook])
9051	set->cnt = 0;
9052	tab->sets[hook] = set;
9053
9054	/* Concatenate the two sets */
9055	memcpy(set->pairs + set->cnt, add_set->pairs, add_set->cnt * sizeof(set->pairs[0]));
9056	/* Now that the set is copied, update with relocated BTF ids */
9057	for (i = set->cnt; i < set->cnt + add_set->cnt; i++)
9058	set->pairs[i].id = btf_relocate_id(btf, set->pairs[i].id);
9059
9060	set->cnt += add_set->cnt;
9061
9062	sort(set->pairs, set->cnt, sizeof(set->pairs[0]), btf_id_cmp_func, NULL);
9063
9064	if (add_filter) {
9065	hook_filter = &tab->hook_filters[hook];
9066	hook_filter->filters[hook_filter->nr_filters++] = kset->filter;
9067	}
9068	return 0;
9069	end:
9070	btf_free_kfunc_set_tab(btf);
9071	return ret;
9072	}
9073
9074	static u32 btf_kfunc_id_set_contains(const struct btf btf,
9075	enum btf_kfunc_hook hook,
9076	u32 kfunc_btf_id)
9077	{
9078	struct btf_id_set8 *set;
9079	u32 *id;
9080
9081	if (hook >= BTF_KFUNC_HOOK_MAX)
9082	return NULL;
9083	if (!btf->kfunc_set_tab)
9084	return NULL;
9085	set = btf->kfunc_set_tab->sets[hook];
9086	if (!set)
9087	return NULL;
9088	id = btf_id_set8_contains(set, kfunc_btf_id);
9089	if (!id)
9090	return NULL;
9091	/* The flags for BTF ID are located next to it */
9092	return id + 1;
9093	}
9094
9095	static bool __btf_kfunc_is_allowed(const struct btf *btf,
9096	enum btf_kfunc_hook hook,
9097	u32 kfunc_btf_id,
9098	const struct bpf_prog *prog)
9099	{
9100	struct btf_kfunc_hook_filter *hook_filter;
9101	int i;
9102
9103	if (hook >= BTF_KFUNC_HOOK_MAX)
9104	return false;
9105	if (!btf->kfunc_set_tab)
9106	return false;
9107
9108	hook_filter = &btf->kfunc_set_tab->hook_filters[hook];
9109	for (i = 0; i < hook_filter->nr_filters; i++) {
9110	if (hook_filter->filters[i](prog, kfunc_btf_id))
9111	return false;
9112	}
9113
9114	return true;
9115	}
9116
9117	static int bpf_prog_type_to_kfunc_hook(enum bpf_prog_type prog_type)
9118	{
9119	switch (prog_type) {
9120	case BPF_PROG_TYPE_UNSPEC:
9121	return BTF_KFUNC_HOOK_COMMON;
9122	case BPF_PROG_TYPE_XDP:
9123	return BTF_KFUNC_HOOK_XDP;
9124	case BPF_PROG_TYPE_SCHED_CLS:
9125	return BTF_KFUNC_HOOK_TC;
9126	case BPF_PROG_TYPE_STRUCT_OPS:
9127	return BTF_KFUNC_HOOK_STRUCT_OPS;
9128	case BPF_PROG_TYPE_TRACING:
9129	case BPF_PROG_TYPE_TRACEPOINT:
9130	case BPF_PROG_TYPE_RAW_TRACEPOINT:
9131	case BPF_PROG_TYPE_PERF_EVENT:
9132	case BPF_PROG_TYPE_LSM:
9133	return BTF_KFUNC_HOOK_TRACING;
9134	case BPF_PROG_TYPE_SYSCALL:
9135	return BTF_KFUNC_HOOK_SYSCALL;
9136	case BPF_PROG_TYPE_CGROUP_SKB:
9137	case BPF_PROG_TYPE_CGROUP_SOCK:
9138	case BPF_PROG_TYPE_CGROUP_DEVICE:
9139	case BPF_PROG_TYPE_CGROUP_SOCK_ADDR:
9140	case BPF_PROG_TYPE_CGROUP_SOCKOPT:
9141	case BPF_PROG_TYPE_CGROUP_SYSCTL:
9142	case BPF_PROG_TYPE_SOCK_OPS:
9143	return BTF_KFUNC_HOOK_CGROUP;
9144	case BPF_PROG_TYPE_SCHED_ACT:
9145	return BTF_KFUNC_HOOK_SCHED_ACT;
9146	case BPF_PROG_TYPE_SK_SKB:
9147	return BTF_KFUNC_HOOK_SK_SKB;
9148	case BPF_PROG_TYPE_SOCKET_FILTER:
9149	return BTF_KFUNC_HOOK_SOCKET_FILTER;
9150	case BPF_PROG_TYPE_LWT_OUT:
9151	case BPF_PROG_TYPE_LWT_IN:
9152	case BPF_PROG_TYPE_LWT_XMIT:
9153	case BPF_PROG_TYPE_LWT_SEG6LOCAL:
9154	return BTF_KFUNC_HOOK_LWT;
9155	case BPF_PROG_TYPE_NETFILTER:
9156	return BTF_KFUNC_HOOK_NETFILTER;
9157	case BPF_PROG_TYPE_KPROBE:
9158	return BTF_KFUNC_HOOK_KPROBE;
9159	default:
9160	return BTF_KFUNC_HOOK_MAX;
9161	}
9162	}
9163
9164	bool btf_kfunc_is_allowed(const struct btf *btf,
9165	u32 kfunc_btf_id,
9166	const struct bpf_prog *prog)
9167	{
9168	enum bpf_prog_type prog_type = resolve_prog_type(prog);
9169	enum btf_kfunc_hook hook;
9170	u32 *kfunc_flags;
9171
9172	kfunc_flags = btf_kfunc_id_set_contains(btf, BTF_KFUNC_HOOK_COMMON, kfunc_btf_id);
9173	if (kfunc_flags && __btf_kfunc_is_allowed(btf, BTF_KFUNC_HOOK_COMMON, kfunc_btf_id, prog))
9174	return true;
9175
9176	hook = bpf_prog_type_to_kfunc_hook(prog_type);
9177	kfunc_flags = btf_kfunc_id_set_contains(btf, hook, kfunc_btf_id);
9178	if (kfunc_flags && __btf_kfunc_is_allowed(btf, hook, kfunc_btf_id, prog))
9179	return true;
9180
9181	return false;
9182	}
9183
9184	/* Caution:
9185	* Reference to the module (obtained using btf_try_get_module) corresponding to
9186	* the struct btf MUST be held when calling this function from verifier
9187	* context. This is usually true as we stash references in prog's kfunc_btf_tab;
9188	* keeping the reference for the duration of the call provides the necessary
9189	* protection for looking up a well-formed btf->kfunc_set_tab.
9190	*/
9191	u32 btf_kfunc_flags(const struct btf btf, u32 kfunc_btf_id, const struct bpf_prog *prog)
9192	{
9193	enum bpf_prog_type prog_type = resolve_prog_type(prog);
9194	enum btf_kfunc_hook hook;
9195	u32 *kfunc_flags;
9196
9197	kfunc_flags = btf_kfunc_id_set_contains(btf, BTF_KFUNC_HOOK_COMMON, kfunc_btf_id);
9198	if (kfunc_flags)
9199	return kfunc_flags;
9200
9201	hook = bpf_prog_type_to_kfunc_hook(prog_type);
9202	return btf_kfunc_id_set_contains(btf, hook, kfunc_btf_id);
9203	}
9204
9205	/*
9206	* Check a single KF_* @flag on a kfunc across all of its hook sets.
9207	* Returns:
9208	* * 1 if @flag is set
9209	* * 0 if @flag is not set
9210	* * -EINVAL if @flag is set inconsistently across the sets
9211	* * -ENOENT if kfunc_btf_id is not a registered kfunc
9212	*/
9213	int btf_kfunc_check_flag(const struct btf *btf, u32 kfunc_btf_id, u32 flag)
9214	{
9215	enum btf_kfunc_hook hook;
9216	int res = -ENOENT;
9217	bool is_set;
9218	u32 *flags;
9219
9220	for (hook = 0; hook < BTF_KFUNC_HOOK_MAX; hook++) {
9221	flags = btf_kfunc_id_set_contains(btf, hook, kfunc_btf_id);
9222	if (!flags)
9223	continue;
9224	is_set = *flags & flag;
9225	if (res < 0)
9226	res = is_set;
9227	else if (res != is_set)
9228	return -EINVAL;
9229	}
9230
9231	return res;
9232	}
9233
9234	u32 btf_kfunc_is_modify_return(const struct btf btf, u32 kfunc_btf_id,
9235	const struct bpf_prog *prog)
9236	{
9237	if (!__btf_kfunc_is_allowed(btf, BTF_KFUNC_HOOK_FMODRET, kfunc_btf_id, prog))
9238	return NULL;
9239
9240	return btf_kfunc_id_set_contains(btf, BTF_KFUNC_HOOK_FMODRET, kfunc_btf_id);
9241	}
9242
9243	static int __register_btf_kfunc_id_set(enum btf_kfunc_hook hook,
9244	const struct btf_kfunc_id_set *kset)
9245	{
9246	struct btf *btf;
9247	int ret, i;
9248
9249	btf = btf_get_module_btf(kset->owner);
9250	if (!btf)
9251	return check_btf_kconfigs(kset->owner, "kfunc");
9252	if (IS_ERR(btf))
9253	return PTR_ERR(btf);
9254
9255	for (i = 0; i < kset->set->cnt; i++) {
9256	ret = btf_check_kfunc_protos(btf, btf_relocate_id(btf, kset->set->pairs[i].id),
9257	kset->set->pairs[i].flags);
9258	if (ret)
9259	goto err_out;
9260	}
9261
9262	ret = btf_populate_kfunc_set(btf, hook, kset);
9263
9264	err_out:
9265	btf_put(btf);
9266	return ret;
9267	}
9268
9269	/* This function must be invoked only from initcalls/module init functions */
9270	int register_btf_kfunc_id_set(enum bpf_prog_type prog_type,
9271	const struct btf_kfunc_id_set *kset)
9272	{
9273	enum btf_kfunc_hook hook;
9274
9275	/* All kfuncs need to be tagged as such in BTF.
9276	* WARN() for initcall registrations that do not check errors.
9277	*/
9278	if (!(kset->set->flags & BTF_SET8_KFUNCS)) {
9279	WARN_ON(!kset->owner);
9280	return -EINVAL;
9281	}
9282
9283	hook = bpf_prog_type_to_kfunc_hook(prog_type);
9284	return __register_btf_kfunc_id_set(hook, kset);
9285	}
9286	EXPORT_SYMBOL_GPL(register_btf_kfunc_id_set);
9287
9288	/* This function must be invoked only from initcalls/module init functions */
9289	int register_btf_fmodret_id_set(const struct btf_kfunc_id_set *kset)
9290	{
9291	return __register_btf_kfunc_id_set(BTF_KFUNC_HOOK_FMODRET, kset);
9292	}
9293	EXPORT_SYMBOL_GPL(register_btf_fmodret_id_set);
9294
9295	s32 btf_find_dtor_kfunc(struct btf *btf, u32 btf_id)
9296	{
9297	struct btf_id_dtor_kfunc_tab *tab = btf->dtor_kfunc_tab;
9298	struct btf_id_dtor_kfunc *dtor;
9299
9300	if (!tab)
9301	return -ENOENT;
9302	/* Even though the size of tab->dtors[0] is > sizeof(u32), we only need
9303	* to compare the first u32 with btf_id, so we can reuse btf_id_cmp_func.
9304	*/
9305	BUILD_BUG_ON(offsetof(struct btf_id_dtor_kfunc, btf_id) != 0);
9306	dtor = bsearch(&btf_id, tab->dtors, tab->cnt, sizeof(tab->dtors[0]), btf_id_cmp_func);
9307	if (!dtor)
9308	return -ENOENT;
9309	return dtor->kfunc_btf_id;
9310	}
9311
9312	static int btf_check_dtor_kfuncs(struct btf btf, const struct btf_id_dtor_kfunc dtors, u32 cnt)
9313	{
9314	const struct btf_type dtor_func, dtor_func_proto, *t;
9315	const struct btf_param *args;
9316	s32 dtor_btf_id;
9317	u32 nr_args, i;
9318
9319	for (i = 0; i < cnt; i++) {
9320	dtor_btf_id = btf_relocate_id(btf, dtors[i].kfunc_btf_id);
9321
9322	dtor_func = btf_type_by_id(btf, dtor_btf_id);
9323	if (!dtor_func \|\| !btf_type_is_func(dtor_func))
9324	return -EINVAL;
9325
9326	dtor_func_proto = btf_type_by_id(btf, dtor_func->type);
9327	if (!dtor_func_proto \|\| !btf_type_is_func_proto(dtor_func_proto))
9328	return -EINVAL;
9329
9330	/* Make sure the prototype of the destructor kfunc is 'void func(type )' /
9331	t = btf_type_by_id(btf, dtor_func_proto->type);
9332	if (!t \|\| !btf_type_is_void(t))
9333	return -EINVAL;
9334
9335	nr_args = btf_type_vlen(dtor_func_proto);
9336	if (nr_args != 1)
9337	return -EINVAL;
9338	args = btf_params(dtor_func_proto);
9339	t = btf_type_by_id(btf, args[0].type);
9340	/* Allow any pointer type, as width on targets Linux supports
9341	* will be same for all pointer types (i.e. sizeof(void *))
9342	*/
9343	if (!t \|\| !btf_type_is_ptr(t))
9344	return -EINVAL;
9345
9346	if (IS_ENABLED(CONFIG_CFI)) {
9347	/* Ensure the destructor kfunc type matches btf_dtor_kfunc_t */
9348	t = btf_type_by_id(btf, t->type);
9349	if (!btf_type_is_void(t))
9350	return -EINVAL;
9351	}
9352	}
9353	return 0;
9354	}
9355
9356	/* This function must be invoked only from initcalls/module init functions */
9357	int register_btf_id_dtor_kfuncs(const struct btf_id_dtor_kfunc *dtors, u32 add_cnt,
9358	struct module *owner)
9359	{
9360	struct btf_id_dtor_kfunc_tab *tab;
9361	struct btf *btf;
9362	u32 tab_cnt, i;
9363	int ret;
9364
9365	btf = btf_get_module_btf(owner);
9366	if (!btf)
9367	return check_btf_kconfigs(owner, "dtor kfuncs");
9368	if (IS_ERR(btf))
9369	return PTR_ERR(btf);
9370
9371	if (add_cnt >= BTF_DTOR_KFUNC_MAX_CNT) {
9372	pr_err("cannot register more than %d kfunc destructors\n", BTF_DTOR_KFUNC_MAX_CNT);
9373	ret = -E2BIG;
9374	goto end;
9375	}
9376
9377	/* Ensure that the prototype of dtor kfuncs being registered is sane */
9378	ret = btf_check_dtor_kfuncs(btf, dtors, add_cnt);
9379	if (ret < 0)
9380	goto end;
9381
9382	tab = btf->dtor_kfunc_tab;
9383	/* Only one call allowed for modules */
9384	if (WARN_ON_ONCE(tab && btf_is_module(btf))) {
9385	ret = -EINVAL;
9386	goto end;
9387	}
9388
9389	tab_cnt = tab ? tab->cnt : 0;
9390	if (tab_cnt > U32_MAX - add_cnt) {
9391	ret = -EOVERFLOW;
9392	goto end;
9393	}
9394	if (tab_cnt + add_cnt >= BTF_DTOR_KFUNC_MAX_CNT) {
9395	pr_err("cannot register more than %d kfunc destructors\n", BTF_DTOR_KFUNC_MAX_CNT);
9396	ret = -E2BIG;
9397	goto end;
9398	}
9399
9400	tab = krealloc(btf->dtor_kfunc_tab,
9401	struct_size(tab, dtors, tab_cnt + add_cnt),
9402	GFP_KERNEL \| __GFP_NOWARN);
9403	if (!tab) {
9404	ret = -ENOMEM;
9405	goto end;
9406	}
9407
9408	if (!btf->dtor_kfunc_tab)
9409	tab->cnt = 0;
9410	btf->dtor_kfunc_tab = tab;
9411
9412	memcpy(tab->dtors + tab->cnt, dtors, add_cnt * sizeof(tab->dtors[0]));
9413
9414	/* remap BTF ids based on BTF relocation (if any) */
9415	for (i = tab_cnt; i < tab_cnt + add_cnt; i++) {
9416	tab->dtors[i].btf_id = btf_relocate_id(btf, tab->dtors[i].btf_id);
9417	tab->dtors[i].kfunc_btf_id = btf_relocate_id(btf, tab->dtors[i].kfunc_btf_id);
9418	}
9419
9420	tab->cnt += add_cnt;
9421
9422	sort(tab->dtors, tab->cnt, sizeof(tab->dtors[0]), btf_id_cmp_func, NULL);
9423
9424	end:
9425	if (ret)
9426	btf_free_dtor_kfunc_tab(btf);
9427	btf_put(btf);
9428	return ret;
9429	}
9430	EXPORT_SYMBOL_GPL(register_btf_id_dtor_kfuncs);
9431
9432	#define MAX_TYPES_ARE_COMPAT_DEPTH 2
9433
9434	/* Check local and target types for compatibility. This check is used for
9435	* type-based CO-RE relocations and follow slightly different rules than
9436	* field-based relocations. This function assumes that root types were already
9437	* checked for name match. Beyond that initial root-level name check, names
9438	* are completely ignored. Compatibility rules are as follows:
9439	* - any two STRUCTs/UNIONs/FWDs/ENUMs/INTs/ENUM64s are considered compatible, but
9440	* kind should match for local and target types (i.e., STRUCT is not
9441	* compatible with UNION);
9442	* - for ENUMs/ENUM64s, the size is ignored;
9443	* - for INT, size and signedness are ignored;
9444	* - for ARRAY, dimensionality is ignored, element types are checked for
9445	* compatibility recursively;
9446	* - CONST/VOLATILE/RESTRICT modifiers are ignored;
9447	* - TYPEDEFs/PTRs are compatible if types they pointing to are compatible;
9448	* - FUNC_PROTOs are compatible if they have compatible signature: same
9449	* number of input args and compatible return and argument types.
9450	* These rules are not set in stone and probably will be adjusted as we get
9451	* more experience with using BPF CO-RE relocations.
9452	*/
9453	int bpf_core_types_are_compat(const struct btf *local_btf, __u32 local_id,
9454	const struct btf *targ_btf, __u32 targ_id)
9455	{
9456	return __bpf_core_types_are_compat(local_btf, local_id, targ_btf, targ_id,
9457	MAX_TYPES_ARE_COMPAT_DEPTH);
9458	}
9459
9460	#define MAX_TYPES_MATCH_DEPTH 2
9461
9462	int bpf_core_types_match(const struct btf *local_btf, u32 local_id,
9463	const struct btf *targ_btf, u32 targ_id)
9464	{
9465	return __bpf_core_types_match(local_btf, local_id, targ_btf, targ_id, false,
9466	MAX_TYPES_MATCH_DEPTH);
9467	}
9468
9469	static bool bpf_core_is_flavor_sep(const char *s)
9470	{
9471	/* check X___Y name pattern, where X and Y are not underscores */
9472	return s[0] != '_' && /* X */
9473	s[1] == '_' && s[2] == '_' && s[3] == '_' && /* ___ */
9474	s[4] != '_'; /* Y */
9475	}
9476
9477	size_t bpf_core_essential_name_len(const char *name)
9478	{
9479	size_t n = strlen(name);
9480	int i;
9481
9482	for (i = n - 5; i >= 0; i--) {
9483	if (bpf_core_is_flavor_sep(name + i))
9484	return i + 1;
9485	}
9486	return n;
9487	}
9488
9489	static void bpf_free_cands(struct bpf_cand_cache *cands)
9490	{
9491	if (!cands->cnt)
9492	/* empty candidate array was allocated on stack */
9493	return;
9494	kfree(cands);
9495	}
9496
9497	static void bpf_free_cands_from_cache(struct bpf_cand_cache *cands)
9498	{
9499	kfree(cands->name);
9500	kfree(cands);
9501	}
9502
9503	#define VMLINUX_CAND_CACHE_SIZE 31
9504	static struct bpf_cand_cache *vmlinux_cand_cache[VMLINUX_CAND_CACHE_SIZE];
9505
9506	#define MODULE_CAND_CACHE_SIZE 31
9507	static struct bpf_cand_cache *module_cand_cache[MODULE_CAND_CACHE_SIZE];
9508
9509	static void __print_cand_cache(struct bpf_verifier_log *log,
9510	struct bpf_cand_cache **cache,
9511	int cache_size)
9512	{
9513	struct bpf_cand_cache *cc;
9514	int i, j;
9515
9516	for (i = 0; i < cache_size; i++) {
9517	cc = cache[i];
9518	if (!cc)
9519	continue;
9520	bpf_log(log, "[%d]%s(", i, cc->name);
9521	for (j = 0; j < cc->cnt; j++) {
9522	bpf_log(log, "%d", cc->cands[j].id);
9523	if (j < cc->cnt - 1)
9524	bpf_log(log, " ");
9525	}
9526	bpf_log(log, "), ");
9527	}
9528	}
9529
9530	static void print_cand_cache(struct bpf_verifier_log *log)
9531	{
9532	mutex_lock(&cand_cache_mutex);
9533	bpf_log(log, "vmlinux_cand_cache:");
9534	__print_cand_cache(log, vmlinux_cand_cache, VMLINUX_CAND_CACHE_SIZE);
9535	bpf_log(log, "\nmodule_cand_cache:");
9536	__print_cand_cache(log, module_cand_cache, MODULE_CAND_CACHE_SIZE);
9537	bpf_log(log, "\n");
9538	mutex_unlock(&cand_cache_mutex);
9539	}
9540
9541	static u32 hash_cands(struct bpf_cand_cache *cands)
9542	{
9543	return jhash(cands->name, cands->name_len, 0);
9544	}
9545
9546	static struct bpf_cand_cache check_cand_cache(struct bpf_cand_cache cands,
9547	struct bpf_cand_cache **cache,
9548	int cache_size)
9549	{
9550	struct bpf_cand_cache *cc = cache[hash_cands(cands) % cache_size];
9551
9552	if (cc && cc->name_len == cands->name_len &&
9553	!strncmp(cc->name, cands->name, cands->name_len))
9554	return cc;
9555	return NULL;
9556	}
9557
9558	static size_t sizeof_cands(int cnt)
9559	{
9560	return offsetof(struct bpf_cand_cache, cands[cnt]);
9561	}
9562
9563	static struct bpf_cand_cache populate_cand_cache(struct bpf_cand_cache cands,
9564	struct bpf_cand_cache **cache,
9565	int cache_size)
9566	{
9567	struct bpf_cand_cache *cc = &cache[hash_cands(cands) % cache_size], new_cands;
9568
9569	if (*cc) {
9570	bpf_free_cands_from_cache(*cc);
9571	*cc = NULL;
9572	}
9573	new_cands = kmemdup(cands, sizeof_cands(cands->cnt), GFP_KERNEL_ACCOUNT);
9574	if (!new_cands) {
9575	bpf_free_cands(cands);
9576	return ERR_PTR(-ENOMEM);
9577	}
9578	/* strdup the name, since it will stay in cache.
9579	* the cands->name points to strings in prog's BTF and the prog can be unloaded.
9580	*/
9581	new_cands->name = kmemdup_nul(cands->name, cands->name_len, GFP_KERNEL_ACCOUNT);
9582	bpf_free_cands(cands);
9583	if (!new_cands->name) {
9584	kfree(new_cands);
9585	return ERR_PTR(-ENOMEM);
9586	}
9587	*cc = new_cands;
9588	return new_cands;
9589	}
9590
9591	#ifdef CONFIG_DEBUG_INFO_BTF_MODULES
9592	static void __purge_cand_cache(struct btf btf, struct bpf_cand_cache *cache,
9593	int cache_size)
9594	{
9595	struct bpf_cand_cache *cc;
9596	int i, j;
9597
9598	for (i = 0; i < cache_size; i++) {
9599	cc = cache[i];
9600	if (!cc)
9601	continue;
9602	if (!btf) {
9603	/* when new module is loaded purge all of module_cand_cache,
9604	* since new module might have candidates with the name
9605	* that matches cached cands.
9606	*/
9607	bpf_free_cands_from_cache(cc);
9608	cache[i] = NULL;
9609	continue;
9610	}
9611	/* when module is unloaded purge cache entries
9612	* that match module's btf
9613	*/
9614	for (j = 0; j < cc->cnt; j++)
9615	if (cc->cands[j].btf == btf) {
9616	bpf_free_cands_from_cache(cc);
9617	cache[i] = NULL;
9618	break;
9619	}
9620	}
9621
9622	}
9623
9624	static void purge_cand_cache(struct btf *btf)
9625	{
9626	mutex_lock(&cand_cache_mutex);
9627	__purge_cand_cache(btf, module_cand_cache, MODULE_CAND_CACHE_SIZE);
9628	mutex_unlock(&cand_cache_mutex);
9629	}
9630	#endif
9631
9632	static struct bpf_cand_cache *
9633	bpf_core_add_cands(struct bpf_cand_cache cands, const struct btf targ_btf,
9634	int targ_start_id)
9635	{
9636	struct bpf_cand_cache *new_cands;
9637	const struct btf_type *t;
9638	const char *targ_name;
9639	size_t targ_essent_len;
9640	int n, i;
9641
9642	n = btf_nr_types(targ_btf);
9643	for (i = targ_start_id; i < n; i++) {
9644	t = btf_type_by_id(targ_btf, i);
9645	if (btf_kind(t) != cands->kind)
9646	continue;
9647
9648	targ_name = btf_name_by_offset(targ_btf, t->name_off);
9649	if (!targ_name)
9650	continue;
9651
9652	/* the resched point is before strncmp to make sure that search
9653	* for non-existing name will have a chance to schedule().
9654	*/
9655	cond_resched();
9656
9657	if (strncmp(cands->name, targ_name, cands->name_len) != 0)
9658	continue;
9659
9660	targ_essent_len = bpf_core_essential_name_len(targ_name);
9661	if (targ_essent_len != cands->name_len)
9662	continue;
9663
9664	/* most of the time there is only one candidate for a given kind+name pair */
9665	new_cands = kmalloc(sizeof_cands(cands->cnt + 1), GFP_KERNEL_ACCOUNT);
9666	if (!new_cands) {
9667	bpf_free_cands(cands);
9668	return ERR_PTR(-ENOMEM);
9669	}
9670
9671	memcpy(new_cands, cands, sizeof_cands(cands->cnt));
9672	bpf_free_cands(cands);
9673	cands = new_cands;
9674	cands->cands[cands->cnt].btf = targ_btf;
9675	cands->cands[cands->cnt].id = i;
9676	cands->cnt++;
9677	}
9678	return cands;
9679	}
9680
9681	static struct bpf_cand_cache *
9682	bpf_core_find_cands(struct bpf_core_ctx *ctx, u32 local_type_id)
9683	{
9684	struct bpf_cand_cache cands, cc, local_cand = {};
9685	const struct btf *local_btf = ctx->btf;
9686	const struct btf_type *local_type;
9687	const struct btf *main_btf;
9688	size_t local_essent_len;
9689	struct btf *mod_btf;
9690	const char *name;
9691	int id;
9692
9693	main_btf = bpf_get_btf_vmlinux();
9694	if (IS_ERR(main_btf))
9695	return ERR_CAST(main_btf);
9696	if (!main_btf)
9697	return ERR_PTR(-EINVAL);
9698
9699	local_type = btf_type_by_id(local_btf, local_type_id);
9700	if (!local_type)
9701	return ERR_PTR(-EINVAL);
9702
9703	name = btf_name_by_offset(local_btf, local_type->name_off);
9704	if (str_is_empty(name))
9705	return ERR_PTR(-EINVAL);
9706	local_essent_len = bpf_core_essential_name_len(name);
9707
9708	cands = &local_cand;
9709	cands->name = name;
9710	cands->kind = btf_kind(local_type);
9711	cands->name_len = local_essent_len;
9712
9713	cc = check_cand_cache(cands, vmlinux_cand_cache, VMLINUX_CAND_CACHE_SIZE);
9714	/* cands is a pointer to stack here */
9715	if (cc) {
9716	if (cc->cnt)
9717	return cc;
9718	goto check_modules;
9719	}
9720
9721	/* Attempt to find target candidates in vmlinux BTF first */
9722	cands = bpf_core_add_cands(cands, main_btf, btf_named_start_id(main_btf, true));
9723	if (IS_ERR(cands))
9724	return ERR_CAST(cands);
9725
9726	/* cands is a pointer to kmalloced memory here if cands->cnt > 0 */
9727
9728	/* populate cache even when cands->cnt == 0 */
9729	cc = populate_cand_cache(cands, vmlinux_cand_cache, VMLINUX_CAND_CACHE_SIZE);
9730	if (IS_ERR(cc))
9731	return ERR_CAST(cc);
9732
9733	/* if vmlinux BTF has any candidate, don't go for module BTFs */
9734	if (cc->cnt)
9735	return cc;
9736
9737	check_modules:
9738	/* cands is a pointer to stack here and cands->cnt == 0 */
9739	cc = check_cand_cache(cands, module_cand_cache, MODULE_CAND_CACHE_SIZE);
9740	if (cc)
9741	/* if cache has it return it even if cc->cnt == 0 */
9742	return cc;
9743
9744	/* If candidate is not found in vmlinux's BTF then search in module's BTFs */
9745	spin_lock_bh(&btf_idr_lock);
9746	idr_for_each_entry(&btf_idr, mod_btf, id) {
9747	if (!btf_is_module(mod_btf))
9748	continue;
9749	/* linear search could be slow hence unlock/lock
9750	* the IDR to avoiding holding it for too long
9751	*/
9752	btf_get(mod_btf);
9753	spin_unlock_bh(&btf_idr_lock);
9754	cands = bpf_core_add_cands(cands, mod_btf, btf_named_start_id(mod_btf, true));
9755	btf_put(mod_btf);
9756	if (IS_ERR(cands))
9757	return ERR_CAST(cands);
9758	spin_lock_bh(&btf_idr_lock);
9759	}
9760	spin_unlock_bh(&btf_idr_lock);
9761	/* cands is a pointer to kmalloced memory here if cands->cnt > 0
9762	* or pointer to stack if cands->cnd == 0.
9763	* Copy it into the cache even when cands->cnt == 0 and
9764	* return the result.
9765	*/
9766	return populate_cand_cache(cands, module_cand_cache, MODULE_CAND_CACHE_SIZE);
9767	}
9768
9769	int bpf_core_apply(struct bpf_core_ctx ctx, const struct bpf_core_relo relo,
9770	int relo_idx, void *insn)
9771	{
9772	bool need_cands = relo->kind != BPF_CORE_TYPE_ID_LOCAL;
9773	struct bpf_core_cand_list cands = {};
9774	struct bpf_core_relo_res targ_res;
9775	struct bpf_core_spec *specs;
9776	const struct btf_type *type;
9777	int err;
9778
9779	/* ~4k of temp memory necessary to convert LLVM spec like "0:1:0:5"
9780	* into arrays of btf_ids of struct fields and array indices.
9781	*/
9782	specs = kzalloc_objs(*specs, 3, GFP_KERNEL_ACCOUNT);
9783	if (!specs)
9784	return -ENOMEM;
9785
9786	type = btf_type_by_id(ctx->btf, relo->type_id);
9787	if (!type) {
9788	bpf_log(ctx->log, "relo #%u: bad type id %u\n",
9789	relo_idx, relo->type_id);
9790	kfree(specs);
9791	return -EINVAL;
9792	}
9793
9794	if (need_cands) {
9795	struct bpf_cand_cache *cc;
9796	int i;
9797
9798	mutex_lock(&cand_cache_mutex);
9799	cc = bpf_core_find_cands(ctx, relo->type_id);
9800	if (IS_ERR(cc)) {
9801	bpf_log(ctx->log, "target candidate search failed for %d\n",
9802	relo->type_id);
9803	err = PTR_ERR(cc);
9804	goto out;
9805	}
9806	if (cc->cnt) {
9807	cands.cands = kzalloc_objs(*cands.cands, cc->cnt,
9808	GFP_KERNEL_ACCOUNT);
9809	if (!cands.cands) {
9810	err = -ENOMEM;
9811	goto out;
9812	}
9813	}
9814	for (i = 0; i < cc->cnt; i++) {
9815	bpf_log(ctx->log,
9816	"CO-RE relocating %s %s: found target candidate [%d]\n",
9817	btf_kind_str[cc->kind], cc->name, cc->cands[i].id);
9818	cands.cands[i].btf = cc->cands[i].btf;
9819	cands.cands[i].id = cc->cands[i].id;
9820	}
9821	cands.len = cc->cnt;
9822	/* cand_cache_mutex needs to span the cache lookup and
9823	* copy of btf pointer into bpf_core_cand_list,
9824	* since module can be unloaded while bpf_core_calc_relo_insn
9825	* is working with module's btf.
9826	*/
9827	}
9828
9829	err = bpf_core_calc_relo_insn((void *)ctx->log, relo, relo_idx, ctx->btf, &cands, specs,
9830	&targ_res);
9831	if (err)
9832	goto out;
9833
9834	err = bpf_core_patch_insn((void *)ctx->log, insn, relo->insn_off / 8, relo, relo_idx,
9835	&targ_res);
9836
9837	out:
9838	kfree(specs);
9839	if (need_cands) {
9840	kfree(cands.cands);
9841	mutex_unlock(&cand_cache_mutex);
9842	if (ctx->log->level & BPF_LOG_LEVEL2)
9843	print_cand_cache(ctx->log);
9844	}
9845	return err;
9846	}
9847
9848	bool btf_nested_type_is_trusted(struct bpf_verifier_log *log,
9849	const struct bpf_reg_state *reg,
9850	const char field_name, u32 btf_id, const char suffix)
9851	{
9852	struct btf *btf = reg->btf;
9853	const struct btf_type walk_type, safe_type;
9854	const char *tname;
9855	char safe_tname[64];
9856	long ret, safe_id;
9857	const struct btf_member *member;
9858	u32 i;
9859
9860	walk_type = btf_type_by_id(btf, reg->btf_id);
9861	if (!walk_type)
9862	return false;
9863
9864	tname = btf_name_by_offset(btf, walk_type->name_off);
9865
9866	ret = snprintf(safe_tname, sizeof(safe_tname), "%s%s", tname, suffix);
9867	if (ret >= sizeof(safe_tname))
9868	return false;
9869
9870	safe_id = btf_find_by_name_kind(btf, safe_tname, BTF_INFO_KIND(walk_type->info));
9871	if (safe_id < 0)
9872	return false;
9873
9874	safe_type = btf_type_by_id(btf, safe_id);
9875	if (!safe_type)
9876	return false;
9877
9878	for_each_member(i, safe_type, member) {
9879	const char *m_name = __btf_name_by_offset(btf, member->name_off);
9880	const struct btf_type *mtype = btf_type_by_id(btf, member->type);
9881	u32 id;
9882
9883	if (!btf_type_is_ptr(mtype))
9884	continue;
9885
9886	btf_type_skip_modifiers(btf, mtype->type, &id);
9887	/* If we match on both type and name, the field is considered trusted. */
9888	if (btf_id == id && !strcmp(field_name, m_name))
9889	return true;
9890	}
9891
9892	return false;
9893	}
9894
9895	bool btf_type_ids_nocast_alias(struct bpf_verifier_log *log,
9896	const struct btf *reg_btf, u32 reg_id,
9897	const struct btf *arg_btf, u32 arg_id)
9898	{
9899	const char reg_name, arg_name, *search_needle;
9900	const struct btf_type reg_type, arg_type;
9901	int reg_len, arg_len, cmp_len;
9902	size_t pattern_len = sizeof(NOCAST_ALIAS_SUFFIX) - sizeof(char);
9903
9904	reg_type = btf_type_by_id(reg_btf, reg_id);
9905	if (!reg_type)
9906	return false;
9907
9908	arg_type = btf_type_by_id(arg_btf, arg_id);
9909	if (!arg_type)
9910	return false;
9911
9912	reg_name = btf_name_by_offset(reg_btf, reg_type->name_off);
9913	arg_name = btf_name_by_offset(arg_btf, arg_type->name_off);
9914
9915	reg_len = strlen(reg_name);
9916	arg_len = strlen(arg_name);
9917
9918	/* Exactly one of the two type names may be suffixed with ___init, so
9919	* if the strings are the same size, they can't possibly be no-cast
9920	* aliases of one another. If you have two of the same type names, e.g.
9921	* they're both nf_conn___init, it would be improper to return true
9922	* because they are _not_ no-cast aliases, they are the same type.
9923	*/
9924	if (reg_len == arg_len)
9925	return false;
9926
9927	/* Either of the two names must be the other name, suffixed with ___init. */
9928	if ((reg_len != arg_len + pattern_len) &&
9929	(arg_len != reg_len + pattern_len))
9930	return false;
9931
9932	if (reg_len < arg_len) {
9933	search_needle = strstr(arg_name, NOCAST_ALIAS_SUFFIX);
9934	cmp_len = reg_len;
9935	} else {
9936	search_needle = strstr(reg_name, NOCAST_ALIAS_SUFFIX);
9937	cmp_len = arg_len;
9938	}
9939
9940	if (!search_needle)
9941	return false;
9942
9943	/* ___init suffix must come at the end of the name */
9944	if (*(search_needle + pattern_len) != '\0')
9945	return false;
9946
9947	return !strncmp(reg_name, arg_name, cmp_len);
9948	}
9949
9950	#ifdef CONFIG_BPF_JIT
9951	static int
9952	btf_add_struct_ops(struct btf btf, struct bpf_struct_ops st_ops,
9953	struct bpf_verifier_log *log)
9954	{
9955	struct btf_struct_ops_tab tab, new_tab;
9956	int i, err;
9957
9958	tab = btf->struct_ops_tab;
9959	if (!tab) {
9960	tab = kzalloc_flex(*tab, ops, 4);
9961	if (!tab)
9962	return -ENOMEM;
9963	tab->capacity = 4;
9964	btf->struct_ops_tab = tab;
9965	}
9966
9967	for (i = 0; i < tab->cnt; i++)
9968	if (tab->ops[i].st_ops == st_ops)
9969	return -EEXIST;
9970
9971	if (tab->cnt == tab->capacity) {
9972	new_tab = krealloc(tab,
9973	struct_size(tab, ops, tab->capacity * 2),
9974	GFP_KERNEL);
9975	if (!new_tab)
9976	return -ENOMEM;
9977	tab = new_tab;
9978	tab->capacity *= 2;
9979	btf->struct_ops_tab = tab;
9980	}
9981
9982	tab->ops[btf->struct_ops_tab->cnt].st_ops = st_ops;
9983
9984	err = bpf_struct_ops_desc_init(&tab->ops[btf->struct_ops_tab->cnt], btf, log);
9985	if (err)
9986	return err;
9987
9988	btf->struct_ops_tab->cnt++;
9989
9990	return 0;
9991	}
9992
9993	const struct bpf_struct_ops_desc *
9994	bpf_struct_ops_find_value(struct btf *btf, u32 value_id)
9995	{
9996	const struct bpf_struct_ops_desc *st_ops_list;
9997	unsigned int i;
9998	u32 cnt;
9999
10000	if (!value_id)
10001	return NULL;
10002	if (!btf->struct_ops_tab)
10003	return NULL;
10004
10005	cnt = btf->struct_ops_tab->cnt;
10006	st_ops_list = btf->struct_ops_tab->ops;
10007	for (i = 0; i < cnt; i++) {
10008	if (st_ops_list[i].value_id == value_id)
10009	return &st_ops_list[i];
10010	}
10011
10012	return NULL;
10013	}
10014
10015	const struct bpf_struct_ops_desc *
10016	bpf_struct_ops_find(struct btf *btf, u32 type_id)
10017	{
10018	const struct bpf_struct_ops_desc *st_ops_list;
10019	unsigned int i;
10020	u32 cnt;
10021
10022	if (!type_id)
10023	return NULL;
10024	if (!btf->struct_ops_tab)
10025	return NULL;
10026
10027	cnt = btf->struct_ops_tab->cnt;
10028	st_ops_list = btf->struct_ops_tab->ops;
10029	for (i = 0; i < cnt; i++) {
10030	if (st_ops_list[i].type_id == type_id)
10031	return &st_ops_list[i];
10032	}
10033
10034	return NULL;
10035	}
10036
10037	int __register_bpf_struct_ops(struct bpf_struct_ops *st_ops)
10038	{
10039	struct bpf_verifier_log *log;
10040	struct btf *btf;
10041	int err = 0;
10042
10043	btf = btf_get_module_btf(st_ops->owner);
10044	if (!btf)
10045	return check_btf_kconfigs(st_ops->owner, "struct_ops");
10046	if (IS_ERR(btf))
10047	return PTR_ERR(btf);
10048
10049	log = kzalloc_obj(*log, GFP_KERNEL \| __GFP_NOWARN);
10050	if (!log) {
10051	err = -ENOMEM;
10052	goto errout;
10053	}
10054
10055	log->level = BPF_LOG_KERNEL;
10056
10057	err = btf_add_struct_ops(btf, st_ops, log);
10058
10059	errout:
10060	kfree(log);
10061	btf_put(btf);
10062
10063	return err;
10064	}
10065	EXPORT_SYMBOL_GPL(__register_bpf_struct_ops);
10066	#endif
10067
10068	bool btf_param_match_suffix(const struct btf *btf,
10069	const struct btf_param *arg,
10070	const char *suffix)
10071	{
10072	int suffix_len = strlen(suffix), len;
10073	const char *param_name;
10074
10075	/* In the future, this can be ported to use BTF tagging */
10076	param_name = btf_name_by_offset(btf, arg->name_off);
10077	if (str_is_empty(param_name))
10078	return false;
10079	len = strlen(param_name);
10080	if (len <= suffix_len)
10081	return false;
10082	param_name += len - suffix_len;
10083	return !strncmp(param_name, suffix, suffix_len);
10084	}