linux - include/linux/bpf.h

3235 lines · 100.3 KiB

1	/* SPDX-License-Identifier: GPL-2.0-only */
2	/* Copyright (c) 2011-2014 PLUMgrid, http://plumgrid.com
3	*/
4	#ifndef _LINUX_BPF_H
5	#define _LINUX_BPF_H 1
6
7	#include <uapi/linux/bpf.h>
8	#include <uapi/linux/filter.h>
9
10	#include <linux/workqueue.h>
11	#include <linux/file.h>
12	#include <linux/percpu.h>
13	#include <linux/err.h>
14	#include <linux/rbtree_latch.h>
15	#include <linux/numa.h>
16	#include <linux/mm_types.h>
17	#include <linux/wait.h>
18	#include <linux/refcount.h>
19	#include <linux/mutex.h>
20	#include <linux/module.h>
21	#include <linux/kallsyms.h>
22	#include <linux/capability.h>
23	#include <linux/sched/mm.h>
24	#include <linux/slab.h>
25	#include <linux/percpu-refcount.h>
26	#include <linux/stddef.h>
27	#include <linux/bpfptr.h>
28	#include <linux/btf.h>
29	#include <linux/rcupdate_trace.h>
30	#include <linux/static_call.h>
31	#include <linux/memcontrol.h>
32
33	struct bpf_verifier_env;
34	struct bpf_verifier_log;
35	struct perf_event;
36	struct bpf_prog;
37	struct bpf_prog_aux;
38	struct bpf_map;
39	struct sock;
40	struct seq_file;
41	struct btf;
42	struct btf_type;
43	struct exception_table_entry;
44	struct seq_operations;
45	struct bpf_iter_aux_info;
46	struct bpf_local_storage;
47	struct bpf_local_storage_map;
48	struct kobject;
49	struct mem_cgroup;
50	struct module;
51	struct bpf_func_state;
52	struct ftrace_ops;
53	struct cgroup;
54
55	extern struct idr btf_idr;
56	extern spinlock_t btf_idr_lock;
57	extern struct kobject *btf_kobj;
58	extern struct bpf_mem_alloc bpf_global_ma, bpf_global_percpu_ma;
59	extern bool bpf_global_ma_set;
60
61	typedef u64 (*bpf_callback_t)(u64, u64, u64, u64, u64);
62	typedef int (bpf_iter_init_seq_priv_t)(void private_data,
63	struct bpf_iter_aux_info *aux);
64	typedef void (bpf_iter_fini_seq_priv_t)(void private_data);
65	typedef unsigned int (bpf_func_t)(const void ,
66	const struct bpf_insn *);
67	struct bpf_iter_seq_info {
68	const struct seq_operations *seq_ops;
69	bpf_iter_init_seq_priv_t init_seq_private;
70	bpf_iter_fini_seq_priv_t fini_seq_private;
71	u32 seq_priv_size;
72	};
73
74	/* map is generic key/value storage optionally accessible by eBPF programs */
75	struct bpf_map_ops {
76	/* funcs callable from userspace (via syscall) */
77	int (map_alloc_check)(union bpf_attr attr);
78	struct bpf_map (map_alloc)(union bpf_attr *attr);
79	void (map_release)(struct bpf_map map, struct file *map_file);
80	void (map_free)(struct bpf_map map);
81	int (map_get_next_key)(struct bpf_map map, void key, void next_key);
82	void (map_release_uref)(struct bpf_map map);
83	void (map_lookup_elem_sys_only)(struct bpf_map map, void key);
84	int (map_lookup_batch)(struct bpf_map map, const union bpf_attr *attr,
85	union bpf_attr __user *uattr);
86	int (map_lookup_and_delete_elem)(struct bpf_map map, void *key,
87	void *value, u64 flags);
88	int (map_lookup_and_delete_batch)(struct bpf_map map,
89	const union bpf_attr *attr,
90	union bpf_attr __user *uattr);
91	int (map_update_batch)(struct bpf_map map, struct file *map_file,
92	const union bpf_attr *attr,
93	union bpf_attr __user *uattr);
94	int (map_delete_batch)(struct bpf_map map, const union bpf_attr *attr,
95	union bpf_attr __user *uattr);
96
97	/* funcs callable from userspace and from eBPF programs */
98	void (map_lookup_elem)(struct bpf_map map, void key);
99	long (map_update_elem)(struct bpf_map map, void key, void value, u64 flags);
100	long (map_delete_elem)(struct bpf_map map, void *key);
101	long (map_push_elem)(struct bpf_map map, void *value, u64 flags);
102	long (map_pop_elem)(struct bpf_map map, void *value);
103	long (map_peek_elem)(struct bpf_map map, void *value);
104	void (map_lookup_percpu_elem)(struct bpf_map map, void key, u32 cpu);
105
106	/* funcs called by prog_array and perf_event_array map */
107	void (map_fd_get_ptr)(struct bpf_map map, struct file map_file,
108	int fd);
109	void (map_fd_put_ptr)(void ptr);
110	int (map_gen_lookup)(struct bpf_map map, struct bpf_insn *insn_buf);
111	u32 (map_fd_sys_lookup_elem)(void ptr);
112	void (map_seq_show_elem)(struct bpf_map map, void *key,
113	struct seq_file *m);
114	int (map_check_btf)(const struct bpf_map map,
115	const struct btf *btf,
116	const struct btf_type *key_type,
117	const struct btf_type *value_type);
118
119	/* Prog poke tracking helpers. */
120	int (map_poke_track)(struct bpf_map map, struct bpf_prog_aux *aux);
121	void (map_poke_untrack)(struct bpf_map map, struct bpf_prog_aux *aux);
122	void (map_poke_run)(struct bpf_map map, u32 key, struct bpf_prog *old,
123	struct bpf_prog *new);
124
125	/* Direct value access helpers. */
126	int (map_direct_value_addr)(const struct bpf_map map,
127	u64 *imm, u32 off);
128	int (map_direct_value_meta)(const struct bpf_map map,
129	u64 imm, u32 *off);
130	int (map_mmap)(struct bpf_map map, struct vm_area_struct *vma);
131	__poll_t (map_poll)(struct bpf_map map, struct file *filp,
132	struct poll_table_struct *pts);
133
134	/* Functions called by bpf_local_storage maps */
135	int (map_local_storage_charge)(struct bpf_local_storage_map smap,
136	void *owner, u32 size);
137	void (map_local_storage_uncharge)(struct bpf_local_storage_map smap,
138	void *owner, u32 size);
139	struct bpf_local_storage __rcu ** (map_owner_storage_ptr)(void owner);
140
141	/* Misc helpers.*/
142	long (map_redirect)(struct bpf_map map, u64 key, u64 flags);
143
144	/* map_meta_equal must be implemented for maps that can be
145	* used as an inner map. It is a runtime check to ensure
146	* an inner map can be inserted to an outer map.
147	*
148	* Some properties of the inner map has been used during the
149	* verification time. When inserting an inner map at the runtime,
150	* map_meta_equal has to ensure the inserting map has the same
151	* properties that the verifier has used earlier.
152	*/
153	bool (map_meta_equal)(const struct bpf_map meta0,
154	const struct bpf_map *meta1);
155
156
157	int (map_set_for_each_callback_args)(struct bpf_verifier_env env,
158	struct bpf_func_state *caller,
159	struct bpf_func_state *callee);
160	long (map_for_each_callback)(struct bpf_map map,
161	bpf_callback_t callback_fn,
162	void *callback_ctx, u64 flags);
163
164	u64 (map_mem_usage)(const struct bpf_map map);
165
166	/* BTF id of struct allocated by map_alloc */
167	int *map_btf_id;
168
169	/* bpf_iter info used to open a seq_file */
170	const struct bpf_iter_seq_info *iter_seq_info;
171	};
172
173	enum {
174	/* Support at most 10 fields in a BTF type */
175	BTF_FIELDS_MAX = 10,
176	};
177
178	enum btf_field_type {
179	BPF_SPIN_LOCK = (1 << 0),
180	BPF_TIMER = (1 << 1),
181	BPF_KPTR_UNREF = (1 << 2),
182	BPF_KPTR_REF = (1 << 3),
183	BPF_KPTR_PERCPU = (1 << 4),
184	BPF_KPTR = BPF_KPTR_UNREF \| BPF_KPTR_REF \| BPF_KPTR_PERCPU,
185	BPF_LIST_HEAD = (1 << 5),
186	BPF_LIST_NODE = (1 << 6),
187	BPF_RB_ROOT = (1 << 7),
188	BPF_RB_NODE = (1 << 8),
189	BPF_GRAPH_NODE_OR_ROOT = BPF_LIST_NODE \| BPF_LIST_HEAD \|
190	BPF_RB_NODE \| BPF_RB_ROOT,
191	BPF_REFCOUNT = (1 << 9),
192	};
193
194	typedef void (btf_dtor_kfunc_t)(void );
195
196	struct btf_field_kptr {
197	struct btf *btf;
198	struct module *module;
199	/* dtor used if btf_is_kernel(btf), otherwise the type is
200	* program-allocated, dtor is NULL, and __bpf_obj_drop_impl is used
201	*/
202	btf_dtor_kfunc_t dtor;
203	u32 btf_id;
204	};
205
206	struct btf_field_graph_root {
207	struct btf *btf;
208	u32 value_btf_id;
209	u32 node_offset;
210	struct btf_record *value_rec;
211	};
212
213	struct btf_field {
214	u32 offset;
215	u32 size;
216	enum btf_field_type type;
217	union {
218	struct btf_field_kptr kptr;
219	struct btf_field_graph_root graph_root;
220	};
221	};
222
223	struct btf_record {
224	u32 cnt;
225	u32 field_mask;
226	int spin_lock_off;
227	int timer_off;
228	int refcount_off;
229	struct btf_field fields[];
230	};
231
232	/* Non-opaque version of bpf_rb_node in uapi/linux/bpf.h */
233	struct bpf_rb_node_kern {
234	struct rb_node rb_node;
235	void *owner;
236	} __attribute__((aligned(8)));
237
238	/* Non-opaque version of bpf_list_node in uapi/linux/bpf.h */
239	struct bpf_list_node_kern {
240	struct list_head list_head;
241	void *owner;
242	} __attribute__((aligned(8)));
243
244	struct bpf_map {
245	/* The first two cachelines with read-mostly members of which some
246	* are also accessed in fast-path (e.g. ops, max_entries).
247	*/
248	const struct bpf_map_ops *ops ____cacheline_aligned;
249	struct bpf_map *inner_map_meta;
250	#ifdef CONFIG_SECURITY
251	void *security;
252	#endif
253	enum bpf_map_type map_type;
254	u32 key_size;
255	u32 value_size;
256	u32 max_entries;
257	u64 map_extra; /* any per-map-type extra fields */
258	u32 map_flags;
259	u32 id;
260	struct btf_record *record;
261	int numa_node;
262	u32 btf_key_type_id;
263	u32 btf_value_type_id;
264	u32 btf_vmlinux_value_type_id;
265	struct btf *btf;
266	#ifdef CONFIG_MEMCG_KMEM
267	struct obj_cgroup *objcg;
268	#endif
269	char name[BPF_OBJ_NAME_LEN];
270	/* The 3rd and 4th cacheline with misc members to avoid false sharing
271	* particularly with refcounting.
272	*/
273	atomic64_t refcnt ____cacheline_aligned;
274	atomic64_t usercnt;
275	struct work_struct work;
276	struct mutex freeze_mutex;
277	atomic64_t writecnt;
278	/* 'Ownership' of program-containing map is claimed by the first program
279	* that is going to use this map or by the first program which FD is
280	* stored in the map to make sure that all callers and callees have the
281	* same prog type, JITed flag and xdp_has_frags flag.
282	*/
283	struct {
284	spinlock_t lock;
285	enum bpf_prog_type type;
286	bool jited;
287	bool xdp_has_frags;
288	} owner;
289	bool bypass_spec_v1;
290	bool frozen; /* write-once; write-protected by freeze_mutex */
291	s64 __percpu *elem_count;
292	};
293
294	static inline const char *btf_field_type_name(enum btf_field_type type)
295	{
296	switch (type) {
297	case BPF_SPIN_LOCK:
298	return "bpf_spin_lock";
299	case BPF_TIMER:
300	return "bpf_timer";
301	case BPF_KPTR_UNREF:
302	case BPF_KPTR_REF:
303	return "kptr";
304	case BPF_KPTR_PERCPU:
305	return "percpu_kptr";
306	case BPF_LIST_HEAD:
307	return "bpf_list_head";
308	case BPF_LIST_NODE:
309	return "bpf_list_node";
310	case BPF_RB_ROOT:
311	return "bpf_rb_root";
312	case BPF_RB_NODE:
313	return "bpf_rb_node";
314	case BPF_REFCOUNT:
315	return "bpf_refcount";
316	default:
317	WARN_ON_ONCE(1);
318	return "unknown";
319	}
320	}
321
322	static inline u32 btf_field_type_size(enum btf_field_type type)
323	{
324	switch (type) {
325	case BPF_SPIN_LOCK:
326	return sizeof(struct bpf_spin_lock);
327	case BPF_TIMER:
328	return sizeof(struct bpf_timer);
329	case BPF_KPTR_UNREF:
330	case BPF_KPTR_REF:
331	case BPF_KPTR_PERCPU:
332	return sizeof(u64);
333	case BPF_LIST_HEAD:
334	return sizeof(struct bpf_list_head);
335	case BPF_LIST_NODE:
336	return sizeof(struct bpf_list_node);
337	case BPF_RB_ROOT:
338	return sizeof(struct bpf_rb_root);
339	case BPF_RB_NODE:
340	return sizeof(struct bpf_rb_node);
341	case BPF_REFCOUNT:
342	return sizeof(struct bpf_refcount);
343	default:
344	WARN_ON_ONCE(1);
345	return 0;
346	}
347	}
348
349	static inline u32 btf_field_type_align(enum btf_field_type type)
350	{
351	switch (type) {
352	case BPF_SPIN_LOCK:
353	return __alignof__(struct bpf_spin_lock);
354	case BPF_TIMER:
355	return __alignof__(struct bpf_timer);
356	case BPF_KPTR_UNREF:
357	case BPF_KPTR_REF:
358	case BPF_KPTR_PERCPU:
359	return __alignof__(u64);
360	case BPF_LIST_HEAD:
361	return __alignof__(struct bpf_list_head);
362	case BPF_LIST_NODE:
363	return __alignof__(struct bpf_list_node);
364	case BPF_RB_ROOT:
365	return __alignof__(struct bpf_rb_root);
366	case BPF_RB_NODE:
367	return __alignof__(struct bpf_rb_node);
368	case BPF_REFCOUNT:
369	return __alignof__(struct bpf_refcount);
370	default:
371	WARN_ON_ONCE(1);
372	return 0;
373	}
374	}
375
376	static inline void bpf_obj_init_field(const struct btf_field field, void addr)
377	{
378	memset(addr, 0, field->size);
379
380	switch (field->type) {
381	case BPF_REFCOUNT:
382	refcount_set((refcount_t *)addr, 1);
383	break;
384	case BPF_RB_NODE:
385	RB_CLEAR_NODE((struct rb_node *)addr);
386	break;
387	case BPF_LIST_HEAD:
388	case BPF_LIST_NODE:
389	INIT_LIST_HEAD((struct list_head *)addr);
390	break;
391	case BPF_RB_ROOT:
392	/* RB_ROOT_CACHED 0-inits, no need to do anything after memset */
393	case BPF_SPIN_LOCK:
394	case BPF_TIMER:
395	case BPF_KPTR_UNREF:
396	case BPF_KPTR_REF:
397	case BPF_KPTR_PERCPU:
398	break;
399	default:
400	WARN_ON_ONCE(1);
401	return;
402	}
403	}
404
405	static inline bool btf_record_has_field(const struct btf_record *rec, enum btf_field_type type)
406	{
407	if (IS_ERR_OR_NULL(rec))
408	return false;
409	return rec->field_mask & type;
410	}
411
412	static inline void bpf_obj_init(const struct btf_record rec, void obj)
413	{
414	int i;
415
416	if (IS_ERR_OR_NULL(rec))
417	return;
418	for (i = 0; i < rec->cnt; i++)
419	bpf_obj_init_field(&rec->fields[i], obj + rec->fields[i].offset);
420	}
421
422	/* 'dst' must be a temporary buffer and should not point to memory that is being
423	* used in parallel by a bpf program or bpf syscall, otherwise the access from
424	* the bpf program or bpf syscall may be corrupted by the reinitialization,
425	* leading to weird problems. Even 'dst' is newly-allocated from bpf memory
426	* allocator, it is still possible for 'dst' to be used in parallel by a bpf
427	* program or bpf syscall.
428	*/
429	static inline void check_and_init_map_value(struct bpf_map map, void dst)
430	{
431	bpf_obj_init(map->record, dst);
432	}
433
434	/* memcpy that is used with 8-byte aligned pointers, power-of-8 size and
435	* forced to use 'long' read/writes to try to atomically copy long counters.
436	* Best-effort only. No barriers here, since it _will_ race with concurrent
437	* updates from BPF programs. Called from bpf syscall and mostly used with
438	* size 8 or 16 bytes, so ask compiler to inline it.
439	*/
440	static inline void bpf_long_memcpy(void dst, const void src, u32 size)
441	{
442	const long *lsrc = src;
443	long *ldst = dst;
444
445	size /= sizeof(long);
446	while (size--)
447	data_race(ldst++ = lsrc++);
448	}
449
450	/* copy everything but bpf_spin_lock, bpf_timer, and kptrs. There could be one of each. */
451	static inline void bpf_obj_memcpy(struct btf_record *rec,
452	void dst, void src, u32 size,
453	bool long_memcpy)
454	{
455	u32 curr_off = 0;
456	int i;
457
458	if (IS_ERR_OR_NULL(rec)) {
459	if (long_memcpy)
460	bpf_long_memcpy(dst, src, round_up(size, 8));
461	else
462	memcpy(dst, src, size);
463	return;
464	}
465
466	for (i = 0; i < rec->cnt; i++) {
467	u32 next_off = rec->fields[i].offset;
468	u32 sz = next_off - curr_off;
469
470	memcpy(dst + curr_off, src + curr_off, sz);
471	curr_off += rec->fields[i].size + sz;
472	}
473	memcpy(dst + curr_off, src + curr_off, size - curr_off);
474	}
475
476	static inline void copy_map_value(struct bpf_map map, void dst, void *src)
477	{
478	bpf_obj_memcpy(map->record, dst, src, map->value_size, false);
479	}
480
481	static inline void copy_map_value_long(struct bpf_map map, void dst, void *src)
482	{
483	bpf_obj_memcpy(map->record, dst, src, map->value_size, true);
484	}
485
486	static inline void bpf_obj_memzero(struct btf_record rec, void dst, u32 size)
487	{
488	u32 curr_off = 0;
489	int i;
490
491	if (IS_ERR_OR_NULL(rec)) {
492	memset(dst, 0, size);
493	return;
494	}
495
496	for (i = 0; i < rec->cnt; i++) {
497	u32 next_off = rec->fields[i].offset;
498	u32 sz = next_off - curr_off;
499
500	memset(dst + curr_off, 0, sz);
501	curr_off += rec->fields[i].size + sz;
502	}
503	memset(dst + curr_off, 0, size - curr_off);
504	}
505
506	static inline void zero_map_value(struct bpf_map map, void dst)
507	{
508	bpf_obj_memzero(map->record, dst, map->value_size);
509	}
510
511	void copy_map_value_locked(struct bpf_map map, void dst, void *src,
512	bool lock_src);
513	void bpf_timer_cancel_and_free(void *timer);
514	void bpf_list_head_free(const struct btf_field field, void list_head,
515	struct bpf_spin_lock *spin_lock);
516	void bpf_rb_root_free(const struct btf_field field, void rb_root,
517	struct bpf_spin_lock *spin_lock);
518
519
520	int bpf_obj_name_cpy(char dst, const char src, unsigned int size);
521
522	struct bpf_offload_dev;
523	struct bpf_offloaded_map;
524
525	struct bpf_map_dev_ops {
526	int (map_get_next_key)(struct bpf_offloaded_map map,
527	void key, void next_key);
528	int (map_lookup_elem)(struct bpf_offloaded_map map,
529	void key, void value);
530	int (map_update_elem)(struct bpf_offloaded_map map,
531	void key, void value, u64 flags);
532	int (map_delete_elem)(struct bpf_offloaded_map map, void *key);
533	};
534
535	struct bpf_offloaded_map {
536	struct bpf_map map;
537	struct net_device *netdev;
538	const struct bpf_map_dev_ops *dev_ops;
539	void *dev_priv;
540	struct list_head offloads;
541	};
542
543	static inline struct bpf_offloaded_map map_to_offmap(struct bpf_map map)
544	{
545	return container_of(map, struct bpf_offloaded_map, map);
546	}
547
548	static inline bool bpf_map_offload_neutral(const struct bpf_map *map)
549	{
550	return map->map_type == BPF_MAP_TYPE_PERF_EVENT_ARRAY;
551	}
552
553	static inline bool bpf_map_support_seq_show(const struct bpf_map *map)
554	{
555	return (map->btf_value_type_id \|\| map->btf_vmlinux_value_type_id) &&
556	map->ops->map_seq_show_elem;
557	}
558
559	int map_check_no_btf(const struct bpf_map *map,
560	const struct btf *btf,
561	const struct btf_type *key_type,
562	const struct btf_type *value_type);
563
564	bool bpf_map_meta_equal(const struct bpf_map *meta0,
565	const struct bpf_map *meta1);
566
567	extern const struct bpf_map_ops bpf_map_offload_ops;
568
569	/* bpf_type_flag contains a set of flags that are applicable to the values of
570	* arg_type, ret_type and reg_type. For example, a pointer value may be null,
571	* or a memory is read-only. We classify types into two categories: base types
572	* and extended types. Extended types are base types combined with a type flag.
573	*
574	* Currently there are no more than 32 base types in arg_type, ret_type and
575	* reg_types.
576	*/
577	#define BPF_BASE_TYPE_BITS 8
578
579	enum bpf_type_flag {
580	/* PTR may be NULL. */
581	PTR_MAYBE_NULL = BIT(0 + BPF_BASE_TYPE_BITS),
582
583	/* MEM is read-only. When applied on bpf_arg, it indicates the arg is
584	* compatible with both mutable and immutable memory.
585	*/
586	MEM_RDONLY = BIT(1 + BPF_BASE_TYPE_BITS),
587
588	/* MEM points to BPF ring buffer reservation. */
589	MEM_RINGBUF = BIT(2 + BPF_BASE_TYPE_BITS),
590
591	/* MEM is in user address space. */
592	MEM_USER = BIT(3 + BPF_BASE_TYPE_BITS),
593
594	/* MEM is a percpu memory. MEM_PERCPU tags PTR_TO_BTF_ID. When tagged
595	* with MEM_PERCPU, PTR_TO_BTF_ID _cannot_ be directly accessed. In
596	* order to drop this tag, it must be passed into bpf_per_cpu_ptr()
597	* or bpf_this_cpu_ptr(), which will return the pointer corresponding
598	* to the specified cpu.
599	*/
600	MEM_PERCPU = BIT(4 + BPF_BASE_TYPE_BITS),
601
602	/* Indicates that the argument will be released. */
603	OBJ_RELEASE = BIT(5 + BPF_BASE_TYPE_BITS),
604
605	/* PTR is not trusted. This is only used with PTR_TO_BTF_ID, to mark
606	* unreferenced and referenced kptr loaded from map value using a load
607	* instruction, so that they can only be dereferenced but not escape the
608	* BPF program into the kernel (i.e. cannot be passed as arguments to
609	* kfunc or bpf helpers).
610	*/
611	PTR_UNTRUSTED = BIT(6 + BPF_BASE_TYPE_BITS),
612
613	MEM_UNINIT = BIT(7 + BPF_BASE_TYPE_BITS),
614
615	/* DYNPTR points to memory local to the bpf program. */
616	DYNPTR_TYPE_LOCAL = BIT(8 + BPF_BASE_TYPE_BITS),
617
618	/* DYNPTR points to a kernel-produced ringbuf record. */
619	DYNPTR_TYPE_RINGBUF = BIT(9 + BPF_BASE_TYPE_BITS),
620
621	/* Size is known at compile time. */
622	MEM_FIXED_SIZE = BIT(10 + BPF_BASE_TYPE_BITS),
623
624	/* MEM is of an allocated object of type in program BTF. This is used to
625	* tag PTR_TO_BTF_ID allocated using bpf_obj_new.
626	*/
627	MEM_ALLOC = BIT(11 + BPF_BASE_TYPE_BITS),
628
629	/* PTR was passed from the kernel in a trusted context, and may be
630	* passed to KF_TRUSTED_ARGS kfuncs or BPF helper functions.
631	* Confusingly, this is _not_ the opposite of PTR_UNTRUSTED above.
632	* PTR_UNTRUSTED refers to a kptr that was read directly from a map
633	* without invoking bpf_kptr_xchg(). What we really need to know is
634	* whether a pointer is safe to pass to a kfunc or BPF helper function.
635	* While PTR_UNTRUSTED pointers are unsafe to pass to kfuncs and BPF
636	* helpers, they do not cover all possible instances of unsafe
637	* pointers. For example, a pointer that was obtained from walking a
638	* struct will _not_ get the PTR_UNTRUSTED type modifier, despite the
639	* fact that it may be NULL, invalid, etc. This is due to backwards
640	* compatibility requirements, as this was the behavior that was first
641	* introduced when kptrs were added. The behavior is now considered
642	* deprecated, and PTR_UNTRUSTED will eventually be removed.
643	*
644	* PTR_TRUSTED, on the other hand, is a pointer that the kernel
645	* guarantees to be valid and safe to pass to kfuncs and BPF helpers.
646	* For example, pointers passed to tracepoint arguments are considered
647	* PTR_TRUSTED, as are pointers that are passed to struct_ops
648	* callbacks. As alluded to above, pointers that are obtained from
649	* walking PTR_TRUSTED pointers are _not_ trusted. For example, if a
650	* struct task_struct *task is PTR_TRUSTED, then accessing
651	* task->last_wakee will lose the PTR_TRUSTED modifier when it's stored
652	* in a BPF register. Similarly, pointers passed to certain programs
653	* types such as kretprobes are not guaranteed to be valid, as they may
654	* for example contain an object that was recently freed.
655	*/
656	PTR_TRUSTED = BIT(12 + BPF_BASE_TYPE_BITS),
657
658	/* MEM is tagged with rcu and memory access needs rcu_read_lock protection. */
659	MEM_RCU = BIT(13 + BPF_BASE_TYPE_BITS),
660
661	/* Used to tag PTR_TO_BTF_ID \| MEM_ALLOC references which are non-owning.
662	* Currently only valid for linked-list and rbtree nodes. If the nodes
663	* have a bpf_refcount_field, they must be tagged MEM_RCU as well.
664	*/
665	NON_OWN_REF = BIT(14 + BPF_BASE_TYPE_BITS),
666
667	/* DYNPTR points to sk_buff */
668	DYNPTR_TYPE_SKB = BIT(15 + BPF_BASE_TYPE_BITS),
669
670	/* DYNPTR points to xdp_buff */
671	DYNPTR_TYPE_XDP = BIT(16 + BPF_BASE_TYPE_BITS),
672
673	__BPF_TYPE_FLAG_MAX,
674	__BPF_TYPE_LAST_FLAG = __BPF_TYPE_FLAG_MAX - 1,
675	};
676
677	#define DYNPTR_TYPE_FLAG_MASK (DYNPTR_TYPE_LOCAL \| DYNPTR_TYPE_RINGBUF \| DYNPTR_TYPE_SKB \
678	\| DYNPTR_TYPE_XDP)
679
680	/* Max number of base types. */
681	#define BPF_BASE_TYPE_LIMIT (1UL << BPF_BASE_TYPE_BITS)
682
683	/* Max number of all types. */
684	#define BPF_TYPE_LIMIT (__BPF_TYPE_LAST_FLAG \| (__BPF_TYPE_LAST_FLAG - 1))
685
686	/* function argument constraints */
687	enum bpf_arg_type {
688	ARG_DONTCARE = 0, /* unused argument in helper function */
689
690	/* the following constraints used to prototype
691	* bpf_map_lookup/update/delete_elem() functions
692	*/
693	ARG_CONST_MAP_PTR, /* const argument used as pointer to bpf_map */
694	ARG_PTR_TO_MAP_KEY, /* pointer to stack used as map key */
695	ARG_PTR_TO_MAP_VALUE, /* pointer to stack used as map value */
696
697	/* Used to prototype bpf_memcmp() and other functions that access data
698	* on eBPF program stack
699	*/
700	ARG_PTR_TO_MEM, /* pointer to valid memory (stack, packet, map value) */
701
702	ARG_CONST_SIZE, /* number of bytes accessed from memory */
703	ARG_CONST_SIZE_OR_ZERO, /* number of bytes accessed from memory or 0 */
704
705	ARG_PTR_TO_CTX, /* pointer to context */
706	ARG_ANYTHING, /* any (initialized) argument is ok */
707	ARG_PTR_TO_SPIN_LOCK, /* pointer to bpf_spin_lock */
708	ARG_PTR_TO_SOCK_COMMON, /* pointer to sock_common */
709	ARG_PTR_TO_INT, /* pointer to int */
710	ARG_PTR_TO_LONG, /* pointer to long */
711	ARG_PTR_TO_SOCKET, /* pointer to bpf_sock (fullsock) */
712	ARG_PTR_TO_BTF_ID, /* pointer to in-kernel struct */
713	ARG_PTR_TO_RINGBUF_MEM, /* pointer to dynamically reserved ringbuf memory */
714	ARG_CONST_ALLOC_SIZE_OR_ZERO, /* number of allocated bytes requested */
715	ARG_PTR_TO_BTF_ID_SOCK_COMMON, /* pointer to in-kernel sock_common or bpf-mirrored bpf_sock */
716	ARG_PTR_TO_PERCPU_BTF_ID, /* pointer to in-kernel percpu type */
717	ARG_PTR_TO_FUNC, /* pointer to a bpf program function */
718	ARG_PTR_TO_STACK, /* pointer to stack */
719	ARG_PTR_TO_CONST_STR, /* pointer to a null terminated read-only string */
720	ARG_PTR_TO_TIMER, /* pointer to bpf_timer */
721	ARG_PTR_TO_KPTR, /* pointer to referenced kptr */
722	ARG_PTR_TO_DYNPTR, /* pointer to bpf_dynptr. See bpf_type_flag for dynptr type */
723	__BPF_ARG_TYPE_MAX,
724
725	/* Extended arg_types. */
726	ARG_PTR_TO_MAP_VALUE_OR_NULL = PTR_MAYBE_NULL \| ARG_PTR_TO_MAP_VALUE,
727	ARG_PTR_TO_MEM_OR_NULL = PTR_MAYBE_NULL \| ARG_PTR_TO_MEM,
728	ARG_PTR_TO_CTX_OR_NULL = PTR_MAYBE_NULL \| ARG_PTR_TO_CTX,
729	ARG_PTR_TO_SOCKET_OR_NULL = PTR_MAYBE_NULL \| ARG_PTR_TO_SOCKET,
730	ARG_PTR_TO_STACK_OR_NULL = PTR_MAYBE_NULL \| ARG_PTR_TO_STACK,
731	ARG_PTR_TO_BTF_ID_OR_NULL = PTR_MAYBE_NULL \| ARG_PTR_TO_BTF_ID,
732	/* pointer to memory does not need to be initialized, helper function must fill
733	* all bytes or clear them in error case.
734	*/
735	ARG_PTR_TO_UNINIT_MEM = MEM_UNINIT \| ARG_PTR_TO_MEM,
736	/* Pointer to valid memory of size known at compile time. */
737	ARG_PTR_TO_FIXED_SIZE_MEM = MEM_FIXED_SIZE \| ARG_PTR_TO_MEM,
738
739	/* This must be the last entry. Its purpose is to ensure the enum is
740	* wide enough to hold the higher bits reserved for bpf_type_flag.
741	*/
742	__BPF_ARG_TYPE_LIMIT = BPF_TYPE_LIMIT,
743	};
744	static_assert(__BPF_ARG_TYPE_MAX <= BPF_BASE_TYPE_LIMIT);
745
746	/* type of values returned from helper functions */
747	enum bpf_return_type {
748	RET_INTEGER, /* function returns integer */
749	RET_VOID, /* function doesn't return anything */
750	RET_PTR_TO_MAP_VALUE, /* returns a pointer to map elem value */
751	RET_PTR_TO_SOCKET, /* returns a pointer to a socket */
752	RET_PTR_TO_TCP_SOCK, /* returns a pointer to a tcp_sock */
753	RET_PTR_TO_SOCK_COMMON, /* returns a pointer to a sock_common */
754	RET_PTR_TO_MEM, /* returns a pointer to memory */
755	RET_PTR_TO_MEM_OR_BTF_ID, /* returns a pointer to a valid memory or a btf_id */
756	RET_PTR_TO_BTF_ID, /* returns a pointer to a btf_id */
757	__BPF_RET_TYPE_MAX,
758
759	/* Extended ret_types. */
760	RET_PTR_TO_MAP_VALUE_OR_NULL = PTR_MAYBE_NULL \| RET_PTR_TO_MAP_VALUE,
761	RET_PTR_TO_SOCKET_OR_NULL = PTR_MAYBE_NULL \| RET_PTR_TO_SOCKET,
762	RET_PTR_TO_TCP_SOCK_OR_NULL = PTR_MAYBE_NULL \| RET_PTR_TO_TCP_SOCK,
763	RET_PTR_TO_SOCK_COMMON_OR_NULL = PTR_MAYBE_NULL \| RET_PTR_TO_SOCK_COMMON,
764	RET_PTR_TO_RINGBUF_MEM_OR_NULL = PTR_MAYBE_NULL \| MEM_RINGBUF \| RET_PTR_TO_MEM,
765	RET_PTR_TO_DYNPTR_MEM_OR_NULL = PTR_MAYBE_NULL \| RET_PTR_TO_MEM,
766	RET_PTR_TO_BTF_ID_OR_NULL = PTR_MAYBE_NULL \| RET_PTR_TO_BTF_ID,
767	RET_PTR_TO_BTF_ID_TRUSTED = PTR_TRUSTED \| RET_PTR_TO_BTF_ID,
768
769	/* This must be the last entry. Its purpose is to ensure the enum is
770	* wide enough to hold the higher bits reserved for bpf_type_flag.
771	*/
772	__BPF_RET_TYPE_LIMIT = BPF_TYPE_LIMIT,
773	};
774	static_assert(__BPF_RET_TYPE_MAX <= BPF_BASE_TYPE_LIMIT);
775
776	/* eBPF function prototype used by verifier to allow BPF_CALLs from eBPF programs
777	* to in-kernel helper functions and for adjusting imm32 field in BPF_CALL
778	* instructions after verifying
779	*/
780	struct bpf_func_proto {
781	u64 (*func)(u64 r1, u64 r2, u64 r3, u64 r4, u64 r5);
782	bool gpl_only;
783	bool pkt_access;
784	bool might_sleep;
785	enum bpf_return_type ret_type;
786	union {
787	struct {
788	enum bpf_arg_type arg1_type;
789	enum bpf_arg_type arg2_type;
790	enum bpf_arg_type arg3_type;
791	enum bpf_arg_type arg4_type;
792	enum bpf_arg_type arg5_type;
793	};
794	enum bpf_arg_type arg_type[5];
795	};
796	union {
797	struct {
798	u32 *arg1_btf_id;
799	u32 *arg2_btf_id;
800	u32 *arg3_btf_id;
801	u32 *arg4_btf_id;
802	u32 *arg5_btf_id;
803	};
804	u32 *arg_btf_id[5];
805	struct {
806	size_t arg1_size;
807	size_t arg2_size;
808	size_t arg3_size;
809	size_t arg4_size;
810	size_t arg5_size;
811	};
812	size_t arg_size[5];
813	};
814	int ret_btf_id; / return value btf_id */
815	bool (allowed)(const struct bpf_prog prog);
816	};
817
818	/* bpf_context is intentionally undefined structure. Pointer to bpf_context is
819	* the first argument to eBPF programs.
820	* For socket filters: 'struct bpf_context ' == 'struct sk_buff '
821	*/
822	struct bpf_context;
823
824	enum bpf_access_type {
825	BPF_READ = 1,
826	BPF_WRITE = 2
827	};
828
829	/* types of values stored in eBPF registers */
830	/* Pointer types represent:
831	* pointer
832	* pointer + imm
833	* pointer + (u16) var
834	* pointer + (u16) var + imm
835	* if (range > 0) then [ptr, ptr + range - off) is safe to access
836	* if (id > 0) means that some 'var' was added
837	* if (off > 0) means that 'imm' was added
838	*/
839	enum bpf_reg_type {
840	NOT_INIT = 0, /* nothing was written into register */
841	SCALAR_VALUE, /* reg doesn't contain a valid pointer */
842	PTR_TO_CTX, /* reg points to bpf_context */
843	CONST_PTR_TO_MAP, /* reg points to struct bpf_map */
844	PTR_TO_MAP_VALUE, /* reg points to map element value */
845	PTR_TO_MAP_KEY, /* reg points to a map element key */
846	PTR_TO_STACK, /* reg == frame_pointer + offset */
847	PTR_TO_PACKET_META, /* skb->data - meta_len */
848	PTR_TO_PACKET, /* reg points to skb->data */
849	PTR_TO_PACKET_END, /* skb->data + headlen */
850	PTR_TO_FLOW_KEYS, /* reg points to bpf_flow_keys */
851	PTR_TO_SOCKET, /* reg points to struct bpf_sock */
852	PTR_TO_SOCK_COMMON, /* reg points to sock_common */
853	PTR_TO_TCP_SOCK, /* reg points to struct tcp_sock */
854	PTR_TO_TP_BUFFER, /* reg points to a writable raw tp's buffer */
855	PTR_TO_XDP_SOCK, /* reg points to struct xdp_sock */
856	/* PTR_TO_BTF_ID points to a kernel struct that does not need
857	* to be null checked by the BPF program. This does not imply the
858	* pointer is _not_ null and in practice this can easily be a null
859	* pointer when reading pointer chains. The assumption is program
860	* context will handle null pointer dereference typically via fault
861	* handling. The verifier must keep this in mind and can make no
862	* assumptions about null or non-null when doing branch analysis.
863	* Further, when passed into helpers the helpers can not, without
864	* additional context, assume the value is non-null.
865	*/
866	PTR_TO_BTF_ID,
867	/* PTR_TO_BTF_ID_OR_NULL points to a kernel struct that has not
868	* been checked for null. Used primarily to inform the verifier
869	* an explicit null check is required for this struct.
870	*/
871	PTR_TO_MEM, /* reg points to valid memory region */
872	PTR_TO_BUF, /* reg points to a read/write buffer */
873	PTR_TO_FUNC, /* reg points to a bpf program function */
874	CONST_PTR_TO_DYNPTR, /* reg points to a const struct bpf_dynptr */
875	__BPF_REG_TYPE_MAX,
876
877	/* Extended reg_types. */
878	PTR_TO_MAP_VALUE_OR_NULL = PTR_MAYBE_NULL \| PTR_TO_MAP_VALUE,
879	PTR_TO_SOCKET_OR_NULL = PTR_MAYBE_NULL \| PTR_TO_SOCKET,
880	PTR_TO_SOCK_COMMON_OR_NULL = PTR_MAYBE_NULL \| PTR_TO_SOCK_COMMON,
881	PTR_TO_TCP_SOCK_OR_NULL = PTR_MAYBE_NULL \| PTR_TO_TCP_SOCK,
882	PTR_TO_BTF_ID_OR_NULL = PTR_MAYBE_NULL \| PTR_TO_BTF_ID,
883
884	/* This must be the last entry. Its purpose is to ensure the enum is
885	* wide enough to hold the higher bits reserved for bpf_type_flag.
886	*/
887	__BPF_REG_TYPE_LIMIT = BPF_TYPE_LIMIT,
888	};
889	static_assert(__BPF_REG_TYPE_MAX <= BPF_BASE_TYPE_LIMIT);
890
891	/* The information passed from prog-specific *_is_valid_access
892	* back to the verifier.
893	*/
894	struct bpf_insn_access_aux {
895	enum bpf_reg_type reg_type;
896	union {
897	int ctx_field_size;
898	struct {
899	struct btf *btf;
900	u32 btf_id;
901	};
902	};
903	struct bpf_verifier_log log; / for verbose logs */
904	};
905
906	static inline void
907	bpf_ctx_record_field_size(struct bpf_insn_access_aux *aux, u32 size)
908	{
909	aux->ctx_field_size = size;
910	}
911
912	static bool bpf_is_ldimm64(const struct bpf_insn *insn)
913	{
914	return insn->code == (BPF_LD \| BPF_IMM \| BPF_DW);
915	}
916
917	static inline bool bpf_pseudo_func(const struct bpf_insn *insn)
918	{
919	return bpf_is_ldimm64(insn) && insn->src_reg == BPF_PSEUDO_FUNC;
920	}
921
922	struct bpf_prog_ops {
923	int (test_run)(struct bpf_prog prog, const union bpf_attr *kattr,
924	union bpf_attr __user *uattr);
925	};
926
927	struct bpf_reg_state;
928	struct bpf_verifier_ops {
929	/* return eBPF function prototype for verification */
930	const struct bpf_func_proto *
931	(*get_func_proto)(enum bpf_func_id func_id,
932	const struct bpf_prog *prog);
933
934	/* return true if 'size' wide access at offset 'off' within bpf_context
935	* with 'type' (read or write) is allowed
936	*/
937	bool (*is_valid_access)(int off, int size, enum bpf_access_type type,
938	const struct bpf_prog *prog,
939	struct bpf_insn_access_aux *info);
940	int (gen_prologue)(struct bpf_insn insn, bool direct_write,
941	const struct bpf_prog *prog);
942	int (gen_ld_abs)(const struct bpf_insn orig,
943	struct bpf_insn *insn_buf);
944	u32 (*convert_ctx_access)(enum bpf_access_type type,
945	const struct bpf_insn *src,
946	struct bpf_insn *dst,
947	struct bpf_prog prog, u32 target_size);
948	int (btf_struct_access)(struct bpf_verifier_log log,
949	const struct bpf_reg_state *reg,
950	int off, int size);
951	};
952
953	struct bpf_prog_offload_ops {
954	/* verifier basic callbacks */
955	int (insn_hook)(struct bpf_verifier_env env,
956	int insn_idx, int prev_insn_idx);
957	int (finalize)(struct bpf_verifier_env env);
958	/* verifier optimization callbacks (called after .finalize) */
959	int (replace_insn)(struct bpf_verifier_env env, u32 off,
960	struct bpf_insn *insn);
961	int (remove_insns)(struct bpf_verifier_env env, u32 off, u32 cnt);
962	/* program management callbacks */
963	int (prepare)(struct bpf_prog prog);
964	int (translate)(struct bpf_prog prog);
965	void (destroy)(struct bpf_prog prog);
966	};
967
968	struct bpf_prog_offload {
969	struct bpf_prog *prog;
970	struct net_device *netdev;
971	struct bpf_offload_dev *offdev;
972	void *dev_priv;
973	struct list_head offloads;
974	bool dev_state;
975	bool opt_failed;
976	void *jited_image;
977	u32 jited_len;
978	};
979
980	enum bpf_cgroup_storage_type {
981	BPF_CGROUP_STORAGE_SHARED,
982	BPF_CGROUP_STORAGE_PERCPU,
983	__BPF_CGROUP_STORAGE_MAX
984	};
985
986	#define MAX_BPF_CGROUP_STORAGE_TYPE __BPF_CGROUP_STORAGE_MAX
987
988	/* The longest tracepoint has 12 args.
989	* See include/trace/bpf_probe.h
990	*/
991	#define MAX_BPF_FUNC_ARGS 12
992
993	/* The maximum number of arguments passed through registers
994	* a single function may have.
995	*/
996	#define MAX_BPF_FUNC_REG_ARGS 5
997
998	/* The argument is a structure. */
999	#define BTF_FMODEL_STRUCT_ARG BIT(0)
1000
1001	/* The argument is signed. */
1002	#define BTF_FMODEL_SIGNED_ARG BIT(1)
1003
1004	struct btf_func_model {
1005	u8 ret_size;
1006	u8 ret_flags;
1007	u8 nr_args;
1008	u8 arg_size[MAX_BPF_FUNC_ARGS];
1009	u8 arg_flags[MAX_BPF_FUNC_ARGS];
1010	};
1011
1012	/* Restore arguments before returning from trampoline to let original function
1013	* continue executing. This flag is used for fentry progs when there are no
1014	* fexit progs.
1015	*/
1016	#define BPF_TRAMP_F_RESTORE_REGS BIT(0)
1017	/* Call original function after fentry progs, but before fexit progs.
1018	* Makes sense for fentry/fexit, normal calls and indirect calls.
1019	*/
1020	#define BPF_TRAMP_F_CALL_ORIG BIT(1)
1021	/* Skip current frame and return to parent. Makes sense for fentry/fexit
1022	* programs only. Should not be used with normal calls and indirect calls.
1023	*/
1024	#define BPF_TRAMP_F_SKIP_FRAME BIT(2)
1025	/* Store IP address of the caller on the trampoline stack,
1026	* so it's available for trampoline's programs.
1027	*/
1028	#define BPF_TRAMP_F_IP_ARG BIT(3)
1029	/* Return the return value of fentry prog. Only used by bpf_struct_ops. */
1030	#define BPF_TRAMP_F_RET_FENTRY_RET BIT(4)
1031
1032	/* Get original function from stack instead of from provided direct address.
1033	* Makes sense for trampolines with fexit or fmod_ret programs.
1034	*/
1035	#define BPF_TRAMP_F_ORIG_STACK BIT(5)
1036
1037	/* This trampoline is on a function with another ftrace_ops with IPMODIFY,
1038	* e.g., a live patch. This flag is set and cleared by ftrace call backs,
1039	*/
1040	#define BPF_TRAMP_F_SHARE_IPMODIFY BIT(6)
1041
1042	/* Indicate that current trampoline is in a tail call context. Then, it has to
1043	* cache and restore tail_call_cnt to avoid infinite tail call loop.
1044	*/
1045	#define BPF_TRAMP_F_TAIL_CALL_CTX BIT(7)
1046
1047	/* Each call __bpf_prog_enter + call bpf_func + call __bpf_prog_exit is ~50
1048	* bytes on x86.
1049	*/
1050	enum {
1051	#if defined(__s390x__)
1052	BPF_MAX_TRAMP_LINKS = 27,
1053	#else
1054	BPF_MAX_TRAMP_LINKS = 38,
1055	#endif
1056	};
1057
1058	struct bpf_tramp_links {
1059	struct bpf_tramp_link *links[BPF_MAX_TRAMP_LINKS];
1060	int nr_links;
1061	};
1062
1063	struct bpf_tramp_run_ctx;
1064
1065	/* Different use cases for BPF trampoline:
1066	* 1. replace nop at the function entry (kprobe equivalent)
1067	* flags = BPF_TRAMP_F_RESTORE_REGS
1068	* fentry = a set of programs to run before returning from trampoline
1069	*
1070	* 2. replace nop at the function entry (kprobe + kretprobe equivalent)
1071	* flags = BPF_TRAMP_F_CALL_ORIG \| BPF_TRAMP_F_SKIP_FRAME
1072	* orig_call = fentry_ip + MCOUNT_INSN_SIZE
1073	* fentry = a set of program to run before calling original function
1074	* fexit = a set of program to run after original function
1075	*
1076	* 3. replace direct call instruction anywhere in the function body
1077	* or assign a function pointer for indirect call (like tcp_congestion_ops->cong_avoid)
1078	* With flags = 0
1079	* fentry = a set of programs to run before returning from trampoline
1080	* With flags = BPF_TRAMP_F_CALL_ORIG
1081	* orig_call = original callback addr or direct function addr
1082	* fentry = a set of program to run before calling original function
1083	* fexit = a set of program to run after original function
1084	*/
1085	struct bpf_tramp_image;
1086	int arch_prepare_bpf_trampoline(struct bpf_tramp_image tr, void image, void *image_end,
1087	const struct btf_func_model *m, u32 flags,
1088	struct bpf_tramp_links *tlinks,
1089	void *orig_call);
1090	u64 notrace __bpf_prog_enter_sleepable_recur(struct bpf_prog *prog,
1091	struct bpf_tramp_run_ctx *run_ctx);
1092	void notrace __bpf_prog_exit_sleepable_recur(struct bpf_prog *prog, u64 start,
1093	struct bpf_tramp_run_ctx *run_ctx);
1094	void notrace __bpf_tramp_enter(struct bpf_tramp_image *tr);
1095	void notrace __bpf_tramp_exit(struct bpf_tramp_image *tr);
1096	typedef u64 (bpf_trampoline_enter_t)(struct bpf_prog prog,
1097	struct bpf_tramp_run_ctx *run_ctx);
1098	typedef void (bpf_trampoline_exit_t)(struct bpf_prog prog, u64 start,
1099	struct bpf_tramp_run_ctx *run_ctx);
1100	bpf_trampoline_enter_t bpf_trampoline_enter(const struct bpf_prog *prog);
1101	bpf_trampoline_exit_t bpf_trampoline_exit(const struct bpf_prog *prog);
1102
1103	struct bpf_ksym {
1104	unsigned long start;
1105	unsigned long end;
1106	char name[KSYM_NAME_LEN];
1107	struct list_head lnode;
1108	struct latch_tree_node tnode;
1109	bool prog;
1110	};
1111
1112	enum bpf_tramp_prog_type {
1113	BPF_TRAMP_FENTRY,
1114	BPF_TRAMP_FEXIT,
1115	BPF_TRAMP_MODIFY_RETURN,
1116	BPF_TRAMP_MAX,
1117	BPF_TRAMP_REPLACE, /* more than MAX */
1118	};
1119
1120	struct bpf_tramp_image {
1121	void *image;
1122	struct bpf_ksym ksym;
1123	struct percpu_ref pcref;
1124	void *ip_after_call;
1125	void *ip_epilogue;
1126	union {
1127	struct rcu_head rcu;
1128	struct work_struct work;
1129	};
1130	};
1131
1132	struct bpf_trampoline {
1133	/* hlist for trampoline_table */
1134	struct hlist_node hlist;
1135	struct ftrace_ops *fops;
1136	/* serializes access to fields of this trampoline */
1137	struct mutex mutex;
1138	refcount_t refcnt;
1139	u32 flags;
1140	u64 key;
1141	struct {
1142	struct btf_func_model model;
1143	void *addr;
1144	bool ftrace_managed;
1145	} func;
1146	/* if !NULL this is BPF_PROG_TYPE_EXT program that extends another BPF
1147	* program by replacing one of its functions. func.addr is the address
1148	* of the function it replaced.
1149	*/
1150	struct bpf_prog *extension_prog;
1151	/* list of BPF programs using this trampoline */
1152	struct hlist_head progs_hlist[BPF_TRAMP_MAX];
1153	/* Number of attached programs. A counter per kind. */
1154	int progs_cnt[BPF_TRAMP_MAX];
1155	/* Executable image of trampoline */
1156	struct bpf_tramp_image *cur_image;
1157	struct module *mod;
1158	};
1159
1160	struct bpf_attach_target_info {
1161	struct btf_func_model fmodel;
1162	long tgt_addr;
1163	struct module *tgt_mod;
1164	const char *tgt_name;
1165	const struct btf_type *tgt_type;
1166	};
1167
1168	#define BPF_DISPATCHER_MAX 48 /* Fits in 2048B */
1169
1170	struct bpf_dispatcher_prog {
1171	struct bpf_prog *prog;
1172	refcount_t users;
1173	};
1174
1175	struct bpf_dispatcher {
1176	/* dispatcher mutex */
1177	struct mutex mutex;
1178	void *func;
1179	struct bpf_dispatcher_prog progs[BPF_DISPATCHER_MAX];
1180	int num_progs;
1181	void *image;
1182	void *rw_image;
1183	u32 image_off;
1184	struct bpf_ksym ksym;
1185	#ifdef CONFIG_HAVE_STATIC_CALL
1186	struct static_call_key *sc_key;
1187	void *sc_tramp;
1188	#endif
1189	};
1190
1191	static __always_inline __nocfi unsigned int bpf_dispatcher_nop_func(
1192	const void *ctx,
1193	const struct bpf_insn *insnsi,
1194	bpf_func_t bpf_func)
1195	{
1196	return bpf_func(ctx, insnsi);
1197	}
1198
1199	/* the implementation of the opaque uapi struct bpf_dynptr */
1200	struct bpf_dynptr_kern {
1201	void *data;
1202	/* Size represents the number of usable bytes of dynptr data.
1203	* If for example the offset is at 4 for a local dynptr whose data is
1204	* of type u64, the number of usable bytes is 4.
1205	*
1206	* The upper 8 bits are reserved. It is as follows:
1207	* Bits 0 - 23 = size
1208	* Bits 24 - 30 = dynptr type
1209	* Bit 31 = whether dynptr is read-only
1210	*/
1211	u32 size;
1212	u32 offset;
1213	} __aligned(8);
1214
1215	enum bpf_dynptr_type {
1216	BPF_DYNPTR_TYPE_INVALID,
1217	/* Points to memory that is local to the bpf program */
1218	BPF_DYNPTR_TYPE_LOCAL,
1219	/* Underlying data is a ringbuf record */
1220	BPF_DYNPTR_TYPE_RINGBUF,
1221	/* Underlying data is a sk_buff */
1222	BPF_DYNPTR_TYPE_SKB,
1223	/* Underlying data is a xdp_buff */
1224	BPF_DYNPTR_TYPE_XDP,
1225	};
1226
1227	int bpf_dynptr_check_size(u32 size);
1228	u32 __bpf_dynptr_size(const struct bpf_dynptr_kern *ptr);
1229
1230	#ifdef CONFIG_BPF_JIT
1231	int bpf_trampoline_link_prog(struct bpf_tramp_link link, struct bpf_trampoline tr);
1232	int bpf_trampoline_unlink_prog(struct bpf_tramp_link link, struct bpf_trampoline tr);
1233	struct bpf_trampoline *bpf_trampoline_get(u64 key,
1234	struct bpf_attach_target_info *tgt_info);
1235	void bpf_trampoline_put(struct bpf_trampoline *tr);
1236	int arch_prepare_bpf_dispatcher(void image, void buf, s64 *funcs, int num_funcs);
1237
1238	/*
1239	* When the architecture supports STATIC_CALL replace the bpf_dispatcher_fn
1240	* indirection with a direct call to the bpf program. If the architecture does
1241	* not have STATIC_CALL, avoid a double-indirection.
1242	*/
1243	#ifdef CONFIG_HAVE_STATIC_CALL
1244
1245	#define __BPF_DISPATCHER_SC_INIT(_name) \
1246	.sc_key = &STATIC_CALL_KEY(_name), \
1247	.sc_tramp = STATIC_CALL_TRAMP_ADDR(_name),
1248
1249	#define __BPF_DISPATCHER_SC(name) \
1250	DEFINE_STATIC_CALL(bpf_dispatcher_##name##_call, bpf_dispatcher_nop_func)
1251
1252	#define __BPF_DISPATCHER_CALL(name) \
1253	static_call(bpf_dispatcher_##name##_call)(ctx, insnsi, bpf_func)
1254
1255	#define __BPF_DISPATCHER_UPDATE(_d, _new) \
1256	__static_call_update((_d)->sc_key, (_d)->sc_tramp, (_new))
1257
1258	#else
1259	#define __BPF_DISPATCHER_SC_INIT(name)
1260	#define __BPF_DISPATCHER_SC(name)
1261	#define __BPF_DISPATCHER_CALL(name) bpf_func(ctx, insnsi)
1262	#define __BPF_DISPATCHER_UPDATE(_d, _new)
1263	#endif
1264
1265	#define BPF_DISPATCHER_INIT(_name) { \
1266	.mutex = __MUTEX_INITIALIZER(_name.mutex), \
1267	.func = &_name##_func, \
1268	.progs = {}, \
1269	.num_progs = 0, \
1270	.image = NULL, \
1271	.image_off = 0, \
1272	.ksym = { \
1273	.name = #_name, \
1274	.lnode = LIST_HEAD_INIT(_name.ksym.lnode), \
1275	}, \
1276	__BPF_DISPATCHER_SC_INIT(_name##_call) \
1277	}
1278
1279	#define DEFINE_BPF_DISPATCHER(name) \
1280	__BPF_DISPATCHER_SC(name); \
1281	noinline __nocfi unsigned int bpf_dispatcher_##name##_func( \
1282	const void *ctx, \
1283	const struct bpf_insn *insnsi, \
1284	bpf_func_t bpf_func) \
1285	{ \
1286	return __BPF_DISPATCHER_CALL(name); \
1287	} \
1288	EXPORT_SYMBOL(bpf_dispatcher_##name##_func); \
1289	struct bpf_dispatcher bpf_dispatcher_##name = \
1290	BPF_DISPATCHER_INIT(bpf_dispatcher_##name);
1291
1292	#define DECLARE_BPF_DISPATCHER(name) \
1293	unsigned int bpf_dispatcher_##name##_func( \
1294	const void *ctx, \
1295	const struct bpf_insn *insnsi, \
1296	bpf_func_t bpf_func); \
1297	extern struct bpf_dispatcher bpf_dispatcher_##name;
1298
1299	#define BPF_DISPATCHER_FUNC(name) bpf_dispatcher_##name##_func
1300	#define BPF_DISPATCHER_PTR(name) (&bpf_dispatcher_##name)
1301	void bpf_dispatcher_change_prog(struct bpf_dispatcher d, struct bpf_prog from,
1302	struct bpf_prog *to);
1303	/* Called only from JIT-enabled code, so there's no need for stubs. */
1304	void bpf_image_ksym_add(void data, struct bpf_ksym ksym);
1305	void bpf_image_ksym_del(struct bpf_ksym *ksym);
1306	void bpf_ksym_add(struct bpf_ksym *ksym);
1307	void bpf_ksym_del(struct bpf_ksym *ksym);
1308	int bpf_jit_charge_modmem(u32 size);
1309	void bpf_jit_uncharge_modmem(u32 size);
1310	bool bpf_prog_has_trampoline(const struct bpf_prog *prog);
1311	#else
1312	static inline int bpf_trampoline_link_prog(struct bpf_tramp_link *link,
1313	struct bpf_trampoline *tr)
1314	{
1315	return -ENOTSUPP;
1316	}
1317	static inline int bpf_trampoline_unlink_prog(struct bpf_tramp_link *link,
1318	struct bpf_trampoline *tr)
1319	{
1320	return -ENOTSUPP;
1321	}
1322	static inline struct bpf_trampoline *bpf_trampoline_get(u64 key,
1323	struct bpf_attach_target_info *tgt_info)
1324	{
1325	return NULL;
1326	}
1327	static inline void bpf_trampoline_put(struct bpf_trampoline *tr) {}
1328	#define DEFINE_BPF_DISPATCHER(name)
1329	#define DECLARE_BPF_DISPATCHER(name)
1330	#define BPF_DISPATCHER_FUNC(name) bpf_dispatcher_nop_func
1331	#define BPF_DISPATCHER_PTR(name) NULL
1332	static inline void bpf_dispatcher_change_prog(struct bpf_dispatcher *d,
1333	struct bpf_prog *from,
1334	struct bpf_prog *to) {}
1335	static inline bool is_bpf_image_address(unsigned long address)
1336	{
1337	return false;
1338	}
1339	static inline bool bpf_prog_has_trampoline(const struct bpf_prog *prog)
1340	{
1341	return false;
1342	}
1343	#endif
1344
1345	struct bpf_func_info_aux {
1346	u16 linkage;
1347	bool unreliable;
1348	};
1349
1350	enum bpf_jit_poke_reason {
1351	BPF_POKE_REASON_TAIL_CALL,
1352	};
1353
1354	/* Descriptor of pokes pointing /into/ the JITed image. */
1355	struct bpf_jit_poke_descriptor {
1356	void *tailcall_target;
1357	void *tailcall_bypass;
1358	void *bypass_addr;
1359	void *aux;
1360	union {
1361	struct {
1362	struct bpf_map *map;
1363	u32 key;
1364	} tail_call;
1365	};
1366	bool tailcall_target_stable;
1367	u8 adj_off;
1368	u16 reason;
1369	u32 insn_idx;
1370	};
1371
1372	/* reg_type info for ctx arguments */
1373	struct bpf_ctx_arg_aux {
1374	u32 offset;
1375	enum bpf_reg_type reg_type;
1376	u32 btf_id;
1377	};
1378
1379	struct btf_mod_pair {
1380	struct btf *btf;
1381	struct module *module;
1382	};
1383
1384	struct bpf_kfunc_desc_tab;
1385
1386	struct bpf_prog_aux {
1387	atomic64_t refcnt;
1388	u32 used_map_cnt;
1389	u32 used_btf_cnt;
1390	u32 max_ctx_offset;
1391	u32 max_pkt_offset;
1392	u32 max_tp_access;
1393	u32 stack_depth;
1394	u32 id;
1395	u32 func_cnt; /* used by non-func prog as the number of func progs */
1396	u32 real_func_cnt; /* includes hidden progs, only used for JIT and freeing progs */
1397	u32 func_idx; /* 0 for non-func prog, the index in func array for func prog */
1398	u32 attach_btf_id; /* in-kernel BTF type id to attach to */
1399	u32 ctx_arg_info_size;
1400	u32 max_rdonly_access;
1401	u32 max_rdwr_access;
1402	struct btf *attach_btf;
1403	const struct bpf_ctx_arg_aux *ctx_arg_info;
1404	struct mutex dst_mutex; /* protects dst_* pointers below, after prog becomes visible */
1405	struct bpf_prog *dst_prog;
1406	struct bpf_trampoline *dst_trampoline;
1407	enum bpf_prog_type saved_dst_prog_type;
1408	enum bpf_attach_type saved_dst_attach_type;
1409	bool verifier_zext; /* Zero extensions has been inserted by verifier. */
1410	bool dev_bound; /* Program is bound to the netdev. */
1411	bool offload_requested; /* Program is bound and offloaded to the netdev. */
1412	bool attach_btf_trace; /* true if attaching to BTF-enabled raw tp */
1413	bool func_proto_unreliable;
1414	bool sleepable;
1415	bool tail_call_reachable;
1416	bool xdp_has_frags;
1417	bool exception_cb;
1418	bool exception_boundary;
1419	/* BTF_KIND_FUNC_PROTO for valid attach_btf_id */
1420	const struct btf_type *attach_func_proto;
1421	/* function name for valid attach_btf_id */
1422	const char *attach_func_name;
1423	struct bpf_prog **func;
1424	void jit_data; / JIT specific data. arch dependent */
1425	struct bpf_jit_poke_descriptor *poke_tab;
1426	struct bpf_kfunc_desc_tab *kfunc_tab;
1427	struct bpf_kfunc_btf_tab *kfunc_btf_tab;
1428	u32 size_poke_tab;
1429	struct bpf_ksym ksym;
1430	const struct bpf_prog_ops *ops;
1431	struct bpf_map **used_maps;
1432	struct mutex used_maps_mutex; /* mutex for used_maps and used_map_cnt */
1433	struct btf_mod_pair *used_btfs;
1434	struct bpf_prog *prog;
1435	struct user_struct *user;
1436	u64 load_time; /* ns since boottime */
1437	u32 verified_insns;
1438	int cgroup_atype; /* enum cgroup_bpf_attach_type */
1439	struct bpf_map *cgroup_storage[MAX_BPF_CGROUP_STORAGE_TYPE];
1440	char name[BPF_OBJ_NAME_LEN];
1441	unsigned int (*bpf_exception_cb)(u64 cookie, u64 sp, u64 bp);
1442	#ifdef CONFIG_SECURITY
1443	void *security;
1444	#endif
1445	struct bpf_prog_offload *offload;
1446	struct btf *btf;
1447	struct bpf_func_info *func_info;
1448	struct bpf_func_info_aux *func_info_aux;
1449	/* bpf_line_info loaded from userspace. linfo->insn_off
1450	* has the xlated insn offset.
1451	* Both the main and sub prog share the same linfo.
1452	* The subprog can access its first linfo by
1453	* using the linfo_idx.
1454	*/
1455	struct bpf_line_info *linfo;
1456	/* jited_linfo is the jited addr of the linfo. It has a
1457	* one to one mapping to linfo:
1458	* jited_linfo[i] is the jited addr for the linfo[i]->insn_off.
1459	* Both the main and sub prog share the same jited_linfo.
1460	* The subprog can access its first jited_linfo by
1461	* using the linfo_idx.
1462	*/
1463	void **jited_linfo;
1464	u32 func_info_cnt;
1465	u32 nr_linfo;
1466	/* subprog can use linfo_idx to access its first linfo and
1467	* jited_linfo.
1468	* main prog always has linfo_idx == 0
1469	*/
1470	u32 linfo_idx;
1471	struct module *mod;
1472	u32 num_exentries;
1473	struct exception_table_entry *extable;
1474	union {
1475	struct work_struct work;
1476	struct rcu_head rcu;
1477	};
1478	};
1479
1480	struct bpf_prog {
1481	u16 pages; /* Number of allocated pages */
1482	u16 jited:1, /* Is our filter JIT'ed? */
1483	jit_requested:1,/* archs need to JIT the prog */
1484	gpl_compatible:1, /* Is filter GPL compatible? */
1485	cb_access:1, /* Is control block accessed? */
1486	dst_needed:1, /* Do we need dst entry? */
1487	blinding_requested:1, /* needs constant blinding */
1488	blinded:1, /* Was blinded */
1489	is_func:1, /* program is a bpf function */
1490	kprobe_override:1, /* Do we override a kprobe? */
1491	has_callchain_buf:1, /* callchain buffer allocated? */
1492	enforce_expected_attach_type:1, /* Enforce expected_attach_type checking at attach time */
1493	call_get_stack:1, /* Do we call bpf_get_stack() or bpf_get_stackid() */
1494	call_get_func_ip:1, /* Do we call get_func_ip() */
1495	tstamp_type_access:1; /* Accessed __sk_buff->tstamp_type */
1496	enum bpf_prog_type type; /* Type of BPF program */
1497	enum bpf_attach_type expected_attach_type; /* For some prog types */
1498	u32 len; /* Number of filter blocks */
1499	u32 jited_len; /* Size of jited insns in bytes */
1500	u8 tag[BPF_TAG_SIZE];
1501	struct bpf_prog_stats __percpu *stats;
1502	int __percpu *active;
1503	unsigned int (bpf_func)(const void ctx,
1504	const struct bpf_insn *insn);
1505	struct bpf_prog_aux aux; / Auxiliary fields */
1506	struct sock_fprog_kern orig_prog; / Original BPF program */
1507	/* Instructions for interpreter */
1508	union {
1509	DECLARE_FLEX_ARRAY(struct sock_filter, insns);
1510	DECLARE_FLEX_ARRAY(struct bpf_insn, insnsi);
1511	};
1512	};
1513
1514	struct bpf_array_aux {
1515	/* Programs with direct jumps into programs part of this array. */
1516	struct list_head poke_progs;
1517	struct bpf_map *map;
1518	struct mutex poke_mutex;
1519	struct work_struct work;
1520	};
1521
1522	struct bpf_link {
1523	atomic64_t refcnt;
1524	u32 id;
1525	enum bpf_link_type type;
1526	const struct bpf_link_ops *ops;
1527	struct bpf_prog *prog;
1528	struct work_struct work;
1529	};
1530
1531	struct bpf_link_ops {
1532	void (release)(struct bpf_link link);
1533	void (dealloc)(struct bpf_link link);
1534	int (detach)(struct bpf_link link);
1535	int (update_prog)(struct bpf_link link, struct bpf_prog *new_prog,
1536	struct bpf_prog *old_prog);
1537	void (show_fdinfo)(const struct bpf_link link, struct seq_file *seq);
1538	int (fill_link_info)(const struct bpf_link link,
1539	struct bpf_link_info *info);
1540	int (update_map)(struct bpf_link link, struct bpf_map *new_map,
1541	struct bpf_map *old_map);
1542	};
1543
1544	struct bpf_tramp_link {
1545	struct bpf_link link;
1546	struct hlist_node tramp_hlist;
1547	u64 cookie;
1548	};
1549
1550	struct bpf_shim_tramp_link {
1551	struct bpf_tramp_link link;
1552	struct bpf_trampoline *trampoline;
1553	};
1554
1555	struct bpf_tracing_link {
1556	struct bpf_tramp_link link;
1557	enum bpf_attach_type attach_type;
1558	struct bpf_trampoline *trampoline;
1559	struct bpf_prog *tgt_prog;
1560	};
1561
1562	struct bpf_link_primer {
1563	struct bpf_link *link;
1564	struct file *file;
1565	int fd;
1566	u32 id;
1567	};
1568
1569	struct bpf_struct_ops_value;
1570	struct btf_member;
1571
1572	#define BPF_STRUCT_OPS_MAX_NR_MEMBERS 64
1573	/**
1574	* struct bpf_struct_ops - A structure of callbacks allowing a subsystem to
1575	* define a BPF_MAP_TYPE_STRUCT_OPS map type composed
1576	* of BPF_PROG_TYPE_STRUCT_OPS progs.
1577	* @verifier_ops: A structure of callbacks that are invoked by the verifier
1578	* when determining whether the struct_ops progs in the
1579	* struct_ops map are valid.
1580	* @init: A callback that is invoked a single time, and before any other
1581	* callback, to initialize the structure. A nonzero return value means
1582	* the subsystem could not be initialized.
1583	* @check_member: When defined, a callback invoked by the verifier to allow
1584	* the subsystem to determine if an entry in the struct_ops map
1585	* is valid. A nonzero return value means that the map is
1586	* invalid and should be rejected by the verifier.
1587	* @init_member: A callback that is invoked for each member of the struct_ops
1588	* map to allow the subsystem to initialize the member. A nonzero
1589	* value means the member could not be initialized. This callback
1590	* is exclusive with the @type, @type_id, @value_type, and
1591	* @value_id fields.
1592	* @reg: A callback that is invoked when the struct_ops map has been
1593	* initialized and is being attached to. Zero means the struct_ops map
1594	* has been successfully registered and is live. A nonzero return value
1595	* means the struct_ops map could not be registered.
1596	* @unreg: A callback that is invoked when the struct_ops map should be
1597	* unregistered.
1598	* @update: A callback that is invoked when the live struct_ops map is being
1599	* updated to contain new values. This callback is only invoked when
1600	* the struct_ops map is loaded with BPF_F_LINK. If not defined, the
1601	* it is assumed that the struct_ops map cannot be updated.
1602	* @validate: A callback that is invoked after all of the members have been
1603	* initialized. This callback should perform static checks on the
1604	* map, meaning that it should either fail or succeed
1605	* deterministically. A struct_ops map that has been validated may
1606	* not necessarily succeed in being registered if the call to @reg
1607	* fails. For example, a valid struct_ops map may be loaded, but
1608	* then fail to be registered due to there being another active
1609	* struct_ops map on the system in the subsystem already. For this
1610	* reason, if this callback is not defined, the check is skipped as
1611	* the struct_ops map will have final verification performed in
1612	* @reg.
1613	* @type: BTF type.
1614	* @value_type: Value type.
1615	* @name: The name of the struct bpf_struct_ops object.
1616	* @func_models: Func models
1617	* @type_id: BTF type id.
1618	* @value_id: BTF value id.
1619	*/
1620	struct bpf_struct_ops {
1621	const struct bpf_verifier_ops *verifier_ops;
1622	int (init)(struct btf btf);
1623	int (check_member)(const struct btf_type t,
1624	const struct btf_member *member,
1625	const struct bpf_prog *prog);
1626	int (init_member)(const struct btf_type t,
1627	const struct btf_member *member,
1628	void kdata, const void udata);
1629	int (reg)(void kdata);
1630	void (unreg)(void kdata);
1631	int (update)(void kdata, void *old_kdata);
1632	int (validate)(void kdata);
1633	const struct btf_type *type;
1634	const struct btf_type *value_type;
1635	const char *name;
1636	struct btf_func_model func_models[BPF_STRUCT_OPS_MAX_NR_MEMBERS];
1637	u32 type_id;
1638	u32 value_id;
1639	};
1640
1641	#if defined(CONFIG_BPF_JIT) && defined(CONFIG_BPF_SYSCALL)
1642	#define BPF_MODULE_OWNER ((void *)((0xeB9FUL << 2) + POISON_POINTER_DELTA))
1643	const struct bpf_struct_ops *bpf_struct_ops_find(u32 type_id);
1644	void bpf_struct_ops_init(struct btf btf, struct bpf_verifier_log log);
1645	bool bpf_struct_ops_get(const void *kdata);
1646	void bpf_struct_ops_put(const void *kdata);
1647	int bpf_struct_ops_map_sys_lookup_elem(struct bpf_map map, void key,
1648	void *value);
1649	int bpf_struct_ops_prepare_trampoline(struct bpf_tramp_links *tlinks,
1650	struct bpf_tramp_link *link,
1651	const struct btf_func_model *model,
1652	void image, void image_end);
1653	static inline bool bpf_try_module_get(const void data, struct module owner)
1654	{
1655	if (owner == BPF_MODULE_OWNER)
1656	return bpf_struct_ops_get(data);
1657	else
1658	return try_module_get(owner);
1659	}
1660	static inline void bpf_module_put(const void data, struct module owner)
1661	{
1662	if (owner == BPF_MODULE_OWNER)
1663	bpf_struct_ops_put(data);
1664	else
1665	module_put(owner);
1666	}
1667	int bpf_struct_ops_link_create(union bpf_attr *attr);
1668
1669	#ifdef CONFIG_NET
1670	/* Define it here to avoid the use of forward declaration */
1671	struct bpf_dummy_ops_state {
1672	int val;
1673	};
1674
1675	struct bpf_dummy_ops {
1676	int (test_1)(struct bpf_dummy_ops_state cb);
1677	int (test_2)(struct bpf_dummy_ops_state cb, int a1, unsigned short a2,
1678	char a3, unsigned long a4);
1679	int (test_sleepable)(struct bpf_dummy_ops_state cb);
1680	};
1681
1682	int bpf_struct_ops_test_run(struct bpf_prog prog, const union bpf_attr kattr,
1683	union bpf_attr __user *uattr);
1684	#endif
1685	#else
1686	static inline const struct bpf_struct_ops *bpf_struct_ops_find(u32 type_id)
1687	{
1688	return NULL;
1689	}
1690	static inline void bpf_struct_ops_init(struct btf *btf,
1691	struct bpf_verifier_log *log)
1692	{
1693	}
1694	static inline bool bpf_try_module_get(const void data, struct module owner)
1695	{
1696	return try_module_get(owner);
1697	}
1698	static inline void bpf_module_put(const void data, struct module owner)
1699	{
1700	module_put(owner);
1701	}
1702	static inline int bpf_struct_ops_map_sys_lookup_elem(struct bpf_map *map,
1703	void *key,
1704	void *value)
1705	{
1706	return -EINVAL;
1707	}
1708	static inline int bpf_struct_ops_link_create(union bpf_attr *attr)
1709	{
1710	return -EOPNOTSUPP;
1711	}
1712
1713	#endif
1714
1715	#if defined(CONFIG_CGROUP_BPF) && defined(CONFIG_BPF_LSM)
1716	int bpf_trampoline_link_cgroup_shim(struct bpf_prog *prog,
1717	int cgroup_atype);
1718	void bpf_trampoline_unlink_cgroup_shim(struct bpf_prog *prog);
1719	#else
1720	static inline int bpf_trampoline_link_cgroup_shim(struct bpf_prog *prog,
1721	int cgroup_atype)
1722	{
1723	return -EOPNOTSUPP;
1724	}
1725	static inline void bpf_trampoline_unlink_cgroup_shim(struct bpf_prog *prog)
1726	{
1727	}
1728	#endif
1729
1730	struct bpf_array {
1731	struct bpf_map map;
1732	u32 elem_size;
1733	u32 index_mask;
1734	struct bpf_array_aux *aux;
1735	union {
1736	DECLARE_FLEX_ARRAY(char, value) __aligned(8);
1737	DECLARE_FLEX_ARRAY(void *, ptrs) __aligned(8);
1738	DECLARE_FLEX_ARRAY(void __percpu *, pptrs) __aligned(8);
1739	};
1740	};
1741
1742	#define BPF_COMPLEXITY_LIMIT_INSNS 1000000 /* yes. 1M insns */
1743	#define MAX_TAIL_CALL_CNT 33
1744
1745	/* Maximum number of loops for bpf_loop and bpf_iter_num.
1746	* It's enum to expose it (and thus make it discoverable) through BTF.
1747	*/
1748	enum {
1749	BPF_MAX_LOOPS = 8 * 1024 * 1024,
1750	};
1751
1752	#define BPF_F_ACCESS_MASK (BPF_F_RDONLY \| \
1753	BPF_F_RDONLY_PROG \| \
1754	BPF_F_WRONLY \| \
1755	BPF_F_WRONLY_PROG)
1756
1757	#define BPF_MAP_CAN_READ BIT(0)
1758	#define BPF_MAP_CAN_WRITE BIT(1)
1759
1760	/* Maximum number of user-producer ring buffer samples that can be drained in
1761	* a call to bpf_user_ringbuf_drain().
1762	*/
1763	#define BPF_MAX_USER_RINGBUF_SAMPLES (128 * 1024)
1764
1765	static inline u32 bpf_map_flags_to_cap(struct bpf_map *map)
1766	{
1767	u32 access_flags = map->map_flags & (BPF_F_RDONLY_PROG \| BPF_F_WRONLY_PROG);
1768
1769	/* Combination of BPF_F_RDONLY_PROG \| BPF_F_WRONLY_PROG is
1770	* not possible.
1771	*/
1772	if (access_flags & BPF_F_RDONLY_PROG)
1773	return BPF_MAP_CAN_READ;
1774	else if (access_flags & BPF_F_WRONLY_PROG)
1775	return BPF_MAP_CAN_WRITE;
1776	else
1777	return BPF_MAP_CAN_READ \| BPF_MAP_CAN_WRITE;
1778	}
1779
1780	static inline bool bpf_map_flags_access_ok(u32 access_flags)
1781	{
1782	return (access_flags & (BPF_F_RDONLY_PROG \| BPF_F_WRONLY_PROG)) !=
1783	(BPF_F_RDONLY_PROG \| BPF_F_WRONLY_PROG);
1784	}
1785
1786	struct bpf_event_entry {
1787	struct perf_event *event;
1788	struct file *perf_file;
1789	struct file *map_file;
1790	struct rcu_head rcu;
1791	};
1792
1793	static inline bool map_type_contains_progs(struct bpf_map *map)
1794	{
1795	return map->map_type == BPF_MAP_TYPE_PROG_ARRAY \|\|
1796	map->map_type == BPF_MAP_TYPE_DEVMAP \|\|
1797	map->map_type == BPF_MAP_TYPE_CPUMAP;
1798	}
1799
1800	bool bpf_prog_map_compatible(struct bpf_map map, const struct bpf_prog fp);
1801	int bpf_prog_calc_tag(struct bpf_prog *fp);
1802
1803	const struct bpf_func_proto *bpf_get_trace_printk_proto(void);
1804	const struct bpf_func_proto *bpf_get_trace_vprintk_proto(void);
1805
1806	typedef unsigned long (bpf_ctx_copy_t)(void dst, const void *src,
1807	unsigned long off, unsigned long len);
1808	typedef u32 (*bpf_convert_ctx_access_t)(enum bpf_access_type type,
1809	const struct bpf_insn *src,
1810	struct bpf_insn *dst,
1811	struct bpf_prog *prog,
1812	u32 *target_size);
1813
1814	u64 bpf_event_output(struct bpf_map map, u64 flags, void meta, u64 meta_size,
1815	void *ctx, u64 ctx_size, bpf_ctx_copy_t ctx_copy);
1816
1817	/* an array of programs to be executed under rcu_lock.
1818	*
1819	* Typical usage:
1820	* ret = bpf_prog_run_array(rcu_dereference(&bpf_prog_array), ctx, bpf_prog_run);
1821	*
1822	* the structure returned by bpf_prog_array_alloc() should be populated
1823	* with program pointers and the last pointer must be NULL.
1824	* The user has to keep refcnt on the program and make sure the program
1825	* is removed from the array before bpf_prog_put().
1826	* The 'struct bpf_prog_array *' should only be replaced with xchg()
1827	* since other cpus are walking the array of pointers in parallel.
1828	*/
1829	struct bpf_prog_array_item {
1830	struct bpf_prog *prog;
1831	union {
1832	struct bpf_cgroup_storage *cgroup_storage[MAX_BPF_CGROUP_STORAGE_TYPE];
1833	u64 bpf_cookie;
1834	};
1835	};
1836
1837	struct bpf_prog_array {
1838	struct rcu_head rcu;
1839	struct bpf_prog_array_item items[];
1840	};
1841
1842	struct bpf_empty_prog_array {
1843	struct bpf_prog_array hdr;
1844	struct bpf_prog *null_prog;
1845	};
1846
1847	/* to avoid allocating empty bpf_prog_array for cgroups that
1848	* don't have bpf program attached use one global 'bpf_empty_prog_array'
1849	* It will not be modified the caller of bpf_prog_array_alloc()
1850	* (since caller requested prog_cnt == 0)
1851	* that pointer should be 'freed' by bpf_prog_array_free()
1852	*/
1853	extern struct bpf_empty_prog_array bpf_empty_prog_array;
1854
1855	struct bpf_prog_array *bpf_prog_array_alloc(u32 prog_cnt, gfp_t flags);
1856	void bpf_prog_array_free(struct bpf_prog_array *progs);
1857	/* Use when traversal over the bpf_prog_array uses tasks_trace rcu */
1858	void bpf_prog_array_free_sleepable(struct bpf_prog_array *progs);
1859	int bpf_prog_array_length(struct bpf_prog_array *progs);
1860	bool bpf_prog_array_is_empty(struct bpf_prog_array *array);
1861	int bpf_prog_array_copy_to_user(struct bpf_prog_array *progs,
1862	__u32 __user *prog_ids, u32 cnt);
1863
1864	void bpf_prog_array_delete_safe(struct bpf_prog_array *progs,
1865	struct bpf_prog *old_prog);
1866	int bpf_prog_array_delete_safe_at(struct bpf_prog_array *array, int index);
1867	int bpf_prog_array_update_at(struct bpf_prog_array *array, int index,
1868	struct bpf_prog *prog);
1869	int bpf_prog_array_copy_info(struct bpf_prog_array *array,
1870	u32 *prog_ids, u32 request_cnt,
1871	u32 *prog_cnt);
1872	int bpf_prog_array_copy(struct bpf_prog_array *old_array,
1873	struct bpf_prog *exclude_prog,
1874	struct bpf_prog *include_prog,
1875	u64 bpf_cookie,
1876	struct bpf_prog_array **new_array);
1877
1878	struct bpf_run_ctx {};
1879
1880	struct bpf_cg_run_ctx {
1881	struct bpf_run_ctx run_ctx;
1882	const struct bpf_prog_array_item *prog_item;
1883	int retval;
1884	};
1885
1886	struct bpf_trace_run_ctx {
1887	struct bpf_run_ctx run_ctx;
1888	u64 bpf_cookie;
1889	bool is_uprobe;
1890	};
1891
1892	struct bpf_tramp_run_ctx {
1893	struct bpf_run_ctx run_ctx;
1894	u64 bpf_cookie;
1895	struct bpf_run_ctx *saved_run_ctx;
1896	};
1897
1898	static inline struct bpf_run_ctx bpf_set_run_ctx(struct bpf_run_ctx new_ctx)
1899	{
1900	struct bpf_run_ctx *old_ctx = NULL;
1901
1902	#ifdef CONFIG_BPF_SYSCALL
1903	old_ctx = current->bpf_ctx;
1904	current->bpf_ctx = new_ctx;
1905	#endif
1906	return old_ctx;
1907	}
1908
1909	static inline void bpf_reset_run_ctx(struct bpf_run_ctx *old_ctx)
1910	{
1911	#ifdef CONFIG_BPF_SYSCALL
1912	current->bpf_ctx = old_ctx;
1913	#endif
1914	}
1915
1916	/* BPF program asks to bypass CAP_NET_BIND_SERVICE in bind. */
1917	#define BPF_RET_BIND_NO_CAP_NET_BIND_SERVICE (1 << 0)
1918	/* BPF program asks to set CN on the packet. */
1919	#define BPF_RET_SET_CN (1 << 0)
1920
1921	typedef u32 (bpf_prog_run_fn)(const struct bpf_prog prog, const void *ctx);
1922
1923	static __always_inline u32
1924	bpf_prog_run_array(const struct bpf_prog_array *array,
1925	const void *ctx, bpf_prog_run_fn run_prog)
1926	{
1927	const struct bpf_prog_array_item *item;
1928	const struct bpf_prog *prog;
1929	struct bpf_run_ctx *old_run_ctx;
1930	struct bpf_trace_run_ctx run_ctx;
1931	u32 ret = 1;
1932
1933	RCU_LOCKDEP_WARN(!rcu_read_lock_held(), "no rcu lock held");
1934
1935	if (unlikely(!array))
1936	return ret;
1937
1938	run_ctx.is_uprobe = false;
1939
1940	migrate_disable();
1941	old_run_ctx = bpf_set_run_ctx(&run_ctx.run_ctx);
1942	item = &array->items[0];
1943	while ((prog = READ_ONCE(item->prog))) {
1944	run_ctx.bpf_cookie = item->bpf_cookie;
1945	ret &= run_prog(prog, ctx);
1946	item++;
1947	}
1948	bpf_reset_run_ctx(old_run_ctx);
1949	migrate_enable();
1950	return ret;
1951	}
1952
1953	/* Notes on RCU design for bpf_prog_arrays containing sleepable programs:
1954	*
1955	* We use the tasks_trace rcu flavor read section to protect the bpf_prog_array
1956	* overall. As a result, we must use the bpf_prog_array_free_sleepable
1957	* in order to use the tasks_trace rcu grace period.
1958	*
1959	* When a non-sleepable program is inside the array, we take the rcu read
1960	* section and disable preemption for that program alone, so it can access
1961	* rcu-protected dynamically sized maps.
1962	*/
1963	static __always_inline u32
1964	bpf_prog_run_array_uprobe(const struct bpf_prog_array __rcu *array_rcu,
1965	const void *ctx, bpf_prog_run_fn run_prog)
1966	{
1967	const struct bpf_prog_array_item *item;
1968	const struct bpf_prog *prog;
1969	const struct bpf_prog_array *array;
1970	struct bpf_run_ctx *old_run_ctx;
1971	struct bpf_trace_run_ctx run_ctx;
1972	u32 ret = 1;
1973
1974	might_fault();
1975
1976	rcu_read_lock_trace();
1977	migrate_disable();
1978
1979	run_ctx.is_uprobe = true;
1980
1981	array = rcu_dereference_check(array_rcu, rcu_read_lock_trace_held());
1982	if (unlikely(!array))
1983	goto out;
1984	old_run_ctx = bpf_set_run_ctx(&run_ctx.run_ctx);
1985	item = &array->items[0];
1986	while ((prog = READ_ONCE(item->prog))) {
1987	if (!prog->aux->sleepable)
1988	rcu_read_lock();
1989
1990	run_ctx.bpf_cookie = item->bpf_cookie;
1991	ret &= run_prog(prog, ctx);
1992	item++;
1993
1994	if (!prog->aux->sleepable)
1995	rcu_read_unlock();
1996	}
1997	bpf_reset_run_ctx(old_run_ctx);
1998	out:
1999	migrate_enable();
2000	rcu_read_unlock_trace();
2001	return ret;
2002	}
2003
2004	#ifdef CONFIG_BPF_SYSCALL
2005	DECLARE_PER_CPU(int, bpf_prog_active);
2006	extern struct mutex bpf_stats_enabled_mutex;
2007
2008	/*
2009	* Block execution of BPF programs attached to instrumentation (perf,
2010	* kprobes, tracepoints) to prevent deadlocks on map operations as any of
2011	* these events can happen inside a region which holds a map bucket lock
2012	* and can deadlock on it.
2013	*/
2014	static inline void bpf_disable_instrumentation(void)
2015	{
2016	migrate_disable();
2017	this_cpu_inc(bpf_prog_active);
2018	}
2019
2020	static inline void bpf_enable_instrumentation(void)
2021	{
2022	this_cpu_dec(bpf_prog_active);
2023	migrate_enable();
2024	}
2025
2026	extern const struct file_operations bpf_map_fops;
2027	extern const struct file_operations bpf_prog_fops;
2028	extern const struct file_operations bpf_iter_fops;
2029
2030	#define BPF_PROG_TYPE(_id, _name, prog_ctx_type, kern_ctx_type) \
2031	extern const struct bpf_prog_ops _name ## _prog_ops; \
2032	extern const struct bpf_verifier_ops _name ## _verifier_ops;
2033	#define BPF_MAP_TYPE(_id, _ops) \
2034	extern const struct bpf_map_ops _ops;
2035	#define BPF_LINK_TYPE(_id, _name)
2036	#include <linux/bpf_types.h>
2037	#undef BPF_PROG_TYPE
2038	#undef BPF_MAP_TYPE
2039	#undef BPF_LINK_TYPE
2040
2041	extern const struct bpf_prog_ops bpf_offload_prog_ops;
2042	extern const struct bpf_verifier_ops tc_cls_act_analyzer_ops;
2043	extern const struct bpf_verifier_ops xdp_analyzer_ops;
2044
2045	struct bpf_prog *bpf_prog_get(u32 ufd);
2046	struct bpf_prog *bpf_prog_get_type_dev(u32 ufd, enum bpf_prog_type type,
2047	bool attach_drv);
2048	void bpf_prog_add(struct bpf_prog *prog, int i);
2049	void bpf_prog_sub(struct bpf_prog *prog, int i);
2050	void bpf_prog_inc(struct bpf_prog *prog);
2051	struct bpf_prog * __must_check bpf_prog_inc_not_zero(struct bpf_prog *prog);
2052	void bpf_prog_put(struct bpf_prog *prog);
2053
2054	void bpf_prog_free_id(struct bpf_prog *prog);
2055	void bpf_map_free_id(struct bpf_map *map);
2056
2057	struct btf_field btf_record_find(const struct btf_record rec,
2058	u32 offset, u32 field_mask);
2059	void btf_record_free(struct btf_record *rec);
2060	void bpf_map_free_record(struct bpf_map *map);
2061	struct btf_record btf_record_dup(const struct btf_record rec);
2062	bool btf_record_equal(const struct btf_record rec_a, const struct btf_record rec_b);
2063	void bpf_obj_free_timer(const struct btf_record rec, void obj);
2064	void bpf_obj_free_fields(const struct btf_record rec, void obj);
2065	void __bpf_obj_drop_impl(void p, const struct btf_record rec, bool percpu);
2066
2067	struct bpf_map *bpf_map_get(u32 ufd);
2068	struct bpf_map *bpf_map_get_with_uref(u32 ufd);
2069	struct bpf_map *__bpf_map_get(struct fd f);
2070	void bpf_map_inc(struct bpf_map *map);
2071	void bpf_map_inc_with_uref(struct bpf_map *map);
2072	struct bpf_map __bpf_map_inc_not_zero(struct bpf_map map, bool uref);
2073	struct bpf_map * __must_check bpf_map_inc_not_zero(struct bpf_map *map);
2074	void bpf_map_put_with_uref(struct bpf_map *map);
2075	void bpf_map_put(struct bpf_map *map);
2076	void *bpf_map_area_alloc(u64 size, int numa_node);
2077	void *bpf_map_area_mmapable_alloc(u64 size, int numa_node);
2078	void bpf_map_area_free(void *base);
2079	bool bpf_map_write_active(const struct bpf_map *map);
2080	void bpf_map_init_from_attr(struct bpf_map map, union bpf_attr attr);
2081	int generic_map_lookup_batch(struct bpf_map *map,
2082	const union bpf_attr *attr,
2083	union bpf_attr __user *uattr);
2084	int generic_map_update_batch(struct bpf_map map, struct file map_file,
2085	const union bpf_attr *attr,
2086	union bpf_attr __user *uattr);
2087	int generic_map_delete_batch(struct bpf_map *map,
2088	const union bpf_attr *attr,
2089	union bpf_attr __user *uattr);
2090	struct bpf_map bpf_map_get_curr_or_next(u32 id);
2091	struct bpf_prog bpf_prog_get_curr_or_next(u32 id);
2092
2093	#ifdef CONFIG_MEMCG_KMEM
2094	void bpf_map_kmalloc_node(const struct bpf_map map, size_t size, gfp_t flags,
2095	int node);
2096	void bpf_map_kzalloc(const struct bpf_map map, size_t size, gfp_t flags);
2097	void bpf_map_kvcalloc(struct bpf_map map, size_t n, size_t size,
2098	gfp_t flags);
2099	void __percpu bpf_map_alloc_percpu(const struct bpf_map map, size_t size,
2100	size_t align, gfp_t flags);
2101	#else
2102	static inline void *
2103	bpf_map_kmalloc_node(const struct bpf_map *map, size_t size, gfp_t flags,
2104	int node)
2105	{
2106	return kmalloc_node(size, flags, node);
2107	}
2108
2109	static inline void *
2110	bpf_map_kzalloc(const struct bpf_map *map, size_t size, gfp_t flags)
2111	{
2112	return kzalloc(size, flags);
2113	}
2114
2115	static inline void *
2116	bpf_map_kvcalloc(struct bpf_map *map, size_t n, size_t size, gfp_t flags)
2117	{
2118	return kvcalloc(n, size, flags);
2119	}
2120
2121	static inline void __percpu *
2122	bpf_map_alloc_percpu(const struct bpf_map *map, size_t size, size_t align,
2123	gfp_t flags)
2124	{
2125	return __alloc_percpu_gfp(size, align, flags);
2126	}
2127	#endif
2128
2129	static inline int
2130	bpf_map_init_elem_count(struct bpf_map *map)
2131	{
2132	size_t size = sizeof(*map->elem_count), align = size;
2133	gfp_t flags = GFP_USER \| __GFP_NOWARN;
2134
2135	map->elem_count = bpf_map_alloc_percpu(map, size, align, flags);
2136	if (!map->elem_count)
2137	return -ENOMEM;
2138
2139	return 0;
2140	}
2141
2142	static inline void
2143	bpf_map_free_elem_count(struct bpf_map *map)
2144	{
2145	free_percpu(map->elem_count);
2146	}
2147
2148	static inline void bpf_map_inc_elem_count(struct bpf_map *map)
2149	{
2150	this_cpu_inc(*map->elem_count);
2151	}
2152
2153	static inline void bpf_map_dec_elem_count(struct bpf_map *map)
2154	{
2155	this_cpu_dec(*map->elem_count);
2156	}
2157
2158	extern int sysctl_unprivileged_bpf_disabled;
2159
2160	static inline bool bpf_allow_ptr_leaks(void)
2161	{
2162	return perfmon_capable();
2163	}
2164
2165	static inline bool bpf_allow_uninit_stack(void)
2166	{
2167	return perfmon_capable();
2168	}
2169
2170	static inline bool bpf_bypass_spec_v1(void)
2171	{
2172	return cpu_mitigations_off() \|\| perfmon_capable();
2173	}
2174
2175	static inline bool bpf_bypass_spec_v4(void)
2176	{
2177	return cpu_mitigations_off() \|\| perfmon_capable();
2178	}
2179
2180	int bpf_map_new_fd(struct bpf_map *map, int flags);
2181	int bpf_prog_new_fd(struct bpf_prog *prog);
2182
2183	void bpf_link_init(struct bpf_link *link, enum bpf_link_type type,
2184	const struct bpf_link_ops ops, struct bpf_prog prog);
2185	int bpf_link_prime(struct bpf_link link, struct bpf_link_primer primer);
2186	int bpf_link_settle(struct bpf_link_primer *primer);
2187	void bpf_link_cleanup(struct bpf_link_primer *primer);
2188	void bpf_link_inc(struct bpf_link *link);
2189	void bpf_link_put(struct bpf_link *link);
2190	int bpf_link_new_fd(struct bpf_link *link);
2191	struct bpf_link *bpf_link_get_from_fd(u32 ufd);
2192	struct bpf_link bpf_link_get_curr_or_next(u32 id);
2193
2194	int bpf_obj_pin_user(u32 ufd, int path_fd, const char __user *pathname);
2195	int bpf_obj_get_user(int path_fd, const char __user *pathname, int flags);
2196
2197	#define BPF_ITER_FUNC_PREFIX "bpf_iter_"
2198	#define DEFINE_BPF_ITER_FUNC(target, args...) \
2199	extern int bpf_iter_ ## target(args); \
2200	int __init bpf_iter_ ## target(args) { return 0; }
2201
2202	/*
2203	* The task type of iterators.
2204	*
2205	* For BPF task iterators, they can be parameterized with various
2206	* parameters to visit only some of tasks.
2207	*
2208	* BPF_TASK_ITER_ALL (default)
2209	* Iterate over resources of every task.
2210	*
2211	* BPF_TASK_ITER_TID
2212	* Iterate over resources of a task/tid.
2213	*
2214	* BPF_TASK_ITER_TGID
2215	* Iterate over resources of every task of a process / task group.
2216	*/
2217	enum bpf_iter_task_type {
2218	BPF_TASK_ITER_ALL = 0,
2219	BPF_TASK_ITER_TID,
2220	BPF_TASK_ITER_TGID,
2221	};
2222
2223	struct bpf_iter_aux_info {
2224	/* for map_elem iter */
2225	struct bpf_map *map;
2226
2227	/* for cgroup iter */
2228	struct {
2229	struct cgroup start; / starting cgroup */
2230	enum bpf_cgroup_iter_order order;
2231	} cgroup;
2232	struct {
2233	enum bpf_iter_task_type type;
2234	u32 pid;
2235	} task;
2236	};
2237
2238	typedef int (bpf_iter_attach_target_t)(struct bpf_prog prog,
2239	union bpf_iter_link_info *linfo,
2240	struct bpf_iter_aux_info *aux);
2241	typedef void (bpf_iter_detach_target_t)(struct bpf_iter_aux_info aux);
2242	typedef void (bpf_iter_show_fdinfo_t) (const struct bpf_iter_aux_info aux,
2243	struct seq_file *seq);
2244	typedef int (bpf_iter_fill_link_info_t)(const struct bpf_iter_aux_info aux,
2245	struct bpf_link_info *info);
2246	typedef const struct bpf_func_proto *
2247	(*bpf_iter_get_func_proto_t)(enum bpf_func_id func_id,
2248	const struct bpf_prog *prog);
2249
2250	enum bpf_iter_feature {
2251	BPF_ITER_RESCHED = BIT(0),
2252	};
2253
2254	#define BPF_ITER_CTX_ARG_MAX 2
2255	struct bpf_iter_reg {
2256	const char *target;
2257	bpf_iter_attach_target_t attach_target;
2258	bpf_iter_detach_target_t detach_target;
2259	bpf_iter_show_fdinfo_t show_fdinfo;
2260	bpf_iter_fill_link_info_t fill_link_info;
2261	bpf_iter_get_func_proto_t get_func_proto;
2262	u32 ctx_arg_info_size;
2263	u32 feature;
2264	struct bpf_ctx_arg_aux ctx_arg_info[BPF_ITER_CTX_ARG_MAX];
2265	const struct bpf_iter_seq_info *seq_info;
2266	};
2267
2268	struct bpf_iter_meta {
2269	__bpf_md_ptr(struct seq_file *, seq);
2270	u64 session_id;
2271	u64 seq_num;
2272	};
2273
2274	struct bpf_iter__bpf_map_elem {
2275	__bpf_md_ptr(struct bpf_iter_meta *, meta);
2276	__bpf_md_ptr(struct bpf_map *, map);
2277	__bpf_md_ptr(void *, key);
2278	__bpf_md_ptr(void *, value);
2279	};
2280
2281	int bpf_iter_reg_target(const struct bpf_iter_reg *reg_info);
2282	void bpf_iter_unreg_target(const struct bpf_iter_reg *reg_info);
2283	bool bpf_iter_prog_supported(struct bpf_prog *prog);
2284	const struct bpf_func_proto *
2285	bpf_iter_get_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog);
2286	int bpf_iter_link_attach(const union bpf_attr attr, bpfptr_t uattr, struct bpf_prog prog);
2287	int bpf_iter_new_fd(struct bpf_link *link);
2288	bool bpf_link_is_iter(struct bpf_link *link);
2289	struct bpf_prog bpf_iter_get_info(struct bpf_iter_meta meta, bool in_stop);
2290	int bpf_iter_run_prog(struct bpf_prog prog, void ctx);
2291	void bpf_iter_map_show_fdinfo(const struct bpf_iter_aux_info *aux,
2292	struct seq_file *seq);
2293	int bpf_iter_map_fill_link_info(const struct bpf_iter_aux_info *aux,
2294	struct bpf_link_info *info);
2295
2296	int map_set_for_each_callback_args(struct bpf_verifier_env *env,
2297	struct bpf_func_state *caller,
2298	struct bpf_func_state *callee);
2299
2300	int bpf_percpu_hash_copy(struct bpf_map map, void key, void *value);
2301	int bpf_percpu_array_copy(struct bpf_map map, void key, void *value);
2302	int bpf_percpu_hash_update(struct bpf_map map, void key, void *value,
2303	u64 flags);
2304	int bpf_percpu_array_update(struct bpf_map map, void key, void *value,
2305	u64 flags);
2306
2307	int bpf_stackmap_copy(struct bpf_map map, void key, void *value);
2308
2309	int bpf_fd_array_map_update_elem(struct bpf_map map, struct file map_file,
2310	void key, void value, u64 map_flags);
2311	int bpf_fd_array_map_lookup_elem(struct bpf_map map, void key, u32 *value);
2312	int bpf_fd_htab_map_update_elem(struct bpf_map map, struct file map_file,
2313	void key, void value, u64 map_flags);
2314	int bpf_fd_htab_map_lookup_elem(struct bpf_map map, void key, u32 *value);
2315
2316	int bpf_get_file_flag(int flags);
2317	int bpf_check_uarg_tail_zero(bpfptr_t uaddr, size_t expected_size,
2318	size_t actual_size);
2319
2320	/* verify correctness of eBPF program */
2321	int bpf_check(struct bpf_prog *fp, union bpf_attr attr, bpfptr_t uattr, u32 uattr_size);
2322
2323	#ifndef CONFIG_BPF_JIT_ALWAYS_ON
2324	void bpf_patch_call_args(struct bpf_insn *insn, u32 stack_depth);
2325	#endif
2326
2327	struct btf *bpf_get_btf_vmlinux(void);
2328
2329	/* Map specifics */
2330	struct xdp_frame;
2331	struct sk_buff;
2332	struct bpf_dtab_netdev;
2333	struct bpf_cpu_map_entry;
2334
2335	void __dev_flush(void);
2336	int dev_xdp_enqueue(struct net_device dev, struct xdp_frame xdpf,
2337	struct net_device *dev_rx);
2338	int dev_map_enqueue(struct bpf_dtab_netdev dst, struct xdp_frame xdpf,
2339	struct net_device *dev_rx);
2340	int dev_map_enqueue_multi(struct xdp_frame xdpf, struct net_device dev_rx,
2341	struct bpf_map *map, bool exclude_ingress);
2342	int dev_map_generic_redirect(struct bpf_dtab_netdev dst, struct sk_buff skb,
2343	struct bpf_prog *xdp_prog);
2344	int dev_map_redirect_multi(struct net_device dev, struct sk_buff skb,
2345	struct bpf_prog xdp_prog, struct bpf_map map,
2346	bool exclude_ingress);
2347
2348	void __cpu_map_flush(void);
2349	int cpu_map_enqueue(struct bpf_cpu_map_entry rcpu, struct xdp_frame xdpf,
2350	struct net_device *dev_rx);
2351	int cpu_map_generic_redirect(struct bpf_cpu_map_entry *rcpu,
2352	struct sk_buff *skb);
2353
2354	/* Return map's numa specified by userspace */
2355	static inline int bpf_map_attr_numa_node(const union bpf_attr *attr)
2356	{
2357	return (attr->map_flags & BPF_F_NUMA_NODE) ?
2358	attr->numa_node : NUMA_NO_NODE;
2359	}
2360
2361	struct bpf_prog bpf_prog_get_type_path(const char name, enum bpf_prog_type type);
2362	int array_map_alloc_check(union bpf_attr *attr);
2363
2364	int bpf_prog_test_run_xdp(struct bpf_prog prog, const union bpf_attr kattr,
2365	union bpf_attr __user *uattr);
2366	int bpf_prog_test_run_skb(struct bpf_prog prog, const union bpf_attr kattr,
2367	union bpf_attr __user *uattr);
2368	int bpf_prog_test_run_tracing(struct bpf_prog *prog,
2369	const union bpf_attr *kattr,
2370	union bpf_attr __user *uattr);
2371	int bpf_prog_test_run_flow_dissector(struct bpf_prog *prog,
2372	const union bpf_attr *kattr,
2373	union bpf_attr __user *uattr);
2374	int bpf_prog_test_run_raw_tp(struct bpf_prog *prog,
2375	const union bpf_attr *kattr,
2376	union bpf_attr __user *uattr);
2377	int bpf_prog_test_run_sk_lookup(struct bpf_prog *prog,
2378	const union bpf_attr *kattr,
2379	union bpf_attr __user *uattr);
2380	int bpf_prog_test_run_nf(struct bpf_prog *prog,
2381	const union bpf_attr *kattr,
2382	union bpf_attr __user *uattr);
2383	bool btf_ctx_access(int off, int size, enum bpf_access_type type,
2384	const struct bpf_prog *prog,
2385	struct bpf_insn_access_aux *info);
2386
2387	static inline bool bpf_tracing_ctx_access(int off, int size,
2388	enum bpf_access_type type)
2389	{
2390	if (off < 0 \|\| off >= sizeof(__u64) * MAX_BPF_FUNC_ARGS)
2391	return false;
2392	if (type != BPF_READ)
2393	return false;
2394	if (off % size != 0)
2395	return false;
2396	return true;
2397	}
2398
2399	static inline bool bpf_tracing_btf_ctx_access(int off, int size,
2400	enum bpf_access_type type,
2401	const struct bpf_prog *prog,
2402	struct bpf_insn_access_aux *info)
2403	{
2404	if (!bpf_tracing_ctx_access(off, size, type))
2405	return false;
2406	return btf_ctx_access(off, size, type, prog, info);
2407	}
2408
2409	int btf_struct_access(struct bpf_verifier_log *log,
2410	const struct bpf_reg_state *reg,
2411	int off, int size, enum bpf_access_type atype,
2412	u32 next_btf_id, enum bpf_type_flag flag, const char **field_name);
2413	bool btf_struct_ids_match(struct bpf_verifier_log *log,
2414	const struct btf *btf, u32 id, int off,
2415	const struct btf *need_btf, u32 need_type_id,
2416	bool strict);
2417
2418	int btf_distill_func_proto(struct bpf_verifier_log *log,
2419	struct btf *btf,
2420	const struct btf_type *func_proto,
2421	const char *func_name,
2422	struct btf_func_model *m);
2423
2424	struct bpf_reg_state;
2425	int btf_check_subprog_arg_match(struct bpf_verifier_env *env, int subprog,
2426	struct bpf_reg_state *regs);
2427	int btf_check_subprog_call(struct bpf_verifier_env *env, int subprog,
2428	struct bpf_reg_state *regs);
2429	int btf_prepare_func_args(struct bpf_verifier_env *env, int subprog,
2430	struct bpf_reg_state *reg, bool is_ex_cb);
2431	int btf_check_type_match(struct bpf_verifier_log log, const struct bpf_prog prog,
2432	struct btf btf, const struct btf_type t);
2433	const char btf_find_decl_tag_value(const struct btf btf, const struct btf_type *pt,
2434	int comp_idx, const char *tag_key);
2435
2436	struct bpf_prog *bpf_prog_by_id(u32 id);
2437	struct bpf_link *bpf_link_by_id(u32 id);
2438
2439	const struct bpf_func_proto *bpf_base_func_proto(enum bpf_func_id func_id);
2440	void bpf_task_storage_free(struct task_struct *task);
2441	void bpf_cgrp_storage_free(struct cgroup *cgroup);
2442	bool bpf_prog_has_kfunc_call(const struct bpf_prog *prog);
2443	const struct btf_func_model *
2444	bpf_jit_find_kfunc_model(const struct bpf_prog *prog,
2445	const struct bpf_insn *insn);
2446	int bpf_get_kfunc_addr(const struct bpf_prog *prog, u32 func_id,
2447	u16 btf_fd_idx, u8 **func_addr);
2448
2449	struct bpf_core_ctx {
2450	struct bpf_verifier_log *log;
2451	const struct btf *btf;
2452	};
2453
2454	bool btf_nested_type_is_trusted(struct bpf_verifier_log *log,
2455	const struct bpf_reg_state *reg,
2456	const char field_name, u32 btf_id, const char suffix);
2457
2458	bool btf_type_ids_nocast_alias(struct bpf_verifier_log *log,
2459	const struct btf *reg_btf, u32 reg_id,
2460	const struct btf *arg_btf, u32 arg_id);
2461
2462	int bpf_core_apply(struct bpf_core_ctx ctx, const struct bpf_core_relo relo,
2463	int relo_idx, void *insn);
2464
2465	static inline bool unprivileged_ebpf_enabled(void)
2466	{
2467	return !sysctl_unprivileged_bpf_disabled;
2468	}
2469
2470	/* Not all bpf prog type has the bpf_ctx.
2471	* For the bpf prog type that has initialized the bpf_ctx,
2472	* this function can be used to decide if a kernel function
2473	* is called by a bpf program.
2474	*/
2475	static inline bool has_current_bpf_ctx(void)
2476	{
2477	return !!current->bpf_ctx;
2478	}
2479
2480	void notrace bpf_prog_inc_misses_counter(struct bpf_prog *prog);
2481
2482	void bpf_dynptr_init(struct bpf_dynptr_kern ptr, void data,
2483	enum bpf_dynptr_type type, u32 offset, u32 size);
2484	void bpf_dynptr_set_null(struct bpf_dynptr_kern *ptr);
2485	void bpf_dynptr_set_rdonly(struct bpf_dynptr_kern *ptr);
2486
2487	bool dev_check_flush(void);
2488	bool cpu_map_check_flush(void);
2489	#else /* !CONFIG_BPF_SYSCALL */
2490	static inline struct bpf_prog *bpf_prog_get(u32 ufd)
2491	{
2492	return ERR_PTR(-EOPNOTSUPP);
2493	}
2494
2495	static inline struct bpf_prog *bpf_prog_get_type_dev(u32 ufd,
2496	enum bpf_prog_type type,
2497	bool attach_drv)
2498	{
2499	return ERR_PTR(-EOPNOTSUPP);
2500	}
2501
2502	static inline void bpf_prog_add(struct bpf_prog *prog, int i)
2503	{
2504	}
2505
2506	static inline void bpf_prog_sub(struct bpf_prog *prog, int i)
2507	{
2508	}
2509
2510	static inline void bpf_prog_put(struct bpf_prog *prog)
2511	{
2512	}
2513
2514	static inline void bpf_prog_inc(struct bpf_prog *prog)
2515	{
2516	}
2517
2518	static inline struct bpf_prog *__must_check
2519	bpf_prog_inc_not_zero(struct bpf_prog *prog)
2520	{
2521	return ERR_PTR(-EOPNOTSUPP);
2522	}
2523
2524	static inline void bpf_link_init(struct bpf_link *link, enum bpf_link_type type,
2525	const struct bpf_link_ops *ops,
2526	struct bpf_prog *prog)
2527	{
2528	}
2529
2530	static inline int bpf_link_prime(struct bpf_link *link,
2531	struct bpf_link_primer *primer)
2532	{
2533	return -EOPNOTSUPP;
2534	}
2535
2536	static inline int bpf_link_settle(struct bpf_link_primer *primer)
2537	{
2538	return -EOPNOTSUPP;
2539	}
2540
2541	static inline void bpf_link_cleanup(struct bpf_link_primer *primer)
2542	{
2543	}
2544
2545	static inline void bpf_link_inc(struct bpf_link *link)
2546	{
2547	}
2548
2549	static inline void bpf_link_put(struct bpf_link *link)
2550	{
2551	}
2552
2553	static inline int bpf_obj_get_user(const char __user *pathname, int flags)
2554	{
2555	return -EOPNOTSUPP;
2556	}
2557
2558	static inline void __dev_flush(void)
2559	{
2560	}
2561
2562	struct xdp_frame;
2563	struct bpf_dtab_netdev;
2564	struct bpf_cpu_map_entry;
2565
2566	static inline
2567	int dev_xdp_enqueue(struct net_device dev, struct xdp_frame xdpf,
2568	struct net_device *dev_rx)
2569	{
2570	return 0;
2571	}
2572
2573	static inline
2574	int dev_map_enqueue(struct bpf_dtab_netdev dst, struct xdp_frame xdpf,
2575	struct net_device *dev_rx)
2576	{
2577	return 0;
2578	}
2579
2580	static inline
2581	int dev_map_enqueue_multi(struct xdp_frame xdpf, struct net_device dev_rx,
2582	struct bpf_map *map, bool exclude_ingress)
2583	{
2584	return 0;
2585	}
2586
2587	struct sk_buff;
2588
2589	static inline int dev_map_generic_redirect(struct bpf_dtab_netdev *dst,
2590	struct sk_buff *skb,
2591	struct bpf_prog *xdp_prog)
2592	{
2593	return 0;
2594	}
2595
2596	static inline
2597	int dev_map_redirect_multi(struct net_device dev, struct sk_buff skb,
2598	struct bpf_prog xdp_prog, struct bpf_map map,
2599	bool exclude_ingress)
2600	{
2601	return 0;
2602	}
2603
2604	static inline void __cpu_map_flush(void)
2605	{
2606	}
2607
2608	static inline int cpu_map_enqueue(struct bpf_cpu_map_entry *rcpu,
2609	struct xdp_frame *xdpf,
2610	struct net_device *dev_rx)
2611	{
2612	return 0;
2613	}
2614
2615	static inline int cpu_map_generic_redirect(struct bpf_cpu_map_entry *rcpu,
2616	struct sk_buff *skb)
2617	{
2618	return -EOPNOTSUPP;
2619	}
2620
2621	static inline struct bpf_prog bpf_prog_get_type_path(const char name,
2622	enum bpf_prog_type type)
2623	{
2624	return ERR_PTR(-EOPNOTSUPP);
2625	}
2626
2627	static inline int bpf_prog_test_run_xdp(struct bpf_prog *prog,
2628	const union bpf_attr *kattr,
2629	union bpf_attr __user *uattr)
2630	{
2631	return -ENOTSUPP;
2632	}
2633
2634	static inline int bpf_prog_test_run_skb(struct bpf_prog *prog,
2635	const union bpf_attr *kattr,
2636	union bpf_attr __user *uattr)
2637	{
2638	return -ENOTSUPP;
2639	}
2640
2641	static inline int bpf_prog_test_run_tracing(struct bpf_prog *prog,
2642	const union bpf_attr *kattr,
2643	union bpf_attr __user *uattr)
2644	{
2645	return -ENOTSUPP;
2646	}
2647
2648	static inline int bpf_prog_test_run_flow_dissector(struct bpf_prog *prog,
2649	const union bpf_attr *kattr,
2650	union bpf_attr __user *uattr)
2651	{
2652	return -ENOTSUPP;
2653	}
2654
2655	static inline int bpf_prog_test_run_sk_lookup(struct bpf_prog *prog,
2656	const union bpf_attr *kattr,
2657	union bpf_attr __user *uattr)
2658	{
2659	return -ENOTSUPP;
2660	}
2661
2662	static inline void bpf_map_put(struct bpf_map *map)
2663	{
2664	}
2665
2666	static inline struct bpf_prog *bpf_prog_by_id(u32 id)
2667	{
2668	return ERR_PTR(-ENOTSUPP);
2669	}
2670
2671	static inline int btf_struct_access(struct bpf_verifier_log *log,
2672	const struct bpf_reg_state *reg,
2673	int off, int size, enum bpf_access_type atype,
2674	u32 next_btf_id, enum bpf_type_flag flag,
2675	const char **field_name)
2676	{
2677	return -EACCES;
2678	}
2679
2680	static inline const struct bpf_func_proto *
2681	bpf_base_func_proto(enum bpf_func_id func_id)
2682	{
2683	return NULL;
2684	}
2685
2686	static inline void bpf_task_storage_free(struct task_struct *task)
2687	{
2688	}
2689
2690	static inline bool bpf_prog_has_kfunc_call(const struct bpf_prog *prog)
2691	{
2692	return false;
2693	}
2694
2695	static inline const struct btf_func_model *
2696	bpf_jit_find_kfunc_model(const struct bpf_prog *prog,
2697	const struct bpf_insn *insn)
2698	{
2699	return NULL;
2700	}
2701
2702	static inline int
2703	bpf_get_kfunc_addr(const struct bpf_prog *prog, u32 func_id,
2704	u16 btf_fd_idx, u8 **func_addr)
2705	{
2706	return -ENOTSUPP;
2707	}
2708
2709	static inline bool unprivileged_ebpf_enabled(void)
2710	{
2711	return false;
2712	}
2713
2714	static inline bool has_current_bpf_ctx(void)
2715	{
2716	return false;
2717	}
2718
2719	static inline void bpf_prog_inc_misses_counter(struct bpf_prog *prog)
2720	{
2721	}
2722
2723	static inline void bpf_cgrp_storage_free(struct cgroup *cgroup)
2724	{
2725	}
2726
2727	static inline void bpf_dynptr_init(struct bpf_dynptr_kern ptr, void data,
2728	enum bpf_dynptr_type type, u32 offset, u32 size)
2729	{
2730	}
2731
2732	static inline void bpf_dynptr_set_null(struct bpf_dynptr_kern *ptr)
2733	{
2734	}
2735
2736	static inline void bpf_dynptr_set_rdonly(struct bpf_dynptr_kern *ptr)
2737	{
2738	}
2739	#endif /* CONFIG_BPF_SYSCALL */
2740
2741	static __always_inline int
2742	bpf_probe_read_kernel_common(void dst, u32 size, const void unsafe_ptr)
2743	{
2744	int ret = -EFAULT;
2745
2746	if (IS_ENABLED(CONFIG_BPF_EVENTS))
2747	ret = copy_from_kernel_nofault(dst, unsafe_ptr, size);
2748	if (unlikely(ret < 0))
2749	memset(dst, 0, size);
2750	return ret;
2751	}
2752
2753	void __bpf_free_used_btfs(struct bpf_prog_aux *aux,
2754	struct btf_mod_pair *used_btfs, u32 len);
2755
2756	static inline struct bpf_prog *bpf_prog_get_type(u32 ufd,
2757	enum bpf_prog_type type)
2758	{
2759	return bpf_prog_get_type_dev(ufd, type, false);
2760	}
2761
2762	void __bpf_free_used_maps(struct bpf_prog_aux *aux,
2763	struct bpf_map **used_maps, u32 len);
2764
2765	bool bpf_prog_get_ok(struct bpf_prog , enum bpf_prog_type , bool);
2766
2767	int bpf_prog_offload_compile(struct bpf_prog *prog);
2768	void bpf_prog_dev_bound_destroy(struct bpf_prog *prog);
2769	int bpf_prog_offload_info_fill(struct bpf_prog_info *info,
2770	struct bpf_prog *prog);
2771
2772	int bpf_map_offload_info_fill(struct bpf_map_info info, struct bpf_map map);
2773
2774	int bpf_map_offload_lookup_elem(struct bpf_map map, void key, void *value);
2775	int bpf_map_offload_update_elem(struct bpf_map *map,
2776	void key, void value, u64 flags);
2777	int bpf_map_offload_delete_elem(struct bpf_map map, void key);
2778	int bpf_map_offload_get_next_key(struct bpf_map *map,
2779	void key, void next_key);
2780
2781	bool bpf_offload_prog_map_match(struct bpf_prog prog, struct bpf_map map);
2782
2783	struct bpf_offload_dev *
2784	bpf_offload_dev_create(const struct bpf_prog_offload_ops ops, void priv);
2785	void bpf_offload_dev_destroy(struct bpf_offload_dev *offdev);
2786	void bpf_offload_dev_priv(struct bpf_offload_dev offdev);
2787	int bpf_offload_dev_netdev_register(struct bpf_offload_dev *offdev,
2788	struct net_device *netdev);
2789	void bpf_offload_dev_netdev_unregister(struct bpf_offload_dev *offdev,
2790	struct net_device *netdev);
2791	bool bpf_offload_dev_match(struct bpf_prog prog, struct net_device netdev);
2792
2793	void unpriv_ebpf_notify(int new_state);
2794
2795	#if defined(CONFIG_NET) && defined(CONFIG_BPF_SYSCALL)
2796	int bpf_dev_bound_kfunc_check(struct bpf_verifier_log *log,
2797	struct bpf_prog_aux *prog_aux);
2798	void bpf_dev_bound_resolve_kfunc(struct bpf_prog prog, u32 func_id);
2799	int bpf_prog_dev_bound_init(struct bpf_prog prog, union bpf_attr attr);
2800	int bpf_prog_dev_bound_inherit(struct bpf_prog new_prog, struct bpf_prog old_prog);
2801	void bpf_dev_bound_netdev_unregister(struct net_device *dev);
2802
2803	static inline bool bpf_prog_is_dev_bound(const struct bpf_prog_aux *aux)
2804	{
2805	return aux->dev_bound;
2806	}
2807
2808	static inline bool bpf_prog_is_offloaded(const struct bpf_prog_aux *aux)
2809	{
2810	return aux->offload_requested;
2811	}
2812
2813	bool bpf_prog_dev_bound_match(const struct bpf_prog lhs, const struct bpf_prog rhs);
2814
2815	static inline bool bpf_map_is_offloaded(struct bpf_map *map)
2816	{
2817	return unlikely(map->ops == &bpf_map_offload_ops);
2818	}
2819
2820	struct bpf_map bpf_map_offload_map_alloc(union bpf_attr attr);
2821	void bpf_map_offload_map_free(struct bpf_map *map);
2822	u64 bpf_map_offload_map_mem_usage(const struct bpf_map *map);
2823	int bpf_prog_test_run_syscall(struct bpf_prog *prog,
2824	const union bpf_attr *kattr,
2825	union bpf_attr __user *uattr);
2826
2827	int sock_map_get_from_fd(const union bpf_attr attr, struct bpf_prog prog);
2828	int sock_map_prog_detach(const union bpf_attr *attr, enum bpf_prog_type ptype);
2829	int sock_map_update_elem_sys(struct bpf_map map, void key, void *value, u64 flags);
2830	int sock_map_bpf_prog_query(const union bpf_attr *attr,
2831	union bpf_attr __user *uattr);
2832
2833	void sock_map_unhash(struct sock *sk);
2834	void sock_map_destroy(struct sock *sk);
2835	void sock_map_close(struct sock *sk, long timeout);
2836	#else
2837	static inline int bpf_dev_bound_kfunc_check(struct bpf_verifier_log *log,
2838	struct bpf_prog_aux *prog_aux)
2839	{
2840	return -EOPNOTSUPP;
2841	}
2842
2843	static inline void bpf_dev_bound_resolve_kfunc(struct bpf_prog prog,
2844	u32 func_id)
2845	{
2846	return NULL;
2847	}
2848
2849	static inline int bpf_prog_dev_bound_init(struct bpf_prog *prog,
2850	union bpf_attr *attr)
2851	{
2852	return -EOPNOTSUPP;
2853	}
2854
2855	static inline int bpf_prog_dev_bound_inherit(struct bpf_prog *new_prog,
2856	struct bpf_prog *old_prog)
2857	{
2858	return -EOPNOTSUPP;
2859	}
2860
2861	static inline void bpf_dev_bound_netdev_unregister(struct net_device *dev)
2862	{
2863	}
2864
2865	static inline bool bpf_prog_is_dev_bound(const struct bpf_prog_aux *aux)
2866	{
2867	return false;
2868	}
2869
2870	static inline bool bpf_prog_is_offloaded(struct bpf_prog_aux *aux)
2871	{
2872	return false;
2873	}
2874
2875	static inline bool bpf_prog_dev_bound_match(const struct bpf_prog lhs, const struct bpf_prog rhs)
2876	{
2877	return false;
2878	}
2879
2880	static inline bool bpf_map_is_offloaded(struct bpf_map *map)
2881	{
2882	return false;
2883	}
2884
2885	static inline struct bpf_map bpf_map_offload_map_alloc(union bpf_attr attr)
2886	{
2887	return ERR_PTR(-EOPNOTSUPP);
2888	}
2889
2890	static inline void bpf_map_offload_map_free(struct bpf_map *map)
2891	{
2892	}
2893
2894	static inline u64 bpf_map_offload_map_mem_usage(const struct bpf_map *map)
2895	{
2896	return 0;
2897	}
2898
2899	static inline int bpf_prog_test_run_syscall(struct bpf_prog *prog,
2900	const union bpf_attr *kattr,
2901	union bpf_attr __user *uattr)
2902	{
2903	return -ENOTSUPP;
2904	}
2905
2906	#ifdef CONFIG_BPF_SYSCALL
2907	static inline int sock_map_get_from_fd(const union bpf_attr *attr,
2908	struct bpf_prog *prog)
2909	{
2910	return -EINVAL;
2911	}
2912
2913	static inline int sock_map_prog_detach(const union bpf_attr *attr,
2914	enum bpf_prog_type ptype)
2915	{
2916	return -EOPNOTSUPP;
2917	}
2918
2919	static inline int sock_map_update_elem_sys(struct bpf_map map, void key, void *value,
2920	u64 flags)
2921	{
2922	return -EOPNOTSUPP;
2923	}
2924
2925	static inline int sock_map_bpf_prog_query(const union bpf_attr *attr,
2926	union bpf_attr __user *uattr)
2927	{
2928	return -EINVAL;
2929	}
2930	#endif /* CONFIG_BPF_SYSCALL */
2931	#endif /* CONFIG_NET && CONFIG_BPF_SYSCALL */
2932
2933	static __always_inline void
2934	bpf_prog_inc_misses_counters(const struct bpf_prog_array *array)
2935	{
2936	const struct bpf_prog_array_item *item;
2937	struct bpf_prog *prog;
2938
2939	if (unlikely(!array))
2940	return;
2941
2942	item = &array->items[0];
2943	while ((prog = READ_ONCE(item->prog))) {
2944	bpf_prog_inc_misses_counter(prog);
2945	item++;
2946	}
2947	}
2948
2949	#if defined(CONFIG_INET) && defined(CONFIG_BPF_SYSCALL)
2950	void bpf_sk_reuseport_detach(struct sock *sk);
2951	int bpf_fd_reuseport_array_lookup_elem(struct bpf_map map, void key,
2952	void *value);
2953	int bpf_fd_reuseport_array_update_elem(struct bpf_map map, void key,
2954	void *value, u64 map_flags);
2955	#else
2956	static inline void bpf_sk_reuseport_detach(struct sock *sk)
2957	{
2958	}
2959
2960	#ifdef CONFIG_BPF_SYSCALL
2961	static inline int bpf_fd_reuseport_array_lookup_elem(struct bpf_map *map,
2962	void key, void value)
2963	{
2964	return -EOPNOTSUPP;
2965	}
2966
2967	static inline int bpf_fd_reuseport_array_update_elem(struct bpf_map *map,
2968	void key, void value,
2969	u64 map_flags)
2970	{
2971	return -EOPNOTSUPP;
2972	}
2973	#endif /* CONFIG_BPF_SYSCALL */
2974	#endif /* defined(CONFIG_INET) && defined(CONFIG_BPF_SYSCALL) */
2975
2976	/* verifier prototypes for helper functions called from eBPF programs */
2977	extern const struct bpf_func_proto bpf_map_lookup_elem_proto;
2978	extern const struct bpf_func_proto bpf_map_update_elem_proto;
2979	extern const struct bpf_func_proto bpf_map_delete_elem_proto;
2980	extern const struct bpf_func_proto bpf_map_push_elem_proto;
2981	extern const struct bpf_func_proto bpf_map_pop_elem_proto;
2982	extern const struct bpf_func_proto bpf_map_peek_elem_proto;
2983	extern const struct bpf_func_proto bpf_map_lookup_percpu_elem_proto;
2984
2985	extern const struct bpf_func_proto bpf_get_prandom_u32_proto;
2986	extern const struct bpf_func_proto bpf_get_smp_processor_id_proto;
2987	extern const struct bpf_func_proto bpf_get_numa_node_id_proto;
2988	extern const struct bpf_func_proto bpf_tail_call_proto;
2989	extern const struct bpf_func_proto bpf_ktime_get_ns_proto;
2990	extern const struct bpf_func_proto bpf_ktime_get_boot_ns_proto;
2991	extern const struct bpf_func_proto bpf_ktime_get_tai_ns_proto;
2992	extern const struct bpf_func_proto bpf_get_current_pid_tgid_proto;
2993	extern const struct bpf_func_proto bpf_get_current_uid_gid_proto;
2994	extern const struct bpf_func_proto bpf_get_current_comm_proto;
2995	extern const struct bpf_func_proto bpf_get_stackid_proto;
2996	extern const struct bpf_func_proto bpf_get_stack_proto;
2997	extern const struct bpf_func_proto bpf_get_task_stack_proto;
2998	extern const struct bpf_func_proto bpf_get_stackid_proto_pe;
2999	extern const struct bpf_func_proto bpf_get_stack_proto_pe;
3000	extern const struct bpf_func_proto bpf_sock_map_update_proto;
3001	extern const struct bpf_func_proto bpf_sock_hash_update_proto;
3002	extern const struct bpf_func_proto bpf_get_current_cgroup_id_proto;
3003	extern const struct bpf_func_proto bpf_get_current_ancestor_cgroup_id_proto;
3004	extern const struct bpf_func_proto bpf_get_cgroup_classid_curr_proto;
3005	extern const struct bpf_func_proto bpf_msg_redirect_hash_proto;
3006	extern const struct bpf_func_proto bpf_msg_redirect_map_proto;
3007	extern const struct bpf_func_proto bpf_sk_redirect_hash_proto;
3008	extern const struct bpf_func_proto bpf_sk_redirect_map_proto;
3009	extern const struct bpf_func_proto bpf_spin_lock_proto;
3010	extern const struct bpf_func_proto bpf_spin_unlock_proto;
3011	extern const struct bpf_func_proto bpf_get_local_storage_proto;
3012	extern const struct bpf_func_proto bpf_strtol_proto;
3013	extern const struct bpf_func_proto bpf_strtoul_proto;
3014	extern const struct bpf_func_proto bpf_tcp_sock_proto;
3015	extern const struct bpf_func_proto bpf_jiffies64_proto;
3016	extern const struct bpf_func_proto bpf_get_ns_current_pid_tgid_proto;
3017	extern const struct bpf_func_proto bpf_event_output_data_proto;
3018	extern const struct bpf_func_proto bpf_ringbuf_output_proto;
3019	extern const struct bpf_func_proto bpf_ringbuf_reserve_proto;
3020	extern const struct bpf_func_proto bpf_ringbuf_submit_proto;
3021	extern const struct bpf_func_proto bpf_ringbuf_discard_proto;
3022	extern const struct bpf_func_proto bpf_ringbuf_query_proto;
3023	extern const struct bpf_func_proto bpf_ringbuf_reserve_dynptr_proto;
3024	extern const struct bpf_func_proto bpf_ringbuf_submit_dynptr_proto;
3025	extern const struct bpf_func_proto bpf_ringbuf_discard_dynptr_proto;
3026	extern const struct bpf_func_proto bpf_skc_to_tcp6_sock_proto;
3027	extern const struct bpf_func_proto bpf_skc_to_tcp_sock_proto;
3028	extern const struct bpf_func_proto bpf_skc_to_tcp_timewait_sock_proto;
3029	extern const struct bpf_func_proto bpf_skc_to_tcp_request_sock_proto;
3030	extern const struct bpf_func_proto bpf_skc_to_udp6_sock_proto;
3031	extern const struct bpf_func_proto bpf_skc_to_unix_sock_proto;
3032	extern const struct bpf_func_proto bpf_skc_to_mptcp_sock_proto;
3033	extern const struct bpf_func_proto bpf_copy_from_user_proto;
3034	extern const struct bpf_func_proto bpf_snprintf_btf_proto;
3035	extern const struct bpf_func_proto bpf_snprintf_proto;
3036	extern const struct bpf_func_proto bpf_per_cpu_ptr_proto;
3037	extern const struct bpf_func_proto bpf_this_cpu_ptr_proto;
3038	extern const struct bpf_func_proto bpf_ktime_get_coarse_ns_proto;
3039	extern const struct bpf_func_proto bpf_sock_from_file_proto;
3040	extern const struct bpf_func_proto bpf_get_socket_ptr_cookie_proto;
3041	extern const struct bpf_func_proto bpf_task_storage_get_recur_proto;
3042	extern const struct bpf_func_proto bpf_task_storage_get_proto;
3043	extern const struct bpf_func_proto bpf_task_storage_delete_recur_proto;
3044	extern const struct bpf_func_proto bpf_task_storage_delete_proto;
3045	extern const struct bpf_func_proto bpf_for_each_map_elem_proto;
3046	extern const struct bpf_func_proto bpf_btf_find_by_name_kind_proto;
3047	extern const struct bpf_func_proto bpf_sk_setsockopt_proto;
3048	extern const struct bpf_func_proto bpf_sk_getsockopt_proto;
3049	extern const struct bpf_func_proto bpf_unlocked_sk_setsockopt_proto;
3050	extern const struct bpf_func_proto bpf_unlocked_sk_getsockopt_proto;
3051	extern const struct bpf_func_proto bpf_find_vma_proto;
3052	extern const struct bpf_func_proto bpf_loop_proto;
3053	extern const struct bpf_func_proto bpf_copy_from_user_task_proto;
3054	extern const struct bpf_func_proto bpf_set_retval_proto;
3055	extern const struct bpf_func_proto bpf_get_retval_proto;
3056	extern const struct bpf_func_proto bpf_user_ringbuf_drain_proto;
3057	extern const struct bpf_func_proto bpf_cgrp_storage_get_proto;
3058	extern const struct bpf_func_proto bpf_cgrp_storage_delete_proto;
3059
3060	const struct bpf_func_proto *tracing_prog_func_proto(
3061	enum bpf_func_id func_id, const struct bpf_prog *prog);
3062
3063	/* Shared helpers among cBPF and eBPF. */
3064	void bpf_user_rnd_init_once(void);
3065	u64 bpf_user_rnd_u32(u64 r1, u64 r2, u64 r3, u64 r4, u64 r5);
3066	u64 bpf_get_raw_cpu_id(u64 r1, u64 r2, u64 r3, u64 r4, u64 r5);
3067
3068	#if defined(CONFIG_NET)
3069	bool bpf_sock_common_is_valid_access(int off, int size,
3070	enum bpf_access_type type,
3071	struct bpf_insn_access_aux *info);
3072	bool bpf_sock_is_valid_access(int off, int size, enum bpf_access_type type,
3073	struct bpf_insn_access_aux *info);
3074	u32 bpf_sock_convert_ctx_access(enum bpf_access_type type,
3075	const struct bpf_insn *si,
3076	struct bpf_insn *insn_buf,
3077	struct bpf_prog *prog,
3078	u32 *target_size);
3079	int bpf_dynptr_from_skb_rdonly(struct sk_buff *skb, u64 flags,
3080	struct bpf_dynptr_kern *ptr);
3081	#else
3082	static inline bool bpf_sock_common_is_valid_access(int off, int size,
3083	enum bpf_access_type type,
3084	struct bpf_insn_access_aux *info)
3085	{
3086	return false;
3087	}
3088	static inline bool bpf_sock_is_valid_access(int off, int size,
3089	enum bpf_access_type type,
3090	struct bpf_insn_access_aux *info)
3091	{
3092	return false;
3093	}
3094	static inline u32 bpf_sock_convert_ctx_access(enum bpf_access_type type,
3095	const struct bpf_insn *si,
3096	struct bpf_insn *insn_buf,
3097	struct bpf_prog *prog,
3098	u32 *target_size)
3099	{
3100	return 0;
3101	}
3102	static inline int bpf_dynptr_from_skb_rdonly(struct sk_buff *skb, u64 flags,
3103	struct bpf_dynptr_kern *ptr)
3104	{
3105	return -EOPNOTSUPP;
3106	}
3107	#endif
3108
3109	#ifdef CONFIG_INET
3110	struct sk_reuseport_kern {
3111	struct sk_buff *skb;
3112	struct sock *sk;
3113	struct sock *selected_sk;
3114	struct sock *migrating_sk;
3115	void *data_end;
3116	u32 hash;
3117	u32 reuseport_id;
3118	bool bind_inany;
3119	};
3120	bool bpf_tcp_sock_is_valid_access(int off, int size, enum bpf_access_type type,
3121	struct bpf_insn_access_aux *info);
3122
3123	u32 bpf_tcp_sock_convert_ctx_access(enum bpf_access_type type,
3124	const struct bpf_insn *si,
3125	struct bpf_insn *insn_buf,
3126	struct bpf_prog *prog,
3127	u32 *target_size);
3128
3129	bool bpf_xdp_sock_is_valid_access(int off, int size, enum bpf_access_type type,
3130	struct bpf_insn_access_aux *info);
3131
3132	u32 bpf_xdp_sock_convert_ctx_access(enum bpf_access_type type,
3133	const struct bpf_insn *si,
3134	struct bpf_insn *insn_buf,
3135	struct bpf_prog *prog,
3136	u32 *target_size);
3137	#else
3138	static inline bool bpf_tcp_sock_is_valid_access(int off, int size,
3139	enum bpf_access_type type,
3140	struct bpf_insn_access_aux *info)
3141	{
3142	return false;
3143	}
3144
3145	static inline u32 bpf_tcp_sock_convert_ctx_access(enum bpf_access_type type,
3146	const struct bpf_insn *si,
3147	struct bpf_insn *insn_buf,
3148	struct bpf_prog *prog,
3149	u32 *target_size)
3150	{
3151	return 0;
3152	}
3153	static inline bool bpf_xdp_sock_is_valid_access(int off, int size,
3154	enum bpf_access_type type,
3155	struct bpf_insn_access_aux *info)
3156	{
3157	return false;
3158	}
3159
3160	static inline u32 bpf_xdp_sock_convert_ctx_access(enum bpf_access_type type,
3161	const struct bpf_insn *si,
3162	struct bpf_insn *insn_buf,
3163	struct bpf_prog *prog,
3164	u32 *target_size)
3165	{
3166	return 0;
3167	}
3168	#endif /* CONFIG_INET */
3169
3170	enum bpf_text_poke_type {
3171	BPF_MOD_CALL,
3172	BPF_MOD_JUMP,
3173	};
3174
3175	int bpf_arch_text_poke(void *ip, enum bpf_text_poke_type t,
3176	void addr1, void addr2);
3177
3178	void bpf_arch_poke_desc_update(struct bpf_jit_poke_descriptor *poke,
3179	struct bpf_prog new, struct bpf_prog old);
3180
3181	void bpf_arch_text_copy(void dst, void *src, size_t len);
3182	int bpf_arch_text_invalidate(void *dst, size_t len);
3183
3184	struct btf_id_set;
3185	bool btf_id_set_contains(const struct btf_id_set *set, u32 id);
3186
3187	#define MAX_BPRINTF_VARARGS 12
3188	#define MAX_BPRINTF_BUF 1024
3189
3190	struct bpf_bprintf_data {
3191	u32 *bin_args;
3192	char *buf;
3193	bool get_bin_args;
3194	bool get_buf;
3195	};
3196
3197	int bpf_bprintf_prepare(char fmt, u32 fmt_size, const u64 raw_args,
3198	u32 num_args, struct bpf_bprintf_data *data);
3199	void bpf_bprintf_cleanup(struct bpf_bprintf_data *data);
3200
3201	#ifdef CONFIG_BPF_LSM
3202	void bpf_cgroup_atype_get(u32 attach_btf_id, int cgroup_atype);
3203	void bpf_cgroup_atype_put(int cgroup_atype);
3204	#else
3205	static inline void bpf_cgroup_atype_get(u32 attach_btf_id, int cgroup_atype) {}
3206	static inline void bpf_cgroup_atype_put(int cgroup_atype) {}
3207	#endif /* CONFIG_BPF_LSM */
3208
3209	struct key;
3210
3211	#ifdef CONFIG_KEYS
3212	struct bpf_key {
3213	struct key *key;
3214	bool has_ref;
3215	};
3216	#endif /* CONFIG_KEYS */
3217
3218	static inline bool type_is_alloc(u32 type)
3219	{
3220	return type & MEM_ALLOC;
3221	}
3222
3223	static inline gfp_t bpf_memcg_flags(gfp_t flags)
3224	{
3225	if (memcg_bpf_enabled())
3226	return flags \| __GFP_ACCOUNT;
3227	return flags;
3228	}
3229
3230	static inline bool bpf_is_subprog(const struct bpf_prog *prog)
3231	{
3232	return prog->aux->func_idx != 0;
3233	}
3234
3235	#endif /* _LINUX_BPF_H */