projects/linux·blob master · refs

6760 lines · 170.0 KiB

1// SPDX-License-Identifier: GPL-2.0-only
2/* Copyright (c) 2011-2014 PLUMgrid, http://plumgrid.com
3 */
4#include <crypto/sha2.h>
5#include <linux/bpf.h>
6#include <linux/bpf-cgroup.h>
7#include <linux/bpf_trace.h>
8#include <linux/bpf_lirc.h>
9#include <linux/bpf_verifier.h>
10#include <linux/bsearch.h>
11#include <linux/btf.h>
12#include <linux/hex.h>
13#include <linux/syscalls.h>
14#include <linux/slab.h>
15#include <linux/sched/signal.h>
16#include <linux/vmalloc.h>
17#include <linux/mmzone.h>
18#include <linux/anon_inodes.h>
19#include <linux/fdtable.h>
20#include <linux/file.h>
21#include <linux/fs.h>
22#include <linux/license.h>
23#include <linux/filter.h>
24#include <linux/kernel.h>
25#include <linux/idr.h>
26#include <linux/cred.h>
27#include <linux/timekeeping.h>
28#include <linux/ctype.h>
29#include <linux/nospec.h>
30#include <linux/audit.h>
31#include <uapi/linux/btf.h>
32#include <linux/pgtable.h>
33#include <linux/bpf_lsm.h>
34#include <linux/poll.h>
35#include <linux/sort.h>
36#include <linux/bpf-netns.h>
37#include <linux/rcupdate_trace.h>
38#include <linux/memcontrol.h>
39#include <linux/trace_events.h>
40#include <linux/tracepoint.h>
41#include <linux/overflow.h>
42#include <linux/cookie.h>
43#include <linux/btf_ids.h>
44
45#include <net/netfilter/nf_bpf_link.h>
46#include <net/netkit.h>
47#include <net/tcx.h>
48
49#define IS_FD_ARRAY(map) ((map)->map_type == BPF_MAP_TYPE_PERF_EVENT_ARRAY || \
50 (map)->map_type == BPF_MAP_TYPE_CGROUP_ARRAY || \
51 (map)->map_type == BPF_MAP_TYPE_ARRAY_OF_MAPS)
52#define IS_FD_PROG_ARRAY(map) ((map)->map_type == BPF_MAP_TYPE_PROG_ARRAY)
53#define IS_FD_HASH(map) ((map)->map_type == BPF_MAP_TYPE_HASH_OF_MAPS)
54#define IS_FD_MAP(map) (IS_FD_ARRAY(map) || IS_FD_PROG_ARRAY(map) || \
55 IS_FD_HASH(map))
56
57#define BPF_OBJ_FLAG_MASK (BPF_F_RDONLY | BPF_F_WRONLY)
58
59DEFINE_PER_CPU(int, bpf_prog_active);
60DEFINE_COOKIE(bpf_map_cookie);
61static DEFINE_IDR(prog_idr);
62static DEFINE_SPINLOCK(prog_idr_lock);
63static DEFINE_IDR(map_idr);
64static DEFINE_SPINLOCK(map_idr_lock);
65static DEFINE_IDR(link_idr);
66static DEFINE_SPINLOCK(link_idr_lock);
67
68int sysctl_unprivileged_bpf_disabled __read_mostly =
69 IS_BUILTIN(CONFIG_BPF_UNPRIV_DEFAULT_OFF) ? 2 : 0;
70
71static const struct bpf_map_ops * const bpf_map_types[] = {
72#define BPF_PROG_TYPE(_id, _name, prog_ctx_type, kern_ctx_type)
73#define BPF_MAP_TYPE(_id, _ops) \
74 [_id] = &_ops,
75#define BPF_LINK_TYPE(_id, _name)
76#include <linux/bpf_types.h>
77#undef BPF_PROG_TYPE
78#undef BPF_MAP_TYPE
79#undef BPF_LINK_TYPE
80};
81
82/*
83 * If we're handed a bigger struct than we know of, ensure all the unknown bits
84 * are 0 - i.e. new user-space does not rely on any kernel feature extensions
85 * we don't know about yet.
86 *
87 * There is a ToCToU between this function call and the following
88 * copy_from_user() call. However, this is not a concern since this function is
89 * meant to be a future-proofing of bits.
90 */
91int bpf_check_uarg_tail_zero(bpfptr_t uaddr,
92 size_t expected_size,
93 size_t actual_size)
94{
95 int res;
96
97 if (unlikely(actual_size > PAGE_SIZE)) /* silly large */
98 return -E2BIG;
99
100 if (actual_size <= expected_size)
101 return 0;
102
103 if (uaddr.is_kernel)
104 res = memchr_inv(uaddr.kernel + expected_size, 0,
105 actual_size - expected_size) == NULL;
106 else
107 res = check_zeroed_user(uaddr.user + expected_size,
108 actual_size - expected_size);
109 if (res < 0)
110 return res;
111 return res ? 0 : -E2BIG;
112}
113
114const struct bpf_map_ops bpf_map_offload_ops = {
115 .map_meta_equal = bpf_map_meta_equal,
116 .map_alloc = bpf_map_offload_map_alloc,
117 .map_free = bpf_map_offload_map_free,
118 .map_check_btf = map_check_no_btf,
119 .map_mem_usage = bpf_map_offload_map_mem_usage,
120};
121
122static void bpf_map_write_active_inc(struct bpf_map *map)
123{
124 atomic64_inc(&map->writecnt);
125}
126
127static void bpf_map_write_active_dec(struct bpf_map *map)
128{
129 atomic64_dec(&map->writecnt);
130}
131
132bool bpf_map_write_active(const struct bpf_map *map)
133{
134 return atomic64_read(&map->writecnt) != 0;
135}
136
137static u32 bpf_map_value_size(const struct bpf_map *map, u64 flags)
138{
139 if (flags & (BPF_F_CPU | BPF_F_ALL_CPUS))
140 return map->value_size;
141 else if (map->map_type == BPF_MAP_TYPE_PERCPU_HASH ||
142 map->map_type == BPF_MAP_TYPE_LRU_PERCPU_HASH ||
143 map->map_type == BPF_MAP_TYPE_PERCPU_ARRAY ||
144 map->map_type == BPF_MAP_TYPE_PERCPU_CGROUP_STORAGE)
145 return round_up(map->value_size, 8) * num_possible_cpus();
146 else if (IS_FD_MAP(map))
147 return sizeof(u32);
148 else
149 return map->value_size;
150}
151
152static void maybe_wait_bpf_programs(struct bpf_map *map)
153{
154 /* Wait for any running non-sleepable BPF programs to complete so that
155 * userspace, when we return to it, knows that all non-sleepable
156 * programs that could be running use the new map value. For sleepable
157 * BPF programs, synchronize_rcu_tasks_trace() should be used to wait
158 * for the completions of these programs, but considering the waiting
159 * time can be very long and userspace may think it will hang forever,
160 * so don't handle sleepable BPF programs now.
161 */
162 if (map->map_type == BPF_MAP_TYPE_HASH_OF_MAPS ||
163 map->map_type == BPF_MAP_TYPE_ARRAY_OF_MAPS)
164 synchronize_rcu_expedited();
165}
166
167static void unpin_uptr_kaddr(void *kaddr)
168{
169 if (kaddr)
170 unpin_user_page(virt_to_page(kaddr));
171}
172
173static void __bpf_obj_unpin_uptrs(struct btf_record *rec, u32 cnt, void *obj)
174{
175 const struct btf_field *field;
176 void **uptr_addr;
177 int i;
178
179 for (i = 0, field = rec->fields; i < cnt; i++, field++) {
180 if (field->type != BPF_UPTR)
181 continue;
182
183 uptr_addr = obj + field->offset;
184 unpin_uptr_kaddr(*uptr_addr);
185 }
186}
187
188static void bpf_obj_unpin_uptrs(struct btf_record *rec, void *obj)
189{
190 if (!btf_record_has_field(rec, BPF_UPTR))
191 return;
192
193 __bpf_obj_unpin_uptrs(rec, rec->cnt, obj);
194}
195
196static int bpf_obj_pin_uptrs(struct btf_record *rec, void *obj)
197{
198 const struct btf_field *field;
199 const struct btf_type *t;
200 unsigned long start, end;
201 struct page *page;
202 void **uptr_addr;
203 int i, err;
204
205 if (!btf_record_has_field(rec, BPF_UPTR))
206 return 0;
207
208 for (i = 0, field = rec->fields; i < rec->cnt; i++, field++) {
209 if (field->type != BPF_UPTR)
210 continue;
211
212 uptr_addr = obj + field->offset;
213 start = *(unsigned long *)uptr_addr;
214 if (!start)
215 continue;
216
217 t = btf_type_by_id(field->kptr.btf, field->kptr.btf_id);
218 /* t->size was checked for zero before */
219 if (check_add_overflow(start, t->size - 1, &end)) {
220 err = -EFAULT;
221 goto unpin_all;
222 }
223
224 /* The uptr's struct cannot span across two pages */
225 if ((start & PAGE_MASK) != (end & PAGE_MASK)) {
226 err = -EOPNOTSUPP;
227 goto unpin_all;
228 }
229
230 err = pin_user_pages_fast(start, 1, FOLL_LONGTERM | FOLL_WRITE, &page);
231 if (err != 1)
232 goto unpin_all;
233
234 if (PageHighMem(page)) {
235 err = -EOPNOTSUPP;
236 unpin_user_page(page);
237 goto unpin_all;
238 }
239
240 *uptr_addr = page_address(page) + offset_in_page(start);
241 }
242
243 return 0;
244
245unpin_all:
246 __bpf_obj_unpin_uptrs(rec, i, obj);
247 return err;
248}
249
250static int bpf_map_update_value(struct bpf_map *map, struct file *map_file,
251 void *key, void *value, __u64 flags)
252{
253 int err;
254
255 /* Need to create a kthread, thus must support schedule */
256 if (bpf_map_is_offloaded(map)) {
257 return bpf_map_offload_update_elem(map, key, value, flags);
258 } else if (map->map_type == BPF_MAP_TYPE_CPUMAP ||
259 map->map_type == BPF_MAP_TYPE_ARENA ||
260 map->map_type == BPF_MAP_TYPE_STRUCT_OPS) {
261 return map->ops->map_update_elem(map, key, value, flags);
262 } else if (map->map_type == BPF_MAP_TYPE_SOCKHASH ||
263 map->map_type == BPF_MAP_TYPE_SOCKMAP) {
264 return sock_map_update_elem_sys(map, key, value, flags);
265 } else if (IS_FD_PROG_ARRAY(map)) {
266 return bpf_fd_array_map_update_elem(map, map_file, key, value,
267 flags);
268 }
269
270 bpf_disable_instrumentation();
271 if (map->map_type == BPF_MAP_TYPE_PERCPU_HASH ||
272 map->map_type == BPF_MAP_TYPE_LRU_PERCPU_HASH) {
273 err = bpf_percpu_hash_update(map, key, value, flags);
274 } else if (map->map_type == BPF_MAP_TYPE_PERCPU_ARRAY) {
275 err = bpf_percpu_array_update(map, key, value, flags);
276 } else if (map->map_type == BPF_MAP_TYPE_PERCPU_CGROUP_STORAGE) {
277 err = bpf_percpu_cgroup_storage_update(map, key, value,
278 flags);
279 } else if (IS_FD_ARRAY(map)) {
280 err = bpf_fd_array_map_update_elem(map, map_file, key, value,
281 flags);
282 } else if (map->map_type == BPF_MAP_TYPE_HASH_OF_MAPS) {
283 err = bpf_fd_htab_map_update_elem(map, map_file, key, value,
284 flags);
285 } else if (map->map_type == BPF_MAP_TYPE_REUSEPORT_SOCKARRAY) {
286 /* rcu_read_lock() is not needed */
287 err = bpf_fd_reuseport_array_update_elem(map, key, value,
288 flags);
289 } else if (map->map_type == BPF_MAP_TYPE_QUEUE ||
290 map->map_type == BPF_MAP_TYPE_STACK ||
291 map->map_type == BPF_MAP_TYPE_BLOOM_FILTER) {
292 err = map->ops->map_push_elem(map, value, flags);
293 } else {
294 err = bpf_obj_pin_uptrs(map->record, value);
295 if (!err) {
296 rcu_read_lock();
297 err = map->ops->map_update_elem(map, key, value, flags);
298 rcu_read_unlock();
299 if (err)
300 bpf_obj_unpin_uptrs(map->record, value);
301 }
302 }
303 bpf_enable_instrumentation();
304
305 return err;
306}
307
308static int bpf_map_copy_value(struct bpf_map *map, void *key, void *value,
309 __u64 flags)
310{
311 void *ptr;
312 int err;
313
314 if (bpf_map_is_offloaded(map))
315 return bpf_map_offload_lookup_elem(map, key, value);
316
317 bpf_disable_instrumentation();
318 if (map->map_type == BPF_MAP_TYPE_PERCPU_HASH ||
319 map->map_type == BPF_MAP_TYPE_LRU_PERCPU_HASH) {
320 err = bpf_percpu_hash_copy(map, key, value, flags);
321 } else if (map->map_type == BPF_MAP_TYPE_PERCPU_ARRAY) {
322 err = bpf_percpu_array_copy(map, key, value, flags);
323 } else if (map->map_type == BPF_MAP_TYPE_PERCPU_CGROUP_STORAGE) {
324 err = bpf_percpu_cgroup_storage_copy(map, key, value, flags);
325 } else if (map->map_type == BPF_MAP_TYPE_STACK_TRACE) {
326 err = bpf_stackmap_extract(map, key, value, false);
327 } else if (IS_FD_ARRAY(map) || IS_FD_PROG_ARRAY(map)) {
328 err = bpf_fd_array_map_lookup_elem(map, key, value);
329 } else if (IS_FD_HASH(map)) {
330 err = bpf_fd_htab_map_lookup_elem(map, key, value);
331 } else if (map->map_type == BPF_MAP_TYPE_REUSEPORT_SOCKARRAY) {
332 err = bpf_fd_reuseport_array_lookup_elem(map, key, value);
333 } else if (map->map_type == BPF_MAP_TYPE_QUEUE ||
334 map->map_type == BPF_MAP_TYPE_STACK ||
335 map->map_type == BPF_MAP_TYPE_BLOOM_FILTER) {
336 err = map->ops->map_peek_elem(map, value);
337 } else if (map->map_type == BPF_MAP_TYPE_STRUCT_OPS) {
338 /* struct_ops map requires directly updating "value" */
339 err = bpf_struct_ops_map_sys_lookup_elem(map, key, value);
340 } else {
341 rcu_read_lock();
342 if (map->ops->map_lookup_elem_sys_only)
343 ptr = map->ops->map_lookup_elem_sys_only(map, key);
344 else
345 ptr = map->ops->map_lookup_elem(map, key);
346 if (IS_ERR(ptr)) {
347 err = PTR_ERR(ptr);
348 } else if (!ptr) {
349 err = -ENOENT;
350 } else {
351 err = 0;
352 if (flags & BPF_F_LOCK)
353 /* lock 'ptr' and copy everything but lock */
354 copy_map_value_locked(map, value, ptr, true);
355 else
356 copy_map_value(map, value, ptr);
357 /* mask lock and timer, since value wasn't zero inited */
358 check_and_init_map_value(map, value);
359 }
360 rcu_read_unlock();
361 }
362
363 bpf_enable_instrumentation();
364
365 return err;
366}
367
368/* Please, do not use this function outside from the map creation path
369 * (e.g. in map update path) without taking care of setting the active
370 * memory cgroup (see at bpf_map_kmalloc_node() for example).
371 */
372static void *__bpf_map_area_alloc(u64 size, int numa_node, bool mmapable)
373{
374 /* We really just want to fail instead of triggering OOM killer
375 * under memory pressure, therefore we set __GFP_NORETRY to kmalloc,
376 * which is used for lower order allocation requests.
377 *
378 * It has been observed that higher order allocation requests done by
379 * vmalloc with __GFP_NORETRY being set might fail due to not trying
380 * to reclaim memory from the page cache, thus we set
381 * __GFP_RETRY_MAYFAIL to avoid such situations.
382 */
383
384 gfp_t gfp = bpf_memcg_flags(__GFP_NOWARN | __GFP_ZERO);
385 unsigned int flags = 0;
386 unsigned long align = 1;
387 void *area;
388
389 if (size >= SIZE_MAX)
390 return NULL;
391
392 /* kmalloc()'ed memory can't be mmap()'ed */
393 if (mmapable) {
394 BUG_ON(!PAGE_ALIGNED(size));
395 align = SHMLBA;
396 flags = VM_USERMAP;
397 } else if (size <= (PAGE_SIZE << PAGE_ALLOC_COSTLY_ORDER)) {
398 area = kmalloc_node(size, gfp | GFP_USER | __GFP_NORETRY,
399 numa_node);
400 if (area != NULL)
401 return area;
402 }
403
404 return __vmalloc_node_range(size, align, VMALLOC_START, VMALLOC_END,
405 gfp | GFP_KERNEL | __GFP_RETRY_MAYFAIL, PAGE_KERNEL,
406 flags, numa_node, __builtin_return_address(0));
407}
408
409void *bpf_map_area_alloc(u64 size, int numa_node)
410{
411 return __bpf_map_area_alloc(size, numa_node, false);
412}
413
414void *bpf_map_area_mmapable_alloc(u64 size, int numa_node)
415{
416 return __bpf_map_area_alloc(size, numa_node, true);
417}
418
419void bpf_map_area_free(void *area)
420{
421 kvfree(area);
422}
423
424static u32 bpf_map_flags_retain_permanent(u32 flags)
425{
426 /* Some map creation flags are not tied to the map object but
427 * rather to the map fd instead, so they have no meaning upon
428 * map object inspection since multiple file descriptors with
429 * different (access) properties can exist here. Thus, given
430 * this has zero meaning for the map itself, lets clear these
431 * from here.
432 */
433 return flags & ~(BPF_F_RDONLY | BPF_F_WRONLY);
434}
435
436void bpf_map_init_from_attr(struct bpf_map *map, union bpf_attr *attr)
437{
438 map->map_type = attr->map_type;
439 map->key_size = attr->key_size;
440 map->value_size = attr->value_size;
441 map->max_entries = attr->max_entries;
442 map->map_flags = bpf_map_flags_retain_permanent(attr->map_flags);
443 map->numa_node = bpf_map_attr_numa_node(attr);
444 map->map_extra = attr->map_extra;
445}
446
447static int bpf_map_alloc_id(struct bpf_map *map)
448{
449 int id;
450
451 idr_preload(GFP_KERNEL);
452 spin_lock_bh(&map_idr_lock);
453 id = idr_alloc_cyclic(&map_idr, map, 1, INT_MAX, GFP_ATOMIC);
454 if (id > 0)
455 map->id = id;
456 spin_unlock_bh(&map_idr_lock);
457 idr_preload_end();
458
459 if (WARN_ON_ONCE(!id))
460 return -ENOSPC;
461
462 return id > 0 ? 0 : id;
463}
464
465void bpf_map_free_id(struct bpf_map *map)
466{
467 unsigned long flags;
468
469 /* Offloaded maps are removed from the IDR store when their device
470 * disappears - even if someone holds an fd to them they are unusable,
471 * the memory is gone, all ops will fail; they are simply waiting for
472 * refcnt to drop to be freed.
473 */
474 if (!map->id)
475 return;
476
477 spin_lock_irqsave(&map_idr_lock, flags);
478
479 idr_remove(&map_idr, map->id);
480 map->id = 0;
481
482 spin_unlock_irqrestore(&map_idr_lock, flags);
483}
484
485#ifdef CONFIG_MEMCG
486static void bpf_map_save_memcg(struct bpf_map *map)
487{
488 /* Currently if a map is created by a process belonging to the root
489 * memory cgroup, get_obj_cgroup_from_current() will return NULL.
490 * So we have to check map->objcg for being NULL each time it's
491 * being used.
492 */
493 if (memcg_bpf_enabled())
494 map->objcg = get_obj_cgroup_from_current();
495}
496
497static void bpf_map_release_memcg(struct bpf_map *map)
498{
499 if (map->objcg)
500 obj_cgroup_put(map->objcg);
501}
502
503static struct mem_cgroup *bpf_map_get_memcg(const struct bpf_map *map)
504{
505 if (map->objcg)
506 return get_mem_cgroup_from_objcg(map->objcg);
507
508 return root_mem_cgroup;
509}
510
511void bpf_map_memcg_enter(const struct bpf_map *map, struct mem_cgroup **old_memcg,
512 struct mem_cgroup **new_memcg)
513{
514 *new_memcg = bpf_map_get_memcg(map);
515 *old_memcg = set_active_memcg(*new_memcg);
516}
517
518void bpf_map_memcg_exit(struct mem_cgroup *old_memcg,
519 struct mem_cgroup *new_memcg)
520{
521 set_active_memcg(old_memcg);
522 mem_cgroup_put(new_memcg);
523}
524
525void *bpf_map_kmalloc_node(const struct bpf_map *map, size_t size, gfp_t flags,
526 int node)
527{
528 struct mem_cgroup *memcg, *old_memcg;
529 void *ptr;
530
531 bpf_map_memcg_enter(map, &old_memcg, &memcg);
532 ptr = kmalloc_node(size, flags | __GFP_ACCOUNT, node);
533 bpf_map_memcg_exit(old_memcg, memcg);
534
535 return ptr;
536}
537
538void *bpf_map_kmalloc_nolock(const struct bpf_map *map, size_t size, gfp_t flags,
539 int node)
540{
541 struct mem_cgroup *memcg, *old_memcg;
542 void *ptr;
543
544 bpf_map_memcg_enter(map, &old_memcg, &memcg);
545 ptr = kmalloc_nolock(size, flags | __GFP_ACCOUNT, node);
546 bpf_map_memcg_exit(old_memcg, memcg);
547
548 return ptr;
549}
550
551void *bpf_map_kzalloc(const struct bpf_map *map, size_t size, gfp_t flags)
552{
553 struct mem_cgroup *memcg, *old_memcg;
554 void *ptr;
555
556 bpf_map_memcg_enter(map, &old_memcg, &memcg);
557 ptr = kzalloc(size, flags | __GFP_ACCOUNT);
558 bpf_map_memcg_exit(old_memcg, memcg);
559
560 return ptr;
561}
562
563void *bpf_map_kvcalloc(struct bpf_map *map, size_t n, size_t size,
564 gfp_t flags)
565{
566 struct mem_cgroup *memcg, *old_memcg;
567 void *ptr;
568
569 bpf_map_memcg_enter(map, &old_memcg, &memcg);
570 ptr = kvcalloc(n, size, flags | __GFP_ACCOUNT);
571 bpf_map_memcg_exit(old_memcg, memcg);
572
573 return ptr;
574}
575
576void __percpu *bpf_map_alloc_percpu(const struct bpf_map *map, size_t size,
577 size_t align, gfp_t flags)
578{
579 struct mem_cgroup *memcg, *old_memcg;
580 void __percpu *ptr;
581
582 bpf_map_memcg_enter(map, &old_memcg, &memcg);
583 ptr = __alloc_percpu_gfp(size, align, flags | __GFP_ACCOUNT);
584 bpf_map_memcg_exit(old_memcg, memcg);
585
586 return ptr;
587}
588
589#else
590static void bpf_map_save_memcg(struct bpf_map *map)
591{
592}
593
594static void bpf_map_release_memcg(struct bpf_map *map)
595{
596}
597#endif
598
599static bool can_alloc_pages(void)
600{
601 return preempt_count() == 0 && !irqs_disabled() &&
602 !IS_ENABLED(CONFIG_PREEMPT_RT);
603}
604
605static struct page *__bpf_alloc_page(int nid)
606{
607 if (!can_alloc_pages())
608 return alloc_pages_nolock(__GFP_ACCOUNT, nid, 0);
609
610 return alloc_pages_node(nid,
611 GFP_KERNEL | __GFP_ZERO | __GFP_ACCOUNT
612 | __GFP_NOWARN,
613 0);
614}
615
616int bpf_map_alloc_pages(const struct bpf_map *map, int nid,
617 unsigned long nr_pages, struct page **pages)
618{
619 unsigned long i, j;
620 struct page *pg;
621 int ret = 0;
622
623 for (i = 0; i < nr_pages; i++) {
624 pg = __bpf_alloc_page(nid);
625
626 if (pg) {
627 pages[i] = pg;
628 continue;
629 }
630 for (j = 0; j < i; j++)
631 free_pages_nolock(pages[j], 0);
632 ret = -ENOMEM;
633 break;
634 }
635
636 return ret;
637}
638
639static int btf_field_cmp(const void *a, const void *b)
640{
641 const struct btf_field *f1 = a, *f2 = b;
642
643 if (f1->offset < f2->offset)
644 return -1;
645 else if (f1->offset > f2->offset)
646 return 1;
647 return 0;
648}
649
650struct btf_field *btf_record_find(const struct btf_record *rec, u32 offset,
651 u32 field_mask)
652{
653 struct btf_field *field;
654
655 if (IS_ERR_OR_NULL(rec) || !(rec->field_mask & field_mask))
656 return NULL;
657 field = bsearch(&offset, rec->fields, rec->cnt, sizeof(rec->fields[0]), btf_field_cmp);
658 if (!field || !(field->type & field_mask))
659 return NULL;
660 return field;
661}
662
663void btf_record_free(struct btf_record *rec)
664{
665 int i;
666
667 if (IS_ERR_OR_NULL(rec))
668 return;
669 for (i = 0; i < rec->cnt; i++) {
670 switch (rec->fields[i].type) {
671 case BPF_KPTR_UNREF:
672 case BPF_KPTR_REF:
673 case BPF_KPTR_PERCPU:
674 case BPF_UPTR:
675 if (rec->fields[i].kptr.module)
676 module_put(rec->fields[i].kptr.module);
677 if (btf_is_kernel(rec->fields[i].kptr.btf))
678 btf_put(rec->fields[i].kptr.btf);
679 break;
680 case BPF_LIST_HEAD:
681 case BPF_LIST_NODE:
682 case BPF_RB_ROOT:
683 case BPF_RB_NODE:
684 case BPF_SPIN_LOCK:
685 case BPF_RES_SPIN_LOCK:
686 case BPF_TIMER:
687 case BPF_REFCOUNT:
688 case BPF_WORKQUEUE:
689 case BPF_TASK_WORK:
690 /* Nothing to release */
691 break;
692 default:
693 WARN_ON_ONCE(1);
694 continue;
695 }
696 }
697 kfree(rec);
698}
699
700void bpf_map_free_record(struct bpf_map *map)
701{
702 btf_record_free(map->record);
703 map->record = NULL;
704}
705
706struct btf_record *btf_record_dup(const struct btf_record *rec)
707{
708 const struct btf_field *fields;
709 struct btf_record *new_rec;
710 int ret, size, i;
711
712 if (IS_ERR_OR_NULL(rec))
713 return NULL;
714 size = struct_size(rec, fields, rec->cnt);
715 new_rec = kmemdup(rec, size, GFP_KERNEL | __GFP_NOWARN);
716 if (!new_rec)
717 return ERR_PTR(-ENOMEM);
718 /* Do a deep copy of the btf_record */
719 fields = rec->fields;
720 new_rec->cnt = 0;
721 for (i = 0; i < rec->cnt; i++) {
722 switch (fields[i].type) {
723 case BPF_KPTR_UNREF:
724 case BPF_KPTR_REF:
725 case BPF_KPTR_PERCPU:
726 case BPF_UPTR:
727 if (btf_is_kernel(fields[i].kptr.btf))
728 btf_get(fields[i].kptr.btf);
729 if (fields[i].kptr.module && !try_module_get(fields[i].kptr.module)) {
730 ret = -ENXIO;
731 goto free;
732 }
733 break;
734 case BPF_LIST_HEAD:
735 case BPF_LIST_NODE:
736 case BPF_RB_ROOT:
737 case BPF_RB_NODE:
738 case BPF_SPIN_LOCK:
739 case BPF_RES_SPIN_LOCK:
740 case BPF_TIMER:
741 case BPF_REFCOUNT:
742 case BPF_WORKQUEUE:
743 case BPF_TASK_WORK:
744 /* Nothing to acquire */
745 break;
746 default:
747 ret = -EFAULT;
748 WARN_ON_ONCE(1);
749 goto free;
750 }
751 new_rec->cnt++;
752 }
753 return new_rec;
754free:
755 btf_record_free(new_rec);
756 return ERR_PTR(ret);
757}
758
759bool btf_record_equal(const struct btf_record *rec_a, const struct btf_record *rec_b)
760{
761 bool a_has_fields = !IS_ERR_OR_NULL(rec_a), b_has_fields = !IS_ERR_OR_NULL(rec_b);
762 int size;
763
764 if (!a_has_fields && !b_has_fields)
765 return true;
766 if (a_has_fields != b_has_fields)
767 return false;
768 if (rec_a->cnt != rec_b->cnt)
769 return false;
770 size = struct_size(rec_a, fields, rec_a->cnt);
771 /* btf_parse_fields uses kzalloc to allocate a btf_record, so unused
772 * members are zeroed out. So memcmp is safe to do without worrying
773 * about padding/unused fields.
774 *
775 * While spin_lock, timer, and kptr have no relation to map BTF,
776 * list_head metadata is specific to map BTF, the btf and value_rec
777 * members in particular. btf is the map BTF, while value_rec points to
778 * btf_record in that map BTF.
779 *
780 * So while by default, we don't rely on the map BTF (which the records
781 * were parsed from) matching for both records, which is not backwards
782 * compatible, in case list_head is part of it, we implicitly rely on
783 * that by way of depending on memcmp succeeding for it.
784 */
785 return !memcmp(rec_a, rec_b, size);
786}
787
788void bpf_obj_free_timer(const struct btf_record *rec, void *obj)
789{
790 if (WARN_ON_ONCE(!btf_record_has_field(rec, BPF_TIMER)))
791 return;
792 bpf_timer_cancel_and_free(obj + rec->timer_off);
793}
794
795void bpf_obj_free_workqueue(const struct btf_record *rec, void *obj)
796{
797 if (WARN_ON_ONCE(!btf_record_has_field(rec, BPF_WORKQUEUE)))
798 return;
799 bpf_wq_cancel_and_free(obj + rec->wq_off);
800}
801
802void bpf_obj_free_task_work(const struct btf_record *rec, void *obj)
803{
804 if (WARN_ON_ONCE(!btf_record_has_field(rec, BPF_TASK_WORK)))
805 return;
806 bpf_task_work_cancel_and_free(obj + rec->task_work_off);
807}
808
809void bpf_obj_cancel_fields(struct bpf_map *map, void *obj)
810{
811 bpf_map_free_internal_structs(map, obj);
812}
813
814void bpf_obj_free_fields(const struct btf_record *rec, void *obj)
815{
816 const struct btf_field *fields;
817 int i;
818
819 if (IS_ERR_OR_NULL(rec))
820 return;
821 fields = rec->fields;
822 for (i = 0; i < rec->cnt; i++) {
823 struct btf_struct_meta *pointee_struct_meta;
824 const struct btf_field *field = &fields[i];
825 void *field_ptr = obj + field->offset;
826 void *xchgd_field;
827
828 switch (fields[i].type) {
829 case BPF_SPIN_LOCK:
830 case BPF_RES_SPIN_LOCK:
831 break;
832 case BPF_TIMER:
833 bpf_timer_cancel_and_free(field_ptr);
834 break;
835 case BPF_WORKQUEUE:
836 bpf_wq_cancel_and_free(field_ptr);
837 break;
838 case BPF_TASK_WORK:
839 bpf_task_work_cancel_and_free(field_ptr);
840 break;
841 case BPF_KPTR_UNREF:
842 WRITE_ONCE(*(u64 *)field_ptr, 0);
843 break;
844 case BPF_KPTR_REF:
845 case BPF_KPTR_PERCPU:
846 xchgd_field = (void *)xchg((unsigned long *)field_ptr, 0);
847 if (!xchgd_field)
848 break;
849
850 if (!btf_is_kernel(field->kptr.btf)) {
851 pointee_struct_meta = btf_find_struct_meta(field->kptr.btf,
852 field->kptr.btf_id);
853 __bpf_obj_drop_impl(xchgd_field, pointee_struct_meta ?
854 pointee_struct_meta->record : NULL,
855 fields[i].type == BPF_KPTR_PERCPU);
856 } else {
857 field->kptr.dtor(xchgd_field);
858 }
859 break;
860 case BPF_UPTR:
861 /* The caller ensured that no one is using the uptr */
862 unpin_uptr_kaddr(*(void **)field_ptr);
863 break;
864 case BPF_LIST_HEAD:
865 if (WARN_ON_ONCE(rec->spin_lock_off < 0))
866 continue;
867 bpf_list_head_free(field, field_ptr, obj + rec->spin_lock_off);
868 break;
869 case BPF_RB_ROOT:
870 if (WARN_ON_ONCE(rec->spin_lock_off < 0))
871 continue;
872 bpf_rb_root_free(field, field_ptr, obj + rec->spin_lock_off);
873 break;
874 case BPF_LIST_NODE:
875 case BPF_RB_NODE:
876 case BPF_REFCOUNT:
877 break;
878 default:
879 WARN_ON_ONCE(1);
880 continue;
881 }
882 }
883}
884
885static void bpf_map_free(struct bpf_map *map)
886{
887 struct btf_record *rec = map->record;
888 struct btf *btf = map->btf;
889
890 /* implementation dependent freeing. Disabling migration to simplify
891 * the free of values or special fields allocated from bpf memory
892 * allocator.
893 */
894 kfree(map->excl_prog_sha);
895 migrate_disable();
896 map->ops->map_free(map);
897 migrate_enable();
898
899 /* Delay freeing of btf_record for maps, as map_free
900 * callback usually needs access to them. It is better to do it here
901 * than require each callback to do the free itself manually.
902 *
903 * Note that the btf_record stashed in map->inner_map_meta->record was
904 * already freed using the map_free callback for map in map case which
905 * eventually calls bpf_map_free_meta, since inner_map_meta is only a
906 * template bpf_map struct used during verification.
907 */
908 btf_record_free(rec);
909 /* Delay freeing of btf for maps, as map_free callback may need
910 * struct_meta info which will be freed with btf_put().
911 */
912 btf_put(btf);
913}
914
915/* called from workqueue */
916static void bpf_map_free_deferred(struct work_struct *work)
917{
918 struct bpf_map *map = container_of(work, struct bpf_map, work);
919
920 security_bpf_map_free(map);
921 bpf_map_release_memcg(map);
922 bpf_map_owner_free(map);
923 bpf_map_free(map);
924}
925
926static void bpf_map_put_uref(struct bpf_map *map)
927{
928 if (atomic64_dec_and_test(&map->usercnt)) {
929 if (map->ops->map_release_uref)
930 map->ops->map_release_uref(map);
931 }
932}
933
934static void bpf_map_free_in_work(struct bpf_map *map)
935{
936 INIT_WORK(&map->work, bpf_map_free_deferred);
937 /* Avoid spawning kworkers, since they all might contend
938 * for the same mutex like slab_mutex.
939 */
940 queue_work(system_dfl_wq, &map->work);
941}
942
943static void bpf_map_free_rcu_gp(struct rcu_head *rcu)
944{
945 bpf_map_free_in_work(container_of(rcu, struct bpf_map, rcu));
946}
947
948/* decrement map refcnt and schedule it for freeing via workqueue
949 * (underlying map implementation ops->map_free() might sleep)
950 */
951void bpf_map_put(struct bpf_map *map)
952{
953 if (atomic64_dec_and_test(&map->refcnt)) {
954 /* bpf_map_free_id() must be called first */
955 bpf_map_free_id(map);
956
957 WARN_ON_ONCE(atomic64_read(&map->sleepable_refcnt));
958 /* RCU tasks trace grace period implies RCU grace period. */
959 if (READ_ONCE(map->free_after_mult_rcu_gp))
960 call_rcu_tasks_trace(&map->rcu, bpf_map_free_rcu_gp);
961 else if (READ_ONCE(map->free_after_rcu_gp))
962 call_rcu(&map->rcu, bpf_map_free_rcu_gp);
963 else
964 bpf_map_free_in_work(map);
965 }
966}
967EXPORT_SYMBOL_GPL(bpf_map_put);
968
969void bpf_map_put_with_uref(struct bpf_map *map)
970{
971 bpf_map_put_uref(map);
972 bpf_map_put(map);
973}
974
975static int bpf_map_release(struct inode *inode, struct file *filp)
976{
977 struct bpf_map *map = filp->private_data;
978
979 if (map->ops->map_release)
980 map->ops->map_release(map, filp);
981
982 bpf_map_put_with_uref(map);
983 return 0;
984}
985
986static fmode_t map_get_sys_perms(struct bpf_map *map, struct fd f)
987{
988 fmode_t mode = fd_file(f)->f_mode;
989
990 /* Our file permissions may have been overridden by global
991 * map permissions facing syscall side.
992 */
993 if (READ_ONCE(map->frozen))
994 mode &= ~FMODE_CAN_WRITE;
995 return mode;
996}
997
998#ifdef CONFIG_PROC_FS
999/* Show the memory usage of a bpf map */
1000static u64 bpf_map_memory_usage(const struct bpf_map *map)
1001{
1002 return map->ops->map_mem_usage(map);
1003}
1004
1005static void bpf_map_show_fdinfo(struct seq_file *m, struct file *filp)
1006{
1007 struct bpf_map *map = filp->private_data;
1008 u32 type = 0, jited = 0;
1009
1010 spin_lock(&map->owner_lock);
1011 if (map->owner) {
1012 type = map->owner->type;
1013 jited = map->owner->jited;
1014 }
1015 spin_unlock(&map->owner_lock);
1016
1017 seq_printf(m,
1018 "map_type:\t%u\n"
1019 "key_size:\t%u\n"
1020 "value_size:\t%u\n"
1021 "max_entries:\t%u\n"
1022 "map_flags:\t%#x\n"
1023 "map_extra:\t%#llx\n"
1024 "memlock:\t%llu\n"
1025 "map_id:\t%u\n"
1026 "frozen:\t%u\n",
1027 map->map_type,
1028 map->key_size,
1029 map->value_size,
1030 map->max_entries,
1031 map->map_flags,
1032 (unsigned long long)map->map_extra,
1033 bpf_map_memory_usage(map),
1034 map->id,
1035 READ_ONCE(map->frozen));
1036 if (type) {
1037 seq_printf(m, "owner_prog_type:\t%u\n", type);
1038 seq_printf(m, "owner_jited:\t%u\n", jited);
1039 }
1040}
1041#endif
1042
1043static ssize_t bpf_dummy_read(struct file *filp, char __user *buf, size_t siz,
1044 loff_t *ppos)
1045{
1046 /* We need this handler such that alloc_file() enables
1047 * f_mode with FMODE_CAN_READ.
1048 */
1049 return -EINVAL;
1050}
1051
1052static ssize_t bpf_dummy_write(struct file *filp, const char __user *buf,
1053 size_t siz, loff_t *ppos)
1054{
1055 /* We need this handler such that alloc_file() enables
1056 * f_mode with FMODE_CAN_WRITE.
1057 */
1058 return -EINVAL;
1059}
1060
1061/* called for any extra memory-mapped regions (except initial) */
1062static void bpf_map_mmap_open(struct vm_area_struct *vma)
1063{
1064 struct bpf_map *map = vma->vm_file->private_data;
1065
1066 if (vma->vm_flags & VM_MAYWRITE)
1067 bpf_map_write_active_inc(map);
1068}
1069
1070/* called for all unmapped memory region (including initial) */
1071static void bpf_map_mmap_close(struct vm_area_struct *vma)
1072{
1073 struct bpf_map *map = vma->vm_file->private_data;
1074
1075 if (vma->vm_flags & VM_MAYWRITE)
1076 bpf_map_write_active_dec(map);
1077}
1078
1079static vm_fault_t bpf_map_mmap_fault(struct vm_fault *vmf)
1080{
1081 struct bpf_map *map = vmf->vma->vm_private_data;
1082
1083 return map->ops->map_mmap_fault(map, vmf);
1084}
1085
1086static const struct vm_operations_struct bpf_map_default_vmops = {
1087 .open = bpf_map_mmap_open,
1088 .close = bpf_map_mmap_close,
1089};
1090
1091static const struct vm_operations_struct bpf_map_lazy_vmops = {
1092 .open = bpf_map_mmap_open,
1093 .close = bpf_map_mmap_close,
1094 .fault = bpf_map_mmap_fault,
1095};
1096
1097static int bpf_map_mmap(struct file *filp, struct vm_area_struct *vma)
1098{
1099 struct bpf_map *map = filp->private_data;
1100 int err = 0;
1101
1102 if (!map->ops->map_mmap || !IS_ERR_OR_NULL(map->record))
1103 return -ENOTSUPP;
1104
1105 if (!(vma->vm_flags & VM_SHARED))
1106 return -EINVAL;
1107
1108 mutex_lock(&map->freeze_mutex);
1109
1110 if (vma->vm_flags & VM_WRITE) {
1111 if (map->frozen) {
1112 err = -EPERM;
1113 goto out;
1114 }
1115 /* map is meant to be read-only, so do not allow mapping as
1116 * writable, because it's possible to leak a writable page
1117 * reference and allows user-space to still modify it after
1118 * freezing, while verifier will assume contents do not change
1119 */
1120 if (map->map_flags & BPF_F_RDONLY_PROG) {
1121 err = -EACCES;
1122 goto out;
1123 }
1124 bpf_map_write_active_inc(map);
1125 }
1126out:
1127 mutex_unlock(&map->freeze_mutex);
1128 if (err)
1129 return err;
1130
1131 /* set default open/close callbacks */
1132 vma->vm_ops = map->ops->map_mmap_fault ? &bpf_map_lazy_vmops : &bpf_map_default_vmops;
1133 vma->vm_private_data = map;
1134 vm_flags_clear(vma, VM_MAYEXEC);
1135 /* If mapping is read-only, then disallow potentially re-mapping with
1136 * PROT_WRITE by dropping VM_MAYWRITE flag. This VM_MAYWRITE clearing
1137 * means that as far as BPF map's memory-mapped VMAs are concerned,
1138 * VM_WRITE and VM_MAYWRITE and equivalent, if one of them is set,
1139 * both should be set, so we can forget about VM_MAYWRITE and always
1140 * check just VM_WRITE
1141 */
1142 if (!(vma->vm_flags & VM_WRITE))
1143 vm_flags_clear(vma, VM_MAYWRITE);
1144
1145 err = map->ops->map_mmap(map, vma);
1146 if (err) {
1147 if (vma->vm_flags & VM_WRITE)
1148 bpf_map_write_active_dec(map);
1149 }
1150
1151 return err;
1152}
1153
1154static __poll_t bpf_map_poll(struct file *filp, struct poll_table_struct *pts)
1155{
1156 struct bpf_map *map = filp->private_data;
1157
1158 if (map->ops->map_poll)
1159 return map->ops->map_poll(map, filp, pts);
1160
1161 return EPOLLERR;
1162}
1163
1164static unsigned long bpf_get_unmapped_area(struct file *filp, unsigned long addr,
1165 unsigned long len, unsigned long pgoff,
1166 unsigned long flags)
1167{
1168 struct bpf_map *map = filp->private_data;
1169
1170 if (map->ops->map_get_unmapped_area)
1171 return map->ops->map_get_unmapped_area(filp, addr, len, pgoff, flags);
1172#ifdef CONFIG_MMU
1173 return mm_get_unmapped_area(filp, addr, len, pgoff, flags);
1174#else
1175 return addr;
1176#endif
1177}
1178
1179const struct file_operations bpf_map_fops = {
1180#ifdef CONFIG_PROC_FS
1181 .show_fdinfo = bpf_map_show_fdinfo,
1182#endif
1183 .release = bpf_map_release,
1184 .read = bpf_dummy_read,
1185 .write = bpf_dummy_write,
1186 .mmap = bpf_map_mmap,
1187 .poll = bpf_map_poll,
1188 .get_unmapped_area = bpf_get_unmapped_area,
1189};
1190
1191int bpf_map_new_fd(struct bpf_map *map, int flags)
1192{
1193 int ret;
1194
1195 ret = security_bpf_map(map, OPEN_FMODE(flags));
1196 if (ret < 0)
1197 return ret;
1198
1199 return anon_inode_getfd("bpf-map", &bpf_map_fops, map,
1200 flags | O_CLOEXEC);
1201}
1202
1203int bpf_get_file_flag(int flags)
1204{
1205 if ((flags & BPF_F_RDONLY) && (flags & BPF_F_WRONLY))
1206 return -EINVAL;
1207 if (flags & BPF_F_RDONLY)
1208 return O_RDONLY;
1209 if (flags & BPF_F_WRONLY)
1210 return O_WRONLY;
1211 return O_RDWR;
1212}
1213
1214/* helper macro to check that unused fields 'union bpf_attr' are zero */
1215#define CHECK_ATTR(CMD) \
1216 memchr_inv((void *) &attr->CMD##_LAST_FIELD + \
1217 sizeof(attr->CMD##_LAST_FIELD), 0, \
1218 sizeof(*attr) - \
1219 offsetof(union bpf_attr, CMD##_LAST_FIELD) - \
1220 sizeof(attr->CMD##_LAST_FIELD)) != NULL
1221
1222/* dst and src must have at least "size" number of bytes.
1223 * Return strlen on success and < 0 on error.
1224 */
1225int bpf_obj_name_cpy(char *dst, const char *src, unsigned int size)
1226{
1227 const char *end = src + size;
1228 const char *orig_src = src;
1229
1230 memset(dst, 0, size);
1231 /* Copy all isalnum(), '_' and '.' chars. */
1232 while (src < end && *src) {
1233 if (!isalnum(*src) &&
1234 *src != '_' && *src != '.')
1235 return -EINVAL;
1236 *dst++ = *src++;
1237 }
1238
1239 /* No '\0' found in "size" number of bytes */
1240 if (src == end)
1241 return -EINVAL;
1242
1243 return src - orig_src;
1244}
1245EXPORT_SYMBOL_GPL(bpf_obj_name_cpy);
1246
1247int map_check_no_btf(struct bpf_map *map,
1248 const struct btf *btf,
1249 const struct btf_type *key_type,
1250 const struct btf_type *value_type)
1251{
1252 return -ENOTSUPP;
1253}
1254
1255static int map_check_btf(struct bpf_map *map, struct bpf_token *token,
1256 const struct btf *btf, u32 btf_key_id, u32 btf_value_id)
1257{
1258 const struct btf_type *key_type, *value_type;
1259 u32 key_size, value_size;
1260 int ret = 0;
1261
1262 /* Some maps allow key to be unspecified. */
1263 if (btf_key_id) {
1264 key_type = btf_type_id_size(btf, &btf_key_id, &key_size);
1265 if (!key_type || key_size != map->key_size)
1266 return -EINVAL;
1267 } else {
1268 key_type = btf_type_by_id(btf, 0);
1269 if (!map->ops->map_check_btf)
1270 return -EINVAL;
1271 }
1272
1273 value_type = btf_type_id_size(btf, &btf_value_id, &value_size);
1274 if (!value_type || value_size != map->value_size)
1275 return -EINVAL;
1276
1277 map->record = btf_parse_fields(btf, value_type,
1278 BPF_SPIN_LOCK | BPF_RES_SPIN_LOCK | BPF_TIMER | BPF_KPTR | BPF_LIST_HEAD |
1279 BPF_RB_ROOT | BPF_REFCOUNT | BPF_WORKQUEUE | BPF_UPTR |
1280 BPF_TASK_WORK,
1281 map->value_size);
1282 if (!IS_ERR_OR_NULL(map->record)) {
1283 int i;
1284
1285 if (!bpf_token_capable(token, CAP_BPF)) {
1286 ret = -EPERM;
1287 goto free_map_tab;
1288 }
1289 if (map->map_flags & (BPF_F_RDONLY_PROG | BPF_F_WRONLY_PROG)) {
1290 ret = -EACCES;
1291 goto free_map_tab;
1292 }
1293 for (i = 0; i < sizeof(map->record->field_mask) * 8; i++) {
1294 switch (map->record->field_mask & (1 << i)) {
1295 case 0:
1296 continue;
1297 case BPF_SPIN_LOCK:
1298 case BPF_RES_SPIN_LOCK:
1299 if (map->map_type != BPF_MAP_TYPE_HASH &&
1300 map->map_type != BPF_MAP_TYPE_RHASH &&
1301 map->map_type != BPF_MAP_TYPE_ARRAY &&
1302 map->map_type != BPF_MAP_TYPE_CGROUP_STORAGE &&
1303 map->map_type != BPF_MAP_TYPE_SK_STORAGE &&
1304 map->map_type != BPF_MAP_TYPE_INODE_STORAGE &&
1305 map->map_type != BPF_MAP_TYPE_TASK_STORAGE &&
1306 map->map_type != BPF_MAP_TYPE_CGRP_STORAGE) {
1307 ret = -EOPNOTSUPP;
1308 goto free_map_tab;
1309 }
1310 break;
1311 case BPF_TIMER:
1312 case BPF_WORKQUEUE:
1313 case BPF_TASK_WORK:
1314 if (map->map_type != BPF_MAP_TYPE_HASH &&
1315 map->map_type != BPF_MAP_TYPE_RHASH &&
1316 map->map_type != BPF_MAP_TYPE_LRU_HASH &&
1317 map->map_type != BPF_MAP_TYPE_ARRAY) {
1318 ret = -EOPNOTSUPP;
1319 goto free_map_tab;
1320 }
1321 break;
1322 case BPF_KPTR_UNREF:
1323 case BPF_KPTR_REF:
1324 case BPF_KPTR_PERCPU:
1325 case BPF_REFCOUNT:
1326 if (map->map_type != BPF_MAP_TYPE_HASH &&
1327 map->map_type != BPF_MAP_TYPE_RHASH &&
1328 map->map_type != BPF_MAP_TYPE_PERCPU_HASH &&
1329 map->map_type != BPF_MAP_TYPE_LRU_HASH &&
1330 map->map_type != BPF_MAP_TYPE_LRU_PERCPU_HASH &&
1331 map->map_type != BPF_MAP_TYPE_ARRAY &&
1332 map->map_type != BPF_MAP_TYPE_PERCPU_ARRAY &&
1333 map->map_type != BPF_MAP_TYPE_SK_STORAGE &&
1334 map->map_type != BPF_MAP_TYPE_INODE_STORAGE &&
1335 map->map_type != BPF_MAP_TYPE_TASK_STORAGE &&
1336 map->map_type != BPF_MAP_TYPE_CGRP_STORAGE) {
1337 ret = -EOPNOTSUPP;
1338 goto free_map_tab;
1339 }
1340 break;
1341 case BPF_UPTR:
1342 if (map->map_type != BPF_MAP_TYPE_TASK_STORAGE) {
1343 ret = -EOPNOTSUPP;
1344 goto free_map_tab;
1345 }
1346 break;
1347 case BPF_LIST_HEAD:
1348 case BPF_RB_ROOT:
1349 if (map->map_type != BPF_MAP_TYPE_HASH &&
1350 map->map_type != BPF_MAP_TYPE_LRU_HASH &&
1351 map->map_type != BPF_MAP_TYPE_ARRAY) {
1352 ret = -EOPNOTSUPP;
1353 goto free_map_tab;
1354 }
1355 break;
1356 default:
1357 /* Fail if map_type checks are missing for a field type */
1358 ret = -EOPNOTSUPP;
1359 goto free_map_tab;
1360 }
1361 }
1362 }
1363
1364 ret = btf_check_and_fixup_fields(btf, map->record);
1365 if (ret < 0)
1366 goto free_map_tab;
1367
1368 if (map->ops->map_check_btf) {
1369 ret = map->ops->map_check_btf(map, btf, key_type, value_type);
1370 if (ret < 0)
1371 goto free_map_tab;
1372 }
1373
1374 return ret;
1375free_map_tab:
1376 bpf_map_free_record(map);
1377 return ret;
1378}
1379
1380#define BPF_MAP_CREATE_LAST_FIELD excl_prog_hash_size
1381/* called via syscall */
1382static int map_create_alloc(union bpf_attr *attr, bpfptr_t uattr, struct bpf_verifier_log *log,
1383 struct bpf_map **mapp, struct bpf_token **tokenp)
1384{
1385 const struct bpf_map_ops *ops;
1386 struct bpf_token *token = NULL;
1387 int numa_node = bpf_map_attr_numa_node(attr);
1388 u32 map_type = attr->map_type;
1389 struct bpf_map *map;
1390 bool token_flag;
1391 int err;
1392
1393 err = CHECK_ATTR(BPF_MAP_CREATE);
1394 if (err) {
1395 bpf_log(log, "Invalid attr.\n");
1396 return -EINVAL;
1397 }
1398
1399 /* check BPF_F_TOKEN_FD flag, remember if it's set, and then clear it
1400 * to avoid per-map type checks tripping on unknown flag
1401 */
1402 token_flag = attr->map_flags & BPF_F_TOKEN_FD;
1403 attr->map_flags &= ~BPF_F_TOKEN_FD;
1404
1405 if (attr->btf_vmlinux_value_type_id) {
1406 if (attr->map_type != BPF_MAP_TYPE_STRUCT_OPS) {
1407 bpf_log(log, "btf_vmlinux_value_type_id can only be used with struct_ops maps.\n");
1408 return -EINVAL;
1409 }
1410 if (attr->btf_key_type_id || attr->btf_value_type_id) {
1411 bpf_log(log, "btf_vmlinux_value_type_id is mutually exclusive with btf_key_type_id and btf_value_type_id.\n");
1412 return -EINVAL;
1413 }
1414 } else if (attr->btf_key_type_id && !attr->btf_value_type_id) {
1415 bpf_log(log, "Invalid btf_value_type_id.\n");
1416 return -EINVAL;
1417 }
1418
1419 if (attr->map_type != BPF_MAP_TYPE_BLOOM_FILTER &&
1420 attr->map_type != BPF_MAP_TYPE_ARENA &&
1421 attr->map_type != BPF_MAP_TYPE_RHASH &&
1422 attr->map_extra != 0) {
1423 bpf_log(log, "Invalid map_extra.\n");
1424 return -EINVAL;
1425 }
1426
1427 if (numa_node != NUMA_NO_NODE &&
1428 ((unsigned int)numa_node >= nr_node_ids ||
1429 !node_online(numa_node))) {
1430 bpf_log(log, "Invalid numa_node.\n");
1431 return -EINVAL;
1432 }
1433
1434 /* find map type and init map: hashtable vs rbtree vs bloom vs ... */
1435 map_type = attr->map_type;
1436 if (map_type >= ARRAY_SIZE(bpf_map_types)) {
1437 bpf_log(log, "Invalid map_type.\n");
1438 return -EINVAL;
1439 }
1440 map_type = array_index_nospec(map_type, ARRAY_SIZE(bpf_map_types));
1441 ops = bpf_map_types[map_type];
1442 if (!ops)
1443 return -EINVAL;
1444
1445 if (ops->map_alloc_check) {
1446 err = ops->map_alloc_check(attr);
1447 if (err)
1448 return err;
1449 }
1450 if (attr->map_ifindex)
1451 ops = &bpf_map_offload_ops;
1452 if (!ops->map_mem_usage)
1453 return -EINVAL;
1454
1455 if (token_flag) {
1456 token = bpf_token_get_from_fd(attr->map_token_fd);
1457 if (IS_ERR(token)) {
1458 bpf_log(log, "Invalid map_token_fd.\n");
1459 return PTR_ERR(token);
1460 }
1461
1462 /* if current token doesn't grant map creation permissions,
1463 * then we can't use this token, so ignore it and rely on
1464 * system-wide capabilities checks
1465 */
1466 if (!bpf_token_allow_cmd(token, BPF_MAP_CREATE) ||
1467 !bpf_token_allow_map_type(token, attr->map_type)) {
1468 bpf_token_put(token);
1469 token = NULL;
1470 }
1471 }
1472
1473 err = -EPERM;
1474
1475 /* Intent here is for unprivileged_bpf_disabled to block BPF map
1476 * creation for unprivileged users; other actions depend
1477 * on fd availability and access to bpffs, so are dependent on
1478 * object creation success. Even with unprivileged BPF disabled,
1479 * capability checks are still carried out.
1480 */
1481 if (sysctl_unprivileged_bpf_disabled && !bpf_token_capable(token, CAP_BPF))
1482 goto put_token;
1483
1484 /* check privileged map type permissions */
1485 switch (map_type) {
1486 case BPF_MAP_TYPE_ARRAY:
1487 case BPF_MAP_TYPE_PERCPU_ARRAY:
1488 case BPF_MAP_TYPE_PROG_ARRAY:
1489 case BPF_MAP_TYPE_PERF_EVENT_ARRAY:
1490 case BPF_MAP_TYPE_CGROUP_ARRAY:
1491 case BPF_MAP_TYPE_ARRAY_OF_MAPS:
1492 case BPF_MAP_TYPE_HASH:
1493 case BPF_MAP_TYPE_RHASH:
1494 case BPF_MAP_TYPE_PERCPU_HASH:
1495 case BPF_MAP_TYPE_HASH_OF_MAPS:
1496 case BPF_MAP_TYPE_RINGBUF:
1497 case BPF_MAP_TYPE_USER_RINGBUF:
1498 case BPF_MAP_TYPE_CGROUP_STORAGE:
1499 case BPF_MAP_TYPE_PERCPU_CGROUP_STORAGE:
1500 /* unprivileged */
1501 break;
1502 case BPF_MAP_TYPE_SK_STORAGE:
1503 case BPF_MAP_TYPE_INODE_STORAGE:
1504 case BPF_MAP_TYPE_TASK_STORAGE:
1505 case BPF_MAP_TYPE_CGRP_STORAGE:
1506 case BPF_MAP_TYPE_BLOOM_FILTER:
1507 case BPF_MAP_TYPE_LPM_TRIE:
1508 case BPF_MAP_TYPE_REUSEPORT_SOCKARRAY:
1509 case BPF_MAP_TYPE_STACK_TRACE:
1510 case BPF_MAP_TYPE_QUEUE:
1511 case BPF_MAP_TYPE_STACK:
1512 case BPF_MAP_TYPE_LRU_HASH:
1513 case BPF_MAP_TYPE_LRU_PERCPU_HASH:
1514 case BPF_MAP_TYPE_STRUCT_OPS:
1515 case BPF_MAP_TYPE_CPUMAP:
1516 case BPF_MAP_TYPE_ARENA:
1517 case BPF_MAP_TYPE_INSN_ARRAY:
1518 if (!bpf_token_capable(token, CAP_BPF))
1519 goto put_token;
1520 break;
1521 case BPF_MAP_TYPE_SOCKMAP:
1522 case BPF_MAP_TYPE_SOCKHASH:
1523 case BPF_MAP_TYPE_DEVMAP:
1524 case BPF_MAP_TYPE_DEVMAP_HASH:
1525 case BPF_MAP_TYPE_XSKMAP:
1526 if (!bpf_token_capable(token, CAP_NET_ADMIN))
1527 goto put_token;
1528 break;
1529 default:
1530 WARN(1, "unsupported map type %d", map_type);
1531 goto put_token;
1532 }
1533
1534 map = ops->map_alloc(attr);
1535 if (IS_ERR(map)) {
1536 err = PTR_ERR(map);
1537 goto put_token;
1538 }
1539 map->ops = ops;
1540 map->map_type = map_type;
1541
1542 err = bpf_obj_name_cpy(map->name, attr->map_name,
1543 sizeof(attr->map_name));
1544 if (err < 0) {
1545 bpf_log(log, "Invalid map_name.\n");
1546 goto free_map;
1547 }
1548
1549 preempt_disable();
1550 map->cookie = gen_cookie_next(&bpf_map_cookie);
1551 preempt_enable();
1552
1553 atomic64_set(&map->refcnt, 1);
1554 atomic64_set(&map->usercnt, 1);
1555 mutex_init(&map->freeze_mutex);
1556 spin_lock_init(&map->owner_lock);
1557
1558 if (attr->btf_key_type_id || attr->btf_value_type_id ||
1559 /* Even the map's value is a kernel's struct,
1560 * the bpf_prog.o must have BTF to begin with
1561 * to figure out the corresponding kernel's
1562 * counter part. Thus, attr->btf_fd has
1563 * to be valid also.
1564 */
1565 attr->btf_vmlinux_value_type_id) {
1566 struct btf *btf;
1567
1568 btf = btf_get_by_fd(attr->btf_fd);
1569 if (IS_ERR(btf)) {
1570 bpf_log(log, "Invalid btf_fd.\n");
1571 err = PTR_ERR(btf);
1572 goto free_map;
1573 }
1574 if (btf_is_kernel(btf)) {
1575 btf_put(btf);
1576 err = -EACCES;
1577 goto free_map;
1578 }
1579 map->btf = btf;
1580
1581 if (attr->btf_value_type_id) {
1582 err = map_check_btf(map, token, btf, attr->btf_key_type_id,
1583 attr->btf_value_type_id);
1584 if (err)
1585 goto free_map;
1586 }
1587
1588 map->btf_key_type_id = attr->btf_key_type_id;
1589 map->btf_value_type_id = attr->btf_value_type_id;
1590 map->btf_vmlinux_value_type_id =
1591 attr->btf_vmlinux_value_type_id;
1592 }
1593
1594 if (attr->excl_prog_hash) {
1595 bpfptr_t uprog_hash = make_bpfptr(attr->excl_prog_hash, uattr.is_kernel);
1596
1597 if (attr->excl_prog_hash_size != SHA256_DIGEST_SIZE) {
1598 bpf_log(log, "Invalid excl_prog_hash_size.\n");
1599 err = -EINVAL;
1600 goto free_map;
1601 }
1602
1603 map->excl_prog_sha = kzalloc(SHA256_DIGEST_SIZE, GFP_KERNEL);
1604 if (!map->excl_prog_sha) {
1605 err = -ENOMEM;
1606 goto free_map;
1607 }
1608
1609 if (copy_from_bpfptr(map->excl_prog_sha, uprog_hash, SHA256_DIGEST_SIZE)) {
1610 err = -EFAULT;
1611 goto free_map;
1612 }
1613 } else if (attr->excl_prog_hash_size) {
1614 bpf_log(log, "Invalid excl_prog_hash_size.\n");
1615 err = -EINVAL;
1616 goto free_map;
1617 }
1618
1619 *mapp = map;
1620 *tokenp = token;
1621 return 0;
1622
1623free_map:
1624 bpf_map_free(map);
1625put_token:
1626 bpf_token_put(token);
1627 return err;
1628}
1629
1630static int map_create(union bpf_attr *attr, bpfptr_t uattr, struct bpf_common_attr *attr_common,
1631 bpfptr_t uattr_common, u32 size_common)
1632{
1633 struct bpf_token *token = NULL;
1634 struct bpf_verifier_log *log;
1635 struct bpf_log_attr attr_log;
1636 struct bpf_map *map = NULL;
1637 int err, ret;
1638 int f_flags;
1639
1640 log = bpf_log_attr_create_vlog(&attr_log, attr_common, uattr_common, size_common);
1641 if (IS_ERR(log))
1642 return PTR_ERR(log);
1643
1644 err = map_create_alloc(attr, uattr, log, &map, &token);
1645
1646 /* preserve original error even if log finalization is successful */
1647 ret = bpf_log_attr_finalize(&attr_log, log);
1648 if (ret)
1649 err = ret;
1650
1651 kfree(log);
1652
1653 if (err)
1654 goto free_map;
1655
1656 f_flags = bpf_get_file_flag(attr->map_flags);
1657 if (f_flags < 0) {
1658 err = f_flags;
1659 goto free_map;
1660 }
1661
1662 err = security_bpf_map_create(map, attr, token, uattr.is_kernel);
1663 if (err)
1664 goto free_map;
1665
1666 err = bpf_map_alloc_id(map);
1667 if (err)
1668 goto free_map_sec;
1669
1670 bpf_map_save_memcg(map);
1671 bpf_token_put(token);
1672
1673 err = bpf_map_new_fd(map, f_flags);
1674 if (err < 0) {
1675 /* failed to allocate fd.
1676 * bpf_map_put_with_uref() is needed because the above
1677 * bpf_map_alloc_id() has published the map
1678 * to the userspace and the userspace may
1679 * have refcnt-ed it through BPF_MAP_GET_FD_BY_ID.
1680 */
1681 bpf_map_put_with_uref(map);
1682 return err;
1683 }
1684
1685 return err;
1686
1687free_map_sec:
1688 security_bpf_map_free(map);
1689free_map:
1690 if (map)
1691 bpf_map_free(map);
1692 bpf_token_put(token);
1693 return err;
1694}
1695
1696void bpf_map_inc(struct bpf_map *map)
1697{
1698 atomic64_inc(&map->refcnt);
1699}
1700EXPORT_SYMBOL_GPL(bpf_map_inc);
1701
1702void bpf_map_inc_with_uref(struct bpf_map *map)
1703{
1704 atomic64_inc(&map->refcnt);
1705 atomic64_inc(&map->usercnt);
1706}
1707EXPORT_SYMBOL_GPL(bpf_map_inc_with_uref);
1708
1709struct bpf_map *bpf_map_get(u32 ufd)
1710{
1711 CLASS(fd, f)(ufd);
1712 struct bpf_map *map = __bpf_map_get(f);
1713
1714 if (!IS_ERR(map))
1715 bpf_map_inc(map);
1716
1717 return map;
1718}
1719EXPORT_SYMBOL_NS(bpf_map_get, "BPF_INTERNAL");
1720
1721struct bpf_map *bpf_map_get_with_uref(u32 ufd)
1722{
1723 CLASS(fd, f)(ufd);
1724 struct bpf_map *map = __bpf_map_get(f);
1725
1726 if (!IS_ERR(map))
1727 bpf_map_inc_with_uref(map);
1728
1729 return map;
1730}
1731
1732/* map_idr_lock should have been held or the map should have been
1733 * protected by rcu read lock.
1734 */
1735struct bpf_map *__bpf_map_inc_not_zero(struct bpf_map *map, bool uref)
1736{
1737 int refold;
1738
1739 refold = atomic64_fetch_add_unless(&map->refcnt, 1, 0);
1740 if (!refold)
1741 return ERR_PTR(-ENOENT);
1742 if (uref)
1743 atomic64_inc(&map->usercnt);
1744
1745 return map;
1746}
1747
1748struct bpf_map *bpf_map_inc_not_zero(struct bpf_map *map)
1749{
1750 lockdep_assert(rcu_read_lock_held());
1751 return __bpf_map_inc_not_zero(map, false);
1752}
1753EXPORT_SYMBOL_GPL(bpf_map_inc_not_zero);
1754
1755int __weak bpf_stackmap_extract(struct bpf_map *map, void *key, void *value,
1756 bool delete)
1757{
1758 return -ENOTSUPP;
1759}
1760
1761static void *__bpf_copy_key(void __user *ukey, u64 key_size)
1762{
1763 if (key_size)
1764 return vmemdup_user(ukey, key_size);
1765
1766 if (ukey)
1767 return ERR_PTR(-EINVAL);
1768
1769 return NULL;
1770}
1771
1772static void *___bpf_copy_key(bpfptr_t ukey, u64 key_size)
1773{
1774 if (key_size)
1775 return kvmemdup_bpfptr(ukey, key_size);
1776
1777 if (!bpfptr_is_null(ukey))
1778 return ERR_PTR(-EINVAL);
1779
1780 return NULL;
1781}
1782
1783/* last field in 'union bpf_attr' used by this command */
1784#define BPF_MAP_LOOKUP_ELEM_LAST_FIELD flags
1785
1786static int map_lookup_elem(union bpf_attr *attr)
1787{
1788 void __user *ukey = u64_to_user_ptr(attr->key);
1789 void __user *uvalue = u64_to_user_ptr(attr->value);
1790 struct bpf_map *map;
1791 void *key, *value;
1792 u32 value_size;
1793 int err;
1794
1795 if (CHECK_ATTR(BPF_MAP_LOOKUP_ELEM))
1796 return -EINVAL;
1797
1798 CLASS(fd, f)(attr->map_fd);
1799 map = __bpf_map_get(f);
1800 if (IS_ERR(map))
1801 return PTR_ERR(map);
1802 if (!(map_get_sys_perms(map, f) & FMODE_CAN_READ))
1803 return -EPERM;
1804
1805 err = bpf_map_check_op_flags(map, attr->flags, BPF_F_LOCK | BPF_F_CPU);
1806 if (err)
1807 return err;
1808
1809 key = __bpf_copy_key(ukey, map->key_size);
1810 if (IS_ERR(key))
1811 return PTR_ERR(key);
1812
1813 value_size = bpf_map_value_size(map, attr->flags);
1814
1815 err = -ENOMEM;
1816 value = kvmalloc(value_size, GFP_USER | __GFP_NOWARN);
1817 if (!value)
1818 goto free_key;
1819
1820 if (map->map_type == BPF_MAP_TYPE_BLOOM_FILTER) {
1821 if (copy_from_user(value, uvalue, value_size))
1822 err = -EFAULT;
1823 else
1824 err = bpf_map_copy_value(map, key, value, attr->flags);
1825 goto free_value;
1826 }
1827
1828 err = bpf_map_copy_value(map, key, value, attr->flags);
1829 if (err)
1830 goto free_value;
1831
1832 err = -EFAULT;
1833 if (copy_to_user(uvalue, value, value_size) != 0)
1834 goto free_value;
1835
1836 err = 0;
1837
1838free_value:
1839 kvfree(value);
1840free_key:
1841 kvfree(key);
1842 return err;
1843}
1844
1845#define BPF_MAP_UPDATE_ELEM_LAST_FIELD flags
1846
1847static int map_update_elem(union bpf_attr *attr, bpfptr_t uattr)
1848{
1849 bpfptr_t ukey = make_bpfptr(attr->key, uattr.is_kernel);
1850 bpfptr_t uvalue = make_bpfptr(attr->value, uattr.is_kernel);
1851 struct bpf_map *map;
1852 void *key, *value;
1853 u32 value_size;
1854 int err;
1855
1856 if (CHECK_ATTR(BPF_MAP_UPDATE_ELEM))
1857 return -EINVAL;
1858
1859 CLASS(fd, f)(attr->map_fd);
1860 map = __bpf_map_get(f);
1861 if (IS_ERR(map))
1862 return PTR_ERR(map);
1863 bpf_map_write_active_inc(map);
1864 if (!(map_get_sys_perms(map, f) & FMODE_CAN_WRITE)) {
1865 err = -EPERM;
1866 goto err_put;
1867 }
1868
1869 err = bpf_map_check_op_flags(map, attr->flags, ~0);
1870 if (err)
1871 goto err_put;
1872
1873 key = ___bpf_copy_key(ukey, map->key_size);
1874 if (IS_ERR(key)) {
1875 err = PTR_ERR(key);
1876 goto err_put;
1877 }
1878
1879 value_size = bpf_map_value_size(map, attr->flags);
1880 value = kvmemdup_bpfptr(uvalue, value_size);
1881 if (IS_ERR(value)) {
1882 err = PTR_ERR(value);
1883 goto free_key;
1884 }
1885
1886 err = bpf_map_update_value(map, fd_file(f), key, value, attr->flags);
1887 if (!err)
1888 maybe_wait_bpf_programs(map);
1889
1890 kvfree(value);
1891free_key:
1892 kvfree(key);
1893err_put:
1894 bpf_map_write_active_dec(map);
1895 return err;
1896}
1897
1898#define BPF_MAP_DELETE_ELEM_LAST_FIELD key
1899
1900static int map_delete_elem(union bpf_attr *attr, bpfptr_t uattr)
1901{
1902 bpfptr_t ukey = make_bpfptr(attr->key, uattr.is_kernel);
1903 struct bpf_map *map;
1904 void *key;
1905 int err;
1906
1907 if (CHECK_ATTR(BPF_MAP_DELETE_ELEM))
1908 return -EINVAL;
1909
1910 CLASS(fd, f)(attr->map_fd);
1911 map = __bpf_map_get(f);
1912 if (IS_ERR(map))
1913 return PTR_ERR(map);
1914 bpf_map_write_active_inc(map);
1915 if (!(map_get_sys_perms(map, f) & FMODE_CAN_WRITE)) {
1916 err = -EPERM;
1917 goto err_put;
1918 }
1919
1920 key = ___bpf_copy_key(ukey, map->key_size);
1921 if (IS_ERR(key)) {
1922 err = PTR_ERR(key);
1923 goto err_put;
1924 }
1925
1926 if (bpf_map_is_offloaded(map)) {
1927 err = bpf_map_offload_delete_elem(map, key);
1928 goto out;
1929 } else if (IS_FD_PROG_ARRAY(map) ||
1930 map->map_type == BPF_MAP_TYPE_STRUCT_OPS) {
1931 /* These maps require sleepable context */
1932 err = map->ops->map_delete_elem(map, key);
1933 goto out;
1934 }
1935
1936 bpf_disable_instrumentation();
1937 rcu_read_lock();
1938 err = map->ops->map_delete_elem(map, key);
1939 rcu_read_unlock();
1940 bpf_enable_instrumentation();
1941 if (!err)
1942 maybe_wait_bpf_programs(map);
1943out:
1944 kvfree(key);
1945err_put:
1946 bpf_map_write_active_dec(map);
1947 return err;
1948}
1949
1950/* last field in 'union bpf_attr' used by this command */
1951#define BPF_MAP_GET_NEXT_KEY_LAST_FIELD next_key
1952
1953static int map_get_next_key(union bpf_attr *attr)
1954{
1955 void __user *ukey = u64_to_user_ptr(attr->key);
1956 void __user *unext_key = u64_to_user_ptr(attr->next_key);
1957 struct bpf_map *map;
1958 void *key, *next_key;
1959 int err;
1960
1961 if (CHECK_ATTR(BPF_MAP_GET_NEXT_KEY))
1962 return -EINVAL;
1963
1964 CLASS(fd, f)(attr->map_fd);
1965 map = __bpf_map_get(f);
1966 if (IS_ERR(map))
1967 return PTR_ERR(map);
1968 if (!(map_get_sys_perms(map, f) & FMODE_CAN_READ))
1969 return -EPERM;
1970
1971 if (ukey) {
1972 key = __bpf_copy_key(ukey, map->key_size);
1973 if (IS_ERR(key))
1974 return PTR_ERR(key);
1975 } else {
1976 key = NULL;
1977 }
1978
1979 err = -ENOMEM;
1980 next_key = kvmalloc(map->key_size, GFP_USER);
1981 if (!next_key)
1982 goto free_key;
1983
1984 if (bpf_map_is_offloaded(map)) {
1985 err = bpf_map_offload_get_next_key(map, key, next_key);
1986 goto out;
1987 }
1988
1989 rcu_read_lock();
1990 err = map->ops->map_get_next_key(map, key, next_key);
1991 rcu_read_unlock();
1992out:
1993 if (err)
1994 goto free_next_key;
1995
1996 err = -EFAULT;
1997 if (copy_to_user(unext_key, next_key, map->key_size) != 0)
1998 goto free_next_key;
1999
2000 err = 0;
2001
2002free_next_key:
2003 kvfree(next_key);
2004free_key:
2005 kvfree(key);
2006 return err;
2007}
2008
2009int generic_map_delete_batch(struct bpf_map *map,
2010 const union bpf_attr *attr,
2011 union bpf_attr __user *uattr)
2012{
2013 void __user *keys = u64_to_user_ptr(attr->batch.keys);
2014 u32 cp, max_count;
2015 int err = 0;
2016 void *key;
2017
2018 if (attr->batch.elem_flags & ~BPF_F_LOCK)
2019 return -EINVAL;
2020
2021 if ((attr->batch.elem_flags & BPF_F_LOCK) &&
2022 !btf_record_has_field(map->record, BPF_SPIN_LOCK)) {
2023 return -EINVAL;
2024 }
2025
2026 max_count = attr->batch.count;
2027 if (!max_count)
2028 return 0;
2029
2030 if (put_user(0, &uattr->batch.count))
2031 return -EFAULT;
2032
2033 key = kvmalloc(map->key_size, GFP_USER | __GFP_NOWARN);
2034 if (!key)
2035 return -ENOMEM;
2036
2037 for (cp = 0; cp < max_count; cp++) {
2038 err = -EFAULT;
2039 if (copy_from_user(key, keys + (size_t)cp * map->key_size,
2040 map->key_size))
2041 break;
2042
2043 if (bpf_map_is_offloaded(map)) {
2044 err = bpf_map_offload_delete_elem(map, key);
2045 break;
2046 }
2047
2048 bpf_disable_instrumentation();
2049 rcu_read_lock();
2050 err = map->ops->map_delete_elem(map, key);
2051 rcu_read_unlock();
2052 bpf_enable_instrumentation();
2053 if (err)
2054 break;
2055 cond_resched();
2056 }
2057 if (copy_to_user(&uattr->batch.count, &cp, sizeof(cp)))
2058 err = -EFAULT;
2059
2060 kvfree(key);
2061
2062 return err;
2063}
2064
2065int generic_map_update_batch(struct bpf_map *map, struct file *map_file,
2066 const union bpf_attr *attr,
2067 union bpf_attr __user *uattr)
2068{
2069 void __user *values = u64_to_user_ptr(attr->batch.values);
2070 void __user *keys = u64_to_user_ptr(attr->batch.keys);
2071 u32 value_size, cp, max_count;
2072 void *key, *value;
2073 int err = 0;
2074
2075 err = bpf_map_check_op_flags(map, attr->batch.elem_flags,
2076 BPF_F_LOCK | BPF_F_CPU | BPF_F_ALL_CPUS);
2077 if (err)
2078 return err;
2079
2080 value_size = bpf_map_value_size(map, attr->batch.elem_flags);
2081
2082 max_count = attr->batch.count;
2083 if (!max_count)
2084 return 0;
2085
2086 if (put_user(0, &uattr->batch.count))
2087 return -EFAULT;
2088
2089 key = kvmalloc(map->key_size, GFP_USER | __GFP_NOWARN);
2090 if (!key)
2091 return -ENOMEM;
2092
2093 value = kvmalloc(value_size, GFP_USER | __GFP_NOWARN);
2094 if (!value) {
2095 kvfree(key);
2096 return -ENOMEM;
2097 }
2098
2099 for (cp = 0; cp < max_count; cp++) {
2100 err = -EFAULT;
2101 if (copy_from_user(key, keys + (size_t)cp * map->key_size,
2102 map->key_size) ||
2103 copy_from_user(value, values + (size_t)cp * value_size, value_size))
2104 break;
2105
2106 err = bpf_map_update_value(map, map_file, key, value,
2107 attr->batch.elem_flags);
2108
2109 if (err)
2110 break;
2111 cond_resched();
2112 }
2113
2114 if (copy_to_user(&uattr->batch.count, &cp, sizeof(cp)))
2115 err = -EFAULT;
2116
2117 kvfree(value);
2118 kvfree(key);
2119
2120 return err;
2121}
2122
2123int generic_map_lookup_batch(struct bpf_map *map,
2124 const union bpf_attr *attr,
2125 union bpf_attr __user *uattr)
2126{
2127 void __user *uobatch = u64_to_user_ptr(attr->batch.out_batch);
2128 void __user *ubatch = u64_to_user_ptr(attr->batch.in_batch);
2129 void __user *values = u64_to_user_ptr(attr->batch.values);
2130 void __user *keys = u64_to_user_ptr(attr->batch.keys);
2131 void *buf, *buf_prevkey, *prev_key, *key, *value;
2132 u32 value_size, cp, max_count;
2133 int err;
2134
2135 err = bpf_map_check_op_flags(map, attr->batch.elem_flags, BPF_F_LOCK | BPF_F_CPU);
2136 if (err)
2137 return err;
2138
2139 value_size = bpf_map_value_size(map, attr->batch.elem_flags);
2140
2141 max_count = attr->batch.count;
2142 if (!max_count)
2143 return 0;
2144
2145 if (put_user(0, &uattr->batch.count))
2146 return -EFAULT;
2147
2148 buf_prevkey = kvmalloc(map->key_size, GFP_USER | __GFP_NOWARN);
2149 if (!buf_prevkey)
2150 return -ENOMEM;
2151
2152 buf = kvmalloc(map->key_size + value_size, GFP_USER | __GFP_NOWARN);
2153 if (!buf) {
2154 kvfree(buf_prevkey);
2155 return -ENOMEM;
2156 }
2157
2158 err = -EFAULT;
2159 prev_key = NULL;
2160 if (ubatch && copy_from_user(buf_prevkey, ubatch, map->key_size))
2161 goto free_buf;
2162 key = buf;
2163 value = key + map->key_size;
2164 if (ubatch)
2165 prev_key = buf_prevkey;
2166
2167 for (cp = 0; cp < max_count;) {
2168 rcu_read_lock();
2169 err = map->ops->map_get_next_key(map, prev_key, key);
2170 rcu_read_unlock();
2171 if (err)
2172 break;
2173 err = bpf_map_copy_value(map, key, value,
2174 attr->batch.elem_flags);
2175
2176 if (err == -ENOENT)
2177 goto next_key;
2178
2179 if (err)
2180 goto free_buf;
2181
2182 if (copy_to_user(keys + (size_t)cp * map->key_size, key,
2183 map->key_size)) {
2184 err = -EFAULT;
2185 goto free_buf;
2186 }
2187 if (copy_to_user(values + (size_t)cp * value_size, value, value_size)) {
2188 err = -EFAULT;
2189 goto free_buf;
2190 }
2191
2192 cp++;
2193next_key:
2194 if (!prev_key)
2195 prev_key = buf_prevkey;
2196
2197 swap(prev_key, key);
2198 cond_resched();
2199 }
2200
2201 if (err == -EFAULT)
2202 goto free_buf;
2203
2204 if ((copy_to_user(&uattr->batch.count, &cp, sizeof(cp)) ||
2205 (cp && copy_to_user(uobatch, prev_key, map->key_size))))
2206 err = -EFAULT;
2207
2208free_buf:
2209 kvfree(buf_prevkey);
2210 kvfree(buf);
2211 return err;
2212}
2213
2214#define BPF_MAP_LOOKUP_AND_DELETE_ELEM_LAST_FIELD flags
2215
2216static int map_lookup_and_delete_elem(union bpf_attr *attr)
2217{
2218 void __user *ukey = u64_to_user_ptr(attr->key);
2219 void __user *uvalue = u64_to_user_ptr(attr->value);
2220 struct bpf_map *map;
2221 void *key, *value;
2222 u32 value_size;
2223 int err;
2224
2225 if (CHECK_ATTR(BPF_MAP_LOOKUP_AND_DELETE_ELEM))
2226 return -EINVAL;
2227
2228 if (attr->flags & ~BPF_F_LOCK)
2229 return -EINVAL;
2230
2231 CLASS(fd, f)(attr->map_fd);
2232 map = __bpf_map_get(f);
2233 if (IS_ERR(map))
2234 return PTR_ERR(map);
2235 bpf_map_write_active_inc(map);
2236 if (!(map_get_sys_perms(map, f) & FMODE_CAN_READ) ||
2237 !(map_get_sys_perms(map, f) & FMODE_CAN_WRITE)) {
2238 err = -EPERM;
2239 goto err_put;
2240 }
2241
2242 if (attr->flags &&
2243 (map->map_type == BPF_MAP_TYPE_QUEUE ||
2244 map->map_type == BPF_MAP_TYPE_STACK)) {
2245 err = -EINVAL;
2246 goto err_put;
2247 }
2248
2249 if ((attr->flags & BPF_F_LOCK) &&
2250 !btf_record_has_field(map->record, BPF_SPIN_LOCK)) {
2251 err = -EINVAL;
2252 goto err_put;
2253 }
2254
2255 key = __bpf_copy_key(ukey, map->key_size);
2256 if (IS_ERR(key)) {
2257 err = PTR_ERR(key);
2258 goto err_put;
2259 }
2260
2261 value_size = bpf_map_value_size(map, 0);
2262
2263 err = -ENOMEM;
2264 value = kvmalloc(value_size, GFP_USER | __GFP_NOWARN);
2265 if (!value)
2266 goto free_key;
2267
2268 err = -ENOTSUPP;
2269 if (map->map_type == BPF_MAP_TYPE_QUEUE ||
2270 map->map_type == BPF_MAP_TYPE_STACK) {
2271 err = map->ops->map_pop_elem(map, value);
2272 } else if (map->map_type == BPF_MAP_TYPE_HASH ||
2273 map->map_type == BPF_MAP_TYPE_PERCPU_HASH ||
2274 map->map_type == BPF_MAP_TYPE_LRU_HASH ||
2275 map->map_type == BPF_MAP_TYPE_LRU_PERCPU_HASH ||
2276 map->map_type == BPF_MAP_TYPE_RHASH ||
2277 map->map_type == BPF_MAP_TYPE_STACK_TRACE) {
2278 if (!bpf_map_is_offloaded(map)) {
2279 bpf_disable_instrumentation();
2280 rcu_read_lock();
2281 err = map->ops->map_lookup_and_delete_elem(map, key, value, attr->flags);
2282 rcu_read_unlock();
2283 bpf_enable_instrumentation();
2284 }
2285 }
2286
2287 if (err)
2288 goto free_value;
2289
2290 if (copy_to_user(uvalue, value, value_size) != 0) {
2291 err = -EFAULT;
2292 goto free_value;
2293 }
2294
2295 err = 0;
2296
2297free_value:
2298 kvfree(value);
2299free_key:
2300 kvfree(key);
2301err_put:
2302 bpf_map_write_active_dec(map);
2303 return err;
2304}
2305
2306#define BPF_MAP_FREEZE_LAST_FIELD map_fd
2307
2308static int map_freeze(const union bpf_attr *attr)
2309{
2310 int err = 0;
2311 struct bpf_map *map;
2312
2313 if (CHECK_ATTR(BPF_MAP_FREEZE))
2314 return -EINVAL;
2315
2316 CLASS(fd, f)(attr->map_fd);
2317 map = __bpf_map_get(f);
2318 if (IS_ERR(map))
2319 return PTR_ERR(map);
2320
2321 if (map->map_type == BPF_MAP_TYPE_STRUCT_OPS || !IS_ERR_OR_NULL(map->record))
2322 return -ENOTSUPP;
2323
2324 if (!(map_get_sys_perms(map, f) & FMODE_CAN_WRITE))
2325 return -EPERM;
2326
2327 mutex_lock(&map->freeze_mutex);
2328 if (bpf_map_write_active(map)) {
2329 err = -EBUSY;
2330 goto err_put;
2331 }
2332 if (READ_ONCE(map->frozen)) {
2333 err = -EBUSY;
2334 goto err_put;
2335 }
2336
2337 WRITE_ONCE(map->frozen, true);
2338err_put:
2339 mutex_unlock(&map->freeze_mutex);
2340 return err;
2341}
2342
2343static const struct bpf_prog_ops * const bpf_prog_types[] = {
2344#define BPF_PROG_TYPE(_id, _name, prog_ctx_type, kern_ctx_type) \
2345 [_id] = & _name ## _prog_ops,
2346#define BPF_MAP_TYPE(_id, _ops)
2347#define BPF_LINK_TYPE(_id, _name)
2348#include <linux/bpf_types.h>
2349#undef BPF_PROG_TYPE
2350#undef BPF_MAP_TYPE
2351#undef BPF_LINK_TYPE
2352};
2353
2354static int find_prog_type(enum bpf_prog_type type, struct bpf_prog *prog)
2355{
2356 const struct bpf_prog_ops *ops;
2357
2358 if (type >= ARRAY_SIZE(bpf_prog_types))
2359 return -EINVAL;
2360 type = array_index_nospec(type, ARRAY_SIZE(bpf_prog_types));
2361 ops = bpf_prog_types[type];
2362 if (!ops)
2363 return -EINVAL;
2364
2365 if (!bpf_prog_is_offloaded(prog->aux))
2366 prog->aux->ops = ops;
2367 else
2368 prog->aux->ops = &bpf_offload_prog_ops;
2369 prog->type = type;
2370 return 0;
2371}
2372
2373enum bpf_audit {
2374 BPF_AUDIT_LOAD,
2375 BPF_AUDIT_UNLOAD,
2376 BPF_AUDIT_MAX,
2377};
2378
2379static const char * const bpf_audit_str[BPF_AUDIT_MAX] = {
2380 [BPF_AUDIT_LOAD] = "LOAD",
2381 [BPF_AUDIT_UNLOAD] = "UNLOAD",
2382};
2383
2384static void bpf_audit_prog(const struct bpf_prog *prog, unsigned int op)
2385{
2386 struct audit_context *ctx = NULL;
2387 struct audit_buffer *ab;
2388
2389 if (WARN_ON_ONCE(op >= BPF_AUDIT_MAX))
2390 return;
2391 if (audit_enabled == AUDIT_OFF)
2392 return;
2393 if (!in_hardirq() && !irqs_disabled())
2394 ctx = audit_context();
2395 ab = audit_log_start(ctx, GFP_ATOMIC, AUDIT_BPF);
2396 if (unlikely(!ab))
2397 return;
2398 audit_log_format(ab, "prog-id=%u op=%s",
2399 prog->aux->id, bpf_audit_str[op]);
2400 audit_log_end(ab);
2401}
2402
2403static int bpf_prog_alloc_id(struct bpf_prog *prog)
2404{
2405 int id;
2406
2407 idr_preload(GFP_KERNEL);
2408 spin_lock_bh(&prog_idr_lock);
2409 id = idr_alloc_cyclic(&prog_idr, prog, 1, INT_MAX, GFP_ATOMIC);
2410 if (id > 0)
2411 prog->aux->id = id;
2412 spin_unlock_bh(&prog_idr_lock);
2413 idr_preload_end();
2414
2415 /* id is in [1, INT_MAX) */
2416 if (WARN_ON_ONCE(!id))
2417 return -ENOSPC;
2418
2419 return id > 0 ? 0 : id;
2420}
2421
2422void bpf_prog_free_id(struct bpf_prog *prog)
2423{
2424 unsigned long flags;
2425
2426 /* cBPF to eBPF migrations are currently not in the idr store.
2427 * Offloaded programs are removed from the store when their device
2428 * disappears - even if someone grabs an fd to them they are unusable,
2429 * simply waiting for refcnt to drop to be freed.
2430 */
2431 if (!prog->aux->id)
2432 return;
2433
2434 spin_lock_irqsave(&prog_idr_lock, flags);
2435 idr_remove(&prog_idr, prog->aux->id);
2436 prog->aux->id = 0;
2437 spin_unlock_irqrestore(&prog_idr_lock, flags);
2438}
2439
2440static void __bpf_prog_put_rcu(struct rcu_head *rcu)
2441{
2442 struct bpf_prog_aux *aux = container_of(rcu, struct bpf_prog_aux, rcu);
2443
2444 kvfree(aux->func_info);
2445 kfree(aux->func_info_aux);
2446 free_uid(aux->user);
2447 security_bpf_prog_free(aux->prog);
2448 bpf_prog_free(aux->prog);
2449}
2450
2451/*
2452 * Progs called from a trampoline can also be reached by a task that was
2453 * preempted in the trampoline before the prog's enter helper took its RCU
2454 * read lock, wait for those first.
2455 */
2456static void __bpf_prog_put_rcu_tasks(struct rcu_head *rcu)
2457{
2458 struct bpf_prog *prog = container_of(rcu, struct bpf_prog_aux, rcu)->prog;
2459
2460 if (prog->sleepable)
2461 call_rcu_tasks_trace(rcu, __bpf_prog_put_rcu);
2462 else
2463 call_rcu(rcu, __bpf_prog_put_rcu);
2464}
2465
2466static void __bpf_prog_put_noref(struct bpf_prog *prog, bool deferred)
2467{
2468 bpf_prog_kallsyms_del_all(prog);
2469 btf_put(prog->aux->btf);
2470 module_put(prog->aux->mod);
2471 kvfree(prog->aux->jited_linfo);
2472 kvfree(prog->aux->linfo);
2473 kfree(prog->aux->kfunc_tab);
2474 kfree(prog->aux->ctx_arg_info);
2475 if (prog->aux->attach_btf)
2476 btf_put(prog->aux->attach_btf);
2477
2478 if (deferred) {
2479 if (IS_ENABLED(CONFIG_TASKS_RCU) && prog->aux->tramp_linked)
2480 call_rcu_tasks(&prog->aux->rcu, __bpf_prog_put_rcu_tasks);
2481 else if (prog->sleepable)
2482 call_rcu_tasks_trace(&prog->aux->rcu, __bpf_prog_put_rcu);
2483 else
2484 call_rcu(&prog->aux->rcu, __bpf_prog_put_rcu);
2485 } else {
2486 __bpf_prog_put_rcu(&prog->aux->rcu);
2487 }
2488}
2489
2490static void bpf_prog_put_deferred(struct work_struct *work)
2491{
2492 struct bpf_prog_aux *aux;
2493 struct bpf_prog *prog;
2494
2495 aux = container_of(work, struct bpf_prog_aux, work);
2496 prog = aux->prog;
2497 perf_event_bpf_event(prog, PERF_BPF_EVENT_PROG_UNLOAD, 0);
2498 bpf_audit_prog(prog, BPF_AUDIT_UNLOAD);
2499 bpf_prog_free_id(prog);
2500 __bpf_prog_put_noref(prog, true);
2501}
2502
2503static void __bpf_prog_put(struct bpf_prog *prog)
2504{
2505 struct bpf_prog_aux *aux = prog->aux;
2506
2507 if (atomic64_dec_and_test(&aux->refcnt)) {
2508 if (in_hardirq() || irqs_disabled()) {
2509 INIT_WORK(&aux->work, bpf_prog_put_deferred);
2510 schedule_work(&aux->work);
2511 } else {
2512 bpf_prog_put_deferred(&aux->work);
2513 }
2514 }
2515}
2516
2517void bpf_prog_put(struct bpf_prog *prog)
2518{
2519 __bpf_prog_put(prog);
2520}
2521EXPORT_SYMBOL_GPL(bpf_prog_put);
2522
2523static int bpf_prog_release(struct inode *inode, struct file *filp)
2524{
2525 struct bpf_prog *prog = filp->private_data;
2526
2527 bpf_prog_put(prog);
2528 return 0;
2529}
2530
2531struct bpf_prog_kstats {
2532 u64 nsecs;
2533 u64 cnt;
2534 u64 misses;
2535};
2536
2537void notrace bpf_prog_inc_misses_counter(struct bpf_prog *prog)
2538{
2539 struct bpf_prog_stats *stats;
2540 unsigned int flags;
2541
2542 if (unlikely(!prog->stats))
2543 return;
2544
2545 stats = this_cpu_ptr(prog->stats);
2546 flags = u64_stats_update_begin_irqsave(&stats->syncp);
2547 u64_stats_inc(&stats->misses);
2548 u64_stats_update_end_irqrestore(&stats->syncp, flags);
2549}
2550
2551static void bpf_prog_get_stats(const struct bpf_prog *prog,
2552 struct bpf_prog_kstats *stats)
2553{
2554 u64 nsecs = 0, cnt = 0, misses = 0;
2555 int cpu;
2556
2557 for_each_possible_cpu(cpu) {
2558 const struct bpf_prog_stats *st;
2559 unsigned int start;
2560 u64 tnsecs, tcnt, tmisses;
2561
2562 st = per_cpu_ptr(prog->stats, cpu);
2563 do {
2564 start = u64_stats_fetch_begin(&st->syncp);
2565 tnsecs = u64_stats_read(&st->nsecs);
2566 tcnt = u64_stats_read(&st->cnt);
2567 tmisses = u64_stats_read(&st->misses);
2568 } while (u64_stats_fetch_retry(&st->syncp, start));
2569 nsecs += tnsecs;
2570 cnt += tcnt;
2571 misses += tmisses;
2572 }
2573 stats->nsecs = nsecs;
2574 stats->cnt = cnt;
2575 stats->misses = misses;
2576}
2577
2578#ifdef CONFIG_PROC_FS
2579static void bpf_prog_show_fdinfo(struct seq_file *m, struct file *filp)
2580{
2581 const struct bpf_prog *prog = filp->private_data;
2582 char prog_tag[sizeof(prog->tag) * 2 + 1] = { };
2583 struct bpf_prog_kstats stats;
2584
2585 bpf_prog_get_stats(prog, &stats);
2586 bin2hex(prog_tag, prog->tag, sizeof(prog->tag));
2587 seq_printf(m,
2588 "prog_type:\t%u\n"
2589 "prog_jited:\t%u\n"
2590 "prog_tag:\t%s\n"
2591 "memlock:\t%llu\n"
2592 "prog_id:\t%u\n"
2593 "run_time_ns:\t%llu\n"
2594 "run_cnt:\t%llu\n"
2595 "recursion_misses:\t%llu\n"
2596 "verified_insns:\t%u\n",
2597 prog->type,
2598 prog->jited,
2599 prog_tag,
2600 prog->pages * 1ULL << PAGE_SHIFT,
2601 prog->aux->id,
2602 stats.nsecs,
2603 stats.cnt,
2604 stats.misses,
2605 prog->aux->verified_insns);
2606}
2607#endif
2608
2609const struct file_operations bpf_prog_fops = {
2610#ifdef CONFIG_PROC_FS
2611 .show_fdinfo = bpf_prog_show_fdinfo,
2612#endif
2613 .release = bpf_prog_release,
2614 .read = bpf_dummy_read,
2615 .write = bpf_dummy_write,
2616};
2617
2618int bpf_prog_new_fd(struct bpf_prog *prog)
2619{
2620 int ret;
2621
2622 ret = security_bpf_prog(prog);
2623 if (ret < 0)
2624 return ret;
2625
2626 return anon_inode_getfd("bpf-prog", &bpf_prog_fops, prog,
2627 O_RDWR | O_CLOEXEC);
2628}
2629
2630void bpf_prog_add(struct bpf_prog *prog, int i)
2631{
2632 atomic64_add(i, &prog->aux->refcnt);
2633}
2634EXPORT_SYMBOL_GPL(bpf_prog_add);
2635
2636void bpf_prog_sub(struct bpf_prog *prog, int i)
2637{
2638 /* Only to be used for undoing previous bpf_prog_add() in some
2639 * error path. We still know that another entity in our call
2640 * path holds a reference to the program, thus atomic_sub() can
2641 * be safely used in such cases!
2642 */
2643 WARN_ON(atomic64_sub_return(i, &prog->aux->refcnt) == 0);
2644}
2645EXPORT_SYMBOL_GPL(bpf_prog_sub);
2646
2647void bpf_prog_inc(struct bpf_prog *prog)
2648{
2649 atomic64_inc(&prog->aux->refcnt);
2650}
2651EXPORT_SYMBOL_GPL(bpf_prog_inc);
2652
2653/* prog_idr_lock should have been held */
2654struct bpf_prog *bpf_prog_inc_not_zero(struct bpf_prog *prog)
2655{
2656 int refold;
2657
2658 refold = atomic64_fetch_add_unless(&prog->aux->refcnt, 1, 0);
2659
2660 if (!refold)
2661 return ERR_PTR(-ENOENT);
2662
2663 return prog;
2664}
2665EXPORT_SYMBOL_GPL(bpf_prog_inc_not_zero);
2666
2667bool bpf_prog_get_ok(struct bpf_prog *prog,
2668 enum bpf_prog_type *attach_type, bool attach_drv)
2669{
2670 /* not an attachment, just a refcount inc, always allow */
2671 if (!attach_type)
2672 return true;
2673
2674 if (prog->type != *attach_type)
2675 return false;
2676 if (bpf_prog_is_offloaded(prog->aux) && !attach_drv)
2677 return false;
2678
2679 return true;
2680}
2681
2682static struct bpf_prog *__bpf_prog_get(u32 ufd, enum bpf_prog_type *attach_type,
2683 bool attach_drv)
2684{
2685 CLASS(fd, f)(ufd);
2686 struct bpf_prog *prog;
2687
2688 if (fd_empty(f))
2689 return ERR_PTR(-EBADF);
2690 if (fd_file(f)->f_op != &bpf_prog_fops)
2691 return ERR_PTR(-EINVAL);
2692
2693 prog = fd_file(f)->private_data;
2694 if (!bpf_prog_get_ok(prog, attach_type, attach_drv))
2695 return ERR_PTR(-EINVAL);
2696
2697 bpf_prog_inc(prog);
2698 return prog;
2699}
2700
2701struct bpf_prog *bpf_prog_get(u32 ufd)
2702{
2703 return __bpf_prog_get(ufd, NULL, false);
2704}
2705
2706struct bpf_prog *bpf_prog_get_type_dev(u32 ufd, enum bpf_prog_type type,
2707 bool attach_drv)
2708{
2709 return __bpf_prog_get(ufd, &type, attach_drv);
2710}
2711EXPORT_SYMBOL_GPL(bpf_prog_get_type_dev);
2712
2713/* Initially all BPF programs could be loaded w/o specifying
2714 * expected_attach_type. Later for some of them specifying expected_attach_type
2715 * at load time became required so that program could be validated properly.
2716 * Programs of types that are allowed to be loaded both w/ and w/o (for
2717 * backward compatibility) expected_attach_type, should have the default attach
2718 * type assigned to expected_attach_type for the latter case, so that it can be
2719 * validated later at attach time.
2720 *
2721 * bpf_prog_load_fixup_attach_type() sets expected_attach_type in @attr if
2722 * prog type requires it but has some attach types that have to be backward
2723 * compatible.
2724 */
2725static void bpf_prog_load_fixup_attach_type(union bpf_attr *attr)
2726{
2727 switch (attr->prog_type) {
2728 case BPF_PROG_TYPE_CGROUP_SOCK:
2729 /* Unfortunately BPF_ATTACH_TYPE_UNSPEC enumeration doesn't
2730 * exist so checking for non-zero is the way to go here.
2731 */
2732 if (!attr->expected_attach_type)
2733 attr->expected_attach_type =
2734 BPF_CGROUP_INET_SOCK_CREATE;
2735 break;
2736 case BPF_PROG_TYPE_SK_REUSEPORT:
2737 if (!attr->expected_attach_type)
2738 attr->expected_attach_type =
2739 BPF_SK_REUSEPORT_SELECT;
2740 break;
2741 }
2742}
2743
2744static int
2745bpf_prog_load_check_attach(enum bpf_prog_type prog_type,
2746 enum bpf_attach_type expected_attach_type,
2747 struct btf *attach_btf, u32 btf_id,
2748 struct bpf_prog *dst_prog,
2749 bool multi_func)
2750{
2751 if (btf_id) {
2752 if (btf_id > BTF_MAX_TYPE)
2753 return -EINVAL;
2754
2755 if (!attach_btf && !dst_prog)
2756 return -EINVAL;
2757
2758 switch (prog_type) {
2759 case BPF_PROG_TYPE_TRACING:
2760 case BPF_PROG_TYPE_LSM:
2761 case BPF_PROG_TYPE_STRUCT_OPS:
2762 case BPF_PROG_TYPE_EXT:
2763 break;
2764 default:
2765 return -EINVAL;
2766 }
2767 }
2768
2769 if (multi_func) {
2770 if (prog_type != BPF_PROG_TYPE_TRACING)
2771 return -EINVAL;
2772 if (!attach_btf || btf_id)
2773 return -EINVAL;
2774 return 0;
2775 }
2776
2777 if (attach_btf && (!btf_id || dst_prog))
2778 return -EINVAL;
2779
2780 if (dst_prog && prog_type != BPF_PROG_TYPE_TRACING &&
2781 prog_type != BPF_PROG_TYPE_EXT)
2782 return -EINVAL;
2783
2784 switch (prog_type) {
2785 case BPF_PROG_TYPE_CGROUP_SOCK:
2786 switch (expected_attach_type) {
2787 case BPF_CGROUP_INET_SOCK_CREATE:
2788 case BPF_CGROUP_INET_SOCK_RELEASE:
2789 case BPF_CGROUP_INET4_POST_BIND:
2790 case BPF_CGROUP_INET6_POST_BIND:
2791 return 0;
2792 default:
2793 return -EINVAL;
2794 }
2795 case BPF_PROG_TYPE_CGROUP_SOCK_ADDR:
2796 switch (expected_attach_type) {
2797 case BPF_CGROUP_INET4_BIND:
2798 case BPF_CGROUP_INET6_BIND:
2799 case BPF_CGROUP_INET4_CONNECT:
2800 case BPF_CGROUP_INET6_CONNECT:
2801 case BPF_CGROUP_UNIX_CONNECT:
2802 case BPF_CGROUP_INET4_GETPEERNAME:
2803 case BPF_CGROUP_INET6_GETPEERNAME:
2804 case BPF_CGROUP_UNIX_GETPEERNAME:
2805 case BPF_CGROUP_INET4_GETSOCKNAME:
2806 case BPF_CGROUP_INET6_GETSOCKNAME:
2807 case BPF_CGROUP_UNIX_GETSOCKNAME:
2808 case BPF_CGROUP_UDP4_SENDMSG:
2809 case BPF_CGROUP_UDP6_SENDMSG:
2810 case BPF_CGROUP_UNIX_SENDMSG:
2811 case BPF_CGROUP_UDP4_RECVMSG:
2812 case BPF_CGROUP_UDP6_RECVMSG:
2813 case BPF_CGROUP_UNIX_RECVMSG:
2814 return 0;
2815 default:
2816 return -EINVAL;
2817 }
2818 case BPF_PROG_TYPE_CGROUP_SKB:
2819 switch (expected_attach_type) {
2820 case BPF_CGROUP_INET_INGRESS:
2821 case BPF_CGROUP_INET_EGRESS:
2822 return 0;
2823 default:
2824 return -EINVAL;
2825 }
2826 case BPF_PROG_TYPE_CGROUP_SOCKOPT:
2827 switch (expected_attach_type) {
2828 case BPF_CGROUP_SETSOCKOPT:
2829 case BPF_CGROUP_GETSOCKOPT:
2830 return 0;
2831 default:
2832 return -EINVAL;
2833 }
2834 case BPF_PROG_TYPE_SK_LOOKUP:
2835 if (expected_attach_type == BPF_SK_LOOKUP)
2836 return 0;
2837 return -EINVAL;
2838 case BPF_PROG_TYPE_SK_REUSEPORT:
2839 switch (expected_attach_type) {
2840 case BPF_SK_REUSEPORT_SELECT:
2841 case BPF_SK_REUSEPORT_SELECT_OR_MIGRATE:
2842 return 0;
2843 default:
2844 return -EINVAL;
2845 }
2846 case BPF_PROG_TYPE_NETFILTER:
2847 if (expected_attach_type == BPF_NETFILTER)
2848 return 0;
2849 return -EINVAL;
2850 case BPF_PROG_TYPE_SYSCALL:
2851 case BPF_PROG_TYPE_EXT:
2852 if (expected_attach_type)
2853 return -EINVAL;
2854 fallthrough;
2855 default:
2856 return 0;
2857 }
2858}
2859
2860static bool is_net_admin_prog_type(enum bpf_prog_type prog_type)
2861{
2862 switch (prog_type) {
2863 case BPF_PROG_TYPE_SCHED_CLS:
2864 case BPF_PROG_TYPE_SCHED_ACT:
2865 case BPF_PROG_TYPE_XDP:
2866 case BPF_PROG_TYPE_LWT_IN:
2867 case BPF_PROG_TYPE_LWT_OUT:
2868 case BPF_PROG_TYPE_LWT_XMIT:
2869 case BPF_PROG_TYPE_LWT_SEG6LOCAL:
2870 case BPF_PROG_TYPE_SK_SKB:
2871 case BPF_PROG_TYPE_SK_MSG:
2872 case BPF_PROG_TYPE_FLOW_DISSECTOR:
2873 case BPF_PROG_TYPE_CGROUP_DEVICE:
2874 case BPF_PROG_TYPE_CGROUP_SOCK:
2875 case BPF_PROG_TYPE_CGROUP_SOCK_ADDR:
2876 case BPF_PROG_TYPE_CGROUP_SOCKOPT:
2877 case BPF_PROG_TYPE_CGROUP_SYSCTL:
2878 case BPF_PROG_TYPE_SOCK_OPS:
2879 case BPF_PROG_TYPE_EXT: /* extends any prog */
2880 case BPF_PROG_TYPE_NETFILTER:
2881 return true;
2882 case BPF_PROG_TYPE_CGROUP_SKB:
2883 /* always unpriv */
2884 case BPF_PROG_TYPE_SK_REUSEPORT:
2885 /* equivalent to SOCKET_FILTER. need CAP_BPF only */
2886 default:
2887 return false;
2888 }
2889}
2890
2891static bool is_perfmon_prog_type(enum bpf_prog_type prog_type)
2892{
2893 switch (prog_type) {
2894 case BPF_PROG_TYPE_KPROBE:
2895 case BPF_PROG_TYPE_TRACEPOINT:
2896 case BPF_PROG_TYPE_PERF_EVENT:
2897 case BPF_PROG_TYPE_RAW_TRACEPOINT:
2898 case BPF_PROG_TYPE_RAW_TRACEPOINT_WRITABLE:
2899 case BPF_PROG_TYPE_TRACING:
2900 case BPF_PROG_TYPE_LSM:
2901 case BPF_PROG_TYPE_STRUCT_OPS: /* has access to struct sock */
2902 case BPF_PROG_TYPE_EXT: /* extends any prog */
2903 return true;
2904 default:
2905 return false;
2906 }
2907}
2908
2909static int bpf_prog_mark_insn_arrays_ready(struct bpf_prog *prog)
2910{
2911 int err;
2912 int i;
2913
2914 for (i = 0; i < prog->aux->used_map_cnt; i++) {
2915 if (prog->aux->used_maps[i]->map_type != BPF_MAP_TYPE_INSN_ARRAY)
2916 continue;
2917
2918 err = bpf_insn_array_ready(prog->aux->used_maps[i]);
2919 if (err)
2920 return err;
2921 }
2922
2923 return 0;
2924}
2925
2926extern int bpf_multi_func(void);
2927int __init __used bpf_multi_func(void) { return 0; }
2928
2929BTF_ID_LIST_GLOBAL_SINGLE(bpf_multi_func_btf_id, func, bpf_multi_func)
2930
2931/* last field in 'union bpf_attr' used by this command */
2932#define BPF_PROG_LOAD_LAST_FIELD keyring_id
2933
2934static int bpf_prog_load(union bpf_attr *attr, bpfptr_t uattr, struct bpf_log_attr *attr_log)
2935{
2936 enum bpf_prog_type type = attr->prog_type;
2937 struct bpf_prog *prog, *dst_prog = NULL;
2938 struct btf *attach_btf = NULL;
2939 struct bpf_token *token = NULL;
2940 bool bpf_cap;
2941 int err;
2942 char license[128];
2943 bool multi_func;
2944
2945 if (CHECK_ATTR(BPF_PROG_LOAD))
2946 return -EINVAL;
2947
2948 if (attr->prog_flags & ~(BPF_F_STRICT_ALIGNMENT |
2949 BPF_F_ANY_ALIGNMENT |
2950 BPF_F_TEST_STATE_FREQ |
2951 BPF_F_SLEEPABLE |
2952 BPF_F_TEST_RND_HI32 |
2953 BPF_F_XDP_HAS_FRAGS |
2954 BPF_F_XDP_DEV_BOUND_ONLY |
2955 BPF_F_TEST_REG_INVARIANTS |
2956 BPF_F_TOKEN_FD))
2957 return -EINVAL;
2958
2959 bpf_prog_load_fixup_attach_type(attr);
2960
2961 if (attr->prog_flags & BPF_F_TOKEN_FD) {
2962 token = bpf_token_get_from_fd(attr->prog_token_fd);
2963 if (IS_ERR(token))
2964 return PTR_ERR(token);
2965 /* if current token doesn't grant prog loading permissions,
2966 * then we can't use this token, so ignore it and rely on
2967 * system-wide capabilities checks
2968 */
2969 if (!bpf_token_allow_cmd(token, BPF_PROG_LOAD) ||
2970 !bpf_token_allow_prog_type(token, attr->prog_type,
2971 attr->expected_attach_type)) {
2972 bpf_token_put(token);
2973 token = NULL;
2974 }
2975 }
2976
2977 bpf_cap = bpf_token_capable(token, CAP_BPF);
2978 err = -EPERM;
2979
2980 if (!IS_ENABLED(CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS) &&
2981 (attr->prog_flags & BPF_F_ANY_ALIGNMENT) &&
2982 !bpf_cap)
2983 goto put_token;
2984
2985 /* Intent here is for unprivileged_bpf_disabled to block BPF program
2986 * creation for unprivileged users; other actions depend
2987 * on fd availability and access to bpffs, so are dependent on
2988 * object creation success. Even with unprivileged BPF disabled,
2989 * capability checks are still carried out for these
2990 * and other operations.
2991 */
2992 if (sysctl_unprivileged_bpf_disabled && !bpf_cap)
2993 goto put_token;
2994
2995 if (attr->insn_cnt == 0 ||
2996 attr->insn_cnt > (bpf_cap ? BPF_COMPLEXITY_LIMIT_INSNS : BPF_MAXINSNS)) {
2997 err = -E2BIG;
2998 goto put_token;
2999 }
3000 if (type != BPF_PROG_TYPE_SOCKET_FILTER &&
3001 type != BPF_PROG_TYPE_CGROUP_SKB &&
3002 !bpf_cap)
3003 goto put_token;
3004
3005 if (is_net_admin_prog_type(type) && !bpf_token_capable(token, CAP_NET_ADMIN))
3006 goto put_token;
3007 if (is_perfmon_prog_type(type) && !bpf_token_capable(token, CAP_PERFMON))
3008 goto put_token;
3009
3010 multi_func = is_tracing_multi(attr->expected_attach_type);
3011
3012 /* attach_prog_fd/attach_btf_obj_fd can specify fd of either bpf_prog
3013 * or btf, we need to check which one it is
3014 */
3015 if (attr->attach_prog_fd) {
3016 dst_prog = bpf_prog_get(attr->attach_prog_fd);
3017 if (IS_ERR(dst_prog)) {
3018 dst_prog = NULL;
3019 attach_btf = btf_get_by_fd(attr->attach_btf_obj_fd);
3020 if (IS_ERR(attach_btf)) {
3021 err = -EINVAL;
3022 goto put_token;
3023 }
3024 if (!btf_is_kernel(attach_btf)) {
3025 /* attaching through specifying bpf_prog's BTF
3026 * objects directly might be supported eventually
3027 */
3028 btf_put(attach_btf);
3029 err = -ENOTSUPP;
3030 goto put_token;
3031 }
3032 }
3033 } else if (attr->attach_btf_id || multi_func) {
3034 /* fall back to vmlinux BTF, if BTF type ID is specified */
3035 attach_btf = bpf_get_btf_vmlinux();
3036 if (IS_ERR(attach_btf)) {
3037 err = PTR_ERR(attach_btf);
3038 goto put_token;
3039 }
3040 if (!attach_btf) {
3041 err = -EINVAL;
3042 goto put_token;
3043 }
3044 btf_get(attach_btf);
3045 }
3046
3047 if (bpf_prog_load_check_attach(type, attr->expected_attach_type,
3048 attach_btf, attr->attach_btf_id,
3049 dst_prog, multi_func)) {
3050 if (dst_prog)
3051 bpf_prog_put(dst_prog);
3052 if (attach_btf)
3053 btf_put(attach_btf);
3054 err = -EINVAL;
3055 goto put_token;
3056 }
3057
3058 /* plain bpf_prog allocation */
3059 prog = bpf_prog_alloc(bpf_prog_size(attr->insn_cnt), GFP_USER);
3060 if (!prog) {
3061 if (dst_prog)
3062 bpf_prog_put(dst_prog);
3063 if (attach_btf)
3064 btf_put(attach_btf);
3065 err = -EINVAL;
3066 goto put_token;
3067 }
3068
3069 prog->expected_attach_type = attr->expected_attach_type;
3070 prog->sleepable = !!(attr->prog_flags & BPF_F_SLEEPABLE);
3071 prog->aux->attach_btf = attach_btf;
3072 prog->aux->attach_btf_id = multi_func ? bpf_multi_func_btf_id[0] : attr->attach_btf_id;
3073 prog->aux->dst_prog = dst_prog;
3074 if (dst_prog) {
3075 prog->aux->saved_dst_prog_type = dst_prog->type;
3076 prog->aux->saved_dst_attach_type = dst_prog->expected_attach_type;
3077 }
3078 prog->aux->dev_bound = !!attr->prog_ifindex;
3079 prog->aux->xdp_has_frags = attr->prog_flags & BPF_F_XDP_HAS_FRAGS;
3080
3081 /* move token into prog->aux, reuse taken refcnt */
3082 prog->aux->token = token;
3083 token = NULL;
3084
3085 prog->aux->user = get_current_user();
3086 prog->len = attr->insn_cnt;
3087
3088 err = -EFAULT;
3089 if (copy_from_bpfptr(prog->insns,
3090 make_bpfptr(attr->insns, uattr.is_kernel),
3091 bpf_prog_insn_size(prog)) != 0)
3092 goto free_prog;
3093 /* copy eBPF program license from user space */
3094 if (strncpy_from_bpfptr(license,
3095 make_bpfptr(attr->license, uattr.is_kernel),
3096 sizeof(license) - 1) < 0)
3097 goto free_prog;
3098 license[sizeof(license) - 1] = 0;
3099
3100 /* eBPF programs must be GPL compatible to use GPL-ed functions */
3101 prog->gpl_compatible = license_is_gpl_compatible(license) ? 1 : 0;
3102 prog->aux->sig.keyring_type = BPF_SIG_KEYRING_NONE;
3103 prog->aux->sig.verdict = BPF_SIG_UNSIGNED;
3104 prog->orig_prog = NULL;
3105 prog->jited = 0;
3106
3107 atomic64_set(&prog->aux->refcnt, 1);
3108
3109 if (bpf_prog_is_dev_bound(prog->aux)) {
3110 err = bpf_prog_dev_bound_init(prog, attr);
3111 if (err)
3112 goto free_prog;
3113 }
3114
3115 if (type == BPF_PROG_TYPE_EXT && dst_prog &&
3116 bpf_prog_is_dev_bound(dst_prog->aux)) {
3117 err = bpf_prog_dev_bound_inherit(prog, dst_prog);
3118 if (err)
3119 goto free_prog;
3120 }
3121
3122 /*
3123 * Bookkeeping for managing the program attachment chain.
3124 *
3125 * It might be tempting to set attach_tracing_prog flag at the attachment
3126 * time, but this will not prevent from loading bunch of tracing prog
3127 * first, then attach them one to another.
3128 *
3129 * The flag attach_tracing_prog is set for the whole program lifecycle, and
3130 * doesn't have to be cleared in bpf_tracing_link_release, since tracing
3131 * programs cannot change attachment target.
3132 */
3133 if (type == BPF_PROG_TYPE_TRACING && dst_prog &&
3134 dst_prog->type == BPF_PROG_TYPE_TRACING) {
3135 prog->aux->attach_tracing_prog = true;
3136 }
3137
3138 /* find program type: socket_filter vs tracing_filter */
3139 err = find_prog_type(type, prog);
3140 if (err < 0)
3141 goto free_prog;
3142
3143 prog->aux->load_time = ktime_get_boottime_ns();
3144 err = bpf_obj_name_cpy(prog->aux->name, attr->prog_name,
3145 sizeof(attr->prog_name));
3146 if (err < 0)
3147 goto free_prog;
3148
3149 /* run eBPF verifier */
3150 err = bpf_check(&prog, attr, uattr, attr_log);
3151 if (err < 0)
3152 goto free_used_maps;
3153
3154 err = bpf_prog_mark_insn_arrays_ready(prog);
3155 if (err < 0)
3156 goto free_used_maps;
3157
3158 err = bpf_prog_alloc_id(prog);
3159 if (err)
3160 goto free_used_maps;
3161
3162 /* Upon success of bpf_prog_alloc_id(), the BPF prog is
3163 * effectively publicly exposed. However, retrieving via
3164 * bpf_prog_get_fd_by_id() will take another reference,
3165 * therefore it cannot be gone underneath us.
3166 *
3167 * Only for the time /after/ successful bpf_prog_new_fd()
3168 * and before returning to userspace, we might just hold
3169 * one reference and any parallel close on that fd could
3170 * rip everything out. Hence, below notifications must
3171 * happen before bpf_prog_new_fd().
3172 *
3173 * Also, any failure handling from this point onwards must
3174 * be using bpf_prog_put() given the program is exposed.
3175 */
3176 bpf_prog_kallsyms_add(prog);
3177 perf_event_bpf_event(prog, PERF_BPF_EVENT_PROG_LOAD, 0);
3178 bpf_audit_prog(prog, BPF_AUDIT_LOAD);
3179
3180 err = bpf_prog_new_fd(prog);
3181 if (err < 0)
3182 bpf_prog_put(prog);
3183 return err;
3184
3185free_used_maps:
3186 /* In case we have subprogs, we need to wait for a grace
3187 * period before we can tear down JIT memory since symbols
3188 * are already exposed under kallsyms.
3189 */
3190 __bpf_prog_put_noref(prog, prog->aux->real_func_cnt);
3191 return err;
3192
3193free_prog:
3194 free_uid(prog->aux->user);
3195 if (prog->aux->attach_btf)
3196 btf_put(prog->aux->attach_btf);
3197 bpf_prog_free(prog);
3198put_token:
3199 bpf_token_put(token);
3200 return err;
3201}
3202
3203#define BPF_OBJ_LAST_FIELD path_fd
3204
3205static int bpf_obj_pin(const union bpf_attr *attr)
3206{
3207 int path_fd;
3208
3209 if (CHECK_ATTR(BPF_OBJ) || attr->file_flags & ~BPF_F_PATH_FD)
3210 return -EINVAL;
3211
3212 /* path_fd has to be accompanied by BPF_F_PATH_FD flag */
3213 if (!(attr->file_flags & BPF_F_PATH_FD) && attr->path_fd)
3214 return -EINVAL;
3215
3216 path_fd = attr->file_flags & BPF_F_PATH_FD ? attr->path_fd : AT_FDCWD;
3217 return bpf_obj_pin_user(attr->bpf_fd, path_fd,
3218 u64_to_user_ptr(attr->pathname));
3219}
3220
3221static int bpf_obj_get(const union bpf_attr *attr)
3222{
3223 int path_fd;
3224
3225 if (CHECK_ATTR(BPF_OBJ) || attr->bpf_fd != 0 ||
3226 attr->file_flags & ~(BPF_OBJ_FLAG_MASK | BPF_F_PATH_FD))
3227 return -EINVAL;
3228
3229 /* path_fd has to be accompanied by BPF_F_PATH_FD flag */
3230 if (!(attr->file_flags & BPF_F_PATH_FD) && attr->path_fd)
3231 return -EINVAL;
3232
3233 path_fd = attr->file_flags & BPF_F_PATH_FD ? attr->path_fd : AT_FDCWD;
3234 return bpf_obj_get_user(path_fd, u64_to_user_ptr(attr->pathname),
3235 attr->file_flags);
3236}
3237
3238/* bpf_link_init_sleepable() allows to specify whether BPF link itself has
3239 * "sleepable" semantics, which normally would mean that BPF link's attach
3240 * hook can dereference link or link's underlying program for some time after
3241 * detachment due to RCU Tasks Trace-based lifetime protection scheme.
3242 * BPF program itself can be non-sleepable, yet, because it's transitively
3243 * reachable through BPF link, its freeing has to be delayed until after RCU
3244 * Tasks Trace GP.
3245 */
3246void bpf_link_init_sleepable(struct bpf_link *link, enum bpf_link_type type,
3247 const struct bpf_link_ops *ops, struct bpf_prog *prog,
3248 enum bpf_attach_type attach_type, bool sleepable)
3249{
3250 WARN_ON(ops->dealloc && ops->dealloc_deferred);
3251 atomic64_set(&link->refcnt, 1);
3252 link->type = type;
3253 link->sleepable = sleepable;
3254 link->id = 0;
3255 link->ops = ops;
3256 link->prog = prog;
3257 link->attach_type = attach_type;
3258}
3259
3260void bpf_link_init(struct bpf_link *link, enum bpf_link_type type,
3261 const struct bpf_link_ops *ops, struct bpf_prog *prog,
3262 enum bpf_attach_type attach_type)
3263{
3264 bpf_link_init_sleepable(link, type, ops, prog, attach_type, false);
3265}
3266
3267void bpf_tramp_link_init(struct bpf_tramp_link *link, enum bpf_link_type type,
3268 const struct bpf_link_ops *ops, struct bpf_prog *prog,
3269 enum bpf_attach_type attach_type, u64 cookie)
3270{
3271 bpf_link_init(&link->link, type, ops, prog, attach_type);
3272 link->node.link = &link->link;
3273 link->node.cookie = cookie;
3274}
3275
3276static void bpf_link_free_id(int id)
3277{
3278 if (!id)
3279 return;
3280
3281 spin_lock_bh(&link_idr_lock);
3282 idr_remove(&link_idr, id);
3283 spin_unlock_bh(&link_idr_lock);
3284}
3285
3286/* Clean up bpf_link and corresponding anon_inode file and FD. After
3287 * anon_inode is created, bpf_link can't be just kfree()'d due to deferred
3288 * anon_inode's release() call. This helper marks bpf_link as
3289 * defunct, releases anon_inode file and puts reserved FD. bpf_prog's refcnt
3290 * is not decremented, it's the responsibility of a calling code that failed
3291 * to complete bpf_link initialization.
3292 * This helper eventually calls link's dealloc callback, but does not call
3293 * link's release callback.
3294 */
3295void bpf_link_cleanup(struct bpf_link_primer *primer)
3296{
3297 primer->link->prog = NULL;
3298 bpf_link_free_id(primer->id);
3299 fput(primer->file);
3300 put_unused_fd(primer->fd);
3301}
3302
3303void bpf_link_inc(struct bpf_link *link)
3304{
3305 atomic64_inc(&link->refcnt);
3306}
3307
3308static void bpf_link_dealloc(struct bpf_link *link)
3309{
3310 /* now that we know that bpf_link itself can't be reached, put underlying BPF program */
3311 if (link->prog)
3312 bpf_prog_put(link->prog);
3313
3314 /* free bpf_link and its containing memory */
3315 if (link->ops->dealloc_deferred)
3316 link->ops->dealloc_deferred(link);
3317 else
3318 link->ops->dealloc(link);
3319}
3320
3321static void bpf_link_defer_dealloc_rcu_gp(struct rcu_head *rcu)
3322{
3323 struct bpf_link *link = container_of(rcu, struct bpf_link, rcu);
3324
3325 bpf_link_dealloc(link);
3326}
3327
3328static bool bpf_link_is_tracepoint(struct bpf_link *link)
3329{
3330 /*
3331 * Only these combinations support a tracepoint bpf_link.
3332 * BPF_LINK_TYPE_TRACING raw_tp progs are hardcoded to use
3333 * bpf_raw_tp_link_lops and thus dealloc_deferred(), see
3334 * bpf_raw_tp_link_attach().
3335 */
3336 return link->type == BPF_LINK_TYPE_RAW_TRACEPOINT ||
3337 (link->type == BPF_LINK_TYPE_TRACING && link->attach_type == BPF_TRACE_RAW_TP);
3338}
3339
3340/* bpf_link_free is guaranteed to be called from process context */
3341static void bpf_link_free(struct bpf_link *link)
3342{
3343 const struct bpf_link_ops *ops = link->ops;
3344
3345 bpf_link_free_id(link->id);
3346 /* detach BPF program, clean up used resources */
3347 if (link->prog)
3348 ops->release(link);
3349 if (ops->dealloc_deferred) {
3350 /*
3351 * Schedule BPF link deallocation, which will only then
3352 * trigger putting BPF program refcount.
3353 * If underlying BPF program is sleepable or BPF link's target
3354 * attach hookpoint is sleepable or otherwise requires RCU GPs
3355 * to ensure link and its underlying BPF program is not
3356 * reachable anymore, we need to first wait for RCU tasks
3357 * trace sync, and then go through "classic" RCU grace period.
3358 *
3359 * For tracepoint BPF links, we need to go through SRCU grace
3360 * period wait instead when non-faultable tracepoint is used. We
3361 * don't need to chain SRCU grace period waits, however, for the
3362 * faultable case, since it exclusively uses RCU Tasks Trace.
3363 */
3364 if (link->sleepable || (link->prog && link->prog->sleepable))
3365 /* RCU Tasks Trace grace period implies RCU grace period. */
3366 call_rcu_tasks_trace(&link->rcu, bpf_link_defer_dealloc_rcu_gp);
3367 /* We need to do a SRCU grace period wait for non-faultable tracepoint BPF links. */
3368 else if (bpf_link_is_tracepoint(link))
3369 call_tracepoint_unregister_atomic(&link->rcu, bpf_link_defer_dealloc_rcu_gp);
3370 else
3371 call_rcu(&link->rcu, bpf_link_defer_dealloc_rcu_gp);
3372 } else if (ops->dealloc) {
3373 bpf_link_dealloc(link);
3374 }
3375}
3376
3377static void bpf_link_put_deferred(struct work_struct *work)
3378{
3379 struct bpf_link *link = container_of(work, struct bpf_link, work);
3380
3381 bpf_link_free(link);
3382}
3383
3384/* bpf_link_put might be called from atomic context. It needs to be called
3385 * from sleepable context in order to acquire sleeping locks during the process.
3386 */
3387void bpf_link_put(struct bpf_link *link)
3388{
3389 if (!atomic64_dec_and_test(&link->refcnt))
3390 return;
3391
3392 INIT_WORK(&link->work, bpf_link_put_deferred);
3393 schedule_work(&link->work);
3394}
3395EXPORT_SYMBOL(bpf_link_put);
3396
3397static void bpf_link_put_direct(struct bpf_link *link)
3398{
3399 if (!atomic64_dec_and_test(&link->refcnt))
3400 return;
3401 bpf_link_free(link);
3402}
3403
3404static int bpf_link_release(struct inode *inode, struct file *filp)
3405{
3406 struct bpf_link *link = filp->private_data;
3407
3408 bpf_link_put_direct(link);
3409 return 0;
3410}
3411
3412#ifdef CONFIG_PROC_FS
3413#define BPF_PROG_TYPE(_id, _name, prog_ctx_type, kern_ctx_type)
3414#define BPF_MAP_TYPE(_id, _ops)
3415#define BPF_LINK_TYPE(_id, _name) [_id] = #_name,
3416static const char *bpf_link_type_strs[] = {
3417 [BPF_LINK_TYPE_UNSPEC] = "<invalid>",
3418#include <linux/bpf_types.h>
3419};
3420#undef BPF_PROG_TYPE
3421#undef BPF_MAP_TYPE
3422#undef BPF_LINK_TYPE
3423
3424static void bpf_link_show_fdinfo(struct seq_file *m, struct file *filp)
3425{
3426 const struct bpf_link *link = filp->private_data;
3427 const struct bpf_prog *prog;
3428 enum bpf_link_type type = link->type;
3429 char prog_tag[sizeof(prog->tag) * 2 + 1] = { };
3430 u32 prog_id = 0;
3431
3432 if (type < ARRAY_SIZE(bpf_link_type_strs) && bpf_link_type_strs[type]) {
3433 if (link->type == BPF_LINK_TYPE_KPROBE_MULTI)
3434 seq_printf(m, "link_type:\t%s\n", link->flags == BPF_F_KPROBE_MULTI_RETURN ?
3435 "kretprobe_multi" : "kprobe_multi");
3436 else if (link->type == BPF_LINK_TYPE_UPROBE_MULTI)
3437 seq_printf(m, "link_type:\t%s\n", link->flags & BPF_F_UPROBE_MULTI_RETURN ?
3438 "uretprobe_multi" : "uprobe_multi");
3439 else
3440 seq_printf(m, "link_type:\t%s\n", bpf_link_type_strs[type]);
3441 } else {
3442 WARN_ONCE(1, "missing BPF_LINK_TYPE(...) for link type %u\n", type);
3443 seq_printf(m, "link_type:\t<%u>\n", type);
3444 }
3445 seq_printf(m, "link_id:\t%u\n", link->id);
3446
3447 rcu_read_lock();
3448 prog = READ_ONCE(link->prog);
3449 if (prog) {
3450 bin2hex(prog_tag, prog->tag, sizeof(prog->tag));
3451 prog_id = prog->aux->id;
3452 }
3453 rcu_read_unlock();
3454
3455 if (prog) {
3456 seq_printf(m,
3457 "prog_tag:\t%s\n"
3458 "prog_id:\t%u\n",
3459 prog_tag,
3460 prog_id);
3461 }
3462 if (link->ops->show_fdinfo)
3463 link->ops->show_fdinfo(link, m);
3464}
3465#endif
3466
3467static __poll_t bpf_link_poll(struct file *file, struct poll_table_struct *pts)
3468{
3469 struct bpf_link *link = file->private_data;
3470
3471 return link->ops->poll(file, pts);
3472}
3473
3474static const struct file_operations bpf_link_fops = {
3475#ifdef CONFIG_PROC_FS
3476 .show_fdinfo = bpf_link_show_fdinfo,
3477#endif
3478 .release = bpf_link_release,
3479 .read = bpf_dummy_read,
3480 .write = bpf_dummy_write,
3481};
3482
3483static const struct file_operations bpf_link_fops_poll = {
3484#ifdef CONFIG_PROC_FS
3485 .show_fdinfo = bpf_link_show_fdinfo,
3486#endif
3487 .release = bpf_link_release,
3488 .read = bpf_dummy_read,
3489 .write = bpf_dummy_write,
3490 .poll = bpf_link_poll,
3491};
3492
3493static int bpf_link_alloc_id(struct bpf_link *link)
3494{
3495 int id;
3496
3497 idr_preload(GFP_KERNEL);
3498 spin_lock_bh(&link_idr_lock);
3499 id = idr_alloc_cyclic(&link_idr, link, 1, INT_MAX, GFP_ATOMIC);
3500 spin_unlock_bh(&link_idr_lock);
3501 idr_preload_end();
3502
3503 return id;
3504}
3505
3506/* Prepare bpf_link to be exposed to user-space by allocating anon_inode file,
3507 * reserving unused FD and allocating ID from link_idr. This is to be paired
3508 * with bpf_link_settle() to install FD and ID and expose bpf_link to
3509 * user-space, if bpf_link is successfully attached. If not, bpf_link and
3510 * pre-allocated resources are to be freed with bpf_cleanup() call. All the
3511 * transient state is passed around in struct bpf_link_primer.
3512 * This is preferred way to create and initialize bpf_link, especially when
3513 * there are complicated and expensive operations in between creating bpf_link
3514 * itself and attaching it to BPF hook. By using bpf_link_prime() and
3515 * bpf_link_settle() kernel code using bpf_link doesn't have to perform
3516 * expensive (and potentially failing) roll back operations in a rare case
3517 * that file, FD, or ID can't be allocated.
3518 */
3519int bpf_link_prime(struct bpf_link *link, struct bpf_link_primer *primer)
3520{
3521 struct file *file;
3522 int fd, id;
3523
3524 fd = get_unused_fd_flags(O_CLOEXEC);
3525 if (fd < 0)
3526 return fd;
3527
3528 id = bpf_link_alloc_id(link);
3529 if (id < 0) {
3530 put_unused_fd(fd);
3531 return id;
3532 }
3533
3534 file = anon_inode_getfile("bpf_link",
3535 link->ops->poll ? &bpf_link_fops_poll : &bpf_link_fops,
3536 link, O_CLOEXEC);
3537 if (IS_ERR(file)) {
3538 bpf_link_free_id(id);
3539 put_unused_fd(fd);
3540 return PTR_ERR(file);
3541 }
3542
3543 primer->link = link;
3544 primer->file = file;
3545 primer->fd = fd;
3546 primer->id = id;
3547 return 0;
3548}
3549
3550int bpf_link_settle(struct bpf_link_primer *primer)
3551{
3552 /* make bpf_link fetchable by ID */
3553 spin_lock_bh(&link_idr_lock);
3554 primer->link->id = primer->id;
3555 spin_unlock_bh(&link_idr_lock);
3556 /* make bpf_link fetchable by FD */
3557 fd_install(primer->fd, primer->file);
3558 /* pass through installed FD */
3559 return primer->fd;
3560}
3561
3562int bpf_link_new_fd(struct bpf_link *link)
3563{
3564 return anon_inode_getfd("bpf-link",
3565 link->ops->poll ? &bpf_link_fops_poll : &bpf_link_fops,
3566 link, O_CLOEXEC);
3567}
3568
3569struct bpf_link *bpf_link_get_from_fd(u32 ufd)
3570{
3571 CLASS(fd, f)(ufd);
3572 struct bpf_link *link;
3573
3574 if (fd_empty(f))
3575 return ERR_PTR(-EBADF);
3576 if (fd_file(f)->f_op != &bpf_link_fops && fd_file(f)->f_op != &bpf_link_fops_poll)
3577 return ERR_PTR(-EINVAL);
3578
3579 link = fd_file(f)->private_data;
3580 bpf_link_inc(link);
3581 return link;
3582}
3583EXPORT_SYMBOL_NS(bpf_link_get_from_fd, "BPF_INTERNAL");
3584
3585static void bpf_tracing_link_release(struct bpf_link *link)
3586{
3587 struct bpf_tracing_link *tr_link =
3588 container_of(link, struct bpf_tracing_link, link.link);
3589 int err;
3590
3591 err = bpf_trampoline_unlink_prog(&tr_link->link.node,
3592 tr_link->trampoline,
3593 tr_link->tgt_prog);
3594 WARN_ONCE(err, "bpf_trampoline_unlink_prog failed: %d\n", err);
3595
3596 bpf_trampoline_put(tr_link->trampoline);
3597
3598 /* tgt_prog is NULL if target is a kernel function */
3599 if (tr_link->tgt_prog)
3600 bpf_prog_put(tr_link->tgt_prog);
3601}
3602
3603static void bpf_tracing_link_dealloc(struct bpf_link *link)
3604{
3605 struct bpf_tracing_link *tr_link = container_of(link, struct bpf_tracing_link, link.link);
3606
3607 kfree(tr_link);
3608}
3609
3610static void bpf_tracing_link_show_fdinfo(const struct bpf_link *link,
3611 struct seq_file *seq)
3612{
3613 struct bpf_tracing_link *tr_link = container_of(link, struct bpf_tracing_link, link.link);
3614
3615 u32 target_btf_id, target_obj_id;
3616
3617 bpf_trampoline_unpack_key(tr_link->trampoline->key,
3618 &target_obj_id, &target_btf_id);
3619 seq_printf(seq,
3620 "attach_type:\t%d\n"
3621 "target_obj_id:\t%u\n"
3622 "target_btf_id:\t%u\n"
3623 "cookie:\t%llu\n",
3624 link->attach_type,
3625 target_obj_id,
3626 target_btf_id,
3627 tr_link->link.node.cookie);
3628}
3629
3630static int bpf_tracing_link_fill_link_info(const struct bpf_link *link,
3631 struct bpf_link_info *info)
3632{
3633 struct bpf_tracing_link *tr_link = container_of(link, struct bpf_tracing_link, link.link);
3634
3635 info->tracing.attach_type = link->attach_type;
3636 info->tracing.cookie = tr_link->link.node.cookie;
3637 bpf_trampoline_unpack_key(tr_link->trampoline->key,
3638 &info->tracing.target_obj_id,
3639 &info->tracing.target_btf_id);
3640
3641 return 0;
3642}
3643
3644static const struct bpf_link_ops bpf_tracing_link_lops = {
3645 .release = bpf_tracing_link_release,
3646 .dealloc = bpf_tracing_link_dealloc,
3647 .show_fdinfo = bpf_tracing_link_show_fdinfo,
3648 .fill_link_info = bpf_tracing_link_fill_link_info,
3649};
3650
3651static int bpf_tracing_prog_attach(struct bpf_prog *prog,
3652 int tgt_prog_fd,
3653 u32 btf_id,
3654 u64 bpf_cookie,
3655 enum bpf_attach_type attach_type)
3656{
3657 struct bpf_link_primer link_primer;
3658 struct bpf_prog *tgt_prog = NULL;
3659 struct bpf_trampoline *tr = NULL;
3660 struct bpf_tracing_link *link;
3661 u64 key = 0;
3662 int err;
3663
3664 switch (prog->type) {
3665 case BPF_PROG_TYPE_TRACING:
3666 if (prog->expected_attach_type != BPF_TRACE_FENTRY &&
3667 prog->expected_attach_type != BPF_TRACE_FEXIT &&
3668 prog->expected_attach_type != BPF_TRACE_FSESSION &&
3669 prog->expected_attach_type != BPF_MODIFY_RETURN) {
3670 err = -EINVAL;
3671 goto out_put_prog;
3672 }
3673 break;
3674 case BPF_PROG_TYPE_EXT:
3675 if (prog->expected_attach_type != 0) {
3676 err = -EINVAL;
3677 goto out_put_prog;
3678 }
3679 break;
3680 case BPF_PROG_TYPE_LSM:
3681 if (prog->expected_attach_type != BPF_LSM_MAC) {
3682 err = -EINVAL;
3683 goto out_put_prog;
3684 }
3685 break;
3686 default:
3687 err = -EINVAL;
3688 goto out_put_prog;
3689 }
3690
3691 if (!!tgt_prog_fd != !!btf_id) {
3692 err = -EINVAL;
3693 goto out_put_prog;
3694 }
3695
3696 if (tgt_prog_fd) {
3697 /*
3698 * For now we only allow new targets for BPF_PROG_TYPE_EXT. If this
3699 * part would be changed to implement the same for
3700 * BPF_PROG_TYPE_TRACING, do not forget to update the way how
3701 * attach_tracing_prog flag is set.
3702 */
3703 if (prog->type != BPF_PROG_TYPE_EXT) {
3704 err = -EINVAL;
3705 goto out_put_prog;
3706 }
3707
3708 tgt_prog = bpf_prog_get(tgt_prog_fd);
3709 if (IS_ERR(tgt_prog)) {
3710 err = PTR_ERR(tgt_prog);
3711 tgt_prog = NULL;
3712 goto out_put_prog;
3713 }
3714
3715 key = bpf_trampoline_compute_key(tgt_prog, NULL, btf_id);
3716 }
3717
3718 link = kzalloc_obj(*link, GFP_USER);
3719 if (!link) {
3720 err = -ENOMEM;
3721 goto out_put_prog;
3722 }
3723 bpf_tramp_link_init(&link->link, BPF_LINK_TYPE_TRACING,
3724 &bpf_tracing_link_lops, prog, attach_type, bpf_cookie);
3725
3726 if (prog->expected_attach_type == BPF_TRACE_FSESSION) {
3727 link->fexit.link = &link->link.link;
3728 link->fexit.cookie = bpf_cookie;
3729 }
3730
3731 mutex_lock(&prog->aux->dst_mutex);
3732
3733 /* There are a few possible cases here:
3734 *
3735 * - if prog->aux->dst_trampoline is set, the program was just loaded
3736 * and not yet attached to anything, so we can use the values stored
3737 * in prog->aux
3738 *
3739 * - if prog->aux->dst_trampoline is NULL, the program has already been
3740 * attached to a target and its initial target was cleared (below)
3741 *
3742 * - if tgt_prog != NULL, the caller specified tgt_prog_fd +
3743 * target_btf_id using the link_create API.
3744 *
3745 * - if tgt_prog == NULL when this function was called using the old
3746 * raw_tracepoint_open API, and we need a target from prog->aux
3747 *
3748 * - if prog->aux->dst_trampoline and tgt_prog is NULL, the program
3749 * was detached and is going for re-attachment.
3750 *
3751 * - if prog->aux->dst_trampoline is NULL and tgt_prog and prog->aux->attach_btf
3752 * are NULL, then program was already attached and user did not provide
3753 * tgt_prog_fd so we have no way to find out or create trampoline
3754 */
3755 if (!prog->aux->dst_trampoline && !tgt_prog) {
3756 /*
3757 * Allow re-attach for TRACING and LSM programs. If it's
3758 * currently linked, bpf_trampoline_link_prog will fail.
3759 * EXT programs need to specify tgt_prog_fd, so they
3760 * re-attach in separate code path.
3761 */
3762 if (prog->type != BPF_PROG_TYPE_TRACING &&
3763 prog->type != BPF_PROG_TYPE_LSM) {
3764 err = -EINVAL;
3765 goto out_unlock;
3766 }
3767 /* We can allow re-attach only if we have valid attach_btf. */
3768 if (!prog->aux->attach_btf) {
3769 err = -EINVAL;
3770 goto out_unlock;
3771 }
3772 btf_id = prog->aux->attach_btf_id;
3773 key = bpf_trampoline_compute_key(NULL, prog->aux->attach_btf, btf_id);
3774 }
3775
3776 if (!prog->aux->dst_trampoline ||
3777 (key && key != prog->aux->dst_trampoline->key)) {
3778 /* If there is no saved target, or the specified target is
3779 * different from the destination specified at load time, we
3780 * need a new trampoline and a check for compatibility
3781 */
3782 struct bpf_attach_target_info tgt_info = {};
3783
3784 err = bpf_check_attach_target(NULL, prog, tgt_prog, btf_id,
3785 &tgt_info);
3786 if (err)
3787 goto out_unlock;
3788
3789 if (tgt_info.tgt_mod) {
3790 module_put(prog->aux->mod);
3791 prog->aux->mod = tgt_info.tgt_mod;
3792 }
3793
3794 tr = bpf_trampoline_get(key, &tgt_info);
3795 if (!tr) {
3796 err = -ENOMEM;
3797 goto out_unlock;
3798 }
3799 } else {
3800 /* The caller didn't specify a target, or the target was the
3801 * same as the destination supplied during program load. This
3802 * means we can reuse the trampoline and reference from program
3803 * load time, and there is no need to allocate a new one. This
3804 * can only happen once for any program, as the saved values in
3805 * prog->aux are cleared below.
3806 */
3807 tr = prog->aux->dst_trampoline;
3808 tgt_prog = prog->aux->dst_prog;
3809 }
3810 /*
3811 * It is to prevent modifying struct pt_regs via kprobe_write_ctx=true
3812 * freplace prog. Without this check, kprobe_write_ctx=true freplace
3813 * prog is allowed to attach to kprobe_write_ctx=false kprobe prog, and
3814 * then modify the registers of the kprobe prog's target kernel
3815 * function.
3816 *
3817 * This also blocks the combination of uprobe+freplace, because it is
3818 * unable to recognize the use of the tgt_prog as an uprobe or a kprobe
3819 * by tgt_prog itself. At attach time, uprobe/kprobe is recognized by
3820 * the target perf event flags in __perf_event_set_bpf_prog().
3821 */
3822 if (prog->type == BPF_PROG_TYPE_EXT &&
3823 prog->aux->kprobe_write_ctx != tgt_prog->aux->kprobe_write_ctx) {
3824 err = -EINVAL;
3825 goto out_unlock;
3826 }
3827
3828 err = bpf_link_prime(&link->link.link, &link_primer);
3829 if (err)
3830 goto out_unlock;
3831
3832 err = bpf_trampoline_link_prog(&link->link.node, tr, tgt_prog);
3833 if (err) {
3834 bpf_link_cleanup(&link_primer);
3835 link = NULL;
3836 goto out_unlock;
3837 }
3838
3839 link->tgt_prog = tgt_prog;
3840 link->trampoline = tr;
3841
3842 /* Always clear the trampoline and target prog from prog->aux to make
3843 * sure the original attach destination is not kept alive after a
3844 * program is (re-)attached to another target.
3845 */
3846 if (prog->aux->dst_prog &&
3847 (tgt_prog_fd || tr != prog->aux->dst_trampoline))
3848 /* got extra prog ref from syscall, or attaching to different prog */
3849 bpf_prog_put(prog->aux->dst_prog);
3850 if (prog->aux->dst_trampoline && tr != prog->aux->dst_trampoline)
3851 /* we allocated a new trampoline, so free the old one */
3852 bpf_trampoline_put(prog->aux->dst_trampoline);
3853
3854 prog->aux->dst_prog = NULL;
3855 prog->aux->dst_trampoline = NULL;
3856 mutex_unlock(&prog->aux->dst_mutex);
3857
3858 return bpf_link_settle(&link_primer);
3859out_unlock:
3860 if (tr && tr != prog->aux->dst_trampoline)
3861 bpf_trampoline_put(tr);
3862 mutex_unlock(&prog->aux->dst_mutex);
3863 kfree(link);
3864out_put_prog:
3865 if (tgt_prog_fd && tgt_prog)
3866 bpf_prog_put(tgt_prog);
3867 return err;
3868}
3869
3870static void bpf_raw_tp_link_release(struct bpf_link *link)
3871{
3872 struct bpf_raw_tp_link *raw_tp =
3873 container_of(link, struct bpf_raw_tp_link, link);
3874
3875 bpf_probe_unregister(raw_tp->btp, raw_tp);
3876 bpf_put_raw_tracepoint(raw_tp->btp);
3877}
3878
3879static void bpf_raw_tp_link_dealloc(struct bpf_link *link)
3880{
3881 struct bpf_raw_tp_link *raw_tp =
3882 container_of(link, struct bpf_raw_tp_link, link);
3883
3884 kfree(raw_tp);
3885}
3886
3887static void bpf_raw_tp_link_show_fdinfo(const struct bpf_link *link,
3888 struct seq_file *seq)
3889{
3890 struct bpf_raw_tp_link *raw_tp_link =
3891 container_of(link, struct bpf_raw_tp_link, link);
3892
3893 seq_printf(seq,
3894 "tp_name:\t%s\n"
3895 "cookie:\t%llu\n",
3896 raw_tp_link->btp->tp->name,
3897 raw_tp_link->cookie);
3898}
3899
3900static int bpf_copy_to_user(char __user *ubuf, const char *buf, u32 ulen,
3901 u32 len)
3902{
3903 if (ulen >= len + 1) {
3904 if (copy_to_user(ubuf, buf, len + 1))
3905 return -EFAULT;
3906 } else {
3907 char zero = '\0';
3908
3909 if (copy_to_user(ubuf, buf, ulen - 1))
3910 return -EFAULT;
3911 if (put_user(zero, ubuf + ulen - 1))
3912 return -EFAULT;
3913 return -ENOSPC;
3914 }
3915
3916 return 0;
3917}
3918
3919static int bpf_raw_tp_link_fill_link_info(const struct bpf_link *link,
3920 struct bpf_link_info *info)
3921{
3922 struct bpf_raw_tp_link *raw_tp_link =
3923 container_of(link, struct bpf_raw_tp_link, link);
3924 char __user *ubuf = u64_to_user_ptr(info->raw_tracepoint.tp_name);
3925 const char *tp_name = raw_tp_link->btp->tp->name;
3926 u32 ulen = info->raw_tracepoint.tp_name_len;
3927 size_t tp_len = strlen(tp_name);
3928
3929 if (!ulen ^ !ubuf)
3930 return -EINVAL;
3931
3932 info->raw_tracepoint.tp_name_len = tp_len + 1;
3933 info->raw_tracepoint.cookie = raw_tp_link->cookie;
3934
3935 if (!ubuf)
3936 return 0;
3937
3938 return bpf_copy_to_user(ubuf, tp_name, ulen, tp_len);
3939}
3940
3941static const struct bpf_link_ops bpf_raw_tp_link_lops = {
3942 .release = bpf_raw_tp_link_release,
3943 .dealloc_deferred = bpf_raw_tp_link_dealloc,
3944 .show_fdinfo = bpf_raw_tp_link_show_fdinfo,
3945 .fill_link_info = bpf_raw_tp_link_fill_link_info,
3946};
3947
3948#ifdef CONFIG_PERF_EVENTS
3949struct bpf_perf_link {
3950 struct bpf_link link;
3951 struct file *perf_file;
3952};
3953
3954static void bpf_perf_link_release(struct bpf_link *link)
3955{
3956 struct bpf_perf_link *perf_link = container_of(link, struct bpf_perf_link, link);
3957 struct perf_event *event = perf_link->perf_file->private_data;
3958
3959 perf_event_free_bpf_prog(event);
3960 fput(perf_link->perf_file);
3961}
3962
3963static void bpf_perf_link_dealloc(struct bpf_link *link)
3964{
3965 struct bpf_perf_link *perf_link = container_of(link, struct bpf_perf_link, link);
3966
3967 kfree(perf_link);
3968}
3969
3970static int bpf_perf_link_fill_common(const struct perf_event *event,
3971 char __user *uname, u32 *ulenp,
3972 u64 *probe_offset, u64 *probe_addr,
3973 u32 *fd_type, unsigned long *missed)
3974{
3975 const char *buf;
3976 u32 prog_id, ulen;
3977 size_t len;
3978 int err;
3979
3980 ulen = *ulenp;
3981 if (!ulen ^ !uname)
3982 return -EINVAL;
3983
3984 err = bpf_get_perf_event_info(event, &prog_id, fd_type, &buf,
3985 probe_offset, probe_addr, missed);
3986 if (err)
3987 return err;
3988
3989 if (buf) {
3990 len = strlen(buf);
3991 *ulenp = len + 1;
3992 } else {
3993 *ulenp = 1;
3994 }
3995 if (!uname)
3996 return 0;
3997
3998 if (buf) {
3999 err = bpf_copy_to_user(uname, buf, ulen, len);
4000 if (err)
4001 return err;
4002 } else {
4003 char zero = '\0';
4004
4005 if (put_user(zero, uname))
4006 return -EFAULT;
4007 }
4008 return 0;
4009}
4010
4011#ifdef CONFIG_KPROBE_EVENTS
4012static int bpf_perf_link_fill_kprobe(const struct perf_event *event,
4013 struct bpf_link_info *info)
4014{
4015 unsigned long missed;
4016 char __user *uname;
4017 u64 addr, offset;
4018 u32 ulen, type;
4019 int err;
4020
4021 uname = u64_to_user_ptr(info->perf_event.kprobe.func_name);
4022 ulen = info->perf_event.kprobe.name_len;
4023 err = bpf_perf_link_fill_common(event, uname, &ulen, &offset, &addr,
4024 &type, &missed);
4025 if (err)
4026 return err;
4027 if (type == BPF_FD_TYPE_KRETPROBE)
4028 info->perf_event.type = BPF_PERF_EVENT_KRETPROBE;
4029 else
4030 info->perf_event.type = BPF_PERF_EVENT_KPROBE;
4031 info->perf_event.kprobe.name_len = ulen;
4032 info->perf_event.kprobe.offset = offset;
4033 info->perf_event.kprobe.missed = missed;
4034 if (!kallsyms_show_value(current_cred()))
4035 addr = 0;
4036 info->perf_event.kprobe.addr = addr;
4037 info->perf_event.kprobe.cookie = event->bpf_cookie;
4038 return 0;
4039}
4040
4041static void bpf_perf_link_fdinfo_kprobe(const struct perf_event *event,
4042 struct seq_file *seq)
4043{
4044 const char *name;
4045 int err;
4046 u32 prog_id, type;
4047 u64 offset, addr;
4048 unsigned long missed;
4049
4050 err = bpf_get_perf_event_info(event, &prog_id, &type, &name,
4051 &offset, &addr, &missed);
4052 if (err)
4053 return;
4054
4055 seq_printf(seq,
4056 "name:\t%s\n"
4057 "offset:\t%#llx\n"
4058 "missed:\t%lu\n"
4059 "addr:\t%#llx\n"
4060 "event_type:\t%s\n"
4061 "cookie:\t%llu\n",
4062 name, offset, missed, addr,
4063 type == BPF_FD_TYPE_KRETPROBE ? "kretprobe" : "kprobe",
4064 event->bpf_cookie);
4065}
4066#endif
4067
4068#ifdef CONFIG_UPROBE_EVENTS
4069static int bpf_perf_link_fill_uprobe(const struct perf_event *event,
4070 struct bpf_link_info *info)
4071{
4072 u64 ref_ctr_offset, offset;
4073 char __user *uname;
4074 u32 ulen, type;
4075 int err;
4076
4077 uname = u64_to_user_ptr(info->perf_event.uprobe.file_name);
4078 ulen = info->perf_event.uprobe.name_len;
4079 err = bpf_perf_link_fill_common(event, uname, &ulen, &offset, &ref_ctr_offset,
4080 &type, NULL);
4081 if (err)
4082 return err;
4083
4084 if (type == BPF_FD_TYPE_URETPROBE)
4085 info->perf_event.type = BPF_PERF_EVENT_URETPROBE;
4086 else
4087 info->perf_event.type = BPF_PERF_EVENT_UPROBE;
4088 info->perf_event.uprobe.name_len = ulen;
4089 info->perf_event.uprobe.offset = offset;
4090 info->perf_event.uprobe.cookie = event->bpf_cookie;
4091 info->perf_event.uprobe.ref_ctr_offset = ref_ctr_offset;
4092 return 0;
4093}
4094
4095static void bpf_perf_link_fdinfo_uprobe(const struct perf_event *event,
4096 struct seq_file *seq)
4097{
4098 const char *name;
4099 int err;
4100 u32 prog_id, type;
4101 u64 offset, ref_ctr_offset;
4102 unsigned long missed;
4103
4104 err = bpf_get_perf_event_info(event, &prog_id, &type, &name,
4105 &offset, &ref_ctr_offset, &missed);
4106 if (err)
4107 return;
4108
4109 seq_printf(seq,
4110 "name:\t%s\n"
4111 "offset:\t%#llx\n"
4112 "ref_ctr_offset:\t%#llx\n"
4113 "event_type:\t%s\n"
4114 "cookie:\t%llu\n",
4115 name, offset, ref_ctr_offset,
4116 type == BPF_FD_TYPE_URETPROBE ? "uretprobe" : "uprobe",
4117 event->bpf_cookie);
4118}
4119#endif
4120
4121static int bpf_perf_link_fill_probe(const struct perf_event *event,
4122 struct bpf_link_info *info)
4123{
4124#ifdef CONFIG_KPROBE_EVENTS
4125 if (event->tp_event->flags & TRACE_EVENT_FL_KPROBE)
4126 return bpf_perf_link_fill_kprobe(event, info);
4127#endif
4128#ifdef CONFIG_UPROBE_EVENTS
4129 if (event->tp_event->flags & TRACE_EVENT_FL_UPROBE)
4130 return bpf_perf_link_fill_uprobe(event, info);
4131#endif
4132 return -EOPNOTSUPP;
4133}
4134
4135static int bpf_perf_link_fill_tracepoint(const struct perf_event *event,
4136 struct bpf_link_info *info)
4137{
4138 char __user *uname;
4139 u32 ulen;
4140 int err;
4141
4142 uname = u64_to_user_ptr(info->perf_event.tracepoint.tp_name);
4143 ulen = info->perf_event.tracepoint.name_len;
4144 err = bpf_perf_link_fill_common(event, uname, &ulen, NULL, NULL, NULL, NULL);
4145 if (err)
4146 return err;
4147
4148 info->perf_event.type = BPF_PERF_EVENT_TRACEPOINT;
4149 info->perf_event.tracepoint.name_len = ulen;
4150 info->perf_event.tracepoint.cookie = event->bpf_cookie;
4151 return 0;
4152}
4153
4154static int bpf_perf_link_fill_perf_event(const struct perf_event *event,
4155 struct bpf_link_info *info)
4156{
4157 info->perf_event.event.type = event->attr.type;
4158 info->perf_event.event.config = event->attr.config;
4159 info->perf_event.event.cookie = event->bpf_cookie;
4160 info->perf_event.type = BPF_PERF_EVENT_EVENT;
4161 return 0;
4162}
4163
4164static int bpf_perf_link_fill_link_info(const struct bpf_link *link,
4165 struct bpf_link_info *info)
4166{
4167 struct bpf_perf_link *perf_link;
4168 const struct perf_event *event;
4169
4170 perf_link = container_of(link, struct bpf_perf_link, link);
4171 event = perf_get_event(perf_link->perf_file);
4172 if (IS_ERR(event))
4173 return PTR_ERR(event);
4174
4175 switch (event->prog->type) {
4176 case BPF_PROG_TYPE_PERF_EVENT:
4177 return bpf_perf_link_fill_perf_event(event, info);
4178 case BPF_PROG_TYPE_TRACEPOINT:
4179 return bpf_perf_link_fill_tracepoint(event, info);
4180 case BPF_PROG_TYPE_KPROBE:
4181 return bpf_perf_link_fill_probe(event, info);
4182 default:
4183 return -EOPNOTSUPP;
4184 }
4185}
4186
4187static void bpf_perf_event_link_show_fdinfo(const struct perf_event *event,
4188 struct seq_file *seq)
4189{
4190 seq_printf(seq,
4191 "type:\t%u\n"
4192 "config:\t%llu\n"
4193 "event_type:\t%s\n"
4194 "cookie:\t%llu\n",
4195 event->attr.type, event->attr.config,
4196 "event", event->bpf_cookie);
4197}
4198
4199static void bpf_tracepoint_link_show_fdinfo(const struct perf_event *event,
4200 struct seq_file *seq)
4201{
4202 int err;
4203 const char *name;
4204 u32 prog_id;
4205
4206 err = bpf_get_perf_event_info(event, &prog_id, NULL, &name, NULL,
4207 NULL, NULL);
4208 if (err)
4209 return;
4210
4211 seq_printf(seq,
4212 "tp_name:\t%s\n"
4213 "event_type:\t%s\n"
4214 "cookie:\t%llu\n",
4215 name, "tracepoint", event->bpf_cookie);
4216}
4217
4218static void bpf_probe_link_show_fdinfo(const struct perf_event *event,
4219 struct seq_file *seq)
4220{
4221#ifdef CONFIG_KPROBE_EVENTS
4222 if (event->tp_event->flags & TRACE_EVENT_FL_KPROBE)
4223 return bpf_perf_link_fdinfo_kprobe(event, seq);
4224#endif
4225
4226#ifdef CONFIG_UPROBE_EVENTS
4227 if (event->tp_event->flags & TRACE_EVENT_FL_UPROBE)
4228 return bpf_perf_link_fdinfo_uprobe(event, seq);
4229#endif
4230}
4231
4232static void bpf_perf_link_show_fdinfo(const struct bpf_link *link,
4233 struct seq_file *seq)
4234{
4235 struct bpf_perf_link *perf_link;
4236 const struct perf_event *event;
4237
4238 perf_link = container_of(link, struct bpf_perf_link, link);
4239 event = perf_get_event(perf_link->perf_file);
4240 if (IS_ERR(event))
4241 return;
4242
4243 switch (event->prog->type) {
4244 case BPF_PROG_TYPE_PERF_EVENT:
4245 return bpf_perf_event_link_show_fdinfo(event, seq);
4246 case BPF_PROG_TYPE_TRACEPOINT:
4247 return bpf_tracepoint_link_show_fdinfo(event, seq);
4248 case BPF_PROG_TYPE_KPROBE:
4249 return bpf_probe_link_show_fdinfo(event, seq);
4250 default:
4251 return;
4252 }
4253}
4254
4255static const struct bpf_link_ops bpf_perf_link_lops = {
4256 .release = bpf_perf_link_release,
4257 .dealloc = bpf_perf_link_dealloc,
4258 .fill_link_info = bpf_perf_link_fill_link_info,
4259 .show_fdinfo = bpf_perf_link_show_fdinfo,
4260};
4261
4262static int bpf_perf_link_attach(const union bpf_attr *attr, struct bpf_prog *prog)
4263{
4264 struct bpf_link_primer link_primer;
4265 struct bpf_perf_link *link;
4266 struct perf_event *event;
4267 struct file *perf_file;
4268 int err;
4269
4270 if (attr->link_create.flags)
4271 return -EINVAL;
4272
4273 perf_file = perf_event_get(attr->link_create.target_fd);
4274 if (IS_ERR(perf_file))
4275 return PTR_ERR(perf_file);
4276
4277 link = kzalloc_obj(*link, GFP_USER);
4278 if (!link) {
4279 err = -ENOMEM;
4280 goto out_put_file;
4281 }
4282 bpf_link_init(&link->link, BPF_LINK_TYPE_PERF_EVENT, &bpf_perf_link_lops, prog,
4283 attr->link_create.attach_type);
4284 link->perf_file = perf_file;
4285
4286 err = bpf_link_prime(&link->link, &link_primer);
4287 if (err) {
4288 kfree(link);
4289 goto out_put_file;
4290 }
4291
4292 event = perf_file->private_data;
4293 err = perf_event_set_bpf_prog(event, prog, attr->link_create.perf_event.bpf_cookie);
4294 if (err) {
4295 bpf_link_cleanup(&link_primer);
4296 goto out_put_file;
4297 }
4298 /* perf_event_set_bpf_prog() doesn't take its own refcnt on prog */
4299 bpf_prog_inc(prog);
4300
4301 return bpf_link_settle(&link_primer);
4302
4303out_put_file:
4304 fput(perf_file);
4305 return err;
4306}
4307#else
4308static int bpf_perf_link_attach(const union bpf_attr *attr, struct bpf_prog *prog)
4309{
4310 return -EOPNOTSUPP;
4311}
4312#endif /* CONFIG_PERF_EVENTS */
4313
4314static int bpf_raw_tp_link_attach(struct bpf_prog *prog,
4315 const char __user *user_tp_name, u64 cookie,
4316 enum bpf_attach_type attach_type)
4317{
4318 struct bpf_link_primer link_primer;
4319 struct bpf_raw_tp_link *link;
4320 struct bpf_raw_event_map *btp;
4321 const char *tp_name;
4322 char buf[128];
4323 int err;
4324
4325 switch (prog->type) {
4326 case BPF_PROG_TYPE_TRACING:
4327 case BPF_PROG_TYPE_EXT:
4328 case BPF_PROG_TYPE_LSM:
4329 if (user_tp_name)
4330 /* The attach point for this category of programs
4331 * should be specified via btf_id during program load.
4332 */
4333 return -EINVAL;
4334 if (prog->type == BPF_PROG_TYPE_TRACING &&
4335 prog->expected_attach_type == BPF_TRACE_RAW_TP) {
4336 tp_name = prog->aux->attach_func_name;
4337 break;
4338 }
4339 return bpf_tracing_prog_attach(prog, 0, 0, 0, attach_type);
4340 case BPF_PROG_TYPE_RAW_TRACEPOINT:
4341 case BPF_PROG_TYPE_RAW_TRACEPOINT_WRITABLE:
4342 if (strncpy_from_user(buf, user_tp_name, sizeof(buf) - 1) < 0)
4343 return -EFAULT;
4344 buf[sizeof(buf) - 1] = 0;
4345 tp_name = buf;
4346 break;
4347 default:
4348 return -EINVAL;
4349 }
4350
4351 btp = bpf_get_raw_tracepoint(tp_name);
4352 if (!btp)
4353 return -ENOENT;
4354
4355 if (prog->sleepable && !tracepoint_is_faultable(btp->tp)) {
4356 bpf_put_raw_tracepoint(btp);
4357 return -EINVAL;
4358 }
4359
4360 link = kzalloc_obj(*link, GFP_USER);
4361 if (!link) {
4362 err = -ENOMEM;
4363 goto out_put_btp;
4364 }
4365 bpf_link_init_sleepable(&link->link, BPF_LINK_TYPE_RAW_TRACEPOINT,
4366 &bpf_raw_tp_link_lops, prog, attach_type,
4367 tracepoint_is_faultable(btp->tp));
4368 link->btp = btp;
4369 link->cookie = cookie;
4370
4371 err = bpf_link_prime(&link->link, &link_primer);
4372 if (err) {
4373 kfree(link);
4374 goto out_put_btp;
4375 }
4376
4377 err = bpf_probe_register(link->btp, link);
4378 if (err) {
4379 bpf_link_cleanup(&link_primer);
4380 goto out_put_btp;
4381 }
4382
4383 return bpf_link_settle(&link_primer);
4384
4385out_put_btp:
4386 bpf_put_raw_tracepoint(btp);
4387 return err;
4388}
4389
4390#define BPF_RAW_TRACEPOINT_OPEN_LAST_FIELD raw_tracepoint.cookie
4391
4392static int bpf_raw_tracepoint_open(const union bpf_attr *attr)
4393{
4394 struct bpf_prog *prog;
4395 void __user *tp_name;
4396 __u64 cookie;
4397 int fd;
4398
4399 if (CHECK_ATTR(BPF_RAW_TRACEPOINT_OPEN))
4400 return -EINVAL;
4401
4402 prog = bpf_prog_get(attr->raw_tracepoint.prog_fd);
4403 if (IS_ERR(prog))
4404 return PTR_ERR(prog);
4405
4406 tp_name = u64_to_user_ptr(attr->raw_tracepoint.name);
4407 cookie = attr->raw_tracepoint.cookie;
4408 fd = bpf_raw_tp_link_attach(prog, tp_name, cookie, prog->expected_attach_type);
4409 if (fd < 0)
4410 bpf_prog_put(prog);
4411 return fd;
4412}
4413
4414static enum bpf_prog_type
4415attach_type_to_prog_type(enum bpf_attach_type attach_type)
4416{
4417 switch (attach_type) {
4418 case BPF_CGROUP_INET_INGRESS:
4419 case BPF_CGROUP_INET_EGRESS:
4420 return BPF_PROG_TYPE_CGROUP_SKB;
4421 case BPF_CGROUP_INET_SOCK_CREATE:
4422 case BPF_CGROUP_INET_SOCK_RELEASE:
4423 case BPF_CGROUP_INET4_POST_BIND:
4424 case BPF_CGROUP_INET6_POST_BIND:
4425 return BPF_PROG_TYPE_CGROUP_SOCK;
4426 case BPF_CGROUP_INET4_BIND:
4427 case BPF_CGROUP_INET6_BIND:
4428 case BPF_CGROUP_INET4_CONNECT:
4429 case BPF_CGROUP_INET6_CONNECT:
4430 case BPF_CGROUP_UNIX_CONNECT:
4431 case BPF_CGROUP_INET4_GETPEERNAME:
4432 case BPF_CGROUP_INET6_GETPEERNAME:
4433 case BPF_CGROUP_UNIX_GETPEERNAME:
4434 case BPF_CGROUP_INET4_GETSOCKNAME:
4435 case BPF_CGROUP_INET6_GETSOCKNAME:
4436 case BPF_CGROUP_UNIX_GETSOCKNAME:
4437 case BPF_CGROUP_UDP4_SENDMSG:
4438 case BPF_CGROUP_UDP6_SENDMSG:
4439 case BPF_CGROUP_UNIX_SENDMSG:
4440 case BPF_CGROUP_UDP4_RECVMSG:
4441 case BPF_CGROUP_UDP6_RECVMSG:
4442 case BPF_CGROUP_UNIX_RECVMSG:
4443 return BPF_PROG_TYPE_CGROUP_SOCK_ADDR;
4444 case BPF_CGROUP_SOCK_OPS:
4445 return BPF_PROG_TYPE_SOCK_OPS;
4446 case BPF_CGROUP_DEVICE:
4447 return BPF_PROG_TYPE_CGROUP_DEVICE;
4448 case BPF_SK_MSG_VERDICT:
4449 return BPF_PROG_TYPE_SK_MSG;
4450 case BPF_SK_SKB_STREAM_PARSER:
4451 case BPF_SK_SKB_STREAM_VERDICT:
4452 case BPF_SK_SKB_VERDICT:
4453 return BPF_PROG_TYPE_SK_SKB;
4454 case BPF_LIRC_MODE2:
4455 return BPF_PROG_TYPE_LIRC_MODE2;
4456 case BPF_FLOW_DISSECTOR:
4457 return BPF_PROG_TYPE_FLOW_DISSECTOR;
4458 case BPF_CGROUP_SYSCTL:
4459 return BPF_PROG_TYPE_CGROUP_SYSCTL;
4460 case BPF_CGROUP_GETSOCKOPT:
4461 case BPF_CGROUP_SETSOCKOPT:
4462 return BPF_PROG_TYPE_CGROUP_SOCKOPT;
4463 case BPF_TRACE_ITER:
4464 case BPF_TRACE_RAW_TP:
4465 case BPF_TRACE_FENTRY:
4466 case BPF_TRACE_FEXIT:
4467 case BPF_TRACE_FSESSION:
4468 case BPF_TRACE_FSESSION_MULTI:
4469 case BPF_TRACE_FENTRY_MULTI:
4470 case BPF_TRACE_FEXIT_MULTI:
4471 case BPF_MODIFY_RETURN:
4472 return BPF_PROG_TYPE_TRACING;
4473 case BPF_LSM_MAC:
4474 return BPF_PROG_TYPE_LSM;
4475 case BPF_SK_LOOKUP:
4476 return BPF_PROG_TYPE_SK_LOOKUP;
4477 case BPF_XDP:
4478 return BPF_PROG_TYPE_XDP;
4479 case BPF_LSM_CGROUP:
4480 return BPF_PROG_TYPE_LSM;
4481 case BPF_TCX_INGRESS:
4482 case BPF_TCX_EGRESS:
4483 case BPF_NETKIT_PRIMARY:
4484 case BPF_NETKIT_PEER:
4485 return BPF_PROG_TYPE_SCHED_CLS;
4486 default:
4487 return BPF_PROG_TYPE_UNSPEC;
4488 }
4489}
4490
4491static int bpf_prog_attach_check_attach_type(const struct bpf_prog *prog,
4492 enum bpf_attach_type attach_type)
4493{
4494 enum bpf_prog_type ptype;
4495
4496 switch (prog->type) {
4497 case BPF_PROG_TYPE_CGROUP_SOCK:
4498 case BPF_PROG_TYPE_CGROUP_SOCK_ADDR:
4499 case BPF_PROG_TYPE_CGROUP_SOCKOPT:
4500 case BPF_PROG_TYPE_SK_LOOKUP:
4501 return attach_type == prog->expected_attach_type ? 0 : -EINVAL;
4502 case BPF_PROG_TYPE_CGROUP_SKB:
4503 if (!bpf_token_capable(prog->aux->token, CAP_NET_ADMIN))
4504 /* cg-skb progs can be loaded by unpriv user.
4505 * check permissions at attach time.
4506 */
4507 return -EPERM;
4508
4509 ptype = attach_type_to_prog_type(attach_type);
4510 if (prog->type != ptype)
4511 return -EINVAL;
4512
4513 return prog->enforce_expected_attach_type &&
4514 prog->expected_attach_type != attach_type ?
4515 -EINVAL : 0;
4516 case BPF_PROG_TYPE_EXT:
4517 return 0;
4518 case BPF_PROG_TYPE_NETFILTER:
4519 if (attach_type != BPF_NETFILTER)
4520 return -EINVAL;
4521 return 0;
4522 case BPF_PROG_TYPE_PERF_EVENT:
4523 case BPF_PROG_TYPE_TRACEPOINT:
4524 if (attach_type != BPF_PERF_EVENT)
4525 return -EINVAL;
4526 return 0;
4527 case BPF_PROG_TYPE_KPROBE:
4528 if (prog->expected_attach_type == BPF_TRACE_KPROBE_MULTI &&
4529 attach_type != BPF_TRACE_KPROBE_MULTI)
4530 return -EINVAL;
4531 if (prog->expected_attach_type == BPF_TRACE_KPROBE_SESSION &&
4532 attach_type != BPF_TRACE_KPROBE_SESSION)
4533 return -EINVAL;
4534 if (prog->expected_attach_type == BPF_TRACE_UPROBE_MULTI &&
4535 attach_type != BPF_TRACE_UPROBE_MULTI)
4536 return -EINVAL;
4537 if (prog->expected_attach_type == BPF_TRACE_UPROBE_SESSION &&
4538 attach_type != BPF_TRACE_UPROBE_SESSION)
4539 return -EINVAL;
4540 if (attach_type != BPF_PERF_EVENT &&
4541 attach_type != BPF_TRACE_KPROBE_MULTI &&
4542 attach_type != BPF_TRACE_KPROBE_SESSION &&
4543 attach_type != BPF_TRACE_UPROBE_MULTI &&
4544 attach_type != BPF_TRACE_UPROBE_SESSION)
4545 return -EINVAL;
4546 return 0;
4547 case BPF_PROG_TYPE_SCHED_CLS:
4548 if (attach_type != BPF_TCX_INGRESS &&
4549 attach_type != BPF_TCX_EGRESS &&
4550 attach_type != BPF_NETKIT_PRIMARY &&
4551 attach_type != BPF_NETKIT_PEER)
4552 return -EINVAL;
4553 return 0;
4554 default:
4555 ptype = attach_type_to_prog_type(attach_type);
4556 if (ptype == BPF_PROG_TYPE_UNSPEC || ptype != prog->type)
4557 return -EINVAL;
4558 return 0;
4559 }
4560}
4561
4562static bool is_cgroup_prog_type(enum bpf_prog_type ptype, enum bpf_attach_type atype,
4563 bool check_atype)
4564{
4565 switch (ptype) {
4566 case BPF_PROG_TYPE_CGROUP_DEVICE:
4567 case BPF_PROG_TYPE_CGROUP_SKB:
4568 case BPF_PROG_TYPE_CGROUP_SOCK:
4569 case BPF_PROG_TYPE_CGROUP_SOCK_ADDR:
4570 case BPF_PROG_TYPE_CGROUP_SOCKOPT:
4571 case BPF_PROG_TYPE_CGROUP_SYSCTL:
4572 case BPF_PROG_TYPE_SOCK_OPS:
4573 return true;
4574 case BPF_PROG_TYPE_LSM:
4575 return check_atype ? atype == BPF_LSM_CGROUP : true;
4576 default:
4577 return false;
4578 }
4579}
4580
4581#define BPF_PROG_ATTACH_LAST_FIELD expected_revision
4582
4583#define BPF_F_ATTACH_MASK_BASE \
4584 (BPF_F_ALLOW_OVERRIDE | \
4585 BPF_F_ALLOW_MULTI | \
4586 BPF_F_REPLACE | \
4587 BPF_F_PREORDER)
4588
4589#define BPF_F_ATTACH_MASK_MPROG \
4590 (BPF_F_REPLACE | \
4591 BPF_F_BEFORE | \
4592 BPF_F_AFTER | \
4593 BPF_F_ID | \
4594 BPF_F_LINK)
4595
4596static int bpf_prog_attach(const union bpf_attr *attr)
4597{
4598 enum bpf_prog_type ptype;
4599 struct bpf_prog *prog;
4600 int ret;
4601
4602 if (CHECK_ATTR(BPF_PROG_ATTACH))
4603 return -EINVAL;
4604
4605 ptype = attach_type_to_prog_type(attr->attach_type);
4606 if (ptype == BPF_PROG_TYPE_UNSPEC)
4607 return -EINVAL;
4608 if (bpf_mprog_supported(ptype)) {
4609 if (attr->attach_flags & ~BPF_F_ATTACH_MASK_MPROG)
4610 return -EINVAL;
4611 } else if (is_cgroup_prog_type(ptype, 0, false)) {
4612 if (attr->attach_flags & ~(BPF_F_ATTACH_MASK_BASE | BPF_F_ATTACH_MASK_MPROG))
4613 return -EINVAL;
4614 } else {
4615 if (attr->attach_flags & ~BPF_F_ATTACH_MASK_BASE)
4616 return -EINVAL;
4617 if (attr->relative_fd ||
4618 attr->expected_revision)
4619 return -EINVAL;
4620 }
4621
4622 prog = bpf_prog_get_type(attr->attach_bpf_fd, ptype);
4623 if (IS_ERR(prog))
4624 return PTR_ERR(prog);
4625
4626 if (bpf_prog_attach_check_attach_type(prog, attr->attach_type)) {
4627 bpf_prog_put(prog);
4628 return -EINVAL;
4629 }
4630
4631 if (is_cgroup_prog_type(ptype, prog->expected_attach_type, true)) {
4632 ret = cgroup_bpf_prog_attach(attr, ptype, prog);
4633 goto out;
4634 }
4635
4636 switch (ptype) {
4637 case BPF_PROG_TYPE_SK_SKB:
4638 case BPF_PROG_TYPE_SK_MSG:
4639 ret = sock_map_get_from_fd(attr, prog);
4640 break;
4641 case BPF_PROG_TYPE_LIRC_MODE2:
4642 ret = lirc_prog_attach(attr, prog);
4643 break;
4644 case BPF_PROG_TYPE_FLOW_DISSECTOR:
4645 ret = netns_bpf_prog_attach(attr, prog);
4646 break;
4647 case BPF_PROG_TYPE_SCHED_CLS:
4648 if (attr->attach_type == BPF_TCX_INGRESS ||
4649 attr->attach_type == BPF_TCX_EGRESS)
4650 ret = tcx_prog_attach(attr, prog);
4651 else
4652 ret = netkit_prog_attach(attr, prog);
4653 break;
4654 default:
4655 ret = -EINVAL;
4656 }
4657out:
4658 if (ret)
4659 bpf_prog_put(prog);
4660 return ret;
4661}
4662
4663#define BPF_PROG_DETACH_LAST_FIELD expected_revision
4664
4665static int bpf_prog_detach(const union bpf_attr *attr)
4666{
4667 struct bpf_prog *prog = NULL;
4668 enum bpf_prog_type ptype;
4669 int ret;
4670
4671 if (CHECK_ATTR(BPF_PROG_DETACH))
4672 return -EINVAL;
4673
4674 ptype = attach_type_to_prog_type(attr->attach_type);
4675 if (bpf_mprog_supported(ptype)) {
4676 if (ptype == BPF_PROG_TYPE_UNSPEC)
4677 return -EINVAL;
4678 if (attr->attach_flags & ~BPF_F_ATTACH_MASK_MPROG)
4679 return -EINVAL;
4680 if (attr->attach_bpf_fd) {
4681 prog = bpf_prog_get_type(attr->attach_bpf_fd, ptype);
4682 if (IS_ERR(prog))
4683 return PTR_ERR(prog);
4684 } else if (!bpf_mprog_detach_empty(ptype)) {
4685 return -EPERM;
4686 }
4687 } else if (is_cgroup_prog_type(ptype, 0, false)) {
4688 if (attr->attach_flags || attr->relative_fd)
4689 return -EINVAL;
4690 } else if (attr->attach_flags ||
4691 attr->relative_fd ||
4692 attr->expected_revision) {
4693 return -EINVAL;
4694 }
4695
4696 switch (ptype) {
4697 case BPF_PROG_TYPE_SK_MSG:
4698 case BPF_PROG_TYPE_SK_SKB:
4699 ret = sock_map_prog_detach(attr, ptype);
4700 break;
4701 case BPF_PROG_TYPE_LIRC_MODE2:
4702 ret = lirc_prog_detach(attr);
4703 break;
4704 case BPF_PROG_TYPE_FLOW_DISSECTOR:
4705 ret = netns_bpf_prog_detach(attr, ptype);
4706 break;
4707 case BPF_PROG_TYPE_CGROUP_DEVICE:
4708 case BPF_PROG_TYPE_CGROUP_SKB:
4709 case BPF_PROG_TYPE_CGROUP_SOCK:
4710 case BPF_PROG_TYPE_CGROUP_SOCK_ADDR:
4711 case BPF_PROG_TYPE_CGROUP_SOCKOPT:
4712 case BPF_PROG_TYPE_CGROUP_SYSCTL:
4713 case BPF_PROG_TYPE_SOCK_OPS:
4714 case BPF_PROG_TYPE_LSM:
4715 ret = cgroup_bpf_prog_detach(attr, ptype);
4716 break;
4717 case BPF_PROG_TYPE_SCHED_CLS:
4718 if (attr->attach_type == BPF_TCX_INGRESS ||
4719 attr->attach_type == BPF_TCX_EGRESS)
4720 ret = tcx_prog_detach(attr, prog);
4721 else
4722 ret = netkit_prog_detach(attr, prog);
4723 break;
4724 default:
4725 ret = -EINVAL;
4726 }
4727
4728 if (prog)
4729 bpf_prog_put(prog);
4730 return ret;
4731}
4732
4733#define BPF_PROG_QUERY_LAST_FIELD query.revision
4734
4735static int bpf_prog_query(const union bpf_attr *attr,
4736 union bpf_attr __user *uattr, u32 uattr_size)
4737{
4738 if (!bpf_net_capable())
4739 return -EPERM;
4740 if (CHECK_ATTR(BPF_PROG_QUERY))
4741 return -EINVAL;
4742 if (attr->query.query_flags & ~BPF_F_QUERY_EFFECTIVE)
4743 return -EINVAL;
4744
4745 switch (attr->query.attach_type) {
4746 case BPF_CGROUP_INET_INGRESS:
4747 case BPF_CGROUP_INET_EGRESS:
4748 case BPF_CGROUP_INET_SOCK_CREATE:
4749 case BPF_CGROUP_INET_SOCK_RELEASE:
4750 case BPF_CGROUP_INET4_BIND:
4751 case BPF_CGROUP_INET6_BIND:
4752 case BPF_CGROUP_INET4_POST_BIND:
4753 case BPF_CGROUP_INET6_POST_BIND:
4754 case BPF_CGROUP_INET4_CONNECT:
4755 case BPF_CGROUP_INET6_CONNECT:
4756 case BPF_CGROUP_UNIX_CONNECT:
4757 case BPF_CGROUP_INET4_GETPEERNAME:
4758 case BPF_CGROUP_INET6_GETPEERNAME:
4759 case BPF_CGROUP_UNIX_GETPEERNAME:
4760 case BPF_CGROUP_INET4_GETSOCKNAME:
4761 case BPF_CGROUP_INET6_GETSOCKNAME:
4762 case BPF_CGROUP_UNIX_GETSOCKNAME:
4763 case BPF_CGROUP_UDP4_SENDMSG:
4764 case BPF_CGROUP_UDP6_SENDMSG:
4765 case BPF_CGROUP_UNIX_SENDMSG:
4766 case BPF_CGROUP_UDP4_RECVMSG:
4767 case BPF_CGROUP_UDP6_RECVMSG:
4768 case BPF_CGROUP_UNIX_RECVMSG:
4769 case BPF_CGROUP_SOCK_OPS:
4770 case BPF_CGROUP_DEVICE:
4771 case BPF_CGROUP_SYSCTL:
4772 case BPF_CGROUP_GETSOCKOPT:
4773 case BPF_CGROUP_SETSOCKOPT:
4774 case BPF_LSM_CGROUP:
4775 return cgroup_bpf_prog_query(attr, uattr, uattr_size);
4776 case BPF_LIRC_MODE2:
4777 return lirc_prog_query(attr, uattr);
4778 case BPF_FLOW_DISSECTOR:
4779 case BPF_SK_LOOKUP:
4780 return netns_bpf_prog_query(attr, uattr);
4781 case BPF_SK_SKB_STREAM_PARSER:
4782 case BPF_SK_SKB_STREAM_VERDICT:
4783 case BPF_SK_MSG_VERDICT:
4784 case BPF_SK_SKB_VERDICT:
4785 return sock_map_bpf_prog_query(attr, uattr);
4786 case BPF_TCX_INGRESS:
4787 case BPF_TCX_EGRESS:
4788 return tcx_prog_query(attr, uattr);
4789 case BPF_NETKIT_PRIMARY:
4790 case BPF_NETKIT_PEER:
4791 return netkit_prog_query(attr, uattr);
4792 default:
4793 return -EINVAL;
4794 }
4795}
4796
4797#define BPF_PROG_TEST_RUN_LAST_FIELD test.batch_size
4798
4799static int bpf_prog_test_run(const union bpf_attr *attr,
4800 union bpf_attr __user *uattr)
4801{
4802 struct bpf_prog *prog;
4803 int ret = -ENOTSUPP;
4804
4805 if (CHECK_ATTR(BPF_PROG_TEST_RUN))
4806 return -EINVAL;
4807
4808 if ((attr->test.ctx_size_in && !attr->test.ctx_in) ||
4809 (!attr->test.ctx_size_in && attr->test.ctx_in))
4810 return -EINVAL;
4811
4812 if ((attr->test.ctx_size_out && !attr->test.ctx_out) ||
4813 (!attr->test.ctx_size_out && attr->test.ctx_out))
4814 return -EINVAL;
4815
4816 prog = bpf_prog_get(attr->test.prog_fd);
4817 if (IS_ERR(prog))
4818 return PTR_ERR(prog);
4819
4820 if (prog->aux->ops->test_run)
4821 ret = prog->aux->ops->test_run(prog, attr, uattr);
4822
4823 bpf_prog_put(prog);
4824 return ret;
4825}
4826
4827#define BPF_OBJ_GET_NEXT_ID_LAST_FIELD next_id
4828
4829static int bpf_obj_get_next_id(const union bpf_attr *attr,
4830 union bpf_attr __user *uattr,
4831 struct idr *idr,
4832 spinlock_t *lock)
4833{
4834 u32 next_id = attr->start_id;
4835 int err = 0;
4836
4837 if (CHECK_ATTR(BPF_OBJ_GET_NEXT_ID) || next_id >= INT_MAX)
4838 return -EINVAL;
4839
4840 if (!capable(CAP_SYS_ADMIN))
4841 return -EPERM;
4842
4843 next_id++;
4844 spin_lock_bh(lock);
4845 if (!idr_get_next(idr, &next_id))
4846 err = -ENOENT;
4847 spin_unlock_bh(lock);
4848
4849 if (!err)
4850 err = put_user(next_id, &uattr->next_id);
4851
4852 return err;
4853}
4854
4855struct bpf_map *bpf_map_get_curr_or_next(u32 *id)
4856{
4857 struct bpf_map *map;
4858
4859 spin_lock_bh(&map_idr_lock);
4860again:
4861 map = idr_get_next(&map_idr, id);
4862 if (map) {
4863 map = __bpf_map_inc_not_zero(map, false);
4864 if (IS_ERR(map)) {
4865 (*id)++;
4866 goto again;
4867 }
4868 }
4869 spin_unlock_bh(&map_idr_lock);
4870
4871 return map;
4872}
4873
4874struct bpf_prog *bpf_prog_get_curr_or_next(u32 *id)
4875{
4876 struct bpf_prog *prog;
4877
4878 spin_lock_bh(&prog_idr_lock);
4879again:
4880 prog = idr_get_next(&prog_idr, id);
4881 if (prog) {
4882 prog = bpf_prog_inc_not_zero(prog);
4883 if (IS_ERR(prog)) {
4884 (*id)++;
4885 goto again;
4886 }
4887 }
4888 spin_unlock_bh(&prog_idr_lock);
4889
4890 return prog;
4891}
4892
4893#define BPF_PROG_GET_FD_BY_ID_LAST_FIELD prog_id
4894
4895struct bpf_prog *bpf_prog_by_id(u32 id)
4896{
4897 struct bpf_prog *prog;
4898
4899 if (!id)
4900 return ERR_PTR(-ENOENT);
4901
4902 spin_lock_bh(&prog_idr_lock);
4903 prog = idr_find(&prog_idr, id);
4904 if (prog)
4905 prog = bpf_prog_inc_not_zero(prog);
4906 else
4907 prog = ERR_PTR(-ENOENT);
4908 spin_unlock_bh(&prog_idr_lock);
4909 return prog;
4910}
4911
4912static int bpf_prog_get_fd_by_id(const union bpf_attr *attr)
4913{
4914 struct bpf_prog *prog;
4915 u32 id = attr->prog_id;
4916 int fd;
4917
4918 if (CHECK_ATTR(BPF_PROG_GET_FD_BY_ID))
4919 return -EINVAL;
4920
4921 if (!capable(CAP_SYS_ADMIN))
4922 return -EPERM;
4923
4924 prog = bpf_prog_by_id(id);
4925 if (IS_ERR(prog))
4926 return PTR_ERR(prog);
4927
4928 fd = bpf_prog_new_fd(prog);
4929 if (fd < 0)
4930 bpf_prog_put(prog);
4931
4932 return fd;
4933}
4934
4935#define BPF_MAP_GET_FD_BY_ID_LAST_FIELD open_flags
4936
4937static int bpf_map_get_fd_by_id(const union bpf_attr *attr)
4938{
4939 struct bpf_map *map;
4940 u32 id = attr->map_id;
4941 int f_flags;
4942 int fd;
4943
4944 if (CHECK_ATTR(BPF_MAP_GET_FD_BY_ID) ||
4945 attr->open_flags & ~BPF_OBJ_FLAG_MASK)
4946 return -EINVAL;
4947
4948 if (!capable(CAP_SYS_ADMIN))
4949 return -EPERM;
4950
4951 f_flags = bpf_get_file_flag(attr->open_flags);
4952 if (f_flags < 0)
4953 return f_flags;
4954
4955 spin_lock_bh(&map_idr_lock);
4956 map = idr_find(&map_idr, id);
4957 if (map)
4958 map = __bpf_map_inc_not_zero(map, true);
4959 else
4960 map = ERR_PTR(-ENOENT);
4961 spin_unlock_bh(&map_idr_lock);
4962
4963 if (IS_ERR(map))
4964 return PTR_ERR(map);
4965
4966 fd = bpf_map_new_fd(map, f_flags);
4967 if (fd < 0)
4968 bpf_map_put_with_uref(map);
4969
4970 return fd;
4971}
4972
4973static const struct bpf_map *bpf_map_from_imm(const struct bpf_prog *prog,
4974 unsigned long addr, u32 *off,
4975 u32 *type)
4976{
4977 const struct bpf_map *map;
4978 int i;
4979
4980 mutex_lock(&prog->aux->used_maps_mutex);
4981 for (i = 0, *off = 0; i < prog->aux->used_map_cnt; i++) {
4982 map = prog->aux->used_maps[i];
4983 if (map == (void *)addr) {
4984 *type = BPF_PSEUDO_MAP_FD;
4985 goto out;
4986 }
4987 if (!map->ops->map_direct_value_meta)
4988 continue;
4989 if (!map->ops->map_direct_value_meta(map, addr, off)) {
4990 *type = BPF_PSEUDO_MAP_VALUE;
4991 goto out;
4992 }
4993 }
4994 map = NULL;
4995
4996out:
4997 mutex_unlock(&prog->aux->used_maps_mutex);
4998 return map;
4999}
5000
5001static void prepare_dump_pseudo_call(struct bpf_insn *insn)
5002{
5003 s32 call_off = insn->imm;
5004
5005 /*
5006 * BPF_CALL_ARGS only exists for interpreter fallback.
5007 * 1. For interpreter (BPF_CALL_ARGS): insn->off is the index of
5008 * interpreters_args array, so here using bpf_call_args_imm()
5009 * to get the real address offset.
5010 * 2. For JIT (BPF_CALL): insn->off is the subprog id.
5011 */
5012 if (insn->code == (BPF_JMP | BPF_CALL_ARGS))
5013 insn->imm = bpf_call_args_imm(insn->off);
5014 else
5015 insn->imm = insn->off;
5016
5017 /* Avoid dumping a truncated and misleading pc-relative offset. */
5018 if (call_off > S16_MAX || call_off < S16_MIN)
5019 insn->off = 0;
5020 else
5021 insn->off = call_off;
5022}
5023
5024static struct bpf_insn *bpf_insn_prepare_dump(const struct bpf_prog *prog,
5025 const struct cred *f_cred)
5026{
5027 const struct bpf_map *map;
5028 struct bpf_insn *insns;
5029 u32 off, type;
5030 u64 imm;
5031 u8 code;
5032 int i;
5033
5034 insns = kmemdup(prog->insnsi, bpf_prog_insn_size(prog),
5035 GFP_USER);
5036 if (!insns)
5037 return insns;
5038
5039 for (i = 0; i < prog->len; i++) {
5040 code = insns[i].code;
5041
5042 if (code == (BPF_JMP | BPF_TAIL_CALL)) {
5043 insns[i].code = BPF_JMP | BPF_CALL;
5044 insns[i].imm = BPF_FUNC_tail_call;
5045 /* fall-through */
5046 }
5047 if (code == (BPF_JMP | BPF_CALL) ||
5048 code == (BPF_JMP | BPF_CALL_ARGS)) {
5049 /* Restore the legacy xlated dump layout. */
5050 if (insns[i].src_reg == BPF_PSEUDO_CALL)
5051 prepare_dump_pseudo_call(&insns[i]);
5052 if (code == (BPF_JMP | BPF_CALL_ARGS))
5053 insns[i].code = BPF_JMP | BPF_CALL;
5054 if (!bpf_dump_raw_ok(f_cred))
5055 insns[i].imm = 0;
5056 continue;
5057 }
5058 if (BPF_CLASS(code) == BPF_LDX && BPF_MODE(code) == BPF_PROBE_MEM) {
5059 insns[i].code = BPF_LDX | BPF_SIZE(code) | BPF_MEM;
5060 continue;
5061 }
5062
5063 if ((BPF_CLASS(code) == BPF_LDX || BPF_CLASS(code) == BPF_STX ||
5064 BPF_CLASS(code) == BPF_ST) && BPF_MODE(code) == BPF_PROBE_MEM32) {
5065 insns[i].code = BPF_CLASS(code) | BPF_SIZE(code) | BPF_MEM;
5066 continue;
5067 }
5068
5069 if (code != (BPF_LD | BPF_IMM | BPF_DW))
5070 continue;
5071
5072 imm = ((u64)insns[i + 1].imm << 32) | (u32)insns[i].imm;
5073 map = bpf_map_from_imm(prog, imm, &off, &type);
5074 if (map) {
5075 insns[i].src_reg = type;
5076 insns[i].imm = map->id;
5077 insns[i + 1].imm = off;
5078 continue;
5079 }
5080 }
5081
5082 return insns;
5083}
5084
5085static int set_info_rec_size(struct bpf_prog_info *info)
5086{
5087 /*
5088 * Ensure info.*_rec_size is the same as kernel expected size
5089 *
5090 * or
5091 *
5092 * Only allow zero *_rec_size if both _rec_size and _cnt are
5093 * zero. In this case, the kernel will set the expected
5094 * _rec_size back to the info.
5095 */
5096
5097 if ((info->nr_func_info || info->func_info_rec_size) &&
5098 info->func_info_rec_size != sizeof(struct bpf_func_info))
5099 return -EINVAL;
5100
5101 if ((info->nr_line_info || info->line_info_rec_size) &&
5102 info->line_info_rec_size != sizeof(struct bpf_line_info))
5103 return -EINVAL;
5104
5105 if ((info->nr_jited_line_info || info->jited_line_info_rec_size) &&
5106 info->jited_line_info_rec_size != sizeof(__u64))
5107 return -EINVAL;
5108
5109 info->func_info_rec_size = sizeof(struct bpf_func_info);
5110 info->line_info_rec_size = sizeof(struct bpf_line_info);
5111 info->jited_line_info_rec_size = sizeof(__u64);
5112
5113 return 0;
5114}
5115
5116static int bpf_prog_get_info_by_fd(struct file *file,
5117 struct bpf_prog *prog,
5118 const union bpf_attr *attr,
5119 union bpf_attr __user *uattr)
5120{
5121 struct bpf_prog_info __user *uinfo = u64_to_user_ptr(attr->info.info);
5122 struct btf *attach_btf = bpf_prog_get_target_btf(prog);
5123 struct bpf_prog_info info;
5124 u32 info_len = attr->info.info_len;
5125 struct bpf_prog_kstats stats;
5126 char __user *uinsns;
5127 u32 ulen, len;
5128 int err;
5129
5130 len = offsetofend(struct bpf_prog_info, attach_btf_id);
5131 err = bpf_check_uarg_tail_zero(USER_BPFPTR(uinfo), len, info_len);
5132 if (err)
5133 return err;
5134 info_len = min_t(u32, sizeof(info), info_len);
5135
5136 memset(&info, 0, sizeof(info));
5137 if (copy_from_user(&info, uinfo, info_len))
5138 return -EFAULT;
5139
5140 info.type = prog->type;
5141 info.id = prog->aux->id;
5142 info.load_time = prog->aux->load_time;
5143 info.created_by_uid = from_kuid_munged(current_user_ns(),
5144 prog->aux->user->uid);
5145 info.gpl_compatible = prog->gpl_compatible;
5146
5147 memcpy(info.tag, prog->tag, sizeof(prog->tag));
5148 memcpy(info.name, prog->aux->name, sizeof(prog->aux->name));
5149
5150 mutex_lock(&prog->aux->used_maps_mutex);
5151 ulen = info.nr_map_ids;
5152 info.nr_map_ids = prog->aux->used_map_cnt;
5153 ulen = min_t(u32, info.nr_map_ids, ulen);
5154 if (ulen) {
5155 u32 __user *user_map_ids = u64_to_user_ptr(info.map_ids);
5156 u32 i;
5157
5158 for (i = 0; i < ulen; i++)
5159 if (put_user(prog->aux->used_maps[i]->id,
5160 &user_map_ids[i])) {
5161 mutex_unlock(&prog->aux->used_maps_mutex);
5162 return -EFAULT;
5163 }
5164 }
5165 mutex_unlock(&prog->aux->used_maps_mutex);
5166
5167 err = set_info_rec_size(&info);
5168 if (err)
5169 return err;
5170
5171 bpf_prog_get_stats(prog, &stats);
5172 info.run_time_ns = stats.nsecs;
5173 info.run_cnt = stats.cnt;
5174 info.recursion_misses = stats.misses;
5175
5176 info.verified_insns = prog->aux->verified_insns;
5177 if (prog->aux->btf)
5178 info.btf_id = btf_obj_id(prog->aux->btf);
5179
5180 if (!bpf_capable()) {
5181 info.jited_prog_len = 0;
5182 info.xlated_prog_len = 0;
5183 info.nr_jited_ksyms = 0;
5184 info.nr_jited_func_lens = 0;
5185 info.nr_func_info = 0;
5186 info.nr_line_info = 0;
5187 info.nr_jited_line_info = 0;
5188 goto done;
5189 }
5190
5191 ulen = info.xlated_prog_len;
5192 info.xlated_prog_len = bpf_prog_insn_size(prog);
5193 if (info.xlated_prog_len && ulen) {
5194 struct bpf_insn *insns_sanitized;
5195 bool fault;
5196
5197 if (!prog->blinded || bpf_dump_raw_ok(file->f_cred)) {
5198 insns_sanitized = bpf_insn_prepare_dump(prog, file->f_cred);
5199 if (!insns_sanitized)
5200 return -ENOMEM;
5201 uinsns = u64_to_user_ptr(info.xlated_prog_insns);
5202 ulen = min_t(u32, info.xlated_prog_len, ulen);
5203 fault = copy_to_user(uinsns, insns_sanitized, ulen);
5204 kfree(insns_sanitized);
5205 if (fault)
5206 return -EFAULT;
5207 } else {
5208 info.xlated_prog_insns = 0;
5209 }
5210 }
5211
5212 if (bpf_prog_is_offloaded(prog->aux)) {
5213 err = bpf_prog_offload_info_fill(&info, prog);
5214 if (err)
5215 return err;
5216 goto done;
5217 }
5218
5219 /* NOTE: the following code is supposed to be skipped for offload.
5220 * bpf_prog_offload_info_fill() is the place to fill similar fields
5221 * for offload.
5222 */
5223 ulen = info.jited_prog_len;
5224 if (prog->aux->func_cnt) {
5225 u32 i;
5226
5227 info.jited_prog_len = 0;
5228 for (i = 0; i < prog->aux->func_cnt; i++)
5229 info.jited_prog_len += prog->aux->func[i]->jited_len;
5230 } else {
5231 info.jited_prog_len = prog->jited_len;
5232 }
5233
5234 if (info.jited_prog_len && ulen) {
5235 if (bpf_dump_raw_ok(file->f_cred)) {
5236 uinsns = u64_to_user_ptr(info.jited_prog_insns);
5237 ulen = min_t(u32, info.jited_prog_len, ulen);
5238
5239 /* for multi-function programs, copy the JITed
5240 * instructions for all the functions
5241 */
5242 if (prog->aux->func_cnt) {
5243 u32 len, free, i;
5244 u8 *img;
5245
5246 free = ulen;
5247 for (i = 0; i < prog->aux->func_cnt; i++) {
5248 len = prog->aux->func[i]->jited_len;
5249 len = min_t(u32, len, free);
5250 img = (u8 *) prog->aux->func[i]->bpf_func;
5251 if (copy_to_user(uinsns, img, len))
5252 return -EFAULT;
5253 uinsns += len;
5254 free -= len;
5255 if (!free)
5256 break;
5257 }
5258 } else {
5259 if (copy_to_user(uinsns, prog->bpf_func, ulen))
5260 return -EFAULT;
5261 }
5262 } else {
5263 info.jited_prog_insns = 0;
5264 }
5265 }
5266
5267 ulen = info.nr_jited_ksyms;
5268 info.nr_jited_ksyms = prog->aux->func_cnt ? : 1;
5269 if (ulen) {
5270 if (bpf_dump_raw_ok(file->f_cred)) {
5271 unsigned long ksym_addr;
5272 u64 __user *user_ksyms;
5273 u32 i;
5274
5275 /* copy the address of the kernel symbol
5276 * corresponding to each function
5277 */
5278 ulen = min_t(u32, info.nr_jited_ksyms, ulen);
5279 user_ksyms = u64_to_user_ptr(info.jited_ksyms);
5280 if (prog->aux->func_cnt) {
5281 for (i = 0; i < ulen; i++) {
5282 ksym_addr = (unsigned long)
5283 prog->aux->func[i]->bpf_func;
5284 if (put_user((u64) ksym_addr,
5285 &user_ksyms[i]))
5286 return -EFAULT;
5287 }
5288 } else {
5289 ksym_addr = (unsigned long) prog->bpf_func;
5290 if (put_user((u64) ksym_addr, &user_ksyms[0]))
5291 return -EFAULT;
5292 }
5293 } else {
5294 info.jited_ksyms = 0;
5295 }
5296 }
5297
5298 ulen = info.nr_jited_func_lens;
5299 info.nr_jited_func_lens = prog->aux->func_cnt ? : 1;
5300 if (ulen) {
5301 if (bpf_dump_raw_ok(file->f_cred)) {
5302 u32 __user *user_lens;
5303 u32 func_len, i;
5304
5305 /* copy the JITed image lengths for each function */
5306 ulen = min_t(u32, info.nr_jited_func_lens, ulen);
5307 user_lens = u64_to_user_ptr(info.jited_func_lens);
5308 if (prog->aux->func_cnt) {
5309 for (i = 0; i < ulen; i++) {
5310 func_len =
5311 prog->aux->func[i]->jited_len;
5312 if (put_user(func_len, &user_lens[i]))
5313 return -EFAULT;
5314 }
5315 } else {
5316 func_len = prog->jited_len;
5317 if (put_user(func_len, &user_lens[0]))
5318 return -EFAULT;
5319 }
5320 } else {
5321 info.jited_func_lens = 0;
5322 }
5323 }
5324
5325 info.attach_btf_id = prog->aux->attach_btf_id;
5326 if (attach_btf)
5327 info.attach_btf_obj_id = btf_obj_id(attach_btf);
5328
5329 ulen = info.nr_func_info;
5330 info.nr_func_info = prog->aux->func_info_cnt;
5331 if (info.nr_func_info && ulen) {
5332 char __user *user_finfo;
5333
5334 user_finfo = u64_to_user_ptr(info.func_info);
5335 ulen = min_t(u32, info.nr_func_info, ulen);
5336 if (copy_to_user(user_finfo, prog->aux->func_info,
5337 info.func_info_rec_size * ulen))
5338 return -EFAULT;
5339 }
5340
5341 ulen = info.nr_line_info;
5342 info.nr_line_info = prog->aux->nr_linfo;
5343 if (info.nr_line_info && ulen) {
5344 __u8 __user *user_linfo;
5345
5346 user_linfo = u64_to_user_ptr(info.line_info);
5347 ulen = min_t(u32, info.nr_line_info, ulen);
5348 if (copy_to_user(user_linfo, prog->aux->linfo,
5349 info.line_info_rec_size * ulen))
5350 return -EFAULT;
5351 }
5352
5353 ulen = info.nr_jited_line_info;
5354 if (prog->aux->jited_linfo)
5355 info.nr_jited_line_info = prog->aux->nr_linfo;
5356 else
5357 info.nr_jited_line_info = 0;
5358 if (info.nr_jited_line_info && ulen) {
5359 if (bpf_dump_raw_ok(file->f_cred)) {
5360 unsigned long line_addr;
5361 __u64 __user *user_linfo;
5362 u32 i;
5363
5364 user_linfo = u64_to_user_ptr(info.jited_line_info);
5365 ulen = min_t(u32, info.nr_jited_line_info, ulen);
5366 for (i = 0; i < ulen; i++) {
5367 line_addr = (unsigned long)prog->aux->jited_linfo[i];
5368 if (put_user((__u64)line_addr, &user_linfo[i]))
5369 return -EFAULT;
5370 }
5371 } else {
5372 info.jited_line_info = 0;
5373 }
5374 }
5375
5376 ulen = info.nr_prog_tags;
5377 info.nr_prog_tags = prog->aux->func_cnt ? : 1;
5378 if (ulen) {
5379 __u8 __user (*user_prog_tags)[BPF_TAG_SIZE];
5380 u32 i;
5381
5382 user_prog_tags = u64_to_user_ptr(info.prog_tags);
5383 ulen = min_t(u32, info.nr_prog_tags, ulen);
5384 if (prog->aux->func_cnt) {
5385 for (i = 0; i < ulen; i++) {
5386 if (copy_to_user(user_prog_tags[i],
5387 prog->aux->func[i]->tag,
5388 BPF_TAG_SIZE))
5389 return -EFAULT;
5390 }
5391 } else {
5392 if (copy_to_user(user_prog_tags[0],
5393 prog->tag, BPF_TAG_SIZE))
5394 return -EFAULT;
5395 }
5396 }
5397
5398done:
5399 if (copy_to_user(uinfo, &info, info_len) ||
5400 put_user(info_len, &uattr->info.info_len))
5401 return -EFAULT;
5402
5403 return 0;
5404}
5405
5406static int bpf_map_get_info_by_fd(struct file *file,
5407 struct bpf_map *map,
5408 const union bpf_attr *attr,
5409 union bpf_attr __user *uattr)
5410{
5411 struct bpf_map_info __user *uinfo = u64_to_user_ptr(attr->info.info);
5412 struct bpf_map_info info;
5413 u32 info_len = attr->info.info_len, len;
5414 int err;
5415
5416 len = offsetofend(struct bpf_map_info, hash_size);
5417 err = bpf_check_uarg_tail_zero(USER_BPFPTR(uinfo), len, info_len);
5418 if (err)
5419 return err;
5420 info_len = min_t(u32, sizeof(info), info_len);
5421
5422 memset(&info, 0, sizeof(info));
5423 if (copy_from_user(&info, uinfo, info_len))
5424 return -EFAULT;
5425
5426 info.type = map->map_type;
5427 info.id = map->id;
5428 info.key_size = map->key_size;
5429 info.value_size = map->value_size;
5430 info.max_entries = map->max_entries;
5431 info.map_flags = map->map_flags;
5432 info.map_extra = map->map_extra;
5433 memcpy(info.name, map->name, sizeof(map->name));
5434
5435 if (map->btf) {
5436 info.btf_id = btf_obj_id(map->btf);
5437 info.btf_key_type_id = map->btf_key_type_id;
5438 info.btf_value_type_id = map->btf_value_type_id;
5439 }
5440 info.btf_vmlinux_value_type_id = map->btf_vmlinux_value_type_id;
5441 if (map->map_type == BPF_MAP_TYPE_STRUCT_OPS)
5442 bpf_map_struct_ops_info_fill(&info, map);
5443
5444 if (bpf_map_is_offloaded(map)) {
5445 err = bpf_map_offload_info_fill(&info, map);
5446 if (err)
5447 return err;
5448 }
5449
5450 if (info.hash) {
5451 char __user *uhash = u64_to_user_ptr(info.hash);
5452
5453 if (!map->ops->map_get_hash)
5454 return -EINVAL;
5455 if (info.hash_size != sizeof(map->sha))
5456 return -EINVAL;
5457 if (!READ_ONCE(map->frozen))
5458 return -EPERM;
5459
5460 err = map->ops->map_get_hash(map);
5461 if (err != 0)
5462 return err;
5463
5464 if (copy_to_user(uhash, map->sha, sizeof(map->sha)) != 0)
5465 return -EFAULT;
5466 } else if (info.hash_size) {
5467 return -EINVAL;
5468 }
5469
5470 if (copy_to_user(uinfo, &info, info_len) ||
5471 put_user(info_len, &uattr->info.info_len))
5472 return -EFAULT;
5473
5474 return 0;
5475}
5476
5477static int bpf_btf_get_info_by_fd(struct file *file,
5478 struct btf *btf,
5479 const union bpf_attr *attr,
5480 union bpf_attr __user *uattr)
5481{
5482 struct bpf_btf_info __user *uinfo = u64_to_user_ptr(attr->info.info);
5483 u32 info_len = attr->info.info_len;
5484 int err;
5485
5486 err = bpf_check_uarg_tail_zero(USER_BPFPTR(uinfo), sizeof(*uinfo), info_len);
5487 if (err)
5488 return err;
5489
5490 return btf_get_info_by_fd(btf, attr, uattr);
5491}
5492
5493static int bpf_link_get_info_by_fd(struct file *file,
5494 struct bpf_link *link,
5495 const union bpf_attr *attr,
5496 union bpf_attr __user *uattr)
5497{
5498 struct bpf_link_info __user *uinfo = u64_to_user_ptr(attr->info.info);
5499 struct bpf_link_info info;
5500 const struct bpf_prog *prog;
5501 u32 info_len = attr->info.info_len;
5502 int err;
5503
5504 err = bpf_check_uarg_tail_zero(USER_BPFPTR(uinfo), sizeof(info), info_len);
5505 if (err)
5506 return err;
5507 info_len = min_t(u32, sizeof(info), info_len);
5508
5509 memset(&info, 0, sizeof(info));
5510 if (copy_from_user(&info, uinfo, info_len))
5511 return -EFAULT;
5512
5513 info.type = link->type;
5514 info.id = link->id;
5515
5516 rcu_read_lock();
5517 prog = READ_ONCE(link->prog);
5518 if (prog)
5519 info.prog_id = prog->aux->id;
5520 rcu_read_unlock();
5521
5522 if (link->ops->fill_link_info) {
5523 err = link->ops->fill_link_info(link, &info);
5524 if (err)
5525 return err;
5526 }
5527
5528 if (copy_to_user(uinfo, &info, info_len) ||
5529 put_user(info_len, &uattr->info.info_len))
5530 return -EFAULT;
5531
5532 return 0;
5533}
5534
5535static int token_get_info_by_fd(struct file *file,
5536 struct bpf_token *token,
5537 const union bpf_attr *attr,
5538 union bpf_attr __user *uattr)
5539{
5540 struct bpf_token_info __user *uinfo = u64_to_user_ptr(attr->info.info);
5541 u32 info_len = attr->info.info_len;
5542 int err;
5543
5544 err = bpf_check_uarg_tail_zero(USER_BPFPTR(uinfo), sizeof(*uinfo), info_len);
5545 if (err)
5546 return err;
5547 return bpf_token_get_info_by_fd(token, attr, uattr);
5548}
5549
5550#define BPF_OBJ_GET_INFO_BY_FD_LAST_FIELD info.info
5551
5552static int bpf_obj_get_info_by_fd(const union bpf_attr *attr,
5553 union bpf_attr __user *uattr)
5554{
5555 if (CHECK_ATTR(BPF_OBJ_GET_INFO_BY_FD))
5556 return -EINVAL;
5557
5558 CLASS(fd, f)(attr->info.bpf_fd);
5559 if (fd_empty(f))
5560 return -EBADFD;
5561
5562 if (fd_file(f)->f_op == &bpf_prog_fops)
5563 return bpf_prog_get_info_by_fd(fd_file(f), fd_file(f)->private_data, attr,
5564 uattr);
5565 else if (fd_file(f)->f_op == &bpf_map_fops)
5566 return bpf_map_get_info_by_fd(fd_file(f), fd_file(f)->private_data, attr,
5567 uattr);
5568 else if (fd_file(f)->f_op == &btf_fops)
5569 return bpf_btf_get_info_by_fd(fd_file(f), fd_file(f)->private_data, attr, uattr);
5570 else if (fd_file(f)->f_op == &bpf_link_fops || fd_file(f)->f_op == &bpf_link_fops_poll)
5571 return bpf_link_get_info_by_fd(fd_file(f), fd_file(f)->private_data,
5572 attr, uattr);
5573 else if (fd_file(f)->f_op == &bpf_token_fops)
5574 return token_get_info_by_fd(fd_file(f), fd_file(f)->private_data,
5575 attr, uattr);
5576 return -EINVAL;
5577}
5578
5579#define BPF_BTF_LOAD_LAST_FIELD btf_token_fd
5580
5581static int bpf_btf_load(const union bpf_attr *attr, bpfptr_t uattr, struct bpf_log_attr *attr_log)
5582{
5583 struct bpf_token *token = NULL;
5584
5585 if (CHECK_ATTR(BPF_BTF_LOAD))
5586 return -EINVAL;
5587
5588 if (attr->btf_flags & ~BPF_F_TOKEN_FD)
5589 return -EINVAL;
5590
5591 if (attr->btf_flags & BPF_F_TOKEN_FD) {
5592 token = bpf_token_get_from_fd(attr->btf_token_fd);
5593 if (IS_ERR(token))
5594 return PTR_ERR(token);
5595 if (!bpf_token_allow_cmd(token, BPF_BTF_LOAD)) {
5596 bpf_token_put(token);
5597 token = NULL;
5598 }
5599 }
5600
5601 if (!bpf_token_capable(token, CAP_BPF)) {
5602 bpf_token_put(token);
5603 return -EPERM;
5604 }
5605
5606 bpf_token_put(token);
5607
5608 return btf_new_fd(attr, uattr, attr_log);
5609}
5610
5611#define BPF_BTF_GET_FD_BY_ID_LAST_FIELD fd_by_id_token_fd
5612
5613static int bpf_btf_get_fd_by_id(const union bpf_attr *attr)
5614{
5615 struct bpf_token *token = NULL;
5616
5617 if (CHECK_ATTR(BPF_BTF_GET_FD_BY_ID))
5618 return -EINVAL;
5619
5620 if (attr->open_flags & ~BPF_F_TOKEN_FD)
5621 return -EINVAL;
5622
5623 if (attr->open_flags & BPF_F_TOKEN_FD) {
5624 token = bpf_token_get_from_fd(attr->fd_by_id_token_fd);
5625 if (IS_ERR(token))
5626 return PTR_ERR(token);
5627 if (!bpf_token_allow_cmd(token, BPF_BTF_GET_FD_BY_ID)) {
5628 bpf_token_put(token);
5629 token = NULL;
5630 }
5631 }
5632
5633 if (!bpf_token_capable(token, CAP_SYS_ADMIN)) {
5634 bpf_token_put(token);
5635 return -EPERM;
5636 }
5637
5638 bpf_token_put(token);
5639
5640 return btf_get_fd_by_id(attr->btf_id);
5641}
5642
5643static int bpf_task_fd_query_copy(const union bpf_attr *attr,
5644 union bpf_attr __user *uattr,
5645 u32 prog_id, u32 fd_type,
5646 const char *buf, u64 probe_offset,
5647 u64 probe_addr)
5648{
5649 char __user *ubuf = u64_to_user_ptr(attr->task_fd_query.buf);
5650 u32 len = buf ? strlen(buf) : 0, input_len;
5651 int err = 0;
5652
5653 if (put_user(len, &uattr->task_fd_query.buf_len))
5654 return -EFAULT;
5655 input_len = attr->task_fd_query.buf_len;
5656 if (input_len && ubuf) {
5657 if (!len) {
5658 /* nothing to copy, just make ubuf NULL terminated */
5659 char zero = '\0';
5660
5661 if (put_user(zero, ubuf))
5662 return -EFAULT;
5663 } else {
5664 err = bpf_copy_to_user(ubuf, buf, input_len, len);
5665 if (err == -EFAULT)
5666 return err;
5667 }
5668 }
5669
5670 if (put_user(prog_id, &uattr->task_fd_query.prog_id) ||
5671 put_user(fd_type, &uattr->task_fd_query.fd_type) ||
5672 put_user(probe_offset, &uattr->task_fd_query.probe_offset) ||
5673 put_user(probe_addr, &uattr->task_fd_query.probe_addr))
5674 return -EFAULT;
5675
5676 return err;
5677}
5678
5679#define BPF_TASK_FD_QUERY_LAST_FIELD task_fd_query.probe_addr
5680
5681static int bpf_task_fd_query(const union bpf_attr *attr,
5682 union bpf_attr __user *uattr)
5683{
5684 pid_t pid = attr->task_fd_query.pid;
5685 u32 fd = attr->task_fd_query.fd;
5686 const struct perf_event *event;
5687 struct task_struct *task;
5688 struct file *file;
5689 int err;
5690
5691 if (CHECK_ATTR(BPF_TASK_FD_QUERY))
5692 return -EINVAL;
5693
5694 if (!capable(CAP_SYS_ADMIN))
5695 return -EPERM;
5696
5697 if (attr->task_fd_query.flags != 0)
5698 return -EINVAL;
5699
5700 rcu_read_lock();
5701 task = get_pid_task(find_vpid(pid), PIDTYPE_PID);
5702 rcu_read_unlock();
5703 if (!task)
5704 return -ENOENT;
5705
5706 err = 0;
5707 file = fget_task(task, fd);
5708 put_task_struct(task);
5709 if (!file)
5710 return -EBADF;
5711
5712 if (file->f_op == &bpf_link_fops || file->f_op == &bpf_link_fops_poll) {
5713 struct bpf_link *link = file->private_data;
5714
5715 if (link->ops == &bpf_raw_tp_link_lops) {
5716 struct bpf_raw_tp_link *raw_tp =
5717 container_of(link, struct bpf_raw_tp_link, link);
5718 struct bpf_raw_event_map *btp = raw_tp->btp;
5719
5720 err = bpf_task_fd_query_copy(attr, uattr,
5721 raw_tp->link.prog->aux->id,
5722 BPF_FD_TYPE_RAW_TRACEPOINT,
5723 btp->tp->name, 0, 0);
5724 goto put_file;
5725 }
5726 goto out_not_supp;
5727 }
5728
5729 event = perf_get_event(file);
5730 if (!IS_ERR(event)) {
5731 u64 probe_offset, probe_addr;
5732 u32 prog_id, fd_type;
5733 const char *buf;
5734
5735 err = bpf_get_perf_event_info(event, &prog_id, &fd_type,
5736 &buf, &probe_offset,
5737 &probe_addr, NULL);
5738 if (!err)
5739 err = bpf_task_fd_query_copy(attr, uattr, prog_id,
5740 fd_type, buf,
5741 probe_offset,
5742 probe_addr);
5743 goto put_file;
5744 }
5745
5746out_not_supp:
5747 err = -ENOTSUPP;
5748put_file:
5749 fput(file);
5750 return err;
5751}
5752
5753#define BPF_MAP_BATCH_LAST_FIELD batch.flags
5754
5755#define BPF_DO_BATCH(fn, ...) \
5756 do { \
5757 if (!fn) { \
5758 err = -ENOTSUPP; \
5759 goto err_put; \
5760 } \
5761 err = fn(__VA_ARGS__); \
5762 } while (0)
5763
5764static int bpf_map_do_batch(const union bpf_attr *attr,
5765 union bpf_attr __user *uattr,
5766 int cmd)
5767{
5768 bool has_read = cmd == BPF_MAP_LOOKUP_BATCH ||
5769 cmd == BPF_MAP_LOOKUP_AND_DELETE_BATCH;
5770 bool has_write = cmd != BPF_MAP_LOOKUP_BATCH;
5771 struct bpf_map *map;
5772 int err;
5773
5774 if (CHECK_ATTR(BPF_MAP_BATCH))
5775 return -EINVAL;
5776
5777 CLASS(fd, f)(attr->batch.map_fd);
5778
5779 map = __bpf_map_get(f);
5780 if (IS_ERR(map))
5781 return PTR_ERR(map);
5782 if (has_write)
5783 bpf_map_write_active_inc(map);
5784 if (has_read && !(map_get_sys_perms(map, f) & FMODE_CAN_READ)) {
5785 err = -EPERM;
5786 goto err_put;
5787 }
5788 if (has_write && !(map_get_sys_perms(map, f) & FMODE_CAN_WRITE)) {
5789 err = -EPERM;
5790 goto err_put;
5791 }
5792
5793 if (cmd == BPF_MAP_LOOKUP_BATCH)
5794 BPF_DO_BATCH(map->ops->map_lookup_batch, map, attr, uattr);
5795 else if (cmd == BPF_MAP_LOOKUP_AND_DELETE_BATCH)
5796 BPF_DO_BATCH(map->ops->map_lookup_and_delete_batch, map, attr, uattr);
5797 else if (cmd == BPF_MAP_UPDATE_BATCH)
5798 BPF_DO_BATCH(map->ops->map_update_batch, map, fd_file(f), attr, uattr);
5799 else
5800 BPF_DO_BATCH(map->ops->map_delete_batch, map, attr, uattr);
5801err_put:
5802 if (has_write) {
5803 maybe_wait_bpf_programs(map);
5804 bpf_map_write_active_dec(map);
5805 }
5806 return err;
5807}
5808
5809#define BPF_LINK_CREATE_LAST_FIELD link_create.uprobe_multi.path_fd
5810static int link_create(union bpf_attr *attr, bpfptr_t uattr)
5811{
5812 struct bpf_prog *prog;
5813 int ret;
5814
5815 if (CHECK_ATTR(BPF_LINK_CREATE))
5816 return -EINVAL;
5817
5818 if (attr->link_create.attach_type == BPF_STRUCT_OPS)
5819 return bpf_struct_ops_link_create(attr);
5820
5821 prog = bpf_prog_get(attr->link_create.prog_fd);
5822 if (IS_ERR(prog))
5823 return PTR_ERR(prog);
5824
5825 ret = bpf_prog_attach_check_attach_type(prog,
5826 attr->link_create.attach_type);
5827 if (ret)
5828 goto out;
5829
5830 switch (prog->type) {
5831 case BPF_PROG_TYPE_CGROUP_SKB:
5832 case BPF_PROG_TYPE_CGROUP_SOCK:
5833 case BPF_PROG_TYPE_CGROUP_SOCK_ADDR:
5834 case BPF_PROG_TYPE_SOCK_OPS:
5835 case BPF_PROG_TYPE_CGROUP_DEVICE:
5836 case BPF_PROG_TYPE_CGROUP_SYSCTL:
5837 case BPF_PROG_TYPE_CGROUP_SOCKOPT:
5838 ret = cgroup_bpf_link_attach(attr, prog);
5839 break;
5840 case BPF_PROG_TYPE_EXT:
5841 ret = bpf_tracing_prog_attach(prog,
5842 attr->link_create.target_fd,
5843 attr->link_create.target_btf_id,
5844 attr->link_create.tracing.cookie,
5845 attr->link_create.attach_type);
5846 break;
5847 case BPF_PROG_TYPE_LSM:
5848 case BPF_PROG_TYPE_TRACING:
5849 if (attr->link_create.attach_type != prog->expected_attach_type) {
5850 ret = -EINVAL;
5851 goto out;
5852 }
5853 if (prog->expected_attach_type == BPF_TRACE_RAW_TP)
5854 ret = bpf_raw_tp_link_attach(prog, NULL, attr->link_create.tracing.cookie,
5855 attr->link_create.attach_type);
5856 else if (prog->expected_attach_type == BPF_TRACE_ITER)
5857 ret = bpf_iter_link_attach(attr, uattr, prog);
5858 else if (prog->expected_attach_type == BPF_LSM_CGROUP)
5859 ret = cgroup_bpf_link_attach(attr, prog);
5860 else if (is_tracing_multi(prog->expected_attach_type))
5861 ret = bpf_tracing_multi_attach(prog, attr);
5862 else
5863 ret = bpf_tracing_prog_attach(prog,
5864 attr->link_create.target_fd,
5865 attr->link_create.target_btf_id,
5866 attr->link_create.tracing.cookie,
5867 attr->link_create.attach_type);
5868 break;
5869 case BPF_PROG_TYPE_FLOW_DISSECTOR:
5870 case BPF_PROG_TYPE_SK_LOOKUP:
5871 ret = netns_bpf_link_create(attr, prog);
5872 break;
5873 case BPF_PROG_TYPE_SK_MSG:
5874 case BPF_PROG_TYPE_SK_SKB:
5875 ret = sock_map_link_create(attr, prog);
5876 break;
5877#ifdef CONFIG_NET
5878 case BPF_PROG_TYPE_XDP:
5879 ret = bpf_xdp_link_attach(attr, prog);
5880 break;
5881 case BPF_PROG_TYPE_SCHED_CLS:
5882 if (attr->link_create.attach_type == BPF_TCX_INGRESS ||
5883 attr->link_create.attach_type == BPF_TCX_EGRESS)
5884 ret = tcx_link_attach(attr, prog);
5885 else
5886 ret = netkit_link_attach(attr, prog);
5887 break;
5888 case BPF_PROG_TYPE_NETFILTER:
5889 ret = bpf_nf_link_attach(attr, prog);
5890 break;
5891#endif
5892 case BPF_PROG_TYPE_PERF_EVENT:
5893 case BPF_PROG_TYPE_TRACEPOINT:
5894 ret = bpf_perf_link_attach(attr, prog);
5895 break;
5896 case BPF_PROG_TYPE_KPROBE:
5897 if (attr->link_create.attach_type == BPF_PERF_EVENT)
5898 ret = bpf_perf_link_attach(attr, prog);
5899 else if (attr->link_create.attach_type == BPF_TRACE_KPROBE_MULTI ||
5900 attr->link_create.attach_type == BPF_TRACE_KPROBE_SESSION)
5901 ret = bpf_kprobe_multi_link_attach(attr, prog);
5902 else if (attr->link_create.attach_type == BPF_TRACE_UPROBE_MULTI ||
5903 attr->link_create.attach_type == BPF_TRACE_UPROBE_SESSION)
5904 ret = bpf_uprobe_multi_link_attach(attr, prog);
5905 break;
5906 default:
5907 ret = -EINVAL;
5908 }
5909
5910out:
5911 if (ret < 0)
5912 bpf_prog_put(prog);
5913 return ret;
5914}
5915
5916static int link_update_map(struct bpf_link *link, union bpf_attr *attr)
5917{
5918 struct bpf_map *new_map, *old_map = NULL;
5919 int ret;
5920
5921 new_map = bpf_map_get(attr->link_update.new_map_fd);
5922 if (IS_ERR(new_map))
5923 return PTR_ERR(new_map);
5924
5925 if (attr->link_update.flags & BPF_F_REPLACE) {
5926 old_map = bpf_map_get(attr->link_update.old_map_fd);
5927 if (IS_ERR(old_map)) {
5928 ret = PTR_ERR(old_map);
5929 goto out_put;
5930 }
5931 } else if (attr->link_update.old_map_fd) {
5932 ret = -EINVAL;
5933 goto out_put;
5934 }
5935
5936 ret = link->ops->update_map(link, new_map, old_map);
5937
5938 if (old_map)
5939 bpf_map_put(old_map);
5940out_put:
5941 bpf_map_put(new_map);
5942 return ret;
5943}
5944
5945#define BPF_LINK_UPDATE_LAST_FIELD link_update.old_prog_fd
5946
5947static int link_update(union bpf_attr *attr)
5948{
5949 struct bpf_prog *old_prog = NULL, *new_prog;
5950 struct bpf_link *link;
5951 u32 flags;
5952 int ret;
5953
5954 if (CHECK_ATTR(BPF_LINK_UPDATE))
5955 return -EINVAL;
5956
5957 flags = attr->link_update.flags;
5958 if (flags & ~BPF_F_REPLACE)
5959 return -EINVAL;
5960
5961 link = bpf_link_get_from_fd(attr->link_update.link_fd);
5962 if (IS_ERR(link))
5963 return PTR_ERR(link);
5964
5965 if (link->ops->update_map) {
5966 ret = link_update_map(link, attr);
5967 goto out_put_link;
5968 }
5969
5970 new_prog = bpf_prog_get(attr->link_update.new_prog_fd);
5971 if (IS_ERR(new_prog)) {
5972 ret = PTR_ERR(new_prog);
5973 goto out_put_link;
5974 }
5975
5976 if (flags & BPF_F_REPLACE) {
5977 old_prog = bpf_prog_get(attr->link_update.old_prog_fd);
5978 if (IS_ERR(old_prog)) {
5979 ret = PTR_ERR(old_prog);
5980 old_prog = NULL;
5981 goto out_put_progs;
5982 }
5983 } else if (attr->link_update.old_prog_fd) {
5984 ret = -EINVAL;
5985 goto out_put_progs;
5986 }
5987
5988 if (link->ops->update_prog)
5989 ret = link->ops->update_prog(link, new_prog, old_prog);
5990 else
5991 ret = -EINVAL;
5992
5993out_put_progs:
5994 if (old_prog)
5995 bpf_prog_put(old_prog);
5996 if (ret)
5997 bpf_prog_put(new_prog);
5998out_put_link:
5999 bpf_link_put_direct(link);
6000 return ret;
6001}
6002
6003#define BPF_LINK_DETACH_LAST_FIELD link_detach.link_fd
6004
6005static int link_detach(union bpf_attr *attr)
6006{
6007 struct bpf_link *link;
6008 int ret;
6009
6010 if (CHECK_ATTR(BPF_LINK_DETACH))
6011 return -EINVAL;
6012
6013 link = bpf_link_get_from_fd(attr->link_detach.link_fd);
6014 if (IS_ERR(link))
6015 return PTR_ERR(link);
6016
6017 if (link->ops->detach)
6018 ret = link->ops->detach(link);
6019 else
6020 ret = -EOPNOTSUPP;
6021
6022 bpf_link_put_direct(link);
6023 return ret;
6024}
6025
6026struct bpf_link *bpf_link_inc_not_zero(struct bpf_link *link)
6027{
6028 return atomic64_fetch_add_unless(&link->refcnt, 1, 0) ? link : ERR_PTR(-ENOENT);
6029}
6030EXPORT_SYMBOL(bpf_link_inc_not_zero);
6031
6032struct bpf_link *bpf_link_by_id(u32 id)
6033{
6034 struct bpf_link *link;
6035
6036 if (!id)
6037 return ERR_PTR(-ENOENT);
6038
6039 spin_lock_bh(&link_idr_lock);
6040 /* before link is "settled", ID is 0, pretend it doesn't exist yet */
6041 link = idr_find(&link_idr, id);
6042 if (link) {
6043 if (link->id)
6044 link = bpf_link_inc_not_zero(link);
6045 else
6046 link = ERR_PTR(-EAGAIN);
6047 } else {
6048 link = ERR_PTR(-ENOENT);
6049 }
6050 spin_unlock_bh(&link_idr_lock);
6051 return link;
6052}
6053
6054struct bpf_link *bpf_link_get_curr_or_next(u32 *id)
6055{
6056 struct bpf_link *link;
6057
6058 spin_lock_bh(&link_idr_lock);
6059again:
6060 link = idr_get_next(&link_idr, id);
6061 if (link) {
6062 if (link->id)
6063 link = bpf_link_inc_not_zero(link);
6064 else
6065 link = ERR_PTR(-EAGAIN);
6066 if (IS_ERR(link)) {
6067 (*id)++;
6068 goto again;
6069 }
6070 }
6071 spin_unlock_bh(&link_idr_lock);
6072
6073 return link;
6074}
6075
6076#define BPF_LINK_GET_FD_BY_ID_LAST_FIELD link_id
6077
6078static int bpf_link_get_fd_by_id(const union bpf_attr *attr)
6079{
6080 struct bpf_link *link;
6081 u32 id = attr->link_id;
6082 int fd;
6083
6084 if (CHECK_ATTR(BPF_LINK_GET_FD_BY_ID))
6085 return -EINVAL;
6086
6087 if (!capable(CAP_SYS_ADMIN))
6088 return -EPERM;
6089
6090 link = bpf_link_by_id(id);
6091 if (IS_ERR(link))
6092 return PTR_ERR(link);
6093
6094 fd = bpf_link_new_fd(link);
6095 if (fd < 0)
6096 bpf_link_put_direct(link);
6097
6098 return fd;
6099}
6100
6101DEFINE_MUTEX(bpf_stats_enabled_mutex);
6102
6103static int bpf_stats_release(struct inode *inode, struct file *file)
6104{
6105 mutex_lock(&bpf_stats_enabled_mutex);
6106 static_key_slow_dec(&bpf_stats_enabled_key.key);
6107 mutex_unlock(&bpf_stats_enabled_mutex);
6108 return 0;
6109}
6110
6111static const struct file_operations bpf_stats_fops = {
6112 .release = bpf_stats_release,
6113};
6114
6115static int bpf_enable_runtime_stats(void)
6116{
6117 int fd;
6118
6119 mutex_lock(&bpf_stats_enabled_mutex);
6120
6121 /* Set a very high limit to avoid overflow */
6122 if (static_key_count(&bpf_stats_enabled_key.key) > INT_MAX / 2) {
6123 mutex_unlock(&bpf_stats_enabled_mutex);
6124 return -EBUSY;
6125 }
6126
6127 fd = anon_inode_getfd("bpf-stats", &bpf_stats_fops, NULL, O_CLOEXEC);
6128 if (fd >= 0)
6129 static_key_slow_inc(&bpf_stats_enabled_key.key);
6130
6131 mutex_unlock(&bpf_stats_enabled_mutex);
6132 return fd;
6133}
6134
6135#define BPF_ENABLE_STATS_LAST_FIELD enable_stats.type
6136
6137static int bpf_enable_stats(union bpf_attr *attr)
6138{
6139
6140 if (CHECK_ATTR(BPF_ENABLE_STATS))
6141 return -EINVAL;
6142
6143 if (!capable(CAP_SYS_ADMIN))
6144 return -EPERM;
6145
6146 switch (attr->enable_stats.type) {
6147 case BPF_STATS_RUN_TIME:
6148 return bpf_enable_runtime_stats();
6149 default:
6150 break;
6151 }
6152 return -EINVAL;
6153}
6154
6155#define BPF_ITER_CREATE_LAST_FIELD iter_create.flags
6156
6157static int bpf_iter_create(union bpf_attr *attr)
6158{
6159 struct bpf_link *link;
6160 int err;
6161
6162 if (CHECK_ATTR(BPF_ITER_CREATE))
6163 return -EINVAL;
6164
6165 if (attr->iter_create.flags)
6166 return -EINVAL;
6167
6168 link = bpf_link_get_from_fd(attr->iter_create.link_fd);
6169 if (IS_ERR(link))
6170 return PTR_ERR(link);
6171
6172 err = bpf_iter_new_fd(link);
6173 bpf_link_put_direct(link);
6174
6175 return err;
6176}
6177
6178#define BPF_PROG_BIND_MAP_LAST_FIELD prog_bind_map.flags
6179
6180static int bpf_prog_bind_map(union bpf_attr *attr)
6181{
6182 struct bpf_prog *prog;
6183 struct bpf_map *map;
6184 struct bpf_map **used_maps_old, **used_maps_new;
6185 int i, ret = 0;
6186
6187 if (CHECK_ATTR(BPF_PROG_BIND_MAP))
6188 return -EINVAL;
6189
6190 if (attr->prog_bind_map.flags)
6191 return -EINVAL;
6192
6193 prog = bpf_prog_get(attr->prog_bind_map.prog_fd);
6194 if (IS_ERR(prog))
6195 return PTR_ERR(prog);
6196
6197 map = bpf_map_get(attr->prog_bind_map.map_fd);
6198 if (IS_ERR(map)) {
6199 ret = PTR_ERR(map);
6200 goto out_prog_put;
6201 }
6202
6203 mutex_lock(&prog->aux->used_maps_mutex);
6204
6205 used_maps_old = prog->aux->used_maps;
6206
6207 for (i = 0; i < prog->aux->used_map_cnt; i++)
6208 if (used_maps_old[i] == map) {
6209 bpf_map_put(map);
6210 goto out_unlock;
6211 }
6212
6213 used_maps_new = kmalloc_objs(used_maps_new[0],
6214 prog->aux->used_map_cnt + 1);
6215 if (!used_maps_new) {
6216 ret = -ENOMEM;
6217 goto out_unlock;
6218 }
6219
6220 /* The bpf program will not access the bpf map, but for the sake of
6221 * simplicity, increase sleepable_refcnt for sleepable program as well.
6222 */
6223 if (prog->sleepable)
6224 atomic64_inc(&map->sleepable_refcnt);
6225 memcpy(used_maps_new, used_maps_old,
6226 sizeof(used_maps_old[0]) * prog->aux->used_map_cnt);
6227 used_maps_new[prog->aux->used_map_cnt] = map;
6228
6229 prog->aux->used_map_cnt++;
6230 prog->aux->used_maps = used_maps_new;
6231
6232 kfree(used_maps_old);
6233
6234out_unlock:
6235 mutex_unlock(&prog->aux->used_maps_mutex);
6236
6237 if (ret)
6238 bpf_map_put(map);
6239out_prog_put:
6240 bpf_prog_put(prog);
6241 return ret;
6242}
6243
6244#define BPF_TOKEN_CREATE_LAST_FIELD token_create.bpffs_fd
6245
6246static int token_create(union bpf_attr *attr)
6247{
6248 if (CHECK_ATTR(BPF_TOKEN_CREATE))
6249 return -EINVAL;
6250
6251 /* no flags are supported yet */
6252 if (attr->token_create.flags)
6253 return -EINVAL;
6254
6255 return bpf_token_create(attr);
6256}
6257
6258#define BPF_PROG_STREAM_READ_BY_FD_LAST_FIELD prog_stream_read.prog_fd
6259
6260static int prog_stream_read(union bpf_attr *attr)
6261{
6262 char __user *buf = u64_to_user_ptr(attr->prog_stream_read.stream_buf);
6263 u32 len = attr->prog_stream_read.stream_buf_len;
6264 struct bpf_prog *prog;
6265 int ret;
6266
6267 if (CHECK_ATTR(BPF_PROG_STREAM_READ_BY_FD))
6268 return -EINVAL;
6269
6270 prog = bpf_prog_get(attr->prog_stream_read.prog_fd);
6271 if (IS_ERR(prog))
6272 return PTR_ERR(prog);
6273
6274 ret = bpf_prog_stream_read(prog, attr->prog_stream_read.stream_id, buf, len);
6275 bpf_prog_put(prog);
6276
6277 return ret;
6278}
6279
6280#define BPF_PROG_ASSOC_STRUCT_OPS_LAST_FIELD prog_assoc_struct_ops.flags
6281
6282static int prog_assoc_struct_ops(union bpf_attr *attr)
6283{
6284 struct bpf_prog *prog;
6285 struct bpf_map *map;
6286 int ret;
6287
6288 if (CHECK_ATTR(BPF_PROG_ASSOC_STRUCT_OPS))
6289 return -EINVAL;
6290
6291 if (attr->prog_assoc_struct_ops.flags)
6292 return -EINVAL;
6293
6294 prog = bpf_prog_get(attr->prog_assoc_struct_ops.prog_fd);
6295 if (IS_ERR(prog))
6296 return PTR_ERR(prog);
6297
6298 if (prog->type == BPF_PROG_TYPE_STRUCT_OPS) {
6299 ret = -EINVAL;
6300 goto put_prog;
6301 }
6302
6303 map = bpf_map_get(attr->prog_assoc_struct_ops.map_fd);
6304 if (IS_ERR(map)) {
6305 ret = PTR_ERR(map);
6306 goto put_prog;
6307 }
6308
6309 if (map->map_type != BPF_MAP_TYPE_STRUCT_OPS) {
6310 ret = -EINVAL;
6311 goto put_map;
6312 }
6313
6314 ret = bpf_prog_assoc_struct_ops(prog, map);
6315
6316put_map:
6317 bpf_map_put(map);
6318put_prog:
6319 bpf_prog_put(prog);
6320 return ret;
6321}
6322
6323static int __sys_bpf(enum bpf_cmd cmd, bpfptr_t uattr, unsigned int size,
6324 bpfptr_t uattr_common, unsigned int size_common)
6325{
6326 struct bpf_common_attr attr_common;
6327 u32 offsetof_log_true_size = 0;
6328 struct bpf_log_attr attr_log;
6329 union bpf_attr attr;
6330 int err;
6331
6332 err = bpf_check_uarg_tail_zero(uattr, sizeof(attr), size);
6333 if (err)
6334 return err;
6335 size = min_t(u32, size, sizeof(attr));
6336
6337 /* copy attributes from user space, may be less than sizeof(bpf_attr) */
6338 memset(&attr, 0, sizeof(attr));
6339 if (copy_from_bpfptr(&attr, uattr, size) != 0)
6340 return -EFAULT;
6341
6342 memset(&attr_common, 0, sizeof(attr_common));
6343 if (cmd & BPF_COMMON_ATTRS) {
6344 err = bpf_check_uarg_tail_zero(uattr_common,
6345 offsetofend(struct bpf_common_attr, log_true_size),
6346 size_common);
6347 if (err)
6348 return err;
6349
6350 cmd &= ~BPF_COMMON_ATTRS;
6351 size_common = min_t(u32, size_common, sizeof(attr_common));
6352 if (copy_from_bpfptr(&attr_common, uattr_common, size_common) != 0)
6353 return -EFAULT;
6354 } else {
6355 size_common = 0;
6356 }
6357
6358 err = security_bpf(cmd, &attr, size, uattr.is_kernel);
6359 if (err < 0)
6360 return err;
6361
6362 switch (cmd) {
6363 case BPF_MAP_CREATE:
6364 err = map_create(&attr, uattr, &attr_common, uattr_common, size_common);
6365 break;
6366 case BPF_MAP_LOOKUP_ELEM:
6367 err = map_lookup_elem(&attr);
6368 break;
6369 case BPF_MAP_UPDATE_ELEM:
6370 err = map_update_elem(&attr, uattr);
6371 break;
6372 case BPF_MAP_DELETE_ELEM:
6373 err = map_delete_elem(&attr, uattr);
6374 break;
6375 case BPF_MAP_GET_NEXT_KEY:
6376 err = map_get_next_key(&attr);
6377 break;
6378 case BPF_MAP_FREEZE:
6379 err = map_freeze(&attr);
6380 break;
6381 case BPF_PROG_LOAD:
6382 if (size >= offsetofend(union bpf_attr, log_true_size))
6383 offsetof_log_true_size = offsetof(union bpf_attr, log_true_size);
6384 err = bpf_log_attr_init(&attr_log, attr.log_buf, attr.log_size, attr.log_level,
6385 offsetof_log_true_size, uattr, &attr_common, uattr_common,
6386 size_common);
6387 err = err ?: bpf_prog_load(&attr, uattr, &attr_log);
6388 break;
6389 case BPF_OBJ_PIN:
6390 err = bpf_obj_pin(&attr);
6391 break;
6392 case BPF_OBJ_GET:
6393 err = bpf_obj_get(&attr);
6394 break;
6395 case BPF_PROG_ATTACH:
6396 err = bpf_prog_attach(&attr);
6397 break;
6398 case BPF_PROG_DETACH:
6399 err = bpf_prog_detach(&attr);
6400 break;
6401 case BPF_PROG_QUERY:
6402 err = bpf_prog_query(&attr, uattr.user, size);
6403 break;
6404 case BPF_PROG_TEST_RUN:
6405 err = bpf_prog_test_run(&attr, uattr.user);
6406 break;
6407 case BPF_PROG_GET_NEXT_ID:
6408 err = bpf_obj_get_next_id(&attr, uattr.user,
6409 &prog_idr, &prog_idr_lock);
6410 break;
6411 case BPF_MAP_GET_NEXT_ID:
6412 err = bpf_obj_get_next_id(&attr, uattr.user,
6413 &map_idr, &map_idr_lock);
6414 break;
6415 case BPF_BTF_GET_NEXT_ID:
6416 err = bpf_obj_get_next_id(&attr, uattr.user,
6417 &btf_idr, &btf_idr_lock);
6418 break;
6419 case BPF_PROG_GET_FD_BY_ID:
6420 err = bpf_prog_get_fd_by_id(&attr);
6421 break;
6422 case BPF_MAP_GET_FD_BY_ID:
6423 err = bpf_map_get_fd_by_id(&attr);
6424 break;
6425 case BPF_OBJ_GET_INFO_BY_FD:
6426 err = bpf_obj_get_info_by_fd(&attr, uattr.user);
6427 break;
6428 case BPF_RAW_TRACEPOINT_OPEN:
6429 err = bpf_raw_tracepoint_open(&attr);
6430 break;
6431 case BPF_BTF_LOAD:
6432 if (size >= offsetofend(union bpf_attr, btf_log_true_size))
6433 offsetof_log_true_size = offsetof(union bpf_attr, btf_log_true_size);
6434 err = bpf_log_attr_init(&attr_log, attr.btf_log_buf, attr.btf_log_size,
6435 attr.btf_log_level, offsetof_log_true_size, uattr,
6436 &attr_common, uattr_common, size_common);
6437 err = err ?: bpf_btf_load(&attr, uattr, &attr_log);
6438 break;
6439 case BPF_BTF_GET_FD_BY_ID:
6440 err = bpf_btf_get_fd_by_id(&attr);
6441 break;
6442 case BPF_TASK_FD_QUERY:
6443 err = bpf_task_fd_query(&attr, uattr.user);
6444 break;
6445 case BPF_MAP_LOOKUP_AND_DELETE_ELEM:
6446 err = map_lookup_and_delete_elem(&attr);
6447 break;
6448 case BPF_MAP_LOOKUP_BATCH:
6449 err = bpf_map_do_batch(&attr, uattr.user, BPF_MAP_LOOKUP_BATCH);
6450 break;
6451 case BPF_MAP_LOOKUP_AND_DELETE_BATCH:
6452 err = bpf_map_do_batch(&attr, uattr.user,
6453 BPF_MAP_LOOKUP_AND_DELETE_BATCH);
6454 break;
6455 case BPF_MAP_UPDATE_BATCH:
6456 err = bpf_map_do_batch(&attr, uattr.user, BPF_MAP_UPDATE_BATCH);
6457 break;
6458 case BPF_MAP_DELETE_BATCH:
6459 err = bpf_map_do_batch(&attr, uattr.user, BPF_MAP_DELETE_BATCH);
6460 break;
6461 case BPF_LINK_CREATE:
6462 err = link_create(&attr, uattr);
6463 break;
6464 case BPF_LINK_UPDATE:
6465 err = link_update(&attr);
6466 break;
6467 case BPF_LINK_GET_FD_BY_ID:
6468 err = bpf_link_get_fd_by_id(&attr);
6469 break;
6470 case BPF_LINK_GET_NEXT_ID:
6471 err = bpf_obj_get_next_id(&attr, uattr.user,
6472 &link_idr, &link_idr_lock);
6473 break;
6474 case BPF_ENABLE_STATS:
6475 err = bpf_enable_stats(&attr);
6476 break;
6477 case BPF_ITER_CREATE:
6478 err = bpf_iter_create(&attr);
6479 break;
6480 case BPF_LINK_DETACH:
6481 err = link_detach(&attr);
6482 break;
6483 case BPF_PROG_BIND_MAP:
6484 err = bpf_prog_bind_map(&attr);
6485 break;
6486 case BPF_TOKEN_CREATE:
6487 err = token_create(&attr);
6488 break;
6489 case BPF_PROG_STREAM_READ_BY_FD:
6490 err = prog_stream_read(&attr);
6491 break;
6492 case BPF_PROG_ASSOC_STRUCT_OPS:
6493 err = prog_assoc_struct_ops(&attr);
6494 break;
6495 default:
6496 err = -EINVAL;
6497 break;
6498 }
6499
6500 return err;
6501}
6502
6503SYSCALL_DEFINE5(bpf, int, cmd, union bpf_attr __user *, uattr, unsigned int, size,
6504 struct bpf_common_attr __user *, uattr_common, unsigned int, size_common)
6505{
6506 return __sys_bpf(cmd, USER_BPFPTR(uattr), size, USER_BPFPTR(uattr_common), size_common);
6507}
6508
6509static bool syscall_prog_is_valid_access(int off, int size,
6510 enum bpf_access_type type,
6511 const struct bpf_prog *prog,
6512 struct bpf_insn_access_aux *info)
6513{
6514 if (off < 0 || off >= U16_MAX)
6515 return false;
6516 /* No alignment requirements for syscall ctx accesses. */
6517 return true;
6518}
6519
6520BPF_CALL_3(bpf_sys_bpf, int, cmd, union bpf_attr *, attr, u32, attr_size)
6521{
6522 switch (cmd) {
6523 case BPF_MAP_CREATE:
6524 case BPF_MAP_DELETE_ELEM:
6525 case BPF_MAP_UPDATE_ELEM:
6526 case BPF_MAP_FREEZE:
6527 case BPF_MAP_GET_FD_BY_ID:
6528 case BPF_PROG_LOAD:
6529 case BPF_BTF_LOAD:
6530 case BPF_LINK_CREATE:
6531 case BPF_RAW_TRACEPOINT_OPEN:
6532 break;
6533 default:
6534 return -EINVAL;
6535 }
6536 return __sys_bpf(cmd, KERNEL_BPFPTR(attr), attr_size, KERNEL_BPFPTR(NULL), 0);
6537}
6538
6539/* To shut up -Wmissing-prototypes.
6540 * This function is used by the kernel light skeleton
6541 * to load bpf programs when modules are loaded or during kernel boot.
6542 * See tools/lib/bpf/skel_internal.h
6543 */
6544int kern_sys_bpf(int cmd, union bpf_attr *attr, unsigned int size);
6545
6546int kern_sys_bpf(int cmd, union bpf_attr *attr, unsigned int size)
6547{
6548 struct bpf_prog * __maybe_unused prog;
6549 struct bpf_tramp_run_ctx __maybe_unused run_ctx;
6550
6551 switch (cmd) {
6552#ifdef CONFIG_BPF_JIT /* __bpf_prog_enter_sleepable used by trampoline and JIT */
6553 case BPF_PROG_TEST_RUN:
6554 if (attr->test.data_in || attr->test.data_out ||
6555 attr->test.ctx_out || attr->test.duration ||
6556 attr->test.repeat || attr->test.flags)
6557 return -EINVAL;
6558
6559 prog = bpf_prog_get_type(attr->test.prog_fd, BPF_PROG_TYPE_SYSCALL);
6560 if (IS_ERR(prog))
6561 return PTR_ERR(prog);
6562
6563 if (attr->test.ctx_size_in < prog->aux->max_ctx_offset ||
6564 attr->test.ctx_size_in > U16_MAX) {
6565 bpf_prog_put(prog);
6566 return -EINVAL;
6567 }
6568
6569 run_ctx.bpf_cookie = 0;
6570 if (!__bpf_prog_enter_sleepable_recur(prog, &run_ctx)) {
6571 /* recursion detected */
6572 __bpf_prog_exit_sleepable_recur(prog, 0, &run_ctx);
6573 bpf_prog_put(prog);
6574 return -EBUSY;
6575 }
6576 attr->test.retval = bpf_prog_run(prog, (void *) (long) attr->test.ctx_in);
6577 __bpf_prog_exit_sleepable_recur(prog, 0 /* bpf_prog_run does runtime stats */,
6578 &run_ctx);
6579 bpf_prog_put(prog);
6580 return 0;
6581#endif
6582 default:
6583 return ____bpf_sys_bpf(cmd, attr, size);
6584 }
6585}
6586EXPORT_SYMBOL_NS(kern_sys_bpf, "BPF_INTERNAL");
6587
6588static const struct bpf_func_proto bpf_sys_bpf_proto = {
6589 .func = bpf_sys_bpf,
6590 .gpl_only = false,
6591 .might_sleep = true,
6592 .ret_type = RET_INTEGER,
6593 .arg1_type = ARG_ANYTHING,
6594 .arg2_type = ARG_PTR_TO_MEM | MEM_RDONLY,
6595 .arg3_type = ARG_MEM_SIZE,
6596};
6597
6598const struct bpf_func_proto * __weak
6599tracing_prog_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog)
6600{
6601 return bpf_base_func_proto(func_id, prog);
6602}
6603
6604BPF_CALL_1(bpf_sys_close, u32, fd)
6605{
6606 /* When bpf program calls this helper there should not be
6607 * an fdget() without matching completed fdput().
6608 * This helper is allowed in the following callchain only:
6609 * sys_bpf->prog_test_run->bpf_prog->bpf_sys_close
6610 */
6611 return close_fd(fd);
6612}
6613
6614static const struct bpf_func_proto bpf_sys_close_proto = {
6615 .func = bpf_sys_close,
6616 .gpl_only = false,
6617 .might_sleep = true,
6618 .ret_type = RET_INTEGER,
6619 .arg1_type = ARG_ANYTHING,
6620};
6621
6622BPF_CALL_4(bpf_kallsyms_lookup_name, const char *, name, int, name_sz, int, flags, u64 *, res)
6623{
6624 *res = 0;
6625 if (flags)
6626 return -EINVAL;
6627
6628 if (name_sz <= 1 || name[name_sz - 1])
6629 return -EINVAL;
6630
6631 if (!bpf_dump_raw_ok(current_cred()))
6632 return -EPERM;
6633
6634 *res = kallsyms_lookup_name(name);
6635 return *res ? 0 : -ENOENT;
6636}
6637
6638static const struct bpf_func_proto bpf_kallsyms_lookup_name_proto = {
6639 .func = bpf_kallsyms_lookup_name,
6640 .gpl_only = false,
6641 .ret_type = RET_INTEGER,
6642 .arg1_type = ARG_PTR_TO_MEM | MEM_RDONLY,
6643 .arg2_type = ARG_MEM_SIZE_OR_ZERO,
6644 .arg3_type = ARG_ANYTHING,
6645 .arg4_type = ARG_PTR_TO_FIXED_SIZE_MEM | MEM_UNINIT | MEM_WRITE | MEM_ALIGNED,
6646 .arg4_size = sizeof(u64),
6647};
6648
6649static const struct bpf_func_proto *
6650syscall_prog_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog)
6651{
6652 switch (func_id) {
6653 case BPF_FUNC_sys_bpf:
6654 return !bpf_token_capable(prog->aux->token, CAP_PERFMON)
6655 ? NULL : &bpf_sys_bpf_proto;
6656 case BPF_FUNC_btf_find_by_name_kind:
6657 return &bpf_btf_find_by_name_kind_proto;
6658 case BPF_FUNC_sys_close:
6659 return &bpf_sys_close_proto;
6660 case BPF_FUNC_kallsyms_lookup_name:
6661 return &bpf_kallsyms_lookup_name_proto;
6662 default:
6663 return tracing_prog_func_proto(func_id, prog);
6664 }
6665}
6666
6667const struct bpf_verifier_ops bpf_syscall_verifier_ops = {
6668 .get_func_proto = syscall_prog_func_proto,
6669 .is_valid_access = syscall_prog_is_valid_access,
6670};
6671
6672const struct bpf_prog_ops bpf_syscall_prog_ops = {
6673 .test_run = bpf_prog_test_run_syscall,
6674};
6675
6676#ifdef CONFIG_SYSCTL
6677static int bpf_stats_handler(const struct ctl_table *table, int write,
6678 void *buffer, size_t *lenp, loff_t *ppos)
6679{
6680 struct static_key *key = (struct static_key *)table->data;
6681 static int saved_val;
6682 int val, ret;
6683 struct ctl_table tmp = {
6684 .data = &val,
6685 .maxlen = sizeof(val),
6686 .mode = table->mode,
6687 .extra1 = SYSCTL_ZERO,
6688 .extra2 = SYSCTL_ONE,
6689 };
6690
6691 if (write && !capable(CAP_SYS_ADMIN))
6692 return -EPERM;
6693
6694 mutex_lock(&bpf_stats_enabled_mutex);
6695 val = saved_val;
6696 ret = proc_dointvec_minmax(&tmp, write, buffer, lenp, ppos);
6697 if (write && !ret && val != saved_val) {
6698 if (val)
6699 static_key_slow_inc(key);
6700 else
6701 static_key_slow_dec(key);
6702 saved_val = val;
6703 }
6704 mutex_unlock(&bpf_stats_enabled_mutex);
6705 return ret;
6706}
6707
6708void __weak unpriv_ebpf_notify(int new_state)
6709{
6710}
6711
6712static int bpf_unpriv_handler(const struct ctl_table *table, int write,
6713 void *buffer, size_t *lenp, loff_t *ppos)
6714{
6715 int ret, unpriv_enable = *(int *)table->data;
6716 bool locked_state = unpriv_enable == 1;
6717 struct ctl_table tmp = *table;
6718
6719 if (write && !capable(CAP_SYS_ADMIN))
6720 return -EPERM;
6721
6722 tmp.data = &unpriv_enable;
6723 ret = proc_dointvec_minmax(&tmp, write, buffer, lenp, ppos);
6724 if (write && !ret) {
6725 if (locked_state && unpriv_enable != 1)
6726 return -EPERM;
6727 *(int *)table->data = unpriv_enable;
6728 }
6729
6730 if (write)
6731 unpriv_ebpf_notify(unpriv_enable);
6732
6733 return ret;
6734}
6735
6736static const struct ctl_table bpf_syscall_table[] = {
6737 {
6738 .procname = "unprivileged_bpf_disabled",
6739 .data = &sysctl_unprivileged_bpf_disabled,
6740 .maxlen = sizeof(sysctl_unprivileged_bpf_disabled),
6741 .mode = 0644,
6742 .proc_handler = bpf_unpriv_handler,
6743 .extra1 = SYSCTL_ZERO,
6744 .extra2 = SYSCTL_TWO,
6745 },
6746 {
6747 .procname = "bpf_stats_enabled",
6748 .data = &bpf_stats_enabled_key.key,
6749 .mode = 0644,
6750 .proc_handler = bpf_stats_handler,
6751 },
6752};
6753
6754static int __init bpf_syscall_sysctl_init(void)
6755{
6756 register_sysctl_init("kernel", bpf_syscall_table);
6757 return 0;
6758}
6759late_initcall(bpf_syscall_sysctl_init);
6760#endif /* CONFIG_SYSCTL */