projects/linux·blob v6.13 · refs

6151 lines · 152.1 KiB

1// SPDX-License-Identifier: GPL-2.0-only
2/* Copyright (c) 2011-2014 PLUMgrid, http://plumgrid.com
3 */
4#include <linux/bpf.h>
5#include <linux/bpf-cgroup.h>
6#include <linux/bpf_trace.h>
7#include <linux/bpf_lirc.h>
8#include <linux/bpf_verifier.h>
9#include <linux/bsearch.h>
10#include <linux/btf.h>
11#include <linux/syscalls.h>
12#include <linux/slab.h>
13#include <linux/sched/signal.h>
14#include <linux/vmalloc.h>
15#include <linux/mmzone.h>
16#include <linux/anon_inodes.h>
17#include <linux/fdtable.h>
18#include <linux/file.h>
19#include <linux/fs.h>
20#include <linux/license.h>
21#include <linux/filter.h>
22#include <linux/kernel.h>
23#include <linux/idr.h>
24#include <linux/cred.h>
25#include <linux/timekeeping.h>
26#include <linux/ctype.h>
27#include <linux/nospec.h>
28#include <linux/audit.h>
29#include <uapi/linux/btf.h>
30#include <linux/pgtable.h>
31#include <linux/bpf_lsm.h>
32#include <linux/poll.h>
33#include <linux/sort.h>
34#include <linux/bpf-netns.h>
35#include <linux/rcupdate_trace.h>
36#include <linux/memcontrol.h>
37#include <linux/trace_events.h>
38#include <linux/tracepoint.h>
39
40#include <net/netfilter/nf_bpf_link.h>
41#include <net/netkit.h>
42#include <net/tcx.h>
43
44#define IS_FD_ARRAY(map) ((map)->map_type == BPF_MAP_TYPE_PERF_EVENT_ARRAY || \
45 (map)->map_type == BPF_MAP_TYPE_CGROUP_ARRAY || \
46 (map)->map_type == BPF_MAP_TYPE_ARRAY_OF_MAPS)
47#define IS_FD_PROG_ARRAY(map) ((map)->map_type == BPF_MAP_TYPE_PROG_ARRAY)
48#define IS_FD_HASH(map) ((map)->map_type == BPF_MAP_TYPE_HASH_OF_MAPS)
49#define IS_FD_MAP(map) (IS_FD_ARRAY(map) || IS_FD_PROG_ARRAY(map) || \
50 IS_FD_HASH(map))
51
52#define BPF_OBJ_FLAG_MASK (BPF_F_RDONLY | BPF_F_WRONLY)
53
54DEFINE_PER_CPU(int, bpf_prog_active);
55static DEFINE_IDR(prog_idr);
56static DEFINE_SPINLOCK(prog_idr_lock);
57static DEFINE_IDR(map_idr);
58static DEFINE_SPINLOCK(map_idr_lock);
59static DEFINE_IDR(link_idr);
60static DEFINE_SPINLOCK(link_idr_lock);
61
62int sysctl_unprivileged_bpf_disabled __read_mostly =
63 IS_BUILTIN(CONFIG_BPF_UNPRIV_DEFAULT_OFF) ? 2 : 0;
64
65static const struct bpf_map_ops * const bpf_map_types[] = {
66#define BPF_PROG_TYPE(_id, _name, prog_ctx_type, kern_ctx_type)
67#define BPF_MAP_TYPE(_id, _ops) \
68 [_id] = &_ops,
69#define BPF_LINK_TYPE(_id, _name)
70#include <linux/bpf_types.h>
71#undef BPF_PROG_TYPE
72#undef BPF_MAP_TYPE
73#undef BPF_LINK_TYPE
74};
75
76/*
77 * If we're handed a bigger struct than we know of, ensure all the unknown bits
78 * are 0 - i.e. new user-space does not rely on any kernel feature extensions
79 * we don't know about yet.
80 *
81 * There is a ToCToU between this function call and the following
82 * copy_from_user() call. However, this is not a concern since this function is
83 * meant to be a future-proofing of bits.
84 */
85int bpf_check_uarg_tail_zero(bpfptr_t uaddr,
86 size_t expected_size,
87 size_t actual_size)
88{
89 int res;
90
91 if (unlikely(actual_size > PAGE_SIZE)) /* silly large */
92 return -E2BIG;
93
94 if (actual_size <= expected_size)
95 return 0;
96
97 if (uaddr.is_kernel)
98 res = memchr_inv(uaddr.kernel + expected_size, 0,
99 actual_size - expected_size) == NULL;
100 else
101 res = check_zeroed_user(uaddr.user + expected_size,
102 actual_size - expected_size);
103 if (res < 0)
104 return res;
105 return res ? 0 : -E2BIG;
106}
107
108const struct bpf_map_ops bpf_map_offload_ops = {
109 .map_meta_equal = bpf_map_meta_equal,
110 .map_alloc = bpf_map_offload_map_alloc,
111 .map_free = bpf_map_offload_map_free,
112 .map_check_btf = map_check_no_btf,
113 .map_mem_usage = bpf_map_offload_map_mem_usage,
114};
115
116static void bpf_map_write_active_inc(struct bpf_map *map)
117{
118 atomic64_inc(&map->writecnt);
119}
120
121static void bpf_map_write_active_dec(struct bpf_map *map)
122{
123 atomic64_dec(&map->writecnt);
124}
125
126bool bpf_map_write_active(const struct bpf_map *map)
127{
128 return atomic64_read(&map->writecnt) != 0;
129}
130
131static u32 bpf_map_value_size(const struct bpf_map *map)
132{
133 if (map->map_type == BPF_MAP_TYPE_PERCPU_HASH ||
134 map->map_type == BPF_MAP_TYPE_LRU_PERCPU_HASH ||
135 map->map_type == BPF_MAP_TYPE_PERCPU_ARRAY ||
136 map->map_type == BPF_MAP_TYPE_PERCPU_CGROUP_STORAGE)
137 return round_up(map->value_size, 8) * num_possible_cpus();
138 else if (IS_FD_MAP(map))
139 return sizeof(u32);
140 else
141 return map->value_size;
142}
143
144static void maybe_wait_bpf_programs(struct bpf_map *map)
145{
146 /* Wait for any running non-sleepable BPF programs to complete so that
147 * userspace, when we return to it, knows that all non-sleepable
148 * programs that could be running use the new map value. For sleepable
149 * BPF programs, synchronize_rcu_tasks_trace() should be used to wait
150 * for the completions of these programs, but considering the waiting
151 * time can be very long and userspace may think it will hang forever,
152 * so don't handle sleepable BPF programs now.
153 */
154 if (map->map_type == BPF_MAP_TYPE_HASH_OF_MAPS ||
155 map->map_type == BPF_MAP_TYPE_ARRAY_OF_MAPS)
156 synchronize_rcu();
157}
158
159static void unpin_uptr_kaddr(void *kaddr)
160{
161 if (kaddr)
162 unpin_user_page(virt_to_page(kaddr));
163}
164
165static void __bpf_obj_unpin_uptrs(struct btf_record *rec, u32 cnt, void *obj)
166{
167 const struct btf_field *field;
168 void **uptr_addr;
169 int i;
170
171 for (i = 0, field = rec->fields; i < cnt; i++, field++) {
172 if (field->type != BPF_UPTR)
173 continue;
174
175 uptr_addr = obj + field->offset;
176 unpin_uptr_kaddr(*uptr_addr);
177 }
178}
179
180static void bpf_obj_unpin_uptrs(struct btf_record *rec, void *obj)
181{
182 if (!btf_record_has_field(rec, BPF_UPTR))
183 return;
184
185 __bpf_obj_unpin_uptrs(rec, rec->cnt, obj);
186}
187
188static int bpf_obj_pin_uptrs(struct btf_record *rec, void *obj)
189{
190 const struct btf_field *field;
191 const struct btf_type *t;
192 unsigned long start, end;
193 struct page *page;
194 void **uptr_addr;
195 int i, err;
196
197 if (!btf_record_has_field(rec, BPF_UPTR))
198 return 0;
199
200 for (i = 0, field = rec->fields; i < rec->cnt; i++, field++) {
201 if (field->type != BPF_UPTR)
202 continue;
203
204 uptr_addr = obj + field->offset;
205 start = *(unsigned long *)uptr_addr;
206 if (!start)
207 continue;
208
209 t = btf_type_by_id(field->kptr.btf, field->kptr.btf_id);
210 /* t->size was checked for zero before */
211 if (check_add_overflow(start, t->size - 1, &end)) {
212 err = -EFAULT;
213 goto unpin_all;
214 }
215
216 /* The uptr's struct cannot span across two pages */
217 if ((start & PAGE_MASK) != (end & PAGE_MASK)) {
218 err = -EOPNOTSUPP;
219 goto unpin_all;
220 }
221
222 err = pin_user_pages_fast(start, 1, FOLL_LONGTERM | FOLL_WRITE, &page);
223 if (err != 1)
224 goto unpin_all;
225
226 if (PageHighMem(page)) {
227 err = -EOPNOTSUPP;
228 unpin_user_page(page);
229 goto unpin_all;
230 }
231
232 *uptr_addr = page_address(page) + offset_in_page(start);
233 }
234
235 return 0;
236
237unpin_all:
238 __bpf_obj_unpin_uptrs(rec, i, obj);
239 return err;
240}
241
242static int bpf_map_update_value(struct bpf_map *map, struct file *map_file,
243 void *key, void *value, __u64 flags)
244{
245 int err;
246
247 /* Need to create a kthread, thus must support schedule */
248 if (bpf_map_is_offloaded(map)) {
249 return bpf_map_offload_update_elem(map, key, value, flags);
250 } else if (map->map_type == BPF_MAP_TYPE_CPUMAP ||
251 map->map_type == BPF_MAP_TYPE_ARENA ||
252 map->map_type == BPF_MAP_TYPE_STRUCT_OPS) {
253 return map->ops->map_update_elem(map, key, value, flags);
254 } else if (map->map_type == BPF_MAP_TYPE_SOCKHASH ||
255 map->map_type == BPF_MAP_TYPE_SOCKMAP) {
256 return sock_map_update_elem_sys(map, key, value, flags);
257 } else if (IS_FD_PROG_ARRAY(map)) {
258 return bpf_fd_array_map_update_elem(map, map_file, key, value,
259 flags);
260 }
261
262 bpf_disable_instrumentation();
263 if (map->map_type == BPF_MAP_TYPE_PERCPU_HASH ||
264 map->map_type == BPF_MAP_TYPE_LRU_PERCPU_HASH) {
265 err = bpf_percpu_hash_update(map, key, value, flags);
266 } else if (map->map_type == BPF_MAP_TYPE_PERCPU_ARRAY) {
267 err = bpf_percpu_array_update(map, key, value, flags);
268 } else if (map->map_type == BPF_MAP_TYPE_PERCPU_CGROUP_STORAGE) {
269 err = bpf_percpu_cgroup_storage_update(map, key, value,
270 flags);
271 } else if (IS_FD_ARRAY(map)) {
272 err = bpf_fd_array_map_update_elem(map, map_file, key, value,
273 flags);
274 } else if (map->map_type == BPF_MAP_TYPE_HASH_OF_MAPS) {
275 err = bpf_fd_htab_map_update_elem(map, map_file, key, value,
276 flags);
277 } else if (map->map_type == BPF_MAP_TYPE_REUSEPORT_SOCKARRAY) {
278 /* rcu_read_lock() is not needed */
279 err = bpf_fd_reuseport_array_update_elem(map, key, value,
280 flags);
281 } else if (map->map_type == BPF_MAP_TYPE_QUEUE ||
282 map->map_type == BPF_MAP_TYPE_STACK ||
283 map->map_type == BPF_MAP_TYPE_BLOOM_FILTER) {
284 err = map->ops->map_push_elem(map, value, flags);
285 } else {
286 err = bpf_obj_pin_uptrs(map->record, value);
287 if (!err) {
288 rcu_read_lock();
289 err = map->ops->map_update_elem(map, key, value, flags);
290 rcu_read_unlock();
291 if (err)
292 bpf_obj_unpin_uptrs(map->record, value);
293 }
294 }
295 bpf_enable_instrumentation();
296
297 return err;
298}
299
300static int bpf_map_copy_value(struct bpf_map *map, void *key, void *value,
301 __u64 flags)
302{
303 void *ptr;
304 int err;
305
306 if (bpf_map_is_offloaded(map))
307 return bpf_map_offload_lookup_elem(map, key, value);
308
309 bpf_disable_instrumentation();
310 if (map->map_type == BPF_MAP_TYPE_PERCPU_HASH ||
311 map->map_type == BPF_MAP_TYPE_LRU_PERCPU_HASH) {
312 err = bpf_percpu_hash_copy(map, key, value);
313 } else if (map->map_type == BPF_MAP_TYPE_PERCPU_ARRAY) {
314 err = bpf_percpu_array_copy(map, key, value);
315 } else if (map->map_type == BPF_MAP_TYPE_PERCPU_CGROUP_STORAGE) {
316 err = bpf_percpu_cgroup_storage_copy(map, key, value);
317 } else if (map->map_type == BPF_MAP_TYPE_STACK_TRACE) {
318 err = bpf_stackmap_copy(map, key, value);
319 } else if (IS_FD_ARRAY(map) || IS_FD_PROG_ARRAY(map)) {
320 err = bpf_fd_array_map_lookup_elem(map, key, value);
321 } else if (IS_FD_HASH(map)) {
322 err = bpf_fd_htab_map_lookup_elem(map, key, value);
323 } else if (map->map_type == BPF_MAP_TYPE_REUSEPORT_SOCKARRAY) {
324 err = bpf_fd_reuseport_array_lookup_elem(map, key, value);
325 } else if (map->map_type == BPF_MAP_TYPE_QUEUE ||
326 map->map_type == BPF_MAP_TYPE_STACK ||
327 map->map_type == BPF_MAP_TYPE_BLOOM_FILTER) {
328 err = map->ops->map_peek_elem(map, value);
329 } else if (map->map_type == BPF_MAP_TYPE_STRUCT_OPS) {
330 /* struct_ops map requires directly updating "value" */
331 err = bpf_struct_ops_map_sys_lookup_elem(map, key, value);
332 } else {
333 rcu_read_lock();
334 if (map->ops->map_lookup_elem_sys_only)
335 ptr = map->ops->map_lookup_elem_sys_only(map, key);
336 else
337 ptr = map->ops->map_lookup_elem(map, key);
338 if (IS_ERR(ptr)) {
339 err = PTR_ERR(ptr);
340 } else if (!ptr) {
341 err = -ENOENT;
342 } else {
343 err = 0;
344 if (flags & BPF_F_LOCK)
345 /* lock 'ptr' and copy everything but lock */
346 copy_map_value_locked(map, value, ptr, true);
347 else
348 copy_map_value(map, value, ptr);
349 /* mask lock and timer, since value wasn't zero inited */
350 check_and_init_map_value(map, value);
351 }
352 rcu_read_unlock();
353 }
354
355 bpf_enable_instrumentation();
356
357 return err;
358}
359
360/* Please, do not use this function outside from the map creation path
361 * (e.g. in map update path) without taking care of setting the active
362 * memory cgroup (see at bpf_map_kmalloc_node() for example).
363 */
364static void *__bpf_map_area_alloc(u64 size, int numa_node, bool mmapable)
365{
366 /* We really just want to fail instead of triggering OOM killer
367 * under memory pressure, therefore we set __GFP_NORETRY to kmalloc,
368 * which is used for lower order allocation requests.
369 *
370 * It has been observed that higher order allocation requests done by
371 * vmalloc with __GFP_NORETRY being set might fail due to not trying
372 * to reclaim memory from the page cache, thus we set
373 * __GFP_RETRY_MAYFAIL to avoid such situations.
374 */
375
376 gfp_t gfp = bpf_memcg_flags(__GFP_NOWARN | __GFP_ZERO);
377 unsigned int flags = 0;
378 unsigned long align = 1;
379 void *area;
380
381 if (size >= SIZE_MAX)
382 return NULL;
383
384 /* kmalloc()'ed memory can't be mmap()'ed */
385 if (mmapable) {
386 BUG_ON(!PAGE_ALIGNED(size));
387 align = SHMLBA;
388 flags = VM_USERMAP;
389 } else if (size <= (PAGE_SIZE << PAGE_ALLOC_COSTLY_ORDER)) {
390 area = kmalloc_node(size, gfp | GFP_USER | __GFP_NORETRY,
391 numa_node);
392 if (area != NULL)
393 return area;
394 }
395
396 return __vmalloc_node_range(size, align, VMALLOC_START, VMALLOC_END,
397 gfp | GFP_KERNEL | __GFP_RETRY_MAYFAIL, PAGE_KERNEL,
398 flags, numa_node, __builtin_return_address(0));
399}
400
401void *bpf_map_area_alloc(u64 size, int numa_node)
402{
403 return __bpf_map_area_alloc(size, numa_node, false);
404}
405
406void *bpf_map_area_mmapable_alloc(u64 size, int numa_node)
407{
408 return __bpf_map_area_alloc(size, numa_node, true);
409}
410
411void bpf_map_area_free(void *area)
412{
413 kvfree(area);
414}
415
416static u32 bpf_map_flags_retain_permanent(u32 flags)
417{
418 /* Some map creation flags are not tied to the map object but
419 * rather to the map fd instead, so they have no meaning upon
420 * map object inspection since multiple file descriptors with
421 * different (access) properties can exist here. Thus, given
422 * this has zero meaning for the map itself, lets clear these
423 * from here.
424 */
425 return flags & ~(BPF_F_RDONLY | BPF_F_WRONLY);
426}
427
428void bpf_map_init_from_attr(struct bpf_map *map, union bpf_attr *attr)
429{
430 map->map_type = attr->map_type;
431 map->key_size = attr->key_size;
432 map->value_size = attr->value_size;
433 map->max_entries = attr->max_entries;
434 map->map_flags = bpf_map_flags_retain_permanent(attr->map_flags);
435 map->numa_node = bpf_map_attr_numa_node(attr);
436 map->map_extra = attr->map_extra;
437}
438
439static int bpf_map_alloc_id(struct bpf_map *map)
440{
441 int id;
442
443 idr_preload(GFP_KERNEL);
444 spin_lock_bh(&map_idr_lock);
445 id = idr_alloc_cyclic(&map_idr, map, 1, INT_MAX, GFP_ATOMIC);
446 if (id > 0)
447 map->id = id;
448 spin_unlock_bh(&map_idr_lock);
449 idr_preload_end();
450
451 if (WARN_ON_ONCE(!id))
452 return -ENOSPC;
453
454 return id > 0 ? 0 : id;
455}
456
457void bpf_map_free_id(struct bpf_map *map)
458{
459 unsigned long flags;
460
461 /* Offloaded maps are removed from the IDR store when their device
462 * disappears - even if someone holds an fd to them they are unusable,
463 * the memory is gone, all ops will fail; they are simply waiting for
464 * refcnt to drop to be freed.
465 */
466 if (!map->id)
467 return;
468
469 spin_lock_irqsave(&map_idr_lock, flags);
470
471 idr_remove(&map_idr, map->id);
472 map->id = 0;
473
474 spin_unlock_irqrestore(&map_idr_lock, flags);
475}
476
477#ifdef CONFIG_MEMCG
478static void bpf_map_save_memcg(struct bpf_map *map)
479{
480 /* Currently if a map is created by a process belonging to the root
481 * memory cgroup, get_obj_cgroup_from_current() will return NULL.
482 * So we have to check map->objcg for being NULL each time it's
483 * being used.
484 */
485 if (memcg_bpf_enabled())
486 map->objcg = get_obj_cgroup_from_current();
487}
488
489static void bpf_map_release_memcg(struct bpf_map *map)
490{
491 if (map->objcg)
492 obj_cgroup_put(map->objcg);
493}
494
495static struct mem_cgroup *bpf_map_get_memcg(const struct bpf_map *map)
496{
497 if (map->objcg)
498 return get_mem_cgroup_from_objcg(map->objcg);
499
500 return root_mem_cgroup;
501}
502
503void *bpf_map_kmalloc_node(const struct bpf_map *map, size_t size, gfp_t flags,
504 int node)
505{
506 struct mem_cgroup *memcg, *old_memcg;
507 void *ptr;
508
509 memcg = bpf_map_get_memcg(map);
510 old_memcg = set_active_memcg(memcg);
511 ptr = kmalloc_node(size, flags | __GFP_ACCOUNT, node);
512 set_active_memcg(old_memcg);
513 mem_cgroup_put(memcg);
514
515 return ptr;
516}
517
518void *bpf_map_kzalloc(const struct bpf_map *map, size_t size, gfp_t flags)
519{
520 struct mem_cgroup *memcg, *old_memcg;
521 void *ptr;
522
523 memcg = bpf_map_get_memcg(map);
524 old_memcg = set_active_memcg(memcg);
525 ptr = kzalloc(size, flags | __GFP_ACCOUNT);
526 set_active_memcg(old_memcg);
527 mem_cgroup_put(memcg);
528
529 return ptr;
530}
531
532void *bpf_map_kvcalloc(struct bpf_map *map, size_t n, size_t size,
533 gfp_t flags)
534{
535 struct mem_cgroup *memcg, *old_memcg;
536 void *ptr;
537
538 memcg = bpf_map_get_memcg(map);
539 old_memcg = set_active_memcg(memcg);
540 ptr = kvcalloc(n, size, flags | __GFP_ACCOUNT);
541 set_active_memcg(old_memcg);
542 mem_cgroup_put(memcg);
543
544 return ptr;
545}
546
547void __percpu *bpf_map_alloc_percpu(const struct bpf_map *map, size_t size,
548 size_t align, gfp_t flags)
549{
550 struct mem_cgroup *memcg, *old_memcg;
551 void __percpu *ptr;
552
553 memcg = bpf_map_get_memcg(map);
554 old_memcg = set_active_memcg(memcg);
555 ptr = __alloc_percpu_gfp(size, align, flags | __GFP_ACCOUNT);
556 set_active_memcg(old_memcg);
557 mem_cgroup_put(memcg);
558
559 return ptr;
560}
561
562#else
563static void bpf_map_save_memcg(struct bpf_map *map)
564{
565}
566
567static void bpf_map_release_memcg(struct bpf_map *map)
568{
569}
570#endif
571
572int bpf_map_alloc_pages(const struct bpf_map *map, gfp_t gfp, int nid,
573 unsigned long nr_pages, struct page **pages)
574{
575 unsigned long i, j;
576 struct page *pg;
577 int ret = 0;
578#ifdef CONFIG_MEMCG
579 struct mem_cgroup *memcg, *old_memcg;
580
581 memcg = bpf_map_get_memcg(map);
582 old_memcg = set_active_memcg(memcg);
583#endif
584 for (i = 0; i < nr_pages; i++) {
585 pg = alloc_pages_node(nid, gfp | __GFP_ACCOUNT, 0);
586
587 if (pg) {
588 pages[i] = pg;
589 continue;
590 }
591 for (j = 0; j < i; j++)
592 __free_page(pages[j]);
593 ret = -ENOMEM;
594 break;
595 }
596
597#ifdef CONFIG_MEMCG
598 set_active_memcg(old_memcg);
599 mem_cgroup_put(memcg);
600#endif
601 return ret;
602}
603
604
605static int btf_field_cmp(const void *a, const void *b)
606{
607 const struct btf_field *f1 = a, *f2 = b;
608
609 if (f1->offset < f2->offset)
610 return -1;
611 else if (f1->offset > f2->offset)
612 return 1;
613 return 0;
614}
615
616struct btf_field *btf_record_find(const struct btf_record *rec, u32 offset,
617 u32 field_mask)
618{
619 struct btf_field *field;
620
621 if (IS_ERR_OR_NULL(rec) || !(rec->field_mask & field_mask))
622 return NULL;
623 field = bsearch(&offset, rec->fields, rec->cnt, sizeof(rec->fields[0]), btf_field_cmp);
624 if (!field || !(field->type & field_mask))
625 return NULL;
626 return field;
627}
628
629void btf_record_free(struct btf_record *rec)
630{
631 int i;
632
633 if (IS_ERR_OR_NULL(rec))
634 return;
635 for (i = 0; i < rec->cnt; i++) {
636 switch (rec->fields[i].type) {
637 case BPF_KPTR_UNREF:
638 case BPF_KPTR_REF:
639 case BPF_KPTR_PERCPU:
640 case BPF_UPTR:
641 if (rec->fields[i].kptr.module)
642 module_put(rec->fields[i].kptr.module);
643 if (btf_is_kernel(rec->fields[i].kptr.btf))
644 btf_put(rec->fields[i].kptr.btf);
645 break;
646 case BPF_LIST_HEAD:
647 case BPF_LIST_NODE:
648 case BPF_RB_ROOT:
649 case BPF_RB_NODE:
650 case BPF_SPIN_LOCK:
651 case BPF_TIMER:
652 case BPF_REFCOUNT:
653 case BPF_WORKQUEUE:
654 /* Nothing to release */
655 break;
656 default:
657 WARN_ON_ONCE(1);
658 continue;
659 }
660 }
661 kfree(rec);
662}
663
664void bpf_map_free_record(struct bpf_map *map)
665{
666 btf_record_free(map->record);
667 map->record = NULL;
668}
669
670struct btf_record *btf_record_dup(const struct btf_record *rec)
671{
672 const struct btf_field *fields;
673 struct btf_record *new_rec;
674 int ret, size, i;
675
676 if (IS_ERR_OR_NULL(rec))
677 return NULL;
678 size = offsetof(struct btf_record, fields[rec->cnt]);
679 new_rec = kmemdup(rec, size, GFP_KERNEL | __GFP_NOWARN);
680 if (!new_rec)
681 return ERR_PTR(-ENOMEM);
682 /* Do a deep copy of the btf_record */
683 fields = rec->fields;
684 new_rec->cnt = 0;
685 for (i = 0; i < rec->cnt; i++) {
686 switch (fields[i].type) {
687 case BPF_KPTR_UNREF:
688 case BPF_KPTR_REF:
689 case BPF_KPTR_PERCPU:
690 case BPF_UPTR:
691 if (btf_is_kernel(fields[i].kptr.btf))
692 btf_get(fields[i].kptr.btf);
693 if (fields[i].kptr.module && !try_module_get(fields[i].kptr.module)) {
694 ret = -ENXIO;
695 goto free;
696 }
697 break;
698 case BPF_LIST_HEAD:
699 case BPF_LIST_NODE:
700 case BPF_RB_ROOT:
701 case BPF_RB_NODE:
702 case BPF_SPIN_LOCK:
703 case BPF_TIMER:
704 case BPF_REFCOUNT:
705 case BPF_WORKQUEUE:
706 /* Nothing to acquire */
707 break;
708 default:
709 ret = -EFAULT;
710 WARN_ON_ONCE(1);
711 goto free;
712 }
713 new_rec->cnt++;
714 }
715 return new_rec;
716free:
717 btf_record_free(new_rec);
718 return ERR_PTR(ret);
719}
720
721bool btf_record_equal(const struct btf_record *rec_a, const struct btf_record *rec_b)
722{
723 bool a_has_fields = !IS_ERR_OR_NULL(rec_a), b_has_fields = !IS_ERR_OR_NULL(rec_b);
724 int size;
725
726 if (!a_has_fields && !b_has_fields)
727 return true;
728 if (a_has_fields != b_has_fields)
729 return false;
730 if (rec_a->cnt != rec_b->cnt)
731 return false;
732 size = offsetof(struct btf_record, fields[rec_a->cnt]);
733 /* btf_parse_fields uses kzalloc to allocate a btf_record, so unused
734 * members are zeroed out. So memcmp is safe to do without worrying
735 * about padding/unused fields.
736 *
737 * While spin_lock, timer, and kptr have no relation to map BTF,
738 * list_head metadata is specific to map BTF, the btf and value_rec
739 * members in particular. btf is the map BTF, while value_rec points to
740 * btf_record in that map BTF.
741 *
742 * So while by default, we don't rely on the map BTF (which the records
743 * were parsed from) matching for both records, which is not backwards
744 * compatible, in case list_head is part of it, we implicitly rely on
745 * that by way of depending on memcmp succeeding for it.
746 */
747 return !memcmp(rec_a, rec_b, size);
748}
749
750void bpf_obj_free_timer(const struct btf_record *rec, void *obj)
751{
752 if (WARN_ON_ONCE(!btf_record_has_field(rec, BPF_TIMER)))
753 return;
754 bpf_timer_cancel_and_free(obj + rec->timer_off);
755}
756
757void bpf_obj_free_workqueue(const struct btf_record *rec, void *obj)
758{
759 if (WARN_ON_ONCE(!btf_record_has_field(rec, BPF_WORKQUEUE)))
760 return;
761 bpf_wq_cancel_and_free(obj + rec->wq_off);
762}
763
764void bpf_obj_free_fields(const struct btf_record *rec, void *obj)
765{
766 const struct btf_field *fields;
767 int i;
768
769 if (IS_ERR_OR_NULL(rec))
770 return;
771 fields = rec->fields;
772 for (i = 0; i < rec->cnt; i++) {
773 struct btf_struct_meta *pointee_struct_meta;
774 const struct btf_field *field = &fields[i];
775 void *field_ptr = obj + field->offset;
776 void *xchgd_field;
777
778 switch (fields[i].type) {
779 case BPF_SPIN_LOCK:
780 break;
781 case BPF_TIMER:
782 bpf_timer_cancel_and_free(field_ptr);
783 break;
784 case BPF_WORKQUEUE:
785 bpf_wq_cancel_and_free(field_ptr);
786 break;
787 case BPF_KPTR_UNREF:
788 WRITE_ONCE(*(u64 *)field_ptr, 0);
789 break;
790 case BPF_KPTR_REF:
791 case BPF_KPTR_PERCPU:
792 xchgd_field = (void *)xchg((unsigned long *)field_ptr, 0);
793 if (!xchgd_field)
794 break;
795
796 if (!btf_is_kernel(field->kptr.btf)) {
797 pointee_struct_meta = btf_find_struct_meta(field->kptr.btf,
798 field->kptr.btf_id);
799 migrate_disable();
800 __bpf_obj_drop_impl(xchgd_field, pointee_struct_meta ?
801 pointee_struct_meta->record : NULL,
802 fields[i].type == BPF_KPTR_PERCPU);
803 migrate_enable();
804 } else {
805 field->kptr.dtor(xchgd_field);
806 }
807 break;
808 case BPF_UPTR:
809 /* The caller ensured that no one is using the uptr */
810 unpin_uptr_kaddr(*(void **)field_ptr);
811 break;
812 case BPF_LIST_HEAD:
813 if (WARN_ON_ONCE(rec->spin_lock_off < 0))
814 continue;
815 bpf_list_head_free(field, field_ptr, obj + rec->spin_lock_off);
816 break;
817 case BPF_RB_ROOT:
818 if (WARN_ON_ONCE(rec->spin_lock_off < 0))
819 continue;
820 bpf_rb_root_free(field, field_ptr, obj + rec->spin_lock_off);
821 break;
822 case BPF_LIST_NODE:
823 case BPF_RB_NODE:
824 case BPF_REFCOUNT:
825 break;
826 default:
827 WARN_ON_ONCE(1);
828 continue;
829 }
830 }
831}
832
833static void bpf_map_free(struct bpf_map *map)
834{
835 struct btf_record *rec = map->record;
836 struct btf *btf = map->btf;
837
838 /* implementation dependent freeing */
839 map->ops->map_free(map);
840 /* Delay freeing of btf_record for maps, as map_free
841 * callback usually needs access to them. It is better to do it here
842 * than require each callback to do the free itself manually.
843 *
844 * Note that the btf_record stashed in map->inner_map_meta->record was
845 * already freed using the map_free callback for map in map case which
846 * eventually calls bpf_map_free_meta, since inner_map_meta is only a
847 * template bpf_map struct used during verification.
848 */
849 btf_record_free(rec);
850 /* Delay freeing of btf for maps, as map_free callback may need
851 * struct_meta info which will be freed with btf_put().
852 */
853 btf_put(btf);
854}
855
856/* called from workqueue */
857static void bpf_map_free_deferred(struct work_struct *work)
858{
859 struct bpf_map *map = container_of(work, struct bpf_map, work);
860
861 security_bpf_map_free(map);
862 bpf_map_release_memcg(map);
863 bpf_map_free(map);
864}
865
866static void bpf_map_put_uref(struct bpf_map *map)
867{
868 if (atomic64_dec_and_test(&map->usercnt)) {
869 if (map->ops->map_release_uref)
870 map->ops->map_release_uref(map);
871 }
872}
873
874static void bpf_map_free_in_work(struct bpf_map *map)
875{
876 INIT_WORK(&map->work, bpf_map_free_deferred);
877 /* Avoid spawning kworkers, since they all might contend
878 * for the same mutex like slab_mutex.
879 */
880 queue_work(system_unbound_wq, &map->work);
881}
882
883static void bpf_map_free_rcu_gp(struct rcu_head *rcu)
884{
885 bpf_map_free_in_work(container_of(rcu, struct bpf_map, rcu));
886}
887
888static void bpf_map_free_mult_rcu_gp(struct rcu_head *rcu)
889{
890 if (rcu_trace_implies_rcu_gp())
891 bpf_map_free_rcu_gp(rcu);
892 else
893 call_rcu(rcu, bpf_map_free_rcu_gp);
894}
895
896/* decrement map refcnt and schedule it for freeing via workqueue
897 * (underlying map implementation ops->map_free() might sleep)
898 */
899void bpf_map_put(struct bpf_map *map)
900{
901 if (atomic64_dec_and_test(&map->refcnt)) {
902 /* bpf_map_free_id() must be called first */
903 bpf_map_free_id(map);
904
905 WARN_ON_ONCE(atomic64_read(&map->sleepable_refcnt));
906 if (READ_ONCE(map->free_after_mult_rcu_gp))
907 call_rcu_tasks_trace(&map->rcu, bpf_map_free_mult_rcu_gp);
908 else if (READ_ONCE(map->free_after_rcu_gp))
909 call_rcu(&map->rcu, bpf_map_free_rcu_gp);
910 else
911 bpf_map_free_in_work(map);
912 }
913}
914EXPORT_SYMBOL_GPL(bpf_map_put);
915
916void bpf_map_put_with_uref(struct bpf_map *map)
917{
918 bpf_map_put_uref(map);
919 bpf_map_put(map);
920}
921
922static int bpf_map_release(struct inode *inode, struct file *filp)
923{
924 struct bpf_map *map = filp->private_data;
925
926 if (map->ops->map_release)
927 map->ops->map_release(map, filp);
928
929 bpf_map_put_with_uref(map);
930 return 0;
931}
932
933static fmode_t map_get_sys_perms(struct bpf_map *map, struct fd f)
934{
935 fmode_t mode = fd_file(f)->f_mode;
936
937 /* Our file permissions may have been overridden by global
938 * map permissions facing syscall side.
939 */
940 if (READ_ONCE(map->frozen))
941 mode &= ~FMODE_CAN_WRITE;
942 return mode;
943}
944
945#ifdef CONFIG_PROC_FS
946/* Show the memory usage of a bpf map */
947static u64 bpf_map_memory_usage(const struct bpf_map *map)
948{
949 return map->ops->map_mem_usage(map);
950}
951
952static void bpf_map_show_fdinfo(struct seq_file *m, struct file *filp)
953{
954 struct bpf_map *map = filp->private_data;
955 u32 type = 0, jited = 0;
956
957 if (map_type_contains_progs(map)) {
958 spin_lock(&map->owner.lock);
959 type = map->owner.type;
960 jited = map->owner.jited;
961 spin_unlock(&map->owner.lock);
962 }
963
964 seq_printf(m,
965 "map_type:\t%u\n"
966 "key_size:\t%u\n"
967 "value_size:\t%u\n"
968 "max_entries:\t%u\n"
969 "map_flags:\t%#x\n"
970 "map_extra:\t%#llx\n"
971 "memlock:\t%llu\n"
972 "map_id:\t%u\n"
973 "frozen:\t%u\n",
974 map->map_type,
975 map->key_size,
976 map->value_size,
977 map->max_entries,
978 map->map_flags,
979 (unsigned long long)map->map_extra,
980 bpf_map_memory_usage(map),
981 map->id,
982 READ_ONCE(map->frozen));
983 if (type) {
984 seq_printf(m, "owner_prog_type:\t%u\n", type);
985 seq_printf(m, "owner_jited:\t%u\n", jited);
986 }
987}
988#endif
989
990static ssize_t bpf_dummy_read(struct file *filp, char __user *buf, size_t siz,
991 loff_t *ppos)
992{
993 /* We need this handler such that alloc_file() enables
994 * f_mode with FMODE_CAN_READ.
995 */
996 return -EINVAL;
997}
998
999static ssize_t bpf_dummy_write(struct file *filp, const char __user *buf,
1000 size_t siz, loff_t *ppos)
1001{
1002 /* We need this handler such that alloc_file() enables
1003 * f_mode with FMODE_CAN_WRITE.
1004 */
1005 return -EINVAL;
1006}
1007
1008/* called for any extra memory-mapped regions (except initial) */
1009static void bpf_map_mmap_open(struct vm_area_struct *vma)
1010{
1011 struct bpf_map *map = vma->vm_file->private_data;
1012
1013 if (vma->vm_flags & VM_MAYWRITE)
1014 bpf_map_write_active_inc(map);
1015}
1016
1017/* called for all unmapped memory region (including initial) */
1018static void bpf_map_mmap_close(struct vm_area_struct *vma)
1019{
1020 struct bpf_map *map = vma->vm_file->private_data;
1021
1022 if (vma->vm_flags & VM_MAYWRITE)
1023 bpf_map_write_active_dec(map);
1024}
1025
1026static const struct vm_operations_struct bpf_map_default_vmops = {
1027 .open = bpf_map_mmap_open,
1028 .close = bpf_map_mmap_close,
1029};
1030
1031static int bpf_map_mmap(struct file *filp, struct vm_area_struct *vma)
1032{
1033 struct bpf_map *map = filp->private_data;
1034 int err;
1035
1036 if (!map->ops->map_mmap || !IS_ERR_OR_NULL(map->record))
1037 return -ENOTSUPP;
1038
1039 if (!(vma->vm_flags & VM_SHARED))
1040 return -EINVAL;
1041
1042 mutex_lock(&map->freeze_mutex);
1043
1044 if (vma->vm_flags & VM_WRITE) {
1045 if (map->frozen) {
1046 err = -EPERM;
1047 goto out;
1048 }
1049 /* map is meant to be read-only, so do not allow mapping as
1050 * writable, because it's possible to leak a writable page
1051 * reference and allows user-space to still modify it after
1052 * freezing, while verifier will assume contents do not change
1053 */
1054 if (map->map_flags & BPF_F_RDONLY_PROG) {
1055 err = -EACCES;
1056 goto out;
1057 }
1058 }
1059
1060 /* set default open/close callbacks */
1061 vma->vm_ops = &bpf_map_default_vmops;
1062 vma->vm_private_data = map;
1063 vm_flags_clear(vma, VM_MAYEXEC);
1064 if (!(vma->vm_flags & VM_WRITE))
1065 /* disallow re-mapping with PROT_WRITE */
1066 vm_flags_clear(vma, VM_MAYWRITE);
1067
1068 err = map->ops->map_mmap(map, vma);
1069 if (err)
1070 goto out;
1071
1072 if (vma->vm_flags & VM_MAYWRITE)
1073 bpf_map_write_active_inc(map);
1074out:
1075 mutex_unlock(&map->freeze_mutex);
1076 return err;
1077}
1078
1079static __poll_t bpf_map_poll(struct file *filp, struct poll_table_struct *pts)
1080{
1081 struct bpf_map *map = filp->private_data;
1082
1083 if (map->ops->map_poll)
1084 return map->ops->map_poll(map, filp, pts);
1085
1086 return EPOLLERR;
1087}
1088
1089static unsigned long bpf_get_unmapped_area(struct file *filp, unsigned long addr,
1090 unsigned long len, unsigned long pgoff,
1091 unsigned long flags)
1092{
1093 struct bpf_map *map = filp->private_data;
1094
1095 if (map->ops->map_get_unmapped_area)
1096 return map->ops->map_get_unmapped_area(filp, addr, len, pgoff, flags);
1097#ifdef CONFIG_MMU
1098 return mm_get_unmapped_area(current->mm, filp, addr, len, pgoff, flags);
1099#else
1100 return addr;
1101#endif
1102}
1103
1104const struct file_operations bpf_map_fops = {
1105#ifdef CONFIG_PROC_FS
1106 .show_fdinfo = bpf_map_show_fdinfo,
1107#endif
1108 .release = bpf_map_release,
1109 .read = bpf_dummy_read,
1110 .write = bpf_dummy_write,
1111 .mmap = bpf_map_mmap,
1112 .poll = bpf_map_poll,
1113 .get_unmapped_area = bpf_get_unmapped_area,
1114};
1115
1116int bpf_map_new_fd(struct bpf_map *map, int flags)
1117{
1118 int ret;
1119
1120 ret = security_bpf_map(map, OPEN_FMODE(flags));
1121 if (ret < 0)
1122 return ret;
1123
1124 return anon_inode_getfd("bpf-map", &bpf_map_fops, map,
1125 flags | O_CLOEXEC);
1126}
1127
1128int bpf_get_file_flag(int flags)
1129{
1130 if ((flags & BPF_F_RDONLY) && (flags & BPF_F_WRONLY))
1131 return -EINVAL;
1132 if (flags & BPF_F_RDONLY)
1133 return O_RDONLY;
1134 if (flags & BPF_F_WRONLY)
1135 return O_WRONLY;
1136 return O_RDWR;
1137}
1138
1139/* helper macro to check that unused fields 'union bpf_attr' are zero */
1140#define CHECK_ATTR(CMD) \
1141 memchr_inv((void *) &attr->CMD##_LAST_FIELD + \
1142 sizeof(attr->CMD##_LAST_FIELD), 0, \
1143 sizeof(*attr) - \
1144 offsetof(union bpf_attr, CMD##_LAST_FIELD) - \
1145 sizeof(attr->CMD##_LAST_FIELD)) != NULL
1146
1147/* dst and src must have at least "size" number of bytes.
1148 * Return strlen on success and < 0 on error.
1149 */
1150int bpf_obj_name_cpy(char *dst, const char *src, unsigned int size)
1151{
1152 const char *end = src + size;
1153 const char *orig_src = src;
1154
1155 memset(dst, 0, size);
1156 /* Copy all isalnum(), '_' and '.' chars. */
1157 while (src < end && *src) {
1158 if (!isalnum(*src) &&
1159 *src != '_' && *src != '.')
1160 return -EINVAL;
1161 *dst++ = *src++;
1162 }
1163
1164 /* No '\0' found in "size" number of bytes */
1165 if (src == end)
1166 return -EINVAL;
1167
1168 return src - orig_src;
1169}
1170
1171int map_check_no_btf(const struct bpf_map *map,
1172 const struct btf *btf,
1173 const struct btf_type *key_type,
1174 const struct btf_type *value_type)
1175{
1176 return -ENOTSUPP;
1177}
1178
1179static int map_check_btf(struct bpf_map *map, struct bpf_token *token,
1180 const struct btf *btf, u32 btf_key_id, u32 btf_value_id)
1181{
1182 const struct btf_type *key_type, *value_type;
1183 u32 key_size, value_size;
1184 int ret = 0;
1185
1186 /* Some maps allow key to be unspecified. */
1187 if (btf_key_id) {
1188 key_type = btf_type_id_size(btf, &btf_key_id, &key_size);
1189 if (!key_type || key_size != map->key_size)
1190 return -EINVAL;
1191 } else {
1192 key_type = btf_type_by_id(btf, 0);
1193 if (!map->ops->map_check_btf)
1194 return -EINVAL;
1195 }
1196
1197 value_type = btf_type_id_size(btf, &btf_value_id, &value_size);
1198 if (!value_type || value_size != map->value_size)
1199 return -EINVAL;
1200
1201 map->record = btf_parse_fields(btf, value_type,
1202 BPF_SPIN_LOCK | BPF_TIMER | BPF_KPTR | BPF_LIST_HEAD |
1203 BPF_RB_ROOT | BPF_REFCOUNT | BPF_WORKQUEUE | BPF_UPTR,
1204 map->value_size);
1205 if (!IS_ERR_OR_NULL(map->record)) {
1206 int i;
1207
1208 if (!bpf_token_capable(token, CAP_BPF)) {
1209 ret = -EPERM;
1210 goto free_map_tab;
1211 }
1212 if (map->map_flags & (BPF_F_RDONLY_PROG | BPF_F_WRONLY_PROG)) {
1213 ret = -EACCES;
1214 goto free_map_tab;
1215 }
1216 for (i = 0; i < sizeof(map->record->field_mask) * 8; i++) {
1217 switch (map->record->field_mask & (1 << i)) {
1218 case 0:
1219 continue;
1220 case BPF_SPIN_LOCK:
1221 if (map->map_type != BPF_MAP_TYPE_HASH &&
1222 map->map_type != BPF_MAP_TYPE_ARRAY &&
1223 map->map_type != BPF_MAP_TYPE_CGROUP_STORAGE &&
1224 map->map_type != BPF_MAP_TYPE_SK_STORAGE &&
1225 map->map_type != BPF_MAP_TYPE_INODE_STORAGE &&
1226 map->map_type != BPF_MAP_TYPE_TASK_STORAGE &&
1227 map->map_type != BPF_MAP_TYPE_CGRP_STORAGE) {
1228 ret = -EOPNOTSUPP;
1229 goto free_map_tab;
1230 }
1231 break;
1232 case BPF_TIMER:
1233 case BPF_WORKQUEUE:
1234 if (map->map_type != BPF_MAP_TYPE_HASH &&
1235 map->map_type != BPF_MAP_TYPE_LRU_HASH &&
1236 map->map_type != BPF_MAP_TYPE_ARRAY) {
1237 ret = -EOPNOTSUPP;
1238 goto free_map_tab;
1239 }
1240 break;
1241 case BPF_KPTR_UNREF:
1242 case BPF_KPTR_REF:
1243 case BPF_KPTR_PERCPU:
1244 case BPF_REFCOUNT:
1245 if (map->map_type != BPF_MAP_TYPE_HASH &&
1246 map->map_type != BPF_MAP_TYPE_PERCPU_HASH &&
1247 map->map_type != BPF_MAP_TYPE_LRU_HASH &&
1248 map->map_type != BPF_MAP_TYPE_LRU_PERCPU_HASH &&
1249 map->map_type != BPF_MAP_TYPE_ARRAY &&
1250 map->map_type != BPF_MAP_TYPE_PERCPU_ARRAY &&
1251 map->map_type != BPF_MAP_TYPE_SK_STORAGE &&
1252 map->map_type != BPF_MAP_TYPE_INODE_STORAGE &&
1253 map->map_type != BPF_MAP_TYPE_TASK_STORAGE &&
1254 map->map_type != BPF_MAP_TYPE_CGRP_STORAGE) {
1255 ret = -EOPNOTSUPP;
1256 goto free_map_tab;
1257 }
1258 break;
1259 case BPF_UPTR:
1260 if (map->map_type != BPF_MAP_TYPE_TASK_STORAGE) {
1261 ret = -EOPNOTSUPP;
1262 goto free_map_tab;
1263 }
1264 break;
1265 case BPF_LIST_HEAD:
1266 case BPF_RB_ROOT:
1267 if (map->map_type != BPF_MAP_TYPE_HASH &&
1268 map->map_type != BPF_MAP_TYPE_LRU_HASH &&
1269 map->map_type != BPF_MAP_TYPE_ARRAY) {
1270 ret = -EOPNOTSUPP;
1271 goto free_map_tab;
1272 }
1273 break;
1274 default:
1275 /* Fail if map_type checks are missing for a field type */
1276 ret = -EOPNOTSUPP;
1277 goto free_map_tab;
1278 }
1279 }
1280 }
1281
1282 ret = btf_check_and_fixup_fields(btf, map->record);
1283 if (ret < 0)
1284 goto free_map_tab;
1285
1286 if (map->ops->map_check_btf) {
1287 ret = map->ops->map_check_btf(map, btf, key_type, value_type);
1288 if (ret < 0)
1289 goto free_map_tab;
1290 }
1291
1292 return ret;
1293free_map_tab:
1294 bpf_map_free_record(map);
1295 return ret;
1296}
1297
1298static bool bpf_net_capable(void)
1299{
1300 return capable(CAP_NET_ADMIN) || capable(CAP_SYS_ADMIN);
1301}
1302
1303#define BPF_MAP_CREATE_LAST_FIELD map_token_fd
1304/* called via syscall */
1305static int map_create(union bpf_attr *attr)
1306{
1307 const struct bpf_map_ops *ops;
1308 struct bpf_token *token = NULL;
1309 int numa_node = bpf_map_attr_numa_node(attr);
1310 u32 map_type = attr->map_type;
1311 struct bpf_map *map;
1312 bool token_flag;
1313 int f_flags;
1314 int err;
1315
1316 err = CHECK_ATTR(BPF_MAP_CREATE);
1317 if (err)
1318 return -EINVAL;
1319
1320 /* check BPF_F_TOKEN_FD flag, remember if it's set, and then clear it
1321 * to avoid per-map type checks tripping on unknown flag
1322 */
1323 token_flag = attr->map_flags & BPF_F_TOKEN_FD;
1324 attr->map_flags &= ~BPF_F_TOKEN_FD;
1325
1326 if (attr->btf_vmlinux_value_type_id) {
1327 if (attr->map_type != BPF_MAP_TYPE_STRUCT_OPS ||
1328 attr->btf_key_type_id || attr->btf_value_type_id)
1329 return -EINVAL;
1330 } else if (attr->btf_key_type_id && !attr->btf_value_type_id) {
1331 return -EINVAL;
1332 }
1333
1334 if (attr->map_type != BPF_MAP_TYPE_BLOOM_FILTER &&
1335 attr->map_type != BPF_MAP_TYPE_ARENA &&
1336 attr->map_extra != 0)
1337 return -EINVAL;
1338
1339 f_flags = bpf_get_file_flag(attr->map_flags);
1340 if (f_flags < 0)
1341 return f_flags;
1342
1343 if (numa_node != NUMA_NO_NODE &&
1344 ((unsigned int)numa_node >= nr_node_ids ||
1345 !node_online(numa_node)))
1346 return -EINVAL;
1347
1348 /* find map type and init map: hashtable vs rbtree vs bloom vs ... */
1349 map_type = attr->map_type;
1350 if (map_type >= ARRAY_SIZE(bpf_map_types))
1351 return -EINVAL;
1352 map_type = array_index_nospec(map_type, ARRAY_SIZE(bpf_map_types));
1353 ops = bpf_map_types[map_type];
1354 if (!ops)
1355 return -EINVAL;
1356
1357 if (ops->map_alloc_check) {
1358 err = ops->map_alloc_check(attr);
1359 if (err)
1360 return err;
1361 }
1362 if (attr->map_ifindex)
1363 ops = &bpf_map_offload_ops;
1364 if (!ops->map_mem_usage)
1365 return -EINVAL;
1366
1367 if (token_flag) {
1368 token = bpf_token_get_from_fd(attr->map_token_fd);
1369 if (IS_ERR(token))
1370 return PTR_ERR(token);
1371
1372 /* if current token doesn't grant map creation permissions,
1373 * then we can't use this token, so ignore it and rely on
1374 * system-wide capabilities checks
1375 */
1376 if (!bpf_token_allow_cmd(token, BPF_MAP_CREATE) ||
1377 !bpf_token_allow_map_type(token, attr->map_type)) {
1378 bpf_token_put(token);
1379 token = NULL;
1380 }
1381 }
1382
1383 err = -EPERM;
1384
1385 /* Intent here is for unprivileged_bpf_disabled to block BPF map
1386 * creation for unprivileged users; other actions depend
1387 * on fd availability and access to bpffs, so are dependent on
1388 * object creation success. Even with unprivileged BPF disabled,
1389 * capability checks are still carried out.
1390 */
1391 if (sysctl_unprivileged_bpf_disabled && !bpf_token_capable(token, CAP_BPF))
1392 goto put_token;
1393
1394 /* check privileged map type permissions */
1395 switch (map_type) {
1396 case BPF_MAP_TYPE_ARRAY:
1397 case BPF_MAP_TYPE_PERCPU_ARRAY:
1398 case BPF_MAP_TYPE_PROG_ARRAY:
1399 case BPF_MAP_TYPE_PERF_EVENT_ARRAY:
1400 case BPF_MAP_TYPE_CGROUP_ARRAY:
1401 case BPF_MAP_TYPE_ARRAY_OF_MAPS:
1402 case BPF_MAP_TYPE_HASH:
1403 case BPF_MAP_TYPE_PERCPU_HASH:
1404 case BPF_MAP_TYPE_HASH_OF_MAPS:
1405 case BPF_MAP_TYPE_RINGBUF:
1406 case BPF_MAP_TYPE_USER_RINGBUF:
1407 case BPF_MAP_TYPE_CGROUP_STORAGE:
1408 case BPF_MAP_TYPE_PERCPU_CGROUP_STORAGE:
1409 /* unprivileged */
1410 break;
1411 case BPF_MAP_TYPE_SK_STORAGE:
1412 case BPF_MAP_TYPE_INODE_STORAGE:
1413 case BPF_MAP_TYPE_TASK_STORAGE:
1414 case BPF_MAP_TYPE_CGRP_STORAGE:
1415 case BPF_MAP_TYPE_BLOOM_FILTER:
1416 case BPF_MAP_TYPE_LPM_TRIE:
1417 case BPF_MAP_TYPE_REUSEPORT_SOCKARRAY:
1418 case BPF_MAP_TYPE_STACK_TRACE:
1419 case BPF_MAP_TYPE_QUEUE:
1420 case BPF_MAP_TYPE_STACK:
1421 case BPF_MAP_TYPE_LRU_HASH:
1422 case BPF_MAP_TYPE_LRU_PERCPU_HASH:
1423 case BPF_MAP_TYPE_STRUCT_OPS:
1424 case BPF_MAP_TYPE_CPUMAP:
1425 case BPF_MAP_TYPE_ARENA:
1426 if (!bpf_token_capable(token, CAP_BPF))
1427 goto put_token;
1428 break;
1429 case BPF_MAP_TYPE_SOCKMAP:
1430 case BPF_MAP_TYPE_SOCKHASH:
1431 case BPF_MAP_TYPE_DEVMAP:
1432 case BPF_MAP_TYPE_DEVMAP_HASH:
1433 case BPF_MAP_TYPE_XSKMAP:
1434 if (!bpf_token_capable(token, CAP_NET_ADMIN))
1435 goto put_token;
1436 break;
1437 default:
1438 WARN(1, "unsupported map type %d", map_type);
1439 goto put_token;
1440 }
1441
1442 map = ops->map_alloc(attr);
1443 if (IS_ERR(map)) {
1444 err = PTR_ERR(map);
1445 goto put_token;
1446 }
1447 map->ops = ops;
1448 map->map_type = map_type;
1449
1450 err = bpf_obj_name_cpy(map->name, attr->map_name,
1451 sizeof(attr->map_name));
1452 if (err < 0)
1453 goto free_map;
1454
1455 atomic64_set(&map->refcnt, 1);
1456 atomic64_set(&map->usercnt, 1);
1457 mutex_init(&map->freeze_mutex);
1458 spin_lock_init(&map->owner.lock);
1459
1460 if (attr->btf_key_type_id || attr->btf_value_type_id ||
1461 /* Even the map's value is a kernel's struct,
1462 * the bpf_prog.o must have BTF to begin with
1463 * to figure out the corresponding kernel's
1464 * counter part. Thus, attr->btf_fd has
1465 * to be valid also.
1466 */
1467 attr->btf_vmlinux_value_type_id) {
1468 struct btf *btf;
1469
1470 btf = btf_get_by_fd(attr->btf_fd);
1471 if (IS_ERR(btf)) {
1472 err = PTR_ERR(btf);
1473 goto free_map;
1474 }
1475 if (btf_is_kernel(btf)) {
1476 btf_put(btf);
1477 err = -EACCES;
1478 goto free_map;
1479 }
1480 map->btf = btf;
1481
1482 if (attr->btf_value_type_id) {
1483 err = map_check_btf(map, token, btf, attr->btf_key_type_id,
1484 attr->btf_value_type_id);
1485 if (err)
1486 goto free_map;
1487 }
1488
1489 map->btf_key_type_id = attr->btf_key_type_id;
1490 map->btf_value_type_id = attr->btf_value_type_id;
1491 map->btf_vmlinux_value_type_id =
1492 attr->btf_vmlinux_value_type_id;
1493 }
1494
1495 err = security_bpf_map_create(map, attr, token);
1496 if (err)
1497 goto free_map_sec;
1498
1499 err = bpf_map_alloc_id(map);
1500 if (err)
1501 goto free_map_sec;
1502
1503 bpf_map_save_memcg(map);
1504 bpf_token_put(token);
1505
1506 err = bpf_map_new_fd(map, f_flags);
1507 if (err < 0) {
1508 /* failed to allocate fd.
1509 * bpf_map_put_with_uref() is needed because the above
1510 * bpf_map_alloc_id() has published the map
1511 * to the userspace and the userspace may
1512 * have refcnt-ed it through BPF_MAP_GET_FD_BY_ID.
1513 */
1514 bpf_map_put_with_uref(map);
1515 return err;
1516 }
1517
1518 return err;
1519
1520free_map_sec:
1521 security_bpf_map_free(map);
1522free_map:
1523 bpf_map_free(map);
1524put_token:
1525 bpf_token_put(token);
1526 return err;
1527}
1528
1529void bpf_map_inc(struct bpf_map *map)
1530{
1531 atomic64_inc(&map->refcnt);
1532}
1533EXPORT_SYMBOL_GPL(bpf_map_inc);
1534
1535void bpf_map_inc_with_uref(struct bpf_map *map)
1536{
1537 atomic64_inc(&map->refcnt);
1538 atomic64_inc(&map->usercnt);
1539}
1540EXPORT_SYMBOL_GPL(bpf_map_inc_with_uref);
1541
1542struct bpf_map *bpf_map_get(u32 ufd)
1543{
1544 CLASS(fd, f)(ufd);
1545 struct bpf_map *map = __bpf_map_get(f);
1546
1547 if (!IS_ERR(map))
1548 bpf_map_inc(map);
1549
1550 return map;
1551}
1552EXPORT_SYMBOL(bpf_map_get);
1553
1554struct bpf_map *bpf_map_get_with_uref(u32 ufd)
1555{
1556 CLASS(fd, f)(ufd);
1557 struct bpf_map *map = __bpf_map_get(f);
1558
1559 if (!IS_ERR(map))
1560 bpf_map_inc_with_uref(map);
1561
1562 return map;
1563}
1564
1565/* map_idr_lock should have been held or the map should have been
1566 * protected by rcu read lock.
1567 */
1568struct bpf_map *__bpf_map_inc_not_zero(struct bpf_map *map, bool uref)
1569{
1570 int refold;
1571
1572 refold = atomic64_fetch_add_unless(&map->refcnt, 1, 0);
1573 if (!refold)
1574 return ERR_PTR(-ENOENT);
1575 if (uref)
1576 atomic64_inc(&map->usercnt);
1577
1578 return map;
1579}
1580
1581struct bpf_map *bpf_map_inc_not_zero(struct bpf_map *map)
1582{
1583 spin_lock_bh(&map_idr_lock);
1584 map = __bpf_map_inc_not_zero(map, false);
1585 spin_unlock_bh(&map_idr_lock);
1586
1587 return map;
1588}
1589EXPORT_SYMBOL_GPL(bpf_map_inc_not_zero);
1590
1591int __weak bpf_stackmap_copy(struct bpf_map *map, void *key, void *value)
1592{
1593 return -ENOTSUPP;
1594}
1595
1596static void *__bpf_copy_key(void __user *ukey, u64 key_size)
1597{
1598 if (key_size)
1599 return vmemdup_user(ukey, key_size);
1600
1601 if (ukey)
1602 return ERR_PTR(-EINVAL);
1603
1604 return NULL;
1605}
1606
1607static void *___bpf_copy_key(bpfptr_t ukey, u64 key_size)
1608{
1609 if (key_size)
1610 return kvmemdup_bpfptr(ukey, key_size);
1611
1612 if (!bpfptr_is_null(ukey))
1613 return ERR_PTR(-EINVAL);
1614
1615 return NULL;
1616}
1617
1618/* last field in 'union bpf_attr' used by this command */
1619#define BPF_MAP_LOOKUP_ELEM_LAST_FIELD flags
1620
1621static int map_lookup_elem(union bpf_attr *attr)
1622{
1623 void __user *ukey = u64_to_user_ptr(attr->key);
1624 void __user *uvalue = u64_to_user_ptr(attr->value);
1625 struct bpf_map *map;
1626 void *key, *value;
1627 u32 value_size;
1628 int err;
1629
1630 if (CHECK_ATTR(BPF_MAP_LOOKUP_ELEM))
1631 return -EINVAL;
1632
1633 if (attr->flags & ~BPF_F_LOCK)
1634 return -EINVAL;
1635
1636 CLASS(fd, f)(attr->map_fd);
1637 map = __bpf_map_get(f);
1638 if (IS_ERR(map))
1639 return PTR_ERR(map);
1640 if (!(map_get_sys_perms(map, f) & FMODE_CAN_READ))
1641 return -EPERM;
1642
1643 if ((attr->flags & BPF_F_LOCK) &&
1644 !btf_record_has_field(map->record, BPF_SPIN_LOCK))
1645 return -EINVAL;
1646
1647 key = __bpf_copy_key(ukey, map->key_size);
1648 if (IS_ERR(key))
1649 return PTR_ERR(key);
1650
1651 value_size = bpf_map_value_size(map);
1652
1653 err = -ENOMEM;
1654 value = kvmalloc(value_size, GFP_USER | __GFP_NOWARN);
1655 if (!value)
1656 goto free_key;
1657
1658 if (map->map_type == BPF_MAP_TYPE_BLOOM_FILTER) {
1659 if (copy_from_user(value, uvalue, value_size))
1660 err = -EFAULT;
1661 else
1662 err = bpf_map_copy_value(map, key, value, attr->flags);
1663 goto free_value;
1664 }
1665
1666 err = bpf_map_copy_value(map, key, value, attr->flags);
1667 if (err)
1668 goto free_value;
1669
1670 err = -EFAULT;
1671 if (copy_to_user(uvalue, value, value_size) != 0)
1672 goto free_value;
1673
1674 err = 0;
1675
1676free_value:
1677 kvfree(value);
1678free_key:
1679 kvfree(key);
1680 return err;
1681}
1682
1683
1684#define BPF_MAP_UPDATE_ELEM_LAST_FIELD flags
1685
1686static int map_update_elem(union bpf_attr *attr, bpfptr_t uattr)
1687{
1688 bpfptr_t ukey = make_bpfptr(attr->key, uattr.is_kernel);
1689 bpfptr_t uvalue = make_bpfptr(attr->value, uattr.is_kernel);
1690 struct bpf_map *map;
1691 void *key, *value;
1692 u32 value_size;
1693 int err;
1694
1695 if (CHECK_ATTR(BPF_MAP_UPDATE_ELEM))
1696 return -EINVAL;
1697
1698 CLASS(fd, f)(attr->map_fd);
1699 map = __bpf_map_get(f);
1700 if (IS_ERR(map))
1701 return PTR_ERR(map);
1702 bpf_map_write_active_inc(map);
1703 if (!(map_get_sys_perms(map, f) & FMODE_CAN_WRITE)) {
1704 err = -EPERM;
1705 goto err_put;
1706 }
1707
1708 if ((attr->flags & BPF_F_LOCK) &&
1709 !btf_record_has_field(map->record, BPF_SPIN_LOCK)) {
1710 err = -EINVAL;
1711 goto err_put;
1712 }
1713
1714 key = ___bpf_copy_key(ukey, map->key_size);
1715 if (IS_ERR(key)) {
1716 err = PTR_ERR(key);
1717 goto err_put;
1718 }
1719
1720 value_size = bpf_map_value_size(map);
1721 value = kvmemdup_bpfptr(uvalue, value_size);
1722 if (IS_ERR(value)) {
1723 err = PTR_ERR(value);
1724 goto free_key;
1725 }
1726
1727 err = bpf_map_update_value(map, fd_file(f), key, value, attr->flags);
1728 if (!err)
1729 maybe_wait_bpf_programs(map);
1730
1731 kvfree(value);
1732free_key:
1733 kvfree(key);
1734err_put:
1735 bpf_map_write_active_dec(map);
1736 return err;
1737}
1738
1739#define BPF_MAP_DELETE_ELEM_LAST_FIELD key
1740
1741static int map_delete_elem(union bpf_attr *attr, bpfptr_t uattr)
1742{
1743 bpfptr_t ukey = make_bpfptr(attr->key, uattr.is_kernel);
1744 struct bpf_map *map;
1745 void *key;
1746 int err;
1747
1748 if (CHECK_ATTR(BPF_MAP_DELETE_ELEM))
1749 return -EINVAL;
1750
1751 CLASS(fd, f)(attr->map_fd);
1752 map = __bpf_map_get(f);
1753 if (IS_ERR(map))
1754 return PTR_ERR(map);
1755 bpf_map_write_active_inc(map);
1756 if (!(map_get_sys_perms(map, f) & FMODE_CAN_WRITE)) {
1757 err = -EPERM;
1758 goto err_put;
1759 }
1760
1761 key = ___bpf_copy_key(ukey, map->key_size);
1762 if (IS_ERR(key)) {
1763 err = PTR_ERR(key);
1764 goto err_put;
1765 }
1766
1767 if (bpf_map_is_offloaded(map)) {
1768 err = bpf_map_offload_delete_elem(map, key);
1769 goto out;
1770 } else if (IS_FD_PROG_ARRAY(map) ||
1771 map->map_type == BPF_MAP_TYPE_STRUCT_OPS) {
1772 /* These maps require sleepable context */
1773 err = map->ops->map_delete_elem(map, key);
1774 goto out;
1775 }
1776
1777 bpf_disable_instrumentation();
1778 rcu_read_lock();
1779 err = map->ops->map_delete_elem(map, key);
1780 rcu_read_unlock();
1781 bpf_enable_instrumentation();
1782 if (!err)
1783 maybe_wait_bpf_programs(map);
1784out:
1785 kvfree(key);
1786err_put:
1787 bpf_map_write_active_dec(map);
1788 return err;
1789}
1790
1791/* last field in 'union bpf_attr' used by this command */
1792#define BPF_MAP_GET_NEXT_KEY_LAST_FIELD next_key
1793
1794static int map_get_next_key(union bpf_attr *attr)
1795{
1796 void __user *ukey = u64_to_user_ptr(attr->key);
1797 void __user *unext_key = u64_to_user_ptr(attr->next_key);
1798 struct bpf_map *map;
1799 void *key, *next_key;
1800 int err;
1801
1802 if (CHECK_ATTR(BPF_MAP_GET_NEXT_KEY))
1803 return -EINVAL;
1804
1805 CLASS(fd, f)(attr->map_fd);
1806 map = __bpf_map_get(f);
1807 if (IS_ERR(map))
1808 return PTR_ERR(map);
1809 if (!(map_get_sys_perms(map, f) & FMODE_CAN_READ))
1810 return -EPERM;
1811
1812 if (ukey) {
1813 key = __bpf_copy_key(ukey, map->key_size);
1814 if (IS_ERR(key))
1815 return PTR_ERR(key);
1816 } else {
1817 key = NULL;
1818 }
1819
1820 err = -ENOMEM;
1821 next_key = kvmalloc(map->key_size, GFP_USER);
1822 if (!next_key)
1823 goto free_key;
1824
1825 if (bpf_map_is_offloaded(map)) {
1826 err = bpf_map_offload_get_next_key(map, key, next_key);
1827 goto out;
1828 }
1829
1830 rcu_read_lock();
1831 err = map->ops->map_get_next_key(map, key, next_key);
1832 rcu_read_unlock();
1833out:
1834 if (err)
1835 goto free_next_key;
1836
1837 err = -EFAULT;
1838 if (copy_to_user(unext_key, next_key, map->key_size) != 0)
1839 goto free_next_key;
1840
1841 err = 0;
1842
1843free_next_key:
1844 kvfree(next_key);
1845free_key:
1846 kvfree(key);
1847 return err;
1848}
1849
1850int generic_map_delete_batch(struct bpf_map *map,
1851 const union bpf_attr *attr,
1852 union bpf_attr __user *uattr)
1853{
1854 void __user *keys = u64_to_user_ptr(attr->batch.keys);
1855 u32 cp, max_count;
1856 int err = 0;
1857 void *key;
1858
1859 if (attr->batch.elem_flags & ~BPF_F_LOCK)
1860 return -EINVAL;
1861
1862 if ((attr->batch.elem_flags & BPF_F_LOCK) &&
1863 !btf_record_has_field(map->record, BPF_SPIN_LOCK)) {
1864 return -EINVAL;
1865 }
1866
1867 max_count = attr->batch.count;
1868 if (!max_count)
1869 return 0;
1870
1871 if (put_user(0, &uattr->batch.count))
1872 return -EFAULT;
1873
1874 key = kvmalloc(map->key_size, GFP_USER | __GFP_NOWARN);
1875 if (!key)
1876 return -ENOMEM;
1877
1878 for (cp = 0; cp < max_count; cp++) {
1879 err = -EFAULT;
1880 if (copy_from_user(key, keys + cp * map->key_size,
1881 map->key_size))
1882 break;
1883
1884 if (bpf_map_is_offloaded(map)) {
1885 err = bpf_map_offload_delete_elem(map, key);
1886 break;
1887 }
1888
1889 bpf_disable_instrumentation();
1890 rcu_read_lock();
1891 err = map->ops->map_delete_elem(map, key);
1892 rcu_read_unlock();
1893 bpf_enable_instrumentation();
1894 if (err)
1895 break;
1896 cond_resched();
1897 }
1898 if (copy_to_user(&uattr->batch.count, &cp, sizeof(cp)))
1899 err = -EFAULT;
1900
1901 kvfree(key);
1902
1903 return err;
1904}
1905
1906int generic_map_update_batch(struct bpf_map *map, struct file *map_file,
1907 const union bpf_attr *attr,
1908 union bpf_attr __user *uattr)
1909{
1910 void __user *values = u64_to_user_ptr(attr->batch.values);
1911 void __user *keys = u64_to_user_ptr(attr->batch.keys);
1912 u32 value_size, cp, max_count;
1913 void *key, *value;
1914 int err = 0;
1915
1916 if (attr->batch.elem_flags & ~BPF_F_LOCK)
1917 return -EINVAL;
1918
1919 if ((attr->batch.elem_flags & BPF_F_LOCK) &&
1920 !btf_record_has_field(map->record, BPF_SPIN_LOCK)) {
1921 return -EINVAL;
1922 }
1923
1924 value_size = bpf_map_value_size(map);
1925
1926 max_count = attr->batch.count;
1927 if (!max_count)
1928 return 0;
1929
1930 if (put_user(0, &uattr->batch.count))
1931 return -EFAULT;
1932
1933 key = kvmalloc(map->key_size, GFP_USER | __GFP_NOWARN);
1934 if (!key)
1935 return -ENOMEM;
1936
1937 value = kvmalloc(value_size, GFP_USER | __GFP_NOWARN);
1938 if (!value) {
1939 kvfree(key);
1940 return -ENOMEM;
1941 }
1942
1943 for (cp = 0; cp < max_count; cp++) {
1944 err = -EFAULT;
1945 if (copy_from_user(key, keys + cp * map->key_size,
1946 map->key_size) ||
1947 copy_from_user(value, values + cp * value_size, value_size))
1948 break;
1949
1950 err = bpf_map_update_value(map, map_file, key, value,
1951 attr->batch.elem_flags);
1952
1953 if (err)
1954 break;
1955 cond_resched();
1956 }
1957
1958 if (copy_to_user(&uattr->batch.count, &cp, sizeof(cp)))
1959 err = -EFAULT;
1960
1961 kvfree(value);
1962 kvfree(key);
1963
1964 return err;
1965}
1966
1967#define MAP_LOOKUP_RETRIES 3
1968
1969int generic_map_lookup_batch(struct bpf_map *map,
1970 const union bpf_attr *attr,
1971 union bpf_attr __user *uattr)
1972{
1973 void __user *uobatch = u64_to_user_ptr(attr->batch.out_batch);
1974 void __user *ubatch = u64_to_user_ptr(attr->batch.in_batch);
1975 void __user *values = u64_to_user_ptr(attr->batch.values);
1976 void __user *keys = u64_to_user_ptr(attr->batch.keys);
1977 void *buf, *buf_prevkey, *prev_key, *key, *value;
1978 int err, retry = MAP_LOOKUP_RETRIES;
1979 u32 value_size, cp, max_count;
1980
1981 if (attr->batch.elem_flags & ~BPF_F_LOCK)
1982 return -EINVAL;
1983
1984 if ((attr->batch.elem_flags & BPF_F_LOCK) &&
1985 !btf_record_has_field(map->record, BPF_SPIN_LOCK))
1986 return -EINVAL;
1987
1988 value_size = bpf_map_value_size(map);
1989
1990 max_count = attr->batch.count;
1991 if (!max_count)
1992 return 0;
1993
1994 if (put_user(0, &uattr->batch.count))
1995 return -EFAULT;
1996
1997 buf_prevkey = kvmalloc(map->key_size, GFP_USER | __GFP_NOWARN);
1998 if (!buf_prevkey)
1999 return -ENOMEM;
2000
2001 buf = kvmalloc(map->key_size + value_size, GFP_USER | __GFP_NOWARN);
2002 if (!buf) {
2003 kvfree(buf_prevkey);
2004 return -ENOMEM;
2005 }
2006
2007 err = -EFAULT;
2008 prev_key = NULL;
2009 if (ubatch && copy_from_user(buf_prevkey, ubatch, map->key_size))
2010 goto free_buf;
2011 key = buf;
2012 value = key + map->key_size;
2013 if (ubatch)
2014 prev_key = buf_prevkey;
2015
2016 for (cp = 0; cp < max_count;) {
2017 rcu_read_lock();
2018 err = map->ops->map_get_next_key(map, prev_key, key);
2019 rcu_read_unlock();
2020 if (err)
2021 break;
2022 err = bpf_map_copy_value(map, key, value,
2023 attr->batch.elem_flags);
2024
2025 if (err == -ENOENT) {
2026 if (retry) {
2027 retry--;
2028 continue;
2029 }
2030 err = -EINTR;
2031 break;
2032 }
2033
2034 if (err)
2035 goto free_buf;
2036
2037 if (copy_to_user(keys + cp * map->key_size, key,
2038 map->key_size)) {
2039 err = -EFAULT;
2040 goto free_buf;
2041 }
2042 if (copy_to_user(values + cp * value_size, value, value_size)) {
2043 err = -EFAULT;
2044 goto free_buf;
2045 }
2046
2047 if (!prev_key)
2048 prev_key = buf_prevkey;
2049
2050 swap(prev_key, key);
2051 retry = MAP_LOOKUP_RETRIES;
2052 cp++;
2053 cond_resched();
2054 }
2055
2056 if (err == -EFAULT)
2057 goto free_buf;
2058
2059 if ((copy_to_user(&uattr->batch.count, &cp, sizeof(cp)) ||
2060 (cp && copy_to_user(uobatch, prev_key, map->key_size))))
2061 err = -EFAULT;
2062
2063free_buf:
2064 kvfree(buf_prevkey);
2065 kvfree(buf);
2066 return err;
2067}
2068
2069#define BPF_MAP_LOOKUP_AND_DELETE_ELEM_LAST_FIELD flags
2070
2071static int map_lookup_and_delete_elem(union bpf_attr *attr)
2072{
2073 void __user *ukey = u64_to_user_ptr(attr->key);
2074 void __user *uvalue = u64_to_user_ptr(attr->value);
2075 struct bpf_map *map;
2076 void *key, *value;
2077 u32 value_size;
2078 int err;
2079
2080 if (CHECK_ATTR(BPF_MAP_LOOKUP_AND_DELETE_ELEM))
2081 return -EINVAL;
2082
2083 if (attr->flags & ~BPF_F_LOCK)
2084 return -EINVAL;
2085
2086 CLASS(fd, f)(attr->map_fd);
2087 map = __bpf_map_get(f);
2088 if (IS_ERR(map))
2089 return PTR_ERR(map);
2090 bpf_map_write_active_inc(map);
2091 if (!(map_get_sys_perms(map, f) & FMODE_CAN_READ) ||
2092 !(map_get_sys_perms(map, f) & FMODE_CAN_WRITE)) {
2093 err = -EPERM;
2094 goto err_put;
2095 }
2096
2097 if (attr->flags &&
2098 (map->map_type == BPF_MAP_TYPE_QUEUE ||
2099 map->map_type == BPF_MAP_TYPE_STACK)) {
2100 err = -EINVAL;
2101 goto err_put;
2102 }
2103
2104 if ((attr->flags & BPF_F_LOCK) &&
2105 !btf_record_has_field(map->record, BPF_SPIN_LOCK)) {
2106 err = -EINVAL;
2107 goto err_put;
2108 }
2109
2110 key = __bpf_copy_key(ukey, map->key_size);
2111 if (IS_ERR(key)) {
2112 err = PTR_ERR(key);
2113 goto err_put;
2114 }
2115
2116 value_size = bpf_map_value_size(map);
2117
2118 err = -ENOMEM;
2119 value = kvmalloc(value_size, GFP_USER | __GFP_NOWARN);
2120 if (!value)
2121 goto free_key;
2122
2123 err = -ENOTSUPP;
2124 if (map->map_type == BPF_MAP_TYPE_QUEUE ||
2125 map->map_type == BPF_MAP_TYPE_STACK) {
2126 err = map->ops->map_pop_elem(map, value);
2127 } else if (map->map_type == BPF_MAP_TYPE_HASH ||
2128 map->map_type == BPF_MAP_TYPE_PERCPU_HASH ||
2129 map->map_type == BPF_MAP_TYPE_LRU_HASH ||
2130 map->map_type == BPF_MAP_TYPE_LRU_PERCPU_HASH) {
2131 if (!bpf_map_is_offloaded(map)) {
2132 bpf_disable_instrumentation();
2133 rcu_read_lock();
2134 err = map->ops->map_lookup_and_delete_elem(map, key, value, attr->flags);
2135 rcu_read_unlock();
2136 bpf_enable_instrumentation();
2137 }
2138 }
2139
2140 if (err)
2141 goto free_value;
2142
2143 if (copy_to_user(uvalue, value, value_size) != 0) {
2144 err = -EFAULT;
2145 goto free_value;
2146 }
2147
2148 err = 0;
2149
2150free_value:
2151 kvfree(value);
2152free_key:
2153 kvfree(key);
2154err_put:
2155 bpf_map_write_active_dec(map);
2156 return err;
2157}
2158
2159#define BPF_MAP_FREEZE_LAST_FIELD map_fd
2160
2161static int map_freeze(const union bpf_attr *attr)
2162{
2163 int err = 0;
2164 struct bpf_map *map;
2165
2166 if (CHECK_ATTR(BPF_MAP_FREEZE))
2167 return -EINVAL;
2168
2169 CLASS(fd, f)(attr->map_fd);
2170 map = __bpf_map_get(f);
2171 if (IS_ERR(map))
2172 return PTR_ERR(map);
2173
2174 if (map->map_type == BPF_MAP_TYPE_STRUCT_OPS || !IS_ERR_OR_NULL(map->record))
2175 return -ENOTSUPP;
2176
2177 if (!(map_get_sys_perms(map, f) & FMODE_CAN_WRITE))
2178 return -EPERM;
2179
2180 mutex_lock(&map->freeze_mutex);
2181 if (bpf_map_write_active(map)) {
2182 err = -EBUSY;
2183 goto err_put;
2184 }
2185 if (READ_ONCE(map->frozen)) {
2186 err = -EBUSY;
2187 goto err_put;
2188 }
2189
2190 WRITE_ONCE(map->frozen, true);
2191err_put:
2192 mutex_unlock(&map->freeze_mutex);
2193 return err;
2194}
2195
2196static const struct bpf_prog_ops * const bpf_prog_types[] = {
2197#define BPF_PROG_TYPE(_id, _name, prog_ctx_type, kern_ctx_type) \
2198 [_id] = & _name ## _prog_ops,
2199#define BPF_MAP_TYPE(_id, _ops)
2200#define BPF_LINK_TYPE(_id, _name)
2201#include <linux/bpf_types.h>
2202#undef BPF_PROG_TYPE
2203#undef BPF_MAP_TYPE
2204#undef BPF_LINK_TYPE
2205};
2206
2207static int find_prog_type(enum bpf_prog_type type, struct bpf_prog *prog)
2208{
2209 const struct bpf_prog_ops *ops;
2210
2211 if (type >= ARRAY_SIZE(bpf_prog_types))
2212 return -EINVAL;
2213 type = array_index_nospec(type, ARRAY_SIZE(bpf_prog_types));
2214 ops = bpf_prog_types[type];
2215 if (!ops)
2216 return -EINVAL;
2217
2218 if (!bpf_prog_is_offloaded(prog->aux))
2219 prog->aux->ops = ops;
2220 else
2221 prog->aux->ops = &bpf_offload_prog_ops;
2222 prog->type = type;
2223 return 0;
2224}
2225
2226enum bpf_audit {
2227 BPF_AUDIT_LOAD,
2228 BPF_AUDIT_UNLOAD,
2229 BPF_AUDIT_MAX,
2230};
2231
2232static const char * const bpf_audit_str[BPF_AUDIT_MAX] = {
2233 [BPF_AUDIT_LOAD] = "LOAD",
2234 [BPF_AUDIT_UNLOAD] = "UNLOAD",
2235};
2236
2237static void bpf_audit_prog(const struct bpf_prog *prog, unsigned int op)
2238{
2239 struct audit_context *ctx = NULL;
2240 struct audit_buffer *ab;
2241
2242 if (WARN_ON_ONCE(op >= BPF_AUDIT_MAX))
2243 return;
2244 if (audit_enabled == AUDIT_OFF)
2245 return;
2246 if (!in_irq() && !irqs_disabled())
2247 ctx = audit_context();
2248 ab = audit_log_start(ctx, GFP_ATOMIC, AUDIT_BPF);
2249 if (unlikely(!ab))
2250 return;
2251 audit_log_format(ab, "prog-id=%u op=%s",
2252 prog->aux->id, bpf_audit_str[op]);
2253 audit_log_end(ab);
2254}
2255
2256static int bpf_prog_alloc_id(struct bpf_prog *prog)
2257{
2258 int id;
2259
2260 idr_preload(GFP_KERNEL);
2261 spin_lock_bh(&prog_idr_lock);
2262 id = idr_alloc_cyclic(&prog_idr, prog, 1, INT_MAX, GFP_ATOMIC);
2263 if (id > 0)
2264 prog->aux->id = id;
2265 spin_unlock_bh(&prog_idr_lock);
2266 idr_preload_end();
2267
2268 /* id is in [1, INT_MAX) */
2269 if (WARN_ON_ONCE(!id))
2270 return -ENOSPC;
2271
2272 return id > 0 ? 0 : id;
2273}
2274
2275void bpf_prog_free_id(struct bpf_prog *prog)
2276{
2277 unsigned long flags;
2278
2279 /* cBPF to eBPF migrations are currently not in the idr store.
2280 * Offloaded programs are removed from the store when their device
2281 * disappears - even if someone grabs an fd to them they are unusable,
2282 * simply waiting for refcnt to drop to be freed.
2283 */
2284 if (!prog->aux->id)
2285 return;
2286
2287 spin_lock_irqsave(&prog_idr_lock, flags);
2288 idr_remove(&prog_idr, prog->aux->id);
2289 prog->aux->id = 0;
2290 spin_unlock_irqrestore(&prog_idr_lock, flags);
2291}
2292
2293static void __bpf_prog_put_rcu(struct rcu_head *rcu)
2294{
2295 struct bpf_prog_aux *aux = container_of(rcu, struct bpf_prog_aux, rcu);
2296
2297 kvfree(aux->func_info);
2298 kfree(aux->func_info_aux);
2299 free_uid(aux->user);
2300 security_bpf_prog_free(aux->prog);
2301 bpf_prog_free(aux->prog);
2302}
2303
2304static void __bpf_prog_put_noref(struct bpf_prog *prog, bool deferred)
2305{
2306 bpf_prog_kallsyms_del_all(prog);
2307 btf_put(prog->aux->btf);
2308 module_put(prog->aux->mod);
2309 kvfree(prog->aux->jited_linfo);
2310 kvfree(prog->aux->linfo);
2311 kfree(prog->aux->kfunc_tab);
2312 if (prog->aux->attach_btf)
2313 btf_put(prog->aux->attach_btf);
2314
2315 if (deferred) {
2316 if (prog->sleepable)
2317 call_rcu_tasks_trace(&prog->aux->rcu, __bpf_prog_put_rcu);
2318 else
2319 call_rcu(&prog->aux->rcu, __bpf_prog_put_rcu);
2320 } else {
2321 __bpf_prog_put_rcu(&prog->aux->rcu);
2322 }
2323}
2324
2325static void bpf_prog_put_deferred(struct work_struct *work)
2326{
2327 struct bpf_prog_aux *aux;
2328 struct bpf_prog *prog;
2329
2330 aux = container_of(work, struct bpf_prog_aux, work);
2331 prog = aux->prog;
2332 perf_event_bpf_event(prog, PERF_BPF_EVENT_PROG_UNLOAD, 0);
2333 bpf_audit_prog(prog, BPF_AUDIT_UNLOAD);
2334 bpf_prog_free_id(prog);
2335 __bpf_prog_put_noref(prog, true);
2336}
2337
2338static void __bpf_prog_put(struct bpf_prog *prog)
2339{
2340 struct bpf_prog_aux *aux = prog->aux;
2341
2342 if (atomic64_dec_and_test(&aux->refcnt)) {
2343 if (in_irq() || irqs_disabled()) {
2344 INIT_WORK(&aux->work, bpf_prog_put_deferred);
2345 schedule_work(&aux->work);
2346 } else {
2347 bpf_prog_put_deferred(&aux->work);
2348 }
2349 }
2350}
2351
2352void bpf_prog_put(struct bpf_prog *prog)
2353{
2354 __bpf_prog_put(prog);
2355}
2356EXPORT_SYMBOL_GPL(bpf_prog_put);
2357
2358static int bpf_prog_release(struct inode *inode, struct file *filp)
2359{
2360 struct bpf_prog *prog = filp->private_data;
2361
2362 bpf_prog_put(prog);
2363 return 0;
2364}
2365
2366struct bpf_prog_kstats {
2367 u64 nsecs;
2368 u64 cnt;
2369 u64 misses;
2370};
2371
2372void notrace bpf_prog_inc_misses_counter(struct bpf_prog *prog)
2373{
2374 struct bpf_prog_stats *stats;
2375 unsigned int flags;
2376
2377 stats = this_cpu_ptr(prog->stats);
2378 flags = u64_stats_update_begin_irqsave(&stats->syncp);
2379 u64_stats_inc(&stats->misses);
2380 u64_stats_update_end_irqrestore(&stats->syncp, flags);
2381}
2382
2383static void bpf_prog_get_stats(const struct bpf_prog *prog,
2384 struct bpf_prog_kstats *stats)
2385{
2386 u64 nsecs = 0, cnt = 0, misses = 0;
2387 int cpu;
2388
2389 for_each_possible_cpu(cpu) {
2390 const struct bpf_prog_stats *st;
2391 unsigned int start;
2392 u64 tnsecs, tcnt, tmisses;
2393
2394 st = per_cpu_ptr(prog->stats, cpu);
2395 do {
2396 start = u64_stats_fetch_begin(&st->syncp);
2397 tnsecs = u64_stats_read(&st->nsecs);
2398 tcnt = u64_stats_read(&st->cnt);
2399 tmisses = u64_stats_read(&st->misses);
2400 } while (u64_stats_fetch_retry(&st->syncp, start));
2401 nsecs += tnsecs;
2402 cnt += tcnt;
2403 misses += tmisses;
2404 }
2405 stats->nsecs = nsecs;
2406 stats->cnt = cnt;
2407 stats->misses = misses;
2408}
2409
2410#ifdef CONFIG_PROC_FS
2411static void bpf_prog_show_fdinfo(struct seq_file *m, struct file *filp)
2412{
2413 const struct bpf_prog *prog = filp->private_data;
2414 char prog_tag[sizeof(prog->tag) * 2 + 1] = { };
2415 struct bpf_prog_kstats stats;
2416
2417 bpf_prog_get_stats(prog, &stats);
2418 bin2hex(prog_tag, prog->tag, sizeof(prog->tag));
2419 seq_printf(m,
2420 "prog_type:\t%u\n"
2421 "prog_jited:\t%u\n"
2422 "prog_tag:\t%s\n"
2423 "memlock:\t%llu\n"
2424 "prog_id:\t%u\n"
2425 "run_time_ns:\t%llu\n"
2426 "run_cnt:\t%llu\n"
2427 "recursion_misses:\t%llu\n"
2428 "verified_insns:\t%u\n",
2429 prog->type,
2430 prog->jited,
2431 prog_tag,
2432 prog->pages * 1ULL << PAGE_SHIFT,
2433 prog->aux->id,
2434 stats.nsecs,
2435 stats.cnt,
2436 stats.misses,
2437 prog->aux->verified_insns);
2438}
2439#endif
2440
2441const struct file_operations bpf_prog_fops = {
2442#ifdef CONFIG_PROC_FS
2443 .show_fdinfo = bpf_prog_show_fdinfo,
2444#endif
2445 .release = bpf_prog_release,
2446 .read = bpf_dummy_read,
2447 .write = bpf_dummy_write,
2448};
2449
2450int bpf_prog_new_fd(struct bpf_prog *prog)
2451{
2452 int ret;
2453
2454 ret = security_bpf_prog(prog);
2455 if (ret < 0)
2456 return ret;
2457
2458 return anon_inode_getfd("bpf-prog", &bpf_prog_fops, prog,
2459 O_RDWR | O_CLOEXEC);
2460}
2461
2462void bpf_prog_add(struct bpf_prog *prog, int i)
2463{
2464 atomic64_add(i, &prog->aux->refcnt);
2465}
2466EXPORT_SYMBOL_GPL(bpf_prog_add);
2467
2468void bpf_prog_sub(struct bpf_prog *prog, int i)
2469{
2470 /* Only to be used for undoing previous bpf_prog_add() in some
2471 * error path. We still know that another entity in our call
2472 * path holds a reference to the program, thus atomic_sub() can
2473 * be safely used in such cases!
2474 */
2475 WARN_ON(atomic64_sub_return(i, &prog->aux->refcnt) == 0);
2476}
2477EXPORT_SYMBOL_GPL(bpf_prog_sub);
2478
2479void bpf_prog_inc(struct bpf_prog *prog)
2480{
2481 atomic64_inc(&prog->aux->refcnt);
2482}
2483EXPORT_SYMBOL_GPL(bpf_prog_inc);
2484
2485/* prog_idr_lock should have been held */
2486struct bpf_prog *bpf_prog_inc_not_zero(struct bpf_prog *prog)
2487{
2488 int refold;
2489
2490 refold = atomic64_fetch_add_unless(&prog->aux->refcnt, 1, 0);
2491
2492 if (!refold)
2493 return ERR_PTR(-ENOENT);
2494
2495 return prog;
2496}
2497EXPORT_SYMBOL_GPL(bpf_prog_inc_not_zero);
2498
2499bool bpf_prog_get_ok(struct bpf_prog *prog,
2500 enum bpf_prog_type *attach_type, bool attach_drv)
2501{
2502 /* not an attachment, just a refcount inc, always allow */
2503 if (!attach_type)
2504 return true;
2505
2506 if (prog->type != *attach_type)
2507 return false;
2508 if (bpf_prog_is_offloaded(prog->aux) && !attach_drv)
2509 return false;
2510
2511 return true;
2512}
2513
2514static struct bpf_prog *__bpf_prog_get(u32 ufd, enum bpf_prog_type *attach_type,
2515 bool attach_drv)
2516{
2517 CLASS(fd, f)(ufd);
2518 struct bpf_prog *prog;
2519
2520 if (fd_empty(f))
2521 return ERR_PTR(-EBADF);
2522 if (fd_file(f)->f_op != &bpf_prog_fops)
2523 return ERR_PTR(-EINVAL);
2524
2525 prog = fd_file(f)->private_data;
2526 if (!bpf_prog_get_ok(prog, attach_type, attach_drv))
2527 return ERR_PTR(-EINVAL);
2528
2529 bpf_prog_inc(prog);
2530 return prog;
2531}
2532
2533struct bpf_prog *bpf_prog_get(u32 ufd)
2534{
2535 return __bpf_prog_get(ufd, NULL, false);
2536}
2537
2538struct bpf_prog *bpf_prog_get_type_dev(u32 ufd, enum bpf_prog_type type,
2539 bool attach_drv)
2540{
2541 return __bpf_prog_get(ufd, &type, attach_drv);
2542}
2543EXPORT_SYMBOL_GPL(bpf_prog_get_type_dev);
2544
2545/* Initially all BPF programs could be loaded w/o specifying
2546 * expected_attach_type. Later for some of them specifying expected_attach_type
2547 * at load time became required so that program could be validated properly.
2548 * Programs of types that are allowed to be loaded both w/ and w/o (for
2549 * backward compatibility) expected_attach_type, should have the default attach
2550 * type assigned to expected_attach_type for the latter case, so that it can be
2551 * validated later at attach time.
2552 *
2553 * bpf_prog_load_fixup_attach_type() sets expected_attach_type in @attr if
2554 * prog type requires it but has some attach types that have to be backward
2555 * compatible.
2556 */
2557static void bpf_prog_load_fixup_attach_type(union bpf_attr *attr)
2558{
2559 switch (attr->prog_type) {
2560 case BPF_PROG_TYPE_CGROUP_SOCK:
2561 /* Unfortunately BPF_ATTACH_TYPE_UNSPEC enumeration doesn't
2562 * exist so checking for non-zero is the way to go here.
2563 */
2564 if (!attr->expected_attach_type)
2565 attr->expected_attach_type =
2566 BPF_CGROUP_INET_SOCK_CREATE;
2567 break;
2568 case BPF_PROG_TYPE_SK_REUSEPORT:
2569 if (!attr->expected_attach_type)
2570 attr->expected_attach_type =
2571 BPF_SK_REUSEPORT_SELECT;
2572 break;
2573 }
2574}
2575
2576static int
2577bpf_prog_load_check_attach(enum bpf_prog_type prog_type,
2578 enum bpf_attach_type expected_attach_type,
2579 struct btf *attach_btf, u32 btf_id,
2580 struct bpf_prog *dst_prog)
2581{
2582 if (btf_id) {
2583 if (btf_id > BTF_MAX_TYPE)
2584 return -EINVAL;
2585
2586 if (!attach_btf && !dst_prog)
2587 return -EINVAL;
2588
2589 switch (prog_type) {
2590 case BPF_PROG_TYPE_TRACING:
2591 case BPF_PROG_TYPE_LSM:
2592 case BPF_PROG_TYPE_STRUCT_OPS:
2593 case BPF_PROG_TYPE_EXT:
2594 break;
2595 default:
2596 return -EINVAL;
2597 }
2598 }
2599
2600 if (attach_btf && (!btf_id || dst_prog))
2601 return -EINVAL;
2602
2603 if (dst_prog && prog_type != BPF_PROG_TYPE_TRACING &&
2604 prog_type != BPF_PROG_TYPE_EXT)
2605 return -EINVAL;
2606
2607 switch (prog_type) {
2608 case BPF_PROG_TYPE_CGROUP_SOCK:
2609 switch (expected_attach_type) {
2610 case BPF_CGROUP_INET_SOCK_CREATE:
2611 case BPF_CGROUP_INET_SOCK_RELEASE:
2612 case BPF_CGROUP_INET4_POST_BIND:
2613 case BPF_CGROUP_INET6_POST_BIND:
2614 return 0;
2615 default:
2616 return -EINVAL;
2617 }
2618 case BPF_PROG_TYPE_CGROUP_SOCK_ADDR:
2619 switch (expected_attach_type) {
2620 case BPF_CGROUP_INET4_BIND:
2621 case BPF_CGROUP_INET6_BIND:
2622 case BPF_CGROUP_INET4_CONNECT:
2623 case BPF_CGROUP_INET6_CONNECT:
2624 case BPF_CGROUP_UNIX_CONNECT:
2625 case BPF_CGROUP_INET4_GETPEERNAME:
2626 case BPF_CGROUP_INET6_GETPEERNAME:
2627 case BPF_CGROUP_UNIX_GETPEERNAME:
2628 case BPF_CGROUP_INET4_GETSOCKNAME:
2629 case BPF_CGROUP_INET6_GETSOCKNAME:
2630 case BPF_CGROUP_UNIX_GETSOCKNAME:
2631 case BPF_CGROUP_UDP4_SENDMSG:
2632 case BPF_CGROUP_UDP6_SENDMSG:
2633 case BPF_CGROUP_UNIX_SENDMSG:
2634 case BPF_CGROUP_UDP4_RECVMSG:
2635 case BPF_CGROUP_UDP6_RECVMSG:
2636 case BPF_CGROUP_UNIX_RECVMSG:
2637 return 0;
2638 default:
2639 return -EINVAL;
2640 }
2641 case BPF_PROG_TYPE_CGROUP_SKB:
2642 switch (expected_attach_type) {
2643 case BPF_CGROUP_INET_INGRESS:
2644 case BPF_CGROUP_INET_EGRESS:
2645 return 0;
2646 default:
2647 return -EINVAL;
2648 }
2649 case BPF_PROG_TYPE_CGROUP_SOCKOPT:
2650 switch (expected_attach_type) {
2651 case BPF_CGROUP_SETSOCKOPT:
2652 case BPF_CGROUP_GETSOCKOPT:
2653 return 0;
2654 default:
2655 return -EINVAL;
2656 }
2657 case BPF_PROG_TYPE_SK_LOOKUP:
2658 if (expected_attach_type == BPF_SK_LOOKUP)
2659 return 0;
2660 return -EINVAL;
2661 case BPF_PROG_TYPE_SK_REUSEPORT:
2662 switch (expected_attach_type) {
2663 case BPF_SK_REUSEPORT_SELECT:
2664 case BPF_SK_REUSEPORT_SELECT_OR_MIGRATE:
2665 return 0;
2666 default:
2667 return -EINVAL;
2668 }
2669 case BPF_PROG_TYPE_NETFILTER:
2670 if (expected_attach_type == BPF_NETFILTER)
2671 return 0;
2672 return -EINVAL;
2673 case BPF_PROG_TYPE_SYSCALL:
2674 case BPF_PROG_TYPE_EXT:
2675 if (expected_attach_type)
2676 return -EINVAL;
2677 fallthrough;
2678 default:
2679 return 0;
2680 }
2681}
2682
2683static bool is_net_admin_prog_type(enum bpf_prog_type prog_type)
2684{
2685 switch (prog_type) {
2686 case BPF_PROG_TYPE_SCHED_CLS:
2687 case BPF_PROG_TYPE_SCHED_ACT:
2688 case BPF_PROG_TYPE_XDP:
2689 case BPF_PROG_TYPE_LWT_IN:
2690 case BPF_PROG_TYPE_LWT_OUT:
2691 case BPF_PROG_TYPE_LWT_XMIT:
2692 case BPF_PROG_TYPE_LWT_SEG6LOCAL:
2693 case BPF_PROG_TYPE_SK_SKB:
2694 case BPF_PROG_TYPE_SK_MSG:
2695 case BPF_PROG_TYPE_FLOW_DISSECTOR:
2696 case BPF_PROG_TYPE_CGROUP_DEVICE:
2697 case BPF_PROG_TYPE_CGROUP_SOCK:
2698 case BPF_PROG_TYPE_CGROUP_SOCK_ADDR:
2699 case BPF_PROG_TYPE_CGROUP_SOCKOPT:
2700 case BPF_PROG_TYPE_CGROUP_SYSCTL:
2701 case BPF_PROG_TYPE_SOCK_OPS:
2702 case BPF_PROG_TYPE_EXT: /* extends any prog */
2703 case BPF_PROG_TYPE_NETFILTER:
2704 return true;
2705 case BPF_PROG_TYPE_CGROUP_SKB:
2706 /* always unpriv */
2707 case BPF_PROG_TYPE_SK_REUSEPORT:
2708 /* equivalent to SOCKET_FILTER. need CAP_BPF only */
2709 default:
2710 return false;
2711 }
2712}
2713
2714static bool is_perfmon_prog_type(enum bpf_prog_type prog_type)
2715{
2716 switch (prog_type) {
2717 case BPF_PROG_TYPE_KPROBE:
2718 case BPF_PROG_TYPE_TRACEPOINT:
2719 case BPF_PROG_TYPE_PERF_EVENT:
2720 case BPF_PROG_TYPE_RAW_TRACEPOINT:
2721 case BPF_PROG_TYPE_RAW_TRACEPOINT_WRITABLE:
2722 case BPF_PROG_TYPE_TRACING:
2723 case BPF_PROG_TYPE_LSM:
2724 case BPF_PROG_TYPE_STRUCT_OPS: /* has access to struct sock */
2725 case BPF_PROG_TYPE_EXT: /* extends any prog */
2726 return true;
2727 default:
2728 return false;
2729 }
2730}
2731
2732/* last field in 'union bpf_attr' used by this command */
2733#define BPF_PROG_LOAD_LAST_FIELD prog_token_fd
2734
2735static int bpf_prog_load(union bpf_attr *attr, bpfptr_t uattr, u32 uattr_size)
2736{
2737 enum bpf_prog_type type = attr->prog_type;
2738 struct bpf_prog *prog, *dst_prog = NULL;
2739 struct btf *attach_btf = NULL;
2740 struct bpf_token *token = NULL;
2741 bool bpf_cap;
2742 int err;
2743 char license[128];
2744
2745 if (CHECK_ATTR(BPF_PROG_LOAD))
2746 return -EINVAL;
2747
2748 if (attr->prog_flags & ~(BPF_F_STRICT_ALIGNMENT |
2749 BPF_F_ANY_ALIGNMENT |
2750 BPF_F_TEST_STATE_FREQ |
2751 BPF_F_SLEEPABLE |
2752 BPF_F_TEST_RND_HI32 |
2753 BPF_F_XDP_HAS_FRAGS |
2754 BPF_F_XDP_DEV_BOUND_ONLY |
2755 BPF_F_TEST_REG_INVARIANTS |
2756 BPF_F_TOKEN_FD))
2757 return -EINVAL;
2758
2759 bpf_prog_load_fixup_attach_type(attr);
2760
2761 if (attr->prog_flags & BPF_F_TOKEN_FD) {
2762 token = bpf_token_get_from_fd(attr->prog_token_fd);
2763 if (IS_ERR(token))
2764 return PTR_ERR(token);
2765 /* if current token doesn't grant prog loading permissions,
2766 * then we can't use this token, so ignore it and rely on
2767 * system-wide capabilities checks
2768 */
2769 if (!bpf_token_allow_cmd(token, BPF_PROG_LOAD) ||
2770 !bpf_token_allow_prog_type(token, attr->prog_type,
2771 attr->expected_attach_type)) {
2772 bpf_token_put(token);
2773 token = NULL;
2774 }
2775 }
2776
2777 bpf_cap = bpf_token_capable(token, CAP_BPF);
2778 err = -EPERM;
2779
2780 if (!IS_ENABLED(CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS) &&
2781 (attr->prog_flags & BPF_F_ANY_ALIGNMENT) &&
2782 !bpf_cap)
2783 goto put_token;
2784
2785 /* Intent here is for unprivileged_bpf_disabled to block BPF program
2786 * creation for unprivileged users; other actions depend
2787 * on fd availability and access to bpffs, so are dependent on
2788 * object creation success. Even with unprivileged BPF disabled,
2789 * capability checks are still carried out for these
2790 * and other operations.
2791 */
2792 if (sysctl_unprivileged_bpf_disabled && !bpf_cap)
2793 goto put_token;
2794
2795 if (attr->insn_cnt == 0 ||
2796 attr->insn_cnt > (bpf_cap ? BPF_COMPLEXITY_LIMIT_INSNS : BPF_MAXINSNS)) {
2797 err = -E2BIG;
2798 goto put_token;
2799 }
2800 if (type != BPF_PROG_TYPE_SOCKET_FILTER &&
2801 type != BPF_PROG_TYPE_CGROUP_SKB &&
2802 !bpf_cap)
2803 goto put_token;
2804
2805 if (is_net_admin_prog_type(type) && !bpf_token_capable(token, CAP_NET_ADMIN))
2806 goto put_token;
2807 if (is_perfmon_prog_type(type) && !bpf_token_capable(token, CAP_PERFMON))
2808 goto put_token;
2809
2810 /* attach_prog_fd/attach_btf_obj_fd can specify fd of either bpf_prog
2811 * or btf, we need to check which one it is
2812 */
2813 if (attr->attach_prog_fd) {
2814 dst_prog = bpf_prog_get(attr->attach_prog_fd);
2815 if (IS_ERR(dst_prog)) {
2816 dst_prog = NULL;
2817 attach_btf = btf_get_by_fd(attr->attach_btf_obj_fd);
2818 if (IS_ERR(attach_btf)) {
2819 err = -EINVAL;
2820 goto put_token;
2821 }
2822 if (!btf_is_kernel(attach_btf)) {
2823 /* attaching through specifying bpf_prog's BTF
2824 * objects directly might be supported eventually
2825 */
2826 btf_put(attach_btf);
2827 err = -ENOTSUPP;
2828 goto put_token;
2829 }
2830 }
2831 } else if (attr->attach_btf_id) {
2832 /* fall back to vmlinux BTF, if BTF type ID is specified */
2833 attach_btf = bpf_get_btf_vmlinux();
2834 if (IS_ERR(attach_btf)) {
2835 err = PTR_ERR(attach_btf);
2836 goto put_token;
2837 }
2838 if (!attach_btf) {
2839 err = -EINVAL;
2840 goto put_token;
2841 }
2842 btf_get(attach_btf);
2843 }
2844
2845 if (bpf_prog_load_check_attach(type, attr->expected_attach_type,
2846 attach_btf, attr->attach_btf_id,
2847 dst_prog)) {
2848 if (dst_prog)
2849 bpf_prog_put(dst_prog);
2850 if (attach_btf)
2851 btf_put(attach_btf);
2852 err = -EINVAL;
2853 goto put_token;
2854 }
2855
2856 /* plain bpf_prog allocation */
2857 prog = bpf_prog_alloc(bpf_prog_size(attr->insn_cnt), GFP_USER);
2858 if (!prog) {
2859 if (dst_prog)
2860 bpf_prog_put(dst_prog);
2861 if (attach_btf)
2862 btf_put(attach_btf);
2863 err = -EINVAL;
2864 goto put_token;
2865 }
2866
2867 prog->expected_attach_type = attr->expected_attach_type;
2868 prog->sleepable = !!(attr->prog_flags & BPF_F_SLEEPABLE);
2869 prog->aux->attach_btf = attach_btf;
2870 prog->aux->attach_btf_id = attr->attach_btf_id;
2871 prog->aux->dst_prog = dst_prog;
2872 prog->aux->dev_bound = !!attr->prog_ifindex;
2873 prog->aux->xdp_has_frags = attr->prog_flags & BPF_F_XDP_HAS_FRAGS;
2874
2875 /* move token into prog->aux, reuse taken refcnt */
2876 prog->aux->token = token;
2877 token = NULL;
2878
2879 prog->aux->user = get_current_user();
2880 prog->len = attr->insn_cnt;
2881
2882 err = -EFAULT;
2883 if (copy_from_bpfptr(prog->insns,
2884 make_bpfptr(attr->insns, uattr.is_kernel),
2885 bpf_prog_insn_size(prog)) != 0)
2886 goto free_prog;
2887 /* copy eBPF program license from user space */
2888 if (strncpy_from_bpfptr(license,
2889 make_bpfptr(attr->license, uattr.is_kernel),
2890 sizeof(license) - 1) < 0)
2891 goto free_prog;
2892 license[sizeof(license) - 1] = 0;
2893
2894 /* eBPF programs must be GPL compatible to use GPL-ed functions */
2895 prog->gpl_compatible = license_is_gpl_compatible(license) ? 1 : 0;
2896
2897 prog->orig_prog = NULL;
2898 prog->jited = 0;
2899
2900 atomic64_set(&prog->aux->refcnt, 1);
2901
2902 if (bpf_prog_is_dev_bound(prog->aux)) {
2903 err = bpf_prog_dev_bound_init(prog, attr);
2904 if (err)
2905 goto free_prog;
2906 }
2907
2908 if (type == BPF_PROG_TYPE_EXT && dst_prog &&
2909 bpf_prog_is_dev_bound(dst_prog->aux)) {
2910 err = bpf_prog_dev_bound_inherit(prog, dst_prog);
2911 if (err)
2912 goto free_prog;
2913 }
2914
2915 /*
2916 * Bookkeeping for managing the program attachment chain.
2917 *
2918 * It might be tempting to set attach_tracing_prog flag at the attachment
2919 * time, but this will not prevent from loading bunch of tracing prog
2920 * first, then attach them one to another.
2921 *
2922 * The flag attach_tracing_prog is set for the whole program lifecycle, and
2923 * doesn't have to be cleared in bpf_tracing_link_release, since tracing
2924 * programs cannot change attachment target.
2925 */
2926 if (type == BPF_PROG_TYPE_TRACING && dst_prog &&
2927 dst_prog->type == BPF_PROG_TYPE_TRACING) {
2928 prog->aux->attach_tracing_prog = true;
2929 }
2930
2931 /* find program type: socket_filter vs tracing_filter */
2932 err = find_prog_type(type, prog);
2933 if (err < 0)
2934 goto free_prog;
2935
2936 prog->aux->load_time = ktime_get_boottime_ns();
2937 err = bpf_obj_name_cpy(prog->aux->name, attr->prog_name,
2938 sizeof(attr->prog_name));
2939 if (err < 0)
2940 goto free_prog;
2941
2942 err = security_bpf_prog_load(prog, attr, token);
2943 if (err)
2944 goto free_prog_sec;
2945
2946 /* run eBPF verifier */
2947 err = bpf_check(&prog, attr, uattr, uattr_size);
2948 if (err < 0)
2949 goto free_used_maps;
2950
2951 prog = bpf_prog_select_runtime(prog, &err);
2952 if (err < 0)
2953 goto free_used_maps;
2954
2955 err = bpf_prog_alloc_id(prog);
2956 if (err)
2957 goto free_used_maps;
2958
2959 /* Upon success of bpf_prog_alloc_id(), the BPF prog is
2960 * effectively publicly exposed. However, retrieving via
2961 * bpf_prog_get_fd_by_id() will take another reference,
2962 * therefore it cannot be gone underneath us.
2963 *
2964 * Only for the time /after/ successful bpf_prog_new_fd()
2965 * and before returning to userspace, we might just hold
2966 * one reference and any parallel close on that fd could
2967 * rip everything out. Hence, below notifications must
2968 * happen before bpf_prog_new_fd().
2969 *
2970 * Also, any failure handling from this point onwards must
2971 * be using bpf_prog_put() given the program is exposed.
2972 */
2973 bpf_prog_kallsyms_add(prog);
2974 perf_event_bpf_event(prog, PERF_BPF_EVENT_PROG_LOAD, 0);
2975 bpf_audit_prog(prog, BPF_AUDIT_LOAD);
2976
2977 err = bpf_prog_new_fd(prog);
2978 if (err < 0)
2979 bpf_prog_put(prog);
2980 return err;
2981
2982free_used_maps:
2983 /* In case we have subprogs, we need to wait for a grace
2984 * period before we can tear down JIT memory since symbols
2985 * are already exposed under kallsyms.
2986 */
2987 __bpf_prog_put_noref(prog, prog->aux->real_func_cnt);
2988 return err;
2989
2990free_prog_sec:
2991 security_bpf_prog_free(prog);
2992free_prog:
2993 free_uid(prog->aux->user);
2994 if (prog->aux->attach_btf)
2995 btf_put(prog->aux->attach_btf);
2996 bpf_prog_free(prog);
2997put_token:
2998 bpf_token_put(token);
2999 return err;
3000}
3001
3002#define BPF_OBJ_LAST_FIELD path_fd
3003
3004static int bpf_obj_pin(const union bpf_attr *attr)
3005{
3006 int path_fd;
3007
3008 if (CHECK_ATTR(BPF_OBJ) || attr->file_flags & ~BPF_F_PATH_FD)
3009 return -EINVAL;
3010
3011 /* path_fd has to be accompanied by BPF_F_PATH_FD flag */
3012 if (!(attr->file_flags & BPF_F_PATH_FD) && attr->path_fd)
3013 return -EINVAL;
3014
3015 path_fd = attr->file_flags & BPF_F_PATH_FD ? attr->path_fd : AT_FDCWD;
3016 return bpf_obj_pin_user(attr->bpf_fd, path_fd,
3017 u64_to_user_ptr(attr->pathname));
3018}
3019
3020static int bpf_obj_get(const union bpf_attr *attr)
3021{
3022 int path_fd;
3023
3024 if (CHECK_ATTR(BPF_OBJ) || attr->bpf_fd != 0 ||
3025 attr->file_flags & ~(BPF_OBJ_FLAG_MASK | BPF_F_PATH_FD))
3026 return -EINVAL;
3027
3028 /* path_fd has to be accompanied by BPF_F_PATH_FD flag */
3029 if (!(attr->file_flags & BPF_F_PATH_FD) && attr->path_fd)
3030 return -EINVAL;
3031
3032 path_fd = attr->file_flags & BPF_F_PATH_FD ? attr->path_fd : AT_FDCWD;
3033 return bpf_obj_get_user(path_fd, u64_to_user_ptr(attr->pathname),
3034 attr->file_flags);
3035}
3036
3037/* bpf_link_init_sleepable() allows to specify whether BPF link itself has
3038 * "sleepable" semantics, which normally would mean that BPF link's attach
3039 * hook can dereference link or link's underlying program for some time after
3040 * detachment due to RCU Tasks Trace-based lifetime protection scheme.
3041 * BPF program itself can be non-sleepable, yet, because it's transitively
3042 * reachable through BPF link, its freeing has to be delayed until after RCU
3043 * Tasks Trace GP.
3044 */
3045void bpf_link_init_sleepable(struct bpf_link *link, enum bpf_link_type type,
3046 const struct bpf_link_ops *ops, struct bpf_prog *prog,
3047 bool sleepable)
3048{
3049 WARN_ON(ops->dealloc && ops->dealloc_deferred);
3050 atomic64_set(&link->refcnt, 1);
3051 link->type = type;
3052 link->sleepable = sleepable;
3053 link->id = 0;
3054 link->ops = ops;
3055 link->prog = prog;
3056}
3057
3058void bpf_link_init(struct bpf_link *link, enum bpf_link_type type,
3059 const struct bpf_link_ops *ops, struct bpf_prog *prog)
3060{
3061 bpf_link_init_sleepable(link, type, ops, prog, false);
3062}
3063
3064static void bpf_link_free_id(int id)
3065{
3066 if (!id)
3067 return;
3068
3069 spin_lock_bh(&link_idr_lock);
3070 idr_remove(&link_idr, id);
3071 spin_unlock_bh(&link_idr_lock);
3072}
3073
3074/* Clean up bpf_link and corresponding anon_inode file and FD. After
3075 * anon_inode is created, bpf_link can't be just kfree()'d due to deferred
3076 * anon_inode's release() call. This helper marks bpf_link as
3077 * defunct, releases anon_inode file and puts reserved FD. bpf_prog's refcnt
3078 * is not decremented, it's the responsibility of a calling code that failed
3079 * to complete bpf_link initialization.
3080 * This helper eventually calls link's dealloc callback, but does not call
3081 * link's release callback.
3082 */
3083void bpf_link_cleanup(struct bpf_link_primer *primer)
3084{
3085 primer->link->prog = NULL;
3086 bpf_link_free_id(primer->id);
3087 fput(primer->file);
3088 put_unused_fd(primer->fd);
3089}
3090
3091void bpf_link_inc(struct bpf_link *link)
3092{
3093 atomic64_inc(&link->refcnt);
3094}
3095
3096static void bpf_link_dealloc(struct bpf_link *link)
3097{
3098 /* now that we know that bpf_link itself can't be reached, put underlying BPF program */
3099 if (link->prog)
3100 bpf_prog_put(link->prog);
3101
3102 /* free bpf_link and its containing memory */
3103 if (link->ops->dealloc_deferred)
3104 link->ops->dealloc_deferred(link);
3105 else
3106 link->ops->dealloc(link);
3107}
3108
3109static void bpf_link_defer_dealloc_rcu_gp(struct rcu_head *rcu)
3110{
3111 struct bpf_link *link = container_of(rcu, struct bpf_link, rcu);
3112
3113 bpf_link_dealloc(link);
3114}
3115
3116static void bpf_link_defer_dealloc_mult_rcu_gp(struct rcu_head *rcu)
3117{
3118 if (rcu_trace_implies_rcu_gp())
3119 bpf_link_defer_dealloc_rcu_gp(rcu);
3120 else
3121 call_rcu(rcu, bpf_link_defer_dealloc_rcu_gp);
3122}
3123
3124/* bpf_link_free is guaranteed to be called from process context */
3125static void bpf_link_free(struct bpf_link *link)
3126{
3127 const struct bpf_link_ops *ops = link->ops;
3128
3129 bpf_link_free_id(link->id);
3130 /* detach BPF program, clean up used resources */
3131 if (link->prog)
3132 ops->release(link);
3133 if (ops->dealloc_deferred) {
3134 /* Schedule BPF link deallocation, which will only then
3135 * trigger putting BPF program refcount.
3136 * If underlying BPF program is sleepable or BPF link's target
3137 * attach hookpoint is sleepable or otherwise requires RCU GPs
3138 * to ensure link and its underlying BPF program is not
3139 * reachable anymore, we need to first wait for RCU tasks
3140 * trace sync, and then go through "classic" RCU grace period
3141 */
3142 if (link->sleepable || (link->prog && link->prog->sleepable))
3143 call_rcu_tasks_trace(&link->rcu, bpf_link_defer_dealloc_mult_rcu_gp);
3144 else
3145 call_rcu(&link->rcu, bpf_link_defer_dealloc_rcu_gp);
3146 } else if (ops->dealloc) {
3147 bpf_link_dealloc(link);
3148 }
3149}
3150
3151static void bpf_link_put_deferred(struct work_struct *work)
3152{
3153 struct bpf_link *link = container_of(work, struct bpf_link, work);
3154
3155 bpf_link_free(link);
3156}
3157
3158/* bpf_link_put might be called from atomic context. It needs to be called
3159 * from sleepable context in order to acquire sleeping locks during the process.
3160 */
3161void bpf_link_put(struct bpf_link *link)
3162{
3163 if (!atomic64_dec_and_test(&link->refcnt))
3164 return;
3165
3166 INIT_WORK(&link->work, bpf_link_put_deferred);
3167 schedule_work(&link->work);
3168}
3169EXPORT_SYMBOL(bpf_link_put);
3170
3171static void bpf_link_put_direct(struct bpf_link *link)
3172{
3173 if (!atomic64_dec_and_test(&link->refcnt))
3174 return;
3175 bpf_link_free(link);
3176}
3177
3178static int bpf_link_release(struct inode *inode, struct file *filp)
3179{
3180 struct bpf_link *link = filp->private_data;
3181
3182 bpf_link_put_direct(link);
3183 return 0;
3184}
3185
3186#ifdef CONFIG_PROC_FS
3187#define BPF_PROG_TYPE(_id, _name, prog_ctx_type, kern_ctx_type)
3188#define BPF_MAP_TYPE(_id, _ops)
3189#define BPF_LINK_TYPE(_id, _name) [_id] = #_name,
3190static const char *bpf_link_type_strs[] = {
3191 [BPF_LINK_TYPE_UNSPEC] = "<invalid>",
3192#include <linux/bpf_types.h>
3193};
3194#undef BPF_PROG_TYPE
3195#undef BPF_MAP_TYPE
3196#undef BPF_LINK_TYPE
3197
3198static void bpf_link_show_fdinfo(struct seq_file *m, struct file *filp)
3199{
3200 const struct bpf_link *link = filp->private_data;
3201 const struct bpf_prog *prog = link->prog;
3202 enum bpf_link_type type = link->type;
3203 char prog_tag[sizeof(prog->tag) * 2 + 1] = { };
3204
3205 if (type < ARRAY_SIZE(bpf_link_type_strs) && bpf_link_type_strs[type]) {
3206 seq_printf(m, "link_type:\t%s\n", bpf_link_type_strs[type]);
3207 } else {
3208 WARN_ONCE(1, "missing BPF_LINK_TYPE(...) for link type %u\n", type);
3209 seq_printf(m, "link_type:\t<%u>\n", type);
3210 }
3211 seq_printf(m, "link_id:\t%u\n", link->id);
3212
3213 if (prog) {
3214 bin2hex(prog_tag, prog->tag, sizeof(prog->tag));
3215 seq_printf(m,
3216 "prog_tag:\t%s\n"
3217 "prog_id:\t%u\n",
3218 prog_tag,
3219 prog->aux->id);
3220 }
3221 if (link->ops->show_fdinfo)
3222 link->ops->show_fdinfo(link, m);
3223}
3224#endif
3225
3226static __poll_t bpf_link_poll(struct file *file, struct poll_table_struct *pts)
3227{
3228 struct bpf_link *link = file->private_data;
3229
3230 return link->ops->poll(file, pts);
3231}
3232
3233static const struct file_operations bpf_link_fops = {
3234#ifdef CONFIG_PROC_FS
3235 .show_fdinfo = bpf_link_show_fdinfo,
3236#endif
3237 .release = bpf_link_release,
3238 .read = bpf_dummy_read,
3239 .write = bpf_dummy_write,
3240};
3241
3242static const struct file_operations bpf_link_fops_poll = {
3243#ifdef CONFIG_PROC_FS
3244 .show_fdinfo = bpf_link_show_fdinfo,
3245#endif
3246 .release = bpf_link_release,
3247 .read = bpf_dummy_read,
3248 .write = bpf_dummy_write,
3249 .poll = bpf_link_poll,
3250};
3251
3252static int bpf_link_alloc_id(struct bpf_link *link)
3253{
3254 int id;
3255
3256 idr_preload(GFP_KERNEL);
3257 spin_lock_bh(&link_idr_lock);
3258 id = idr_alloc_cyclic(&link_idr, link, 1, INT_MAX, GFP_ATOMIC);
3259 spin_unlock_bh(&link_idr_lock);
3260 idr_preload_end();
3261
3262 return id;
3263}
3264
3265/* Prepare bpf_link to be exposed to user-space by allocating anon_inode file,
3266 * reserving unused FD and allocating ID from link_idr. This is to be paired
3267 * with bpf_link_settle() to install FD and ID and expose bpf_link to
3268 * user-space, if bpf_link is successfully attached. If not, bpf_link and
3269 * pre-allocated resources are to be freed with bpf_cleanup() call. All the
3270 * transient state is passed around in struct bpf_link_primer.
3271 * This is preferred way to create and initialize bpf_link, especially when
3272 * there are complicated and expensive operations in between creating bpf_link
3273 * itself and attaching it to BPF hook. By using bpf_link_prime() and
3274 * bpf_link_settle() kernel code using bpf_link doesn't have to perform
3275 * expensive (and potentially failing) roll back operations in a rare case
3276 * that file, FD, or ID can't be allocated.
3277 */
3278int bpf_link_prime(struct bpf_link *link, struct bpf_link_primer *primer)
3279{
3280 struct file *file;
3281 int fd, id;
3282
3283 fd = get_unused_fd_flags(O_CLOEXEC);
3284 if (fd < 0)
3285 return fd;
3286
3287
3288 id = bpf_link_alloc_id(link);
3289 if (id < 0) {
3290 put_unused_fd(fd);
3291 return id;
3292 }
3293
3294 file = anon_inode_getfile("bpf_link",
3295 link->ops->poll ? &bpf_link_fops_poll : &bpf_link_fops,
3296 link, O_CLOEXEC);
3297 if (IS_ERR(file)) {
3298 bpf_link_free_id(id);
3299 put_unused_fd(fd);
3300 return PTR_ERR(file);
3301 }
3302
3303 primer->link = link;
3304 primer->file = file;
3305 primer->fd = fd;
3306 primer->id = id;
3307 return 0;
3308}
3309
3310int bpf_link_settle(struct bpf_link_primer *primer)
3311{
3312 /* make bpf_link fetchable by ID */
3313 spin_lock_bh(&link_idr_lock);
3314 primer->link->id = primer->id;
3315 spin_unlock_bh(&link_idr_lock);
3316 /* make bpf_link fetchable by FD */
3317 fd_install(primer->fd, primer->file);
3318 /* pass through installed FD */
3319 return primer->fd;
3320}
3321
3322int bpf_link_new_fd(struct bpf_link *link)
3323{
3324 return anon_inode_getfd("bpf-link",
3325 link->ops->poll ? &bpf_link_fops_poll : &bpf_link_fops,
3326 link, O_CLOEXEC);
3327}
3328
3329struct bpf_link *bpf_link_get_from_fd(u32 ufd)
3330{
3331 CLASS(fd, f)(ufd);
3332 struct bpf_link *link;
3333
3334 if (fd_empty(f))
3335 return ERR_PTR(-EBADF);
3336 if (fd_file(f)->f_op != &bpf_link_fops && fd_file(f)->f_op != &bpf_link_fops_poll)
3337 return ERR_PTR(-EINVAL);
3338
3339 link = fd_file(f)->private_data;
3340 bpf_link_inc(link);
3341 return link;
3342}
3343EXPORT_SYMBOL(bpf_link_get_from_fd);
3344
3345static void bpf_tracing_link_release(struct bpf_link *link)
3346{
3347 struct bpf_tracing_link *tr_link =
3348 container_of(link, struct bpf_tracing_link, link.link);
3349
3350 WARN_ON_ONCE(bpf_trampoline_unlink_prog(&tr_link->link,
3351 tr_link->trampoline,
3352 tr_link->tgt_prog));
3353
3354 bpf_trampoline_put(tr_link->trampoline);
3355
3356 /* tgt_prog is NULL if target is a kernel function */
3357 if (tr_link->tgt_prog)
3358 bpf_prog_put(tr_link->tgt_prog);
3359}
3360
3361static void bpf_tracing_link_dealloc(struct bpf_link *link)
3362{
3363 struct bpf_tracing_link *tr_link =
3364 container_of(link, struct bpf_tracing_link, link.link);
3365
3366 kfree(tr_link);
3367}
3368
3369static void bpf_tracing_link_show_fdinfo(const struct bpf_link *link,
3370 struct seq_file *seq)
3371{
3372 struct bpf_tracing_link *tr_link =
3373 container_of(link, struct bpf_tracing_link, link.link);
3374 u32 target_btf_id, target_obj_id;
3375
3376 bpf_trampoline_unpack_key(tr_link->trampoline->key,
3377 &target_obj_id, &target_btf_id);
3378 seq_printf(seq,
3379 "attach_type:\t%d\n"
3380 "target_obj_id:\t%u\n"
3381 "target_btf_id:\t%u\n",
3382 tr_link->attach_type,
3383 target_obj_id,
3384 target_btf_id);
3385}
3386
3387static int bpf_tracing_link_fill_link_info(const struct bpf_link *link,
3388 struct bpf_link_info *info)
3389{
3390 struct bpf_tracing_link *tr_link =
3391 container_of(link, struct bpf_tracing_link, link.link);
3392
3393 info->tracing.attach_type = tr_link->attach_type;
3394 bpf_trampoline_unpack_key(tr_link->trampoline->key,
3395 &info->tracing.target_obj_id,
3396 &info->tracing.target_btf_id);
3397
3398 return 0;
3399}
3400
3401static const struct bpf_link_ops bpf_tracing_link_lops = {
3402 .release = bpf_tracing_link_release,
3403 .dealloc = bpf_tracing_link_dealloc,
3404 .show_fdinfo = bpf_tracing_link_show_fdinfo,
3405 .fill_link_info = bpf_tracing_link_fill_link_info,
3406};
3407
3408static int bpf_tracing_prog_attach(struct bpf_prog *prog,
3409 int tgt_prog_fd,
3410 u32 btf_id,
3411 u64 bpf_cookie)
3412{
3413 struct bpf_link_primer link_primer;
3414 struct bpf_prog *tgt_prog = NULL;
3415 struct bpf_trampoline *tr = NULL;
3416 struct bpf_tracing_link *link;
3417 u64 key = 0;
3418 int err;
3419
3420 switch (prog->type) {
3421 case BPF_PROG_TYPE_TRACING:
3422 if (prog->expected_attach_type != BPF_TRACE_FENTRY &&
3423 prog->expected_attach_type != BPF_TRACE_FEXIT &&
3424 prog->expected_attach_type != BPF_MODIFY_RETURN) {
3425 err = -EINVAL;
3426 goto out_put_prog;
3427 }
3428 break;
3429 case BPF_PROG_TYPE_EXT:
3430 if (prog->expected_attach_type != 0) {
3431 err = -EINVAL;
3432 goto out_put_prog;
3433 }
3434 break;
3435 case BPF_PROG_TYPE_LSM:
3436 if (prog->expected_attach_type != BPF_LSM_MAC) {
3437 err = -EINVAL;
3438 goto out_put_prog;
3439 }
3440 break;
3441 default:
3442 err = -EINVAL;
3443 goto out_put_prog;
3444 }
3445
3446 if (!!tgt_prog_fd != !!btf_id) {
3447 err = -EINVAL;
3448 goto out_put_prog;
3449 }
3450
3451 if (tgt_prog_fd) {
3452 /*
3453 * For now we only allow new targets for BPF_PROG_TYPE_EXT. If this
3454 * part would be changed to implement the same for
3455 * BPF_PROG_TYPE_TRACING, do not forget to update the way how
3456 * attach_tracing_prog flag is set.
3457 */
3458 if (prog->type != BPF_PROG_TYPE_EXT) {
3459 err = -EINVAL;
3460 goto out_put_prog;
3461 }
3462
3463 tgt_prog = bpf_prog_get(tgt_prog_fd);
3464 if (IS_ERR(tgt_prog)) {
3465 err = PTR_ERR(tgt_prog);
3466 tgt_prog = NULL;
3467 goto out_put_prog;
3468 }
3469
3470 key = bpf_trampoline_compute_key(tgt_prog, NULL, btf_id);
3471 }
3472
3473 link = kzalloc(sizeof(*link), GFP_USER);
3474 if (!link) {
3475 err = -ENOMEM;
3476 goto out_put_prog;
3477 }
3478 bpf_link_init(&link->link.link, BPF_LINK_TYPE_TRACING,
3479 &bpf_tracing_link_lops, prog);
3480 link->attach_type = prog->expected_attach_type;
3481 link->link.cookie = bpf_cookie;
3482
3483 mutex_lock(&prog->aux->dst_mutex);
3484
3485 /* There are a few possible cases here:
3486 *
3487 * - if prog->aux->dst_trampoline is set, the program was just loaded
3488 * and not yet attached to anything, so we can use the values stored
3489 * in prog->aux
3490 *
3491 * - if prog->aux->dst_trampoline is NULL, the program has already been
3492 * attached to a target and its initial target was cleared (below)
3493 *
3494 * - if tgt_prog != NULL, the caller specified tgt_prog_fd +
3495 * target_btf_id using the link_create API.
3496 *
3497 * - if tgt_prog == NULL when this function was called using the old
3498 * raw_tracepoint_open API, and we need a target from prog->aux
3499 *
3500 * - if prog->aux->dst_trampoline and tgt_prog is NULL, the program
3501 * was detached and is going for re-attachment.
3502 *
3503 * - if prog->aux->dst_trampoline is NULL and tgt_prog and prog->aux->attach_btf
3504 * are NULL, then program was already attached and user did not provide
3505 * tgt_prog_fd so we have no way to find out or create trampoline
3506 */
3507 if (!prog->aux->dst_trampoline && !tgt_prog) {
3508 /*
3509 * Allow re-attach for TRACING and LSM programs. If it's
3510 * currently linked, bpf_trampoline_link_prog will fail.
3511 * EXT programs need to specify tgt_prog_fd, so they
3512 * re-attach in separate code path.
3513 */
3514 if (prog->type != BPF_PROG_TYPE_TRACING &&
3515 prog->type != BPF_PROG_TYPE_LSM) {
3516 err = -EINVAL;
3517 goto out_unlock;
3518 }
3519 /* We can allow re-attach only if we have valid attach_btf. */
3520 if (!prog->aux->attach_btf) {
3521 err = -EINVAL;
3522 goto out_unlock;
3523 }
3524 btf_id = prog->aux->attach_btf_id;
3525 key = bpf_trampoline_compute_key(NULL, prog->aux->attach_btf, btf_id);
3526 }
3527
3528 if (!prog->aux->dst_trampoline ||
3529 (key && key != prog->aux->dst_trampoline->key)) {
3530 /* If there is no saved target, or the specified target is
3531 * different from the destination specified at load time, we
3532 * need a new trampoline and a check for compatibility
3533 */
3534 struct bpf_attach_target_info tgt_info = {};
3535
3536 err = bpf_check_attach_target(NULL, prog, tgt_prog, btf_id,
3537 &tgt_info);
3538 if (err)
3539 goto out_unlock;
3540
3541 if (tgt_info.tgt_mod) {
3542 module_put(prog->aux->mod);
3543 prog->aux->mod = tgt_info.tgt_mod;
3544 }
3545
3546 tr = bpf_trampoline_get(key, &tgt_info);
3547 if (!tr) {
3548 err = -ENOMEM;
3549 goto out_unlock;
3550 }
3551 } else {
3552 /* The caller didn't specify a target, or the target was the
3553 * same as the destination supplied during program load. This
3554 * means we can reuse the trampoline and reference from program
3555 * load time, and there is no need to allocate a new one. This
3556 * can only happen once for any program, as the saved values in
3557 * prog->aux are cleared below.
3558 */
3559 tr = prog->aux->dst_trampoline;
3560 tgt_prog = prog->aux->dst_prog;
3561 }
3562
3563 err = bpf_link_prime(&link->link.link, &link_primer);
3564 if (err)
3565 goto out_unlock;
3566
3567 err = bpf_trampoline_link_prog(&link->link, tr, tgt_prog);
3568 if (err) {
3569 bpf_link_cleanup(&link_primer);
3570 link = NULL;
3571 goto out_unlock;
3572 }
3573
3574 link->tgt_prog = tgt_prog;
3575 link->trampoline = tr;
3576
3577 /* Always clear the trampoline and target prog from prog->aux to make
3578 * sure the original attach destination is not kept alive after a
3579 * program is (re-)attached to another target.
3580 */
3581 if (prog->aux->dst_prog &&
3582 (tgt_prog_fd || tr != prog->aux->dst_trampoline))
3583 /* got extra prog ref from syscall, or attaching to different prog */
3584 bpf_prog_put(prog->aux->dst_prog);
3585 if (prog->aux->dst_trampoline && tr != prog->aux->dst_trampoline)
3586 /* we allocated a new trampoline, so free the old one */
3587 bpf_trampoline_put(prog->aux->dst_trampoline);
3588
3589 prog->aux->dst_prog = NULL;
3590 prog->aux->dst_trampoline = NULL;
3591 mutex_unlock(&prog->aux->dst_mutex);
3592
3593 return bpf_link_settle(&link_primer);
3594out_unlock:
3595 if (tr && tr != prog->aux->dst_trampoline)
3596 bpf_trampoline_put(tr);
3597 mutex_unlock(&prog->aux->dst_mutex);
3598 kfree(link);
3599out_put_prog:
3600 if (tgt_prog_fd && tgt_prog)
3601 bpf_prog_put(tgt_prog);
3602 return err;
3603}
3604
3605static void bpf_raw_tp_link_release(struct bpf_link *link)
3606{
3607 struct bpf_raw_tp_link *raw_tp =
3608 container_of(link, struct bpf_raw_tp_link, link);
3609
3610 bpf_probe_unregister(raw_tp->btp, raw_tp);
3611 bpf_put_raw_tracepoint(raw_tp->btp);
3612}
3613
3614static void bpf_raw_tp_link_dealloc(struct bpf_link *link)
3615{
3616 struct bpf_raw_tp_link *raw_tp =
3617 container_of(link, struct bpf_raw_tp_link, link);
3618
3619 kfree(raw_tp);
3620}
3621
3622static void bpf_raw_tp_link_show_fdinfo(const struct bpf_link *link,
3623 struct seq_file *seq)
3624{
3625 struct bpf_raw_tp_link *raw_tp_link =
3626 container_of(link, struct bpf_raw_tp_link, link);
3627
3628 seq_printf(seq,
3629 "tp_name:\t%s\n",
3630 raw_tp_link->btp->tp->name);
3631}
3632
3633static int bpf_copy_to_user(char __user *ubuf, const char *buf, u32 ulen,
3634 u32 len)
3635{
3636 if (ulen >= len + 1) {
3637 if (copy_to_user(ubuf, buf, len + 1))
3638 return -EFAULT;
3639 } else {
3640 char zero = '\0';
3641
3642 if (copy_to_user(ubuf, buf, ulen - 1))
3643 return -EFAULT;
3644 if (put_user(zero, ubuf + ulen - 1))
3645 return -EFAULT;
3646 return -ENOSPC;
3647 }
3648
3649 return 0;
3650}
3651
3652static int bpf_raw_tp_link_fill_link_info(const struct bpf_link *link,
3653 struct bpf_link_info *info)
3654{
3655 struct bpf_raw_tp_link *raw_tp_link =
3656 container_of(link, struct bpf_raw_tp_link, link);
3657 char __user *ubuf = u64_to_user_ptr(info->raw_tracepoint.tp_name);
3658 const char *tp_name = raw_tp_link->btp->tp->name;
3659 u32 ulen = info->raw_tracepoint.tp_name_len;
3660 size_t tp_len = strlen(tp_name);
3661
3662 if (!ulen ^ !ubuf)
3663 return -EINVAL;
3664
3665 info->raw_tracepoint.tp_name_len = tp_len + 1;
3666
3667 if (!ubuf)
3668 return 0;
3669
3670 return bpf_copy_to_user(ubuf, tp_name, ulen, tp_len);
3671}
3672
3673static const struct bpf_link_ops bpf_raw_tp_link_lops = {
3674 .release = bpf_raw_tp_link_release,
3675 .dealloc_deferred = bpf_raw_tp_link_dealloc,
3676 .show_fdinfo = bpf_raw_tp_link_show_fdinfo,
3677 .fill_link_info = bpf_raw_tp_link_fill_link_info,
3678};
3679
3680#ifdef CONFIG_PERF_EVENTS
3681struct bpf_perf_link {
3682 struct bpf_link link;
3683 struct file *perf_file;
3684};
3685
3686static void bpf_perf_link_release(struct bpf_link *link)
3687{
3688 struct bpf_perf_link *perf_link = container_of(link, struct bpf_perf_link, link);
3689 struct perf_event *event = perf_link->perf_file->private_data;
3690
3691 perf_event_free_bpf_prog(event);
3692 fput(perf_link->perf_file);
3693}
3694
3695static void bpf_perf_link_dealloc(struct bpf_link *link)
3696{
3697 struct bpf_perf_link *perf_link = container_of(link, struct bpf_perf_link, link);
3698
3699 kfree(perf_link);
3700}
3701
3702static int bpf_perf_link_fill_common(const struct perf_event *event,
3703 char __user *uname, u32 *ulenp,
3704 u64 *probe_offset, u64 *probe_addr,
3705 u32 *fd_type, unsigned long *missed)
3706{
3707 const char *buf;
3708 u32 prog_id, ulen;
3709 size_t len;
3710 int err;
3711
3712 ulen = *ulenp;
3713 if (!ulen ^ !uname)
3714 return -EINVAL;
3715
3716 err = bpf_get_perf_event_info(event, &prog_id, fd_type, &buf,
3717 probe_offset, probe_addr, missed);
3718 if (err)
3719 return err;
3720
3721 if (buf) {
3722 len = strlen(buf);
3723 *ulenp = len + 1;
3724 } else {
3725 *ulenp = 1;
3726 }
3727 if (!uname)
3728 return 0;
3729
3730 if (buf) {
3731 err = bpf_copy_to_user(uname, buf, ulen, len);
3732 if (err)
3733 return err;
3734 } else {
3735 char zero = '\0';
3736
3737 if (put_user(zero, uname))
3738 return -EFAULT;
3739 }
3740 return 0;
3741}
3742
3743#ifdef CONFIG_KPROBE_EVENTS
3744static int bpf_perf_link_fill_kprobe(const struct perf_event *event,
3745 struct bpf_link_info *info)
3746{
3747 unsigned long missed;
3748 char __user *uname;
3749 u64 addr, offset;
3750 u32 ulen, type;
3751 int err;
3752
3753 uname = u64_to_user_ptr(info->perf_event.kprobe.func_name);
3754 ulen = info->perf_event.kprobe.name_len;
3755 err = bpf_perf_link_fill_common(event, uname, &ulen, &offset, &addr,
3756 &type, &missed);
3757 if (err)
3758 return err;
3759 if (type == BPF_FD_TYPE_KRETPROBE)
3760 info->perf_event.type = BPF_PERF_EVENT_KRETPROBE;
3761 else
3762 info->perf_event.type = BPF_PERF_EVENT_KPROBE;
3763 info->perf_event.kprobe.name_len = ulen;
3764 info->perf_event.kprobe.offset = offset;
3765 info->perf_event.kprobe.missed = missed;
3766 if (!kallsyms_show_value(current_cred()))
3767 addr = 0;
3768 info->perf_event.kprobe.addr = addr;
3769 info->perf_event.kprobe.cookie = event->bpf_cookie;
3770 return 0;
3771}
3772#endif
3773
3774#ifdef CONFIG_UPROBE_EVENTS
3775static int bpf_perf_link_fill_uprobe(const struct perf_event *event,
3776 struct bpf_link_info *info)
3777{
3778 char __user *uname;
3779 u64 addr, offset;
3780 u32 ulen, type;
3781 int err;
3782
3783 uname = u64_to_user_ptr(info->perf_event.uprobe.file_name);
3784 ulen = info->perf_event.uprobe.name_len;
3785 err = bpf_perf_link_fill_common(event, uname, &ulen, &offset, &addr,
3786 &type, NULL);
3787 if (err)
3788 return err;
3789
3790 if (type == BPF_FD_TYPE_URETPROBE)
3791 info->perf_event.type = BPF_PERF_EVENT_URETPROBE;
3792 else
3793 info->perf_event.type = BPF_PERF_EVENT_UPROBE;
3794 info->perf_event.uprobe.name_len = ulen;
3795 info->perf_event.uprobe.offset = offset;
3796 info->perf_event.uprobe.cookie = event->bpf_cookie;
3797 return 0;
3798}
3799#endif
3800
3801static int bpf_perf_link_fill_probe(const struct perf_event *event,
3802 struct bpf_link_info *info)
3803{
3804#ifdef CONFIG_KPROBE_EVENTS
3805 if (event->tp_event->flags & TRACE_EVENT_FL_KPROBE)
3806 return bpf_perf_link_fill_kprobe(event, info);
3807#endif
3808#ifdef CONFIG_UPROBE_EVENTS
3809 if (event->tp_event->flags & TRACE_EVENT_FL_UPROBE)
3810 return bpf_perf_link_fill_uprobe(event, info);
3811#endif
3812 return -EOPNOTSUPP;
3813}
3814
3815static int bpf_perf_link_fill_tracepoint(const struct perf_event *event,
3816 struct bpf_link_info *info)
3817{
3818 char __user *uname;
3819 u32 ulen;
3820 int err;
3821
3822 uname = u64_to_user_ptr(info->perf_event.tracepoint.tp_name);
3823 ulen = info->perf_event.tracepoint.name_len;
3824 err = bpf_perf_link_fill_common(event, uname, &ulen, NULL, NULL, NULL, NULL);
3825 if (err)
3826 return err;
3827
3828 info->perf_event.type = BPF_PERF_EVENT_TRACEPOINT;
3829 info->perf_event.tracepoint.name_len = ulen;
3830 info->perf_event.tracepoint.cookie = event->bpf_cookie;
3831 return 0;
3832}
3833
3834static int bpf_perf_link_fill_perf_event(const struct perf_event *event,
3835 struct bpf_link_info *info)
3836{
3837 info->perf_event.event.type = event->attr.type;
3838 info->perf_event.event.config = event->attr.config;
3839 info->perf_event.event.cookie = event->bpf_cookie;
3840 info->perf_event.type = BPF_PERF_EVENT_EVENT;
3841 return 0;
3842}
3843
3844static int bpf_perf_link_fill_link_info(const struct bpf_link *link,
3845 struct bpf_link_info *info)
3846{
3847 struct bpf_perf_link *perf_link;
3848 const struct perf_event *event;
3849
3850 perf_link = container_of(link, struct bpf_perf_link, link);
3851 event = perf_get_event(perf_link->perf_file);
3852 if (IS_ERR(event))
3853 return PTR_ERR(event);
3854
3855 switch (event->prog->type) {
3856 case BPF_PROG_TYPE_PERF_EVENT:
3857 return bpf_perf_link_fill_perf_event(event, info);
3858 case BPF_PROG_TYPE_TRACEPOINT:
3859 return bpf_perf_link_fill_tracepoint(event, info);
3860 case BPF_PROG_TYPE_KPROBE:
3861 return bpf_perf_link_fill_probe(event, info);
3862 default:
3863 return -EOPNOTSUPP;
3864 }
3865}
3866
3867static const struct bpf_link_ops bpf_perf_link_lops = {
3868 .release = bpf_perf_link_release,
3869 .dealloc = bpf_perf_link_dealloc,
3870 .fill_link_info = bpf_perf_link_fill_link_info,
3871};
3872
3873static int bpf_perf_link_attach(const union bpf_attr *attr, struct bpf_prog *prog)
3874{
3875 struct bpf_link_primer link_primer;
3876 struct bpf_perf_link *link;
3877 struct perf_event *event;
3878 struct file *perf_file;
3879 int err;
3880
3881 if (attr->link_create.flags)
3882 return -EINVAL;
3883
3884 perf_file = perf_event_get(attr->link_create.target_fd);
3885 if (IS_ERR(perf_file))
3886 return PTR_ERR(perf_file);
3887
3888 link = kzalloc(sizeof(*link), GFP_USER);
3889 if (!link) {
3890 err = -ENOMEM;
3891 goto out_put_file;
3892 }
3893 bpf_link_init(&link->link, BPF_LINK_TYPE_PERF_EVENT, &bpf_perf_link_lops, prog);
3894 link->perf_file = perf_file;
3895
3896 err = bpf_link_prime(&link->link, &link_primer);
3897 if (err) {
3898 kfree(link);
3899 goto out_put_file;
3900 }
3901
3902 event = perf_file->private_data;
3903 err = perf_event_set_bpf_prog(event, prog, attr->link_create.perf_event.bpf_cookie);
3904 if (err) {
3905 bpf_link_cleanup(&link_primer);
3906 goto out_put_file;
3907 }
3908 /* perf_event_set_bpf_prog() doesn't take its own refcnt on prog */
3909 bpf_prog_inc(prog);
3910
3911 return bpf_link_settle(&link_primer);
3912
3913out_put_file:
3914 fput(perf_file);
3915 return err;
3916}
3917#else
3918static int bpf_perf_link_attach(const union bpf_attr *attr, struct bpf_prog *prog)
3919{
3920 return -EOPNOTSUPP;
3921}
3922#endif /* CONFIG_PERF_EVENTS */
3923
3924static int bpf_raw_tp_link_attach(struct bpf_prog *prog,
3925 const char __user *user_tp_name, u64 cookie)
3926{
3927 struct bpf_link_primer link_primer;
3928 struct bpf_raw_tp_link *link;
3929 struct bpf_raw_event_map *btp;
3930 const char *tp_name;
3931 char buf[128];
3932 int err;
3933
3934 switch (prog->type) {
3935 case BPF_PROG_TYPE_TRACING:
3936 case BPF_PROG_TYPE_EXT:
3937 case BPF_PROG_TYPE_LSM:
3938 if (user_tp_name)
3939 /* The attach point for this category of programs
3940 * should be specified via btf_id during program load.
3941 */
3942 return -EINVAL;
3943 if (prog->type == BPF_PROG_TYPE_TRACING &&
3944 prog->expected_attach_type == BPF_TRACE_RAW_TP) {
3945 tp_name = prog->aux->attach_func_name;
3946 break;
3947 }
3948 return bpf_tracing_prog_attach(prog, 0, 0, 0);
3949 case BPF_PROG_TYPE_RAW_TRACEPOINT:
3950 case BPF_PROG_TYPE_RAW_TRACEPOINT_WRITABLE:
3951 if (strncpy_from_user(buf, user_tp_name, sizeof(buf) - 1) < 0)
3952 return -EFAULT;
3953 buf[sizeof(buf) - 1] = 0;
3954 tp_name = buf;
3955 break;
3956 default:
3957 return -EINVAL;
3958 }
3959
3960 btp = bpf_get_raw_tracepoint(tp_name);
3961 if (!btp)
3962 return -ENOENT;
3963
3964 link = kzalloc(sizeof(*link), GFP_USER);
3965 if (!link) {
3966 err = -ENOMEM;
3967 goto out_put_btp;
3968 }
3969 bpf_link_init_sleepable(&link->link, BPF_LINK_TYPE_RAW_TRACEPOINT,
3970 &bpf_raw_tp_link_lops, prog,
3971 tracepoint_is_faultable(btp->tp));
3972 link->btp = btp;
3973 link->cookie = cookie;
3974
3975 err = bpf_link_prime(&link->link, &link_primer);
3976 if (err) {
3977 kfree(link);
3978 goto out_put_btp;
3979 }
3980
3981 err = bpf_probe_register(link->btp, link);
3982 if (err) {
3983 bpf_link_cleanup(&link_primer);
3984 goto out_put_btp;
3985 }
3986
3987 return bpf_link_settle(&link_primer);
3988
3989out_put_btp:
3990 bpf_put_raw_tracepoint(btp);
3991 return err;
3992}
3993
3994#define BPF_RAW_TRACEPOINT_OPEN_LAST_FIELD raw_tracepoint.cookie
3995
3996static int bpf_raw_tracepoint_open(const union bpf_attr *attr)
3997{
3998 struct bpf_prog *prog;
3999 void __user *tp_name;
4000 __u64 cookie;
4001 int fd;
4002
4003 if (CHECK_ATTR(BPF_RAW_TRACEPOINT_OPEN))
4004 return -EINVAL;
4005
4006 prog = bpf_prog_get(attr->raw_tracepoint.prog_fd);
4007 if (IS_ERR(prog))
4008 return PTR_ERR(prog);
4009
4010 tp_name = u64_to_user_ptr(attr->raw_tracepoint.name);
4011 cookie = attr->raw_tracepoint.cookie;
4012 fd = bpf_raw_tp_link_attach(prog, tp_name, cookie);
4013 if (fd < 0)
4014 bpf_prog_put(prog);
4015 return fd;
4016}
4017
4018static enum bpf_prog_type
4019attach_type_to_prog_type(enum bpf_attach_type attach_type)
4020{
4021 switch (attach_type) {
4022 case BPF_CGROUP_INET_INGRESS:
4023 case BPF_CGROUP_INET_EGRESS:
4024 return BPF_PROG_TYPE_CGROUP_SKB;
4025 case BPF_CGROUP_INET_SOCK_CREATE:
4026 case BPF_CGROUP_INET_SOCK_RELEASE:
4027 case BPF_CGROUP_INET4_POST_BIND:
4028 case BPF_CGROUP_INET6_POST_BIND:
4029 return BPF_PROG_TYPE_CGROUP_SOCK;
4030 case BPF_CGROUP_INET4_BIND:
4031 case BPF_CGROUP_INET6_BIND:
4032 case BPF_CGROUP_INET4_CONNECT:
4033 case BPF_CGROUP_INET6_CONNECT:
4034 case BPF_CGROUP_UNIX_CONNECT:
4035 case BPF_CGROUP_INET4_GETPEERNAME:
4036 case BPF_CGROUP_INET6_GETPEERNAME:
4037 case BPF_CGROUP_UNIX_GETPEERNAME:
4038 case BPF_CGROUP_INET4_GETSOCKNAME:
4039 case BPF_CGROUP_INET6_GETSOCKNAME:
4040 case BPF_CGROUP_UNIX_GETSOCKNAME:
4041 case BPF_CGROUP_UDP4_SENDMSG:
4042 case BPF_CGROUP_UDP6_SENDMSG:
4043 case BPF_CGROUP_UNIX_SENDMSG:
4044 case BPF_CGROUP_UDP4_RECVMSG:
4045 case BPF_CGROUP_UDP6_RECVMSG:
4046 case BPF_CGROUP_UNIX_RECVMSG:
4047 return BPF_PROG_TYPE_CGROUP_SOCK_ADDR;
4048 case BPF_CGROUP_SOCK_OPS:
4049 return BPF_PROG_TYPE_SOCK_OPS;
4050 case BPF_CGROUP_DEVICE:
4051 return BPF_PROG_TYPE_CGROUP_DEVICE;
4052 case BPF_SK_MSG_VERDICT:
4053 return BPF_PROG_TYPE_SK_MSG;
4054 case BPF_SK_SKB_STREAM_PARSER:
4055 case BPF_SK_SKB_STREAM_VERDICT:
4056 case BPF_SK_SKB_VERDICT:
4057 return BPF_PROG_TYPE_SK_SKB;
4058 case BPF_LIRC_MODE2:
4059 return BPF_PROG_TYPE_LIRC_MODE2;
4060 case BPF_FLOW_DISSECTOR:
4061 return BPF_PROG_TYPE_FLOW_DISSECTOR;
4062 case BPF_CGROUP_SYSCTL:
4063 return BPF_PROG_TYPE_CGROUP_SYSCTL;
4064 case BPF_CGROUP_GETSOCKOPT:
4065 case BPF_CGROUP_SETSOCKOPT:
4066 return BPF_PROG_TYPE_CGROUP_SOCKOPT;
4067 case BPF_TRACE_ITER:
4068 case BPF_TRACE_RAW_TP:
4069 case BPF_TRACE_FENTRY:
4070 case BPF_TRACE_FEXIT:
4071 case BPF_MODIFY_RETURN:
4072 return BPF_PROG_TYPE_TRACING;
4073 case BPF_LSM_MAC:
4074 return BPF_PROG_TYPE_LSM;
4075 case BPF_SK_LOOKUP:
4076 return BPF_PROG_TYPE_SK_LOOKUP;
4077 case BPF_XDP:
4078 return BPF_PROG_TYPE_XDP;
4079 case BPF_LSM_CGROUP:
4080 return BPF_PROG_TYPE_LSM;
4081 case BPF_TCX_INGRESS:
4082 case BPF_TCX_EGRESS:
4083 case BPF_NETKIT_PRIMARY:
4084 case BPF_NETKIT_PEER:
4085 return BPF_PROG_TYPE_SCHED_CLS;
4086 default:
4087 return BPF_PROG_TYPE_UNSPEC;
4088 }
4089}
4090
4091static int bpf_prog_attach_check_attach_type(const struct bpf_prog *prog,
4092 enum bpf_attach_type attach_type)
4093{
4094 enum bpf_prog_type ptype;
4095
4096 switch (prog->type) {
4097 case BPF_PROG_TYPE_CGROUP_SOCK:
4098 case BPF_PROG_TYPE_CGROUP_SOCK_ADDR:
4099 case BPF_PROG_TYPE_CGROUP_SOCKOPT:
4100 case BPF_PROG_TYPE_SK_LOOKUP:
4101 return attach_type == prog->expected_attach_type ? 0 : -EINVAL;
4102 case BPF_PROG_TYPE_CGROUP_SKB:
4103 if (!bpf_token_capable(prog->aux->token, CAP_NET_ADMIN))
4104 /* cg-skb progs can be loaded by unpriv user.
4105 * check permissions at attach time.
4106 */
4107 return -EPERM;
4108
4109 ptype = attach_type_to_prog_type(attach_type);
4110 if (prog->type != ptype)
4111 return -EINVAL;
4112
4113 return prog->enforce_expected_attach_type &&
4114 prog->expected_attach_type != attach_type ?
4115 -EINVAL : 0;
4116 case BPF_PROG_TYPE_EXT:
4117 return 0;
4118 case BPF_PROG_TYPE_NETFILTER:
4119 if (attach_type != BPF_NETFILTER)
4120 return -EINVAL;
4121 return 0;
4122 case BPF_PROG_TYPE_PERF_EVENT:
4123 case BPF_PROG_TYPE_TRACEPOINT:
4124 if (attach_type != BPF_PERF_EVENT)
4125 return -EINVAL;
4126 return 0;
4127 case BPF_PROG_TYPE_KPROBE:
4128 if (prog->expected_attach_type == BPF_TRACE_KPROBE_MULTI &&
4129 attach_type != BPF_TRACE_KPROBE_MULTI)
4130 return -EINVAL;
4131 if (prog->expected_attach_type == BPF_TRACE_KPROBE_SESSION &&
4132 attach_type != BPF_TRACE_KPROBE_SESSION)
4133 return -EINVAL;
4134 if (prog->expected_attach_type == BPF_TRACE_UPROBE_MULTI &&
4135 attach_type != BPF_TRACE_UPROBE_MULTI)
4136 return -EINVAL;
4137 if (prog->expected_attach_type == BPF_TRACE_UPROBE_SESSION &&
4138 attach_type != BPF_TRACE_UPROBE_SESSION)
4139 return -EINVAL;
4140 if (attach_type != BPF_PERF_EVENT &&
4141 attach_type != BPF_TRACE_KPROBE_MULTI &&
4142 attach_type != BPF_TRACE_KPROBE_SESSION &&
4143 attach_type != BPF_TRACE_UPROBE_MULTI &&
4144 attach_type != BPF_TRACE_UPROBE_SESSION)
4145 return -EINVAL;
4146 return 0;
4147 case BPF_PROG_TYPE_SCHED_CLS:
4148 if (attach_type != BPF_TCX_INGRESS &&
4149 attach_type != BPF_TCX_EGRESS &&
4150 attach_type != BPF_NETKIT_PRIMARY &&
4151 attach_type != BPF_NETKIT_PEER)
4152 return -EINVAL;
4153 return 0;
4154 default:
4155 ptype = attach_type_to_prog_type(attach_type);
4156 if (ptype == BPF_PROG_TYPE_UNSPEC || ptype != prog->type)
4157 return -EINVAL;
4158 return 0;
4159 }
4160}
4161
4162#define BPF_PROG_ATTACH_LAST_FIELD expected_revision
4163
4164#define BPF_F_ATTACH_MASK_BASE \
4165 (BPF_F_ALLOW_OVERRIDE | \
4166 BPF_F_ALLOW_MULTI | \
4167 BPF_F_REPLACE)
4168
4169#define BPF_F_ATTACH_MASK_MPROG \
4170 (BPF_F_REPLACE | \
4171 BPF_F_BEFORE | \
4172 BPF_F_AFTER | \
4173 BPF_F_ID | \
4174 BPF_F_LINK)
4175
4176static int bpf_prog_attach(const union bpf_attr *attr)
4177{
4178 enum bpf_prog_type ptype;
4179 struct bpf_prog *prog;
4180 int ret;
4181
4182 if (CHECK_ATTR(BPF_PROG_ATTACH))
4183 return -EINVAL;
4184
4185 ptype = attach_type_to_prog_type(attr->attach_type);
4186 if (ptype == BPF_PROG_TYPE_UNSPEC)
4187 return -EINVAL;
4188 if (bpf_mprog_supported(ptype)) {
4189 if (attr->attach_flags & ~BPF_F_ATTACH_MASK_MPROG)
4190 return -EINVAL;
4191 } else {
4192 if (attr->attach_flags & ~BPF_F_ATTACH_MASK_BASE)
4193 return -EINVAL;
4194 if (attr->relative_fd ||
4195 attr->expected_revision)
4196 return -EINVAL;
4197 }
4198
4199 prog = bpf_prog_get_type(attr->attach_bpf_fd, ptype);
4200 if (IS_ERR(prog))
4201 return PTR_ERR(prog);
4202
4203 if (bpf_prog_attach_check_attach_type(prog, attr->attach_type)) {
4204 bpf_prog_put(prog);
4205 return -EINVAL;
4206 }
4207
4208 switch (ptype) {
4209 case BPF_PROG_TYPE_SK_SKB:
4210 case BPF_PROG_TYPE_SK_MSG:
4211 ret = sock_map_get_from_fd(attr, prog);
4212 break;
4213 case BPF_PROG_TYPE_LIRC_MODE2:
4214 ret = lirc_prog_attach(attr, prog);
4215 break;
4216 case BPF_PROG_TYPE_FLOW_DISSECTOR:
4217 ret = netns_bpf_prog_attach(attr, prog);
4218 break;
4219 case BPF_PROG_TYPE_CGROUP_DEVICE:
4220 case BPF_PROG_TYPE_CGROUP_SKB:
4221 case BPF_PROG_TYPE_CGROUP_SOCK:
4222 case BPF_PROG_TYPE_CGROUP_SOCK_ADDR:
4223 case BPF_PROG_TYPE_CGROUP_SOCKOPT:
4224 case BPF_PROG_TYPE_CGROUP_SYSCTL:
4225 case BPF_PROG_TYPE_SOCK_OPS:
4226 case BPF_PROG_TYPE_LSM:
4227 if (ptype == BPF_PROG_TYPE_LSM &&
4228 prog->expected_attach_type != BPF_LSM_CGROUP)
4229 ret = -EINVAL;
4230 else
4231 ret = cgroup_bpf_prog_attach(attr, ptype, prog);
4232 break;
4233 case BPF_PROG_TYPE_SCHED_CLS:
4234 if (attr->attach_type == BPF_TCX_INGRESS ||
4235 attr->attach_type == BPF_TCX_EGRESS)
4236 ret = tcx_prog_attach(attr, prog);
4237 else
4238 ret = netkit_prog_attach(attr, prog);
4239 break;
4240 default:
4241 ret = -EINVAL;
4242 }
4243
4244 if (ret)
4245 bpf_prog_put(prog);
4246 return ret;
4247}
4248
4249#define BPF_PROG_DETACH_LAST_FIELD expected_revision
4250
4251static int bpf_prog_detach(const union bpf_attr *attr)
4252{
4253 struct bpf_prog *prog = NULL;
4254 enum bpf_prog_type ptype;
4255 int ret;
4256
4257 if (CHECK_ATTR(BPF_PROG_DETACH))
4258 return -EINVAL;
4259
4260 ptype = attach_type_to_prog_type(attr->attach_type);
4261 if (bpf_mprog_supported(ptype)) {
4262 if (ptype == BPF_PROG_TYPE_UNSPEC)
4263 return -EINVAL;
4264 if (attr->attach_flags & ~BPF_F_ATTACH_MASK_MPROG)
4265 return -EINVAL;
4266 if (attr->attach_bpf_fd) {
4267 prog = bpf_prog_get_type(attr->attach_bpf_fd, ptype);
4268 if (IS_ERR(prog))
4269 return PTR_ERR(prog);
4270 }
4271 } else if (attr->attach_flags ||
4272 attr->relative_fd ||
4273 attr->expected_revision) {
4274 return -EINVAL;
4275 }
4276
4277 switch (ptype) {
4278 case BPF_PROG_TYPE_SK_MSG:
4279 case BPF_PROG_TYPE_SK_SKB:
4280 ret = sock_map_prog_detach(attr, ptype);
4281 break;
4282 case BPF_PROG_TYPE_LIRC_MODE2:
4283 ret = lirc_prog_detach(attr);
4284 break;
4285 case BPF_PROG_TYPE_FLOW_DISSECTOR:
4286 ret = netns_bpf_prog_detach(attr, ptype);
4287 break;
4288 case BPF_PROG_TYPE_CGROUP_DEVICE:
4289 case BPF_PROG_TYPE_CGROUP_SKB:
4290 case BPF_PROG_TYPE_CGROUP_SOCK:
4291 case BPF_PROG_TYPE_CGROUP_SOCK_ADDR:
4292 case BPF_PROG_TYPE_CGROUP_SOCKOPT:
4293 case BPF_PROG_TYPE_CGROUP_SYSCTL:
4294 case BPF_PROG_TYPE_SOCK_OPS:
4295 case BPF_PROG_TYPE_LSM:
4296 ret = cgroup_bpf_prog_detach(attr, ptype);
4297 break;
4298 case BPF_PROG_TYPE_SCHED_CLS:
4299 if (attr->attach_type == BPF_TCX_INGRESS ||
4300 attr->attach_type == BPF_TCX_EGRESS)
4301 ret = tcx_prog_detach(attr, prog);
4302 else
4303 ret = netkit_prog_detach(attr, prog);
4304 break;
4305 default:
4306 ret = -EINVAL;
4307 }
4308
4309 if (prog)
4310 bpf_prog_put(prog);
4311 return ret;
4312}
4313
4314#define BPF_PROG_QUERY_LAST_FIELD query.revision
4315
4316static int bpf_prog_query(const union bpf_attr *attr,
4317 union bpf_attr __user *uattr)
4318{
4319 if (!bpf_net_capable())
4320 return -EPERM;
4321 if (CHECK_ATTR(BPF_PROG_QUERY))
4322 return -EINVAL;
4323 if (attr->query.query_flags & ~BPF_F_QUERY_EFFECTIVE)
4324 return -EINVAL;
4325
4326 switch (attr->query.attach_type) {
4327 case BPF_CGROUP_INET_INGRESS:
4328 case BPF_CGROUP_INET_EGRESS:
4329 case BPF_CGROUP_INET_SOCK_CREATE:
4330 case BPF_CGROUP_INET_SOCK_RELEASE:
4331 case BPF_CGROUP_INET4_BIND:
4332 case BPF_CGROUP_INET6_BIND:
4333 case BPF_CGROUP_INET4_POST_BIND:
4334 case BPF_CGROUP_INET6_POST_BIND:
4335 case BPF_CGROUP_INET4_CONNECT:
4336 case BPF_CGROUP_INET6_CONNECT:
4337 case BPF_CGROUP_UNIX_CONNECT:
4338 case BPF_CGROUP_INET4_GETPEERNAME:
4339 case BPF_CGROUP_INET6_GETPEERNAME:
4340 case BPF_CGROUP_UNIX_GETPEERNAME:
4341 case BPF_CGROUP_INET4_GETSOCKNAME:
4342 case BPF_CGROUP_INET6_GETSOCKNAME:
4343 case BPF_CGROUP_UNIX_GETSOCKNAME:
4344 case BPF_CGROUP_UDP4_SENDMSG:
4345 case BPF_CGROUP_UDP6_SENDMSG:
4346 case BPF_CGROUP_UNIX_SENDMSG:
4347 case BPF_CGROUP_UDP4_RECVMSG:
4348 case BPF_CGROUP_UDP6_RECVMSG:
4349 case BPF_CGROUP_UNIX_RECVMSG:
4350 case BPF_CGROUP_SOCK_OPS:
4351 case BPF_CGROUP_DEVICE:
4352 case BPF_CGROUP_SYSCTL:
4353 case BPF_CGROUP_GETSOCKOPT:
4354 case BPF_CGROUP_SETSOCKOPT:
4355 case BPF_LSM_CGROUP:
4356 return cgroup_bpf_prog_query(attr, uattr);
4357 case BPF_LIRC_MODE2:
4358 return lirc_prog_query(attr, uattr);
4359 case BPF_FLOW_DISSECTOR:
4360 case BPF_SK_LOOKUP:
4361 return netns_bpf_prog_query(attr, uattr);
4362 case BPF_SK_SKB_STREAM_PARSER:
4363 case BPF_SK_SKB_STREAM_VERDICT:
4364 case BPF_SK_MSG_VERDICT:
4365 case BPF_SK_SKB_VERDICT:
4366 return sock_map_bpf_prog_query(attr, uattr);
4367 case BPF_TCX_INGRESS:
4368 case BPF_TCX_EGRESS:
4369 return tcx_prog_query(attr, uattr);
4370 case BPF_NETKIT_PRIMARY:
4371 case BPF_NETKIT_PEER:
4372 return netkit_prog_query(attr, uattr);
4373 default:
4374 return -EINVAL;
4375 }
4376}
4377
4378#define BPF_PROG_TEST_RUN_LAST_FIELD test.batch_size
4379
4380static int bpf_prog_test_run(const union bpf_attr *attr,
4381 union bpf_attr __user *uattr)
4382{
4383 struct bpf_prog *prog;
4384 int ret = -ENOTSUPP;
4385
4386 if (CHECK_ATTR(BPF_PROG_TEST_RUN))
4387 return -EINVAL;
4388
4389 if ((attr->test.ctx_size_in && !attr->test.ctx_in) ||
4390 (!attr->test.ctx_size_in && attr->test.ctx_in))
4391 return -EINVAL;
4392
4393 if ((attr->test.ctx_size_out && !attr->test.ctx_out) ||
4394 (!attr->test.ctx_size_out && attr->test.ctx_out))
4395 return -EINVAL;
4396
4397 prog = bpf_prog_get(attr->test.prog_fd);
4398 if (IS_ERR(prog))
4399 return PTR_ERR(prog);
4400
4401 if (prog->aux->ops->test_run)
4402 ret = prog->aux->ops->test_run(prog, attr, uattr);
4403
4404 bpf_prog_put(prog);
4405 return ret;
4406}
4407
4408#define BPF_OBJ_GET_NEXT_ID_LAST_FIELD next_id
4409
4410static int bpf_obj_get_next_id(const union bpf_attr *attr,
4411 union bpf_attr __user *uattr,
4412 struct idr *idr,
4413 spinlock_t *lock)
4414{
4415 u32 next_id = attr->start_id;
4416 int err = 0;
4417
4418 if (CHECK_ATTR(BPF_OBJ_GET_NEXT_ID) || next_id >= INT_MAX)
4419 return -EINVAL;
4420
4421 if (!capable(CAP_SYS_ADMIN))
4422 return -EPERM;
4423
4424 next_id++;
4425 spin_lock_bh(lock);
4426 if (!idr_get_next(idr, &next_id))
4427 err = -ENOENT;
4428 spin_unlock_bh(lock);
4429
4430 if (!err)
4431 err = put_user(next_id, &uattr->next_id);
4432
4433 return err;
4434}
4435
4436struct bpf_map *bpf_map_get_curr_or_next(u32 *id)
4437{
4438 struct bpf_map *map;
4439
4440 spin_lock_bh(&map_idr_lock);
4441again:
4442 map = idr_get_next(&map_idr, id);
4443 if (map) {
4444 map = __bpf_map_inc_not_zero(map, false);
4445 if (IS_ERR(map)) {
4446 (*id)++;
4447 goto again;
4448 }
4449 }
4450 spin_unlock_bh(&map_idr_lock);
4451
4452 return map;
4453}
4454
4455struct bpf_prog *bpf_prog_get_curr_or_next(u32 *id)
4456{
4457 struct bpf_prog *prog;
4458
4459 spin_lock_bh(&prog_idr_lock);
4460again:
4461 prog = idr_get_next(&prog_idr, id);
4462 if (prog) {
4463 prog = bpf_prog_inc_not_zero(prog);
4464 if (IS_ERR(prog)) {
4465 (*id)++;
4466 goto again;
4467 }
4468 }
4469 spin_unlock_bh(&prog_idr_lock);
4470
4471 return prog;
4472}
4473
4474#define BPF_PROG_GET_FD_BY_ID_LAST_FIELD prog_id
4475
4476struct bpf_prog *bpf_prog_by_id(u32 id)
4477{
4478 struct bpf_prog *prog;
4479
4480 if (!id)
4481 return ERR_PTR(-ENOENT);
4482
4483 spin_lock_bh(&prog_idr_lock);
4484 prog = idr_find(&prog_idr, id);
4485 if (prog)
4486 prog = bpf_prog_inc_not_zero(prog);
4487 else
4488 prog = ERR_PTR(-ENOENT);
4489 spin_unlock_bh(&prog_idr_lock);
4490 return prog;
4491}
4492
4493static int bpf_prog_get_fd_by_id(const union bpf_attr *attr)
4494{
4495 struct bpf_prog *prog;
4496 u32 id = attr->prog_id;
4497 int fd;
4498
4499 if (CHECK_ATTR(BPF_PROG_GET_FD_BY_ID))
4500 return -EINVAL;
4501
4502 if (!capable(CAP_SYS_ADMIN))
4503 return -EPERM;
4504
4505 prog = bpf_prog_by_id(id);
4506 if (IS_ERR(prog))
4507 return PTR_ERR(prog);
4508
4509 fd = bpf_prog_new_fd(prog);
4510 if (fd < 0)
4511 bpf_prog_put(prog);
4512
4513 return fd;
4514}
4515
4516#define BPF_MAP_GET_FD_BY_ID_LAST_FIELD open_flags
4517
4518static int bpf_map_get_fd_by_id(const union bpf_attr *attr)
4519{
4520 struct bpf_map *map;
4521 u32 id = attr->map_id;
4522 int f_flags;
4523 int fd;
4524
4525 if (CHECK_ATTR(BPF_MAP_GET_FD_BY_ID) ||
4526 attr->open_flags & ~BPF_OBJ_FLAG_MASK)
4527 return -EINVAL;
4528
4529 if (!capable(CAP_SYS_ADMIN))
4530 return -EPERM;
4531
4532 f_flags = bpf_get_file_flag(attr->open_flags);
4533 if (f_flags < 0)
4534 return f_flags;
4535
4536 spin_lock_bh(&map_idr_lock);
4537 map = idr_find(&map_idr, id);
4538 if (map)
4539 map = __bpf_map_inc_not_zero(map, true);
4540 else
4541 map = ERR_PTR(-ENOENT);
4542 spin_unlock_bh(&map_idr_lock);
4543
4544 if (IS_ERR(map))
4545 return PTR_ERR(map);
4546
4547 fd = bpf_map_new_fd(map, f_flags);
4548 if (fd < 0)
4549 bpf_map_put_with_uref(map);
4550
4551 return fd;
4552}
4553
4554static const struct bpf_map *bpf_map_from_imm(const struct bpf_prog *prog,
4555 unsigned long addr, u32 *off,
4556 u32 *type)
4557{
4558 const struct bpf_map *map;
4559 int i;
4560
4561 mutex_lock(&prog->aux->used_maps_mutex);
4562 for (i = 0, *off = 0; i < prog->aux->used_map_cnt; i++) {
4563 map = prog->aux->used_maps[i];
4564 if (map == (void *)addr) {
4565 *type = BPF_PSEUDO_MAP_FD;
4566 goto out;
4567 }
4568 if (!map->ops->map_direct_value_meta)
4569 continue;
4570 if (!map->ops->map_direct_value_meta(map, addr, off)) {
4571 *type = BPF_PSEUDO_MAP_VALUE;
4572 goto out;
4573 }
4574 }
4575 map = NULL;
4576
4577out:
4578 mutex_unlock(&prog->aux->used_maps_mutex);
4579 return map;
4580}
4581
4582static struct bpf_insn *bpf_insn_prepare_dump(const struct bpf_prog *prog,
4583 const struct cred *f_cred)
4584{
4585 const struct bpf_map *map;
4586 struct bpf_insn *insns;
4587 u32 off, type;
4588 u64 imm;
4589 u8 code;
4590 int i;
4591
4592 insns = kmemdup(prog->insnsi, bpf_prog_insn_size(prog),
4593 GFP_USER);
4594 if (!insns)
4595 return insns;
4596
4597 for (i = 0; i < prog->len; i++) {
4598 code = insns[i].code;
4599
4600 if (code == (BPF_JMP | BPF_TAIL_CALL)) {
4601 insns[i].code = BPF_JMP | BPF_CALL;
4602 insns[i].imm = BPF_FUNC_tail_call;
4603 /* fall-through */
4604 }
4605 if (code == (BPF_JMP | BPF_CALL) ||
4606 code == (BPF_JMP | BPF_CALL_ARGS)) {
4607 if (code == (BPF_JMP | BPF_CALL_ARGS))
4608 insns[i].code = BPF_JMP | BPF_CALL;
4609 if (!bpf_dump_raw_ok(f_cred))
4610 insns[i].imm = 0;
4611 continue;
4612 }
4613 if (BPF_CLASS(code) == BPF_LDX && BPF_MODE(code) == BPF_PROBE_MEM) {
4614 insns[i].code = BPF_LDX | BPF_SIZE(code) | BPF_MEM;
4615 continue;
4616 }
4617
4618 if ((BPF_CLASS(code) == BPF_LDX || BPF_CLASS(code) == BPF_STX ||
4619 BPF_CLASS(code) == BPF_ST) && BPF_MODE(code) == BPF_PROBE_MEM32) {
4620 insns[i].code = BPF_CLASS(code) | BPF_SIZE(code) | BPF_MEM;
4621 continue;
4622 }
4623
4624 if (code != (BPF_LD | BPF_IMM | BPF_DW))
4625 continue;
4626
4627 imm = ((u64)insns[i + 1].imm << 32) | (u32)insns[i].imm;
4628 map = bpf_map_from_imm(prog, imm, &off, &type);
4629 if (map) {
4630 insns[i].src_reg = type;
4631 insns[i].imm = map->id;
4632 insns[i + 1].imm = off;
4633 continue;
4634 }
4635 }
4636
4637 return insns;
4638}
4639
4640static int set_info_rec_size(struct bpf_prog_info *info)
4641{
4642 /*
4643 * Ensure info.*_rec_size is the same as kernel expected size
4644 *
4645 * or
4646 *
4647 * Only allow zero *_rec_size if both _rec_size and _cnt are
4648 * zero. In this case, the kernel will set the expected
4649 * _rec_size back to the info.
4650 */
4651
4652 if ((info->nr_func_info || info->func_info_rec_size) &&
4653 info->func_info_rec_size != sizeof(struct bpf_func_info))
4654 return -EINVAL;
4655
4656 if ((info->nr_line_info || info->line_info_rec_size) &&
4657 info->line_info_rec_size != sizeof(struct bpf_line_info))
4658 return -EINVAL;
4659
4660 if ((info->nr_jited_line_info || info->jited_line_info_rec_size) &&
4661 info->jited_line_info_rec_size != sizeof(__u64))
4662 return -EINVAL;
4663
4664 info->func_info_rec_size = sizeof(struct bpf_func_info);
4665 info->line_info_rec_size = sizeof(struct bpf_line_info);
4666 info->jited_line_info_rec_size = sizeof(__u64);
4667
4668 return 0;
4669}
4670
4671static int bpf_prog_get_info_by_fd(struct file *file,
4672 struct bpf_prog *prog,
4673 const union bpf_attr *attr,
4674 union bpf_attr __user *uattr)
4675{
4676 struct bpf_prog_info __user *uinfo = u64_to_user_ptr(attr->info.info);
4677 struct btf *attach_btf = bpf_prog_get_target_btf(prog);
4678 struct bpf_prog_info info;
4679 u32 info_len = attr->info.info_len;
4680 struct bpf_prog_kstats stats;
4681 char __user *uinsns;
4682 u32 ulen;
4683 int err;
4684
4685 err = bpf_check_uarg_tail_zero(USER_BPFPTR(uinfo), sizeof(info), info_len);
4686 if (err)
4687 return err;
4688 info_len = min_t(u32, sizeof(info), info_len);
4689
4690 memset(&info, 0, sizeof(info));
4691 if (copy_from_user(&info, uinfo, info_len))
4692 return -EFAULT;
4693
4694 info.type = prog->type;
4695 info.id = prog->aux->id;
4696 info.load_time = prog->aux->load_time;
4697 info.created_by_uid = from_kuid_munged(current_user_ns(),
4698 prog->aux->user->uid);
4699 info.gpl_compatible = prog->gpl_compatible;
4700
4701 memcpy(info.tag, prog->tag, sizeof(prog->tag));
4702 memcpy(info.name, prog->aux->name, sizeof(prog->aux->name));
4703
4704 mutex_lock(&prog->aux->used_maps_mutex);
4705 ulen = info.nr_map_ids;
4706 info.nr_map_ids = prog->aux->used_map_cnt;
4707 ulen = min_t(u32, info.nr_map_ids, ulen);
4708 if (ulen) {
4709 u32 __user *user_map_ids = u64_to_user_ptr(info.map_ids);
4710 u32 i;
4711
4712 for (i = 0; i < ulen; i++)
4713 if (put_user(prog->aux->used_maps[i]->id,
4714 &user_map_ids[i])) {
4715 mutex_unlock(&prog->aux->used_maps_mutex);
4716 return -EFAULT;
4717 }
4718 }
4719 mutex_unlock(&prog->aux->used_maps_mutex);
4720
4721 err = set_info_rec_size(&info);
4722 if (err)
4723 return err;
4724
4725 bpf_prog_get_stats(prog, &stats);
4726 info.run_time_ns = stats.nsecs;
4727 info.run_cnt = stats.cnt;
4728 info.recursion_misses = stats.misses;
4729
4730 info.verified_insns = prog->aux->verified_insns;
4731
4732 if (!bpf_capable()) {
4733 info.jited_prog_len = 0;
4734 info.xlated_prog_len = 0;
4735 info.nr_jited_ksyms = 0;
4736 info.nr_jited_func_lens = 0;
4737 info.nr_func_info = 0;
4738 info.nr_line_info = 0;
4739 info.nr_jited_line_info = 0;
4740 goto done;
4741 }
4742
4743 ulen = info.xlated_prog_len;
4744 info.xlated_prog_len = bpf_prog_insn_size(prog);
4745 if (info.xlated_prog_len && ulen) {
4746 struct bpf_insn *insns_sanitized;
4747 bool fault;
4748
4749 if (prog->blinded && !bpf_dump_raw_ok(file->f_cred)) {
4750 info.xlated_prog_insns = 0;
4751 goto done;
4752 }
4753 insns_sanitized = bpf_insn_prepare_dump(prog, file->f_cred);
4754 if (!insns_sanitized)
4755 return -ENOMEM;
4756 uinsns = u64_to_user_ptr(info.xlated_prog_insns);
4757 ulen = min_t(u32, info.xlated_prog_len, ulen);
4758 fault = copy_to_user(uinsns, insns_sanitized, ulen);
4759 kfree(insns_sanitized);
4760 if (fault)
4761 return -EFAULT;
4762 }
4763
4764 if (bpf_prog_is_offloaded(prog->aux)) {
4765 err = bpf_prog_offload_info_fill(&info, prog);
4766 if (err)
4767 return err;
4768 goto done;
4769 }
4770
4771 /* NOTE: the following code is supposed to be skipped for offload.
4772 * bpf_prog_offload_info_fill() is the place to fill similar fields
4773 * for offload.
4774 */
4775 ulen = info.jited_prog_len;
4776 if (prog->aux->func_cnt) {
4777 u32 i;
4778
4779 info.jited_prog_len = 0;
4780 for (i = 0; i < prog->aux->func_cnt; i++)
4781 info.jited_prog_len += prog->aux->func[i]->jited_len;
4782 } else {
4783 info.jited_prog_len = prog->jited_len;
4784 }
4785
4786 if (info.jited_prog_len && ulen) {
4787 if (bpf_dump_raw_ok(file->f_cred)) {
4788 uinsns = u64_to_user_ptr(info.jited_prog_insns);
4789 ulen = min_t(u32, info.jited_prog_len, ulen);
4790
4791 /* for multi-function programs, copy the JITed
4792 * instructions for all the functions
4793 */
4794 if (prog->aux->func_cnt) {
4795 u32 len, free, i;
4796 u8 *img;
4797
4798 free = ulen;
4799 for (i = 0; i < prog->aux->func_cnt; i++) {
4800 len = prog->aux->func[i]->jited_len;
4801 len = min_t(u32, len, free);
4802 img = (u8 *) prog->aux->func[i]->bpf_func;
4803 if (copy_to_user(uinsns, img, len))
4804 return -EFAULT;
4805 uinsns += len;
4806 free -= len;
4807 if (!free)
4808 break;
4809 }
4810 } else {
4811 if (copy_to_user(uinsns, prog->bpf_func, ulen))
4812 return -EFAULT;
4813 }
4814 } else {
4815 info.jited_prog_insns = 0;
4816 }
4817 }
4818
4819 ulen = info.nr_jited_ksyms;
4820 info.nr_jited_ksyms = prog->aux->func_cnt ? : 1;
4821 if (ulen) {
4822 if (bpf_dump_raw_ok(file->f_cred)) {
4823 unsigned long ksym_addr;
4824 u64 __user *user_ksyms;
4825 u32 i;
4826
4827 /* copy the address of the kernel symbol
4828 * corresponding to each function
4829 */
4830 ulen = min_t(u32, info.nr_jited_ksyms, ulen);
4831 user_ksyms = u64_to_user_ptr(info.jited_ksyms);
4832 if (prog->aux->func_cnt) {
4833 for (i = 0; i < ulen; i++) {
4834 ksym_addr = (unsigned long)
4835 prog->aux->func[i]->bpf_func;
4836 if (put_user((u64) ksym_addr,
4837 &user_ksyms[i]))
4838 return -EFAULT;
4839 }
4840 } else {
4841 ksym_addr = (unsigned long) prog->bpf_func;
4842 if (put_user((u64) ksym_addr, &user_ksyms[0]))
4843 return -EFAULT;
4844 }
4845 } else {
4846 info.jited_ksyms = 0;
4847 }
4848 }
4849
4850 ulen = info.nr_jited_func_lens;
4851 info.nr_jited_func_lens = prog->aux->func_cnt ? : 1;
4852 if (ulen) {
4853 if (bpf_dump_raw_ok(file->f_cred)) {
4854 u32 __user *user_lens;
4855 u32 func_len, i;
4856
4857 /* copy the JITed image lengths for each function */
4858 ulen = min_t(u32, info.nr_jited_func_lens, ulen);
4859 user_lens = u64_to_user_ptr(info.jited_func_lens);
4860 if (prog->aux->func_cnt) {
4861 for (i = 0; i < ulen; i++) {
4862 func_len =
4863 prog->aux->func[i]->jited_len;
4864 if (put_user(func_len, &user_lens[i]))
4865 return -EFAULT;
4866 }
4867 } else {
4868 func_len = prog->jited_len;
4869 if (put_user(func_len, &user_lens[0]))
4870 return -EFAULT;
4871 }
4872 } else {
4873 info.jited_func_lens = 0;
4874 }
4875 }
4876
4877 if (prog->aux->btf)
4878 info.btf_id = btf_obj_id(prog->aux->btf);
4879 info.attach_btf_id = prog->aux->attach_btf_id;
4880 if (attach_btf)
4881 info.attach_btf_obj_id = btf_obj_id(attach_btf);
4882
4883 ulen = info.nr_func_info;
4884 info.nr_func_info = prog->aux->func_info_cnt;
4885 if (info.nr_func_info && ulen) {
4886 char __user *user_finfo;
4887
4888 user_finfo = u64_to_user_ptr(info.func_info);
4889 ulen = min_t(u32, info.nr_func_info, ulen);
4890 if (copy_to_user(user_finfo, prog->aux->func_info,
4891 info.func_info_rec_size * ulen))
4892 return -EFAULT;
4893 }
4894
4895 ulen = info.nr_line_info;
4896 info.nr_line_info = prog->aux->nr_linfo;
4897 if (info.nr_line_info && ulen) {
4898 __u8 __user *user_linfo;
4899
4900 user_linfo = u64_to_user_ptr(info.line_info);
4901 ulen = min_t(u32, info.nr_line_info, ulen);
4902 if (copy_to_user(user_linfo, prog->aux->linfo,
4903 info.line_info_rec_size * ulen))
4904 return -EFAULT;
4905 }
4906
4907 ulen = info.nr_jited_line_info;
4908 if (prog->aux->jited_linfo)
4909 info.nr_jited_line_info = prog->aux->nr_linfo;
4910 else
4911 info.nr_jited_line_info = 0;
4912 if (info.nr_jited_line_info && ulen) {
4913 if (bpf_dump_raw_ok(file->f_cred)) {
4914 unsigned long line_addr;
4915 __u64 __user *user_linfo;
4916 u32 i;
4917
4918 user_linfo = u64_to_user_ptr(info.jited_line_info);
4919 ulen = min_t(u32, info.nr_jited_line_info, ulen);
4920 for (i = 0; i < ulen; i++) {
4921 line_addr = (unsigned long)prog->aux->jited_linfo[i];
4922 if (put_user((__u64)line_addr, &user_linfo[i]))
4923 return -EFAULT;
4924 }
4925 } else {
4926 info.jited_line_info = 0;
4927 }
4928 }
4929
4930 ulen = info.nr_prog_tags;
4931 info.nr_prog_tags = prog->aux->func_cnt ? : 1;
4932 if (ulen) {
4933 __u8 __user (*user_prog_tags)[BPF_TAG_SIZE];
4934 u32 i;
4935
4936 user_prog_tags = u64_to_user_ptr(info.prog_tags);
4937 ulen = min_t(u32, info.nr_prog_tags, ulen);
4938 if (prog->aux->func_cnt) {
4939 for (i = 0; i < ulen; i++) {
4940 if (copy_to_user(user_prog_tags[i],
4941 prog->aux->func[i]->tag,
4942 BPF_TAG_SIZE))
4943 return -EFAULT;
4944 }
4945 } else {
4946 if (copy_to_user(user_prog_tags[0],
4947 prog->tag, BPF_TAG_SIZE))
4948 return -EFAULT;
4949 }
4950 }
4951
4952done:
4953 if (copy_to_user(uinfo, &info, info_len) ||
4954 put_user(info_len, &uattr->info.info_len))
4955 return -EFAULT;
4956
4957 return 0;
4958}
4959
4960static int bpf_map_get_info_by_fd(struct file *file,
4961 struct bpf_map *map,
4962 const union bpf_attr *attr,
4963 union bpf_attr __user *uattr)
4964{
4965 struct bpf_map_info __user *uinfo = u64_to_user_ptr(attr->info.info);
4966 struct bpf_map_info info;
4967 u32 info_len = attr->info.info_len;
4968 int err;
4969
4970 err = bpf_check_uarg_tail_zero(USER_BPFPTR(uinfo), sizeof(info), info_len);
4971 if (err)
4972 return err;
4973 info_len = min_t(u32, sizeof(info), info_len);
4974
4975 memset(&info, 0, sizeof(info));
4976 info.type = map->map_type;
4977 info.id = map->id;
4978 info.key_size = map->key_size;
4979 info.value_size = map->value_size;
4980 info.max_entries = map->max_entries;
4981 info.map_flags = map->map_flags;
4982 info.map_extra = map->map_extra;
4983 memcpy(info.name, map->name, sizeof(map->name));
4984
4985 if (map->btf) {
4986 info.btf_id = btf_obj_id(map->btf);
4987 info.btf_key_type_id = map->btf_key_type_id;
4988 info.btf_value_type_id = map->btf_value_type_id;
4989 }
4990 info.btf_vmlinux_value_type_id = map->btf_vmlinux_value_type_id;
4991 if (map->map_type == BPF_MAP_TYPE_STRUCT_OPS)
4992 bpf_map_struct_ops_info_fill(&info, map);
4993
4994 if (bpf_map_is_offloaded(map)) {
4995 err = bpf_map_offload_info_fill(&info, map);
4996 if (err)
4997 return err;
4998 }
4999
5000 if (copy_to_user(uinfo, &info, info_len) ||
5001 put_user(info_len, &uattr->info.info_len))
5002 return -EFAULT;
5003
5004 return 0;
5005}
5006
5007static int bpf_btf_get_info_by_fd(struct file *file,
5008 struct btf *btf,
5009 const union bpf_attr *attr,
5010 union bpf_attr __user *uattr)
5011{
5012 struct bpf_btf_info __user *uinfo = u64_to_user_ptr(attr->info.info);
5013 u32 info_len = attr->info.info_len;
5014 int err;
5015
5016 err = bpf_check_uarg_tail_zero(USER_BPFPTR(uinfo), sizeof(*uinfo), info_len);
5017 if (err)
5018 return err;
5019
5020 return btf_get_info_by_fd(btf, attr, uattr);
5021}
5022
5023static int bpf_link_get_info_by_fd(struct file *file,
5024 struct bpf_link *link,
5025 const union bpf_attr *attr,
5026 union bpf_attr __user *uattr)
5027{
5028 struct bpf_link_info __user *uinfo = u64_to_user_ptr(attr->info.info);
5029 struct bpf_link_info info;
5030 u32 info_len = attr->info.info_len;
5031 int err;
5032
5033 err = bpf_check_uarg_tail_zero(USER_BPFPTR(uinfo), sizeof(info), info_len);
5034 if (err)
5035 return err;
5036 info_len = min_t(u32, sizeof(info), info_len);
5037
5038 memset(&info, 0, sizeof(info));
5039 if (copy_from_user(&info, uinfo, info_len))
5040 return -EFAULT;
5041
5042 info.type = link->type;
5043 info.id = link->id;
5044 if (link->prog)
5045 info.prog_id = link->prog->aux->id;
5046
5047 if (link->ops->fill_link_info) {
5048 err = link->ops->fill_link_info(link, &info);
5049 if (err)
5050 return err;
5051 }
5052
5053 if (copy_to_user(uinfo, &info, info_len) ||
5054 put_user(info_len, &uattr->info.info_len))
5055 return -EFAULT;
5056
5057 return 0;
5058}
5059
5060
5061#define BPF_OBJ_GET_INFO_BY_FD_LAST_FIELD info.info
5062
5063static int bpf_obj_get_info_by_fd(const union bpf_attr *attr,
5064 union bpf_attr __user *uattr)
5065{
5066 if (CHECK_ATTR(BPF_OBJ_GET_INFO_BY_FD))
5067 return -EINVAL;
5068
5069 CLASS(fd, f)(attr->info.bpf_fd);
5070 if (fd_empty(f))
5071 return -EBADFD;
5072
5073 if (fd_file(f)->f_op == &bpf_prog_fops)
5074 return bpf_prog_get_info_by_fd(fd_file(f), fd_file(f)->private_data, attr,
5075 uattr);
5076 else if (fd_file(f)->f_op == &bpf_map_fops)
5077 return bpf_map_get_info_by_fd(fd_file(f), fd_file(f)->private_data, attr,
5078 uattr);
5079 else if (fd_file(f)->f_op == &btf_fops)
5080 return bpf_btf_get_info_by_fd(fd_file(f), fd_file(f)->private_data, attr, uattr);
5081 else if (fd_file(f)->f_op == &bpf_link_fops || fd_file(f)->f_op == &bpf_link_fops_poll)
5082 return bpf_link_get_info_by_fd(fd_file(f), fd_file(f)->private_data,
5083 attr, uattr);
5084 return -EINVAL;
5085}
5086
5087#define BPF_BTF_LOAD_LAST_FIELD btf_token_fd
5088
5089static int bpf_btf_load(const union bpf_attr *attr, bpfptr_t uattr, __u32 uattr_size)
5090{
5091 struct bpf_token *token = NULL;
5092
5093 if (CHECK_ATTR(BPF_BTF_LOAD))
5094 return -EINVAL;
5095
5096 if (attr->btf_flags & ~BPF_F_TOKEN_FD)
5097 return -EINVAL;
5098
5099 if (attr->btf_flags & BPF_F_TOKEN_FD) {
5100 token = bpf_token_get_from_fd(attr->btf_token_fd);
5101 if (IS_ERR(token))
5102 return PTR_ERR(token);
5103 if (!bpf_token_allow_cmd(token, BPF_BTF_LOAD)) {
5104 bpf_token_put(token);
5105 token = NULL;
5106 }
5107 }
5108
5109 if (!bpf_token_capable(token, CAP_BPF)) {
5110 bpf_token_put(token);
5111 return -EPERM;
5112 }
5113
5114 bpf_token_put(token);
5115
5116 return btf_new_fd(attr, uattr, uattr_size);
5117}
5118
5119#define BPF_BTF_GET_FD_BY_ID_LAST_FIELD btf_id
5120
5121static int bpf_btf_get_fd_by_id(const union bpf_attr *attr)
5122{
5123 if (CHECK_ATTR(BPF_BTF_GET_FD_BY_ID))
5124 return -EINVAL;
5125
5126 if (!capable(CAP_SYS_ADMIN))
5127 return -EPERM;
5128
5129 return btf_get_fd_by_id(attr->btf_id);
5130}
5131
5132static int bpf_task_fd_query_copy(const union bpf_attr *attr,
5133 union bpf_attr __user *uattr,
5134 u32 prog_id, u32 fd_type,
5135 const char *buf, u64 probe_offset,
5136 u64 probe_addr)
5137{
5138 char __user *ubuf = u64_to_user_ptr(attr->task_fd_query.buf);
5139 u32 len = buf ? strlen(buf) : 0, input_len;
5140 int err = 0;
5141
5142 if (put_user(len, &uattr->task_fd_query.buf_len))
5143 return -EFAULT;
5144 input_len = attr->task_fd_query.buf_len;
5145 if (input_len && ubuf) {
5146 if (!len) {
5147 /* nothing to copy, just make ubuf NULL terminated */
5148 char zero = '\0';
5149
5150 if (put_user(zero, ubuf))
5151 return -EFAULT;
5152 } else if (input_len >= len + 1) {
5153 /* ubuf can hold the string with NULL terminator */
5154 if (copy_to_user(ubuf, buf, len + 1))
5155 return -EFAULT;
5156 } else {
5157 /* ubuf cannot hold the string with NULL terminator,
5158 * do a partial copy with NULL terminator.
5159 */
5160 char zero = '\0';
5161
5162 err = -ENOSPC;
5163 if (copy_to_user(ubuf, buf, input_len - 1))
5164 return -EFAULT;
5165 if (put_user(zero, ubuf + input_len - 1))
5166 return -EFAULT;
5167 }
5168 }
5169
5170 if (put_user(prog_id, &uattr->task_fd_query.prog_id) ||
5171 put_user(fd_type, &uattr->task_fd_query.fd_type) ||
5172 put_user(probe_offset, &uattr->task_fd_query.probe_offset) ||
5173 put_user(probe_addr, &uattr->task_fd_query.probe_addr))
5174 return -EFAULT;
5175
5176 return err;
5177}
5178
5179#define BPF_TASK_FD_QUERY_LAST_FIELD task_fd_query.probe_addr
5180
5181static int bpf_task_fd_query(const union bpf_attr *attr,
5182 union bpf_attr __user *uattr)
5183{
5184 pid_t pid = attr->task_fd_query.pid;
5185 u32 fd = attr->task_fd_query.fd;
5186 const struct perf_event *event;
5187 struct task_struct *task;
5188 struct file *file;
5189 int err;
5190
5191 if (CHECK_ATTR(BPF_TASK_FD_QUERY))
5192 return -EINVAL;
5193
5194 if (!capable(CAP_SYS_ADMIN))
5195 return -EPERM;
5196
5197 if (attr->task_fd_query.flags != 0)
5198 return -EINVAL;
5199
5200 rcu_read_lock();
5201 task = get_pid_task(find_vpid(pid), PIDTYPE_PID);
5202 rcu_read_unlock();
5203 if (!task)
5204 return -ENOENT;
5205
5206 err = 0;
5207 file = fget_task(task, fd);
5208 put_task_struct(task);
5209 if (!file)
5210 return -EBADF;
5211
5212 if (file->f_op == &bpf_link_fops || file->f_op == &bpf_link_fops_poll) {
5213 struct bpf_link *link = file->private_data;
5214
5215 if (link->ops == &bpf_raw_tp_link_lops) {
5216 struct bpf_raw_tp_link *raw_tp =
5217 container_of(link, struct bpf_raw_tp_link, link);
5218 struct bpf_raw_event_map *btp = raw_tp->btp;
5219
5220 err = bpf_task_fd_query_copy(attr, uattr,
5221 raw_tp->link.prog->aux->id,
5222 BPF_FD_TYPE_RAW_TRACEPOINT,
5223 btp->tp->name, 0, 0);
5224 goto put_file;
5225 }
5226 goto out_not_supp;
5227 }
5228
5229 event = perf_get_event(file);
5230 if (!IS_ERR(event)) {
5231 u64 probe_offset, probe_addr;
5232 u32 prog_id, fd_type;
5233 const char *buf;
5234
5235 err = bpf_get_perf_event_info(event, &prog_id, &fd_type,
5236 &buf, &probe_offset,
5237 &probe_addr, NULL);
5238 if (!err)
5239 err = bpf_task_fd_query_copy(attr, uattr, prog_id,
5240 fd_type, buf,
5241 probe_offset,
5242 probe_addr);
5243 goto put_file;
5244 }
5245
5246out_not_supp:
5247 err = -ENOTSUPP;
5248put_file:
5249 fput(file);
5250 return err;
5251}
5252
5253#define BPF_MAP_BATCH_LAST_FIELD batch.flags
5254
5255#define BPF_DO_BATCH(fn, ...) \
5256 do { \
5257 if (!fn) { \
5258 err = -ENOTSUPP; \
5259 goto err_put; \
5260 } \
5261 err = fn(__VA_ARGS__); \
5262 } while (0)
5263
5264static int bpf_map_do_batch(const union bpf_attr *attr,
5265 union bpf_attr __user *uattr,
5266 int cmd)
5267{
5268 bool has_read = cmd == BPF_MAP_LOOKUP_BATCH ||
5269 cmd == BPF_MAP_LOOKUP_AND_DELETE_BATCH;
5270 bool has_write = cmd != BPF_MAP_LOOKUP_BATCH;
5271 struct bpf_map *map;
5272 int err;
5273
5274 if (CHECK_ATTR(BPF_MAP_BATCH))
5275 return -EINVAL;
5276
5277 CLASS(fd, f)(attr->batch.map_fd);
5278
5279 map = __bpf_map_get(f);
5280 if (IS_ERR(map))
5281 return PTR_ERR(map);
5282 if (has_write)
5283 bpf_map_write_active_inc(map);
5284 if (has_read && !(map_get_sys_perms(map, f) & FMODE_CAN_READ)) {
5285 err = -EPERM;
5286 goto err_put;
5287 }
5288 if (has_write && !(map_get_sys_perms(map, f) & FMODE_CAN_WRITE)) {
5289 err = -EPERM;
5290 goto err_put;
5291 }
5292
5293 if (cmd == BPF_MAP_LOOKUP_BATCH)
5294 BPF_DO_BATCH(map->ops->map_lookup_batch, map, attr, uattr);
5295 else if (cmd == BPF_MAP_LOOKUP_AND_DELETE_BATCH)
5296 BPF_DO_BATCH(map->ops->map_lookup_and_delete_batch, map, attr, uattr);
5297 else if (cmd == BPF_MAP_UPDATE_BATCH)
5298 BPF_DO_BATCH(map->ops->map_update_batch, map, fd_file(f), attr, uattr);
5299 else
5300 BPF_DO_BATCH(map->ops->map_delete_batch, map, attr, uattr);
5301err_put:
5302 if (has_write) {
5303 maybe_wait_bpf_programs(map);
5304 bpf_map_write_active_dec(map);
5305 }
5306 return err;
5307}
5308
5309#define BPF_LINK_CREATE_LAST_FIELD link_create.uprobe_multi.pid
5310static int link_create(union bpf_attr *attr, bpfptr_t uattr)
5311{
5312 struct bpf_prog *prog;
5313 int ret;
5314
5315 if (CHECK_ATTR(BPF_LINK_CREATE))
5316 return -EINVAL;
5317
5318 if (attr->link_create.attach_type == BPF_STRUCT_OPS)
5319 return bpf_struct_ops_link_create(attr);
5320
5321 prog = bpf_prog_get(attr->link_create.prog_fd);
5322 if (IS_ERR(prog))
5323 return PTR_ERR(prog);
5324
5325 ret = bpf_prog_attach_check_attach_type(prog,
5326 attr->link_create.attach_type);
5327 if (ret)
5328 goto out;
5329
5330 switch (prog->type) {
5331 case BPF_PROG_TYPE_CGROUP_SKB:
5332 case BPF_PROG_TYPE_CGROUP_SOCK:
5333 case BPF_PROG_TYPE_CGROUP_SOCK_ADDR:
5334 case BPF_PROG_TYPE_SOCK_OPS:
5335 case BPF_PROG_TYPE_CGROUP_DEVICE:
5336 case BPF_PROG_TYPE_CGROUP_SYSCTL:
5337 case BPF_PROG_TYPE_CGROUP_SOCKOPT:
5338 ret = cgroup_bpf_link_attach(attr, prog);
5339 break;
5340 case BPF_PROG_TYPE_EXT:
5341 ret = bpf_tracing_prog_attach(prog,
5342 attr->link_create.target_fd,
5343 attr->link_create.target_btf_id,
5344 attr->link_create.tracing.cookie);
5345 break;
5346 case BPF_PROG_TYPE_LSM:
5347 case BPF_PROG_TYPE_TRACING:
5348 if (attr->link_create.attach_type != prog->expected_attach_type) {
5349 ret = -EINVAL;
5350 goto out;
5351 }
5352 if (prog->expected_attach_type == BPF_TRACE_RAW_TP)
5353 ret = bpf_raw_tp_link_attach(prog, NULL, attr->link_create.tracing.cookie);
5354 else if (prog->expected_attach_type == BPF_TRACE_ITER)
5355 ret = bpf_iter_link_attach(attr, uattr, prog);
5356 else if (prog->expected_attach_type == BPF_LSM_CGROUP)
5357 ret = cgroup_bpf_link_attach(attr, prog);
5358 else
5359 ret = bpf_tracing_prog_attach(prog,
5360 attr->link_create.target_fd,
5361 attr->link_create.target_btf_id,
5362 attr->link_create.tracing.cookie);
5363 break;
5364 case BPF_PROG_TYPE_FLOW_DISSECTOR:
5365 case BPF_PROG_TYPE_SK_LOOKUP:
5366 ret = netns_bpf_link_create(attr, prog);
5367 break;
5368 case BPF_PROG_TYPE_SK_MSG:
5369 case BPF_PROG_TYPE_SK_SKB:
5370 ret = sock_map_link_create(attr, prog);
5371 break;
5372#ifdef CONFIG_NET
5373 case BPF_PROG_TYPE_XDP:
5374 ret = bpf_xdp_link_attach(attr, prog);
5375 break;
5376 case BPF_PROG_TYPE_SCHED_CLS:
5377 if (attr->link_create.attach_type == BPF_TCX_INGRESS ||
5378 attr->link_create.attach_type == BPF_TCX_EGRESS)
5379 ret = tcx_link_attach(attr, prog);
5380 else
5381 ret = netkit_link_attach(attr, prog);
5382 break;
5383 case BPF_PROG_TYPE_NETFILTER:
5384 ret = bpf_nf_link_attach(attr, prog);
5385 break;
5386#endif
5387 case BPF_PROG_TYPE_PERF_EVENT:
5388 case BPF_PROG_TYPE_TRACEPOINT:
5389 ret = bpf_perf_link_attach(attr, prog);
5390 break;
5391 case BPF_PROG_TYPE_KPROBE:
5392 if (attr->link_create.attach_type == BPF_PERF_EVENT)
5393 ret = bpf_perf_link_attach(attr, prog);
5394 else if (attr->link_create.attach_type == BPF_TRACE_KPROBE_MULTI ||
5395 attr->link_create.attach_type == BPF_TRACE_KPROBE_SESSION)
5396 ret = bpf_kprobe_multi_link_attach(attr, prog);
5397 else if (attr->link_create.attach_type == BPF_TRACE_UPROBE_MULTI ||
5398 attr->link_create.attach_type == BPF_TRACE_UPROBE_SESSION)
5399 ret = bpf_uprobe_multi_link_attach(attr, prog);
5400 break;
5401 default:
5402 ret = -EINVAL;
5403 }
5404
5405out:
5406 if (ret < 0)
5407 bpf_prog_put(prog);
5408 return ret;
5409}
5410
5411static int link_update_map(struct bpf_link *link, union bpf_attr *attr)
5412{
5413 struct bpf_map *new_map, *old_map = NULL;
5414 int ret;
5415
5416 new_map = bpf_map_get(attr->link_update.new_map_fd);
5417 if (IS_ERR(new_map))
5418 return PTR_ERR(new_map);
5419
5420 if (attr->link_update.flags & BPF_F_REPLACE) {
5421 old_map = bpf_map_get(attr->link_update.old_map_fd);
5422 if (IS_ERR(old_map)) {
5423 ret = PTR_ERR(old_map);
5424 goto out_put;
5425 }
5426 } else if (attr->link_update.old_map_fd) {
5427 ret = -EINVAL;
5428 goto out_put;
5429 }
5430
5431 ret = link->ops->update_map(link, new_map, old_map);
5432
5433 if (old_map)
5434 bpf_map_put(old_map);
5435out_put:
5436 bpf_map_put(new_map);
5437 return ret;
5438}
5439
5440#define BPF_LINK_UPDATE_LAST_FIELD link_update.old_prog_fd
5441
5442static int link_update(union bpf_attr *attr)
5443{
5444 struct bpf_prog *old_prog = NULL, *new_prog;
5445 struct bpf_link *link;
5446 u32 flags;
5447 int ret;
5448
5449 if (CHECK_ATTR(BPF_LINK_UPDATE))
5450 return -EINVAL;
5451
5452 flags = attr->link_update.flags;
5453 if (flags & ~BPF_F_REPLACE)
5454 return -EINVAL;
5455
5456 link = bpf_link_get_from_fd(attr->link_update.link_fd);
5457 if (IS_ERR(link))
5458 return PTR_ERR(link);
5459
5460 if (link->ops->update_map) {
5461 ret = link_update_map(link, attr);
5462 goto out_put_link;
5463 }
5464
5465 new_prog = bpf_prog_get(attr->link_update.new_prog_fd);
5466 if (IS_ERR(new_prog)) {
5467 ret = PTR_ERR(new_prog);
5468 goto out_put_link;
5469 }
5470
5471 if (flags & BPF_F_REPLACE) {
5472 old_prog = bpf_prog_get(attr->link_update.old_prog_fd);
5473 if (IS_ERR(old_prog)) {
5474 ret = PTR_ERR(old_prog);
5475 old_prog = NULL;
5476 goto out_put_progs;
5477 }
5478 } else if (attr->link_update.old_prog_fd) {
5479 ret = -EINVAL;
5480 goto out_put_progs;
5481 }
5482
5483 if (link->ops->update_prog)
5484 ret = link->ops->update_prog(link, new_prog, old_prog);
5485 else
5486 ret = -EINVAL;
5487
5488out_put_progs:
5489 if (old_prog)
5490 bpf_prog_put(old_prog);
5491 if (ret)
5492 bpf_prog_put(new_prog);
5493out_put_link:
5494 bpf_link_put_direct(link);
5495 return ret;
5496}
5497
5498#define BPF_LINK_DETACH_LAST_FIELD link_detach.link_fd
5499
5500static int link_detach(union bpf_attr *attr)
5501{
5502 struct bpf_link *link;
5503 int ret;
5504
5505 if (CHECK_ATTR(BPF_LINK_DETACH))
5506 return -EINVAL;
5507
5508 link = bpf_link_get_from_fd(attr->link_detach.link_fd);
5509 if (IS_ERR(link))
5510 return PTR_ERR(link);
5511
5512 if (link->ops->detach)
5513 ret = link->ops->detach(link);
5514 else
5515 ret = -EOPNOTSUPP;
5516
5517 bpf_link_put_direct(link);
5518 return ret;
5519}
5520
5521struct bpf_link *bpf_link_inc_not_zero(struct bpf_link *link)
5522{
5523 return atomic64_fetch_add_unless(&link->refcnt, 1, 0) ? link : ERR_PTR(-ENOENT);
5524}
5525EXPORT_SYMBOL(bpf_link_inc_not_zero);
5526
5527struct bpf_link *bpf_link_by_id(u32 id)
5528{
5529 struct bpf_link *link;
5530
5531 if (!id)
5532 return ERR_PTR(-ENOENT);
5533
5534 spin_lock_bh(&link_idr_lock);
5535 /* before link is "settled", ID is 0, pretend it doesn't exist yet */
5536 link = idr_find(&link_idr, id);
5537 if (link) {
5538 if (link->id)
5539 link = bpf_link_inc_not_zero(link);
5540 else
5541 link = ERR_PTR(-EAGAIN);
5542 } else {
5543 link = ERR_PTR(-ENOENT);
5544 }
5545 spin_unlock_bh(&link_idr_lock);
5546 return link;
5547}
5548
5549struct bpf_link *bpf_link_get_curr_or_next(u32 *id)
5550{
5551 struct bpf_link *link;
5552
5553 spin_lock_bh(&link_idr_lock);
5554again:
5555 link = idr_get_next(&link_idr, id);
5556 if (link) {
5557 link = bpf_link_inc_not_zero(link);
5558 if (IS_ERR(link)) {
5559 (*id)++;
5560 goto again;
5561 }
5562 }
5563 spin_unlock_bh(&link_idr_lock);
5564
5565 return link;
5566}
5567
5568#define BPF_LINK_GET_FD_BY_ID_LAST_FIELD link_id
5569
5570static int bpf_link_get_fd_by_id(const union bpf_attr *attr)
5571{
5572 struct bpf_link *link;
5573 u32 id = attr->link_id;
5574 int fd;
5575
5576 if (CHECK_ATTR(BPF_LINK_GET_FD_BY_ID))
5577 return -EINVAL;
5578
5579 if (!capable(CAP_SYS_ADMIN))
5580 return -EPERM;
5581
5582 link = bpf_link_by_id(id);
5583 if (IS_ERR(link))
5584 return PTR_ERR(link);
5585
5586 fd = bpf_link_new_fd(link);
5587 if (fd < 0)
5588 bpf_link_put_direct(link);
5589
5590 return fd;
5591}
5592
5593DEFINE_MUTEX(bpf_stats_enabled_mutex);
5594
5595static int bpf_stats_release(struct inode *inode, struct file *file)
5596{
5597 mutex_lock(&bpf_stats_enabled_mutex);
5598 static_key_slow_dec(&bpf_stats_enabled_key.key);
5599 mutex_unlock(&bpf_stats_enabled_mutex);
5600 return 0;
5601}
5602
5603static const struct file_operations bpf_stats_fops = {
5604 .release = bpf_stats_release,
5605};
5606
5607static int bpf_enable_runtime_stats(void)
5608{
5609 int fd;
5610
5611 mutex_lock(&bpf_stats_enabled_mutex);
5612
5613 /* Set a very high limit to avoid overflow */
5614 if (static_key_count(&bpf_stats_enabled_key.key) > INT_MAX / 2) {
5615 mutex_unlock(&bpf_stats_enabled_mutex);
5616 return -EBUSY;
5617 }
5618
5619 fd = anon_inode_getfd("bpf-stats", &bpf_stats_fops, NULL, O_CLOEXEC);
5620 if (fd >= 0)
5621 static_key_slow_inc(&bpf_stats_enabled_key.key);
5622
5623 mutex_unlock(&bpf_stats_enabled_mutex);
5624 return fd;
5625}
5626
5627#define BPF_ENABLE_STATS_LAST_FIELD enable_stats.type
5628
5629static int bpf_enable_stats(union bpf_attr *attr)
5630{
5631
5632 if (CHECK_ATTR(BPF_ENABLE_STATS))
5633 return -EINVAL;
5634
5635 if (!capable(CAP_SYS_ADMIN))
5636 return -EPERM;
5637
5638 switch (attr->enable_stats.type) {
5639 case BPF_STATS_RUN_TIME:
5640 return bpf_enable_runtime_stats();
5641 default:
5642 break;
5643 }
5644 return -EINVAL;
5645}
5646
5647#define BPF_ITER_CREATE_LAST_FIELD iter_create.flags
5648
5649static int bpf_iter_create(union bpf_attr *attr)
5650{
5651 struct bpf_link *link;
5652 int err;
5653
5654 if (CHECK_ATTR(BPF_ITER_CREATE))
5655 return -EINVAL;
5656
5657 if (attr->iter_create.flags)
5658 return -EINVAL;
5659
5660 link = bpf_link_get_from_fd(attr->iter_create.link_fd);
5661 if (IS_ERR(link))
5662 return PTR_ERR(link);
5663
5664 err = bpf_iter_new_fd(link);
5665 bpf_link_put_direct(link);
5666
5667 return err;
5668}
5669
5670#define BPF_PROG_BIND_MAP_LAST_FIELD prog_bind_map.flags
5671
5672static int bpf_prog_bind_map(union bpf_attr *attr)
5673{
5674 struct bpf_prog *prog;
5675 struct bpf_map *map;
5676 struct bpf_map **used_maps_old, **used_maps_new;
5677 int i, ret = 0;
5678
5679 if (CHECK_ATTR(BPF_PROG_BIND_MAP))
5680 return -EINVAL;
5681
5682 if (attr->prog_bind_map.flags)
5683 return -EINVAL;
5684
5685 prog = bpf_prog_get(attr->prog_bind_map.prog_fd);
5686 if (IS_ERR(prog))
5687 return PTR_ERR(prog);
5688
5689 map = bpf_map_get(attr->prog_bind_map.map_fd);
5690 if (IS_ERR(map)) {
5691 ret = PTR_ERR(map);
5692 goto out_prog_put;
5693 }
5694
5695 mutex_lock(&prog->aux->used_maps_mutex);
5696
5697 used_maps_old = prog->aux->used_maps;
5698
5699 for (i = 0; i < prog->aux->used_map_cnt; i++)
5700 if (used_maps_old[i] == map) {
5701 bpf_map_put(map);
5702 goto out_unlock;
5703 }
5704
5705 used_maps_new = kmalloc_array(prog->aux->used_map_cnt + 1,
5706 sizeof(used_maps_new[0]),
5707 GFP_KERNEL);
5708 if (!used_maps_new) {
5709 ret = -ENOMEM;
5710 goto out_unlock;
5711 }
5712
5713 /* The bpf program will not access the bpf map, but for the sake of
5714 * simplicity, increase sleepable_refcnt for sleepable program as well.
5715 */
5716 if (prog->sleepable)
5717 atomic64_inc(&map->sleepable_refcnt);
5718 memcpy(used_maps_new, used_maps_old,
5719 sizeof(used_maps_old[0]) * prog->aux->used_map_cnt);
5720 used_maps_new[prog->aux->used_map_cnt] = map;
5721
5722 prog->aux->used_map_cnt++;
5723 prog->aux->used_maps = used_maps_new;
5724
5725 kfree(used_maps_old);
5726
5727out_unlock:
5728 mutex_unlock(&prog->aux->used_maps_mutex);
5729
5730 if (ret)
5731 bpf_map_put(map);
5732out_prog_put:
5733 bpf_prog_put(prog);
5734 return ret;
5735}
5736
5737#define BPF_TOKEN_CREATE_LAST_FIELD token_create.bpffs_fd
5738
5739static int token_create(union bpf_attr *attr)
5740{
5741 if (CHECK_ATTR(BPF_TOKEN_CREATE))
5742 return -EINVAL;
5743
5744 /* no flags are supported yet */
5745 if (attr->token_create.flags)
5746 return -EINVAL;
5747
5748 return bpf_token_create(attr);
5749}
5750
5751static int __sys_bpf(enum bpf_cmd cmd, bpfptr_t uattr, unsigned int size)
5752{
5753 union bpf_attr attr;
5754 int err;
5755
5756 err = bpf_check_uarg_tail_zero(uattr, sizeof(attr), size);
5757 if (err)
5758 return err;
5759 size = min_t(u32, size, sizeof(attr));
5760
5761 /* copy attributes from user space, may be less than sizeof(bpf_attr) */
5762 memset(&attr, 0, sizeof(attr));
5763 if (copy_from_bpfptr(&attr, uattr, size) != 0)
5764 return -EFAULT;
5765
5766 err = security_bpf(cmd, &attr, size);
5767 if (err < 0)
5768 return err;
5769
5770 switch (cmd) {
5771 case BPF_MAP_CREATE:
5772 err = map_create(&attr);
5773 break;
5774 case BPF_MAP_LOOKUP_ELEM:
5775 err = map_lookup_elem(&attr);
5776 break;
5777 case BPF_MAP_UPDATE_ELEM:
5778 err = map_update_elem(&attr, uattr);
5779 break;
5780 case BPF_MAP_DELETE_ELEM:
5781 err = map_delete_elem(&attr, uattr);
5782 break;
5783 case BPF_MAP_GET_NEXT_KEY:
5784 err = map_get_next_key(&attr);
5785 break;
5786 case BPF_MAP_FREEZE:
5787 err = map_freeze(&attr);
5788 break;
5789 case BPF_PROG_LOAD:
5790 err = bpf_prog_load(&attr, uattr, size);
5791 break;
5792 case BPF_OBJ_PIN:
5793 err = bpf_obj_pin(&attr);
5794 break;
5795 case BPF_OBJ_GET:
5796 err = bpf_obj_get(&attr);
5797 break;
5798 case BPF_PROG_ATTACH:
5799 err = bpf_prog_attach(&attr);
5800 break;
5801 case BPF_PROG_DETACH:
5802 err = bpf_prog_detach(&attr);
5803 break;
5804 case BPF_PROG_QUERY:
5805 err = bpf_prog_query(&attr, uattr.user);
5806 break;
5807 case BPF_PROG_TEST_RUN:
5808 err = bpf_prog_test_run(&attr, uattr.user);
5809 break;
5810 case BPF_PROG_GET_NEXT_ID:
5811 err = bpf_obj_get_next_id(&attr, uattr.user,
5812 &prog_idr, &prog_idr_lock);
5813 break;
5814 case BPF_MAP_GET_NEXT_ID:
5815 err = bpf_obj_get_next_id(&attr, uattr.user,
5816 &map_idr, &map_idr_lock);
5817 break;
5818 case BPF_BTF_GET_NEXT_ID:
5819 err = bpf_obj_get_next_id(&attr, uattr.user,
5820 &btf_idr, &btf_idr_lock);
5821 break;
5822 case BPF_PROG_GET_FD_BY_ID:
5823 err = bpf_prog_get_fd_by_id(&attr);
5824 break;
5825 case BPF_MAP_GET_FD_BY_ID:
5826 err = bpf_map_get_fd_by_id(&attr);
5827 break;
5828 case BPF_OBJ_GET_INFO_BY_FD:
5829 err = bpf_obj_get_info_by_fd(&attr, uattr.user);
5830 break;
5831 case BPF_RAW_TRACEPOINT_OPEN:
5832 err = bpf_raw_tracepoint_open(&attr);
5833 break;
5834 case BPF_BTF_LOAD:
5835 err = bpf_btf_load(&attr, uattr, size);
5836 break;
5837 case BPF_BTF_GET_FD_BY_ID:
5838 err = bpf_btf_get_fd_by_id(&attr);
5839 break;
5840 case BPF_TASK_FD_QUERY:
5841 err = bpf_task_fd_query(&attr, uattr.user);
5842 break;
5843 case BPF_MAP_LOOKUP_AND_DELETE_ELEM:
5844 err = map_lookup_and_delete_elem(&attr);
5845 break;
5846 case BPF_MAP_LOOKUP_BATCH:
5847 err = bpf_map_do_batch(&attr, uattr.user, BPF_MAP_LOOKUP_BATCH);
5848 break;
5849 case BPF_MAP_LOOKUP_AND_DELETE_BATCH:
5850 err = bpf_map_do_batch(&attr, uattr.user,
5851 BPF_MAP_LOOKUP_AND_DELETE_BATCH);
5852 break;
5853 case BPF_MAP_UPDATE_BATCH:
5854 err = bpf_map_do_batch(&attr, uattr.user, BPF_MAP_UPDATE_BATCH);
5855 break;
5856 case BPF_MAP_DELETE_BATCH:
5857 err = bpf_map_do_batch(&attr, uattr.user, BPF_MAP_DELETE_BATCH);
5858 break;
5859 case BPF_LINK_CREATE:
5860 err = link_create(&attr, uattr);
5861 break;
5862 case BPF_LINK_UPDATE:
5863 err = link_update(&attr);
5864 break;
5865 case BPF_LINK_GET_FD_BY_ID:
5866 err = bpf_link_get_fd_by_id(&attr);
5867 break;
5868 case BPF_LINK_GET_NEXT_ID:
5869 err = bpf_obj_get_next_id(&attr, uattr.user,
5870 &link_idr, &link_idr_lock);
5871 break;
5872 case BPF_ENABLE_STATS:
5873 err = bpf_enable_stats(&attr);
5874 break;
5875 case BPF_ITER_CREATE:
5876 err = bpf_iter_create(&attr);
5877 break;
5878 case BPF_LINK_DETACH:
5879 err = link_detach(&attr);
5880 break;
5881 case BPF_PROG_BIND_MAP:
5882 err = bpf_prog_bind_map(&attr);
5883 break;
5884 case BPF_TOKEN_CREATE:
5885 err = token_create(&attr);
5886 break;
5887 default:
5888 err = -EINVAL;
5889 break;
5890 }
5891
5892 return err;
5893}
5894
5895SYSCALL_DEFINE3(bpf, int, cmd, union bpf_attr __user *, uattr, unsigned int, size)
5896{
5897 return __sys_bpf(cmd, USER_BPFPTR(uattr), size);
5898}
5899
5900static bool syscall_prog_is_valid_access(int off, int size,
5901 enum bpf_access_type type,
5902 const struct bpf_prog *prog,
5903 struct bpf_insn_access_aux *info)
5904{
5905 if (off < 0 || off >= U16_MAX)
5906 return false;
5907 if (off % size != 0)
5908 return false;
5909 return true;
5910}
5911
5912BPF_CALL_3(bpf_sys_bpf, int, cmd, union bpf_attr *, attr, u32, attr_size)
5913{
5914 switch (cmd) {
5915 case BPF_MAP_CREATE:
5916 case BPF_MAP_DELETE_ELEM:
5917 case BPF_MAP_UPDATE_ELEM:
5918 case BPF_MAP_FREEZE:
5919 case BPF_MAP_GET_FD_BY_ID:
5920 case BPF_PROG_LOAD:
5921 case BPF_BTF_LOAD:
5922 case BPF_LINK_CREATE:
5923 case BPF_RAW_TRACEPOINT_OPEN:
5924 break;
5925 default:
5926 return -EINVAL;
5927 }
5928 return __sys_bpf(cmd, KERNEL_BPFPTR(attr), attr_size);
5929}
5930
5931
5932/* To shut up -Wmissing-prototypes.
5933 * This function is used by the kernel light skeleton
5934 * to load bpf programs when modules are loaded or during kernel boot.
5935 * See tools/lib/bpf/skel_internal.h
5936 */
5937int kern_sys_bpf(int cmd, union bpf_attr *attr, unsigned int size);
5938
5939int kern_sys_bpf(int cmd, union bpf_attr *attr, unsigned int size)
5940{
5941 struct bpf_prog * __maybe_unused prog;
5942 struct bpf_tramp_run_ctx __maybe_unused run_ctx;
5943
5944 switch (cmd) {
5945#ifdef CONFIG_BPF_JIT /* __bpf_prog_enter_sleepable used by trampoline and JIT */
5946 case BPF_PROG_TEST_RUN:
5947 if (attr->test.data_in || attr->test.data_out ||
5948 attr->test.ctx_out || attr->test.duration ||
5949 attr->test.repeat || attr->test.flags)
5950 return -EINVAL;
5951
5952 prog = bpf_prog_get_type(attr->test.prog_fd, BPF_PROG_TYPE_SYSCALL);
5953 if (IS_ERR(prog))
5954 return PTR_ERR(prog);
5955
5956 if (attr->test.ctx_size_in < prog->aux->max_ctx_offset ||
5957 attr->test.ctx_size_in > U16_MAX) {
5958 bpf_prog_put(prog);
5959 return -EINVAL;
5960 }
5961
5962 run_ctx.bpf_cookie = 0;
5963 if (!__bpf_prog_enter_sleepable_recur(prog, &run_ctx)) {
5964 /* recursion detected */
5965 __bpf_prog_exit_sleepable_recur(prog, 0, &run_ctx);
5966 bpf_prog_put(prog);
5967 return -EBUSY;
5968 }
5969 attr->test.retval = bpf_prog_run(prog, (void *) (long) attr->test.ctx_in);
5970 __bpf_prog_exit_sleepable_recur(prog, 0 /* bpf_prog_run does runtime stats */,
5971 &run_ctx);
5972 bpf_prog_put(prog);
5973 return 0;
5974#endif
5975 default:
5976 return ____bpf_sys_bpf(cmd, attr, size);
5977 }
5978}
5979EXPORT_SYMBOL(kern_sys_bpf);
5980
5981static const struct bpf_func_proto bpf_sys_bpf_proto = {
5982 .func = bpf_sys_bpf,
5983 .gpl_only = false,
5984 .ret_type = RET_INTEGER,
5985 .arg1_type = ARG_ANYTHING,
5986 .arg2_type = ARG_PTR_TO_MEM | MEM_RDONLY,
5987 .arg3_type = ARG_CONST_SIZE,
5988};
5989
5990const struct bpf_func_proto * __weak
5991tracing_prog_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog)
5992{
5993 return bpf_base_func_proto(func_id, prog);
5994}
5995
5996BPF_CALL_1(bpf_sys_close, u32, fd)
5997{
5998 /* When bpf program calls this helper there should not be
5999 * an fdget() without matching completed fdput().
6000 * This helper is allowed in the following callchain only:
6001 * sys_bpf->prog_test_run->bpf_prog->bpf_sys_close
6002 */
6003 return close_fd(fd);
6004}
6005
6006static const struct bpf_func_proto bpf_sys_close_proto = {
6007 .func = bpf_sys_close,
6008 .gpl_only = false,
6009 .ret_type = RET_INTEGER,
6010 .arg1_type = ARG_ANYTHING,
6011};
6012
6013BPF_CALL_4(bpf_kallsyms_lookup_name, const char *, name, int, name_sz, int, flags, u64 *, res)
6014{
6015 *res = 0;
6016 if (flags)
6017 return -EINVAL;
6018
6019 if (name_sz <= 1 || name[name_sz - 1])
6020 return -EINVAL;
6021
6022 if (!bpf_dump_raw_ok(current_cred()))
6023 return -EPERM;
6024
6025 *res = kallsyms_lookup_name(name);
6026 return *res ? 0 : -ENOENT;
6027}
6028
6029static const struct bpf_func_proto bpf_kallsyms_lookup_name_proto = {
6030 .func = bpf_kallsyms_lookup_name,
6031 .gpl_only = false,
6032 .ret_type = RET_INTEGER,
6033 .arg1_type = ARG_PTR_TO_MEM,
6034 .arg2_type = ARG_CONST_SIZE_OR_ZERO,
6035 .arg3_type = ARG_ANYTHING,
6036 .arg4_type = ARG_PTR_TO_FIXED_SIZE_MEM | MEM_UNINIT | MEM_WRITE | MEM_ALIGNED,
6037 .arg4_size = sizeof(u64),
6038};
6039
6040static const struct bpf_func_proto *
6041syscall_prog_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog)
6042{
6043 switch (func_id) {
6044 case BPF_FUNC_sys_bpf:
6045 return !bpf_token_capable(prog->aux->token, CAP_PERFMON)
6046 ? NULL : &bpf_sys_bpf_proto;
6047 case BPF_FUNC_btf_find_by_name_kind:
6048 return &bpf_btf_find_by_name_kind_proto;
6049 case BPF_FUNC_sys_close:
6050 return &bpf_sys_close_proto;
6051 case BPF_FUNC_kallsyms_lookup_name:
6052 return &bpf_kallsyms_lookup_name_proto;
6053 default:
6054 return tracing_prog_func_proto(func_id, prog);
6055 }
6056}
6057
6058const struct bpf_verifier_ops bpf_syscall_verifier_ops = {
6059 .get_func_proto = syscall_prog_func_proto,
6060 .is_valid_access = syscall_prog_is_valid_access,
6061};
6062
6063const struct bpf_prog_ops bpf_syscall_prog_ops = {
6064 .test_run = bpf_prog_test_run_syscall,
6065};
6066
6067#ifdef CONFIG_SYSCTL
6068static int bpf_stats_handler(const struct ctl_table *table, int write,
6069 void *buffer, size_t *lenp, loff_t *ppos)
6070{
6071 struct static_key *key = (struct static_key *)table->data;
6072 static int saved_val;
6073 int val, ret;
6074 struct ctl_table tmp = {
6075 .data = &val,
6076 .maxlen = sizeof(val),
6077 .mode = table->mode,
6078 .extra1 = SYSCTL_ZERO,
6079 .extra2 = SYSCTL_ONE,
6080 };
6081
6082 if (write && !capable(CAP_SYS_ADMIN))
6083 return -EPERM;
6084
6085 mutex_lock(&bpf_stats_enabled_mutex);
6086 val = saved_val;
6087 ret = proc_dointvec_minmax(&tmp, write, buffer, lenp, ppos);
6088 if (write && !ret && val != saved_val) {
6089 if (val)
6090 static_key_slow_inc(key);
6091 else
6092 static_key_slow_dec(key);
6093 saved_val = val;
6094 }
6095 mutex_unlock(&bpf_stats_enabled_mutex);
6096 return ret;
6097}
6098
6099void __weak unpriv_ebpf_notify(int new_state)
6100{
6101}
6102
6103static int bpf_unpriv_handler(const struct ctl_table *table, int write,
6104 void *buffer, size_t *lenp, loff_t *ppos)
6105{
6106 int ret, unpriv_enable = *(int *)table->data;
6107 bool locked_state = unpriv_enable == 1;
6108 struct ctl_table tmp = *table;
6109
6110 if (write && !capable(CAP_SYS_ADMIN))
6111 return -EPERM;
6112
6113 tmp.data = &unpriv_enable;
6114 ret = proc_dointvec_minmax(&tmp, write, buffer, lenp, ppos);
6115 if (write && !ret) {
6116 if (locked_state && unpriv_enable != 1)
6117 return -EPERM;
6118 *(int *)table->data = unpriv_enable;
6119 }
6120
6121 if (write)
6122 unpriv_ebpf_notify(unpriv_enable);
6123
6124 return ret;
6125}
6126
6127static struct ctl_table bpf_syscall_table[] = {
6128 {
6129 .procname = "unprivileged_bpf_disabled",
6130 .data = &sysctl_unprivileged_bpf_disabled,
6131 .maxlen = sizeof(sysctl_unprivileged_bpf_disabled),
6132 .mode = 0644,
6133 .proc_handler = bpf_unpriv_handler,
6134 .extra1 = SYSCTL_ZERO,
6135 .extra2 = SYSCTL_TWO,
6136 },
6137 {
6138 .procname = "bpf_stats_enabled",
6139 .data = &bpf_stats_enabled_key.key,
6140 .mode = 0644,
6141 .proc_handler = bpf_stats_handler,
6142 },
6143};
6144
6145static int __init bpf_syscall_sysctl_init(void)
6146{
6147 register_sysctl_init("kernel", bpf_syscall_table);
6148 return 0;
6149}
6150late_initcall(bpf_syscall_sysctl_init);
6151#endif /* CONFIG_SYSCTL */