projects/linux·blob master · refs

1569 lines · 41.6 KiB

1// SPDX-License-Identifier: GPL-2.0-only
2/* Copyright (c) 2019 Facebook */
3
4#include <linux/bpf.h>
5#include <linux/bpf_verifier.h>
6#include <linux/btf.h>
7#include <linux/filter.h>
8#include <linux/slab.h>
9#include <linux/numa.h>
10#include <linux/seq_file.h>
11#include <linux/refcount.h>
12#include <linux/mutex.h>
13#include <linux/btf_ids.h>
14#include <linux/rcupdate_wait.h>
15#include <linux/poll.h>
16
17struct bpf_struct_ops_value {
18 struct bpf_struct_ops_common_value common;
19 char data[] ____cacheline_aligned_in_smp;
20};
21
22#define MAX_TRAMP_IMAGE_PAGES 8
23
24struct bpf_struct_ops_map {
25 struct bpf_map map;
26 const struct bpf_struct_ops_desc *st_ops_desc;
27 /* protect map_update */
28 struct mutex lock;
29 /* link has all the bpf_links that is populated
30 * to the func ptr of the kernel's struct
31 * (in kvalue.data).
32 */
33 struct bpf_link **links;
34 /* ksyms for bpf trampolines */
35 struct bpf_ksym **ksyms;
36 u32 funcs_cnt;
37 u32 image_pages_cnt;
38 /* image_pages is an array of pages that has all the trampolines
39 * that stores the func args before calling the bpf_prog.
40 */
41 void *image_pages[MAX_TRAMP_IMAGE_PAGES];
42 /* The owner moduler's btf. */
43 struct btf *btf;
44 /* uvalue->data stores the kernel struct
45 * (e.g. tcp_congestion_ops) that is more useful
46 * to userspace than the kvalue. For example,
47 * the bpf_prog's id is stored instead of the kernel
48 * address of a func ptr.
49 */
50 struct bpf_struct_ops_value *uvalue;
51 /* kvalue.data stores the actual kernel's struct
52 * (e.g. tcp_congestion_ops) that will be
53 * registered to the kernel subsystem.
54 */
55 struct bpf_struct_ops_value kvalue;
56};
57
58struct bpf_struct_ops_link {
59 struct bpf_link link;
60 struct bpf_map __rcu *map;
61 wait_queue_head_t wait_hup;
62};
63
64static DEFINE_MUTEX(update_mutex);
65
66#define VALUE_PREFIX "bpf_struct_ops_"
67#define VALUE_PREFIX_LEN (sizeof(VALUE_PREFIX) - 1)
68
69const struct bpf_verifier_ops bpf_struct_ops_verifier_ops = {
70};
71
72const struct bpf_prog_ops bpf_struct_ops_prog_ops = {
73#ifdef CONFIG_NET
74 .test_run = bpf_struct_ops_test_run,
75#endif
76};
77
78BTF_ID_LIST(st_ops_ids)
79BTF_ID(struct, module)
80BTF_ID(struct, bpf_struct_ops_common_value)
81
82enum {
83 IDX_MODULE_ID,
84 IDX_ST_OPS_COMMON_VALUE_ID,
85};
86
87extern struct btf *btf_vmlinux;
88
89static bool is_valid_value_type(struct btf *btf, s32 value_id,
90 const struct btf_type *type,
91 const char *value_name)
92{
93 const struct btf_type *common_value_type;
94 const struct btf_member *member;
95 const struct btf_type *vt, *mt;
96
97 vt = btf_type_by_id(btf, value_id);
98 if (btf_vlen(vt) != 2) {
99 pr_warn("The number of %s's members should be 2, but we get %d\n",
100 value_name, btf_vlen(vt));
101 return false;
102 }
103 member = btf_type_member(vt);
104 mt = btf_type_by_id(btf, member->type);
105 common_value_type = btf_type_by_id(btf_vmlinux,
106 st_ops_ids[IDX_ST_OPS_COMMON_VALUE_ID]);
107 if (mt != common_value_type) {
108 pr_warn("The first member of %s should be bpf_struct_ops_common_value\n",
109 value_name);
110 return false;
111 }
112 member++;
113 mt = btf_type_by_id(btf, member->type);
114 if (mt != type) {
115 pr_warn("The second member of %s should be %s\n",
116 value_name, btf_name_by_offset(btf, type->name_off));
117 return false;
118 }
119
120 return true;
121}
122
123static void *bpf_struct_ops_image_alloc(void)
124{
125 void *image;
126 int err;
127
128 err = bpf_jit_charge_modmem(PAGE_SIZE);
129 if (err)
130 return ERR_PTR(err);
131 image = arch_alloc_bpf_trampoline(PAGE_SIZE);
132 if (!image) {
133 bpf_jit_uncharge_modmem(PAGE_SIZE);
134 return ERR_PTR(-ENOMEM);
135 }
136
137 return image;
138}
139
140void bpf_struct_ops_image_free(void *image)
141{
142 if (image) {
143 arch_free_bpf_trampoline(image, PAGE_SIZE);
144 bpf_jit_uncharge_modmem(PAGE_SIZE);
145 }
146}
147
148#define MAYBE_NULL_SUFFIX "__nullable"
149#define REFCOUNTED_SUFFIX "__ref"
150#define ARENA_SUFFIX "__arena"
151#define ARENA_MAYBE_NULL_SUFFIX "__arena__nullable"
152
153/* Prepare argument info for every nullable argument of a member of a
154 * struct_ops type.
155 *
156 * Initialize a struct bpf_struct_ops_arg_info according to type info of
157 * the arguments of a stub function. (Check kCFI for more information about
158 * stub functions.)
159 *
160 * Each member in the struct_ops type has a struct bpf_struct_ops_arg_info
161 * to provide an array of struct bpf_ctx_arg_aux, which in turn provides
162 * the information that used by the verifier to check the arguments of the
163 * BPF struct_ops program assigned to the member. Here, we only care about
164 * the arguments that are marked as __nullable, __ref or __arena.
165 *
166 * The array of struct bpf_ctx_arg_aux is eventually assigned to
167 * prog->aux->ctx_arg_info of BPF struct_ops programs and passed to the
168 * verifier. (See check_struct_ops_btf_id())
169 *
170 * arg_info->info will be the list of struct bpf_ctx_arg_aux if success. If
171 * fails, it will be kept untouched.
172 */
173static int prepare_arg_info(struct btf *btf,
174 const char *st_ops_name,
175 const char *member_name,
176 const struct btf_type *func_proto, void *stub_func_addr,
177 struct btf_func_model *model,
178 struct bpf_struct_ops_arg_info *arg_info)
179{
180 const struct btf_type *stub_func_proto, *pointed_type;
181 bool is_nullable = false, is_refcounted = false, is_arena = false;
182 bool is_arena_nullable = false;
183 const struct btf_param *stub_args, *args;
184 struct bpf_ctx_arg_aux *info, *info_buf;
185 u32 nargs, arg_no, info_cnt = 0;
186 char ksym[KSYM_SYMBOL_LEN];
187 const char *stub_fname;
188 const char *suffix;
189 s32 stub_func_id;
190 u32 arg_btf_id;
191 int offset;
192
193 stub_fname = kallsyms_lookup((unsigned long)stub_func_addr, NULL, NULL, NULL, ksym);
194 if (!stub_fname) {
195 pr_warn("Cannot find the stub function name for the %s in struct %s\n",
196 member_name, st_ops_name);
197 return -ENOENT;
198 }
199
200 stub_func_id = btf_find_by_name_kind(btf, stub_fname, BTF_KIND_FUNC);
201 if (stub_func_id < 0) {
202 pr_warn("Cannot find the stub function %s in btf\n", stub_fname);
203 return -ENOENT;
204 }
205
206 stub_func_proto = btf_type_by_id(btf, stub_func_id);
207 stub_func_proto = btf_type_by_id(btf, stub_func_proto->type);
208
209 /* Check if the number of arguments of the stub function is the same
210 * as the number of arguments of the function pointer.
211 */
212 nargs = btf_type_vlen(func_proto);
213 if (nargs != btf_type_vlen(stub_func_proto)) {
214 pr_warn("the number of arguments of the stub function %s does not match the number of arguments of the member %s of struct %s\n",
215 stub_fname, member_name, st_ops_name);
216 return -EINVAL;
217 }
218
219 if (!nargs)
220 return 0;
221
222 args = btf_params(func_proto);
223 stub_args = btf_params(stub_func_proto);
224
225 info_buf = kzalloc_objs(*info_buf, nargs);
226 if (!info_buf)
227 return -ENOMEM;
228
229 /* Prepare info for every nullable argument */
230 info = info_buf;
231 for (arg_no = 0; arg_no < nargs; arg_no++) {
232 bool ptr_to_arena, ptr_to_struct;
233
234 /*
235 * Skip arguments that are not suffixed with "__arena__nullable",
236 * "__arena", "__nullable", or "__ref".
237 */
238 is_arena_nullable = btf_param_match_suffix(btf, &stub_args[arg_no],
239 ARENA_MAYBE_NULL_SUFFIX);
240 is_arena = btf_param_match_suffix(btf, &stub_args[arg_no], ARENA_SUFFIX);
241 is_nullable = !is_arena_nullable &&
242 btf_param_match_suffix(btf, &stub_args[arg_no], MAYBE_NULL_SUFFIX);
243 is_refcounted = btf_param_match_suffix(btf, &stub_args[arg_no],
244 REFCOUNTED_SUFFIX);
245
246 if (is_arena_nullable)
247 suffix = ARENA_MAYBE_NULL_SUFFIX;
248 else if (is_arena)
249 suffix = ARENA_SUFFIX;
250 else if (is_nullable)
251 suffix = MAYBE_NULL_SUFFIX;
252 else if (is_refcounted)
253 suffix = REFCOUNTED_SUFFIX;
254 else
255 continue;
256
257 /*
258 * Should be a pointer to struct, or any pointer for __arena or
259 * __arena__nullable.
260 */
261 pointed_type = btf_type_resolve_ptr(btf, args[arg_no].type, &arg_btf_id);
262 ptr_to_arena = pointed_type && (is_arena || is_arena_nullable);
263 ptr_to_struct = pointed_type && btf_type_is_struct(pointed_type);
264 if (!ptr_to_arena && !ptr_to_struct) {
265 pr_warn("stub function %s has %s tagging to an unsupported type\n",
266 stub_fname, suffix);
267 goto err_out;
268 }
269
270 offset = btf_ctx_arg_offset(btf, func_proto, arg_no);
271 if (offset < 0) {
272 pr_warn("stub function %s has an invalid trampoline ctx offset for arg#%u\n",
273 stub_fname, arg_no);
274 goto err_out;
275 }
276
277 if (args[arg_no].type != stub_args[arg_no].type) {
278 pr_warn("arg#%u type in stub function %s does not match with its original func_proto\n",
279 arg_no, stub_fname);
280 goto err_out;
281 }
282
283 /* Fill the information of the new argument */
284 info->btf_id = arg_btf_id;
285 info->btf = btf;
286 info->offset = offset;
287 if (is_arena || is_arena_nullable) {
288 /*
289 * Both types get PTR_TO_ARENA. In verifier state,
290 * PTR_TO_ARENA encompasses potential NULL values, but
291 * we do not force the program to check it, or maintain
292 * precision around it, since it has no safety implication.
293 */
294 info->reg_type = PTR_TO_ARENA;
295 model->arg_flags[arg_no] |= BTF_FMODEL_ARENA_ARG;
296 if (is_arena_nullable)
297 model->arg_flags[arg_no] |= BTF_FMODEL_NULLABLE_ARG;
298 } else if (is_nullable) {
299 info->reg_type = PTR_TRUSTED | PTR_TO_BTF_ID | PTR_MAYBE_NULL;
300 } else if (is_refcounted) {
301 info->reg_type = PTR_TRUSTED | PTR_TO_BTF_ID;
302 info->refcounted = true;
303 }
304
305 info++;
306 info_cnt++;
307 }
308
309 if (info_cnt) {
310 arg_info->info = info_buf;
311 arg_info->cnt = info_cnt;
312 } else {
313 kfree(info_buf);
314 }
315
316 return 0;
317
318err_out:
319 kfree(info_buf);
320
321 return -EINVAL;
322}
323
324/* Clean up the arg_info in a struct bpf_struct_ops_desc. */
325void bpf_struct_ops_desc_release(struct bpf_struct_ops_desc *st_ops_desc)
326{
327 struct bpf_struct_ops_arg_info *arg_info;
328 int i;
329
330 arg_info = st_ops_desc->arg_info;
331 for (i = 0; i < btf_type_vlen(st_ops_desc->type); i++)
332 kfree(arg_info[i].info);
333
334 kfree(arg_info);
335}
336
337static bool is_module_member(const struct btf *btf, u32 id)
338{
339 const struct btf_type *t;
340
341 t = btf_type_resolve_ptr(btf, id, NULL);
342 if (!t)
343 return false;
344
345 if (!__btf_type_is_struct(t) && !btf_type_is_fwd(t))
346 return false;
347
348 return !strcmp(btf_name_by_offset(btf, t->name_off), "module");
349}
350
351int bpf_struct_ops_supported(const struct bpf_struct_ops *st_ops, u32 moff)
352{
353 void *func_ptr = *(void **)(st_ops->cfi_stubs + moff);
354
355 return func_ptr ? 0 : -ENOTSUPP;
356}
357
358int bpf_struct_ops_desc_init(struct bpf_struct_ops_desc *st_ops_desc,
359 struct btf *btf,
360 struct bpf_verifier_log *log)
361{
362 struct bpf_struct_ops *st_ops = st_ops_desc->st_ops;
363 struct bpf_struct_ops_arg_info *arg_info;
364 const struct btf_member *member;
365 const struct btf_type *t;
366 s32 type_id, value_id;
367 char value_name[128];
368 const char *mname;
369 int i, err;
370
371 if (strlen(st_ops->name) + VALUE_PREFIX_LEN >=
372 sizeof(value_name)) {
373 pr_warn("struct_ops name %s is too long\n",
374 st_ops->name);
375 return -EINVAL;
376 }
377 sprintf(value_name, "%s%s", VALUE_PREFIX, st_ops->name);
378
379 if (!st_ops->cfi_stubs) {
380 pr_warn("struct_ops for %s has no cfi_stubs\n", st_ops->name);
381 return -EINVAL;
382 }
383
384 type_id = btf_find_by_name_kind(btf, st_ops->name,
385 BTF_KIND_STRUCT);
386 if (type_id < 0) {
387 pr_warn("Cannot find struct %s in %s\n",
388 st_ops->name, btf_get_name(btf));
389 return -EINVAL;
390 }
391 t = btf_type_by_id(btf, type_id);
392 if (btf_type_vlen(t) > BPF_STRUCT_OPS_MAX_NR_MEMBERS) {
393 pr_warn("Cannot support #%u members in struct %s\n",
394 btf_type_vlen(t), st_ops->name);
395 return -EINVAL;
396 }
397
398 value_id = btf_find_by_name_kind(btf, value_name,
399 BTF_KIND_STRUCT);
400 if (value_id < 0) {
401 pr_warn("Cannot find struct %s in %s\n",
402 value_name, btf_get_name(btf));
403 return -EINVAL;
404 }
405 if (!is_valid_value_type(btf, value_id, t, value_name))
406 return -EINVAL;
407
408 arg_info = kzalloc_objs(*arg_info, btf_type_vlen(t));
409 if (!arg_info)
410 return -ENOMEM;
411
412 st_ops_desc->arg_info = arg_info;
413 st_ops_desc->type = t;
414 st_ops_desc->type_id = type_id;
415 st_ops_desc->value_id = value_id;
416 st_ops_desc->value_type = btf_type_by_id(btf, value_id);
417
418 for_each_member(i, t, member) {
419 const struct btf_type *func_proto, *ret_type;
420 void **stub_func_addr;
421 u32 moff;
422
423 moff = __btf_member_bit_offset(t, member) / 8;
424 mname = btf_name_by_offset(btf, member->name_off);
425 if (!*mname) {
426 pr_warn("anon member in struct %s is not supported\n",
427 st_ops->name);
428 err = -EOPNOTSUPP;
429 goto errout;
430 }
431
432 if (__btf_member_bitfield_size(t, member)) {
433 pr_warn("bit field member %s in struct %s is not supported\n",
434 mname, st_ops->name);
435 err = -EOPNOTSUPP;
436 goto errout;
437 }
438
439 if (!st_ops_ids[IDX_MODULE_ID] && is_module_member(btf, member->type)) {
440 pr_warn("'struct module' btf id not found. Is CONFIG_MODULES enabled? bpf_struct_ops '%s' needs module support.\n",
441 st_ops->name);
442 err = -EOPNOTSUPP;
443 goto errout;
444 }
445
446 func_proto = btf_type_resolve_func_ptr(btf,
447 member->type,
448 NULL);
449
450 /* The member is not a function pointer or
451 * the function pointer is not supported.
452 */
453 if (!func_proto || bpf_struct_ops_supported(st_ops, moff))
454 continue;
455
456 if (func_proto->type) {
457 ret_type = btf_type_resolve_ptr(btf, func_proto->type, NULL);
458 if (ret_type && !__btf_type_is_struct(ret_type)) {
459 pr_warn("func ptr %s in struct %s returns non-struct pointer, which is not supported\n",
460 mname, st_ops->name);
461 err = -EOPNOTSUPP;
462 goto errout;
463 }
464 }
465
466 if (btf_distill_func_proto(log, btf,
467 func_proto, mname,
468 &st_ops->func_models[i])) {
469 pr_warn("Error in parsing func ptr %s in struct %s\n",
470 mname, st_ops->name);
471 err = -EINVAL;
472 goto errout;
473 }
474
475 /*
476 * A >8 byte return value is passed back in a register pair,
477 * which the struct_ops trampoline does not preserve (only
478 * 8 bytes of the return value are saved and restored).
479 */
480 if (st_ops->func_models[i].ret_size > 8) {
481 pr_warn("func ptr %s in struct %s has a >8 byte return value, which is not supported\n",
482 mname, st_ops->name);
483 err = -EOPNOTSUPP;
484 goto errout;
485 }
486
487 stub_func_addr = *(void **)(st_ops->cfi_stubs + moff);
488 err = prepare_arg_info(btf, st_ops->name, mname,
489 func_proto, stub_func_addr,
490 &st_ops->func_models[i],
491 arg_info + i);
492 if (err)
493 goto errout;
494 }
495
496 if (st_ops->init(btf)) {
497 pr_warn("Error in init bpf_struct_ops %s\n",
498 st_ops->name);
499 err = -EINVAL;
500 goto errout;
501 }
502
503 return 0;
504
505errout:
506 bpf_struct_ops_desc_release(st_ops_desc);
507
508 return err;
509}
510
511static int bpf_struct_ops_map_get_next_key(struct bpf_map *map, void *key,
512 void *next_key)
513{
514 if (key && *(u32 *)key == 0)
515 return -ENOENT;
516
517 *(u32 *)next_key = 0;
518 return 0;
519}
520
521int bpf_struct_ops_map_sys_lookup_elem(struct bpf_map *map, void *key,
522 void *value)
523{
524 struct bpf_struct_ops_map *st_map = (struct bpf_struct_ops_map *)map;
525 struct bpf_struct_ops_value *uvalue, *kvalue;
526 enum bpf_struct_ops_state state;
527 s64 refcnt;
528
529 if (unlikely(*(u32 *)key != 0))
530 return -ENOENT;
531
532 kvalue = &st_map->kvalue;
533 /* Pair with smp_store_release() during map_update */
534 state = smp_load_acquire(&kvalue->common.state);
535 if (state == BPF_STRUCT_OPS_STATE_INIT) {
536 memset(value, 0, map->value_size);
537 return 0;
538 }
539
540 /* No lock is needed. state and refcnt do not need
541 * to be updated together under atomic context.
542 */
543 uvalue = value;
544 memcpy(uvalue, st_map->uvalue, map->value_size);
545 uvalue->common.state = state;
546
547 /* This value offers the user space a general estimate of how
548 * many sockets are still utilizing this struct_ops for TCP
549 * congestion control. The number might not be exact, but it
550 * should sufficiently meet our present goals.
551 */
552 refcnt = atomic64_read(&map->refcnt) - atomic64_read(&map->usercnt);
553 refcount_set(&uvalue->common.refcnt, max_t(s64, refcnt, 0));
554
555 return 0;
556}
557
558static void *bpf_struct_ops_map_lookup_elem(struct bpf_map *map, void *key)
559{
560 return ERR_PTR(-EINVAL);
561}
562
563static void bpf_struct_ops_map_put_progs(struct bpf_struct_ops_map *st_map)
564{
565 u32 i;
566
567 for (i = 0; i < st_map->funcs_cnt; i++) {
568 if (!st_map->links[i])
569 break;
570 bpf_link_put(st_map->links[i]);
571 st_map->links[i] = NULL;
572 }
573}
574
575static void bpf_struct_ops_map_dissoc_progs(struct bpf_struct_ops_map *st_map)
576{
577 u32 i;
578
579 for (i = 0; i < st_map->funcs_cnt; i++) {
580 if (!st_map->links[i])
581 break;
582 bpf_prog_disassoc_struct_ops(st_map->links[i]->prog);
583 }
584}
585
586static void bpf_struct_ops_map_free_image(struct bpf_struct_ops_map *st_map)
587{
588 int i;
589
590 for (i = 0; i < st_map->image_pages_cnt; i++)
591 bpf_struct_ops_image_free(st_map->image_pages[i]);
592 st_map->image_pages_cnt = 0;
593}
594
595static int check_zero_holes(const struct btf *btf, const struct btf_type *t, void *data)
596{
597 const struct btf_member *member;
598 u32 i, moff, msize, prev_mend = 0;
599 const struct btf_type *mtype;
600
601 for_each_member(i, t, member) {
602 moff = __btf_member_bit_offset(t, member) / 8;
603 if (moff > prev_mend &&
604 memchr_inv(data + prev_mend, 0, moff - prev_mend))
605 return -EINVAL;
606
607 mtype = btf_type_by_id(btf, member->type);
608 mtype = btf_resolve_size(btf, mtype, &msize);
609 if (IS_ERR(mtype))
610 return PTR_ERR(mtype);
611 prev_mend = moff + msize;
612 }
613
614 if (t->size > prev_mend &&
615 memchr_inv(data + prev_mend, 0, t->size - prev_mend))
616 return -EINVAL;
617
618 return 0;
619}
620
621static void bpf_struct_ops_link_release(struct bpf_link *link)
622{
623}
624
625static void bpf_struct_ops_link_dealloc(struct bpf_link *link)
626{
627 struct bpf_tramp_link *tlink = container_of(link, struct bpf_tramp_link, link);
628
629 kfree(tlink);
630}
631
632const struct bpf_link_ops bpf_struct_ops_link_lops = {
633 .release = bpf_struct_ops_link_release,
634 .dealloc = bpf_struct_ops_link_dealloc,
635};
636
637int bpf_struct_ops_prepare_trampoline(struct bpf_tramp_nodes *tnodes,
638 struct bpf_tramp_node *node,
639 const struct btf_func_model *model,
640 void *stub_func,
641 void **_image, u32 *_image_off,
642 bool allow_alloc)
643{
644 u32 image_off = *_image_off, flags = BPF_TRAMP_F_INDIRECT;
645 void *image = *_image;
646 int size;
647
648 tnodes[BPF_TRAMP_FENTRY].nodes[0] = node;
649 tnodes[BPF_TRAMP_FENTRY].nr_nodes = 1;
650
651 if (model->ret_size > 0)
652 flags |= BPF_TRAMP_F_RET_FENTRY_RET;
653
654 size = arch_bpf_trampoline_size(model, flags, tnodes, stub_func);
655 if (size <= 0)
656 return size ? : -EFAULT;
657
658 /* Allocate image buffer if necessary */
659 if (!image || size > PAGE_SIZE - image_off) {
660 if (!allow_alloc)
661 return -E2BIG;
662
663 image = bpf_struct_ops_image_alloc();
664 if (IS_ERR(image))
665 return PTR_ERR(image);
666 image_off = 0;
667 }
668
669 size = arch_prepare_bpf_trampoline(NULL, image + image_off,
670 image + image_off + size,
671 model, flags, tnodes, stub_func);
672 if (size <= 0) {
673 if (image != *_image)
674 bpf_struct_ops_image_free(image);
675 return size ? : -EFAULT;
676 }
677
678 *_image = image;
679 *_image_off = image_off + size;
680 return 0;
681}
682
683static void bpf_struct_ops_ksym_init(const char *tname, const char *mname,
684 void *image, unsigned int size,
685 struct bpf_ksym *ksym)
686{
687 snprintf(ksym->name, KSYM_NAME_LEN, "bpf__%s_%s", tname, mname);
688 INIT_LIST_HEAD_RCU(&ksym->lnode);
689 bpf_image_ksym_init(image, size, ksym);
690}
691
692static void bpf_struct_ops_map_add_ksyms(struct bpf_struct_ops_map *st_map)
693{
694 u32 i;
695
696 for (i = 0; i < st_map->funcs_cnt; i++) {
697 if (!st_map->ksyms[i])
698 break;
699 bpf_image_ksym_add(st_map->ksyms[i]);
700 }
701}
702
703static void bpf_struct_ops_map_del_ksyms(struct bpf_struct_ops_map *st_map)
704{
705 u32 i;
706
707 for (i = 0; i < st_map->funcs_cnt; i++) {
708 if (!st_map->ksyms[i])
709 break;
710 bpf_image_ksym_del(st_map->ksyms[i]);
711 }
712}
713
714static void bpf_struct_ops_map_free_ksyms(struct bpf_struct_ops_map *st_map)
715{
716 u32 i;
717
718 for (i = 0; i < st_map->funcs_cnt; i++) {
719 if (!st_map->ksyms[i])
720 break;
721 kfree(st_map->ksyms[i]);
722 st_map->ksyms[i] = NULL;
723 }
724}
725
726static long bpf_struct_ops_map_update_elem(struct bpf_map *map, void *key,
727 void *value, u64 flags)
728{
729 struct bpf_struct_ops_map *st_map = (struct bpf_struct_ops_map *)map;
730 const struct bpf_struct_ops_desc *st_ops_desc = st_map->st_ops_desc;
731 const struct bpf_struct_ops *st_ops = st_ops_desc->st_ops;
732 struct bpf_struct_ops_value *uvalue, *kvalue;
733 const struct btf_type *module_type;
734 const struct btf_member *member;
735 const struct btf_type *t = st_ops_desc->type;
736 struct bpf_tramp_nodes *tnodes;
737 void *udata, *kdata;
738 int prog_fd, err;
739 u32 i, trampoline_start, image_off = 0;
740 void *cur_image = NULL, *image = NULL;
741 struct bpf_link **plink;
742 struct bpf_ksym **pksym;
743 const char *tname, *mname;
744
745 if (flags)
746 return -EINVAL;
747
748 if (*(u32 *)key != 0)
749 return -E2BIG;
750
751 err = check_zero_holes(st_map->btf, st_ops_desc->value_type, value);
752 if (err)
753 return err;
754
755 uvalue = value;
756 err = check_zero_holes(st_map->btf, t, uvalue->data);
757 if (err)
758 return err;
759
760 if (uvalue->common.state || refcount_read(&uvalue->common.refcnt))
761 return -EINVAL;
762
763 tnodes = kzalloc_objs(*tnodes, BPF_TRAMP_MAX);
764 if (!tnodes)
765 return -ENOMEM;
766
767 uvalue = (struct bpf_struct_ops_value *)st_map->uvalue;
768 kvalue = (struct bpf_struct_ops_value *)&st_map->kvalue;
769
770 mutex_lock(&st_map->lock);
771
772 if (kvalue->common.state != BPF_STRUCT_OPS_STATE_INIT) {
773 err = -EBUSY;
774 goto unlock;
775 }
776
777 memcpy(uvalue, value, map->value_size);
778
779 udata = &uvalue->data;
780 kdata = &kvalue->data;
781
782 plink = st_map->links;
783 pksym = st_map->ksyms;
784 tname = btf_name_by_offset(st_map->btf, t->name_off);
785 module_type = btf_type_by_id(btf_vmlinux, st_ops_ids[IDX_MODULE_ID]);
786 for_each_member(i, t, member) {
787 const struct btf_type *mtype, *ptype;
788 struct bpf_prog *prog;
789 struct bpf_tramp_link *link;
790 struct bpf_ksym *ksym;
791 u32 moff;
792
793 moff = __btf_member_bit_offset(t, member) / 8;
794 mname = btf_name_by_offset(st_map->btf, member->name_off);
795 ptype = btf_type_resolve_ptr(st_map->btf, member->type, NULL);
796 if (ptype == module_type) {
797 if (*(void **)(udata + moff))
798 goto reset_unlock;
799 *(void **)(kdata + moff) = BPF_MODULE_OWNER;
800 continue;
801 }
802
803 err = st_ops->init_member(t, member, kdata, udata);
804 if (err < 0)
805 goto reset_unlock;
806
807 /* The ->init_member() has handled this member */
808 if (err > 0)
809 continue;
810
811 /* If st_ops->init_member does not handle it,
812 * we will only handle func ptrs and zero-ed members
813 * here. Reject everything else.
814 */
815
816 /* All non func ptr member must be 0 */
817 if (!ptype || !btf_type_is_func_proto(ptype)) {
818 u32 msize;
819
820 mtype = btf_type_by_id(st_map->btf, member->type);
821 mtype = btf_resolve_size(st_map->btf, mtype, &msize);
822 if (IS_ERR(mtype)) {
823 err = PTR_ERR(mtype);
824 goto reset_unlock;
825 }
826
827 if (memchr_inv(udata + moff, 0, msize)) {
828 err = -EINVAL;
829 goto reset_unlock;
830 }
831
832 continue;
833 }
834
835 prog_fd = (int)(*(unsigned long *)(udata + moff));
836 /* Similar check as the attr->attach_prog_fd */
837 if (!prog_fd)
838 continue;
839
840 prog = bpf_prog_get(prog_fd);
841 if (IS_ERR(prog)) {
842 err = PTR_ERR(prog);
843 goto reset_unlock;
844 }
845
846 if (prog->type != BPF_PROG_TYPE_STRUCT_OPS ||
847 prog->aux->attach_btf_id != st_ops_desc->type_id ||
848 prog->expected_attach_type != i) {
849 bpf_prog_put(prog);
850 err = -EINVAL;
851 goto reset_unlock;
852 }
853
854 link = kzalloc_obj(*link, GFP_USER);
855 if (!link) {
856 bpf_prog_put(prog);
857 err = -ENOMEM;
858 goto reset_unlock;
859 }
860 bpf_tramp_link_init(link, BPF_LINK_TYPE_STRUCT_OPS,
861 &bpf_struct_ops_link_lops, prog, prog->expected_attach_type, 0);
862
863 *plink++ = &link->link;
864
865 /* Poison pointer on error instead of return for backward compatibility */
866 bpf_prog_assoc_struct_ops(prog, &st_map->map);
867
868 ksym = kzalloc_obj(*ksym, GFP_USER);
869 if (!ksym) {
870 err = -ENOMEM;
871 goto reset_unlock;
872 }
873 *pksym++ = ksym;
874
875 trampoline_start = image_off;
876 err = bpf_struct_ops_prepare_trampoline(tnodes, &link->node,
877 &st_ops->func_models[i],
878 *(void **)(st_ops->cfi_stubs + moff),
879 &image, &image_off,
880 st_map->image_pages_cnt < MAX_TRAMP_IMAGE_PAGES);
881 if (err)
882 goto reset_unlock;
883
884 if (cur_image != image) {
885 st_map->image_pages[st_map->image_pages_cnt++] = image;
886 cur_image = image;
887 trampoline_start = 0;
888 }
889
890 *(void **)(kdata + moff) = image + trampoline_start + cfi_get_offset();
891
892 /* put prog_id to udata */
893 *(unsigned long *)(udata + moff) = prog->aux->id;
894
895 /* init ksym for this trampoline */
896 bpf_struct_ops_ksym_init(tname, mname,
897 image + trampoline_start,
898 image_off - trampoline_start,
899 ksym);
900 }
901
902 if (st_ops->validate) {
903 err = st_ops->validate(kdata);
904 if (err)
905 goto reset_unlock;
906 }
907 for (i = 0; i < st_map->image_pages_cnt; i++) {
908 err = arch_protect_bpf_trampoline(st_map->image_pages[i],
909 PAGE_SIZE);
910 if (err)
911 goto reset_unlock;
912 }
913
914 if (st_map->map.map_flags & BPF_F_LINK) {
915 err = 0;
916 /* Let bpf_link handle registration & unregistration.
917 *
918 * Pair with smp_load_acquire() during lookup_elem().
919 */
920 smp_store_release(&kvalue->common.state, BPF_STRUCT_OPS_STATE_READY);
921 goto unlock;
922 }
923
924 err = st_ops->reg(kdata, NULL);
925 if (likely(!err)) {
926 /* This refcnt increment on the map here after
927 * 'st_ops->reg()' is secure since the state of the
928 * map must be set to INIT at this moment, and thus
929 * bpf_struct_ops_map_delete_elem() can't unregister
930 * or transition it to TOBEFREE concurrently.
931 */
932 bpf_map_inc(map);
933 /* Pair with smp_load_acquire() during lookup_elem().
934 * It ensures the above udata updates (e.g. prog->aux->id)
935 * can be seen once BPF_STRUCT_OPS_STATE_INUSE is set.
936 */
937 smp_store_release(&kvalue->common.state, BPF_STRUCT_OPS_STATE_INUSE);
938 goto unlock;
939 }
940
941 /* Error during st_ops->reg(). Can happen if this struct_ops needs to be
942 * verified as a whole, after all init_member() calls. Can also happen if
943 * there was a race in registering the struct_ops (under the same name) to
944 * a sub-system through different struct_ops's maps.
945 */
946
947reset_unlock:
948 bpf_struct_ops_map_free_ksyms(st_map);
949 bpf_struct_ops_map_free_image(st_map);
950 bpf_struct_ops_map_dissoc_progs(st_map);
951 bpf_struct_ops_map_put_progs(st_map);
952 memset(uvalue, 0, map->value_size);
953 memset(kvalue, 0, map->value_size);
954unlock:
955 kfree(tnodes);
956 mutex_unlock(&st_map->lock);
957 if (!err)
958 bpf_struct_ops_map_add_ksyms(st_map);
959 return err;
960}
961
962static long bpf_struct_ops_map_delete_elem(struct bpf_map *map, void *key)
963{
964 enum bpf_struct_ops_state prev_state;
965 struct bpf_struct_ops_map *st_map;
966
967 st_map = (struct bpf_struct_ops_map *)map;
968 if (st_map->map.map_flags & BPF_F_LINK)
969 return -EOPNOTSUPP;
970
971 prev_state = cmpxchg(&st_map->kvalue.common.state,
972 BPF_STRUCT_OPS_STATE_INUSE,
973 BPF_STRUCT_OPS_STATE_TOBEFREE);
974 switch (prev_state) {
975 case BPF_STRUCT_OPS_STATE_INUSE:
976 st_map->st_ops_desc->st_ops->unreg(&st_map->kvalue.data, NULL);
977 bpf_map_put(map);
978 return 0;
979 case BPF_STRUCT_OPS_STATE_TOBEFREE:
980 return -EINPROGRESS;
981 case BPF_STRUCT_OPS_STATE_INIT:
982 return -ENOENT;
983 default:
984 WARN_ON_ONCE(1);
985 /* Should never happen. Treat it as not found. */
986 return -ENOENT;
987 }
988}
989
990static void bpf_struct_ops_map_seq_show_elem(struct bpf_map *map, void *key,
991 struct seq_file *m)
992{
993 struct bpf_struct_ops_map *st_map = (struct bpf_struct_ops_map *)map;
994 void *value;
995 int err;
996
997 value = kmalloc(map->value_size, GFP_USER | __GFP_NOWARN);
998 if (!value)
999 return;
1000
1001 err = bpf_struct_ops_map_sys_lookup_elem(map, key, value);
1002 if (!err) {
1003 btf_type_seq_show(st_map->btf,
1004 map->btf_vmlinux_value_type_id,
1005 value, m);
1006 seq_putc(m, '\n');
1007 }
1008
1009 kfree(value);
1010}
1011
1012static void __bpf_struct_ops_map_free(struct bpf_map *map)
1013{
1014 struct bpf_struct_ops_map *st_map = (struct bpf_struct_ops_map *)map;
1015
1016 if (st_map->links)
1017 bpf_struct_ops_map_put_progs(st_map);
1018 if (st_map->ksyms)
1019 bpf_struct_ops_map_free_ksyms(st_map);
1020 bpf_map_area_free(st_map->links);
1021 bpf_map_area_free(st_map->ksyms);
1022 bpf_struct_ops_map_free_image(st_map);
1023 bpf_map_area_free(st_map->uvalue);
1024 bpf_map_area_free(st_map);
1025}
1026
1027static void bpf_struct_ops_map_free(struct bpf_map *map)
1028{
1029 struct bpf_struct_ops_map *st_map = (struct bpf_struct_ops_map *)map;
1030
1031 /* st_ops->owner was acquired during map_alloc to implicitly holds
1032 * the btf's refcnt. The acquire was only done when btf_is_module()
1033 * st_map->btf cannot be NULL here.
1034 */
1035 if (btf_is_module(st_map->btf))
1036 module_put(st_map->st_ops_desc->st_ops->owner);
1037
1038 bpf_struct_ops_map_dissoc_progs(st_map);
1039
1040 bpf_struct_ops_map_del_ksyms(st_map);
1041
1042 /* The struct_ops's function may switch to another struct_ops.
1043 *
1044 * For example, bpf_tcp_cc_x->init() may switch to
1045 * another tcp_cc_y by calling
1046 * setsockopt(TCP_CONGESTION, "tcp_cc_y").
1047 * During the switch, bpf_struct_ops_put(tcp_cc_x) is called
1048 * and its refcount may reach 0 which then free its
1049 * trampoline image while tcp_cc_x is still running.
1050 *
1051 * A vanilla rcu gp is to wait for all bpf-tcp-cc prog
1052 * to finish. bpf-tcp-cc prog is non sleepable.
1053 * A rcu_tasks gp is to wait for the last few insn
1054 * in the tramopline image to finish before releasing
1055 * the trampoline image.
1056 */
1057 synchronize_rcu_mult(call_rcu, call_rcu_tasks);
1058
1059 __bpf_struct_ops_map_free(map);
1060}
1061
1062static int bpf_struct_ops_map_alloc_check(union bpf_attr *attr)
1063{
1064 if (attr->key_size != sizeof(unsigned int) || attr->max_entries != 1 ||
1065 (attr->map_flags & ~(BPF_F_LINK | BPF_F_VTYPE_BTF_OBJ_FD)) ||
1066 !attr->btf_vmlinux_value_type_id)
1067 return -EINVAL;
1068 return 0;
1069}
1070
1071static u32 count_func_ptrs(const struct btf *btf, const struct btf_type *t)
1072{
1073 int i;
1074 u32 count;
1075 const struct btf_member *member;
1076
1077 count = 0;
1078 for_each_member(i, t, member)
1079 if (btf_type_resolve_func_ptr(btf, member->type, NULL))
1080 count++;
1081 return count;
1082}
1083
1084static struct bpf_map *bpf_struct_ops_map_alloc(union bpf_attr *attr)
1085{
1086 const struct bpf_struct_ops_desc *st_ops_desc;
1087 size_t st_map_size;
1088 struct bpf_struct_ops_map *st_map;
1089 const struct btf_type *t, *vt;
1090 struct module *mod = NULL;
1091 struct bpf_map *map;
1092 struct btf *btf;
1093 int ret;
1094
1095 if (attr->map_flags & BPF_F_VTYPE_BTF_OBJ_FD) {
1096 /* The map holds btf for its whole life time. */
1097 btf = btf_get_by_fd(attr->value_type_btf_obj_fd);
1098 if (IS_ERR(btf))
1099 return ERR_CAST(btf);
1100 if (!btf_is_module(btf)) {
1101 btf_put(btf);
1102 return ERR_PTR(-EINVAL);
1103 }
1104
1105 mod = btf_try_get_module(btf);
1106 /* mod holds a refcnt to btf. We don't need an extra refcnt
1107 * here.
1108 */
1109 btf_put(btf);
1110 if (!mod)
1111 return ERR_PTR(-EINVAL);
1112 } else {
1113 btf = bpf_get_btf_vmlinux();
1114 if (IS_ERR(btf))
1115 return ERR_CAST(btf);
1116 if (!btf)
1117 return ERR_PTR(-ENOTSUPP);
1118 }
1119
1120 st_ops_desc = bpf_struct_ops_find_value(btf, attr->btf_vmlinux_value_type_id);
1121 if (!st_ops_desc) {
1122 ret = -ENOTSUPP;
1123 goto errout;
1124 }
1125
1126 vt = st_ops_desc->value_type;
1127 if (attr->value_size != vt->size) {
1128 ret = -EINVAL;
1129 goto errout;
1130 }
1131
1132 t = st_ops_desc->type;
1133
1134 st_map_size = sizeof(*st_map) +
1135 /* kvalue stores the
1136 * struct bpf_struct_ops_tcp_congestions_ops
1137 */
1138 (vt->size - sizeof(struct bpf_struct_ops_value));
1139
1140 st_map = bpf_map_area_alloc(st_map_size, NUMA_NO_NODE);
1141 if (!st_map) {
1142 ret = -ENOMEM;
1143 goto errout;
1144 }
1145
1146 st_map->st_ops_desc = st_ops_desc;
1147 map = &st_map->map;
1148
1149 st_map->uvalue = bpf_map_area_alloc(vt->size, NUMA_NO_NODE);
1150 st_map->funcs_cnt = count_func_ptrs(btf, t);
1151 st_map->links =
1152 bpf_map_area_alloc(st_map->funcs_cnt * sizeof(struct bpf_link *),
1153 NUMA_NO_NODE);
1154
1155 st_map->ksyms =
1156 bpf_map_area_alloc(st_map->funcs_cnt * sizeof(struct bpf_ksym *),
1157 NUMA_NO_NODE);
1158 if (!st_map->uvalue || !st_map->links || !st_map->ksyms) {
1159 ret = -ENOMEM;
1160 goto errout_free;
1161 }
1162 st_map->btf = btf;
1163
1164 mutex_init(&st_map->lock);
1165 bpf_map_init_from_attr(map, attr);
1166
1167 return map;
1168
1169errout_free:
1170 __bpf_struct_ops_map_free(map);
1171errout:
1172 module_put(mod);
1173
1174 return ERR_PTR(ret);
1175}
1176
1177static u64 bpf_struct_ops_map_mem_usage(const struct bpf_map *map)
1178{
1179 struct bpf_struct_ops_map *st_map = (struct bpf_struct_ops_map *)map;
1180 const struct bpf_struct_ops_desc *st_ops_desc = st_map->st_ops_desc;
1181 const struct btf_type *vt = st_ops_desc->value_type;
1182 u64 usage;
1183
1184 usage = sizeof(*st_map) +
1185 vt->size - sizeof(struct bpf_struct_ops_value);
1186 usage += vt->size;
1187 usage += st_map->funcs_cnt * sizeof(struct bpf_link *);
1188 usage += st_map->funcs_cnt * sizeof(struct bpf_ksym *);
1189 usage += PAGE_SIZE;
1190 return usage;
1191}
1192
1193BTF_ID_LIST_SINGLE(bpf_struct_ops_map_btf_ids, struct, bpf_struct_ops_map)
1194const struct bpf_map_ops bpf_struct_ops_map_ops = {
1195 .map_alloc_check = bpf_struct_ops_map_alloc_check,
1196 .map_alloc = bpf_struct_ops_map_alloc,
1197 .map_free = bpf_struct_ops_map_free,
1198 .map_get_next_key = bpf_struct_ops_map_get_next_key,
1199 .map_lookup_elem = bpf_struct_ops_map_lookup_elem,
1200 .map_delete_elem = bpf_struct_ops_map_delete_elem,
1201 .map_update_elem = bpf_struct_ops_map_update_elem,
1202 .map_seq_show_elem = bpf_struct_ops_map_seq_show_elem,
1203 .map_mem_usage = bpf_struct_ops_map_mem_usage,
1204 .map_btf_id = &bpf_struct_ops_map_btf_ids[0],
1205};
1206
1207/* "const void *" because some subsystem is
1208 * passing a const (e.g. const struct tcp_congestion_ops *)
1209 */
1210bool bpf_struct_ops_get(const void *kdata)
1211{
1212 struct bpf_struct_ops_value *kvalue;
1213 struct bpf_struct_ops_map *st_map;
1214 struct bpf_map *map;
1215
1216 kvalue = container_of(kdata, struct bpf_struct_ops_value, data);
1217 st_map = container_of(kvalue, struct bpf_struct_ops_map, kvalue);
1218
1219 map = __bpf_map_inc_not_zero(&st_map->map, false);
1220 return !IS_ERR(map);
1221}
1222EXPORT_SYMBOL_GPL(bpf_struct_ops_get);
1223
1224void bpf_struct_ops_put(const void *kdata)
1225{
1226 struct bpf_struct_ops_value *kvalue;
1227 struct bpf_struct_ops_map *st_map;
1228
1229 kvalue = container_of(kdata, struct bpf_struct_ops_value, data);
1230 st_map = container_of(kvalue, struct bpf_struct_ops_map, kvalue);
1231
1232 bpf_map_put(&st_map->map);
1233}
1234EXPORT_SYMBOL_GPL(bpf_struct_ops_put);
1235
1236u32 bpf_struct_ops_id(const void *kdata)
1237{
1238 struct bpf_struct_ops_value *kvalue;
1239 struct bpf_struct_ops_map *st_map;
1240
1241 kvalue = container_of(kdata, struct bpf_struct_ops_value, data);
1242 st_map = container_of(kvalue, struct bpf_struct_ops_map, kvalue);
1243
1244 return st_map->map.id;
1245}
1246EXPORT_SYMBOL_GPL(bpf_struct_ops_id);
1247
1248/**
1249 * bpf_struct_ops_for_each_prog - Invoke @cb for each member prog
1250 * @kdata: kernel-side struct_ops vmtable (the @kdata arg to ->reg/->update/->unreg)
1251 * @cb: callback invoked once per member prog; non-zero return stops iteration
1252 * @data: opaque argument passed to @cb
1253 *
1254 * Walks the struct_ops member progs registered on the map containing @kdata.
1255 * Intended for use from struct_ops ->reg() callbacks (and similar) that need to
1256 * inspect the loaded BPF programs (for example to discover maps they reference
1257 * via @prog->aux->used_maps).
1258 *
1259 * Return 0 if iteration completed, otherwise the first non-zero @cb return.
1260 */
1261int bpf_struct_ops_for_each_prog(const void *kdata,
1262 int (*cb)(struct bpf_prog *prog, void *data),
1263 void *data)
1264{
1265 struct bpf_struct_ops_value *kvalue;
1266 struct bpf_struct_ops_map *st_map;
1267 u32 i;
1268 int ret;
1269
1270 kvalue = container_of(kdata, struct bpf_struct_ops_value, data);
1271 st_map = container_of(kvalue, struct bpf_struct_ops_map, kvalue);
1272
1273 for (i = 0; i < st_map->funcs_cnt; i++) {
1274 if (!st_map->links[i])
1275 continue;
1276 ret = cb(st_map->links[i]->prog, data);
1277 if (ret)
1278 return ret;
1279 }
1280 return 0;
1281}
1282EXPORT_SYMBOL_GPL(bpf_struct_ops_for_each_prog);
1283
1284static bool bpf_struct_ops_valid_to_reg(struct bpf_map *map)
1285{
1286 struct bpf_struct_ops_map *st_map = (struct bpf_struct_ops_map *)map;
1287
1288 return map->map_type == BPF_MAP_TYPE_STRUCT_OPS &&
1289 map->map_flags & BPF_F_LINK &&
1290 /* Pair with smp_store_release() during map_update */
1291 smp_load_acquire(&st_map->kvalue.common.state) == BPF_STRUCT_OPS_STATE_READY;
1292}
1293
1294static void bpf_struct_ops_map_link_dealloc(struct bpf_link *link)
1295{
1296 struct bpf_struct_ops_link *st_link;
1297 struct bpf_struct_ops_map *st_map;
1298
1299 st_link = container_of(link, struct bpf_struct_ops_link, link);
1300 st_map = (struct bpf_struct_ops_map *)
1301 rcu_dereference_protected(st_link->map, true);
1302 if (st_map) {
1303 st_map->st_ops_desc->st_ops->unreg(&st_map->kvalue.data, link);
1304 bpf_map_put(&st_map->map);
1305 }
1306 kfree(st_link);
1307}
1308
1309static void bpf_struct_ops_map_link_show_fdinfo(const struct bpf_link *link,
1310 struct seq_file *seq)
1311{
1312 struct bpf_struct_ops_link *st_link;
1313 struct bpf_map *map;
1314
1315 st_link = container_of(link, struct bpf_struct_ops_link, link);
1316 rcu_read_lock();
1317 map = rcu_dereference(st_link->map);
1318 if (map)
1319 seq_printf(seq, "map_id:\t%d\n", map->id);
1320 rcu_read_unlock();
1321}
1322
1323static int bpf_struct_ops_map_link_fill_link_info(const struct bpf_link *link,
1324 struct bpf_link_info *info)
1325{
1326 struct bpf_struct_ops_link *st_link;
1327 struct bpf_map *map;
1328
1329 st_link = container_of(link, struct bpf_struct_ops_link, link);
1330 rcu_read_lock();
1331 map = rcu_dereference(st_link->map);
1332 if (map)
1333 info->struct_ops.map_id = map->id;
1334 rcu_read_unlock();
1335 return 0;
1336}
1337
1338static int bpf_struct_ops_map_link_update(struct bpf_link *link, struct bpf_map *new_map,
1339 struct bpf_map *expected_old_map)
1340{
1341 struct bpf_struct_ops_map *st_map, *old_st_map;
1342 struct bpf_map *old_map;
1343 struct bpf_struct_ops_link *st_link;
1344 int err;
1345
1346 st_link = container_of(link, struct bpf_struct_ops_link, link);
1347 st_map = container_of(new_map, struct bpf_struct_ops_map, map);
1348
1349 if (!bpf_struct_ops_valid_to_reg(new_map))
1350 return -EINVAL;
1351
1352 if (!st_map->st_ops_desc->st_ops->update)
1353 return -EOPNOTSUPP;
1354
1355 mutex_lock(&update_mutex);
1356
1357 old_map = rcu_dereference_protected(st_link->map, lockdep_is_held(&update_mutex));
1358 if (!old_map) {
1359 err = -ENOLINK;
1360 goto err_out;
1361 }
1362 if (expected_old_map && old_map != expected_old_map) {
1363 err = -EPERM;
1364 goto err_out;
1365 }
1366
1367 old_st_map = container_of(old_map, struct bpf_struct_ops_map, map);
1368 /* The new and old struct_ops must be the same type. */
1369 if (st_map->st_ops_desc != old_st_map->st_ops_desc) {
1370 err = -EINVAL;
1371 goto err_out;
1372 }
1373
1374 err = st_map->st_ops_desc->st_ops->update(st_map->kvalue.data, old_st_map->kvalue.data, link);
1375 if (err)
1376 goto err_out;
1377
1378 bpf_map_inc(new_map);
1379 rcu_assign_pointer(st_link->map, new_map);
1380 bpf_map_put(old_map);
1381
1382err_out:
1383 mutex_unlock(&update_mutex);
1384
1385 return err;
1386}
1387
1388static int bpf_struct_ops_map_link_detach(struct bpf_link *link)
1389{
1390 struct bpf_struct_ops_link *st_link = container_of(link, struct bpf_struct_ops_link, link);
1391 struct bpf_struct_ops_map *st_map;
1392 struct bpf_map *map;
1393
1394 mutex_lock(&update_mutex);
1395
1396 map = rcu_dereference_protected(st_link->map, lockdep_is_held(&update_mutex));
1397 if (!map) {
1398 mutex_unlock(&update_mutex);
1399 return 0;
1400 }
1401 st_map = container_of(map, struct bpf_struct_ops_map, map);
1402
1403 st_map->st_ops_desc->st_ops->unreg(&st_map->kvalue.data, link);
1404
1405 RCU_INIT_POINTER(st_link->map, NULL);
1406 /* Pair with bpf_map_get() in bpf_struct_ops_link_create() or
1407 * bpf_map_inc() in bpf_struct_ops_map_link_update().
1408 */
1409 bpf_map_put(&st_map->map);
1410
1411 mutex_unlock(&update_mutex);
1412
1413 wake_up_interruptible_poll(&st_link->wait_hup, EPOLLHUP);
1414
1415 return 0;
1416}
1417
1418static __poll_t bpf_struct_ops_map_link_poll(struct file *file,
1419 struct poll_table_struct *pts)
1420{
1421 struct bpf_struct_ops_link *st_link = file->private_data;
1422
1423 poll_wait(file, &st_link->wait_hup, pts);
1424
1425 return rcu_access_pointer(st_link->map) ? 0 : EPOLLHUP;
1426}
1427
1428static const struct bpf_link_ops bpf_struct_ops_map_lops = {
1429 .dealloc = bpf_struct_ops_map_link_dealloc,
1430 .detach = bpf_struct_ops_map_link_detach,
1431 .show_fdinfo = bpf_struct_ops_map_link_show_fdinfo,
1432 .fill_link_info = bpf_struct_ops_map_link_fill_link_info,
1433 .update_map = bpf_struct_ops_map_link_update,
1434 .poll = bpf_struct_ops_map_link_poll,
1435};
1436
1437int bpf_struct_ops_link_create(union bpf_attr *attr)
1438{
1439 struct bpf_struct_ops_link *link = NULL;
1440 struct bpf_link_primer link_primer;
1441 struct bpf_struct_ops_map *st_map;
1442 struct bpf_map *map;
1443 int err;
1444
1445 map = bpf_map_get(attr->link_create.map_fd);
1446 if (IS_ERR(map))
1447 return PTR_ERR(map);
1448
1449 st_map = (struct bpf_struct_ops_map *)map;
1450
1451 if (!bpf_struct_ops_valid_to_reg(map)) {
1452 err = -EINVAL;
1453 goto err_out;
1454 }
1455
1456 link = kzalloc_obj(*link, GFP_USER);
1457 if (!link) {
1458 err = -ENOMEM;
1459 goto err_out;
1460 }
1461 bpf_link_init(&link->link, BPF_LINK_TYPE_STRUCT_OPS, &bpf_struct_ops_map_lops, NULL,
1462 attr->link_create.attach_type);
1463
1464 err = bpf_link_prime(&link->link, &link_primer);
1465 if (err)
1466 goto err_out;
1467
1468 init_waitqueue_head(&link->wait_hup);
1469
1470 /* Hold the update_mutex such that the subsystem cannot
1471 * do link->ops->detach() before the link is fully initialized.
1472 */
1473 mutex_lock(&update_mutex);
1474 err = st_map->st_ops_desc->st_ops->reg(st_map->kvalue.data, &link->link);
1475 if (err) {
1476 mutex_unlock(&update_mutex);
1477 bpf_link_cleanup(&link_primer);
1478 link = NULL;
1479 goto err_out;
1480 }
1481 RCU_INIT_POINTER(link->map, map);
1482 mutex_unlock(&update_mutex);
1483
1484 return bpf_link_settle(&link_primer);
1485
1486err_out:
1487 bpf_map_put(map);
1488 kfree(link);
1489 return err;
1490}
1491
1492int bpf_prog_assoc_struct_ops(struct bpf_prog *prog, struct bpf_map *map)
1493{
1494 struct bpf_map *st_ops_assoc;
1495
1496 guard(mutex)(&prog->aux->st_ops_assoc_mutex);
1497
1498 st_ops_assoc = rcu_dereference_protected(prog->aux->st_ops_assoc,
1499 lockdep_is_held(&prog->aux->st_ops_assoc_mutex));
1500 if (st_ops_assoc && st_ops_assoc == map)
1501 return 0;
1502
1503 if (st_ops_assoc) {
1504 if (prog->type != BPF_PROG_TYPE_STRUCT_OPS)
1505 return -EBUSY;
1506
1507 rcu_assign_pointer(prog->aux->st_ops_assoc, BPF_PTR_POISON);
1508 } else {
1509 /*
1510 * struct_ops map does not track associated non-struct_ops programs.
1511 * Bump the refcount to make sure st_ops_assoc is always valid.
1512 */
1513 if (prog->type != BPF_PROG_TYPE_STRUCT_OPS)
1514 bpf_map_inc(map);
1515
1516 rcu_assign_pointer(prog->aux->st_ops_assoc, map);
1517 }
1518
1519 return 0;
1520}
1521
1522void bpf_prog_disassoc_struct_ops(struct bpf_prog *prog)
1523{
1524 struct bpf_map *st_ops_assoc;
1525
1526 guard(mutex)(&prog->aux->st_ops_assoc_mutex);
1527
1528 st_ops_assoc = rcu_dereference_protected(prog->aux->st_ops_assoc,
1529 lockdep_is_held(&prog->aux->st_ops_assoc_mutex));
1530 if (!st_ops_assoc || st_ops_assoc == BPF_PTR_POISON)
1531 return;
1532
1533 if (prog->type != BPF_PROG_TYPE_STRUCT_OPS)
1534 bpf_map_put(st_ops_assoc);
1535
1536 RCU_INIT_POINTER(prog->aux->st_ops_assoc, NULL);
1537}
1538
1539/*
1540 * Get a reference to the struct_ops struct (i.e., kdata) associated with a
1541 * program. Should only be called in BPF program context (e.g., in a kfunc).
1542 *
1543 * If the returned pointer is not NULL, it must points to a valid struct_ops.
1544 * The struct_ops map is not guaranteed to be initialized nor attached.
1545 * Kernel struct_ops implementers are responsible for tracking and checking
1546 * the state of the struct_ops if the use case requires an initialized or
1547 * attached struct_ops.
1548 */
1549void *bpf_prog_get_assoc_struct_ops(const struct bpf_prog_aux *aux)
1550{
1551 struct bpf_struct_ops_map *st_map;
1552 struct bpf_map *st_ops_assoc;
1553
1554 st_ops_assoc = rcu_dereference_check(aux->st_ops_assoc, bpf_rcu_lock_held());
1555 if (!st_ops_assoc || st_ops_assoc == BPF_PTR_POISON)
1556 return NULL;
1557
1558 st_map = (struct bpf_struct_ops_map *)st_ops_assoc;
1559
1560 return &st_map->kvalue.data;
1561}
1562EXPORT_SYMBOL_GPL(bpf_prog_get_assoc_struct_ops);
1563
1564void bpf_map_struct_ops_info_fill(struct bpf_map_info *info, struct bpf_map *map)
1565{
1566 struct bpf_struct_ops_map *st_map = (struct bpf_struct_ops_map *)map;
1567
1568 info->btf_vmlinux_id = btf_obj_id(st_map->btf);
1569}