projects/linux·blob v6.4 · refs

862 lines · 19.9 KiB

1/*
2 * Copyright (C) 2017-2018 Netronome Systems, Inc.
3 *
4 * This software is licensed under the GNU General License Version 2,
5 * June 1991 as shown in the file COPYING in the top-level directory of this
6 * source tree.
7 *
8 * THE COPYRIGHT HOLDERS AND/OR OTHER PARTIES PROVIDE THE PROGRAM "AS IS"
9 * WITHOUT WARRANTY OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING,
10 * BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
11 * FOR A PARTICULAR PURPOSE. THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE
12 * OF THE PROGRAM IS WITH YOU. SHOULD THE PROGRAM PROVE DEFECTIVE, YOU ASSUME
13 * THE COST OF ALL NECESSARY SERVICING, REPAIR OR CORRECTION.
14 */
15
16#include <linux/bpf.h>
17#include <linux/bpf_verifier.h>
18#include <linux/bug.h>
19#include <linux/kdev_t.h>
20#include <linux/list.h>
21#include <linux/lockdep.h>
22#include <linux/netdevice.h>
23#include <linux/printk.h>
24#include <linux/proc_ns.h>
25#include <linux/rhashtable.h>
26#include <linux/rtnetlink.h>
27#include <linux/rwsem.h>
28
29/* Protects offdevs, members of bpf_offload_netdev and offload members
30 * of all progs.
31 * RTNL lock cannot be taken when holding this lock.
32 */
33static DECLARE_RWSEM(bpf_devs_lock);
34
35struct bpf_offload_dev {
36 const struct bpf_prog_offload_ops *ops;
37 struct list_head netdevs;
38 void *priv;
39};
40
41struct bpf_offload_netdev {
42 struct rhash_head l;
43 struct net_device *netdev;
44 struct bpf_offload_dev *offdev; /* NULL when bound-only */
45 struct list_head progs;
46 struct list_head maps;
47 struct list_head offdev_netdevs;
48};
49
50static const struct rhashtable_params offdevs_params = {
51 .nelem_hint = 4,
52 .key_len = sizeof(struct net_device *),
53 .key_offset = offsetof(struct bpf_offload_netdev, netdev),
54 .head_offset = offsetof(struct bpf_offload_netdev, l),
55 .automatic_shrinking = true,
56};
57
58static struct rhashtable offdevs;
59
60static int bpf_dev_offload_check(struct net_device *netdev)
61{
62 if (!netdev)
63 return -EINVAL;
64 if (!netdev->netdev_ops->ndo_bpf)
65 return -EOPNOTSUPP;
66 return 0;
67}
68
69static struct bpf_offload_netdev *
70bpf_offload_find_netdev(struct net_device *netdev)
71{
72 lockdep_assert_held(&bpf_devs_lock);
73
74 return rhashtable_lookup_fast(&offdevs, &netdev, offdevs_params);
75}
76
77static int __bpf_offload_dev_netdev_register(struct bpf_offload_dev *offdev,
78 struct net_device *netdev)
79{
80 struct bpf_offload_netdev *ondev;
81 int err;
82
83 ondev = kzalloc(sizeof(*ondev), GFP_KERNEL);
84 if (!ondev)
85 return -ENOMEM;
86
87 ondev->netdev = netdev;
88 ondev->offdev = offdev;
89 INIT_LIST_HEAD(&ondev->progs);
90 INIT_LIST_HEAD(&ondev->maps);
91
92 err = rhashtable_insert_fast(&offdevs, &ondev->l, offdevs_params);
93 if (err) {
94 netdev_warn(netdev, "failed to register for BPF offload\n");
95 goto err_free;
96 }
97
98 if (offdev)
99 list_add(&ondev->offdev_netdevs, &offdev->netdevs);
100 return 0;
101
102err_free:
103 kfree(ondev);
104 return err;
105}
106
107static void __bpf_prog_offload_destroy(struct bpf_prog *prog)
108{
109 struct bpf_prog_offload *offload = prog->aux->offload;
110
111 if (offload->dev_state)
112 offload->offdev->ops->destroy(prog);
113
114 list_del_init(&offload->offloads);
115 kfree(offload);
116 prog->aux->offload = NULL;
117}
118
119static int bpf_map_offload_ndo(struct bpf_offloaded_map *offmap,
120 enum bpf_netdev_command cmd)
121{
122 struct netdev_bpf data = {};
123 struct net_device *netdev;
124
125 ASSERT_RTNL();
126
127 data.command = cmd;
128 data.offmap = offmap;
129 /* Caller must make sure netdev is valid */
130 netdev = offmap->netdev;
131
132 return netdev->netdev_ops->ndo_bpf(netdev, &data);
133}
134
135static void __bpf_map_offload_destroy(struct bpf_offloaded_map *offmap)
136{
137 WARN_ON(bpf_map_offload_ndo(offmap, BPF_OFFLOAD_MAP_FREE));
138 /* Make sure BPF_MAP_GET_NEXT_ID can't find this dead map */
139 bpf_map_free_id(&offmap->map);
140 list_del_init(&offmap->offloads);
141 offmap->netdev = NULL;
142}
143
144static void __bpf_offload_dev_netdev_unregister(struct bpf_offload_dev *offdev,
145 struct net_device *netdev)
146{
147 struct bpf_offload_netdev *ondev, *altdev = NULL;
148 struct bpf_offloaded_map *offmap, *mtmp;
149 struct bpf_prog_offload *offload, *ptmp;
150
151 ASSERT_RTNL();
152
153 ondev = rhashtable_lookup_fast(&offdevs, &netdev, offdevs_params);
154 if (WARN_ON(!ondev))
155 return;
156
157 WARN_ON(rhashtable_remove_fast(&offdevs, &ondev->l, offdevs_params));
158
159 /* Try to move the objects to another netdev of the device */
160 if (offdev) {
161 list_del(&ondev->offdev_netdevs);
162 altdev = list_first_entry_or_null(&offdev->netdevs,
163 struct bpf_offload_netdev,
164 offdev_netdevs);
165 }
166
167 if (altdev) {
168 list_for_each_entry(offload, &ondev->progs, offloads)
169 offload->netdev = altdev->netdev;
170 list_splice_init(&ondev->progs, &altdev->progs);
171
172 list_for_each_entry(offmap, &ondev->maps, offloads)
173 offmap->netdev = altdev->netdev;
174 list_splice_init(&ondev->maps, &altdev->maps);
175 } else {
176 list_for_each_entry_safe(offload, ptmp, &ondev->progs, offloads)
177 __bpf_prog_offload_destroy(offload->prog);
178 list_for_each_entry_safe(offmap, mtmp, &ondev->maps, offloads)
179 __bpf_map_offload_destroy(offmap);
180 }
181
182 WARN_ON(!list_empty(&ondev->progs));
183 WARN_ON(!list_empty(&ondev->maps));
184 kfree(ondev);
185}
186
187static int __bpf_prog_dev_bound_init(struct bpf_prog *prog, struct net_device *netdev)
188{
189 struct bpf_offload_netdev *ondev;
190 struct bpf_prog_offload *offload;
191 int err;
192
193 offload = kzalloc(sizeof(*offload), GFP_USER);
194 if (!offload)
195 return -ENOMEM;
196
197 offload->prog = prog;
198 offload->netdev = netdev;
199
200 ondev = bpf_offload_find_netdev(offload->netdev);
201 if (!ondev) {
202 if (bpf_prog_is_offloaded(prog->aux)) {
203 err = -EINVAL;
204 goto err_free;
205 }
206
207 /* When only binding to the device, explicitly
208 * create an entry in the hashtable.
209 */
210 err = __bpf_offload_dev_netdev_register(NULL, offload->netdev);
211 if (err)
212 goto err_free;
213 ondev = bpf_offload_find_netdev(offload->netdev);
214 }
215 offload->offdev = ondev->offdev;
216 prog->aux->offload = offload;
217 list_add_tail(&offload->offloads, &ondev->progs);
218
219 return 0;
220err_free:
221 kfree(offload);
222 return err;
223}
224
225int bpf_prog_dev_bound_init(struct bpf_prog *prog, union bpf_attr *attr)
226{
227 struct net_device *netdev;
228 int err;
229
230 if (attr->prog_type != BPF_PROG_TYPE_SCHED_CLS &&
231 attr->prog_type != BPF_PROG_TYPE_XDP)
232 return -EINVAL;
233
234 if (attr->prog_flags & ~BPF_F_XDP_DEV_BOUND_ONLY)
235 return -EINVAL;
236
237 if (attr->prog_type == BPF_PROG_TYPE_SCHED_CLS &&
238 attr->prog_flags & BPF_F_XDP_DEV_BOUND_ONLY)
239 return -EINVAL;
240
241 netdev = dev_get_by_index(current->nsproxy->net_ns, attr->prog_ifindex);
242 if (!netdev)
243 return -EINVAL;
244
245 err = bpf_dev_offload_check(netdev);
246 if (err)
247 goto out;
248
249 prog->aux->offload_requested = !(attr->prog_flags & BPF_F_XDP_DEV_BOUND_ONLY);
250
251 down_write(&bpf_devs_lock);
252 err = __bpf_prog_dev_bound_init(prog, netdev);
253 up_write(&bpf_devs_lock);
254
255out:
256 dev_put(netdev);
257 return err;
258}
259
260int bpf_prog_dev_bound_inherit(struct bpf_prog *new_prog, struct bpf_prog *old_prog)
261{
262 int err;
263
264 if (!bpf_prog_is_dev_bound(old_prog->aux))
265 return 0;
266
267 if (bpf_prog_is_offloaded(old_prog->aux))
268 return -EINVAL;
269
270 new_prog->aux->dev_bound = old_prog->aux->dev_bound;
271 new_prog->aux->offload_requested = old_prog->aux->offload_requested;
272
273 down_write(&bpf_devs_lock);
274 if (!old_prog->aux->offload) {
275 err = -EINVAL;
276 goto out;
277 }
278
279 err = __bpf_prog_dev_bound_init(new_prog, old_prog->aux->offload->netdev);
280
281out:
282 up_write(&bpf_devs_lock);
283 return err;
284}
285
286int bpf_prog_offload_verifier_prep(struct bpf_prog *prog)
287{
288 struct bpf_prog_offload *offload;
289 int ret = -ENODEV;
290
291 down_read(&bpf_devs_lock);
292 offload = prog->aux->offload;
293 if (offload) {
294 ret = offload->offdev->ops->prepare(prog);
295 offload->dev_state = !ret;
296 }
297 up_read(&bpf_devs_lock);
298
299 return ret;
300}
301
302int bpf_prog_offload_verify_insn(struct bpf_verifier_env *env,
303 int insn_idx, int prev_insn_idx)
304{
305 struct bpf_prog_offload *offload;
306 int ret = -ENODEV;
307
308 down_read(&bpf_devs_lock);
309 offload = env->prog->aux->offload;
310 if (offload)
311 ret = offload->offdev->ops->insn_hook(env, insn_idx,
312 prev_insn_idx);
313 up_read(&bpf_devs_lock);
314
315 return ret;
316}
317
318int bpf_prog_offload_finalize(struct bpf_verifier_env *env)
319{
320 struct bpf_prog_offload *offload;
321 int ret = -ENODEV;
322
323 down_read(&bpf_devs_lock);
324 offload = env->prog->aux->offload;
325 if (offload) {
326 if (offload->offdev->ops->finalize)
327 ret = offload->offdev->ops->finalize(env);
328 else
329 ret = 0;
330 }
331 up_read(&bpf_devs_lock);
332
333 return ret;
334}
335
336void
337bpf_prog_offload_replace_insn(struct bpf_verifier_env *env, u32 off,
338 struct bpf_insn *insn)
339{
340 const struct bpf_prog_offload_ops *ops;
341 struct bpf_prog_offload *offload;
342 int ret = -EOPNOTSUPP;
343
344 down_read(&bpf_devs_lock);
345 offload = env->prog->aux->offload;
346 if (offload) {
347 ops = offload->offdev->ops;
348 if (!offload->opt_failed && ops->replace_insn)
349 ret = ops->replace_insn(env, off, insn);
350 offload->opt_failed |= ret;
351 }
352 up_read(&bpf_devs_lock);
353}
354
355void
356bpf_prog_offload_remove_insns(struct bpf_verifier_env *env, u32 off, u32 cnt)
357{
358 struct bpf_prog_offload *offload;
359 int ret = -EOPNOTSUPP;
360
361 down_read(&bpf_devs_lock);
362 offload = env->prog->aux->offload;
363 if (offload) {
364 if (!offload->opt_failed && offload->offdev->ops->remove_insns)
365 ret = offload->offdev->ops->remove_insns(env, off, cnt);
366 offload->opt_failed |= ret;
367 }
368 up_read(&bpf_devs_lock);
369}
370
371void bpf_prog_dev_bound_destroy(struct bpf_prog *prog)
372{
373 struct bpf_offload_netdev *ondev;
374 struct net_device *netdev;
375
376 rtnl_lock();
377 down_write(&bpf_devs_lock);
378 if (prog->aux->offload) {
379 list_del_init(&prog->aux->offload->offloads);
380
381 netdev = prog->aux->offload->netdev;
382 __bpf_prog_offload_destroy(prog);
383
384 ondev = bpf_offload_find_netdev(netdev);
385 if (!ondev->offdev && list_empty(&ondev->progs))
386 __bpf_offload_dev_netdev_unregister(NULL, netdev);
387 }
388 up_write(&bpf_devs_lock);
389 rtnl_unlock();
390}
391
392static int bpf_prog_offload_translate(struct bpf_prog *prog)
393{
394 struct bpf_prog_offload *offload;
395 int ret = -ENODEV;
396
397 down_read(&bpf_devs_lock);
398 offload = prog->aux->offload;
399 if (offload)
400 ret = offload->offdev->ops->translate(prog);
401 up_read(&bpf_devs_lock);
402
403 return ret;
404}
405
406static unsigned int bpf_prog_warn_on_exec(const void *ctx,
407 const struct bpf_insn *insn)
408{
409 WARN(1, "attempt to execute device eBPF program on the host!");
410 return 0;
411}
412
413int bpf_prog_offload_compile(struct bpf_prog *prog)
414{
415 prog->bpf_func = bpf_prog_warn_on_exec;
416
417 return bpf_prog_offload_translate(prog);
418}
419
420struct ns_get_path_bpf_prog_args {
421 struct bpf_prog *prog;
422 struct bpf_prog_info *info;
423};
424
425static struct ns_common *bpf_prog_offload_info_fill_ns(void *private_data)
426{
427 struct ns_get_path_bpf_prog_args *args = private_data;
428 struct bpf_prog_aux *aux = args->prog->aux;
429 struct ns_common *ns;
430 struct net *net;
431
432 rtnl_lock();
433 down_read(&bpf_devs_lock);
434
435 if (aux->offload) {
436 args->info->ifindex = aux->offload->netdev->ifindex;
437 net = dev_net(aux->offload->netdev);
438 get_net(net);
439 ns = &net->ns;
440 } else {
441 args->info->ifindex = 0;
442 ns = NULL;
443 }
444
445 up_read(&bpf_devs_lock);
446 rtnl_unlock();
447
448 return ns;
449}
450
451int bpf_prog_offload_info_fill(struct bpf_prog_info *info,
452 struct bpf_prog *prog)
453{
454 struct ns_get_path_bpf_prog_args args = {
455 .prog = prog,
456 .info = info,
457 };
458 struct bpf_prog_aux *aux = prog->aux;
459 struct inode *ns_inode;
460 struct path ns_path;
461 char __user *uinsns;
462 int res;
463 u32 ulen;
464
465 res = ns_get_path_cb(&ns_path, bpf_prog_offload_info_fill_ns, &args);
466 if (res) {
467 if (!info->ifindex)
468 return -ENODEV;
469 return res;
470 }
471
472 down_read(&bpf_devs_lock);
473
474 if (!aux->offload) {
475 up_read(&bpf_devs_lock);
476 return -ENODEV;
477 }
478
479 ulen = info->jited_prog_len;
480 info->jited_prog_len = aux->offload->jited_len;
481 if (info->jited_prog_len && ulen) {
482 uinsns = u64_to_user_ptr(info->jited_prog_insns);
483 ulen = min_t(u32, info->jited_prog_len, ulen);
484 if (copy_to_user(uinsns, aux->offload->jited_image, ulen)) {
485 up_read(&bpf_devs_lock);
486 return -EFAULT;
487 }
488 }
489
490 up_read(&bpf_devs_lock);
491
492 ns_inode = ns_path.dentry->d_inode;
493 info->netns_dev = new_encode_dev(ns_inode->i_sb->s_dev);
494 info->netns_ino = ns_inode->i_ino;
495 path_put(&ns_path);
496
497 return 0;
498}
499
500const struct bpf_prog_ops bpf_offload_prog_ops = {
501};
502
503struct bpf_map *bpf_map_offload_map_alloc(union bpf_attr *attr)
504{
505 struct net *net = current->nsproxy->net_ns;
506 struct bpf_offload_netdev *ondev;
507 struct bpf_offloaded_map *offmap;
508 int err;
509
510 if (!capable(CAP_SYS_ADMIN))
511 return ERR_PTR(-EPERM);
512 if (attr->map_type != BPF_MAP_TYPE_ARRAY &&
513 attr->map_type != BPF_MAP_TYPE_HASH)
514 return ERR_PTR(-EINVAL);
515
516 offmap = bpf_map_area_alloc(sizeof(*offmap), NUMA_NO_NODE);
517 if (!offmap)
518 return ERR_PTR(-ENOMEM);
519
520 bpf_map_init_from_attr(&offmap->map, attr);
521
522 rtnl_lock();
523 down_write(&bpf_devs_lock);
524 offmap->netdev = __dev_get_by_index(net, attr->map_ifindex);
525 err = bpf_dev_offload_check(offmap->netdev);
526 if (err)
527 goto err_unlock;
528
529 ondev = bpf_offload_find_netdev(offmap->netdev);
530 if (!ondev) {
531 err = -EINVAL;
532 goto err_unlock;
533 }
534
535 err = bpf_map_offload_ndo(offmap, BPF_OFFLOAD_MAP_ALLOC);
536 if (err)
537 goto err_unlock;
538
539 list_add_tail(&offmap->offloads, &ondev->maps);
540 up_write(&bpf_devs_lock);
541 rtnl_unlock();
542
543 return &offmap->map;
544
545err_unlock:
546 up_write(&bpf_devs_lock);
547 rtnl_unlock();
548 bpf_map_area_free(offmap);
549 return ERR_PTR(err);
550}
551
552void bpf_map_offload_map_free(struct bpf_map *map)
553{
554 struct bpf_offloaded_map *offmap = map_to_offmap(map);
555
556 rtnl_lock();
557 down_write(&bpf_devs_lock);
558 if (offmap->netdev)
559 __bpf_map_offload_destroy(offmap);
560 up_write(&bpf_devs_lock);
561 rtnl_unlock();
562
563 bpf_map_area_free(offmap);
564}
565
566u64 bpf_map_offload_map_mem_usage(const struct bpf_map *map)
567{
568 /* The memory dynamically allocated in netdev dev_ops is not counted */
569 return sizeof(struct bpf_offloaded_map);
570}
571
572int bpf_map_offload_lookup_elem(struct bpf_map *map, void *key, void *value)
573{
574 struct bpf_offloaded_map *offmap = map_to_offmap(map);
575 int ret = -ENODEV;
576
577 down_read(&bpf_devs_lock);
578 if (offmap->netdev)
579 ret = offmap->dev_ops->map_lookup_elem(offmap, key, value);
580 up_read(&bpf_devs_lock);
581
582 return ret;
583}
584
585int bpf_map_offload_update_elem(struct bpf_map *map,
586 void *key, void *value, u64 flags)
587{
588 struct bpf_offloaded_map *offmap = map_to_offmap(map);
589 int ret = -ENODEV;
590
591 if (unlikely(flags > BPF_EXIST))
592 return -EINVAL;
593
594 down_read(&bpf_devs_lock);
595 if (offmap->netdev)
596 ret = offmap->dev_ops->map_update_elem(offmap, key, value,
597 flags);
598 up_read(&bpf_devs_lock);
599
600 return ret;
601}
602
603int bpf_map_offload_delete_elem(struct bpf_map *map, void *key)
604{
605 struct bpf_offloaded_map *offmap = map_to_offmap(map);
606 int ret = -ENODEV;
607
608 down_read(&bpf_devs_lock);
609 if (offmap->netdev)
610 ret = offmap->dev_ops->map_delete_elem(offmap, key);
611 up_read(&bpf_devs_lock);
612
613 return ret;
614}
615
616int bpf_map_offload_get_next_key(struct bpf_map *map, void *key, void *next_key)
617{
618 struct bpf_offloaded_map *offmap = map_to_offmap(map);
619 int ret = -ENODEV;
620
621 down_read(&bpf_devs_lock);
622 if (offmap->netdev)
623 ret = offmap->dev_ops->map_get_next_key(offmap, key, next_key);
624 up_read(&bpf_devs_lock);
625
626 return ret;
627}
628
629struct ns_get_path_bpf_map_args {
630 struct bpf_offloaded_map *offmap;
631 struct bpf_map_info *info;
632};
633
634static struct ns_common *bpf_map_offload_info_fill_ns(void *private_data)
635{
636 struct ns_get_path_bpf_map_args *args = private_data;
637 struct ns_common *ns;
638 struct net *net;
639
640 rtnl_lock();
641 down_read(&bpf_devs_lock);
642
643 if (args->offmap->netdev) {
644 args->info->ifindex = args->offmap->netdev->ifindex;
645 net = dev_net(args->offmap->netdev);
646 get_net(net);
647 ns = &net->ns;
648 } else {
649 args->info->ifindex = 0;
650 ns = NULL;
651 }
652
653 up_read(&bpf_devs_lock);
654 rtnl_unlock();
655
656 return ns;
657}
658
659int bpf_map_offload_info_fill(struct bpf_map_info *info, struct bpf_map *map)
660{
661 struct ns_get_path_bpf_map_args args = {
662 .offmap = map_to_offmap(map),
663 .info = info,
664 };
665 struct inode *ns_inode;
666 struct path ns_path;
667 int res;
668
669 res = ns_get_path_cb(&ns_path, bpf_map_offload_info_fill_ns, &args);
670 if (res) {
671 if (!info->ifindex)
672 return -ENODEV;
673 return res;
674 }
675
676 ns_inode = ns_path.dentry->d_inode;
677 info->netns_dev = new_encode_dev(ns_inode->i_sb->s_dev);
678 info->netns_ino = ns_inode->i_ino;
679 path_put(&ns_path);
680
681 return 0;
682}
683
684static bool __bpf_offload_dev_match(struct bpf_prog *prog,
685 struct net_device *netdev)
686{
687 struct bpf_offload_netdev *ondev1, *ondev2;
688 struct bpf_prog_offload *offload;
689
690 if (!bpf_prog_is_dev_bound(prog->aux))
691 return false;
692
693 offload = prog->aux->offload;
694 if (!offload)
695 return false;
696 if (offload->netdev == netdev)
697 return true;
698
699 ondev1 = bpf_offload_find_netdev(offload->netdev);
700 ondev2 = bpf_offload_find_netdev(netdev);
701
702 return ondev1 && ondev2 && ondev1->offdev == ondev2->offdev;
703}
704
705bool bpf_offload_dev_match(struct bpf_prog *prog, struct net_device *netdev)
706{
707 bool ret;
708
709 down_read(&bpf_devs_lock);
710 ret = __bpf_offload_dev_match(prog, netdev);
711 up_read(&bpf_devs_lock);
712
713 return ret;
714}
715EXPORT_SYMBOL_GPL(bpf_offload_dev_match);
716
717bool bpf_prog_dev_bound_match(const struct bpf_prog *lhs, const struct bpf_prog *rhs)
718{
719 bool ret;
720
721 if (bpf_prog_is_offloaded(lhs->aux) != bpf_prog_is_offloaded(rhs->aux))
722 return false;
723
724 down_read(&bpf_devs_lock);
725 ret = lhs->aux->offload && rhs->aux->offload &&
726 lhs->aux->offload->netdev &&
727 lhs->aux->offload->netdev == rhs->aux->offload->netdev;
728 up_read(&bpf_devs_lock);
729
730 return ret;
731}
732
733bool bpf_offload_prog_map_match(struct bpf_prog *prog, struct bpf_map *map)
734{
735 struct bpf_offloaded_map *offmap;
736 bool ret;
737
738 if (!bpf_map_is_offloaded(map))
739 return bpf_map_offload_neutral(map);
740 offmap = map_to_offmap(map);
741
742 down_read(&bpf_devs_lock);
743 ret = __bpf_offload_dev_match(prog, offmap->netdev);
744 up_read(&bpf_devs_lock);
745
746 return ret;
747}
748
749int bpf_offload_dev_netdev_register(struct bpf_offload_dev *offdev,
750 struct net_device *netdev)
751{
752 int err;
753
754 down_write(&bpf_devs_lock);
755 err = __bpf_offload_dev_netdev_register(offdev, netdev);
756 up_write(&bpf_devs_lock);
757 return err;
758}
759EXPORT_SYMBOL_GPL(bpf_offload_dev_netdev_register);
760
761void bpf_offload_dev_netdev_unregister(struct bpf_offload_dev *offdev,
762 struct net_device *netdev)
763{
764 down_write(&bpf_devs_lock);
765 __bpf_offload_dev_netdev_unregister(offdev, netdev);
766 up_write(&bpf_devs_lock);
767}
768EXPORT_SYMBOL_GPL(bpf_offload_dev_netdev_unregister);
769
770struct bpf_offload_dev *
771bpf_offload_dev_create(const struct bpf_prog_offload_ops *ops, void *priv)
772{
773 struct bpf_offload_dev *offdev;
774
775 offdev = kzalloc(sizeof(*offdev), GFP_KERNEL);
776 if (!offdev)
777 return ERR_PTR(-ENOMEM);
778
779 offdev->ops = ops;
780 offdev->priv = priv;
781 INIT_LIST_HEAD(&offdev->netdevs);
782
783 return offdev;
784}
785EXPORT_SYMBOL_GPL(bpf_offload_dev_create);
786
787void bpf_offload_dev_destroy(struct bpf_offload_dev *offdev)
788{
789 WARN_ON(!list_empty(&offdev->netdevs));
790 kfree(offdev);
791}
792EXPORT_SYMBOL_GPL(bpf_offload_dev_destroy);
793
794void *bpf_offload_dev_priv(struct bpf_offload_dev *offdev)
795{
796 return offdev->priv;
797}
798EXPORT_SYMBOL_GPL(bpf_offload_dev_priv);
799
800void bpf_dev_bound_netdev_unregister(struct net_device *dev)
801{
802 struct bpf_offload_netdev *ondev;
803
804 ASSERT_RTNL();
805
806 down_write(&bpf_devs_lock);
807 ondev = bpf_offload_find_netdev(dev);
808 if (ondev && !ondev->offdev)
809 __bpf_offload_dev_netdev_unregister(NULL, ondev->netdev);
810 up_write(&bpf_devs_lock);
811}
812
813int bpf_dev_bound_kfunc_check(struct bpf_verifier_log *log,
814 struct bpf_prog_aux *prog_aux)
815{
816 if (!bpf_prog_is_dev_bound(prog_aux)) {
817 bpf_log(log, "metadata kfuncs require device-bound program\n");
818 return -EINVAL;
819 }
820
821 if (bpf_prog_is_offloaded(prog_aux)) {
822 bpf_log(log, "metadata kfuncs can't be offloaded\n");
823 return -EINVAL;
824 }
825
826 return 0;
827}
828
829void *bpf_dev_bound_resolve_kfunc(struct bpf_prog *prog, u32 func_id)
830{
831 const struct xdp_metadata_ops *ops;
832 void *p = NULL;
833
834 /* We don't hold bpf_devs_lock while resolving several
835 * kfuncs and can race with the unregister_netdevice().
836 * We rely on bpf_dev_bound_match() check at attach
837 * to render this program unusable.
838 */
839 down_read(&bpf_devs_lock);
840 if (!prog->aux->offload)
841 goto out;
842
843 ops = prog->aux->offload->netdev->xdp_metadata_ops;
844 if (!ops)
845 goto out;
846
847 if (func_id == bpf_xdp_metadata_kfunc_id(XDP_METADATA_KFUNC_RX_TIMESTAMP))
848 p = ops->xmo_rx_timestamp;
849 else if (func_id == bpf_xdp_metadata_kfunc_id(XDP_METADATA_KFUNC_RX_HASH))
850 p = ops->xmo_rx_hash;
851out:
852 up_read(&bpf_devs_lock);
853
854 return p;
855}
856
857static int __init bpf_offload_init(void)
858{
859 return rhashtable_init(&offdevs, &offdevs_params);
860}
861
862core_initcall(bpf_offload_init);