projects/linux·blob master · refs

683 lines · 15.6 KiB

1// SPDX-License-Identifier: GPL-2.0-only
2/* Copyright (c) 2016 Thomas Graf <[email protected]>
3 */
4
5#include <linux/filter.h>
6#include <linux/kernel.h>
7#include <linux/module.h>
8#include <linux/skbuff.h>
9#include <linux/types.h>
10#include <linux/bpf.h>
11#include <net/flow.h>
12#include <net/lwtunnel.h>
13#include <net/gre.h>
14#include <net/ip.h>
15#include <net/ip6_route.h>
16
17struct bpf_lwt_prog {
18 struct bpf_prog *prog;
19 char *name;
20};
21
22struct bpf_lwt {
23 struct bpf_lwt_prog in;
24 struct bpf_lwt_prog out;
25 struct bpf_lwt_prog xmit;
26 int family;
27};
28
29#define MAX_PROG_NAME 256
30
31static inline struct bpf_lwt *bpf_lwt_lwtunnel(struct lwtunnel_state *lwt)
32{
33 return (struct bpf_lwt *)lwt->data;
34}
35
36#define NO_REDIRECT false
37#define CAN_REDIRECT true
38
39static int run_lwt_bpf(struct sk_buff *skb, struct bpf_lwt_prog *lwt,
40 struct dst_entry *dst, bool can_redirect)
41{
42 struct bpf_net_context __bpf_net_ctx, *bpf_net_ctx;
43 int ret;
44
45 /* Disabling BH is needed to protect per-CPU bpf_redirect_info between
46 * BPF prog and skb_do_redirect().
47 */
48 local_bh_disable();
49 bpf_net_ctx = bpf_net_ctx_set(&__bpf_net_ctx);
50 bpf_compute_data_pointers(skb);
51 ret = bpf_prog_run_save_cb(lwt->prog, skb);
52
53 switch (ret) {
54 case BPF_OK:
55 case BPF_LWT_REROUTE:
56 break;
57
58 case BPF_REDIRECT:
59 if (unlikely(!can_redirect)) {
60 pr_warn_once("Illegal redirect return code in prog %s\n",
61 lwt->name ? : "<unknown>");
62 ret = BPF_OK;
63 } else {
64 skb_reset_mac_header(skb);
65 skb_do_redirect(skb);
66 ret = BPF_REDIRECT;
67 }
68 break;
69
70 case BPF_DROP:
71 kfree_skb(skb);
72 ret = -EPERM;
73 break;
74
75 default:
76 pr_warn_once("bpf-lwt: Illegal return value %u, expect packet loss\n", ret);
77 kfree_skb(skb);
78 ret = -EINVAL;
79 break;
80 }
81
82 bpf_net_ctx_clear(bpf_net_ctx);
83 local_bh_enable();
84
85 return ret;
86}
87
88static int bpf_lwt_input_reroute(struct sk_buff *skb)
89{
90 enum skb_drop_reason reason;
91 int err = -EINVAL;
92
93 if (skb->protocol == htons(ETH_P_IP)) {
94 struct net_device *dev = skb_dst(skb)->dev;
95 const struct iphdr *iph = ip_hdr(skb);
96
97 dev_hold(dev);
98 skb_dst_drop(skb);
99 reason = ip_route_input_noref(skb, iph->daddr, iph->saddr,
100 ip4h_dscp(iph), dev);
101 err = reason ? -EINVAL : 0;
102 dev_put(dev);
103 } else if (skb->protocol == htons(ETH_P_IPV6)) {
104 skb_dst_drop(skb);
105 if (IS_ENABLED(CONFIG_IPV6)) {
106 ip6_route_input(skb);
107 err = skb_dst(skb)->error;
108 } else {
109 err = -EAFNOSUPPORT;
110 }
111 } else {
112 err = -EAFNOSUPPORT;
113 }
114
115 if (err)
116 goto err;
117 return dst_input(skb);
118
119err:
120 kfree_skb(skb);
121 return err;
122}
123
124static int bpf_input(struct sk_buff *skb)
125{
126 struct dst_entry *dst = skb_dst(skb);
127 struct bpf_lwt *bpf;
128 int ret;
129
130 bpf = bpf_lwt_lwtunnel(dst->lwtstate);
131 if (bpf->in.prog) {
132 ret = run_lwt_bpf(skb, &bpf->in, dst, NO_REDIRECT);
133 if (ret < 0)
134 return ret;
135 if (ret == BPF_LWT_REROUTE)
136 return bpf_lwt_input_reroute(skb);
137 }
138
139 if (unlikely(!dst->lwtstate->orig_input)) {
140 kfree_skb(skb);
141 return -EINVAL;
142 }
143
144 return dst->lwtstate->orig_input(skb);
145}
146
147static int bpf_output(struct net *net, struct sock *sk, struct sk_buff *skb)
148{
149 struct dst_entry *dst = skb_dst(skb);
150 struct bpf_lwt *bpf;
151 int ret;
152
153 bpf = bpf_lwt_lwtunnel(dst->lwtstate);
154 if (bpf->out.prog) {
155 ret = run_lwt_bpf(skb, &bpf->out, dst, NO_REDIRECT);
156 if (ret < 0)
157 return ret;
158 }
159
160 if (unlikely(!dst->lwtstate->orig_output)) {
161 pr_warn_once("orig_output not set on dst for prog %s\n",
162 bpf->out.name);
163 kfree_skb(skb);
164 return -EINVAL;
165 }
166
167 return dst->lwtstate->orig_output(net, sk, skb);
168}
169
170static int xmit_check_headroom(struct sk_buff *skb, int hroom)
171{
172 if (skb_headroom(skb) < hroom) {
173 int nhead = hroom - skb_headroom(skb);
174
175 if (pskb_expand_head(skb, nhead, 0, GFP_ATOMIC))
176 return -ENOMEM;
177 }
178
179 return 0;
180}
181
182static int bpf_lwt_xmit_reroute(struct sk_buff *skb)
183{
184 struct net_device *l3mdev = l3mdev_master_dev_rcu(skb_dst(skb)->dev);
185 int oif = l3mdev ? l3mdev->ifindex : 0;
186 struct dst_entry *dst = NULL;
187 int err = -EAFNOSUPPORT;
188 struct sock *sk;
189 struct net *net;
190 bool ipv4;
191
192 if (skb->protocol == htons(ETH_P_IP))
193 ipv4 = true;
194 else if (skb->protocol == htons(ETH_P_IPV6))
195 ipv4 = false;
196 else
197 goto err;
198
199 sk = sk_to_full_sk(skb->sk);
200 if (sk) {
201 if (sk->sk_bound_dev_if)
202 oif = sk->sk_bound_dev_if;
203 net = sock_net(sk);
204 } else {
205 net = dev_net(skb_dst(skb)->dev);
206 }
207
208 if (ipv4) {
209 struct iphdr *iph = ip_hdr(skb);
210 struct flowi4 fl4 = {};
211 struct rtable *rt;
212
213 fl4.flowi4_oif = oif;
214 fl4.flowi4_mark = skb->mark;
215 fl4.flowi4_uid = sock_net_uid(net, sk);
216 fl4.flowi4_dscp = ip4h_dscp(iph);
217 fl4.flowi4_flags = FLOWI_FLAG_ANYSRC;
218 fl4.flowi4_proto = iph->protocol;
219 fl4.daddr = iph->daddr;
220 fl4.saddr = iph->saddr;
221
222 rt = ip_route_output_key(net, &fl4);
223 if (IS_ERR(rt)) {
224 err = PTR_ERR(rt);
225 goto err;
226 }
227 dst = &rt->dst;
228 } else {
229 struct ipv6hdr *iph6 = ipv6_hdr(skb);
230 struct flowi6 fl6 = {};
231
232 fl6.flowi6_oif = oif;
233 fl6.flowi6_mark = skb->mark;
234 fl6.flowi6_uid = sock_net_uid(net, sk);
235 fl6.flowlabel = ip6_flowinfo(iph6);
236 fl6.flowi6_proto = iph6->nexthdr;
237 fl6.daddr = iph6->daddr;
238 fl6.saddr = iph6->saddr;
239
240 dst = ip6_dst_lookup_flow(net, skb->sk, &fl6, NULL);
241 if (IS_ERR(dst)) {
242 err = PTR_ERR(dst);
243 goto err;
244 }
245 }
246 if (unlikely(dst->error)) {
247 err = dst->error;
248 dst_release(dst);
249 goto err;
250 }
251
252 /* Although skb header was reserved in bpf_lwt_push_ip_encap(), it
253 * was done for the previous dst, so we are doing it here again, in
254 * case the new dst needs much more space. The call below is a noop
255 * if there is enough header space in skb.
256 */
257 err = skb_cow_head(skb, LL_RESERVED_SPACE(dst->dev));
258 if (unlikely(err)) {
259 dst_release(dst);
260 goto err;
261 }
262
263 skb_dst_drop(skb);
264 skb_dst_set(skb, dst);
265
266 err = dst_output(dev_net(skb_dst(skb)->dev), skb->sk, skb);
267 if (unlikely(err))
268 return net_xmit_errno(err);
269
270 /* ip[6]_finish_output2 understand LWTUNNEL_XMIT_DONE */
271 return LWTUNNEL_XMIT_DONE;
272
273err:
274 kfree_skb(skb);
275 return err;
276}
277
278static int bpf_xmit(struct sk_buff *skb)
279{
280 struct dst_entry *dst = skb_dst(skb);
281 struct bpf_lwt *bpf;
282
283 bpf = bpf_lwt_lwtunnel(dst->lwtstate);
284 if (bpf->xmit.prog) {
285 int hroom = LL_RESERVED_SPACE(dst->dev);
286 __be16 proto = skb->protocol;
287 int ret;
288
289 ret = run_lwt_bpf(skb, &bpf->xmit, dst, CAN_REDIRECT);
290 switch (ret) {
291 case BPF_OK:
292 /* If the header changed, e.g. via bpf_lwt_push_encap,
293 * BPF_LWT_REROUTE below should have been used if the
294 * protocol was also changed.
295 */
296 if (skb->protocol != proto) {
297 kfree_skb(skb);
298 return -EINVAL;
299 }
300 /* If the header was expanded, headroom might be too
301 * small for the L2 header to come, expand as needed.
302 * neigh_hh_output() copies the cached header in
303 * HH_DATA_MOD aligned chunks, so match the reservation
304 * made before LWT xmit.
305 */
306 ret = xmit_check_headroom(skb, hroom);
307 if (unlikely(ret))
308 return ret;
309
310 return LWTUNNEL_XMIT_CONTINUE;
311 case BPF_REDIRECT:
312 return LWTUNNEL_XMIT_DONE;
313 case BPF_LWT_REROUTE:
314 return bpf_lwt_xmit_reroute(skb);
315 default:
316 return ret;
317 }
318 }
319
320 return LWTUNNEL_XMIT_CONTINUE;
321}
322
323static void bpf_lwt_prog_destroy(struct bpf_lwt_prog *prog)
324{
325 if (prog->prog)
326 bpf_prog_put(prog->prog);
327
328 kfree(prog->name);
329}
330
331static void bpf_destroy_state(struct lwtunnel_state *lwt)
332{
333 struct bpf_lwt *bpf = bpf_lwt_lwtunnel(lwt);
334
335 bpf_lwt_prog_destroy(&bpf->in);
336 bpf_lwt_prog_destroy(&bpf->out);
337 bpf_lwt_prog_destroy(&bpf->xmit);
338}
339
340static const struct nla_policy bpf_prog_policy[LWT_BPF_PROG_MAX + 1] = {
341 [LWT_BPF_PROG_FD] = { .type = NLA_U32, },
342 [LWT_BPF_PROG_NAME] = { .type = NLA_NUL_STRING,
343 .len = MAX_PROG_NAME },
344};
345
346static int bpf_parse_prog(struct nlattr *attr, struct bpf_lwt_prog *prog,
347 enum bpf_prog_type type)
348{
349 struct nlattr *tb[LWT_BPF_PROG_MAX + 1];
350 struct bpf_prog *p;
351 int ret;
352 u32 fd;
353
354 ret = nla_parse_nested_deprecated(tb, LWT_BPF_PROG_MAX, attr,
355 bpf_prog_policy, NULL);
356 if (ret < 0)
357 return ret;
358
359 if (!tb[LWT_BPF_PROG_FD] || !tb[LWT_BPF_PROG_NAME])
360 return -EINVAL;
361
362 prog->name = nla_memdup(tb[LWT_BPF_PROG_NAME], GFP_ATOMIC);
363 if (!prog->name)
364 return -ENOMEM;
365
366 fd = nla_get_u32(tb[LWT_BPF_PROG_FD]);
367 p = bpf_prog_get_type(fd, type);
368 if (IS_ERR(p))
369 return PTR_ERR(p);
370
371 prog->prog = p;
372
373 return 0;
374}
375
376static const struct nla_policy bpf_nl_policy[LWT_BPF_MAX + 1] = {
377 [LWT_BPF_IN] = { .type = NLA_NESTED, },
378 [LWT_BPF_OUT] = { .type = NLA_NESTED, },
379 [LWT_BPF_XMIT] = { .type = NLA_NESTED, },
380 [LWT_BPF_XMIT_HEADROOM] = { .type = NLA_U32 },
381};
382
383static int bpf_build_state(struct net *net, struct nlattr *nla,
384 unsigned int family, const void *cfg,
385 struct lwtunnel_state **ts,
386 struct netlink_ext_ack *extack)
387{
388 struct nlattr *tb[LWT_BPF_MAX + 1];
389 struct lwtunnel_state *newts;
390 struct bpf_lwt *bpf;
391 int ret;
392
393 if (family != AF_INET && family != AF_INET6)
394 return -EAFNOSUPPORT;
395
396 ret = nla_parse_nested_deprecated(tb, LWT_BPF_MAX, nla, bpf_nl_policy,
397 extack);
398 if (ret < 0)
399 return ret;
400
401 if (!tb[LWT_BPF_IN] && !tb[LWT_BPF_OUT] && !tb[LWT_BPF_XMIT])
402 return -EINVAL;
403
404 newts = lwtunnel_state_alloc(sizeof(*bpf));
405 if (!newts)
406 return -ENOMEM;
407
408 newts->type = LWTUNNEL_ENCAP_BPF;
409 bpf = bpf_lwt_lwtunnel(newts);
410
411 if (tb[LWT_BPF_IN]) {
412 newts->flags |= LWTUNNEL_STATE_INPUT_REDIRECT;
413 ret = bpf_parse_prog(tb[LWT_BPF_IN], &bpf->in,
414 BPF_PROG_TYPE_LWT_IN);
415 if (ret < 0)
416 goto errout;
417 }
418
419 if (tb[LWT_BPF_OUT]) {
420 newts->flags |= LWTUNNEL_STATE_OUTPUT_REDIRECT;
421 ret = bpf_parse_prog(tb[LWT_BPF_OUT], &bpf->out,
422 BPF_PROG_TYPE_LWT_OUT);
423 if (ret < 0)
424 goto errout;
425 }
426
427 if (tb[LWT_BPF_XMIT]) {
428 newts->flags |= LWTUNNEL_STATE_XMIT_REDIRECT;
429 ret = bpf_parse_prog(tb[LWT_BPF_XMIT], &bpf->xmit,
430 BPF_PROG_TYPE_LWT_XMIT);
431 if (ret < 0)
432 goto errout;
433 }
434
435 if (tb[LWT_BPF_XMIT_HEADROOM]) {
436 u32 headroom = nla_get_u32(tb[LWT_BPF_XMIT_HEADROOM]);
437
438 if (headroom > LWT_BPF_MAX_HEADROOM) {
439 ret = -ERANGE;
440 goto errout;
441 }
442
443 newts->headroom = headroom;
444 }
445
446 bpf->family = family;
447 *ts = newts;
448
449 return 0;
450
451errout:
452 bpf_destroy_state(newts);
453 kfree(newts);
454 return ret;
455}
456
457static int bpf_fill_lwt_prog(struct sk_buff *skb, int attr,
458 struct bpf_lwt_prog *prog)
459{
460 struct nlattr *nest;
461
462 if (!prog->prog)
463 return 0;
464
465 nest = nla_nest_start_noflag(skb, attr);
466 if (!nest)
467 return -EMSGSIZE;
468
469 if (prog->name &&
470 nla_put_string(skb, LWT_BPF_PROG_NAME, prog->name))
471 return -EMSGSIZE;
472
473 return nla_nest_end(skb, nest);
474}
475
476static int bpf_fill_encap_info(struct sk_buff *skb, struct lwtunnel_state *lwt)
477{
478 struct bpf_lwt *bpf = bpf_lwt_lwtunnel(lwt);
479
480 if (bpf_fill_lwt_prog(skb, LWT_BPF_IN, &bpf->in) < 0 ||
481 bpf_fill_lwt_prog(skb, LWT_BPF_OUT, &bpf->out) < 0 ||
482 bpf_fill_lwt_prog(skb, LWT_BPF_XMIT, &bpf->xmit) < 0)
483 return -EMSGSIZE;
484
485 return 0;
486}
487
488static int bpf_encap_nlsize(struct lwtunnel_state *lwtstate)
489{
490 int nest_len = nla_total_size(sizeof(struct nlattr)) +
491 nla_total_size(MAX_PROG_NAME) + /* LWT_BPF_PROG_NAME */
492 0;
493
494 return nest_len + /* LWT_BPF_IN */
495 nest_len + /* LWT_BPF_OUT */
496 nest_len + /* LWT_BPF_XMIT */
497 0;
498}
499
500static int bpf_lwt_prog_cmp(struct bpf_lwt_prog *a, struct bpf_lwt_prog *b)
501{
502 /* FIXME:
503 * The LWT state is currently rebuilt for delete requests which
504 * results in a new bpf_prog instance. Comparing names for now.
505 */
506 if (!a->name && !b->name)
507 return 0;
508
509 if (!a->name || !b->name)
510 return 1;
511
512 return strcmp(a->name, b->name);
513}
514
515static int bpf_encap_cmp(struct lwtunnel_state *a, struct lwtunnel_state *b)
516{
517 struct bpf_lwt *a_bpf = bpf_lwt_lwtunnel(a);
518 struct bpf_lwt *b_bpf = bpf_lwt_lwtunnel(b);
519
520 return bpf_lwt_prog_cmp(&a_bpf->in, &b_bpf->in) ||
521 bpf_lwt_prog_cmp(&a_bpf->out, &b_bpf->out) ||
522 bpf_lwt_prog_cmp(&a_bpf->xmit, &b_bpf->xmit);
523}
524
525static const struct lwtunnel_encap_ops bpf_encap_ops = {
526 .build_state = bpf_build_state,
527 .destroy_state = bpf_destroy_state,
528 .input = bpf_input,
529 .output = bpf_output,
530 .xmit = bpf_xmit,
531 .fill_encap = bpf_fill_encap_info,
532 .get_encap_size = bpf_encap_nlsize,
533 .cmp_encap = bpf_encap_cmp,
534 .owner = THIS_MODULE,
535};
536
537static int handle_gso_type(struct sk_buff *skb, unsigned int gso_type,
538 int encap_len)
539{
540 struct skb_shared_info *shinfo = skb_shinfo(skb);
541
542 gso_type |= SKB_GSO_DODGY;
543 shinfo->gso_type |= gso_type;
544 skb_decrease_gso_size(shinfo, encap_len);
545 shinfo->gso_segs = 0;
546 return 0;
547}
548
549static int handle_gso_encap(struct sk_buff *skb, bool ipv4, int encap_len)
550{
551 int next_hdr_offset;
552 void *next_hdr;
553 __u8 protocol;
554
555 /* SCTP and UDP_L4 gso need more nuanced handling than what
556 * handle_gso_type() does above: skb_decrease_gso_size() is not enough.
557 * So at the moment only TCP GSO packets are let through.
558 */
559 if (!(skb_shinfo(skb)->gso_type & (SKB_GSO_TCPV4 | SKB_GSO_TCPV6)))
560 return -ENOTSUPP;
561
562 if (ipv4) {
563 protocol = ip_hdr(skb)->protocol;
564 next_hdr_offset = sizeof(struct iphdr);
565 next_hdr = skb_network_header(skb) + next_hdr_offset;
566 } else {
567 protocol = ipv6_hdr(skb)->nexthdr;
568 next_hdr_offset = sizeof(struct ipv6hdr);
569 next_hdr = skb_network_header(skb) + next_hdr_offset;
570 }
571
572 switch (protocol) {
573 case IPPROTO_GRE:
574 next_hdr_offset += sizeof(struct gre_base_hdr);
575 if (next_hdr_offset > encap_len)
576 return -EINVAL;
577
578 if (((struct gre_base_hdr *)next_hdr)->flags & GRE_CSUM)
579 return handle_gso_type(skb, SKB_GSO_GRE_CSUM,
580 encap_len);
581 return handle_gso_type(skb, SKB_GSO_GRE, encap_len);
582
583 case IPPROTO_UDP:
584 next_hdr_offset += sizeof(struct udphdr);
585 if (next_hdr_offset > encap_len)
586 return -EINVAL;
587
588 if (((struct udphdr *)next_hdr)->check)
589 return handle_gso_type(skb, SKB_GSO_UDP_TUNNEL_CSUM,
590 encap_len);
591 return handle_gso_type(skb, SKB_GSO_UDP_TUNNEL, encap_len);
592
593 case IPPROTO_IP:
594 case IPPROTO_IPV6:
595 if (ipv4)
596 return handle_gso_type(skb, SKB_GSO_IPXIP4, encap_len);
597 else
598 return handle_gso_type(skb, SKB_GSO_IPXIP6, encap_len);
599
600 default:
601 return -EPROTONOSUPPORT;
602 }
603}
604
605int bpf_lwt_push_ip_encap(struct sk_buff *skb, void *hdr, u32 len, bool ingress)
606{
607 bool is_udp_tunnel;
608 struct iphdr *iph;
609 bool ipv4;
610 int err;
611
612 if (unlikely(len < sizeof(struct iphdr) || len > LWT_BPF_MAX_HEADROOM))
613 return -EINVAL;
614
615 /* validate protocol and length */
616 iph = (struct iphdr *)hdr;
617 if (iph->version == 4) {
618 ipv4 = true;
619 if (unlikely(len < iph->ihl * 4))
620 return -EINVAL;
621 is_udp_tunnel = iph->protocol == IPPROTO_UDP;
622 if (unlikely(is_udp_tunnel && len < iph->ihl * 4 + sizeof(struct udphdr)))
623 return -EINVAL;
624 } else if (iph->version == 6) {
625 ipv4 = false;
626 if (unlikely(len < sizeof(struct ipv6hdr)))
627 return -EINVAL;
628 is_udp_tunnel = ((struct ipv6hdr *)iph)->nexthdr == NEXTHDR_UDP;
629 if (unlikely(is_udp_tunnel && len < sizeof(struct ipv6hdr) + sizeof(struct udphdr)))
630 return -EINVAL;
631 } else {
632 return -EINVAL;
633 }
634
635 if (ingress)
636 err = skb_cow_head(skb, len + skb->mac_len);
637 else
638 err = skb_cow_head(skb,
639 len + LL_RESERVED_SPACE(skb_dst(skb)->dev));
640 if (unlikely(err))
641 return err;
642
643 /* push the encap headers and fix pointers */
644 skb_reset_inner_headers(skb);
645 skb_reset_inner_mac_header(skb); /* mac header is not yet set */
646 skb_set_inner_protocol(skb, skb->protocol);
647 skb->encapsulation = 1;
648 skb_push(skb, len);
649 if (ingress)
650 skb_postpush_rcsum(skb, iph, len);
651 skb_reset_network_header(skb);
652 if (is_udp_tunnel) {
653 size_t iph_sz = ipv4 ? iph->ihl * 4 : sizeof(struct ipv6hdr);
654
655 skb_set_transport_header(skb, skb_network_offset(skb) + iph_sz);
656 }
657 memcpy(skb_network_header(skb), hdr, len);
658 bpf_compute_data_pointers(skb);
659 skb_clear_hash(skb);
660
661 if (ipv4) {
662 skb->protocol = htons(ETH_P_IP);
663 iph = ip_hdr(skb);
664
665 if (!iph->check)
666 iph->check = ip_fast_csum((unsigned char *)iph,
667 iph->ihl);
668 } else {
669 skb->protocol = htons(ETH_P_IPV6);
670 }
671
672 if (skb_is_gso(skb))
673 return handle_gso_encap(skb, ipv4, len);
674
675 return 0;
676}
677
678static int __init bpf_lwt_init(void)
679{
680 return lwtunnel_encap_add_ops(&bpf_encap_ops, LWTUNNEL_ENCAP_BPF);
681}
682
683subsys_initcall(bpf_lwt_init)