projects/linux·blob master · refs

2104 lines · 53.5 KiB

1// SPDX-License-Identifier: GPL-2.0-or-later
2/*
3 * IPv6 output functions
4 * Linux INET6 implementation
5 *
6 * Authors:
7 * Pedro Roque <[email protected]>
8 *
9 * Based on linux/net/ipv4/ip_output.c
10 *
11 * Changes:
12 * A.N.Kuznetsov : airthmetics in fragmentation.
13 * extension headers are implemented.
14 * route changes now work.
15 * ip6_forward does not confuse sniffers.
16 * etc.
17 *
18 * H. von Brand : Added missing #include <linux/string.h>
19 * Imran Patel : frag id should be in NBO
20 * Kazunori MIYAZAWA @USAGI
21 * : add ip6_append_data and related functions
22 * for datagram xmit
23 */
24
25#include <linux/errno.h>
26#include <linux/kernel.h>
27#include <linux/string.h>
28#include <linux/socket.h>
29#include <linux/net.h>
30#include <linux/netdevice.h>
31#include <linux/if_arp.h>
32#include <linux/in6.h>
33#include <linux/tcp.h>
34#include <linux/route.h>
35#include <linux/module.h>
36#include <linux/slab.h>
37
38#include <linux/bpf-cgroup.h>
39#include <linux/netfilter.h>
40#include <linux/netfilter_ipv6.h>
41
42#include <net/sock.h>
43#include <net/snmp.h>
44
45#include <net/gso.h>
46#include <net/ipv6.h>
47#include <net/ndisc.h>
48#include <net/protocol.h>
49#include <net/ip6_route.h>
50#include <net/addrconf.h>
51#include <net/rawv6.h>
52#include <net/icmp.h>
53#include <net/xfrm.h>
54#include <net/checksum.h>
55#include <linux/mroute6.h>
56#include <net/l3mdev.h>
57#include <net/lwtunnel.h>
58#include <net/ip_tunnels.h>
59
60static int ip6_finish_output2(struct net *net, struct sock *sk, struct sk_buff *skb)
61{
62 struct dst_entry *dst = skb_dst(skb);
63 struct net_device *dev = dst_dev_rcu(dst);
64 struct inet6_dev *idev = ip6_dst_idev(dst);
65 unsigned int hh_len = LL_RESERVED_SPACE(dev);
66 const struct in6_addr *daddr, *nexthop;
67 struct ipv6hdr *hdr;
68 struct neighbour *neigh;
69 int ret;
70
71 /* Be paranoid, rather than too clever. */
72 if (unlikely(hh_len > skb_headroom(skb)) && dev->header_ops) {
73 /* idev stays alive because we hold rcu_read_lock(). */
74 skb = skb_expand_head(skb, hh_len);
75 if (!skb) {
76 IP6_INC_STATS(net, idev, IPSTATS_MIB_OUTDISCARDS);
77 return -ENOMEM;
78 }
79 }
80
81 hdr = ipv6_hdr(skb);
82 daddr = &hdr->daddr;
83 if (unlikely(ipv6_addr_is_multicast(daddr))) {
84 if (!(dev->flags & IFF_LOOPBACK) && sk_mc_loop(sk) &&
85 ((mroute6_is_socket(net, skb) &&
86 !(IP6CB(skb)->flags & IP6SKB_FORWARDED)) ||
87 ipv6_chk_mcast_addr(dev, daddr, &hdr->saddr))) {
88 struct sk_buff *newskb = skb_clone(skb, GFP_ATOMIC);
89
90 /* Do not check for IFF_ALLMULTI; multicast routing
91 is not supported in any case.
92 */
93 if (newskb)
94 NF_HOOK(NFPROTO_IPV6, NF_INET_POST_ROUTING,
95 net, sk, newskb, NULL, newskb->dev,
96 dev_loopback_xmit);
97
98 if (hdr->hop_limit == 0) {
99 IP6_INC_STATS(net, idev,
100 IPSTATS_MIB_OUTDISCARDS);
101 kfree_skb(skb);
102 return 0;
103 }
104 }
105
106 IP6_UPD_PO_STATS(net, idev, IPSTATS_MIB_OUTMCAST, skb->len);
107 if (IPV6_ADDR_MC_SCOPE(daddr) <= IPV6_ADDR_SCOPE_NODELOCAL &&
108 !(dev->flags & IFF_LOOPBACK)) {
109 kfree_skb(skb);
110 return 0;
111 }
112 }
113
114 if (lwtunnel_xmit_redirect(dst->lwtstate)) {
115 int res = lwtunnel_xmit(skb);
116
117 if (res != LWTUNNEL_XMIT_CONTINUE)
118 return res;
119 hdr = ipv6_hdr(skb);
120 daddr = &hdr->daddr;
121 }
122
123 IP6_UPD_PO_STATS(net, idev, IPSTATS_MIB_OUT, skb->len);
124
125 nexthop = rt6_nexthop(dst_rt6_info(dst), daddr);
126 neigh = __ipv6_neigh_lookup_noref(dev, nexthop);
127
128 if (IS_ERR_OR_NULL(neigh)) {
129 if (unlikely(!neigh))
130 neigh = __neigh_create(&nd_tbl, nexthop, dev, false);
131 if (IS_ERR(neigh)) {
132 IP6_INC_STATS(net, idev, IPSTATS_MIB_OUTNOROUTES);
133 kfree_skb_reason(skb, SKB_DROP_REASON_NEIGH_CREATEFAIL);
134 return -EINVAL;
135 }
136 }
137 sock_confirm_neigh(skb, neigh);
138 ret = neigh_output(neigh, skb, false);
139 return ret;
140}
141
142static int
143ip6_finish_output_gso_slowpath_drop(struct net *net, struct sock *sk,
144 struct sk_buff *skb, unsigned int mtu)
145{
146 struct sk_buff *segs, *nskb;
147 netdev_features_t features;
148 int ret = 0;
149
150 /* Please see corresponding comment in ip_finish_output_gso
151 * describing the cases where GSO segment length exceeds the
152 * egress MTU.
153 */
154 features = netif_skb_features(skb);
155 segs = skb_gso_segment(skb, features & ~NETIF_F_GSO_MASK);
156 if (IS_ERR_OR_NULL(segs)) {
157 kfree_skb(skb);
158 return -ENOMEM;
159 }
160
161 consume_skb(skb);
162
163 skb_list_walk_safe(segs, segs, nskb) {
164 int err;
165
166 skb_mark_not_on_list(segs);
167 /* Last GSO segment can be smaller than gso_size (and MTU).
168 * Adding a fragment header would produce an "atomic fragment",
169 * which is considered harmful (RFC-8021). Avoid that.
170 */
171 err = segs->len > mtu ?
172 ip6_fragment(net, sk, segs, ip6_finish_output2) :
173 ip6_finish_output2(net, sk, segs);
174 if (err && ret == 0)
175 ret = err;
176 }
177
178 return ret;
179}
180
181static int ip6_finish_output_gso(struct net *net, struct sock *sk,
182 struct sk_buff *skb, unsigned int mtu)
183{
184 if (unlikely(!skb_gso_validate_network_len(skb, mtu)))
185 return ip6_finish_output_gso_slowpath_drop(net, sk, skb, mtu);
186
187 return ip6_finish_output2(net, sk, skb);
188}
189
190static int __ip6_finish_output(struct net *net, struct sock *sk, struct sk_buff *skb)
191{
192 unsigned int mtu;
193
194#if defined(CONFIG_NETFILTER) && defined(CONFIG_XFRM)
195 /* Policy lookup after SNAT yielded a new policy */
196 if (skb_dst(skb)->xfrm) {
197 IP6CB(skb)->flags |= IP6SKB_REROUTED;
198 return dst_output(net, sk, skb);
199 }
200#endif
201
202 mtu = ip6_skb_dst_mtu(skb);
203 if (skb_is_gso(skb))
204 return ip6_finish_output_gso(net, sk, skb, mtu);
205
206 if (unlikely(skb->len > mtu ||
207 (IP6CB(skb)->frag_max_size && skb->len > IP6CB(skb)->frag_max_size)))
208 return ip6_fragment(net, sk, skb, ip6_finish_output2);
209
210 return ip6_finish_output2(net, sk, skb);
211}
212
213static int ip6_finish_output(struct net *net, struct sock *sk, struct sk_buff *skb)
214{
215 int ret;
216
217 ret = BPF_CGROUP_RUN_PROG_INET_EGRESS(sk, skb);
218 switch (ret) {
219 case NET_XMIT_SUCCESS:
220 case NET_XMIT_CN:
221 return __ip6_finish_output(net, sk, skb) ? : ret;
222 default:
223 kfree_skb_reason(skb, SKB_DROP_REASON_BPF_CGROUP_EGRESS);
224 return ret;
225 }
226}
227
228int ip6_output(struct net *net, struct sock *sk, struct sk_buff *skb)
229{
230 struct dst_entry *dst = skb_dst(skb);
231 struct net_device *dev, *indev = skb->dev;
232 struct inet6_dev *idev;
233 int ret;
234
235 skb->protocol = htons(ETH_P_IPV6);
236 rcu_read_lock();
237 dev = dst_dev_rcu(dst);
238 idev = ip6_dst_idev(dst);
239 skb->dev = dev;
240
241 if (unlikely(!idev || READ_ONCE(idev->cnf.disable_ipv6))) {
242 IP6_INC_STATS(net, idev, IPSTATS_MIB_OUTDISCARDS);
243 rcu_read_unlock();
244 kfree_skb_reason(skb, SKB_DROP_REASON_IPV6DISABLED);
245 return 0;
246 }
247
248 ret = NF_HOOK_COND(NFPROTO_IPV6, NF_INET_POST_ROUTING,
249 net, sk, skb, indev, dev,
250 ip6_finish_output,
251 !(IP6CB(skb)->flags & IP6SKB_REROUTED));
252 rcu_read_unlock();
253 return ret;
254}
255EXPORT_SYMBOL(ip6_output);
256
257bool ip6_autoflowlabel(struct net *net, const struct sock *sk)
258{
259 if (!inet6_test_bit(AUTOFLOWLABEL_SET, sk))
260 return ip6_default_np_autolabel(net);
261 return inet6_test_bit(AUTOFLOWLABEL, sk);
262}
263
264int ip6_dst_hoplimit(struct dst_entry *dst)
265{
266 int hoplimit = dst_metric_raw(dst, RTAX_HOPLIMIT);
267
268 rcu_read_lock();
269 if (hoplimit == 0) {
270 struct net_device *dev = dst_dev_rcu(dst);
271 struct inet6_dev *idev;
272
273 idev = __in6_dev_get(dev);
274 if (idev)
275 hoplimit = READ_ONCE(idev->cnf.hop_limit);
276 else
277 hoplimit = READ_ONCE(dev_net(dev)->ipv6.devconf_all->hop_limit);
278 }
279 rcu_read_unlock();
280
281 return hoplimit;
282}
283EXPORT_SYMBOL(ip6_dst_hoplimit);
284
285/*
286 * xmit an sk_buff (used by TCP and SCTP)
287 * Note : socket lock is not held for SYNACK packets, but might be modified
288 * by calls to skb_set_owner_w() and ipv6_local_error(),
289 * which are using proper atomic operations or spinlocks.
290 */
291int ip6_xmit(const struct sock *sk, struct sk_buff *skb, struct flowi6 *fl6,
292 __u32 mark, struct ipv6_txoptions *opt, int tclass, u32 priority)
293{
294 const struct ipv6_pinfo *np = inet6_sk(sk);
295 struct in6_addr *first_hop = &fl6->daddr;
296 struct dst_entry *dst = skb_dst(skb);
297 struct inet6_dev *idev = ip6_dst_idev(dst);
298 struct net *net = sock_net(sk);
299 unsigned int head_room;
300 struct net_device *dev;
301 struct ipv6hdr *hdr;
302 u8 proto = fl6->flowi6_proto;
303 int seg_len = skb->len;
304 int ret, hlimit = -1;
305 u32 mtu;
306
307 rcu_read_lock();
308
309 dev = dst_dev_rcu(dst);
310 head_room = sizeof(struct ipv6hdr) + LL_RESERVED_SPACE(dev);
311 if (opt)
312 head_room += opt->opt_nflen + opt->opt_flen;
313
314 if (unlikely(head_room > skb_headroom(skb))) {
315 /* idev stays alive while we hold rcu_read_lock(). */
316 skb = skb_expand_head(skb, head_room);
317 if (!skb) {
318 IP6_INC_STATS(net, idev, IPSTATS_MIB_OUTDISCARDS);
319 ret = -ENOBUFS;
320 goto unlock;
321 }
322 }
323
324 if (unlikely(opt)) {
325 seg_len += opt->opt_nflen + opt->opt_flen;
326
327 if (opt->opt_flen)
328 proto = ipv6_push_frag_opts(skb, opt, proto);
329
330 if (opt->opt_nflen)
331 proto = ipv6_push_nfrag_opts(skb, opt, proto,
332 &first_hop,
333 &fl6->saddr);
334 }
335
336 if (unlikely(seg_len > IPV6_MAXPLEN))
337 seg_len = 0;
338
339 __skb_push(skb, sizeof(struct ipv6hdr));
340 skb_reset_network_header(skb);
341 hdr = ipv6_hdr(skb);
342
343 /*
344 * Fill in the IPv6 header
345 */
346 if (np)
347 hlimit = READ_ONCE(np->hop_limit);
348 if (hlimit < 0)
349 hlimit = ip6_dst_hoplimit(dst);
350
351 ip6_flow_hdr(hdr, tclass, ip6_make_flowlabel(net, skb, fl6->flowlabel,
352 ip6_autoflowlabel(net, sk), fl6));
353
354 hdr->payload_len = htons(seg_len);
355 hdr->nexthdr = proto;
356 hdr->hop_limit = hlimit;
357
358 hdr->saddr = fl6->saddr;
359 hdr->daddr = *first_hop;
360
361 skb->protocol = htons(ETH_P_IPV6);
362 skb->priority = priority;
363 skb->mark = mark;
364
365 mtu = dst6_mtu(dst);
366 if (likely((skb->len <= mtu) || skb->ignore_df || skb_is_gso(skb))) {
367 IP6_INC_STATS(net, idev, IPSTATS_MIB_OUTREQUESTS);
368
369 /* if egress device is enslaved to an L3 master device pass the
370 * skb to its handler for processing
371 */
372 skb = l3mdev_ip6_out((struct sock *)sk, skb);
373 if (unlikely(!skb)) {
374 ret = 0;
375 goto unlock;
376 }
377
378 /* hooks should never assume socket lock is held.
379 * we promote our socket to non const
380 */
381 ret = NF_HOOK(NFPROTO_IPV6, NF_INET_LOCAL_OUT,
382 net, (struct sock *)sk, skb, NULL, dev,
383 dst_output);
384 goto unlock;
385 }
386
387 ret = -EMSGSIZE;
388 skb->dev = dev;
389 /* ipv6_local_error() does not require socket lock,
390 * we promote our socket to non const
391 */
392 ipv6_local_error((struct sock *)sk, EMSGSIZE, fl6, mtu);
393
394 IP6_INC_STATS(net, idev, IPSTATS_MIB_FRAGFAILS);
395 kfree_skb_reason(skb, SKB_DROP_REASON_PKT_TOO_BIG);
396unlock:
397 rcu_read_unlock();
398 return ret;
399}
400EXPORT_SYMBOL(ip6_xmit);
401
402static int ip6_call_ra_chain(struct sk_buff *skb, int sel)
403{
404 struct ip6_ra_chain *ra;
405 struct sock *last = NULL;
406
407 read_lock(&ip6_ra_lock);
408 for (ra = ip6_ra_chain; ra; ra = ra->next) {
409 struct sock *sk = ra->sk;
410 if (sk && ra->sel == sel &&
411 (!sk->sk_bound_dev_if ||
412 sk->sk_bound_dev_if == skb->dev->ifindex)) {
413
414 if (inet6_test_bit(RTALERT_ISOLATE, sk) &&
415 !net_eq(sock_net(sk), dev_net(skb->dev))) {
416 continue;
417 }
418 if (last) {
419 struct sk_buff *skb2 = skb_clone(skb, GFP_ATOMIC);
420 if (skb2)
421 rawv6_rcv(last, skb2);
422 }
423 last = sk;
424 }
425 }
426
427 if (last) {
428 rawv6_rcv(last, skb);
429 read_unlock(&ip6_ra_lock);
430 return 1;
431 }
432 read_unlock(&ip6_ra_lock);
433 return 0;
434}
435
436static int ip6_forward_proxy_check(struct sk_buff *skb)
437{
438 struct ipv6hdr *hdr = ipv6_hdr(skb);
439 u8 nexthdr = hdr->nexthdr;
440 __be16 frag_off;
441 int offset;
442
443 if (ipv6_ext_hdr(nexthdr)) {
444 offset = ipv6_skip_exthdr(skb, sizeof(*hdr), &nexthdr, &frag_off);
445 if (offset < 0)
446 return 0;
447 } else
448 offset = sizeof(struct ipv6hdr);
449
450 if (nexthdr == IPPROTO_ICMPV6) {
451 struct icmp6hdr *icmp6;
452
453 if (!pskb_may_pull(skb, (skb_network_header(skb) +
454 offset + 1 - skb->data)))
455 return 0;
456
457 icmp6 = (struct icmp6hdr *)(skb_network_header(skb) + offset);
458
459 switch (icmp6->icmp6_type) {
460 case NDISC_ROUTER_SOLICITATION:
461 case NDISC_ROUTER_ADVERTISEMENT:
462 case NDISC_NEIGHBOUR_SOLICITATION:
463 case NDISC_NEIGHBOUR_ADVERTISEMENT:
464 case NDISC_REDIRECT:
465 /* For reaction involving unicast neighbor discovery
466 * message destined to the proxied address, pass it to
467 * input function.
468 */
469 return 1;
470 default:
471 break;
472 }
473 hdr = ipv6_hdr(skb);
474 }
475
476 /*
477 * The proxying router can't forward traffic sent to a link-local
478 * address, so signal the sender and discard the packet. This
479 * behavior is clarified by the MIPv6 specification.
480 */
481 if (ipv6_addr_type(&hdr->daddr) & IPV6_ADDR_LINKLOCAL) {
482 dst_link_failure(skb);
483 return -1;
484 }
485
486 return 0;
487}
488
489static inline int ip6_forward_finish(struct net *net, struct sock *sk,
490 struct sk_buff *skb)
491{
492#ifdef CONFIG_NET_SWITCHDEV
493 if (skb->offload_l3_fwd_mark) {
494 consume_skb(skb);
495 return 0;
496 }
497#endif
498
499 skb_clear_tstamp(skb);
500 return dst_output(net, sk, skb);
501}
502
503static bool ip6_pkt_too_big(const struct sk_buff *skb, unsigned int mtu)
504{
505 if (skb->len <= mtu)
506 return false;
507
508 /* ipv6 conntrack defrag sets max_frag_size + ignore_df */
509 if (IP6CB(skb)->frag_max_size && IP6CB(skb)->frag_max_size > mtu)
510 return true;
511
512 if (skb->ignore_df)
513 return false;
514
515 if (skb_is_gso(skb) && skb_gso_validate_network_len(skb, mtu))
516 return false;
517
518 return true;
519}
520
521int ip6_forward(struct sk_buff *skb)
522{
523 struct dst_entry *dst = skb_dst(skb);
524 struct ipv6hdr *hdr = ipv6_hdr(skb);
525 struct inet6_skb_parm *opt = IP6CB(skb);
526 struct net *net = dev_net(dst_dev(dst));
527 struct net_device *dev;
528 struct inet6_dev *idev;
529 SKB_DR(reason);
530 u32 mtu;
531
532 idev = __in6_dev_get_safely(dev_get_by_index_rcu(net, IP6CB(skb)->iif));
533 if (!READ_ONCE(net->ipv6.devconf_all->forwarding) &&
534 (!idev || !READ_ONCE(idev->cnf.force_forwarding)))
535 goto error;
536
537 if (skb->pkt_type != PACKET_HOST)
538 goto drop;
539
540 if (unlikely(skb->sk))
541 goto drop;
542
543 if (skb_warn_if_lro(skb))
544 goto drop;
545
546 if (!READ_ONCE(net->ipv6.devconf_all->disable_policy) &&
547 (!idev || !READ_ONCE(idev->cnf.disable_policy)) &&
548 !xfrm6_policy_check(NULL, XFRM_POLICY_FWD, skb)) {
549 __IP6_INC_STATS(net, idev, IPSTATS_MIB_INDISCARDS);
550 goto drop;
551 }
552
553 skb_forward_csum(skb);
554
555 /*
556 * We DO NOT make any processing on
557 * RA packets, pushing them to user level AS IS
558 * without ane WARRANTY that application will be able
559 * to interpret them. The reason is that we
560 * cannot make anything clever here.
561 *
562 * We are not end-node, so that if packet contains
563 * AH/ESP, we cannot make anything.
564 * Defragmentation also would be mistake, RA packets
565 * cannot be fragmented, because there is no warranty
566 * that different fragments will go along one path. --ANK
567 */
568 if (unlikely(opt->flags & IP6SKB_ROUTERALERT)) {
569 if (ip6_call_ra_chain(skb, ntohs(opt->ra)))
570 return 0;
571 }
572
573 /*
574 * check and decrement ttl
575 */
576 if (hdr->hop_limit <= 1) {
577 icmpv6_send(skb, ICMPV6_TIME_EXCEED, ICMPV6_EXC_HOPLIMIT, 0);
578 __IP6_INC_STATS(net, idev, IPSTATS_MIB_INHDRERRORS);
579
580 kfree_skb_reason(skb, SKB_DROP_REASON_IP_INHDR);
581 return -ETIMEDOUT;
582 }
583
584 /* XXX: idev->cnf.proxy_ndp? */
585 if (READ_ONCE(net->ipv6.devconf_all->proxy_ndp) &&
586 pneigh_lookup(&nd_tbl, net, &hdr->daddr, skb->dev)) {
587 int proxied = ip6_forward_proxy_check(skb);
588
589 hdr = ipv6_hdr(skb);
590 if (proxied > 0) {
591 /* It's tempting to decrease the hop limit
592 * here by 1, as we do at the end of the
593 * function too.
594 *
595 * But that would be incorrect, as proxying is
596 * not forwarding. The ip6_input function
597 * will handle this packet locally, and it
598 * depends on the hop limit being unchanged.
599 *
600 * One example is the NDP hop limit, that
601 * always has to stay 255, but other would be
602 * similar checks around RA packets, where the
603 * user can even change the desired limit.
604 */
605 return ip6_input(skb);
606 } else if (proxied < 0) {
607 __IP6_INC_STATS(net, idev, IPSTATS_MIB_INDISCARDS);
608 goto drop;
609 }
610 }
611
612 if (!xfrm6_route_forward(skb)) {
613 __IP6_INC_STATS(net, idev, IPSTATS_MIB_INDISCARDS);
614 SKB_DR_SET(reason, XFRM_POLICY);
615 goto drop;
616 }
617 dst = skb_dst(skb);
618 dev = dst_dev(dst);
619 /* IPv6 specs say nothing about it, but it is clear that we cannot
620 send redirects to source routed frames.
621 We don't send redirects to frames decapsulated from IPsec.
622 */
623 if (IP6CB(skb)->iif == dev->ifindex &&
624 opt->srcrt == 0 && !skb_sec_path(skb)) {
625 struct in6_addr *target = NULL;
626 struct inet_peer *peer;
627 struct rt6_info *rt;
628
629 /*
630 * incoming and outgoing devices are the same
631 * send a redirect.
632 */
633
634 rt = dst_rt6_info(dst);
635 if (rt->rt6i_flags & RTF_GATEWAY)
636 target = &rt->rt6i_gateway;
637 else
638 target = &hdr->daddr;
639
640 rcu_read_lock();
641 peer = inet_getpeer_v6(net->ipv6.peers, &hdr->daddr);
642
643 /* Limit redirects both by destination (here)
644 and by source (inside ndisc_send_redirect)
645 */
646 if (peer && inet_peer_xrlim_allow(peer, 1*HZ))
647 ndisc_send_redirect(skb, target);
648 rcu_read_unlock();
649 } else {
650 int addrtype = ipv6_addr_type(&hdr->saddr);
651
652 /* This check is security critical. */
653 if (addrtype == IPV6_ADDR_ANY ||
654 addrtype & (IPV6_ADDR_MULTICAST | IPV6_ADDR_LOOPBACK))
655 goto error;
656 if (addrtype & IPV6_ADDR_LINKLOCAL) {
657 icmpv6_send(skb, ICMPV6_DEST_UNREACH,
658 ICMPV6_NOT_NEIGHBOUR, 0);
659 goto error;
660 }
661 }
662
663 __IP6_INC_STATS(net, ip6_dst_idev(dst), IPSTATS_MIB_OUTFORWDATAGRAMS);
664
665 mtu = ip6_dst_mtu_maybe_forward(dst, true);
666 if (mtu < IPV6_MIN_MTU)
667 mtu = IPV6_MIN_MTU;
668
669 if (unlikely(ip6_pkt_too_big(skb, mtu))) {
670 /* Again, force OUTPUT device used as source address */
671 skb->dev = dev;
672 icmpv6_send(skb, ICMPV6_PKT_TOOBIG, 0, mtu);
673 __IP6_INC_STATS(net, idev, IPSTATS_MIB_INTOOBIGERRORS);
674 __IP6_INC_STATS(net, ip6_dst_idev(dst),
675 IPSTATS_MIB_FRAGFAILS);
676 kfree_skb_reason(skb, SKB_DROP_REASON_PKT_TOO_BIG);
677 return -EMSGSIZE;
678 }
679
680 if (skb_cow(skb, dev->hard_header_len)) {
681 __IP6_INC_STATS(net, ip6_dst_idev(dst),
682 IPSTATS_MIB_OUTDISCARDS);
683 goto drop;
684 }
685
686 hdr = ipv6_hdr(skb);
687
688 /* Mangling hops number delayed to point after skb COW */
689
690 hdr->hop_limit--;
691
692 return NF_HOOK(NFPROTO_IPV6, NF_INET_FORWARD,
693 net, NULL, skb, skb->dev, dev,
694 ip6_forward_finish);
695
696error:
697 __IP6_INC_STATS(net, idev, IPSTATS_MIB_INADDRERRORS);
698 SKB_DR_SET(reason, IP_INADDRERRORS);
699drop:
700 kfree_skb_reason(skb, reason);
701 return -EINVAL;
702}
703
704static void ip6_copy_metadata(struct sk_buff *to, struct sk_buff *from)
705{
706 to->pkt_type = from->pkt_type;
707 to->priority = from->priority;
708 to->protocol = from->protocol;
709 skb_dst_drop(to);
710 skb_dst_set(to, dst_clone(skb_dst(from)));
711 to->dev = from->dev;
712 to->mark = from->mark;
713
714 skb_copy_hash(to, from);
715
716#ifdef CONFIG_NET_SCHED
717 to->tc_index = from->tc_index;
718#endif
719 nf_copy(to, from);
720 skb_ext_copy(to, from);
721 skb_copy_secmark(to, from);
722}
723
724int ip6_fraglist_init(struct sk_buff *skb, unsigned int hlen, u8 *prevhdr,
725 u8 nexthdr, __be32 frag_id,
726 struct ip6_fraglist_iter *iter)
727{
728 unsigned int first_len;
729 struct frag_hdr *fh;
730
731 /* BUILD HEADER */
732 *prevhdr = NEXTHDR_FRAGMENT;
733 iter->tmp_hdr = kmemdup(skb_network_header(skb), hlen, GFP_ATOMIC);
734 if (!iter->tmp_hdr)
735 return -ENOMEM;
736
737 iter->frag = skb_shinfo(skb)->frag_list;
738 skb_frag_list_init(skb);
739
740 iter->offset = 0;
741 iter->hlen = hlen;
742 iter->frag_id = frag_id;
743 iter->nexthdr = nexthdr;
744
745 __skb_pull(skb, hlen);
746 fh = __skb_push(skb, sizeof(struct frag_hdr));
747 __skb_push(skb, hlen);
748 skb_reset_network_header(skb);
749 memcpy(skb_network_header(skb), iter->tmp_hdr, hlen);
750
751 fh->nexthdr = nexthdr;
752 fh->reserved = 0;
753 fh->frag_off = htons(IP6_MF);
754 fh->identification = frag_id;
755
756 first_len = skb_pagelen(skb);
757 skb->data_len = first_len - skb_headlen(skb);
758 skb->len = first_len;
759 ipv6_hdr(skb)->payload_len = htons(first_len - sizeof(struct ipv6hdr));
760
761 return 0;
762}
763EXPORT_SYMBOL(ip6_fraglist_init);
764
765void ip6_fraglist_prepare(struct sk_buff *skb,
766 struct ip6_fraglist_iter *iter)
767{
768 struct sk_buff *frag = iter->frag;
769 unsigned int hlen = iter->hlen;
770 struct frag_hdr *fh;
771
772 frag->ip_summed = CHECKSUM_NONE;
773 skb_reset_transport_header(frag);
774 fh = __skb_push(frag, sizeof(struct frag_hdr));
775 __skb_push(frag, hlen);
776 skb_reset_network_header(frag);
777 memcpy(skb_network_header(frag), iter->tmp_hdr, hlen);
778 iter->offset += skb->len - hlen - sizeof(struct frag_hdr);
779 fh->nexthdr = iter->nexthdr;
780 fh->reserved = 0;
781 fh->frag_off = htons(iter->offset);
782 if (frag->next)
783 fh->frag_off |= htons(IP6_MF);
784 fh->identification = iter->frag_id;
785 ipv6_hdr(frag)->payload_len = htons(frag->len - sizeof(struct ipv6hdr));
786 ip6_copy_metadata(frag, skb);
787}
788EXPORT_SYMBOL(ip6_fraglist_prepare);
789
790void ip6_frag_init(struct sk_buff *skb, unsigned int hlen, unsigned int mtu,
791 unsigned short needed_tailroom, int hdr_room, u8 *prevhdr,
792 u8 nexthdr, __be32 frag_id, struct ip6_frag_state *state)
793{
794 state->prevhdr = prevhdr;
795 state->nexthdr = nexthdr;
796 state->frag_id = frag_id;
797
798 state->hlen = hlen;
799 state->mtu = mtu;
800
801 state->left = skb->len - hlen; /* Space per frame */
802 state->ptr = hlen; /* Where to start from */
803
804 state->hroom = hdr_room;
805 state->troom = needed_tailroom;
806
807 state->offset = 0;
808}
809EXPORT_SYMBOL(ip6_frag_init);
810
811struct sk_buff *ip6_frag_next(struct sk_buff *skb, struct ip6_frag_state *state)
812{
813 u8 *prevhdr = state->prevhdr, *fragnexthdr_offset;
814 struct sk_buff *frag;
815 struct frag_hdr *fh;
816 unsigned int len;
817
818 len = state->left;
819 /* IF: it doesn't fit, use 'mtu' - the data space left */
820 if (len > state->mtu)
821 len = state->mtu;
822 /* IF: we are not sending up to and including the packet end
823 then align the next start on an eight byte boundary */
824 if (len < state->left)
825 len &= ~7;
826
827 /* Allocate buffer */
828 frag = alloc_skb(len + state->hlen + sizeof(struct frag_hdr) +
829 state->hroom + state->troom, GFP_ATOMIC);
830 if (!frag)
831 return ERR_PTR(-ENOMEM);
832
833 /*
834 * Set up data on packet
835 */
836
837 ip6_copy_metadata(frag, skb);
838 skb_reserve(frag, state->hroom);
839 skb_put(frag, len + state->hlen + sizeof(struct frag_hdr));
840 skb_reset_network_header(frag);
841 fh = (struct frag_hdr *)(skb_network_header(frag) + state->hlen);
842 frag->transport_header = (frag->network_header + state->hlen +
843 sizeof(struct frag_hdr));
844
845 /*
846 * Charge the memory for the fragment to any owner
847 * it might possess
848 */
849 if (skb->sk)
850 skb_set_owner_w(frag, skb->sk);
851
852 /*
853 * Copy the packet header into the new buffer.
854 */
855 skb_copy_from_linear_data(skb, skb_network_header(frag), state->hlen);
856
857 fragnexthdr_offset = skb_network_header(frag);
858 fragnexthdr_offset += prevhdr - skb_network_header(skb);
859 *fragnexthdr_offset = NEXTHDR_FRAGMENT;
860
861 /*
862 * Build fragment header.
863 */
864 fh->nexthdr = state->nexthdr;
865 fh->reserved = 0;
866 fh->identification = state->frag_id;
867
868 /*
869 * Copy a block of the IP datagram.
870 */
871 BUG_ON(skb_copy_bits(skb, state->ptr, skb_transport_header(frag),
872 len));
873 state->left -= len;
874
875 fh->frag_off = htons(state->offset);
876 if (state->left > 0)
877 fh->frag_off |= htons(IP6_MF);
878 ipv6_hdr(frag)->payload_len = htons(frag->len - sizeof(struct ipv6hdr));
879
880 state->ptr += len;
881 state->offset += len;
882
883 return frag;
884}
885EXPORT_SYMBOL(ip6_frag_next);
886
887int ip6_fragment(struct net *net, struct sock *sk, struct sk_buff *skb,
888 int (*output)(struct net *, struct sock *, struct sk_buff *))
889{
890 struct sk_buff *frag;
891 struct rt6_info *rt = dst_rt6_info(skb_dst(skb));
892 struct ipv6_pinfo *np = skb->sk && !dev_recursion_level() ?
893 inet6_sk(skb->sk) : NULL;
894 u8 tstamp_type = skb->tstamp_type;
895 struct ip6_frag_state state;
896 unsigned int mtu, hlen, nexthdr_offset;
897 ktime_t tstamp = skb->tstamp;
898 int hroom, err = 0;
899 __be32 frag_id;
900 u8 *prevhdr, nexthdr = 0;
901
902 if (!ipv6_mod_enabled()) {
903 kfree_skb(skb);
904 return -EAFNOSUPPORT;
905 }
906
907 err = ip6_find_1stfragopt(skb, &prevhdr);
908 if (err < 0)
909 goto fail;
910 hlen = err;
911 nexthdr = *prevhdr;
912 nexthdr_offset = prevhdr - skb_network_header(skb);
913
914 mtu = ip6_skb_dst_mtu(skb);
915
916 /* We must not fragment if the socket is set to force MTU discovery
917 * or if the skb it not generated by a local socket.
918 */
919 if (unlikely(!skb->ignore_df && skb->len > mtu))
920 goto fail_toobig;
921
922 if (IP6CB(skb)->frag_max_size) {
923 if (IP6CB(skb)->frag_max_size > mtu)
924 goto fail_toobig;
925
926 /* don't send fragments larger than what we received */
927 mtu = IP6CB(skb)->frag_max_size;
928 if (mtu < IPV6_MIN_MTU)
929 mtu = IPV6_MIN_MTU;
930 }
931
932 if (np) {
933 u32 frag_size = READ_ONCE(np->frag_size);
934
935 if (frag_size && frag_size < mtu)
936 mtu = frag_size;
937 }
938 if (mtu < hlen + sizeof(struct frag_hdr) + 8)
939 goto fail_toobig;
940 mtu -= hlen + sizeof(struct frag_hdr);
941
942 frag_id = ipv6_select_ident(net, &ipv6_hdr(skb)->daddr,
943 &ipv6_hdr(skb)->saddr);
944
945 if (skb->ip_summed == CHECKSUM_PARTIAL &&
946 (err = skb_checksum_help(skb)))
947 goto fail;
948
949 prevhdr = skb_network_header(skb) + nexthdr_offset;
950 hroom = LL_RESERVED_SPACE(rt->dst.dev);
951 if (skb_has_frag_list(skb)) {
952 unsigned int first_len = skb_pagelen(skb);
953 struct ip6_fraglist_iter iter;
954 struct sk_buff *frag2;
955
956 if (first_len - hlen > mtu ||
957 ((first_len - hlen) & 7) ||
958 skb_cloned(skb) ||
959 skb_headroom(skb) < (hroom + sizeof(struct frag_hdr)))
960 goto slow_path;
961
962 skb_walk_frags(skb, frag) {
963 /* Correct geometry. */
964 if (frag->len > mtu ||
965 ((frag->len & 7) && frag->next) ||
966 skb_headroom(frag) < (hlen + hroom + sizeof(struct frag_hdr)))
967 goto slow_path_clean;
968
969 /* Partially cloned skb? */
970 if (skb_shared(frag))
971 goto slow_path_clean;
972
973 BUG_ON(frag->sk);
974 if (skb->sk) {
975 frag->sk = skb->sk;
976 frag->destructor = sock_wfree;
977 }
978 skb->truesize -= frag->truesize;
979 }
980
981 err = ip6_fraglist_init(skb, hlen, prevhdr, nexthdr, frag_id,
982 &iter);
983 if (err < 0)
984 goto fail;
985
986 /* We prevent @rt from being freed. */
987 rcu_read_lock();
988
989 for (;;) {
990 /* Prepare header of the next frame,
991 * before previous one went down. */
992 if (iter.frag)
993 ip6_fraglist_prepare(skb, &iter);
994
995 skb_set_delivery_time(skb, tstamp, tstamp_type);
996 err = output(net, sk, skb);
997 if (!err)
998 IP6_INC_STATS(net, ip6_dst_idev(&rt->dst),
999 IPSTATS_MIB_FRAGCREATES);
1000
1001 if (err || !iter.frag)
1002 break;
1003
1004 skb = ip6_fraglist_next(&iter);
1005 }
1006
1007 kfree(iter.tmp_hdr);
1008
1009 if (err == 0) {
1010 IP6_INC_STATS(net, ip6_dst_idev(&rt->dst),
1011 IPSTATS_MIB_FRAGOKS);
1012 rcu_read_unlock();
1013 return 0;
1014 }
1015
1016 kfree_skb_list(iter.frag);
1017
1018 IP6_INC_STATS(net, ip6_dst_idev(&rt->dst),
1019 IPSTATS_MIB_FRAGFAILS);
1020 rcu_read_unlock();
1021 return err;
1022
1023slow_path_clean:
1024 skb_walk_frags(skb, frag2) {
1025 if (frag2 == frag)
1026 break;
1027 frag2->sk = NULL;
1028 frag2->destructor = NULL;
1029 skb->truesize += frag2->truesize;
1030 }
1031 }
1032
1033slow_path:
1034 /*
1035 * Fragment the datagram.
1036 */
1037
1038 ip6_frag_init(skb, hlen, mtu, rt->dst.dev->needed_tailroom,
1039 LL_RESERVED_SPACE(rt->dst.dev), prevhdr, nexthdr, frag_id,
1040 &state);
1041
1042 /*
1043 * Keep copying data until we run out.
1044 */
1045
1046 while (state.left > 0) {
1047 frag = ip6_frag_next(skb, &state);
1048 if (IS_ERR(frag)) {
1049 err = PTR_ERR(frag);
1050 goto fail;
1051 }
1052
1053 /*
1054 * Put this fragment into the sending queue.
1055 */
1056 skb_set_delivery_time(frag, tstamp, tstamp_type);
1057 err = output(net, sk, frag);
1058 if (err)
1059 goto fail;
1060
1061 IP6_INC_STATS(net, ip6_dst_idev(skb_dst(skb)),
1062 IPSTATS_MIB_FRAGCREATES);
1063 }
1064 IP6_INC_STATS(net, ip6_dst_idev(skb_dst(skb)),
1065 IPSTATS_MIB_FRAGOKS);
1066 consume_skb(skb);
1067 return err;
1068
1069fail_toobig:
1070 icmpv6_send(skb, ICMPV6_PKT_TOOBIG, 0, mtu);
1071 err = -EMSGSIZE;
1072
1073fail:
1074 IP6_INC_STATS(net, ip6_dst_idev(skb_dst(skb)),
1075 IPSTATS_MIB_FRAGFAILS);
1076 kfree_skb(skb);
1077 return err;
1078}
1079EXPORT_SYMBOL_GPL(ip6_fragment);
1080
1081static inline int ip6_rt_check(const struct rt6key *rt_key,
1082 const struct in6_addr *fl_addr,
1083 const struct in6_addr *addr_cache)
1084{
1085 return (rt_key->plen != 128 || !ipv6_addr_equal(fl_addr, &rt_key->addr)) &&
1086 (!addr_cache || !ipv6_addr_equal(fl_addr, addr_cache));
1087}
1088
1089static struct dst_entry *ip6_sk_dst_check(struct sock *sk,
1090 struct dst_entry *dst,
1091 const struct flowi6 *fl6)
1092{
1093 struct ipv6_pinfo *np = inet6_sk(sk);
1094 struct rt6_info *rt;
1095
1096 if (!dst)
1097 goto out;
1098
1099 if (dst->ops->family != AF_INET6) {
1100 dst_release(dst);
1101 return NULL;
1102 }
1103
1104 rt = dst_rt6_info(dst);
1105 /* Yes, checking route validity in not connected
1106 * case is not very simple. Take into account,
1107 * that we do not support routing by source, TOS,
1108 * and MSG_DONTROUTE --ANK (980726)
1109 *
1110 * 1. ip6_rt_check(): If route was host route,
1111 * check that cached destination is current.
1112 * If it is network route, we still may
1113 * check its validity using saved pointer
1114 * to the last used address: daddr_cache.
1115 * We do not want to save whole address now,
1116 * (because main consumer of this service
1117 * is tcp, which has not this problem),
1118 * so that the last trick works only on connected
1119 * sockets.
1120 * 2. oif also should be the same.
1121 */
1122 if (ip6_rt_check(&rt->rt6i_dst, &fl6->daddr,
1123 np->daddr_cache ? &sk->sk_v6_daddr : NULL) ||
1124#ifdef CONFIG_IPV6_SUBTREES
1125 ip6_rt_check(&rt->rt6i_src, &fl6->saddr,
1126 np->saddr_cache ? &np->saddr : NULL) ||
1127#endif
1128 (fl6->flowi6_oif && fl6->flowi6_oif != dst_dev(dst)->ifindex)) {
1129 dst_release(dst);
1130 dst = NULL;
1131 }
1132
1133out:
1134 return dst;
1135}
1136
1137static int ip6_dst_lookup_tail(struct net *net, const struct sock *sk,
1138 struct dst_entry **dst, struct flowi6 *fl6)
1139{
1140#ifdef CONFIG_IPV6_OPTIMISTIC_DAD
1141 struct neighbour *n;
1142 struct rt6_info *rt;
1143#endif
1144 int err;
1145 int flags = 0;
1146
1147 /* The correct way to handle this would be to do
1148 * ip6_route_get_saddr, and then ip6_route_output; however,
1149 * the route-specific preferred source forces the
1150 * ip6_route_output call _before_ ip6_route_get_saddr.
1151 *
1152 * In source specific routing (no src=any default route),
1153 * ip6_route_output will fail given src=any saddr, though, so
1154 * that's why we try it again later.
1155 */
1156 if (ipv6_addr_any(&fl6->saddr)) {
1157 struct fib6_info *from;
1158 struct rt6_info *rt;
1159
1160 *dst = ip6_route_output(net, sk, fl6);
1161 rt = (*dst)->error ? NULL : dst_rt6_info(*dst);
1162
1163 rcu_read_lock();
1164 from = rt ? rcu_dereference(rt->from) : NULL;
1165 err = ip6_route_get_saddr(net, from, &fl6->daddr,
1166 sk ? READ_ONCE(inet6_sk(sk)->srcprefs) : 0,
1167 fl6->flowi6_l3mdev,
1168 &fl6->saddr);
1169 rcu_read_unlock();
1170
1171 if (err)
1172 goto out_err_release;
1173
1174 /* If we had an erroneous initial result, pretend it
1175 * never existed and let the SA-enabled version take
1176 * over.
1177 */
1178 if ((*dst)->error) {
1179 dst_release(*dst);
1180 *dst = NULL;
1181 }
1182
1183 if (fl6->flowi6_oif)
1184 flags |= RT6_LOOKUP_F_IFACE;
1185 }
1186
1187 if (!*dst)
1188 *dst = ip6_route_output_flags(net, sk, fl6, flags);
1189
1190 err = (*dst)->error;
1191 if (err)
1192 goto out_err_release;
1193
1194#ifdef CONFIG_IPV6_OPTIMISTIC_DAD
1195 /*
1196 * Here if the dst entry we've looked up
1197 * has a neighbour entry that is in the INCOMPLETE
1198 * state and the src address from the flow is
1199 * marked as OPTIMISTIC, we release the found
1200 * dst entry and replace it instead with the
1201 * dst entry of the nexthop router
1202 */
1203 rt = dst_rt6_info(*dst);
1204 rcu_read_lock();
1205 n = __ipv6_neigh_lookup_noref(rt->dst.dev,
1206 rt6_nexthop(rt, &fl6->daddr));
1207 err = n && !(READ_ONCE(n->nud_state) & NUD_VALID) ? -EINVAL : 0;
1208 rcu_read_unlock();
1209
1210 if (err) {
1211 struct inet6_ifaddr *ifp;
1212 struct flowi6 fl_gw6;
1213 int redirect;
1214
1215 ifp = ipv6_get_ifaddr(net, &fl6->saddr,
1216 (*dst)->dev, 1);
1217
1218 redirect = (ifp && ifp->flags & IFA_F_OPTIMISTIC);
1219 if (ifp)
1220 in6_ifa_put(ifp);
1221
1222 if (redirect) {
1223 /*
1224 * We need to get the dst entry for the
1225 * default router instead
1226 */
1227 dst_release(*dst);
1228 memcpy(&fl_gw6, fl6, sizeof(struct flowi6));
1229 memset(&fl_gw6.daddr, 0, sizeof(struct in6_addr));
1230 *dst = ip6_route_output(net, sk, &fl_gw6);
1231 err = (*dst)->error;
1232 if (err)
1233 goto out_err_release;
1234 }
1235 }
1236#endif
1237 if (ipv6_addr_v4mapped(&fl6->saddr) &&
1238 !(ipv6_addr_v4mapped(&fl6->daddr) || ipv6_addr_any(&fl6->daddr))) {
1239 err = -EAFNOSUPPORT;
1240 goto out_err_release;
1241 }
1242
1243 return 0;
1244
1245out_err_release:
1246 dst_release(*dst);
1247 *dst = NULL;
1248
1249 if (err == -ENETUNREACH)
1250 IP6_INC_STATS(net, NULL, IPSTATS_MIB_OUTNOROUTES);
1251 return err;
1252}
1253
1254/**
1255 * ip6_dst_lookup - perform route lookup on flow
1256 * @net: Network namespace to perform lookup in
1257 * @sk: socket which provides route info
1258 * @dst: pointer to dst_entry * for result
1259 * @fl6: flow to lookup
1260 *
1261 * This function performs a route lookup on the given flow.
1262 *
1263 * It returns zero on success, or a standard errno code on error.
1264 */
1265int ip6_dst_lookup(struct net *net, struct sock *sk, struct dst_entry **dst,
1266 struct flowi6 *fl6)
1267{
1268 *dst = NULL;
1269 return ip6_dst_lookup_tail(net, sk, dst, fl6);
1270}
1271EXPORT_SYMBOL_GPL(ip6_dst_lookup);
1272
1273/**
1274 * ip6_dst_lookup_flow - perform route lookup on flow with ipsec
1275 * @net: Network namespace to perform lookup in
1276 * @sk: socket which provides route info
1277 * @fl6: flow to lookup
1278 * @final_dst: final destination address for ipsec lookup
1279 *
1280 * This function performs a route lookup on the given flow.
1281 *
1282 * It returns a valid dst pointer on success, or a pointer encoded
1283 * error code.
1284 */
1285struct dst_entry *ip6_dst_lookup_flow(struct net *net, const struct sock *sk, struct flowi6 *fl6,
1286 const struct in6_addr *final_dst)
1287{
1288 struct dst_entry *dst = NULL;
1289 int err;
1290
1291 if (!ipv6_mod_enabled())
1292 return ERR_PTR(-EAFNOSUPPORT);
1293 err = ip6_dst_lookup_tail(net, sk, &dst, fl6);
1294 if (err)
1295 return ERR_PTR(err);
1296 if (final_dst)
1297 fl6->daddr = *final_dst;
1298
1299 return xfrm_lookup_route(net, dst, flowi6_to_flowi(fl6), sk, 0);
1300}
1301EXPORT_SYMBOL_GPL(ip6_dst_lookup_flow);
1302
1303/**
1304 * ip6_sk_dst_lookup_flow - perform socket cached route lookup on flow
1305 * @sk: socket which provides the dst cache and route info
1306 * @fl6: flow to lookup
1307 * @final_dst: final destination address for ipsec lookup
1308 * @connected: whether @sk is connected or not
1309 *
1310 * This function performs a route lookup on the given flow with the
1311 * possibility of using the cached route in the socket if it is valid.
1312 * It will take the socket dst lock when operating on the dst cache.
1313 * As a result, this function can only be used in process context.
1314 *
1315 * In addition, for a connected socket, cache the dst in the socket
1316 * if the current cache is not valid.
1317 *
1318 * It returns a valid dst pointer on success, or a pointer encoded
1319 * error code.
1320 */
1321struct dst_entry *ip6_sk_dst_lookup_flow(struct sock *sk, struct flowi6 *fl6,
1322 const struct in6_addr *final_dst,
1323 bool connected)
1324{
1325 struct dst_entry *dst = sk_dst_check(sk, inet6_sk(sk)->dst_cookie);
1326
1327 dst = ip6_sk_dst_check(sk, dst, fl6);
1328 if (dst)
1329 return dst;
1330
1331 dst = ip6_dst_lookup_flow(sock_net(sk), sk, fl6, final_dst);
1332 if (connected && !IS_ERR(dst))
1333 ip6_sk_dst_store_flow(sk, dst_clone(dst), fl6);
1334
1335 return dst;
1336}
1337
1338static inline struct ipv6_opt_hdr *ip6_opt_dup(struct ipv6_opt_hdr *src,
1339 gfp_t gfp)
1340{
1341 return src ? kmemdup(src, (src->hdrlen + 1) * 8, gfp) : NULL;
1342}
1343
1344static inline struct ipv6_rt_hdr *ip6_rthdr_dup(struct ipv6_rt_hdr *src,
1345 gfp_t gfp)
1346{
1347 return src ? kmemdup(src, (src->hdrlen + 1) * 8, gfp) : NULL;
1348}
1349
1350static void ip6_append_data_mtu(unsigned int *mtu,
1351 int *maxfraglen,
1352 unsigned int fragheaderlen,
1353 struct sk_buff *skb,
1354 struct rt6_info *rt,
1355 unsigned int orig_mtu)
1356{
1357 if (!(rt->dst.flags & DST_XFRM_TUNNEL)) {
1358 if (!skb) {
1359 /* first fragment, reserve header_len */
1360 *mtu = orig_mtu - rt->dst.header_len;
1361
1362 } else {
1363 /*
1364 * this fragment is not first, the headers
1365 * space is regarded as data space.
1366 */
1367 *mtu = orig_mtu;
1368 }
1369 *maxfraglen = ((*mtu - fragheaderlen) & ~7)
1370 + fragheaderlen - sizeof(struct frag_hdr);
1371 }
1372}
1373
1374static int ip6_setup_cork(struct sock *sk, struct inet_cork_full *cork,
1375 struct ipcm6_cookie *ipc6,
1376 struct rt6_info *rt)
1377{
1378 struct ipv6_txoptions *nopt, *opt = ipc6->opt;
1379 struct inet6_cork *v6_cork = &cork->base6;
1380 struct ipv6_pinfo *np = inet6_sk(sk);
1381 unsigned int mtu, frag_size;
1382
1383 /* callers pass dst together with a reference, set it first so
1384 * ip6_cork_release() can put it down even in case of an error.
1385 */
1386 cork->base.dst = &rt->dst;
1387
1388 /*
1389 * setup for corking
1390 */
1391 if (unlikely(opt)) {
1392 if (WARN_ON(v6_cork->opt))
1393 return -EINVAL;
1394
1395 nopt = v6_cork->opt = kzalloc_obj(*opt, sk->sk_allocation);
1396 if (unlikely(!nopt))
1397 return -ENOBUFS;
1398
1399 nopt->tot_len = sizeof(*opt);
1400 nopt->opt_flen = opt->opt_flen;
1401 nopt->opt_nflen = opt->opt_nflen;
1402
1403 nopt->dst0opt = ip6_opt_dup(opt->dst0opt, sk->sk_allocation);
1404 if (opt->dst0opt && !nopt->dst0opt)
1405 return -ENOBUFS;
1406
1407 nopt->dst1opt = ip6_opt_dup(opt->dst1opt, sk->sk_allocation);
1408 if (opt->dst1opt && !nopt->dst1opt)
1409 return -ENOBUFS;
1410
1411 nopt->hopopt = ip6_opt_dup(opt->hopopt, sk->sk_allocation);
1412 if (opt->hopopt && !nopt->hopopt)
1413 return -ENOBUFS;
1414
1415 nopt->srcrt = ip6_rthdr_dup(opt->srcrt, sk->sk_allocation);
1416 if (opt->srcrt && !nopt->srcrt)
1417 return -ENOBUFS;
1418
1419 /* need source address above miyazawa*/
1420 }
1421 v6_cork->hop_limit = ipc6->hlimit;
1422 v6_cork->tclass = ipc6->tclass;
1423 v6_cork->dontfrag = ipc6->dontfrag;
1424 if (rt->dst.flags & DST_XFRM_TUNNEL)
1425 mtu = READ_ONCE(np->pmtudisc) >= IPV6_PMTUDISC_PROBE ?
1426 READ_ONCE(rt->dst.dev->mtu) : dst6_mtu(&rt->dst);
1427 else
1428 mtu = READ_ONCE(np->pmtudisc) >= IPV6_PMTUDISC_PROBE ?
1429 READ_ONCE(rt->dst.dev->mtu) : dst6_mtu(xfrm_dst_path(&rt->dst));
1430
1431 frag_size = READ_ONCE(np->frag_size);
1432 if (frag_size && frag_size < mtu)
1433 mtu = frag_size;
1434
1435 if (sk_is_udp(sk))
1436 mtu = min(mtu, IP6_MAX_MTU);
1437 cork->base.fragsize = mtu;
1438 cork->base.gso_size = ipc6->gso_size;
1439 cork->base.tx_flags = 0;
1440 cork->base.mark = ipc6->sockc.mark;
1441 cork->base.priority = ipc6->sockc.priority;
1442 sock_tx_timestamp(sk, &ipc6->sockc, &cork->base.tx_flags);
1443 if (ipc6->sockc.tsflags & SOCKCM_FLAG_TS_OPT_ID) {
1444 cork->base.flags |= IPCORK_TS_OPT_ID;
1445 cork->base.ts_opt_id = ipc6->sockc.ts_opt_id;
1446 }
1447 cork->base.length = 0;
1448 cork->base.transmit_time = ipc6->sockc.transmit_time;
1449
1450 return 0;
1451}
1452
1453static int __ip6_append_data(struct sock *sk,
1454 struct sk_buff_head *queue,
1455 struct inet_cork_full *cork_full,
1456 struct page_frag *pfrag,
1457 int getfrag(void *from, char *to, int offset,
1458 int len, int odd, struct sk_buff *skb),
1459 void *from, size_t length, int transhdrlen,
1460 unsigned int flags)
1461{
1462 unsigned int maxfraglen, fragheaderlen, mtu, orig_mtu, pmtu;
1463 struct inet6_cork *v6_cork = &cork_full->base6;
1464 struct inet_cork *cork = &cork_full->base;
1465 struct flowi6 *fl6 = &cork_full->fl.u.ip6;
1466 struct sk_buff *skb, *skb_prev = NULL;
1467 struct ubuf_info *uarg = NULL;
1468 int exthdrlen = 0;
1469 int dst_exthdrlen = 0;
1470 int hh_len;
1471 int copy;
1472 int err;
1473 int offset = 0;
1474 bool zc = false;
1475 u32 tskey = 0;
1476 struct rt6_info *rt = dst_rt6_info(cork->dst);
1477 bool paged, hold_tskey = false, extra_uref = false;
1478 struct ipv6_txoptions *opt = v6_cork->opt;
1479 int csummode = CHECKSUM_NONE;
1480 unsigned int maxnonfragsize, headersize;
1481 unsigned int wmem_alloc_delta = 0;
1482
1483 skb = skb_peek_tail(queue);
1484 if (!skb) {
1485 exthdrlen = opt ? opt->opt_flen : 0;
1486 dst_exthdrlen = rt->dst.header_len - rt->rt6i_nfheader_len;
1487 }
1488
1489 paged = !!cork->gso_size;
1490 mtu = cork->gso_size ? IP6_MAX_MTU : cork->fragsize;
1491 orig_mtu = mtu;
1492
1493 hh_len = LL_RESERVED_SPACE(rt->dst.dev);
1494
1495 fragheaderlen = sizeof(struct ipv6hdr) + rt->rt6i_nfheader_len +
1496 (opt ? opt->opt_nflen : 0);
1497
1498 headersize = sizeof(struct ipv6hdr) +
1499 (opt ? opt->opt_flen + opt->opt_nflen : 0) +
1500 rt->rt6i_nfheader_len;
1501
1502 if (mtu <= fragheaderlen ||
1503 ((mtu - fragheaderlen) & ~7) + fragheaderlen <= sizeof(struct frag_hdr))
1504 goto emsgsize;
1505
1506 maxfraglen = ((mtu - fragheaderlen) & ~7) + fragheaderlen -
1507 sizeof(struct frag_hdr);
1508
1509 /* as per RFC 7112 section 5, the entire IPv6 Header Chain must fit
1510 * the first fragment
1511 */
1512 if (headersize + transhdrlen > mtu)
1513 goto emsgsize;
1514
1515 if (cork->length + length > mtu - headersize && v6_cork->dontfrag &&
1516 (sk->sk_protocol == IPPROTO_UDP ||
1517 sk->sk_protocol == IPPROTO_ICMPV6 ||
1518 sk->sk_protocol == IPPROTO_RAW)) {
1519 ipv6_local_rxpmtu(sk, fl6, mtu - headersize +
1520 sizeof(struct ipv6hdr));
1521 goto emsgsize;
1522 }
1523
1524 if (ip6_sk_ignore_df(sk))
1525 maxnonfragsize = sizeof(struct ipv6hdr) + IPV6_MAXPLEN;
1526 else
1527 maxnonfragsize = mtu;
1528
1529 if (cork->length + length > maxnonfragsize - headersize) {
1530emsgsize:
1531 pmtu = max_t(int, mtu - headersize + sizeof(struct ipv6hdr), 0);
1532 ipv6_local_error(sk, EMSGSIZE, fl6, pmtu);
1533 return -EMSGSIZE;
1534 }
1535
1536 /* CHECKSUM_PARTIAL only with no extension headers and when
1537 * we are not going to fragment
1538 */
1539 if (transhdrlen && sk->sk_protocol == IPPROTO_UDP &&
1540 headersize == sizeof(struct ipv6hdr) &&
1541 length <= mtu - headersize &&
1542 (!(flags & MSG_MORE) || cork->gso_size) &&
1543 rt->dst.dev->features & (NETIF_F_IPV6_CSUM | NETIF_F_HW_CSUM))
1544 csummode = CHECKSUM_PARTIAL;
1545
1546 if ((flags & MSG_ZEROCOPY) && length) {
1547 struct msghdr *msg = from;
1548
1549 if (getfrag == ip_generic_getfrag && msg->msg_ubuf) {
1550 if (skb_zcopy(skb) && msg->msg_ubuf != skb_zcopy(skb))
1551 return -EINVAL;
1552
1553 /* Leave uarg NULL if can't zerocopy, callers should
1554 * be able to handle it.
1555 */
1556 if ((rt->dst.dev->features & NETIF_F_SG) &&
1557 csummode == CHECKSUM_PARTIAL) {
1558 paged = true;
1559 zc = true;
1560 uarg = msg->msg_ubuf;
1561 }
1562 } else if (sock_flag(sk, SOCK_ZEROCOPY)) {
1563 uarg = msg_zerocopy_realloc(sk, length, skb_zcopy(skb),
1564 false);
1565 if (!uarg)
1566 return -ENOBUFS;
1567 extra_uref = !skb_zcopy(skb); /* only ref on new uarg */
1568 if (rt->dst.dev->features & NETIF_F_SG &&
1569 csummode == CHECKSUM_PARTIAL) {
1570 paged = true;
1571 zc = true;
1572 } else {
1573 uarg_to_msgzc(uarg)->zerocopy = 0;
1574 skb_zcopy_set(skb, uarg, &extra_uref);
1575 }
1576 }
1577 } else if ((flags & MSG_SPLICE_PAGES) && length) {
1578 if (inet_test_bit(HDRINCL, sk))
1579 return -EPERM;
1580 if (rt->dst.dev->features & NETIF_F_SG &&
1581 getfrag == ip_generic_getfrag)
1582 /* We need an empty buffer to attach stuff to */
1583 paged = true;
1584 else
1585 flags &= ~MSG_SPLICE_PAGES;
1586 }
1587
1588 if (cork->tx_flags & SKBTX_ANY_TSTAMP &&
1589 READ_ONCE(sk->sk_tsflags) & SOF_TIMESTAMPING_OPT_ID) {
1590 if (cork->flags & IPCORK_TS_OPT_ID) {
1591 tskey = cork->ts_opt_id;
1592 } else {
1593 tskey = atomic_inc_return(&sk->sk_tskey) - 1;
1594 hold_tskey = true;
1595 }
1596 }
1597
1598 /*
1599 * Let's try using as much space as possible.
1600 * Use MTU if total length of the message fits into the MTU.
1601 * Otherwise, we need to reserve fragment header and
1602 * fragment alignment (= 8-15 octects, in total).
1603 *
1604 * Note that we may need to "move" the data from the tail
1605 * of the buffer to the new fragment when we split
1606 * the message.
1607 *
1608 * FIXME: It may be fragmented into multiple chunks
1609 * at once if non-fragmentable extension headers
1610 * are too large.
1611 * --yoshfuji
1612 */
1613
1614 cork->length += length;
1615 if (!skb)
1616 goto alloc_new_skb;
1617
1618 while (length > 0) {
1619 /* Check if the remaining data fits into current packet. */
1620 copy = (cork->length <= mtu ? mtu : maxfraglen) - skb->len;
1621 if (copy < length)
1622 copy = maxfraglen - skb->len;
1623
1624 if (copy <= 0) {
1625 char *data;
1626 unsigned int datalen;
1627 unsigned int fraglen;
1628 unsigned int fraggap;
1629 unsigned int alloclen, alloc_extra;
1630 unsigned int pagedlen;
1631alloc_new_skb:
1632 /* There's no room in the current skb */
1633 if (skb)
1634 fraggap = skb->len - maxfraglen;
1635 else
1636 fraggap = 0;
1637 /* update mtu and maxfraglen if necessary */
1638 if (!skb || !skb_prev)
1639 ip6_append_data_mtu(&mtu, &maxfraglen,
1640 fragheaderlen, skb, rt,
1641 orig_mtu);
1642
1643 skb_prev = skb;
1644
1645 /*
1646 * If remaining data exceeds the mtu,
1647 * we know we need more fragment(s).
1648 */
1649 datalen = length + fraggap;
1650
1651 if (datalen > (cork->length <= mtu ? mtu : maxfraglen) - fragheaderlen)
1652 datalen = maxfraglen - fragheaderlen - rt->dst.trailer_len;
1653 fraglen = datalen + fragheaderlen;
1654 pagedlen = 0;
1655
1656 alloc_extra = hh_len;
1657 alloc_extra += dst_exthdrlen;
1658 alloc_extra += rt->dst.trailer_len;
1659
1660 /* We just reserve space for fragment header.
1661 * Note: this may be overallocation if the message
1662 * (without MSG_MORE) fits into the MTU.
1663 */
1664 alloc_extra += sizeof(struct frag_hdr);
1665
1666 if ((flags & MSG_MORE) &&
1667 !(rt->dst.dev->features&NETIF_F_SG))
1668 alloclen = mtu;
1669 else if (!paged &&
1670 (fraglen + alloc_extra < SKB_MAX_ALLOC ||
1671 !(rt->dst.dev->features & NETIF_F_SG)))
1672 alloclen = fraglen;
1673 else {
1674 alloclen = fragheaderlen + transhdrlen + fraggap;
1675 pagedlen = datalen - transhdrlen - fraggap;
1676 }
1677 alloclen += alloc_extra;
1678
1679 if (datalen != length + fraggap) {
1680 /*
1681 * this is not the last fragment, the trailer
1682 * space is regarded as data space.
1683 */
1684 datalen += rt->dst.trailer_len;
1685 }
1686
1687 fraglen = datalen + fragheaderlen;
1688
1689 copy = datalen - transhdrlen - fraggap - pagedlen;
1690 if (copy < 0) {
1691 err = -EINVAL;
1692 goto error;
1693 }
1694 if (transhdrlen) {
1695 skb = sock_alloc_send_skb(sk, alloclen,
1696 (flags & MSG_DONTWAIT), &err);
1697 } else {
1698 skb = NULL;
1699 if (refcount_read(&sk->sk_wmem_alloc) + wmem_alloc_delta <=
1700 2 * sk->sk_sndbuf)
1701 skb = alloc_skb(alloclen,
1702 sk->sk_allocation);
1703 if (unlikely(!skb))
1704 err = -ENOBUFS;
1705 }
1706 if (!skb)
1707 goto error;
1708 /*
1709 * Fill in the control structures
1710 */
1711 skb->protocol = htons(ETH_P_IPV6);
1712 skb->ip_summed = csummode;
1713 skb->csum = 0;
1714 /* reserve for fragmentation and ipsec header */
1715 skb_reserve(skb, hh_len + sizeof(struct frag_hdr) +
1716 dst_exthdrlen);
1717
1718 /*
1719 * Find where to start putting bytes
1720 */
1721 data = skb_put(skb, fraglen - pagedlen);
1722 skb_set_network_header(skb, exthdrlen);
1723 data += fragheaderlen;
1724 skb->transport_header = (skb->network_header +
1725 fragheaderlen);
1726 if (fraggap) {
1727 skb->csum = skb_copy_and_csum_bits(
1728 skb_prev, maxfraglen,
1729 data + transhdrlen, fraggap);
1730 skb_prev->csum = csum_sub(skb_prev->csum,
1731 skb->csum);
1732 data += fraggap;
1733 pskb_trim_unique(skb_prev, maxfraglen);
1734 }
1735 if (copy > 0 &&
1736 INDIRECT_CALL_1(getfrag, ip_generic_getfrag,
1737 from, data + transhdrlen, offset,
1738 copy, fraggap, skb) < 0) {
1739 err = -EFAULT;
1740 kfree_skb(skb);
1741 goto error;
1742 } else if (flags & MSG_SPLICE_PAGES) {
1743 copy = 0;
1744 }
1745
1746 offset += copy;
1747 length -= copy + transhdrlen;
1748 transhdrlen = 0;
1749 exthdrlen = 0;
1750 dst_exthdrlen = 0;
1751
1752 /* Only the initial fragment is time stamped */
1753 skb_shinfo(skb)->tx_flags = cork->tx_flags;
1754 cork->tx_flags = 0;
1755 skb_shinfo(skb)->tskey = tskey;
1756 tskey = 0;
1757 skb_zcopy_set(skb, uarg, &extra_uref);
1758
1759 if ((flags & MSG_CONFIRM) && !skb_prev)
1760 skb_set_dst_pending_confirm(skb, 1);
1761
1762 /*
1763 * Put the packet on the pending queue
1764 */
1765 if (!skb->destructor) {
1766 skb->destructor = sock_wfree;
1767 skb->sk = sk;
1768 wmem_alloc_delta += skb->truesize;
1769 }
1770 __skb_queue_tail(queue, skb);
1771 continue;
1772 }
1773
1774 if (copy > length)
1775 copy = length;
1776
1777 if (!(rt->dst.dev->features&NETIF_F_SG) &&
1778 skb_tailroom(skb) >= copy) {
1779 unsigned int off;
1780
1781 off = skb->len;
1782 if (INDIRECT_CALL_1(getfrag, ip_generic_getfrag,
1783 from, skb_put(skb, copy),
1784 offset, copy, off, skb) < 0) {
1785 __skb_trim(skb, off);
1786 err = -EFAULT;
1787 goto error;
1788 }
1789 } else if (flags & MSG_SPLICE_PAGES) {
1790 struct msghdr *msg = from;
1791
1792 err = -EIO;
1793 if (WARN_ON_ONCE(copy > msg->msg_iter.count))
1794 goto error;
1795
1796 err = skb_splice_from_iter(skb, &msg->msg_iter, copy);
1797 if (err < 0)
1798 goto error;
1799 copy = err;
1800 if (!(flags & MSG_NO_SHARED_FRAGS))
1801 skb_shinfo(skb)->flags |= SKBFL_SHARED_FRAG;
1802 wmem_alloc_delta += copy;
1803 } else if (!zc) {
1804 int i = skb_shinfo(skb)->nr_frags;
1805
1806 err = -ENOMEM;
1807 if (!sk_page_frag_refill(sk, pfrag))
1808 goto error;
1809
1810 skb_zcopy_downgrade_managed(skb);
1811 if (!skb_can_coalesce(skb, i, pfrag->page,
1812 pfrag->offset)) {
1813 err = -EMSGSIZE;
1814 if (i == MAX_SKB_FRAGS)
1815 goto error;
1816
1817 __skb_fill_page_desc(skb, i, pfrag->page,
1818 pfrag->offset, 0);
1819 skb_shinfo(skb)->nr_frags = ++i;
1820 get_page(pfrag->page);
1821 }
1822 copy = min_t(int, copy, pfrag->size - pfrag->offset);
1823 if (INDIRECT_CALL_1(getfrag, ip_generic_getfrag,
1824 from,
1825 page_address(pfrag->page) + pfrag->offset,
1826 offset, copy, skb->len, skb) < 0)
1827 goto error_efault;
1828
1829 pfrag->offset += copy;
1830 skb_frag_size_add(&skb_shinfo(skb)->frags[i - 1], copy);
1831 skb->len += copy;
1832 skb->data_len += copy;
1833 skb->truesize += copy;
1834 wmem_alloc_delta += copy;
1835 } else {
1836 err = skb_zerocopy_iter_dgram(skb, from, copy);
1837 if (err < 0)
1838 goto error;
1839 }
1840 offset += copy;
1841 length -= copy;
1842 }
1843
1844 if (wmem_alloc_delta)
1845 refcount_add(wmem_alloc_delta, &sk->sk_wmem_alloc);
1846 return 0;
1847
1848error_efault:
1849 err = -EFAULT;
1850error:
1851 net_zcopy_put_abort(uarg, extra_uref);
1852 cork->length -= length;
1853 IP6_INC_STATS(sock_net(sk), rt->rt6i_idev, IPSTATS_MIB_OUTDISCARDS);
1854 refcount_add(wmem_alloc_delta, &sk->sk_wmem_alloc);
1855 if (hold_tskey)
1856 atomic_dec(&sk->sk_tskey);
1857 return err;
1858}
1859
1860int ip6_append_data(struct sock *sk,
1861 int getfrag(void *from, char *to, int offset, int len,
1862 int odd, struct sk_buff *skb),
1863 void *from, size_t length, int transhdrlen,
1864 struct ipcm6_cookie *ipc6, struct flowi6 *fl6,
1865 struct rt6_info *rt, unsigned int flags)
1866{
1867 struct inet_sock *inet = inet_sk(sk);
1868 int exthdrlen;
1869 int err;
1870
1871 if (flags&MSG_PROBE)
1872 return 0;
1873 if (skb_queue_empty(&sk->sk_write_queue)) {
1874 /*
1875 * setup for corking
1876 */
1877 dst_hold(&rt->dst);
1878 err = ip6_setup_cork(sk, &inet->cork,
1879 ipc6, rt);
1880 if (err)
1881 return err;
1882
1883 inet->cork.fl.u.ip6 = *fl6;
1884 exthdrlen = (ipc6->opt ? ipc6->opt->opt_flen : 0);
1885 length += exthdrlen;
1886 transhdrlen += exthdrlen;
1887 } else {
1888 transhdrlen = 0;
1889 }
1890
1891 return __ip6_append_data(sk, &sk->sk_write_queue, &inet->cork,
1892 sk_page_frag(sk), getfrag,
1893 from, length, transhdrlen, flags);
1894}
1895EXPORT_SYMBOL_GPL(ip6_append_data);
1896
1897static void ip6_cork_steal_dst(struct sk_buff *skb, struct inet_cork_full *cork)
1898{
1899 struct dst_entry *dst = cork->base.dst;
1900
1901 cork->base.dst = NULL;
1902 skb_dst_set(skb, dst);
1903}
1904
1905static void ip6_cork_release(struct inet_cork_full *cork)
1906{
1907 struct inet6_cork *v6_cork = &cork->base6;
1908
1909 if (unlikely(v6_cork->opt)) {
1910 struct ipv6_txoptions *opt = v6_cork->opt;
1911
1912 kfree(opt->dst0opt);
1913 kfree(opt->dst1opt);
1914 kfree(opt->hopopt);
1915 kfree(opt->srcrt);
1916 kfree(opt);
1917 v6_cork->opt = NULL;
1918 }
1919
1920 if (cork->base.dst) {
1921 dst_release(cork->base.dst);
1922 cork->base.dst = NULL;
1923 }
1924}
1925
1926struct sk_buff *__ip6_make_skb(struct sock *sk,
1927 struct sk_buff_head *queue,
1928 struct inet_cork_full *cork)
1929{
1930 struct sk_buff *skb, *tmp_skb;
1931 struct sk_buff **tail_skb;
1932 struct in6_addr *final_dst;
1933 struct net *net = sock_net(sk);
1934 struct ipv6hdr *hdr;
1935 struct ipv6_txoptions *opt;
1936 struct rt6_info *rt = dst_rt6_info(cork->base.dst);
1937 struct flowi6 *fl6 = &cork->fl.u.ip6;
1938 unsigned char proto = fl6->flowi6_proto;
1939
1940 skb = __skb_dequeue(queue);
1941 if (!skb)
1942 goto out;
1943 tail_skb = &(skb_shinfo(skb)->frag_list);
1944
1945 /* move skb->data to ip header from ext header */
1946 if (skb->data < skb_network_header(skb))
1947 __skb_pull(skb, skb_network_offset(skb));
1948 while ((tmp_skb = __skb_dequeue(queue)) != NULL) {
1949 __skb_pull(tmp_skb, skb_network_header_len(skb));
1950 *tail_skb = tmp_skb;
1951 tail_skb = &(tmp_skb->next);
1952 skb->len += tmp_skb->len;
1953 skb->data_len += tmp_skb->len;
1954 skb->truesize += tmp_skb->truesize;
1955 tmp_skb->destructor = NULL;
1956 tmp_skb->sk = NULL;
1957 }
1958
1959 /* Allow local fragmentation. */
1960 skb->ignore_df = ip6_sk_ignore_df(sk);
1961 __skb_pull(skb, skb_network_header_len(skb));
1962
1963 final_dst = &fl6->daddr;
1964 opt = cork->base6.opt;
1965 if (unlikely(opt)) {
1966 if (opt->opt_flen)
1967 proto = ipv6_push_frag_opts(skb, opt, proto);
1968 if (opt->opt_nflen)
1969 proto = ipv6_push_nfrag_opts(skb, opt, proto,
1970 &final_dst, &fl6->saddr);
1971 }
1972 skb_push(skb, sizeof(struct ipv6hdr));
1973 skb_reset_network_header(skb);
1974 hdr = ipv6_hdr(skb);
1975
1976 ip6_flow_hdr(hdr, cork->base6.tclass,
1977 ip6_make_flowlabel(net, skb, fl6->flowlabel,
1978 ip6_autoflowlabel(net, sk), fl6));
1979 hdr->hop_limit = cork->base6.hop_limit;
1980 hdr->nexthdr = proto;
1981 hdr->saddr = fl6->saddr;
1982 hdr->daddr = *final_dst;
1983
1984 skb->priority = cork->base.priority;
1985 skb->mark = cork->base.mark;
1986 if (sk_is_tcp(sk))
1987 skb_set_delivery_time(skb, cork->base.transmit_time, SKB_CLOCK_MONOTONIC);
1988 else
1989 skb_set_delivery_type_by_clockid(skb, cork->base.transmit_time, sk->sk_clockid);
1990
1991 ip6_cork_steal_dst(skb, cork);
1992 IP6_INC_STATS(net, rt->rt6i_idev, IPSTATS_MIB_OUTREQUESTS);
1993 if (unlikely(proto == IPPROTO_ICMPV6)) {
1994 struct inet6_dev *idev = ip6_dst_idev(skb_dst(skb));
1995 u8 icmp6_type;
1996
1997 if (sk->sk_socket->type == SOCK_RAW &&
1998 !(fl6->flowi6_flags & FLOWI_FLAG_KNOWN_NH))
1999 icmp6_type = fl6->fl6_icmp_type;
2000 else
2001 icmp6_type = icmp6_hdr(skb)->icmp6_type;
2002 ICMP6MSGOUT_INC_STATS(net, idev, icmp6_type);
2003 ICMP6_INC_STATS(net, idev, ICMP6_MIB_OUTMSGS);
2004 }
2005
2006 ip6_cork_release(cork);
2007out:
2008 return skb;
2009}
2010
2011int ip6_send_skb(struct sk_buff *skb)
2012{
2013 struct net *net = sock_net(skb->sk);
2014 struct rt6_info *rt = dst_rt6_info(skb_dst(skb));
2015 int err;
2016
2017 rcu_read_lock();
2018 err = ip6_local_out(net, skb->sk, skb);
2019 if (err) {
2020 if (err > 0)
2021 err = net_xmit_errno(err);
2022 if (err)
2023 IP6_INC_STATS(net, rt->rt6i_idev,
2024 IPSTATS_MIB_OUTDISCARDS);
2025 }
2026
2027 rcu_read_unlock();
2028 return err;
2029}
2030
2031int ip6_push_pending_frames(struct sock *sk)
2032{
2033 struct sk_buff *skb;
2034
2035 skb = ip6_finish_skb(sk);
2036 if (!skb)
2037 return 0;
2038
2039 return ip6_send_skb(skb);
2040}
2041EXPORT_SYMBOL_GPL(ip6_push_pending_frames);
2042
2043static void __ip6_flush_pending_frames(struct sock *sk,
2044 struct sk_buff_head *queue,
2045 struct inet_cork_full *cork)
2046{
2047 struct sk_buff *skb;
2048
2049 while ((skb = __skb_dequeue_tail(queue)) != NULL) {
2050 if (skb_dst(skb))
2051 IP6_INC_STATS(sock_net(sk), ip6_dst_idev(skb_dst(skb)),
2052 IPSTATS_MIB_OUTDISCARDS);
2053 kfree_skb(skb);
2054 }
2055
2056 ip6_cork_release(cork);
2057}
2058
2059void ip6_flush_pending_frames(struct sock *sk)
2060{
2061 __ip6_flush_pending_frames(sk, &sk->sk_write_queue,
2062 &inet_sk(sk)->cork);
2063}
2064EXPORT_SYMBOL_GPL(ip6_flush_pending_frames);
2065
2066struct sk_buff *ip6_make_skb(struct sock *sk,
2067 int getfrag(void *from, char *to, int offset,
2068 int len, int odd, struct sk_buff *skb),
2069 void *from, size_t length, int transhdrlen,
2070 struct ipcm6_cookie *ipc6, struct rt6_info *rt,
2071 unsigned int flags, struct inet_cork_full *cork)
2072{
2073 int exthdrlen = (ipc6->opt ? ipc6->opt->opt_flen : 0);
2074 struct sk_buff_head queue;
2075 int err;
2076
2077 if (flags & MSG_PROBE) {
2078 dst_release(&rt->dst);
2079 return NULL;
2080 }
2081
2082 __skb_queue_head_init(&queue);
2083
2084 cork->base.flags = 0;
2085 cork->base.addr = 0;
2086 cork->base.opt = NULL;
2087 cork->base6.opt = NULL;
2088 err = ip6_setup_cork(sk, cork, ipc6, rt);
2089 if (err) {
2090 ip6_cork_release(cork);
2091 return ERR_PTR(err);
2092 }
2093
2094 err = __ip6_append_data(sk, &queue, cork,
2095 &current->task_frag, getfrag, from,
2096 length + exthdrlen, transhdrlen + exthdrlen,
2097 flags);
2098 if (err) {
2099 __ip6_flush_pending_frames(sk, &queue, cork);
2100 return ERR_PTR(err);
2101 }
2102
2103 return __ip6_make_skb(sk, &queue, cork);
2104}