projects/linux·blob master · refs

849 lines · 20.6 KiB

1// SPDX-License-Identifier: GPL-2.0-or-later
2#include <net/psp.h>
3#include <net/gro.h>
4#include <net/dst_metadata.h>
5#include <net/busy_poll.h>
6#include <trace/events/net.h>
7#include <linux/skbuff_ref.h>
8
9#define MAX_GRO_SKBS 8
10
11static DEFINE_SPINLOCK(offload_lock);
12
13/**
14 * dev_add_offload - register offload handlers
15 * @po: protocol offload declaration
16 *
17 * Add protocol offload handlers to the networking stack. The passed
18 * &proto_offload is linked into kernel lists and may not be freed until
19 * it has been removed from the kernel lists.
20 *
21 * This call does not sleep therefore it can not
22 * guarantee all CPU's that are in middle of receiving packets
23 * will see the new offload handlers (until the next received packet).
24 */
25void dev_add_offload(struct packet_offload *po)
26{
27 struct packet_offload *elem;
28
29 spin_lock(&offload_lock);
30 list_for_each_entry(elem, &net_hotdata.offload_base, list) {
31 if (po->priority < elem->priority)
32 break;
33 }
34 list_add_rcu(&po->list, elem->list.prev);
35 spin_unlock(&offload_lock);
36}
37EXPORT_SYMBOL(dev_add_offload);
38
39/**
40 * __dev_remove_offload - remove offload handler
41 * @po: packet offload declaration
42 *
43 * Remove a protocol offload handler that was previously added to the
44 * kernel offload handlers by dev_add_offload(). The passed &offload_type
45 * is removed from the kernel lists and can be freed or reused once this
46 * function returns.
47 *
48 * The packet type might still be in use by receivers
49 * and must not be freed until after all the CPU's have gone
50 * through a quiescent state.
51 */
52static void __dev_remove_offload(struct packet_offload *po)
53{
54 struct list_head *head = &net_hotdata.offload_base;
55 struct packet_offload *po1;
56
57 spin_lock(&offload_lock);
58
59 list_for_each_entry(po1, head, list) {
60 if (po == po1) {
61 list_del_rcu(&po->list);
62 goto out;
63 }
64 }
65
66 pr_warn("dev_remove_offload: %p not found\n", po);
67out:
68 spin_unlock(&offload_lock);
69}
70
71/**
72 * dev_remove_offload - remove packet offload handler
73 * @po: packet offload declaration
74 *
75 * Remove a packet offload handler that was previously added to the kernel
76 * offload handlers by dev_add_offload(). The passed &offload_type is
77 * removed from the kernel lists and can be freed or reused once this
78 * function returns.
79 *
80 * This call sleeps to guarantee that no CPU is looking at the packet
81 * type after return.
82 */
83void dev_remove_offload(struct packet_offload *po)
84{
85 __dev_remove_offload(po);
86
87 synchronize_net();
88}
89EXPORT_SYMBOL(dev_remove_offload);
90
91
92int skb_gro_receive(struct sk_buff *p, struct sk_buff *skb)
93{
94 struct skb_shared_info *pinfo, *skbinfo = skb_shinfo(skb);
95 unsigned int offset = skb_gro_offset(skb);
96 unsigned int headlen = skb_headlen(skb);
97 unsigned int len = skb_gro_len(skb);
98 unsigned int delta_truesize;
99 unsigned int new_truesize;
100 struct sk_buff *lp;
101 int segs;
102
103 /* Do not splice page pool based packets w/ non-page pool
104 * packets. This can result in reference count issues as page
105 * pool pages will not decrement the reference count and will
106 * instead be immediately returned to the pool or have frag
107 * count decremented.
108 */
109 if (p->pp_recycle != skb->pp_recycle)
110 return -ETOOMANYREFS;
111
112 if (skb_zcopy(p) || skb_zcopy(skb))
113 return -ETOOMANYREFS;
114
115 if (unlikely(p->len + len >= netif_get_gro_max_size(p->dev, p) ||
116 NAPI_GRO_CB(skb)->flush))
117 return -E2BIG;
118
119 if (unlikely(p->len + len >= GRO_LEGACY_MAX_SIZE)) {
120 if (NAPI_GRO_CB(skb)->proto != IPPROTO_TCP ||
121 p->encapsulation)
122 return -E2BIG;
123 }
124
125 segs = NAPI_GRO_CB(skb)->count;
126 lp = NAPI_GRO_CB(p)->last;
127 pinfo = skb_shinfo(lp);
128
129 if (headlen <= offset) {
130 skb_frag_t *frag;
131 skb_frag_t *frag2;
132 int i = skbinfo->nr_frags;
133 int nr_frags = pinfo->nr_frags + i;
134
135 if (nr_frags > MAX_SKB_FRAGS)
136 goto merge;
137
138 offset -= headlen;
139 pinfo->nr_frags = nr_frags;
140 skbinfo->nr_frags = 0;
141
142 frag = pinfo->frags + nr_frags;
143 frag2 = skbinfo->frags + i;
144 do {
145 *--frag = *--frag2;
146 } while (--i);
147
148 skb_frag_off_add(frag, offset);
149 skb_frag_size_sub(frag, offset);
150
151 /* all fragments truesize : remove (head size + sk_buff) */
152 new_truesize = SKB_TRUESIZE(skb_end_offset(skb));
153 delta_truesize = skb->truesize - new_truesize;
154
155 skb->truesize = new_truesize;
156 skb->len -= skb->data_len;
157 skb->data_len = 0;
158
159 NAPI_GRO_CB(skb)->free = NAPI_GRO_FREE;
160 goto done;
161 } else if (skb->head_frag) {
162 int nr_frags = pinfo->nr_frags;
163 skb_frag_t *frag = pinfo->frags + nr_frags;
164 struct page *page = virt_to_head_page(skb->head);
165 unsigned int first_size = headlen - offset;
166 unsigned int first_offset;
167
168 if (nr_frags + 1 + skbinfo->nr_frags > MAX_SKB_FRAGS)
169 goto merge;
170
171 first_offset = skb->data -
172 (unsigned char *)page_address(page) +
173 offset;
174
175 pinfo->nr_frags = nr_frags + 1 + skbinfo->nr_frags;
176
177 skb_frag_fill_page_desc(frag, page, first_offset, first_size);
178
179 memcpy(frag + 1, skbinfo->frags, sizeof(*frag) * skbinfo->nr_frags);
180 /* We dont need to clear skbinfo->nr_frags here */
181
182 new_truesize = SKB_DATA_ALIGN(sizeof(struct sk_buff));
183 delta_truesize = skb->truesize - new_truesize;
184 skb->truesize = new_truesize;
185 NAPI_GRO_CB(skb)->free = NAPI_GRO_FREE_STOLEN_HEAD;
186 goto done;
187 }
188
189merge:
190 /* sk ownership - if any - completely transferred to the aggregated packet */
191 skb->destructor = NULL;
192 skb->sk = NULL;
193 delta_truesize = skb->truesize;
194 if (offset > headlen) {
195 unsigned int eat = offset - headlen;
196
197 skb_frag_off_add(&skbinfo->frags[0], eat);
198 skb_frag_size_sub(&skbinfo->frags[0], eat);
199 skb->data_len -= eat;
200 skb->len -= eat;
201 offset = headlen;
202 }
203
204 __skb_pull(skb, offset);
205
206 if (NAPI_GRO_CB(p)->last == p)
207 skb_shinfo(p)->frag_list = skb;
208 else
209 NAPI_GRO_CB(p)->last->next = skb;
210 NAPI_GRO_CB(p)->last = skb;
211 __skb_header_release(skb);
212 lp = p;
213
214done:
215 NAPI_GRO_CB(p)->count += segs;
216 p->data_len += len;
217 p->truesize += delta_truesize;
218 p->len += len;
219 skb_shinfo(p)->flags |= skbinfo->flags & SKBFL_SHARED_FRAG;
220 if (lp != p) {
221 lp->data_len += len;
222 lp->truesize += delta_truesize;
223 lp->len += len;
224 skb_shinfo(lp)->flags |= skbinfo->flags & SKBFL_SHARED_FRAG;
225 }
226 NAPI_GRO_CB(skb)->same_flow = 1;
227 return 0;
228}
229
230int skb_gro_receive_list(struct sk_buff *p, struct sk_buff *skb)
231{
232 /* make sure to check flush flag and to not merge */
233 if (unlikely(p->len + skb->len >= 65536 ||
234 NAPI_GRO_CB(skb)->flush))
235 return -E2BIG;
236
237 if (!pskb_may_pull(skb, skb_gro_offset(skb))) {
238 NAPI_GRO_CB(skb)->flush = 1;
239 return -ENOMEM;
240 }
241
242 if (NAPI_GRO_CB(p)->last == p)
243 skb_shinfo(p)->frag_list = skb;
244 else
245 NAPI_GRO_CB(p)->last->next = skb;
246
247 skb_pull(skb, skb_gro_offset(skb));
248
249 NAPI_GRO_CB(p)->last = skb;
250 NAPI_GRO_CB(p)->count++;
251 p->data_len += skb->len;
252
253 /* sk ownership - if any - completely transferred to the aggregated packet */
254 skb->destructor = NULL;
255 skb->sk = NULL;
256 p->truesize += skb->truesize;
257 p->len += skb->len;
258
259 skb_shinfo(p)->flags |= skb_shinfo(skb)->flags & SKBFL_SHARED_FRAG;
260
261 NAPI_GRO_CB(skb)->same_flow = 1;
262
263 return 0;
264}
265
266static void gro_complete(struct gro_node *gro, struct sk_buff *skb)
267{
268 struct list_head *head = &net_hotdata.offload_base;
269 struct packet_offload *ptype;
270 __be16 type = skb->protocol;
271 int err = -ENOENT;
272
273 BUILD_BUG_ON(sizeof(struct napi_gro_cb) > sizeof(skb->cb));
274
275 if (NAPI_GRO_CB(skb)->count == 1) {
276 skb_shinfo(skb)->gso_size = 0;
277 goto out;
278 }
279
280 /* NICs can feed encapsulated packets into GRO */
281 skb->encapsulation = 0;
282 rcu_read_lock();
283 list_for_each_entry_rcu(ptype, head, list) {
284 if (ptype->type != type || !ptype->callbacks.gro_complete)
285 continue;
286
287 err = INDIRECT_CALL_INET(ptype->callbacks.gro_complete,
288 ipv6_gro_complete, inet_gro_complete,
289 skb, 0);
290 break;
291 }
292 rcu_read_unlock();
293
294 if (err) {
295 WARN_ON(&ptype->list == head);
296 kfree_skb(skb);
297 return;
298 }
299
300out:
301 gro_normal_one(gro, skb, NAPI_GRO_CB(skb)->count);
302}
303
304static void __gro_flush_chain(struct gro_node *gro, u32 index, bool flush_old)
305{
306 struct list_head *head = &gro->hash[index].list;
307 struct sk_buff *skb, *p;
308
309 list_for_each_entry_safe_reverse(skb, p, head, list) {
310 if (flush_old && NAPI_GRO_CB(skb)->age == jiffies)
311 return;
312 skb_list_del_init(skb);
313 gro_complete(gro, skb);
314 gro->hash[index].count--;
315 }
316
317 if (!gro->hash[index].count)
318 __clear_bit(index, &gro->bitmask);
319}
320
321/*
322 * gro->hash[].list contains packets ordered by age.
323 * youngest packets at the head of it.
324 * Complete skbs in reverse order to reduce latencies.
325 */
326void __gro_flush(struct gro_node *gro, bool flush_old)
327{
328 unsigned long bitmask = gro->bitmask;
329 unsigned int i, base = ~0U;
330
331 while ((i = ffs(bitmask)) != 0) {
332 bitmask >>= i;
333 base += i;
334 __gro_flush_chain(gro, base, flush_old);
335 }
336}
337EXPORT_SYMBOL(__gro_flush);
338
339static unsigned long gro_list_prepare_tc_ext(const struct sk_buff *skb,
340 const struct sk_buff *p,
341 unsigned long diffs)
342{
343#if IS_ENABLED(CONFIG_NET_TC_SKB_EXT)
344 struct tc_skb_ext *skb_ext;
345 struct tc_skb_ext *p_ext;
346
347 skb_ext = skb_ext_find(skb, TC_SKB_EXT);
348 p_ext = skb_ext_find(p, TC_SKB_EXT);
349
350 diffs |= (!!p_ext) ^ (!!skb_ext);
351 if (!diffs && unlikely(skb_ext))
352 diffs |= p_ext->chain ^ skb_ext->chain;
353#endif
354 return diffs;
355}
356
357static void gro_list_prepare(const struct list_head *head,
358 const struct sk_buff *skb)
359{
360 unsigned int maclen = skb->dev->hard_header_len;
361 u32 hash = skb_get_hash_raw(skb);
362 struct sk_buff *p;
363
364 list_for_each_entry(p, head, list) {
365 unsigned long diffs;
366
367 if (hash != skb_get_hash_raw(p)) {
368 NAPI_GRO_CB(p)->same_flow = 0;
369 continue;
370 }
371
372 diffs = (unsigned long)p->dev ^ (unsigned long)skb->dev;
373 diffs |= p->vlan_all ^ skb->vlan_all;
374 diffs |= skb_metadata_differs(p, skb);
375 if (maclen == ETH_HLEN)
376 diffs |= compare_ether_header(skb_mac_header(p),
377 skb_mac_header(skb));
378 else if (!diffs)
379 diffs = memcmp(skb_mac_header(p),
380 skb_mac_header(skb),
381 maclen);
382
383 /* in most common scenarios 'slow_gro' is 0
384 * otherwise we are already on some slower paths
385 * either skip all the infrequent tests altogether or
386 * avoid trying too hard to skip each of them individually
387 */
388 if (!diffs && unlikely(skb->slow_gro | p->slow_gro)) {
389 diffs |= p->sk != skb->sk;
390 diffs |= skb_metadata_dst_cmp(p, skb);
391 diffs |= skb_get_nfct(p) ^ skb_get_nfct(skb);
392
393 diffs |= gro_list_prepare_tc_ext(skb, p, diffs);
394 diffs |= __psp_skb_coalesce_diff(skb, p, diffs);
395 }
396
397 NAPI_GRO_CB(p)->same_flow = !diffs;
398 }
399}
400
401static inline void skb_gro_reset_offset(struct sk_buff *skb, u32 nhoff)
402{
403 const struct skb_shared_info *pinfo;
404 const skb_frag_t *frag0;
405 unsigned int headlen;
406
407 NAPI_GRO_CB(skb)->network_offset = 0;
408 NAPI_GRO_CB(skb)->data_offset = 0;
409 headlen = skb_headlen(skb);
410 NAPI_GRO_CB(skb)->frag0 = skb->data;
411 NAPI_GRO_CB(skb)->frag0_len = headlen;
412 if (headlen)
413 return;
414
415 pinfo = skb_shinfo(skb);
416 frag0 = &pinfo->frags[0];
417
418 if (pinfo->nr_frags && skb_frag_page(frag0) &&
419 !PageHighMem(skb_frag_page(frag0)) &&
420 (!NET_IP_ALIGN || !((skb_frag_off(frag0) + nhoff) & 3))) {
421 NAPI_GRO_CB(skb)->frag0 = skb_frag_address(frag0);
422 NAPI_GRO_CB(skb)->frag0_len = min_t(unsigned int,
423 skb_frag_size(frag0),
424 skb->end - skb->tail);
425 }
426}
427
428static void gro_pull_from_frag0(struct sk_buff *skb, int grow)
429{
430 struct skb_shared_info *pinfo = skb_shinfo(skb);
431
432 DEBUG_NET_WARN_ON_ONCE(skb->end - skb->tail < grow);
433
434 memcpy(skb_tail_pointer(skb), NAPI_GRO_CB(skb)->frag0, grow);
435
436 skb->data_len -= grow;
437 skb->tail += grow;
438
439 skb_frag_off_add(&pinfo->frags[0], grow);
440 skb_frag_size_sub(&pinfo->frags[0], grow);
441
442 if (unlikely(!skb_frag_size(&pinfo->frags[0]))) {
443 skb_frag_unref(skb, 0);
444 memmove(pinfo->frags, pinfo->frags + 1,
445 --pinfo->nr_frags * sizeof(pinfo->frags[0]));
446 }
447}
448
449static void gro_try_pull_from_frag0(struct sk_buff *skb)
450{
451 int grow = skb_gro_offset(skb) - skb_headlen(skb);
452
453 if (grow > 0)
454 gro_pull_from_frag0(skb, grow);
455}
456
457static void gro_flush_oldest(struct gro_node *gro, struct list_head *head)
458{
459 struct sk_buff *oldest;
460
461 oldest = list_last_entry(head, struct sk_buff, list);
462
463 /* We are called with head length >= MAX_GRO_SKBS, so this is
464 * impossible.
465 */
466 if (WARN_ON_ONCE(!oldest))
467 return;
468
469 /* Do not adjust napi->gro_hash[].count, caller is adding a new
470 * SKB to the chain.
471 */
472 skb_list_del_init(oldest);
473 gro_complete(gro, oldest);
474}
475
476static enum gro_result dev_gro_receive(struct gro_node *gro,
477 struct sk_buff *skb)
478{
479 u32 bucket = skb_get_hash_raw(skb) & (GRO_HASH_BUCKETS - 1);
480 struct list_head *head = &net_hotdata.offload_base;
481 struct gro_list *gro_list = &gro->hash[bucket];
482 struct packet_offload *ptype;
483 __be16 type = skb->protocol;
484 struct sk_buff *pp = NULL;
485 enum gro_result ret;
486 int same_flow;
487
488 if (netif_elide_gro(skb->dev))
489 goto normal;
490
491 gro_list_prepare(&gro_list->list, skb);
492
493 rcu_read_lock();
494 list_for_each_entry_rcu(ptype, head, list) {
495 if (ptype->type == type && ptype->callbacks.gro_receive)
496 goto found_ptype;
497 }
498 rcu_read_unlock();
499 goto normal;
500
501found_ptype:
502 skb_set_network_header(skb, skb_gro_offset(skb));
503 skb_reset_mac_len(skb);
504 BUILD_BUG_ON(sizeof_field(struct napi_gro_cb, zeroed) != sizeof(u32));
505 BUILD_BUG_ON(!IS_ALIGNED(offsetof(struct napi_gro_cb, zeroed),
506 sizeof(u32))); /* Avoid slow unaligned acc */
507 *(u32 *)&NAPI_GRO_CB(skb)->zeroed = 0;
508 NAPI_GRO_CB(skb)->flush = skb_has_frag_list(skb);
509 NAPI_GRO_CB(skb)->count = 1;
510 if (unlikely(skb_is_gso(skb))) {
511 NAPI_GRO_CB(skb)->count = skb_shinfo(skb)->gso_segs;
512 /* Only support TCP and non DODGY users. */
513 if (!skb_is_gso_tcp(skb) ||
514 (skb_shinfo(skb)->gso_type & SKB_GSO_DODGY))
515 NAPI_GRO_CB(skb)->flush = 1;
516 }
517
518 /* Setup for GRO checksum validation */
519 switch (skb->ip_summed) {
520 case CHECKSUM_COMPLETE:
521 NAPI_GRO_CB(skb)->csum = skb->csum;
522 NAPI_GRO_CB(skb)->csum_valid = 1;
523 break;
524 case CHECKSUM_UNNECESSARY:
525 NAPI_GRO_CB(skb)->csum_cnt = skb->csum_level + 1;
526 break;
527 }
528
529 pp = INDIRECT_CALL_INET(ptype->callbacks.gro_receive,
530 ipv6_gro_receive, inet_gro_receive,
531 &gro_list->list, skb);
532
533 rcu_read_unlock();
534
535 if (PTR_ERR(pp) == -EINPROGRESS) {
536 ret = GRO_CONSUMED;
537 goto ok;
538 }
539
540 same_flow = NAPI_GRO_CB(skb)->same_flow;
541 ret = NAPI_GRO_CB(skb)->free ? GRO_MERGED_FREE : GRO_MERGED;
542
543 if (pp) {
544 skb_list_del_init(pp);
545 gro_complete(gro, pp);
546 gro_list->count--;
547 }
548
549 if (same_flow)
550 goto ok;
551
552 if (NAPI_GRO_CB(skb)->flush)
553 goto normal;
554
555 if (unlikely(gro_list->count >= MAX_GRO_SKBS))
556 gro_flush_oldest(gro, &gro_list->list);
557 else
558 gro_list->count++;
559
560 /* Must be called before setting NAPI_GRO_CB(skb)->{age|last} */
561 gro_try_pull_from_frag0(skb);
562 NAPI_GRO_CB(skb)->age = jiffies;
563 NAPI_GRO_CB(skb)->last = skb;
564 if (!skb_is_gso(skb))
565 skb_shinfo(skb)->gso_size = skb_gro_len(skb);
566 list_add(&skb->list, &gro_list->list);
567 ret = GRO_HELD;
568ok:
569 if (gro_list->count) {
570 if (!test_bit(bucket, &gro->bitmask))
571 __set_bit(bucket, &gro->bitmask);
572 } else if (test_bit(bucket, &gro->bitmask)) {
573 __clear_bit(bucket, &gro->bitmask);
574 }
575
576 return ret;
577
578normal:
579 ret = GRO_NORMAL;
580 gro_try_pull_from_frag0(skb);
581 goto ok;
582}
583
584struct packet_offload *gro_find_receive_by_type(__be16 type)
585{
586 struct list_head *offload_head = &net_hotdata.offload_base;
587 struct packet_offload *ptype;
588
589 list_for_each_entry_rcu(ptype, offload_head, list) {
590 if (ptype->type != type || !ptype->callbacks.gro_receive)
591 continue;
592 return ptype;
593 }
594 return NULL;
595}
596EXPORT_SYMBOL(gro_find_receive_by_type);
597
598struct packet_offload *gro_find_complete_by_type(__be16 type)
599{
600 struct list_head *offload_head = &net_hotdata.offload_base;
601 struct packet_offload *ptype;
602
603 list_for_each_entry_rcu(ptype, offload_head, list) {
604 if (ptype->type != type || !ptype->callbacks.gro_complete)
605 continue;
606 return ptype;
607 }
608 return NULL;
609}
610EXPORT_SYMBOL(gro_find_complete_by_type);
611
612static gro_result_t gro_skb_finish(struct gro_node *gro, struct sk_buff *skb,
613 gro_result_t ret)
614{
615 switch (ret) {
616 case GRO_NORMAL:
617 gro_normal_one(gro, skb, 1);
618 break;
619
620 case GRO_MERGED_FREE:
621 if (NAPI_GRO_CB(skb)->free == NAPI_GRO_FREE_STOLEN_HEAD)
622 napi_skb_free_stolen_head(skb);
623 else if (skb->fclone != SKB_FCLONE_UNAVAILABLE)
624 __kfree_skb(skb);
625 else
626 __napi_kfree_skb(skb, SKB_CONSUMED);
627 break;
628
629 case GRO_HELD:
630 case GRO_MERGED:
631 case GRO_CONSUMED:
632 break;
633 }
634
635 return ret;
636}
637
638gro_result_t gro_receive_skb(struct gro_node *gro, struct sk_buff *skb)
639{
640 gro_result_t ret;
641
642 __skb_mark_napi_id(skb, gro);
643 trace_napi_gro_receive_entry(skb);
644
645 skb_gro_reset_offset(skb, 0);
646
647 ret = gro_skb_finish(gro, skb, dev_gro_receive(gro, skb));
648 trace_napi_gro_receive_exit(ret);
649
650 return ret;
651}
652EXPORT_SYMBOL(gro_receive_skb);
653
654static void napi_reuse_skb(struct napi_struct *napi, struct sk_buff *skb)
655{
656 struct skb_shared_info *shinfo;
657
658 if (unlikely(skb->pfmemalloc)) {
659 consume_skb(skb);
660 return;
661 }
662 __skb_pull(skb, skb_headlen(skb));
663 /* restore the reserve we had after netdev_alloc_skb_ip_align() */
664 skb_reserve(skb, NET_SKB_PAD + NET_IP_ALIGN - skb_headroom(skb));
665 __vlan_hwaccel_clear_tag(skb);
666 skb->dev = napi->dev;
667 skb->skb_iif = 0;
668
669 /* eth_type_trans() assumes pkt_type is PACKET_HOST */
670 skb->pkt_type = PACKET_HOST;
671
672 skb->encapsulation = 0;
673 skb->ip_summed = CHECKSUM_NONE;
674
675 shinfo = skb_shinfo(skb);
676 shinfo->gso_type = 0;
677 shinfo->gso_size = 0;
678 shinfo->hwtstamps.hwtstamp = 0;
679
680 if (unlikely(skb->slow_gro)) {
681 skb_orphan(skb);
682 skb_ext_reset(skb);
683 nf_reset_ct(skb);
684 skb->slow_gro = 0;
685 }
686
687 napi->skb = skb;
688}
689
690struct sk_buff *napi_get_frags(struct napi_struct *napi)
691{
692 struct sk_buff *skb = napi->skb;
693
694 if (!skb) {
695 skb = napi_alloc_skb(napi, GRO_MAX_HEAD);
696 if (skb) {
697 napi->skb = skb;
698 skb_mark_napi_id(skb, napi);
699 }
700 }
701 return skb;
702}
703EXPORT_SYMBOL(napi_get_frags);
704
705static gro_result_t napi_frags_finish(struct napi_struct *napi,
706 struct sk_buff *skb,
707 gro_result_t ret)
708{
709 switch (ret) {
710 case GRO_NORMAL:
711 case GRO_HELD:
712 __skb_push(skb, ETH_HLEN);
713 skb->protocol = eth_type_trans(skb, skb->dev);
714 if (ret == GRO_NORMAL)
715 gro_normal_one(&napi->gro, skb, 1);
716 break;
717
718 case GRO_MERGED_FREE:
719 if (NAPI_GRO_CB(skb)->free == NAPI_GRO_FREE_STOLEN_HEAD)
720 napi_skb_free_stolen_head(skb);
721 else
722 napi_reuse_skb(napi, skb);
723 break;
724
725 case GRO_MERGED:
726 case GRO_CONSUMED:
727 break;
728 }
729
730 return ret;
731}
732
733/* Upper GRO stack assumes network header starts at gro_offset=0
734 * Drivers could call both napi_gro_frags() and napi_gro_receive()
735 * We copy ethernet header into skb->data to have a common layout.
736 */
737static struct sk_buff *napi_frags_skb(struct napi_struct *napi)
738{
739 struct sk_buff *skb = napi->skb;
740 const struct ethhdr *eth;
741 unsigned int hlen = sizeof(*eth);
742
743 napi->skb = NULL;
744
745 skb_reset_mac_header(skb);
746 skb_gro_reset_offset(skb, hlen);
747
748 if (unlikely(!skb_gro_may_pull(skb, hlen))) {
749 eth = skb_gro_header_slow(skb, hlen, 0);
750 if (unlikely(!eth)) {
751 net_warn_ratelimited("%s: dropping impossible skb from %s\n",
752 __func__, napi->dev->name);
753 napi_reuse_skb(napi, skb);
754 return NULL;
755 }
756 } else {
757 eth = (const struct ethhdr *)skb->data;
758
759 if (NAPI_GRO_CB(skb)->frag0 != skb->data)
760 gro_pull_from_frag0(skb, hlen);
761
762 NAPI_GRO_CB(skb)->frag0 += hlen;
763 NAPI_GRO_CB(skb)->frag0_len -= hlen;
764 }
765 __skb_pull(skb, hlen);
766
767 /*
768 * This works because the only protocols we care about don't require
769 * special handling.
770 * We'll fix it up properly in napi_frags_finish()
771 */
772 skb->protocol = eth->h_proto;
773
774 return skb;
775}
776
777gro_result_t napi_gro_frags(struct napi_struct *napi)
778{
779 gro_result_t ret;
780 struct sk_buff *skb = napi_frags_skb(napi);
781
782 trace_napi_gro_frags_entry(skb);
783
784 ret = napi_frags_finish(napi, skb, dev_gro_receive(&napi->gro, skb));
785 trace_napi_gro_frags_exit(ret);
786
787 return ret;
788}
789EXPORT_SYMBOL(napi_gro_frags);
790
791/* Compute the checksum from gro_offset and return the folded value
792 * after adding in any pseudo checksum.
793 */
794__sum16 __skb_gro_checksum_complete(struct sk_buff *skb)
795{
796 __wsum wsum;
797 __sum16 sum;
798
799 wsum = skb_checksum(skb, skb_gro_offset(skb), skb_gro_len(skb), 0);
800
801 /* NAPI_GRO_CB(skb)->csum holds pseudo checksum */
802 sum = csum_fold(csum_add(NAPI_GRO_CB(skb)->csum, wsum));
803 /* See comments in __skb_checksum_complete(). */
804 if (likely(!sum)) {
805 if (unlikely(skb->ip_summed == CHECKSUM_COMPLETE) &&
806 !skb->csum_complete_sw)
807 netdev_rx_csum_fault(skb->dev, skb);
808 }
809
810 NAPI_GRO_CB(skb)->csum = wsum;
811 NAPI_GRO_CB(skb)->csum_valid = 1;
812
813 return sum;
814}
815EXPORT_SYMBOL(__skb_gro_checksum_complete);
816
817void gro_init(struct gro_node *gro)
818{
819 for (u32 i = 0; i < GRO_HASH_BUCKETS; i++) {
820 INIT_LIST_HEAD(&gro->hash[i].list);
821 gro->hash[i].count = 0;
822 }
823
824 gro->bitmask = 0;
825 gro->cached_napi_id = 0;
826
827 INIT_LIST_HEAD(&gro->rx_list);
828 gro->rx_count = 0;
829}
830
831void gro_cleanup(struct gro_node *gro)
832{
833 struct sk_buff *skb, *n;
834
835 for (u32 i = 0; i < GRO_HASH_BUCKETS; i++) {
836 list_for_each_entry_safe(skb, n, &gro->hash[i].list, list)
837 kfree_skb(skb);
838
839 gro->hash[i].count = 0;
840 }
841
842 gro->bitmask = 0;
843 gro->cached_napi_id = 0;
844
845 list_for_each_entry_safe(skb, n, &gro->rx_list, list)
846 kfree_skb(skb);
847
848 gro->rx_count = 0;
849}