projects/linux·blob master · refs

2029 lines · 47.9 KiB

1// SPDX-License-Identifier: GPL-2.0-only
2/*
3 * drivers/net/veth.c
4 *
5 * Copyright (C) 2007 OpenVZ http://openvz.org, SWsoft Inc
6 *
7 * Author: Pavel Emelianov <[email protected]>
8 * Ethtool interface from: Eric W. Biederman <[email protected]>
9 *
10 */
11
12#include <linux/netdevice.h>
13#include <linux/slab.h>
14#include <linux/ethtool.h>
15#include <linux/etherdevice.h>
16#include <linux/u64_stats_sync.h>
17
18#include <net/rtnetlink.h>
19#include <net/dst.h>
20#include <net/netdev_lock.h>
21#include <net/xfrm.h>
22#include <net/xdp.h>
23#include <linux/veth.h>
24#include <linux/module.h>
25#include <linux/bpf.h>
26#include <linux/filter.h>
27#include <linux/ptr_ring.h>
28#include <linux/bpf_trace.h>
29#include <linux/net_tstamp.h>
30#include <linux/skbuff_ref.h>
31#include <net/page_pool/helpers.h>
32
33#define DRV_NAME "veth"
34#define DRV_VERSION "1.0"
35
36#define VETH_XDP_FLAG BIT(0)
37#define VETH_RING_SIZE 256
38#define VETH_XDP_HEADROOM (XDP_PACKET_HEADROOM + NET_IP_ALIGN)
39
40#define VETH_XDP_TX_BULK_SIZE 16
41#define VETH_XDP_BATCH 16
42
43struct veth_stats {
44 u64 rx_drops;
45 /* xdp */
46 u64 xdp_packets;
47 u64 xdp_bytes;
48 u64 xdp_redirect;
49 u64 xdp_drops;
50 u64 xdp_tx;
51 u64 xdp_tx_err;
52 u64 peer_tq_xdp_xmit;
53 u64 peer_tq_xdp_xmit_err;
54};
55
56struct veth_rq_stats {
57 struct veth_stats vs;
58 struct u64_stats_sync syncp;
59};
60
61struct veth_rq {
62 struct napi_struct xdp_napi;
63 struct napi_struct __rcu *napi; /* points to xdp_napi when the latter is initialized */
64 struct net_device *dev;
65 struct bpf_prog __rcu *xdp_prog;
66 struct xdp_mem_info xdp_mem;
67 struct veth_rq_stats stats;
68 bool rx_notify_masked;
69 struct ptr_ring xdp_ring;
70 struct xdp_rxq_info xdp_rxq;
71 struct page_pool *page_pool;
72};
73
74struct veth_priv {
75 struct net_device __rcu *peer;
76 atomic64_t dropped;
77 struct bpf_prog *_xdp_prog;
78 struct veth_rq *rq;
79 unsigned int requested_headroom;
80 netdevice_tracker peer_tracker;
81};
82
83struct veth_xdp_tx_bq {
84 struct xdp_frame *q[VETH_XDP_TX_BULK_SIZE];
85 unsigned int count;
86};
87
88/*
89 * ethtool interface
90 */
91
92struct veth_q_stat_desc {
93 char desc[ETH_GSTRING_LEN];
94 size_t offset;
95};
96
97#define VETH_RQ_STAT(m) offsetof(struct veth_stats, m)
98
99static const struct veth_q_stat_desc veth_rq_stats_desc[] = {
100 { "xdp_packets", VETH_RQ_STAT(xdp_packets) },
101 { "xdp_bytes", VETH_RQ_STAT(xdp_bytes) },
102 { "drops", VETH_RQ_STAT(rx_drops) },
103 { "xdp_redirect", VETH_RQ_STAT(xdp_redirect) },
104 { "xdp_drops", VETH_RQ_STAT(xdp_drops) },
105 { "xdp_tx", VETH_RQ_STAT(xdp_tx) },
106 { "xdp_tx_errors", VETH_RQ_STAT(xdp_tx_err) },
107};
108
109#define VETH_RQ_STATS_LEN ARRAY_SIZE(veth_rq_stats_desc)
110
111static const struct veth_q_stat_desc veth_tq_stats_desc[] = {
112 { "xdp_xmit", VETH_RQ_STAT(peer_tq_xdp_xmit) },
113 { "xdp_xmit_errors", VETH_RQ_STAT(peer_tq_xdp_xmit_err) },
114};
115
116#define VETH_TQ_STATS_LEN ARRAY_SIZE(veth_tq_stats_desc)
117
118static struct {
119 const char string[ETH_GSTRING_LEN];
120} ethtool_stats_keys[] = {
121 { "peer_ifindex" },
122};
123
124struct veth_xdp_buff {
125 struct xdp_buff xdp;
126 struct sk_buff *skb;
127};
128
129static int veth_get_link_ksettings(struct net_device *dev,
130 struct ethtool_link_ksettings *cmd)
131{
132 cmd->base.speed = SPEED_10000;
133 cmd->base.duplex = DUPLEX_FULL;
134 cmd->base.port = PORT_TP;
135 cmd->base.autoneg = AUTONEG_DISABLE;
136 return 0;
137}
138
139static void veth_get_drvinfo(struct net_device *dev, struct ethtool_drvinfo *info)
140{
141 strscpy(info->driver, DRV_NAME, sizeof(info->driver));
142 strscpy(info->version, DRV_VERSION, sizeof(info->version));
143}
144
145static void veth_get_strings(struct net_device *dev, u32 stringset, u8 *buf)
146{
147 u8 *p = buf;
148 int i, j;
149
150 switch(stringset) {
151 case ETH_SS_STATS:
152 memcpy(p, &ethtool_stats_keys, sizeof(ethtool_stats_keys));
153 p += sizeof(ethtool_stats_keys);
154 for (i = 0; i < dev->real_num_rx_queues; i++)
155 for (j = 0; j < VETH_RQ_STATS_LEN; j++)
156 ethtool_sprintf(&p, "rx_queue_%u_%.18s",
157 i, veth_rq_stats_desc[j].desc);
158
159 for (i = 0; i < dev->real_num_tx_queues; i++)
160 for (j = 0; j < VETH_TQ_STATS_LEN; j++)
161 ethtool_sprintf(&p, "tx_queue_%u_%.18s",
162 i, veth_tq_stats_desc[j].desc);
163
164 page_pool_ethtool_stats_get_strings(p);
165 break;
166 }
167}
168
169static int veth_get_sset_count(struct net_device *dev, int sset)
170{
171 switch (sset) {
172 case ETH_SS_STATS:
173 return ARRAY_SIZE(ethtool_stats_keys) +
174 VETH_RQ_STATS_LEN * dev->real_num_rx_queues +
175 VETH_TQ_STATS_LEN * dev->real_num_tx_queues +
176 page_pool_ethtool_stats_get_count();
177 default:
178 return -EOPNOTSUPP;
179 }
180}
181
182static void veth_get_page_pool_stats(struct net_device *dev, u64 *data)
183{
184#ifdef CONFIG_PAGE_POOL_STATS
185 struct veth_priv *priv = netdev_priv(dev);
186 struct page_pool_stats pp_stats = {};
187 int i;
188
189 for (i = 0; i < dev->real_num_rx_queues; i++) {
190 if (!priv->rq[i].page_pool)
191 continue;
192 page_pool_get_stats(priv->rq[i].page_pool, &pp_stats);
193 }
194 page_pool_ethtool_stats_get(data, &pp_stats);
195#endif /* CONFIG_PAGE_POOL_STATS */
196}
197
198static void veth_get_ethtool_stats(struct net_device *dev,
199 struct ethtool_stats *stats, u64 *data)
200{
201 struct veth_priv *rcv_priv, *priv = netdev_priv(dev);
202 struct net_device *peer = rtnl_dereference(priv->peer);
203 int i, j, idx, pp_idx;
204
205 data[0] = peer ? peer->ifindex : 0;
206 idx = 1;
207 for (i = 0; i < dev->real_num_rx_queues; i++) {
208 const struct veth_rq_stats *rq_stats = &priv->rq[i].stats;
209 const void *stats_base = (void *)&rq_stats->vs;
210 unsigned int start;
211 size_t offset;
212
213 do {
214 start = u64_stats_fetch_begin(&rq_stats->syncp);
215 for (j = 0; j < VETH_RQ_STATS_LEN; j++) {
216 offset = veth_rq_stats_desc[j].offset;
217 data[idx + j] = *(u64 *)(stats_base + offset);
218 }
219 } while (u64_stats_fetch_retry(&rq_stats->syncp, start));
220 idx += VETH_RQ_STATS_LEN;
221 }
222 pp_idx = idx;
223
224 if (!peer)
225 goto page_pool_stats;
226
227 rcv_priv = netdev_priv(peer);
228 for (i = 0; i < peer->real_num_rx_queues; i++) {
229 const struct veth_rq_stats *rq_stats = &rcv_priv->rq[i].stats;
230 const void *base = (void *)&rq_stats->vs;
231 unsigned int start, tx_idx = idx;
232 u64 buf[VETH_TQ_STATS_LEN];
233 size_t offset;
234
235 do {
236 start = u64_stats_fetch_begin(&rq_stats->syncp);
237 for (j = 0; j < VETH_TQ_STATS_LEN; j++) {
238 offset = veth_tq_stats_desc[j].offset;
239 buf[j] = *(u64 *)(base + offset);
240 }
241 } while (u64_stats_fetch_retry(&rq_stats->syncp, start));
242
243 tx_idx += (i % dev->real_num_tx_queues) * VETH_TQ_STATS_LEN;
244 for (j = 0; j < VETH_TQ_STATS_LEN; j++)
245 data[tx_idx + j] += buf[j];
246 }
247 pp_idx = idx + dev->real_num_tx_queues * VETH_TQ_STATS_LEN;
248
249page_pool_stats:
250 veth_get_page_pool_stats(dev, &data[pp_idx]);
251}
252
253static void veth_get_channels(struct net_device *dev,
254 struct ethtool_channels *channels)
255{
256 channels->tx_count = dev->real_num_tx_queues;
257 channels->rx_count = dev->real_num_rx_queues;
258 channels->max_tx = dev->num_tx_queues;
259 channels->max_rx = dev->num_rx_queues;
260}
261
262static int veth_set_channels(struct net_device *dev,
263 struct ethtool_channels *ch);
264
265static const struct ethtool_ops veth_ethtool_ops = {
266 .get_drvinfo = veth_get_drvinfo,
267 .get_link = ethtool_op_get_link,
268 .get_strings = veth_get_strings,
269 .get_sset_count = veth_get_sset_count,
270 .get_ethtool_stats = veth_get_ethtool_stats,
271 .get_link_ksettings = veth_get_link_ksettings,
272 .get_ts_info = ethtool_op_get_ts_info,
273 .get_channels = veth_get_channels,
274 .set_channels = veth_set_channels,
275};
276
277/* general routines */
278
279static bool veth_is_xdp_frame(void *ptr)
280{
281 return (unsigned long)ptr & VETH_XDP_FLAG;
282}
283
284static struct xdp_frame *veth_ptr_to_xdp(void *ptr)
285{
286 return (void *)((unsigned long)ptr & ~VETH_XDP_FLAG);
287}
288
289static void *veth_xdp_to_ptr(struct xdp_frame *xdp)
290{
291 return (void *)((unsigned long)xdp | VETH_XDP_FLAG);
292}
293
294static void veth_ptr_free(void *ptr)
295{
296 if (veth_is_xdp_frame(ptr))
297 xdp_return_frame(veth_ptr_to_xdp(ptr));
298 else
299 kfree_skb(ptr);
300}
301
302static void __veth_xdp_flush(struct veth_rq *rq)
303{
304 /* Write ptr_ring before reading rx_notify_masked */
305 smp_mb();
306 if (!READ_ONCE(rq->rx_notify_masked) &&
307 napi_schedule_prep(&rq->xdp_napi)) {
308 WRITE_ONCE(rq->rx_notify_masked, true);
309 __napi_schedule(&rq->xdp_napi);
310 }
311}
312
313static int veth_xdp_rx(struct veth_rq *rq, struct sk_buff *skb)
314{
315 if (unlikely(ptr_ring_produce(&rq->xdp_ring, skb)))
316 return NETDEV_TX_BUSY; /* signal qdisc layer */
317
318 return NET_RX_SUCCESS; /* same as NETDEV_TX_OK */
319}
320
321static int veth_forward_skb(struct net_device *dev, struct sk_buff *skb,
322 struct veth_rq *rq, bool xdp)
323{
324 return __dev_forward_skb(dev, skb) ?: xdp ?
325 veth_xdp_rx(rq, skb) :
326 __netif_rx(skb);
327}
328
329/* return true if the specified skb has chances of GRO aggregation
330 * Don't strive for accuracy, but try to avoid GRO overhead in the most
331 * common scenarios.
332 * When XDP is enabled, all traffic is considered eligible, as the xmit
333 * device has TSO off.
334 * When TSO is enabled on the xmit device, we are likely interested only
335 * in UDP aggregation, explicitly check for that if the skb is suspected
336 * - the sock_wfree destructor is used by UDP, ICMP and XDP sockets -
337 * to belong to locally generated UDP traffic.
338 */
339static bool veth_skb_is_eligible_for_gro(const struct net_device *dev,
340 const struct net_device *rcv,
341 const struct sk_buff *skb)
342{
343 return !(dev->features & NETIF_F_ALL_TSO) ||
344 (skb->destructor == sock_wfree &&
345 rcv->features & (NETIF_F_GRO_FRAGLIST | NETIF_F_GRO_UDP_FWD));
346}
347
348static netdev_tx_t veth_xmit(struct sk_buff *skb, struct net_device *dev)
349{
350 struct veth_priv *rcv_priv, *priv = netdev_priv(dev);
351 struct veth_rq *rq = NULL;
352 struct netdev_queue *txq;
353 struct net_device *rcv;
354 int length = skb->len;
355 bool use_napi = false;
356 int ret, rxq;
357
358 rcu_read_lock();
359 rcv = rcu_dereference(priv->peer);
360 if (unlikely(!rcv) || !pskb_may_pull(skb, ETH_HLEN)) {
361 kfree_skb(skb);
362 goto drop;
363 }
364
365 rcv_priv = netdev_priv(rcv);
366 rxq = skb_get_queue_mapping(skb);
367 if (rxq < rcv->real_num_rx_queues) {
368 rq = &rcv_priv->rq[rxq];
369
370 /* The napi pointer is available when an XDP program is
371 * attached or when GRO is enabled
372 * Don't bother with napi/GRO if the skb can't be aggregated
373 */
374 use_napi = rcu_access_pointer(rq->napi) &&
375 veth_skb_is_eligible_for_gro(dev, rcv, skb);
376 }
377
378 skb_tx_timestamp(skb);
379
380 ret = veth_forward_skb(rcv, skb, rq, use_napi);
381 switch (ret) {
382 case NET_RX_SUCCESS: /* same as NETDEV_TX_OK */
383 if (!use_napi)
384 dev_sw_netstats_tx_add(dev, 1, length);
385 else
386 __veth_xdp_flush(rq);
387 break;
388 case NETDEV_TX_BUSY:
389 /* If a qdisc is attached to our virtual device, returning
390 * NETDEV_TX_BUSY is allowed.
391 */
392 txq = netdev_get_tx_queue(dev, rxq);
393
394 if (qdisc_txq_has_no_queue(txq)) {
395 dev_kfree_skb_any(skb);
396 goto drop;
397 }
398 /* Restore Eth hdr pulled by dev_forward_skb/eth_type_trans */
399 __skb_push(skb, ETH_HLEN);
400 netif_tx_stop_queue(txq);
401 /* Makes sure NAPI peer consumer runs. Consumer is responsible
402 * for starting txq again, until then ndo_start_xmit (this
403 * function) will not be invoked by the netstack again.
404 */
405 __veth_xdp_flush(rq);
406 break;
407 case NET_RX_DROP: /* same as NET_XMIT_DROP */
408drop:
409 atomic64_inc(&priv->dropped);
410 ret = NET_XMIT_DROP;
411 break;
412 default:
413 net_crit_ratelimited("%s(%s): Invalid return code(%d)",
414 __func__, dev->name, ret);
415 }
416 rcu_read_unlock();
417
418 return ret;
419}
420
421static void veth_stats_rx(struct veth_stats *result, struct net_device *dev)
422{
423 struct veth_priv *priv = netdev_priv(dev);
424 int i;
425
426 result->peer_tq_xdp_xmit_err = 0;
427 result->xdp_packets = 0;
428 result->xdp_tx_err = 0;
429 result->xdp_bytes = 0;
430 result->rx_drops = 0;
431 for (i = 0; i < dev->num_rx_queues; i++) {
432 u64 packets, bytes, drops, xdp_tx_err, peer_tq_xdp_xmit_err;
433 struct veth_rq_stats *stats = &priv->rq[i].stats;
434 unsigned int start;
435
436 do {
437 start = u64_stats_fetch_begin(&stats->syncp);
438 peer_tq_xdp_xmit_err = stats->vs.peer_tq_xdp_xmit_err;
439 xdp_tx_err = stats->vs.xdp_tx_err;
440 packets = stats->vs.xdp_packets;
441 bytes = stats->vs.xdp_bytes;
442 drops = stats->vs.rx_drops;
443 } while (u64_stats_fetch_retry(&stats->syncp, start));
444 result->peer_tq_xdp_xmit_err += peer_tq_xdp_xmit_err;
445 result->xdp_tx_err += xdp_tx_err;
446 result->xdp_packets += packets;
447 result->xdp_bytes += bytes;
448 result->rx_drops += drops;
449 }
450}
451
452static void veth_get_stats64(struct net_device *dev,
453 struct rtnl_link_stats64 *tot)
454{
455 struct veth_priv *priv = netdev_priv(dev);
456 struct net_device *peer;
457 struct veth_stats rx;
458
459 tot->tx_dropped = atomic64_read(&priv->dropped);
460 dev_fetch_sw_netstats(tot, dev->tstats);
461
462 veth_stats_rx(&rx, dev);
463 tot->tx_dropped += rx.xdp_tx_err;
464 tot->rx_dropped = rx.rx_drops + rx.peer_tq_xdp_xmit_err;
465 tot->rx_bytes += rx.xdp_bytes;
466 tot->rx_packets += rx.xdp_packets;
467
468 rcu_read_lock();
469 peer = rcu_dereference(priv->peer);
470 if (peer) {
471 struct rtnl_link_stats64 tot_peer = {};
472
473 dev_fetch_sw_netstats(&tot_peer, peer->tstats);
474 tot->rx_bytes += tot_peer.tx_bytes;
475 tot->rx_packets += tot_peer.tx_packets;
476
477 veth_stats_rx(&rx, peer);
478 tot->tx_dropped += rx.peer_tq_xdp_xmit_err;
479 tot->rx_dropped += rx.xdp_tx_err;
480 tot->tx_bytes += rx.xdp_bytes;
481 tot->tx_packets += rx.xdp_packets;
482 }
483 rcu_read_unlock();
484}
485
486/* fake multicast ability */
487static void veth_set_multicast_list(struct net_device *dev)
488{
489}
490
491static int veth_select_rxq(struct net_device *dev)
492{
493 return smp_processor_id() % dev->real_num_rx_queues;
494}
495
496static struct net_device *veth_peer_dev(struct net_device *dev)
497{
498 struct veth_priv *priv = netdev_priv(dev);
499
500 /* Callers must be under RCU read side. */
501 return rcu_dereference(priv->peer);
502}
503
504static int veth_xdp_xmit(struct net_device *dev, int n,
505 struct xdp_frame **frames,
506 u32 flags, bool ndo_xmit)
507{
508 struct veth_priv *rcv_priv, *priv = netdev_priv(dev);
509 int i, ret = -ENXIO, nxmit = 0;
510 struct net_device *rcv;
511 unsigned int max_len;
512 struct veth_rq *rq;
513
514 if (unlikely(flags & ~XDP_XMIT_FLAGS_MASK))
515 return -EINVAL;
516
517 rcu_read_lock();
518 rcv = rcu_dereference(priv->peer);
519 if (unlikely(!rcv))
520 goto out;
521
522 rcv_priv = netdev_priv(rcv);
523 rq = &rcv_priv->rq[veth_select_rxq(rcv)];
524 /* The napi pointer is set if NAPI is enabled, which ensures that
525 * xdp_ring is initialized on receive side and the peer device is up.
526 */
527 if (!rcu_access_pointer(rq->napi))
528 goto out;
529
530 max_len = rcv->mtu + rcv->hard_header_len + VLAN_HLEN;
531
532 spin_lock(&rq->xdp_ring.producer_lock);
533 for (i = 0; i < n; i++) {
534 struct xdp_frame *frame = frames[i];
535 void *ptr = veth_xdp_to_ptr(frame);
536
537 if (unlikely(xdp_get_frame_len(frame) > max_len ||
538 __ptr_ring_produce(&rq->xdp_ring, ptr)))
539 break;
540 nxmit++;
541 }
542 spin_unlock(&rq->xdp_ring.producer_lock);
543
544 if (flags & XDP_XMIT_FLUSH)
545 __veth_xdp_flush(rq);
546
547 ret = nxmit;
548 if (ndo_xmit) {
549 u64_stats_update_begin(&rq->stats.syncp);
550 rq->stats.vs.peer_tq_xdp_xmit += nxmit;
551 rq->stats.vs.peer_tq_xdp_xmit_err += n - nxmit;
552 u64_stats_update_end(&rq->stats.syncp);
553 }
554
555out:
556 rcu_read_unlock();
557
558 return ret;
559}
560
561static int veth_ndo_xdp_xmit(struct net_device *dev, int n,
562 struct xdp_frame **frames, u32 flags)
563{
564 int err;
565
566 err = veth_xdp_xmit(dev, n, frames, flags, true);
567 if (err < 0) {
568 struct veth_priv *priv = netdev_priv(dev);
569
570 atomic64_add(n, &priv->dropped);
571 }
572
573 return err;
574}
575
576static void veth_xdp_flush_bq(struct veth_rq *rq, struct veth_xdp_tx_bq *bq)
577{
578 int sent, i, err = 0, drops;
579
580 sent = veth_xdp_xmit(rq->dev, bq->count, bq->q, 0, false);
581 if (sent < 0) {
582 err = sent;
583 sent = 0;
584 }
585
586 for (i = sent; unlikely(i < bq->count); i++)
587 xdp_return_frame(bq->q[i]);
588
589 drops = bq->count - sent;
590 trace_xdp_bulk_tx(rq->dev, sent, drops, err);
591
592 u64_stats_update_begin(&rq->stats.syncp);
593 rq->stats.vs.xdp_tx += sent;
594 rq->stats.vs.xdp_tx_err += drops;
595 u64_stats_update_end(&rq->stats.syncp);
596
597 bq->count = 0;
598}
599
600static void veth_xdp_flush(struct veth_rq *rq, struct veth_xdp_tx_bq *bq)
601{
602 struct veth_priv *rcv_priv, *priv = netdev_priv(rq->dev);
603 struct net_device *rcv;
604 struct veth_rq *rcv_rq;
605
606 rcu_read_lock();
607 veth_xdp_flush_bq(rq, bq);
608 rcv = rcu_dereference(priv->peer);
609 if (unlikely(!rcv))
610 goto out;
611
612 rcv_priv = netdev_priv(rcv);
613 rcv_rq = &rcv_priv->rq[veth_select_rxq(rcv)];
614 /* xdp_ring is initialized on receive side? */
615 if (unlikely(!rcu_access_pointer(rcv_rq->xdp_prog)))
616 goto out;
617
618 __veth_xdp_flush(rcv_rq);
619out:
620 rcu_read_unlock();
621}
622
623static int veth_xdp_tx(struct veth_rq *rq, struct xdp_buff *xdp,
624 struct veth_xdp_tx_bq *bq)
625{
626 struct xdp_frame *frame = xdp_convert_buff_to_frame(xdp);
627
628 if (unlikely(!frame))
629 return -EOVERFLOW;
630
631 if (unlikely(bq->count == VETH_XDP_TX_BULK_SIZE))
632 veth_xdp_flush_bq(rq, bq);
633
634 bq->q[bq->count++] = frame;
635
636 return 0;
637}
638
639static struct xdp_frame *veth_xdp_rcv_one(struct veth_rq *rq,
640 struct xdp_frame *frame,
641 struct veth_xdp_tx_bq *bq,
642 struct veth_stats *stats)
643{
644 struct xdp_frame orig_frame;
645 struct bpf_prog *xdp_prog;
646
647 rcu_read_lock();
648 xdp_prog = rcu_dereference(rq->xdp_prog);
649 if (likely(xdp_prog)) {
650 struct veth_xdp_buff vxbuf;
651 struct xdp_buff *xdp = &vxbuf.xdp;
652 u32 act;
653
654 xdp_convert_frame_to_buff(frame, xdp);
655 xdp->rxq = &rq->xdp_rxq;
656 vxbuf.skb = NULL;
657
658 act = bpf_prog_run_xdp(xdp_prog, xdp);
659
660 switch (act) {
661 case XDP_PASS:
662 if (xdp_update_frame_from_buff(xdp, frame))
663 goto err_xdp;
664 break;
665 case XDP_TX:
666 orig_frame = *frame;
667 xdp->rxq->mem.type = frame->mem_type;
668 if (unlikely(veth_xdp_tx(rq, xdp, bq) < 0)) {
669 trace_xdp_exception(rq->dev, xdp_prog, act);
670 frame = &orig_frame;
671 stats->rx_drops++;
672 goto err_xdp;
673 }
674 stats->xdp_tx++;
675 rcu_read_unlock();
676 goto xdp_xmit;
677 case XDP_REDIRECT:
678 orig_frame = *frame;
679 xdp->rxq->mem.type = frame->mem_type;
680 if (xdp_do_redirect(rq->dev, xdp, xdp_prog)) {
681 frame = &orig_frame;
682 stats->rx_drops++;
683 goto err_xdp;
684 }
685 stats->xdp_redirect++;
686 rcu_read_unlock();
687 goto xdp_xmit;
688 default:
689 bpf_warn_invalid_xdp_action(rq->dev, xdp_prog, act);
690 fallthrough;
691 case XDP_ABORTED:
692 trace_xdp_exception(rq->dev, xdp_prog, act);
693 fallthrough;
694 case XDP_DROP:
695 stats->xdp_drops++;
696 goto err_xdp;
697 }
698 }
699 rcu_read_unlock();
700
701 return frame;
702err_xdp:
703 rcu_read_unlock();
704 xdp_return_frame(frame);
705xdp_xmit:
706 return NULL;
707}
708
709/* frames array contains VETH_XDP_BATCH at most */
710static void veth_xdp_rcv_bulk_skb(struct veth_rq *rq, void **frames,
711 int n_xdpf, struct veth_xdp_tx_bq *bq,
712 struct veth_stats *stats)
713{
714 void *skbs[VETH_XDP_BATCH];
715 int i;
716
717 if (unlikely(!napi_skb_cache_get_bulk(skbs, n_xdpf))) {
718 for (i = 0; i < n_xdpf; i++)
719 xdp_return_frame(frames[i]);
720 stats->rx_drops += n_xdpf;
721
722 return;
723 }
724
725 for (i = 0; i < n_xdpf; i++) {
726 struct sk_buff *skb = skbs[i];
727
728 skb = __xdp_build_skb_from_frame(frames[i], skb,
729 rq->dev);
730 if (!skb) {
731 xdp_return_frame(frames[i]);
732 stats->rx_drops++;
733 continue;
734 }
735 napi_gro_receive(&rq->xdp_napi, skb);
736 }
737}
738
739static void veth_xdp_get(struct xdp_buff *xdp)
740{
741 struct skb_shared_info *sinfo = xdp_get_shared_info_from_buff(xdp);
742 int i;
743
744 get_page(virt_to_page(xdp->data));
745 if (likely(!xdp_buff_has_frags(xdp)))
746 return;
747
748 for (i = 0; i < sinfo->nr_frags; i++)
749 __skb_frag_ref(&sinfo->frags[i]);
750}
751
752static int veth_convert_skb_to_xdp_buff(struct veth_rq *rq,
753 struct xdp_buff *xdp,
754 struct sk_buff **pskb)
755{
756 struct sk_buff *skb = *pskb;
757 u32 frame_sz;
758
759 if (skb_shared(skb) || skb_head_is_locked(skb) ||
760 skb_is_nonlinear(skb) ||
761 skb_headroom(skb) < XDP_PACKET_HEADROOM) {
762 if (skb_pp_cow_data(rq->page_pool, pskb, XDP_PACKET_HEADROOM))
763 goto drop;
764
765 skb = *pskb;
766 }
767
768 /* SKB "head" area always have tailroom for skb_shared_info */
769 frame_sz = skb_end_pointer(skb) - skb->head;
770 frame_sz += SKB_DATA_ALIGN(sizeof(struct skb_shared_info));
771 xdp_init_buff(xdp, frame_sz, &rq->xdp_rxq);
772 xdp_prepare_buff(xdp, skb->head, skb_headroom(skb),
773 skb_headlen(skb), true);
774
775 if (skb_shinfo(skb)->nr_frags) {
776 skb_shinfo(skb)->xdp_frags_size = skb->data_len;
777 xdp_buff_set_frags_flag(xdp);
778 } else {
779 xdp_buff_clear_frags_flag(xdp);
780 }
781 *pskb = skb;
782
783 return 0;
784drop:
785 consume_skb(skb);
786 *pskb = NULL;
787
788 return -ENOMEM;
789}
790
791static struct sk_buff *veth_xdp_rcv_skb(struct veth_rq *rq,
792 struct sk_buff *skb,
793 struct veth_xdp_tx_bq *bq,
794 struct veth_stats *stats)
795{
796 void *orig_data, *orig_data_end;
797 struct bpf_prog *xdp_prog;
798 struct veth_xdp_buff vxbuf;
799 struct xdp_buff *xdp = &vxbuf.xdp;
800 u32 act, metalen;
801 int off;
802
803 skb_prepare_for_gro(skb);
804
805 rcu_read_lock();
806 xdp_prog = rcu_dereference(rq->xdp_prog);
807 if (unlikely(!xdp_prog)) {
808 rcu_read_unlock();
809 goto out;
810 }
811
812 __skb_push(skb, skb->data - skb_mac_header(skb));
813 if (veth_convert_skb_to_xdp_buff(rq, xdp, &skb))
814 goto drop;
815 vxbuf.skb = skb;
816
817 orig_data = xdp->data;
818 orig_data_end = xdp->data_end;
819
820 act = bpf_prog_run_xdp(xdp_prog, xdp);
821
822 switch (act) {
823 case XDP_PASS:
824 break;
825 case XDP_TX:
826 veth_xdp_get(xdp);
827 consume_skb(skb);
828 xdp->rxq->mem = rq->xdp_mem;
829 if (unlikely(veth_xdp_tx(rq, xdp, bq) < 0)) {
830 trace_xdp_exception(rq->dev, xdp_prog, act);
831 stats->rx_drops++;
832 goto err_xdp;
833 }
834 stats->xdp_tx++;
835 rcu_read_unlock();
836 goto xdp_xmit;
837 case XDP_REDIRECT:
838 veth_xdp_get(xdp);
839 consume_skb(skb);
840 xdp->rxq->mem = rq->xdp_mem;
841 if (xdp_do_redirect(rq->dev, xdp, xdp_prog)) {
842 stats->rx_drops++;
843 goto err_xdp;
844 }
845 stats->xdp_redirect++;
846 rcu_read_unlock();
847 goto xdp_xmit;
848 default:
849 bpf_warn_invalid_xdp_action(rq->dev, xdp_prog, act);
850 fallthrough;
851 case XDP_ABORTED:
852 trace_xdp_exception(rq->dev, xdp_prog, act);
853 fallthrough;
854 case XDP_DROP:
855 stats->xdp_drops++;
856 goto xdp_drop;
857 }
858 rcu_read_unlock();
859
860 /* check if bpf_xdp_adjust_head was used */
861 off = orig_data - xdp->data;
862 if (off > 0)
863 __skb_push(skb, off);
864 else if (off < 0)
865 __skb_pull(skb, -off);
866
867 skb_reset_mac_header(skb);
868
869 /* XDP frag metadata (e.g. nr_frags) are updated in eBPF helpers
870 * (e.g. bpf_xdp_adjust_tail). Remove the old fragment contribution
871 * from skb->len before updating data_len, then add the new one back.
872 */
873 skb->len -= skb->data_len;
874 if (xdp_buff_has_frags(xdp)) {
875 skb->data_len = skb_shinfo(skb)->xdp_frags_size;
876 skb->len += skb->data_len;
877 } else {
878 skb->data_len = 0;
879 }
880
881 /* Synchronize the skb tail with XDP's updated linear area. */
882 off = xdp->data_end - orig_data_end;
883 if (off != 0) {
884 skb_set_tail_pointer(skb, xdp->data_end - xdp->data);
885 skb->len += off; /* positive on grow, negative on shrink */
886 }
887
888 skb->protocol = eth_type_trans(skb, rq->dev);
889
890 metalen = xdp->data - xdp->data_meta;
891 if (metalen)
892 skb_metadata_set(skb, metalen);
893out:
894 return skb;
895drop:
896 stats->rx_drops++;
897xdp_drop:
898 rcu_read_unlock();
899 kfree_skb(skb);
900 return NULL;
901err_xdp:
902 rcu_read_unlock();
903 xdp_return_buff(xdp);
904xdp_xmit:
905 return NULL;
906}
907
908static int veth_xdp_rcv(struct veth_rq *rq, int budget,
909 struct veth_xdp_tx_bq *bq,
910 struct veth_stats *stats)
911{
912 int i, done = 0, n_xdpf = 0;
913 void *xdpf[VETH_XDP_BATCH];
914
915 for (i = 0; i < budget; i++) {
916 void *ptr = __ptr_ring_consume(&rq->xdp_ring);
917
918 if (!ptr)
919 break;
920
921 if (veth_is_xdp_frame(ptr)) {
922 /* ndo_xdp_xmit */
923 struct xdp_frame *frame = veth_ptr_to_xdp(ptr);
924
925 stats->xdp_bytes += xdp_get_frame_len(frame);
926 frame = veth_xdp_rcv_one(rq, frame, bq, stats);
927 if (frame) {
928 /* XDP_PASS */
929 xdpf[n_xdpf++] = frame;
930 if (n_xdpf == VETH_XDP_BATCH) {
931 veth_xdp_rcv_bulk_skb(rq, xdpf, n_xdpf,
932 bq, stats);
933 n_xdpf = 0;
934 }
935 }
936 } else {
937 /* ndo_start_xmit */
938 struct sk_buff *skb = ptr;
939
940 stats->xdp_bytes += skb->len;
941 skb = veth_xdp_rcv_skb(rq, skb, bq, stats);
942 if (skb) {
943 if (skb_shared(skb) || skb_unclone(skb, GFP_ATOMIC))
944 netif_receive_skb(skb);
945 else
946 napi_gro_receive(&rq->xdp_napi, skb);
947 }
948 }
949 done++;
950 }
951
952 if (n_xdpf)
953 veth_xdp_rcv_bulk_skb(rq, xdpf, n_xdpf, bq, stats);
954
955 u64_stats_update_begin(&rq->stats.syncp);
956 rq->stats.vs.xdp_redirect += stats->xdp_redirect;
957 rq->stats.vs.xdp_bytes += stats->xdp_bytes;
958 rq->stats.vs.xdp_drops += stats->xdp_drops;
959 rq->stats.vs.rx_drops += stats->rx_drops;
960 rq->stats.vs.xdp_packets += done;
961 u64_stats_update_end(&rq->stats.syncp);
962
963 return done;
964}
965
966static int veth_poll(struct napi_struct *napi, int budget)
967{
968 struct veth_rq *rq =
969 container_of(napi, struct veth_rq, xdp_napi);
970 struct veth_priv *priv = netdev_priv(rq->dev);
971 int queue_idx = rq - priv->rq;
972 struct netdev_queue *peer_txq;
973 struct veth_stats stats = {};
974 struct net_device *peer_dev;
975 struct veth_xdp_tx_bq bq;
976 int done;
977
978 bq.count = 0;
979
980 /* NAPI functions as RCU section */
981 peer_dev = rcu_dereference_check(priv->peer, rcu_read_lock_bh_held());
982 peer_txq = (peer_dev && queue_idx < peer_dev->real_num_tx_queues) ?
983 netdev_get_tx_queue(peer_dev, queue_idx) : NULL;
984
985 xdp_set_return_frame_no_direct();
986 done = veth_xdp_rcv(rq, budget, &bq, &stats);
987
988 if (stats.xdp_redirect > 0)
989 xdp_do_flush();
990 if (stats.xdp_tx > 0)
991 veth_xdp_flush(rq, &bq);
992 xdp_clear_return_frame_no_direct();
993
994 if (done < budget && napi_complete_done(napi, done)) {
995 /* Write rx_notify_masked before reading ptr_ring */
996 smp_store_mb(rq->rx_notify_masked, false);
997 if (unlikely(!__ptr_ring_empty(&rq->xdp_ring))) {
998 if (napi_schedule_prep(&rq->xdp_napi)) {
999 WRITE_ONCE(rq->rx_notify_masked, true);
1000 __napi_schedule(&rq->xdp_napi);
1001 }
1002 }
1003 }
1004
1005 /* Release backpressure per NAPI poll */
1006 smp_rmb(); /* Paired with netif_tx_stop_queue set_bit */
1007 if (peer_txq && netif_tx_queue_stopped(peer_txq)) {
1008 txq_trans_cond_update(peer_txq);
1009 netif_tx_wake_queue(peer_txq);
1010 }
1011
1012 return done;
1013}
1014
1015static int veth_create_page_pool(struct veth_rq *rq)
1016{
1017 struct page_pool_params pp_params = {
1018 .order = 0,
1019 .pool_size = VETH_RING_SIZE,
1020 .nid = NUMA_NO_NODE,
1021 .dev = &rq->dev->dev,
1022 };
1023
1024 rq->page_pool = page_pool_create(&pp_params);
1025 if (IS_ERR(rq->page_pool)) {
1026 int err = PTR_ERR(rq->page_pool);
1027
1028 rq->page_pool = NULL;
1029 return err;
1030 }
1031
1032 return 0;
1033}
1034
1035static int __veth_napi_enable_range(struct net_device *dev, int start, int end)
1036{
1037 struct veth_priv *priv = netdev_priv(dev);
1038 int err, i;
1039
1040 for (i = start; i < end; i++) {
1041 err = veth_create_page_pool(&priv->rq[i]);
1042 if (err)
1043 goto err_page_pool;
1044 }
1045
1046 for (i = start; i < end; i++) {
1047 struct veth_rq *rq = &priv->rq[i];
1048
1049 err = ptr_ring_init(&rq->xdp_ring, VETH_RING_SIZE, GFP_KERNEL);
1050 if (err)
1051 goto err_xdp_ring;
1052 }
1053
1054 for (i = start; i < end; i++) {
1055 struct veth_rq *rq = &priv->rq[i];
1056
1057 rcu_assign_pointer(rq->xdp_prog, priv->_xdp_prog);
1058 napi_enable(&rq->xdp_napi);
1059 rcu_assign_pointer(priv->rq[i].napi, &priv->rq[i].xdp_napi);
1060 }
1061
1062 return 0;
1063
1064err_xdp_ring:
1065 for (i--; i >= start; i--)
1066 ptr_ring_cleanup(&priv->rq[i].xdp_ring, veth_ptr_free);
1067 i = end;
1068err_page_pool:
1069 for (i--; i >= start; i--) {
1070 page_pool_destroy(priv->rq[i].page_pool);
1071 priv->rq[i].page_pool = NULL;
1072 }
1073
1074 return err;
1075}
1076
1077static int __veth_napi_enable(struct net_device *dev)
1078{
1079 return __veth_napi_enable_range(dev, 0, dev->real_num_rx_queues);
1080}
1081
1082static void veth_napi_del_range(struct net_device *dev, int start, int end)
1083{
1084 struct veth_priv *priv = netdev_priv(dev);
1085 int i;
1086
1087 for (i = start; i < end; i++) {
1088 struct veth_rq *rq = &priv->rq[i];
1089
1090 rcu_assign_pointer(priv->rq[i].napi, NULL);
1091 napi_disable(&rq->xdp_napi);
1092 rcu_assign_pointer(rq->xdp_prog, NULL);
1093 __netif_napi_del(&rq->xdp_napi);
1094 }
1095 synchronize_net();
1096
1097 for (i = start; i < end; i++) {
1098 struct veth_rq *rq = &priv->rq[i];
1099
1100 rq->rx_notify_masked = false;
1101 ptr_ring_cleanup(&rq->xdp_ring, veth_ptr_free);
1102 }
1103
1104 for (i = start; i < end; i++) {
1105 page_pool_destroy(priv->rq[i].page_pool);
1106 priv->rq[i].page_pool = NULL;
1107 }
1108}
1109
1110static void veth_napi_del(struct net_device *dev)
1111{
1112 veth_napi_del_range(dev, 0, dev->real_num_rx_queues);
1113}
1114
1115static bool veth_gro_requested(const struct net_device *dev)
1116{
1117 return !!(dev->wanted_features & NETIF_F_GRO);
1118}
1119
1120static int veth_enable_xdp_range(struct net_device *dev, int start, int end,
1121 bool napi_already_on)
1122{
1123 struct veth_priv *priv = netdev_priv(dev);
1124 int err, i;
1125
1126 for (i = start; i < end; i++) {
1127 struct veth_rq *rq = &priv->rq[i];
1128
1129 if (!napi_already_on)
1130 netif_napi_add(dev, &rq->xdp_napi, veth_poll);
1131 err = xdp_rxq_info_reg(&rq->xdp_rxq, dev, i, rq->xdp_napi.napi_id);
1132 if (err < 0)
1133 goto err_rxq_reg;
1134
1135 err = xdp_rxq_info_reg_mem_model(&rq->xdp_rxq,
1136 MEM_TYPE_PAGE_SHARED,
1137 NULL);
1138 if (err < 0)
1139 goto err_reg_mem;
1140
1141 /* Save original mem info as it can be overwritten */
1142 rq->xdp_mem = rq->xdp_rxq.mem;
1143 }
1144 return 0;
1145
1146err_reg_mem:
1147 xdp_rxq_info_unreg(&priv->rq[i].xdp_rxq);
1148err_rxq_reg:
1149 if (!napi_already_on)
1150 netif_napi_del(&priv->rq[i].xdp_napi);
1151 for (i--; i >= start; i--) {
1152 struct veth_rq *rq = &priv->rq[i];
1153
1154 xdp_rxq_info_unreg(&rq->xdp_rxq);
1155 if (!napi_already_on)
1156 netif_napi_del(&rq->xdp_napi);
1157 }
1158
1159 return err;
1160}
1161
1162static void veth_disable_xdp_range(struct net_device *dev, int start, int end,
1163 bool delete_napi)
1164{
1165 struct veth_priv *priv = netdev_priv(dev);
1166 int i;
1167
1168 for (i = start; i < end; i++) {
1169 struct veth_rq *rq = &priv->rq[i];
1170
1171 rq->xdp_rxq.mem = rq->xdp_mem;
1172 xdp_rxq_info_unreg(&rq->xdp_rxq);
1173
1174 if (delete_napi)
1175 netif_napi_del(&rq->xdp_napi);
1176 }
1177}
1178
1179static int veth_enable_xdp(struct net_device *dev)
1180{
1181 bool napi_already_on = veth_gro_requested(dev) && (dev->flags & IFF_UP);
1182 struct veth_priv *priv = netdev_priv(dev);
1183 int err, i;
1184
1185 if (!xdp_rxq_info_is_reg(&priv->rq[0].xdp_rxq)) {
1186 err = veth_enable_xdp_range(dev, 0, dev->real_num_rx_queues, napi_already_on);
1187 if (err)
1188 return err;
1189
1190 if (!napi_already_on) {
1191 err = __veth_napi_enable(dev);
1192 if (err) {
1193 veth_disable_xdp_range(dev, 0, dev->real_num_rx_queues, true);
1194 return err;
1195 }
1196 }
1197 }
1198
1199 for (i = 0; i < dev->real_num_rx_queues; i++) {
1200 rcu_assign_pointer(priv->rq[i].xdp_prog, priv->_xdp_prog);
1201 rcu_assign_pointer(priv->rq[i].napi, &priv->rq[i].xdp_napi);
1202 }
1203
1204 return 0;
1205}
1206
1207static void veth_disable_xdp(struct net_device *dev)
1208{
1209 struct veth_priv *priv = netdev_priv(dev);
1210 int i;
1211
1212 for (i = 0; i < dev->real_num_rx_queues; i++)
1213 rcu_assign_pointer(priv->rq[i].xdp_prog, NULL);
1214
1215 if (!netif_running(dev) || !veth_gro_requested(dev))
1216 veth_napi_del(dev);
1217
1218 veth_disable_xdp_range(dev, 0, dev->real_num_rx_queues, false);
1219}
1220
1221static int veth_napi_enable_range(struct net_device *dev, int start, int end)
1222{
1223 struct veth_priv *priv = netdev_priv(dev);
1224 int err, i;
1225
1226 for (i = start; i < end; i++) {
1227 struct veth_rq *rq = &priv->rq[i];
1228
1229 netif_napi_add(dev, &rq->xdp_napi, veth_poll);
1230 }
1231
1232 err = __veth_napi_enable_range(dev, start, end);
1233 if (err) {
1234 for (i = start; i < end; i++) {
1235 struct veth_rq *rq = &priv->rq[i];
1236
1237 netif_napi_del(&rq->xdp_napi);
1238 }
1239 return err;
1240 }
1241 return err;
1242}
1243
1244static int veth_napi_enable(struct net_device *dev)
1245{
1246 return veth_napi_enable_range(dev, 0, dev->real_num_rx_queues);
1247}
1248
1249static void veth_disable_range_safe(struct net_device *dev, int start, int end)
1250{
1251 struct veth_priv *priv = netdev_priv(dev);
1252
1253 if (start >= end)
1254 return;
1255
1256 if (priv->_xdp_prog) {
1257 veth_napi_del_range(dev, start, end);
1258 veth_disable_xdp_range(dev, start, end, false);
1259 } else if (veth_gro_requested(dev)) {
1260 veth_napi_del_range(dev, start, end);
1261 }
1262}
1263
1264static int veth_enable_range_safe(struct net_device *dev, int start, int end)
1265{
1266 struct veth_priv *priv = netdev_priv(dev);
1267 int err;
1268
1269 if (start >= end)
1270 return 0;
1271
1272 if (priv->_xdp_prog) {
1273 /* these channels are freshly initialized, napi is not on there even
1274 * when GRO is requeste
1275 */
1276 err = veth_enable_xdp_range(dev, start, end, false);
1277 if (err)
1278 return err;
1279
1280 err = __veth_napi_enable_range(dev, start, end);
1281 if (err) {
1282 /* on error always delete the newly added napis */
1283 veth_disable_xdp_range(dev, start, end, true);
1284 return err;
1285 }
1286 } else if (veth_gro_requested(dev)) {
1287 return veth_napi_enable_range(dev, start, end);
1288 }
1289 return 0;
1290}
1291
1292static void veth_set_xdp_features(struct net_device *dev)
1293{
1294 struct veth_priv *priv = netdev_priv(dev);
1295 struct net_device *peer;
1296
1297 peer = rtnl_dereference(priv->peer);
1298 if (peer && peer->real_num_tx_queues <= dev->real_num_rx_queues) {
1299 struct veth_priv *priv_peer = netdev_priv(peer);
1300 xdp_features_t val = NETDEV_XDP_ACT_BASIC |
1301 NETDEV_XDP_ACT_REDIRECT |
1302 NETDEV_XDP_ACT_RX_SG;
1303
1304 if (priv_peer->_xdp_prog || veth_gro_requested(peer))
1305 val |= NETDEV_XDP_ACT_NDO_XMIT |
1306 NETDEV_XDP_ACT_NDO_XMIT_SG;
1307 xdp_set_features_flag(dev, val);
1308 } else {
1309 xdp_clear_features_flag(dev);
1310 }
1311}
1312
1313static int veth_set_channels(struct net_device *dev,
1314 struct ethtool_channels *ch)
1315{
1316 struct veth_priv *priv = netdev_priv(dev);
1317 unsigned int old_rx_count, new_rx_count;
1318 struct veth_priv *peer_priv;
1319 struct net_device *peer;
1320 int err;
1321
1322 /* sanity check. Upper bounds are already enforced by the caller */
1323 if (!ch->rx_count || !ch->tx_count)
1324 return -EINVAL;
1325
1326 /* avoid braking XDP, if that is enabled */
1327 peer = rtnl_dereference(priv->peer);
1328 peer_priv = peer ? netdev_priv(peer) : NULL;
1329 if (priv->_xdp_prog && peer && ch->rx_count < peer->real_num_tx_queues)
1330 return -EINVAL;
1331
1332 if (peer && peer_priv && peer_priv->_xdp_prog && ch->tx_count > peer->real_num_rx_queues)
1333 return -EINVAL;
1334
1335 old_rx_count = dev->real_num_rx_queues;
1336 new_rx_count = ch->rx_count;
1337 if (netif_running(dev)) {
1338 /* turn device off */
1339 netif_carrier_off(dev);
1340 if (peer)
1341 netif_carrier_off(peer);
1342
1343 /* try to allocate new resources, as needed*/
1344 err = veth_enable_range_safe(dev, old_rx_count, new_rx_count);
1345 if (err)
1346 goto out;
1347 }
1348
1349 err = netif_set_real_num_rx_queues(dev, ch->rx_count);
1350 if (err)
1351 goto revert;
1352
1353 err = netif_set_real_num_tx_queues(dev, ch->tx_count);
1354 if (err) {
1355 int err2 = netif_set_real_num_rx_queues(dev, old_rx_count);
1356
1357 /* this error condition could happen only if rx and tx change
1358 * in opposite directions (e.g. tx nr raises, rx nr decreases)
1359 * and we can't do anything to fully restore the original
1360 * status
1361 */
1362 if (err2)
1363 pr_warn("Can't restore rx queues config %d -> %d %d",
1364 new_rx_count, old_rx_count, err2);
1365 else
1366 goto revert;
1367 }
1368
1369out:
1370 if (netif_running(dev)) {
1371 /* note that we need to swap the arguments WRT the enable part
1372 * to identify the range we have to disable
1373 */
1374 veth_disable_range_safe(dev, new_rx_count, old_rx_count);
1375 netif_carrier_on(dev);
1376 if (peer)
1377 netif_carrier_on(peer);
1378 }
1379
1380 /* update XDP supported features */
1381 veth_set_xdp_features(dev);
1382 if (peer)
1383 veth_set_xdp_features(peer);
1384
1385 return err;
1386
1387revert:
1388 new_rx_count = old_rx_count;
1389 old_rx_count = ch->rx_count;
1390 goto out;
1391}
1392
1393static int veth_open(struct net_device *dev)
1394{
1395 struct veth_priv *priv = netdev_priv(dev);
1396 struct net_device *peer = rtnl_dereference(priv->peer);
1397 int err;
1398
1399 if (!peer)
1400 return -ENOTCONN;
1401
1402 if (priv->_xdp_prog) {
1403 err = veth_enable_xdp(dev);
1404 if (err)
1405 return err;
1406 } else if (veth_gro_requested(dev)) {
1407 err = veth_napi_enable(dev);
1408 if (err)
1409 return err;
1410 }
1411
1412 if (peer->flags & IFF_UP) {
1413 netif_carrier_on(dev);
1414 netif_carrier_on(peer);
1415 }
1416
1417 veth_set_xdp_features(dev);
1418 veth_set_xdp_features(peer);
1419
1420 return 0;
1421}
1422
1423static int veth_close(struct net_device *dev)
1424{
1425 struct veth_priv *priv = netdev_priv(dev);
1426 struct net_device *peer = rtnl_dereference(priv->peer);
1427
1428 netif_carrier_off(dev);
1429 if (peer)
1430 netif_carrier_off(peer);
1431
1432 if (priv->_xdp_prog)
1433 veth_disable_xdp(dev);
1434 else if (veth_gro_requested(dev))
1435 veth_napi_del(dev);
1436
1437 return 0;
1438}
1439
1440static int is_valid_veth_mtu(int mtu)
1441{
1442 return mtu >= ETH_MIN_MTU && mtu <= ETH_MAX_MTU;
1443}
1444
1445static int veth_alloc_queues(struct net_device *dev)
1446{
1447 struct veth_priv *priv = netdev_priv(dev);
1448 int i;
1449
1450 priv->rq = kvzalloc_objs(*priv->rq, dev->num_rx_queues,
1451 GFP_KERNEL_ACCOUNT | __GFP_RETRY_MAYFAIL);
1452 if (!priv->rq)
1453 return -ENOMEM;
1454
1455 for (i = 0; i < dev->num_rx_queues; i++) {
1456 priv->rq[i].dev = dev;
1457 u64_stats_init(&priv->rq[i].stats.syncp);
1458 }
1459
1460 return 0;
1461}
1462
1463static void veth_free_queues(struct net_device *dev)
1464{
1465 struct veth_priv *priv = netdev_priv(dev);
1466
1467 kvfree(priv->rq);
1468}
1469
1470static int veth_dev_init(struct net_device *dev)
1471{
1472 netdev_lockdep_set_classes(dev);
1473 return veth_alloc_queues(dev);
1474}
1475
1476static void veth_dev_free(struct net_device *dev)
1477{
1478 veth_free_queues(dev);
1479}
1480
1481#ifdef CONFIG_NET_POLL_CONTROLLER
1482static void veth_poll_controller(struct net_device *dev)
1483{
1484 /* veth only receives frames when its peer sends one
1485 * Since it has nothing to do with disabling irqs, we are guaranteed
1486 * never to have pending data when we poll for it so
1487 * there is nothing to do here.
1488 *
1489 * We need this though so netpoll recognizes us as an interface that
1490 * supports polling, which enables bridge devices in virt setups to
1491 * still use netconsole
1492 */
1493}
1494#endif /* CONFIG_NET_POLL_CONTROLLER */
1495
1496static int veth_get_iflink(const struct net_device *dev)
1497{
1498 struct veth_priv *priv = netdev_priv(dev);
1499 struct net_device *peer;
1500 int iflink;
1501
1502 rcu_read_lock();
1503 peer = rcu_dereference(priv->peer);
1504 iflink = peer ? READ_ONCE(peer->ifindex) : 0;
1505 rcu_read_unlock();
1506
1507 return iflink;
1508}
1509
1510static netdev_features_t veth_fix_features(struct net_device *dev,
1511 netdev_features_t features)
1512{
1513 struct veth_priv *priv = netdev_priv(dev);
1514 struct net_device *peer;
1515
1516 peer = rtnl_dereference(priv->peer);
1517 if (peer) {
1518 struct veth_priv *peer_priv = netdev_priv(peer);
1519
1520 if (peer_priv->_xdp_prog)
1521 features &= ~NETIF_F_GSO_SOFTWARE;
1522 }
1523
1524 return features;
1525}
1526
1527static int veth_set_features(struct net_device *dev,
1528 netdev_features_t features)
1529{
1530 netdev_features_t changed = features ^ dev->features;
1531 struct veth_priv *priv = netdev_priv(dev);
1532 struct net_device *peer;
1533 int err;
1534
1535 if (!(changed & NETIF_F_GRO) || !(dev->flags & IFF_UP) || priv->_xdp_prog)
1536 return 0;
1537
1538 peer = rtnl_dereference(priv->peer);
1539 if (features & NETIF_F_GRO) {
1540 err = veth_napi_enable(dev);
1541 if (err)
1542 return err;
1543
1544 if (peer)
1545 xdp_features_set_redirect_target(peer, true);
1546 } else {
1547 if (peer)
1548 xdp_features_clear_redirect_target(peer);
1549 veth_napi_del(dev);
1550 }
1551 return 0;
1552}
1553
1554static void veth_set_rx_headroom(struct net_device *dev, int new_hr)
1555{
1556 struct veth_priv *peer_priv, *priv = netdev_priv(dev);
1557 struct net_device *peer;
1558
1559 if (new_hr < 0)
1560 new_hr = 0;
1561
1562 rcu_read_lock();
1563 peer = rcu_dereference(priv->peer);
1564 if (unlikely(!peer))
1565 goto out;
1566
1567 peer_priv = netdev_priv(peer);
1568 priv->requested_headroom = new_hr;
1569 new_hr = max(priv->requested_headroom, peer_priv->requested_headroom);
1570 dev->needed_headroom = new_hr;
1571 peer->needed_headroom = new_hr;
1572
1573out:
1574 rcu_read_unlock();
1575}
1576
1577static int veth_xdp_set(struct net_device *dev, struct bpf_prog *prog,
1578 struct netlink_ext_ack *extack)
1579{
1580 struct veth_priv *priv = netdev_priv(dev);
1581 struct bpf_prog *old_prog;
1582 struct net_device *peer;
1583 unsigned int max_mtu;
1584 int err;
1585
1586 old_prog = priv->_xdp_prog;
1587 priv->_xdp_prog = prog;
1588 peer = rtnl_dereference(priv->peer);
1589
1590 if (prog) {
1591 if (!peer) {
1592 NL_SET_ERR_MSG_MOD(extack, "Cannot set XDP when peer is detached");
1593 err = -ENOTCONN;
1594 goto err;
1595 }
1596
1597 max_mtu = SKB_WITH_OVERHEAD(PAGE_SIZE - VETH_XDP_HEADROOM) -
1598 peer->hard_header_len;
1599 /* Allow increasing the max_mtu if the program supports
1600 * XDP fragments.
1601 */
1602 if (prog->aux->xdp_has_frags)
1603 max_mtu += PAGE_SIZE * MAX_SKB_FRAGS;
1604
1605 if (peer->mtu > max_mtu) {
1606 NL_SET_ERR_MSG_MOD(extack, "Peer MTU is too large to set XDP");
1607 err = -ERANGE;
1608 goto err;
1609 }
1610
1611 if (dev->real_num_rx_queues < peer->real_num_tx_queues) {
1612 NL_SET_ERR_MSG_MOD(extack, "XDP expects number of rx queues not less than peer tx queues");
1613 err = -ENOSPC;
1614 goto err;
1615 }
1616
1617 if (dev->flags & IFF_UP) {
1618 err = veth_enable_xdp(dev);
1619 if (err) {
1620 NL_SET_ERR_MSG_MOD(extack, "Setup for XDP failed");
1621 goto err;
1622 }
1623 }
1624
1625 if (!old_prog) {
1626 peer->hw_features &= ~NETIF_F_GSO_SOFTWARE;
1627 peer->max_mtu = max_mtu;
1628 }
1629
1630 xdp_features_set_redirect_target(peer, true);
1631 }
1632
1633 if (old_prog) {
1634 if (!prog) {
1635 if (peer && !veth_gro_requested(dev))
1636 xdp_features_clear_redirect_target(peer);
1637
1638 if (dev->flags & IFF_UP)
1639 veth_disable_xdp(dev);
1640
1641 if (peer) {
1642 peer->hw_features |= NETIF_F_GSO_SOFTWARE;
1643 peer->max_mtu = ETH_MAX_MTU;
1644 }
1645 }
1646 bpf_prog_put(old_prog);
1647 }
1648
1649 if ((!!old_prog ^ !!prog) && peer)
1650 netdev_update_features(peer);
1651
1652 return 0;
1653err:
1654 priv->_xdp_prog = old_prog;
1655
1656 return err;
1657}
1658
1659static int veth_xdp(struct net_device *dev, struct netdev_bpf *xdp)
1660{
1661 switch (xdp->command) {
1662 case XDP_SETUP_PROG:
1663 return veth_xdp_set(dev, xdp->prog, xdp->extack);
1664 default:
1665 return -EINVAL;
1666 }
1667}
1668
1669static int veth_xdp_rx_timestamp(const struct xdp_md *ctx, u64 *timestamp)
1670{
1671 struct veth_xdp_buff *_ctx = (void *)ctx;
1672
1673 if (!_ctx->skb)
1674 return -ENODATA;
1675
1676 *timestamp = skb_hwtstamps(_ctx->skb)->hwtstamp;
1677 return 0;
1678}
1679
1680static int veth_xdp_rx_hash(const struct xdp_md *ctx, u32 *hash,
1681 enum xdp_rss_hash_type *rss_type)
1682{
1683 struct veth_xdp_buff *_ctx = (void *)ctx;
1684 struct sk_buff *skb = _ctx->skb;
1685
1686 if (!skb)
1687 return -ENODATA;
1688
1689 *hash = skb_get_hash(skb);
1690 *rss_type = skb->l4_hash ? XDP_RSS_TYPE_L4_ANY : XDP_RSS_TYPE_NONE;
1691
1692 return 0;
1693}
1694
1695static int veth_xdp_rx_vlan_tag(const struct xdp_md *ctx, __be16 *vlan_proto,
1696 u16 *vlan_tci)
1697{
1698 const struct veth_xdp_buff *_ctx = (void *)ctx;
1699 const struct sk_buff *skb = _ctx->skb;
1700 int err;
1701
1702 if (!skb)
1703 return -ENODATA;
1704
1705 err = __vlan_hwaccel_get_tag(skb, vlan_tci);
1706 if (err)
1707 return err;
1708
1709 *vlan_proto = skb->vlan_proto;
1710 return err;
1711}
1712
1713static const struct net_device_ops veth_netdev_ops = {
1714 .ndo_init = veth_dev_init,
1715 .ndo_open = veth_open,
1716 .ndo_stop = veth_close,
1717 .ndo_start_xmit = veth_xmit,
1718 .ndo_get_stats64 = veth_get_stats64,
1719 .ndo_set_rx_mode = veth_set_multicast_list,
1720 .ndo_set_mac_address = eth_mac_addr,
1721#ifdef CONFIG_NET_POLL_CONTROLLER
1722 .ndo_poll_controller = veth_poll_controller,
1723#endif
1724 .ndo_get_iflink = veth_get_iflink,
1725 .ndo_fix_features = veth_fix_features,
1726 .ndo_set_features = veth_set_features,
1727 .ndo_features_check = passthru_features_check,
1728 .ndo_set_rx_headroom = veth_set_rx_headroom,
1729 .ndo_bpf = veth_xdp,
1730 .ndo_xdp_xmit = veth_ndo_xdp_xmit,
1731 .ndo_get_peer_dev = veth_peer_dev,
1732};
1733
1734static const struct xdp_metadata_ops veth_xdp_metadata_ops = {
1735 .xmo_rx_timestamp = veth_xdp_rx_timestamp,
1736 .xmo_rx_hash = veth_xdp_rx_hash,
1737 .xmo_rx_vlan_tag = veth_xdp_rx_vlan_tag,
1738};
1739
1740#define VETH_FEATURES (NETIF_F_SG | NETIF_F_FRAGLIST | NETIF_F_HW_CSUM | \
1741 NETIF_F_RXCSUM | NETIF_F_SCTP_CRC | NETIF_F_HIGHDMA | \
1742 NETIF_F_GSO_SOFTWARE | NETIF_F_GSO_ENCAP_ALL | \
1743 NETIF_F_HW_VLAN_CTAG_TX | NETIF_F_HW_VLAN_CTAG_RX | \
1744 NETIF_F_HW_VLAN_STAG_TX | NETIF_F_HW_VLAN_STAG_RX )
1745
1746static void veth_setup(struct net_device *dev)
1747{
1748 ether_setup(dev);
1749
1750 dev->priv_flags &= ~IFF_TX_SKB_SHARING;
1751 dev->priv_flags |= IFF_LIVE_ADDR_CHANGE;
1752 dev->priv_flags |= IFF_NO_QUEUE;
1753 dev->priv_flags |= IFF_PHONY_HEADROOM;
1754 dev->priv_flags |= IFF_DISABLE_NETPOLL;
1755 dev->lltx = true;
1756
1757 dev->netdev_ops = &veth_netdev_ops;
1758 dev->xdp_metadata_ops = &veth_xdp_metadata_ops;
1759 dev->ethtool_ops = &veth_ethtool_ops;
1760 dev->features |= VETH_FEATURES;
1761 dev->vlan_features = dev->features &
1762 ~(NETIF_F_HW_VLAN_CTAG_TX |
1763 NETIF_F_HW_VLAN_STAG_TX |
1764 NETIF_F_HW_VLAN_CTAG_RX |
1765 NETIF_F_HW_VLAN_STAG_RX);
1766 dev->needs_free_netdev = true;
1767 dev->priv_destructor = veth_dev_free;
1768 dev->pcpu_stat_type = NETDEV_PCPU_STAT_TSTATS;
1769 dev->max_mtu = ETH_MAX_MTU;
1770
1771 dev->hw_features = VETH_FEATURES;
1772 dev->hw_enc_features = VETH_FEATURES;
1773 dev->mpls_features = NETIF_F_HW_CSUM | NETIF_F_GSO_SOFTWARE;
1774 netif_set_tso_max_size(dev, GSO_MAX_SIZE);
1775}
1776
1777/*
1778 * netlink interface
1779 */
1780
1781static int veth_validate(struct nlattr *tb[], struct nlattr *data[],
1782 struct netlink_ext_ack *extack)
1783{
1784 if (tb[IFLA_ADDRESS]) {
1785 if (nla_len(tb[IFLA_ADDRESS]) != ETH_ALEN)
1786 return -EINVAL;
1787 if (!is_valid_ether_addr(nla_data(tb[IFLA_ADDRESS])))
1788 return -EADDRNOTAVAIL;
1789 }
1790 if (tb[IFLA_MTU]) {
1791 if (!is_valid_veth_mtu(nla_get_u32(tb[IFLA_MTU])))
1792 return -EINVAL;
1793 }
1794 return 0;
1795}
1796
1797static struct rtnl_link_ops veth_link_ops;
1798
1799static void veth_disable_gro(struct net_device *dev)
1800{
1801 dev->features &= ~NETIF_F_GRO;
1802 dev->wanted_features &= ~NETIF_F_GRO;
1803 netdev_update_features(dev);
1804}
1805
1806static int veth_init_queues(struct net_device *dev, struct nlattr *tb[])
1807{
1808 int err;
1809
1810 if (!tb[IFLA_NUM_TX_QUEUES] && dev->num_tx_queues > 1) {
1811 err = netif_set_real_num_tx_queues(dev, 1);
1812 if (err)
1813 return err;
1814 }
1815 if (!tb[IFLA_NUM_RX_QUEUES] && dev->num_rx_queues > 1) {
1816 err = netif_set_real_num_rx_queues(dev, 1);
1817 if (err)
1818 return err;
1819 }
1820 return 0;
1821}
1822
1823static int veth_newlink(struct net_device *dev,
1824 struct rtnl_newlink_params *params,
1825 struct netlink_ext_ack *extack)
1826{
1827 struct net *peer_net = rtnl_newlink_peer_net(params);
1828 struct nlattr **data = params->data;
1829 struct nlattr **tb = params->tb;
1830 int err;
1831 struct net_device *peer;
1832 struct veth_priv *priv;
1833 char ifname[IFNAMSIZ];
1834 struct nlattr *peer_tb[IFLA_MAX + 1], **tbp;
1835 unsigned char name_assign_type;
1836 struct ifinfomsg *ifmp;
1837
1838 /*
1839 * create and register peer first
1840 */
1841 if (data && data[VETH_INFO_PEER]) {
1842 struct nlattr *nla_peer = data[VETH_INFO_PEER];
1843
1844 ifmp = nla_data(nla_peer);
1845 rtnl_nla_parse_ifinfomsg(peer_tb, nla_peer, extack);
1846 tbp = peer_tb;
1847 } else {
1848 ifmp = NULL;
1849 tbp = tb;
1850 }
1851
1852 if (ifmp && tbp[IFLA_IFNAME]) {
1853 nla_strscpy(ifname, tbp[IFLA_IFNAME], IFNAMSIZ);
1854 name_assign_type = NET_NAME_USER;
1855 } else {
1856 snprintf(ifname, IFNAMSIZ, DRV_NAME "%%d");
1857 name_assign_type = NET_NAME_ENUM;
1858 }
1859
1860 peer = rtnl_create_link(peer_net, ifname, name_assign_type,
1861 &veth_link_ops, tbp, extack);
1862 if (IS_ERR(peer))
1863 return PTR_ERR(peer);
1864
1865 if (!ifmp || !tbp[IFLA_ADDRESS])
1866 eth_hw_addr_random(peer);
1867
1868 if (ifmp && (dev->ifindex != 0))
1869 peer->ifindex = ifmp->ifi_index;
1870
1871 netif_inherit_tso_max(peer, dev);
1872
1873 err = register_netdevice(peer);
1874 if (err < 0)
1875 goto err_register_peer;
1876
1877 /* keep GRO disabled by default to be consistent with the established
1878 * veth behavior
1879 */
1880 veth_disable_gro(peer);
1881 netif_carrier_off(peer);
1882
1883 err = rtnl_configure_link(peer, ifmp, 0, NULL);
1884 if (err < 0)
1885 goto err_configure_peer;
1886
1887 /*
1888 * register dev last
1889 *
1890 * note, that since we've registered new device the dev's name
1891 * should be re-allocated
1892 */
1893
1894 if (tb[IFLA_ADDRESS] == NULL)
1895 eth_hw_addr_random(dev);
1896
1897 if (tb[IFLA_IFNAME])
1898 nla_strscpy(dev->name, tb[IFLA_IFNAME], IFNAMSIZ);
1899 else
1900 snprintf(dev->name, IFNAMSIZ, DRV_NAME "%%d");
1901
1902 err = register_netdevice(dev);
1903 if (err < 0)
1904 goto err_register_dev;
1905
1906 netif_carrier_off(dev);
1907
1908 /*
1909 * tie the deviced together
1910 */
1911
1912 priv = netdev_priv(dev);
1913 rcu_assign_pointer(priv->peer, peer);
1914 netdev_hold(peer, &priv->peer_tracker, GFP_KERNEL);
1915 err = veth_init_queues(dev, tb);
1916 if (err)
1917 goto err_queues;
1918
1919 priv = netdev_priv(peer);
1920 rcu_assign_pointer(priv->peer, dev);
1921 netdev_hold(dev, &priv->peer_tracker, GFP_KERNEL);
1922 err = veth_init_queues(peer, tb);
1923 if (err)
1924 goto err_peer_queues;
1925
1926 veth_disable_gro(dev);
1927 /* update XDP supported features */
1928 veth_set_xdp_features(dev);
1929 veth_set_xdp_features(peer);
1930
1931 return 0;
1932
1933err_peer_queues:
1934 netdev_put(dev, &priv->peer_tracker);
1935 priv = netdev_priv(dev);
1936err_queues:
1937 netdev_put(peer, &priv->peer_tracker);
1938 unregister_netdevice(dev);
1939err_register_dev:
1940 /* nothing to do */
1941err_configure_peer:
1942 unregister_netdevice(peer);
1943 return err;
1944
1945err_register_peer:
1946 free_netdev(peer);
1947 return err;
1948}
1949
1950static void veth_dellink(struct net_device *dev, struct list_head *head)
1951{
1952 netdevice_tracker *peer_tracker;
1953 struct net_device *peer;
1954 struct veth_priv *priv;
1955
1956 priv = netdev_priv(dev);
1957 peer_tracker = &priv->peer_tracker;
1958 peer = unrcu_pointer(xchg(&priv->peer, NULL));
1959 if (!peer)
1960 return;
1961
1962 unregister_netdevice_queue(dev, head);
1963
1964 priv = netdev_priv(peer);
1965 dev = unrcu_pointer(xchg(&priv->peer, NULL));
1966 if (dev)
1967 unregister_netdevice_queue_net(dev_net(dev), peer, head);
1968
1969 netdev_put(peer, peer_tracker);
1970 netdev_put(dev, &priv->peer_tracker);
1971}
1972
1973static const struct nla_policy veth_policy[VETH_INFO_MAX + 1] = {
1974 [VETH_INFO_PEER] = { .len = sizeof(struct ifinfomsg) },
1975};
1976
1977static struct net *veth_get_link_net(const struct net_device *dev)
1978{
1979 struct veth_priv *priv = netdev_priv(dev);
1980 struct net_device *peer = rtnl_dereference(priv->peer);
1981
1982 return peer ? dev_net(peer) : dev_net(dev);
1983}
1984
1985static unsigned int veth_get_num_queues(void)
1986{
1987 /* enforce the same queue limit as rtnl_create_link */
1988 int queues = num_possible_cpus();
1989
1990 if (queues > 4096)
1991 queues = 4096;
1992 return queues;
1993}
1994
1995static struct rtnl_link_ops veth_link_ops = {
1996 .kind = DRV_NAME,
1997 .priv_size = sizeof(struct veth_priv),
1998 .setup = veth_setup,
1999 .validate = veth_validate,
2000 .newlink = veth_newlink,
2001 .dellink = veth_dellink,
2002 .policy = veth_policy,
2003 .peer_type = VETH_INFO_PEER,
2004 .maxtype = VETH_INFO_MAX,
2005 .get_link_net = veth_get_link_net,
2006 .get_num_tx_queues = veth_get_num_queues,
2007 .get_num_rx_queues = veth_get_num_queues,
2008};
2009
2010/*
2011 * init/fini
2012 */
2013
2014static __init int veth_init(void)
2015{
2016 return rtnl_link_register(&veth_link_ops);
2017}
2018
2019static __exit void veth_exit(void)
2020{
2021 rtnl_link_unregister(&veth_link_ops);
2022}
2023
2024module_init(veth_init);
2025module_exit(veth_exit);
2026
2027MODULE_DESCRIPTION("Virtual Ethernet Tunnel");
2028MODULE_LICENSE("GPL v2");
2029MODULE_ALIAS_RTNL_LINK(DRV_NAME);