projects/linux·blob uaccess-cleanups · refs

1278 lines · 29.4 KiB

1/*
2 * Copyright (C) 2017 Netronome Systems, Inc.
3 *
4 * This software is licensed under the GNU General License Version 2,
5 * June 1991 as shown in the file COPYING in the top-level directory of this
6 * source tree.
7 *
8 * THE COPYRIGHT HOLDERS AND/OR OTHER PARTIES PROVIDE THE PROGRAM "AS IS"
9 * WITHOUT WARRANTY OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING,
10 * BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
11 * FOR A PARTICULAR PURPOSE. THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE
12 * OF THE PROGRAM IS WITH YOU. SHOULD THE PROGRAM PROVE DEFECTIVE, YOU ASSUME
13 * THE COST OF ALL NECESSARY SERVICING, REPAIR OR CORRECTION.
14 */
15
16#include <linux/debugfs.h>
17#include <linux/etherdevice.h>
18#include <linux/ethtool_netlink.h>
19#include <linux/kernel.h>
20#include <linux/module.h>
21#include <linux/netdevice.h>
22#include <linux/slab.h>
23#include <net/netdev_queues.h>
24#include <net/netdev_rx_queue.h>
25#include <net/page_pool/helpers.h>
26#include <net/netlink.h>
27#include <net/net_shaper.h>
28#include <net/netdev_lock.h>
29#include <net/pkt_cls.h>
30#include <net/rtnetlink.h>
31#include <net/udp_tunnel.h>
32#include <net/busy_poll.h>
33
34#include "netdevsim.h"
35
36MODULE_IMPORT_NS("NETDEV_INTERNAL");
37
38#define NSIM_RING_SIZE 256
39
40static void nsim_start_peer_tx_queue(struct net_device *dev, struct nsim_rq *rq)
41{
42 struct netdevsim *ns = netdev_priv(dev);
43 struct net_device *peer_dev;
44 struct netdevsim *peer_ns;
45 struct netdev_queue *txq;
46 u16 idx;
47
48 idx = rq->napi.index;
49 rcu_read_lock();
50 peer_ns = rcu_dereference(ns->peer);
51 if (!peer_ns)
52 goto out;
53
54 /* TX device */
55 peer_dev = peer_ns->netdev;
56 if (dev->real_num_tx_queues != peer_dev->num_rx_queues)
57 goto out;
58
59 txq = netdev_get_tx_queue(peer_dev, idx);
60 if (!netif_tx_queue_stopped(txq))
61 goto out;
62
63 netif_tx_wake_queue(txq);
64out:
65 rcu_read_unlock();
66}
67
68static void nsim_stop_tx_queue(struct net_device *tx_dev,
69 struct net_device *rx_dev,
70 struct nsim_rq *rq,
71 u16 idx)
72{
73 /* If different queues size, do not stop, since it is not
74 * easy to find which TX queue is mapped here
75 */
76 if (rx_dev->real_num_tx_queues != tx_dev->num_rx_queues)
77 return;
78
79 /* rq is the queue on the receive side */
80 netif_subqueue_try_stop(tx_dev, idx,
81 NSIM_RING_SIZE - skb_queue_len(&rq->skb_queue),
82 NSIM_RING_SIZE / 2);
83}
84
85static int nsim_napi_rx(struct net_device *tx_dev, struct net_device *rx_dev,
86 struct nsim_rq *rq, struct sk_buff *skb)
87{
88 if (skb_queue_len(&rq->skb_queue) > NSIM_RING_SIZE) {
89 dev_kfree_skb_any(skb);
90 return NET_RX_DROP;
91 }
92
93 skb_queue_tail(&rq->skb_queue, skb);
94
95 /* Stop the peer TX queue avoiding dropping packets later */
96 if (skb_queue_len(&rq->skb_queue) >= NSIM_RING_SIZE)
97 nsim_stop_tx_queue(tx_dev, rx_dev, rq,
98 skb_get_queue_mapping(skb));
99
100 return NET_RX_SUCCESS;
101}
102
103static int nsim_forward_skb(struct net_device *tx_dev,
104 struct net_device *rx_dev,
105 struct sk_buff *skb,
106 struct nsim_rq *rq,
107 struct skb_ext *psp_ext)
108{
109 int ret;
110
111 ret = __dev_forward_skb(rx_dev, skb);
112 if (ret) {
113 if (psp_ext)
114 __skb_ext_put(psp_ext);
115 return ret;
116 }
117
118 nsim_psp_handle_ext(skb, psp_ext);
119
120 return nsim_napi_rx(tx_dev, rx_dev, rq, skb);
121}
122
123static netdev_tx_t nsim_start_xmit(struct sk_buff *skb, struct net_device *dev)
124{
125 struct netdevsim *ns = netdev_priv(dev);
126 struct skb_ext *psp_ext = NULL;
127 struct net_device *peer_dev;
128 unsigned int len = skb->len;
129 struct netdevsim *peer_ns;
130 struct netdev_config *cfg;
131 struct nsim_rq *rq;
132 int rxq;
133 int dr;
134
135 rcu_read_lock();
136 if (!nsim_ipsec_tx(ns, skb))
137 goto out_drop_any;
138
139 /* Check if loopback mode is enabled */
140 if (dev->features & NETIF_F_LOOPBACK) {
141 peer_ns = ns;
142 peer_dev = dev;
143 } else {
144 peer_ns = rcu_dereference(ns->peer);
145 if (!peer_ns)
146 goto out_drop_any;
147 peer_dev = peer_ns->netdev;
148 }
149
150 dr = nsim_do_psp(skb, ns, peer_ns, &psp_ext);
151 if (dr)
152 goto out_drop_free;
153
154 rxq = skb_get_queue_mapping(skb);
155 if (rxq >= peer_dev->num_rx_queues)
156 rxq = rxq % peer_dev->num_rx_queues;
157 rq = peer_ns->rq[rxq];
158
159 cfg = peer_dev->cfg;
160 if (skb_is_nonlinear(skb) &&
161 (cfg->hds_config != ETHTOOL_TCP_DATA_SPLIT_ENABLED ||
162 (cfg->hds_config == ETHTOOL_TCP_DATA_SPLIT_ENABLED &&
163 cfg->hds_thresh > len)))
164 skb_linearize(skb);
165
166 skb_tx_timestamp(skb);
167 if (unlikely(nsim_forward_skb(dev, peer_dev,
168 skb, rq, psp_ext) == NET_RX_DROP))
169 goto out_drop_cnt;
170
171 if (!hrtimer_active(&rq->napi_timer))
172 hrtimer_start(&rq->napi_timer, us_to_ktime(5), HRTIMER_MODE_REL);
173
174 rcu_read_unlock();
175 dev_dstats_tx_add(dev, len);
176 return NETDEV_TX_OK;
177
178out_drop_any:
179 dr = SKB_DROP_REASON_NOT_SPECIFIED;
180out_drop_free:
181 kfree_skb_reason(skb, dr);
182out_drop_cnt:
183 rcu_read_unlock();
184 dev_dstats_tx_dropped(dev);
185 return NETDEV_TX_OK;
186}
187
188static int nsim_set_rx_mode(struct net_device *dev,
189 struct netdev_hw_addr_list *uc,
190 struct netdev_hw_addr_list *mc)
191{
192 return 0;
193}
194
195static int nsim_change_mtu(struct net_device *dev, int new_mtu)
196{
197 struct netdevsim *ns = netdev_priv(dev);
198
199 if (ns->xdp.prog && !ns->xdp.prog->aux->xdp_has_frags &&
200 new_mtu > NSIM_XDP_MAX_MTU)
201 return -EBUSY;
202
203 WRITE_ONCE(dev->mtu, new_mtu);
204
205 return 0;
206}
207
208static int nsim_set_vf_mac(struct net_device *dev, int vf, u8 *mac)
209{
210 struct netdevsim *ns = netdev_priv(dev);
211 struct nsim_dev *nsim_dev = ns->nsim_dev;
212
213 /* Only refuse multicast addresses, zero address can mean unset/any. */
214 if (vf >= nsim_dev_get_vfs(nsim_dev) || is_multicast_ether_addr(mac))
215 return -EINVAL;
216 memcpy(nsim_dev->vfconfigs[vf].vf_mac, mac, ETH_ALEN);
217
218 return 0;
219}
220
221static int nsim_set_vf_vlan(struct net_device *dev, int vf,
222 u16 vlan, u8 qos, __be16 vlan_proto)
223{
224 struct netdevsim *ns = netdev_priv(dev);
225 struct nsim_dev *nsim_dev = ns->nsim_dev;
226
227 if (vf >= nsim_dev_get_vfs(nsim_dev) || vlan > 4095 || qos > 7)
228 return -EINVAL;
229
230 nsim_dev->vfconfigs[vf].vlan = vlan;
231 nsim_dev->vfconfigs[vf].qos = qos;
232 nsim_dev->vfconfigs[vf].vlan_proto = vlan_proto;
233
234 return 0;
235}
236
237static int nsim_set_vf_rate(struct net_device *dev, int vf, int min, int max)
238{
239 struct netdevsim *ns = netdev_priv(dev);
240 struct nsim_dev *nsim_dev = ns->nsim_dev;
241
242 if (nsim_esw_mode_is_switchdev(ns->nsim_dev)) {
243 pr_err("Not supported in switchdev mode. Please use devlink API.\n");
244 return -EOPNOTSUPP;
245 }
246
247 if (vf >= nsim_dev_get_vfs(nsim_dev))
248 return -EINVAL;
249
250 nsim_dev->vfconfigs[vf].min_tx_rate = min;
251 nsim_dev->vfconfigs[vf].max_tx_rate = max;
252
253 return 0;
254}
255
256static int nsim_set_vf_spoofchk(struct net_device *dev, int vf, bool val)
257{
258 struct netdevsim *ns = netdev_priv(dev);
259 struct nsim_dev *nsim_dev = ns->nsim_dev;
260
261 if (vf >= nsim_dev_get_vfs(nsim_dev))
262 return -EINVAL;
263 nsim_dev->vfconfigs[vf].spoofchk_enabled = val;
264
265 return 0;
266}
267
268static int nsim_set_vf_rss_query_en(struct net_device *dev, int vf, bool val)
269{
270 struct netdevsim *ns = netdev_priv(dev);
271 struct nsim_dev *nsim_dev = ns->nsim_dev;
272
273 if (vf >= nsim_dev_get_vfs(nsim_dev))
274 return -EINVAL;
275 nsim_dev->vfconfigs[vf].rss_query_enabled = val;
276
277 return 0;
278}
279
280static int nsim_set_vf_trust(struct net_device *dev, int vf, bool val)
281{
282 struct netdevsim *ns = netdev_priv(dev);
283 struct nsim_dev *nsim_dev = ns->nsim_dev;
284
285 if (vf >= nsim_dev_get_vfs(nsim_dev))
286 return -EINVAL;
287 nsim_dev->vfconfigs[vf].trusted = val;
288
289 return 0;
290}
291
292static int
293nsim_get_vf_config(struct net_device *dev, int vf, struct ifla_vf_info *ivi)
294{
295 struct netdevsim *ns = netdev_priv(dev);
296 struct nsim_dev *nsim_dev = ns->nsim_dev;
297
298 if (vf >= nsim_dev_get_vfs(nsim_dev))
299 return -EINVAL;
300
301 ivi->vf = vf;
302 ivi->linkstate = nsim_dev->vfconfigs[vf].link_state;
303 ivi->min_tx_rate = nsim_dev->vfconfigs[vf].min_tx_rate;
304 ivi->max_tx_rate = nsim_dev->vfconfigs[vf].max_tx_rate;
305 ivi->vlan = nsim_dev->vfconfigs[vf].vlan;
306 ivi->vlan_proto = nsim_dev->vfconfigs[vf].vlan_proto;
307 ivi->qos = nsim_dev->vfconfigs[vf].qos;
308 memcpy(&ivi->mac, nsim_dev->vfconfigs[vf].vf_mac, ETH_ALEN);
309 ivi->spoofchk = nsim_dev->vfconfigs[vf].spoofchk_enabled;
310 ivi->trusted = nsim_dev->vfconfigs[vf].trusted;
311 ivi->rss_query_en = nsim_dev->vfconfigs[vf].rss_query_enabled;
312
313 return 0;
314}
315
316static int nsim_set_vf_link_state(struct net_device *dev, int vf, int state)
317{
318 struct netdevsim *ns = netdev_priv(dev);
319 struct nsim_dev *nsim_dev = ns->nsim_dev;
320
321 if (vf >= nsim_dev_get_vfs(nsim_dev))
322 return -EINVAL;
323
324 switch (state) {
325 case IFLA_VF_LINK_STATE_AUTO:
326 case IFLA_VF_LINK_STATE_ENABLE:
327 case IFLA_VF_LINK_STATE_DISABLE:
328 break;
329 default:
330 return -EINVAL;
331 }
332
333 nsim_dev->vfconfigs[vf].link_state = state;
334
335 return 0;
336}
337
338static int
339nsim_set_features(struct net_device *dev, netdev_features_t features)
340{
341 struct netdevsim *ns = netdev_priv(dev);
342
343 if ((dev->features & NETIF_F_HW_TC) > (features & NETIF_F_HW_TC))
344 return nsim_bpf_disable_tc(ns);
345
346 return 0;
347}
348
349static int nsim_get_iflink(const struct net_device *dev)
350{
351 struct netdevsim *nsim, *peer;
352 int iflink;
353
354 nsim = netdev_priv(dev);
355
356 rcu_read_lock();
357 peer = rcu_dereference(nsim->peer);
358 iflink = peer ? READ_ONCE(peer->netdev->ifindex) :
359 READ_ONCE(dev->ifindex);
360 rcu_read_unlock();
361
362 return iflink;
363}
364
365static int nsim_rcv(struct nsim_rq *rq, int budget)
366{
367 struct net_device *dev = rq->napi.dev;
368 struct bpf_prog *xdp_prog;
369 struct netdevsim *ns;
370 struct sk_buff *skb;
371 unsigned int skblen;
372 int i, ret;
373
374 ns = netdev_priv(dev);
375 xdp_prog = READ_ONCE(ns->xdp.prog);
376
377 for (i = 0; i < budget; i++) {
378 if (skb_queue_empty(&rq->skb_queue))
379 break;
380
381 skb = skb_dequeue(&rq->skb_queue);
382
383 if (xdp_prog) {
384 /* skb might be freed directly by XDP, save the len */
385 skblen = skb->len;
386
387 if (skb->ip_summed == CHECKSUM_PARTIAL)
388 skb_checksum_help(skb);
389 ret = do_xdp_generic(xdp_prog, &skb);
390 if (ret != XDP_PASS) {
391 dev_dstats_rx_add(dev, skblen);
392 continue;
393 }
394 }
395
396 /* skb might be discard at netif_receive_skb, save the len */
397 dev_dstats_rx_add(dev, skb->len);
398 napi_gro_receive(&rq->napi, skb);
399 }
400
401 nsim_start_peer_tx_queue(dev, rq);
402 return i;
403}
404
405static int nsim_poll(struct napi_struct *napi, int budget)
406{
407 struct nsim_rq *rq = container_of(napi, struct nsim_rq, napi);
408 int done;
409
410 done = nsim_rcv(rq, budget);
411 if (done < budget)
412 napi_complete_done(napi, done);
413
414 return done;
415}
416
417static int nsim_create_page_pool(struct page_pool **p, struct napi_struct *napi)
418{
419 struct page_pool_params params = {
420 .order = 0,
421 .pool_size = NSIM_RING_SIZE,
422 .nid = NUMA_NO_NODE,
423 .dev = &napi->dev->dev,
424 .napi = napi,
425 .dma_dir = DMA_BIDIRECTIONAL,
426 .netdev = napi->dev,
427 };
428 struct page_pool *pool;
429
430 pool = page_pool_create(&params);
431 if (IS_ERR(pool))
432 return PTR_ERR(pool);
433
434 *p = pool;
435 return 0;
436}
437
438static int nsim_init_napi(struct netdevsim *ns)
439{
440 struct net_device *dev = ns->netdev;
441 struct nsim_rq *rq;
442 int err, i;
443
444 for (i = 0; i < dev->num_rx_queues; i++) {
445 rq = ns->rq[i];
446
447 netif_napi_add_config_locked(dev, &rq->napi, nsim_poll, i);
448 }
449
450 for (i = 0; i < dev->num_rx_queues; i++) {
451 rq = ns->rq[i];
452
453 err = nsim_create_page_pool(&rq->page_pool, &rq->napi);
454 if (err)
455 goto err_pp_destroy;
456 }
457
458 return 0;
459
460err_pp_destroy:
461 while (i--) {
462 page_pool_destroy(ns->rq[i]->page_pool);
463 ns->rq[i]->page_pool = NULL;
464 }
465
466 for (i = 0; i < dev->num_rx_queues; i++)
467 __netif_napi_del_locked(&ns->rq[i]->napi);
468
469 return err;
470}
471
472static enum hrtimer_restart nsim_napi_schedule(struct hrtimer *timer)
473{
474 struct nsim_rq *rq;
475
476 rq = container_of(timer, struct nsim_rq, napi_timer);
477 napi_schedule(&rq->napi);
478
479 return HRTIMER_NORESTART;
480}
481
482static void nsim_rq_timer_init(struct nsim_rq *rq)
483{
484 hrtimer_setup(&rq->napi_timer, nsim_napi_schedule, CLOCK_MONOTONIC,
485 HRTIMER_MODE_REL);
486}
487
488static void nsim_enable_napi(struct netdevsim *ns)
489{
490 struct net_device *dev = ns->netdev;
491 int i;
492
493 for (i = 0; i < dev->num_rx_queues; i++) {
494 struct nsim_rq *rq = ns->rq[i];
495
496 netif_queue_set_napi(dev, i, NETDEV_QUEUE_TYPE_RX, &rq->napi);
497 napi_enable_locked(&rq->napi);
498 }
499}
500
501static int nsim_open(struct net_device *dev)
502{
503 struct netdevsim *ns = netdev_priv(dev);
504 struct netdevsim *peer;
505 int err;
506
507 netdev_assert_locked(dev);
508
509 err = nsim_init_napi(ns);
510 if (err)
511 return err;
512
513 nsim_enable_napi(ns);
514
515 peer = rtnl_dereference(ns->peer);
516 if (peer && netif_running(peer->netdev)) {
517 netif_carrier_on(dev);
518 netif_carrier_on(peer->netdev);
519 }
520
521 return 0;
522}
523
524static void nsim_del_napi(struct netdevsim *ns)
525{
526 struct net_device *dev = ns->netdev;
527 int i;
528
529 for (i = 0; i < dev->num_rx_queues; i++) {
530 struct nsim_rq *rq = ns->rq[i];
531
532 netif_queue_set_napi(dev, i, NETDEV_QUEUE_TYPE_RX, NULL);
533 napi_disable_locked(&rq->napi);
534 __netif_napi_del_locked(&rq->napi);
535 }
536 synchronize_net();
537
538 for (i = 0; i < dev->num_rx_queues; i++) {
539 page_pool_destroy(ns->rq[i]->page_pool);
540 ns->rq[i]->page_pool = NULL;
541 }
542}
543
544static int nsim_stop(struct net_device *dev)
545{
546 struct netdevsim *ns = netdev_priv(dev);
547 struct netdevsim *peer;
548
549 netdev_assert_locked(dev);
550
551 netif_carrier_off(dev);
552 peer = rtnl_dereference(ns->peer);
553 if (peer)
554 netif_carrier_off(peer->netdev);
555
556 nsim_del_napi(ns);
557
558 return 0;
559}
560
561static int nsim_vlan_rx_add_vid(struct net_device *dev, __be16 proto, u16 vid)
562{
563 struct netdevsim *ns = netdev_priv(dev);
564
565 if (vid >= VLAN_N_VID)
566 return -EINVAL;
567
568 if (proto == htons(ETH_P_8021Q))
569 WARN_ON_ONCE(test_and_set_bit(vid, ns->vlan.ctag));
570 else if (proto == htons(ETH_P_8021AD))
571 WARN_ON_ONCE(test_and_set_bit(vid, ns->vlan.stag));
572
573 return 0;
574}
575
576static int nsim_vlan_rx_kill_vid(struct net_device *dev, __be16 proto, u16 vid)
577{
578 struct netdevsim *ns = netdev_priv(dev);
579
580 if (vid >= VLAN_N_VID)
581 return -EINVAL;
582
583 if (proto == htons(ETH_P_8021Q))
584 WARN_ON_ONCE(!test_and_clear_bit(vid, ns->vlan.ctag));
585 else if (proto == htons(ETH_P_8021AD))
586 WARN_ON_ONCE(!test_and_clear_bit(vid, ns->vlan.stag));
587
588 return 0;
589}
590
591static int nsim_shaper_set(struct net_shaper_binding *binding,
592 const struct net_shaper *shaper,
593 struct netlink_ext_ack *extack)
594{
595 return 0;
596}
597
598static int nsim_shaper_del(struct net_shaper_binding *binding,
599 const struct net_shaper_handle *handle,
600 struct netlink_ext_ack *extack)
601{
602 return 0;
603}
604
605static int nsim_shaper_group(struct net_shaper_binding *binding,
606 int leaves_count,
607 const struct net_shaper *leaves,
608 const struct net_shaper *root,
609 struct netlink_ext_ack *extack)
610{
611 return 0;
612}
613
614static void nsim_shaper_cap(struct net_shaper_binding *binding,
615 enum net_shaper_scope scope,
616 unsigned long *flags)
617{
618 *flags = ULONG_MAX;
619}
620
621static const struct net_shaper_ops nsim_shaper_ops = {
622 .set = nsim_shaper_set,
623 .delete = nsim_shaper_del,
624 .group = nsim_shaper_group,
625 .capabilities = nsim_shaper_cap,
626};
627
628static const struct net_device_ops nsim_netdev_ops = {
629 .ndo_start_xmit = nsim_start_xmit,
630 .ndo_set_rx_mode_async = nsim_set_rx_mode,
631 .ndo_set_mac_address = eth_mac_addr,
632 .ndo_validate_addr = eth_validate_addr,
633 .ndo_change_mtu = nsim_change_mtu,
634 .ndo_set_vf_mac = nsim_set_vf_mac,
635 .ndo_set_vf_vlan = nsim_set_vf_vlan,
636 .ndo_set_vf_rate = nsim_set_vf_rate,
637 .ndo_set_vf_spoofchk = nsim_set_vf_spoofchk,
638 .ndo_set_vf_trust = nsim_set_vf_trust,
639 .ndo_get_vf_config = nsim_get_vf_config,
640 .ndo_set_vf_link_state = nsim_set_vf_link_state,
641 .ndo_set_vf_rss_query_en = nsim_set_vf_rss_query_en,
642 .ndo_setup_tc = nsim_setup_tc,
643 .ndo_set_features = nsim_set_features,
644 .ndo_get_iflink = nsim_get_iflink,
645 .ndo_bpf = nsim_bpf,
646 .ndo_open = nsim_open,
647 .ndo_stop = nsim_stop,
648 .ndo_vlan_rx_add_vid = nsim_vlan_rx_add_vid,
649 .ndo_vlan_rx_kill_vid = nsim_vlan_rx_kill_vid,
650 .net_shaper_ops = &nsim_shaper_ops,
651};
652
653static const struct net_device_ops nsim_vf_netdev_ops = {
654 .ndo_start_xmit = nsim_start_xmit,
655 .ndo_set_rx_mode_async = nsim_set_rx_mode,
656 .ndo_set_mac_address = eth_mac_addr,
657 .ndo_validate_addr = eth_validate_addr,
658 .ndo_change_mtu = nsim_change_mtu,
659 .ndo_setup_tc = nsim_setup_tc,
660 .ndo_set_features = nsim_set_features,
661 .ndo_vlan_rx_add_vid = nsim_vlan_rx_add_vid,
662 .ndo_vlan_rx_kill_vid = nsim_vlan_rx_kill_vid,
663};
664
665/* We don't have true per-queue stats, yet, so do some random fakery here.
666 * Only report stuff for queue 0.
667 */
668static void nsim_get_queue_stats_rx(struct net_device *dev, int idx,
669 struct netdev_queue_stats_rx *stats)
670{
671 struct rtnl_link_stats64 rtstats = {};
672
673 if (!idx)
674 dev_get_stats(dev, &rtstats);
675
676 stats->packets = rtstats.rx_packets - !!rtstats.rx_packets;
677 stats->bytes = rtstats.rx_bytes;
678}
679
680static void nsim_get_queue_stats_tx(struct net_device *dev, int idx,
681 struct netdev_queue_stats_tx *stats)
682{
683 struct rtnl_link_stats64 rtstats = {};
684
685 if (!idx)
686 dev_get_stats(dev, &rtstats);
687
688 stats->packets = rtstats.tx_packets - !!rtstats.tx_packets;
689 stats->bytes = rtstats.tx_bytes;
690}
691
692static void nsim_get_base_stats(struct net_device *dev,
693 struct netdev_queue_stats_rx *rx,
694 struct netdev_queue_stats_tx *tx)
695{
696 struct rtnl_link_stats64 rtstats = {};
697
698 dev_get_stats(dev, &rtstats);
699
700 rx->packets = !!rtstats.rx_packets;
701 rx->bytes = 0;
702 tx->packets = !!rtstats.tx_packets;
703 tx->bytes = 0;
704}
705
706static const struct netdev_stat_ops nsim_stat_ops = {
707 .get_queue_stats_tx = nsim_get_queue_stats_tx,
708 .get_queue_stats_rx = nsim_get_queue_stats_rx,
709 .get_base_stats = nsim_get_base_stats,
710};
711
712static struct nsim_rq *nsim_queue_alloc(void)
713{
714 struct nsim_rq *rq;
715
716 rq = kzalloc_obj(*rq, GFP_KERNEL_ACCOUNT);
717 if (!rq)
718 return NULL;
719
720 skb_queue_head_init(&rq->skb_queue);
721 nsim_rq_timer_init(rq);
722 return rq;
723}
724
725static void nsim_queue_free(struct net_device *dev, struct nsim_rq *rq)
726{
727 hrtimer_cancel(&rq->napi_timer);
728
729 if (rq->skb_queue.qlen) {
730 local_bh_disable();
731 dev_dstats_rx_dropped_add(dev, rq->skb_queue.qlen);
732 local_bh_enable();
733 }
734
735 skb_queue_purge_reason(&rq->skb_queue, SKB_DROP_REASON_QUEUE_PURGE);
736 kfree(rq);
737}
738
739/* Queue reset mode is controlled by ns->rq_reset_mode.
740 * - normal - new NAPI new pool (old NAPI enabled when new added)
741 * - mode 1 - allocate new pool (NAPI is only disabled / enabled)
742 * - mode 2 - new NAPI new pool (old NAPI removed before new added)
743 * - mode 3 - new NAPI new pool (old NAPI disabled when new added)
744 */
745struct nsim_queue_mem {
746 struct nsim_rq *rq;
747 struct page_pool *pp;
748};
749
750static int
751nsim_queue_mem_alloc(struct net_device *dev,
752 struct netdev_queue_config *qcfg,
753 void *per_queue_mem, int idx)
754{
755 struct nsim_queue_mem *qmem = per_queue_mem;
756 struct netdevsim *ns = netdev_priv(dev);
757 int err;
758
759 if (ns->rq_reset_mode > 3)
760 return -EINVAL;
761
762 if (ns->rq_reset_mode == 1) {
763 if (!netif_running(ns->netdev))
764 return -ENETDOWN;
765 return nsim_create_page_pool(&qmem->pp, &ns->rq[idx]->napi);
766 }
767
768 qmem->rq = nsim_queue_alloc();
769 if (!qmem->rq)
770 return -ENOMEM;
771
772 err = nsim_create_page_pool(&qmem->rq->page_pool, &qmem->rq->napi);
773 if (err)
774 goto err_free;
775
776 if (!ns->rq_reset_mode)
777 netif_napi_add_config_locked(dev, &qmem->rq->napi, nsim_poll,
778 idx);
779
780 return 0;
781
782err_free:
783 nsim_queue_free(dev, qmem->rq);
784 return err;
785}
786
787static void nsim_queue_mem_free(struct net_device *dev, void *per_queue_mem)
788{
789 struct nsim_queue_mem *qmem = per_queue_mem;
790 struct netdevsim *ns = netdev_priv(dev);
791
792 page_pool_destroy(qmem->pp);
793 if (qmem->rq) {
794 if (!ns->rq_reset_mode)
795 netif_napi_del_locked(&qmem->rq->napi);
796 page_pool_destroy(qmem->rq->page_pool);
797 nsim_queue_free(dev, qmem->rq);
798 }
799}
800
801static int
802nsim_queue_start(struct net_device *dev, struct netdev_queue_config *qcfg,
803 void *per_queue_mem, int idx)
804{
805 struct nsim_queue_mem *qmem = per_queue_mem;
806 struct netdevsim *ns = netdev_priv(dev);
807
808 netdev_assert_locked(dev);
809
810 if (ns->rq_reset_mode == 1) {
811 ns->rq[idx]->page_pool = qmem->pp;
812 napi_enable_locked(&ns->rq[idx]->napi);
813 return 0;
814 }
815
816 /* netif_napi_add()/_del() should normally be called from alloc/free,
817 * here we want to test various call orders.
818 */
819 if (ns->rq_reset_mode == 2) {
820 netif_napi_del_locked(&ns->rq[idx]->napi);
821 netif_napi_add_config_locked(dev, &qmem->rq->napi, nsim_poll,
822 idx);
823 } else if (ns->rq_reset_mode == 3) {
824 netif_napi_add_config_locked(dev, &qmem->rq->napi, nsim_poll,
825 idx);
826 netif_napi_del_locked(&ns->rq[idx]->napi);
827 }
828
829 ns->rq[idx] = qmem->rq;
830 netif_queue_set_napi(dev, idx, NETDEV_QUEUE_TYPE_RX, &ns->rq[idx]->napi);
831 napi_enable_locked(&ns->rq[idx]->napi);
832
833 return 0;
834}
835
836static int nsim_queue_stop(struct net_device *dev, void *per_queue_mem, int idx)
837{
838 struct nsim_queue_mem *qmem = per_queue_mem;
839 struct netdevsim *ns = netdev_priv(dev);
840
841 netdev_assert_locked(dev);
842
843 napi_disable_locked(&ns->rq[idx]->napi);
844
845 if (ns->rq_reset_mode == 1) {
846 qmem->pp = ns->rq[idx]->page_pool;
847 page_pool_disable_direct_recycling(qmem->pp);
848 } else {
849 qmem->rq = ns->rq[idx];
850 }
851
852 return 0;
853}
854
855static const struct netdev_queue_mgmt_ops nsim_queue_mgmt_ops = {
856 .ndo_queue_mem_size = sizeof(struct nsim_queue_mem),
857 .ndo_queue_mem_alloc = nsim_queue_mem_alloc,
858 .ndo_queue_mem_free = nsim_queue_mem_free,
859 .ndo_queue_start = nsim_queue_start,
860 .ndo_queue_stop = nsim_queue_stop,
861};
862
863static ssize_t
864nsim_qreset_write(struct file *file, const char __user *data,
865 size_t count, loff_t *ppos)
866{
867 struct netdevsim *ns = file->private_data;
868 unsigned int queue, mode;
869 char buf[32];
870 ssize_t ret;
871
872 if (count >= sizeof(buf))
873 return -EINVAL;
874 if (copy_from_user(buf, data, count))
875 return -EFAULT;
876 buf[count] = '\0';
877
878 ret = sscanf(buf, "%u %u", &queue, &mode);
879 if (ret != 2)
880 return -EINVAL;
881
882 netdev_lock(ns->netdev);
883 if (queue >= ns->netdev->real_num_rx_queues) {
884 ret = -EINVAL;
885 goto exit_unlock;
886 }
887
888 ns->rq_reset_mode = mode;
889 ret = netdev_rx_queue_restart(ns->netdev, queue);
890 ns->rq_reset_mode = 0;
891 if (ret)
892 goto exit_unlock;
893
894 ret = count;
895exit_unlock:
896 netdev_unlock(ns->netdev);
897 return ret;
898}
899
900static const struct file_operations nsim_qreset_fops = {
901 .open = simple_open,
902 .write = nsim_qreset_write,
903 .owner = THIS_MODULE,
904};
905
906static ssize_t
907nsim_pp_hold_read(struct file *file, char __user *data,
908 size_t count, loff_t *ppos)
909{
910 struct netdevsim *ns = file->private_data;
911 char buf[3] = "n\n";
912
913 if (ns->page)
914 buf[0] = 'y';
915
916 return simple_read_from_buffer(data, count, ppos, buf, 2);
917}
918
919static ssize_t
920nsim_pp_hold_write(struct file *file, const char __user *data,
921 size_t count, loff_t *ppos)
922{
923 struct netdevsim *ns = file->private_data;
924 ssize_t ret;
925 bool val;
926
927 ret = kstrtobool_from_user(data, count, &val);
928 if (ret)
929 return ret;
930
931 rtnl_lock();
932 ret = count;
933 if (val == !!ns->page)
934 goto exit;
935
936 if (!netif_running(ns->netdev) && val) {
937 ret = -ENETDOWN;
938 } else if (val) {
939 ns->page = page_pool_dev_alloc_pages(ns->rq[0]->page_pool);
940 if (!ns->page)
941 ret = -ENOMEM;
942 } else {
943 page_pool_put_full_page(pp_page_to_nmdesc(ns->page)->pp,
944 ns->page, false);
945 ns->page = NULL;
946 }
947
948exit:
949 rtnl_unlock();
950 return ret;
951}
952
953static const struct file_operations nsim_pp_hold_fops = {
954 .open = simple_open,
955 .read = nsim_pp_hold_read,
956 .write = nsim_pp_hold_write,
957 .llseek = generic_file_llseek,
958 .owner = THIS_MODULE,
959};
960
961static int nsim_vlan_show(struct seq_file *s, void *data)
962{
963 struct netdevsim *ns = s->private;
964 int vid;
965
966 for_each_set_bit(vid, ns->vlan.ctag, VLAN_N_VID)
967 seq_printf(s, "ctag %d\n", vid);
968 for_each_set_bit(vid, ns->vlan.stag, VLAN_N_VID)
969 seq_printf(s, "stag %d\n", vid);
970
971 return 0;
972}
973DEFINE_SHOW_ATTRIBUTE(nsim_vlan);
974
975static void nsim_setup(struct net_device *dev)
976{
977 ether_setup(dev);
978 eth_hw_addr_random(dev);
979
980 dev->flags &= ~IFF_MULTICAST;
981 dev->priv_flags |= IFF_LIVE_ADDR_CHANGE;
982 dev->features |= NETIF_F_HIGHDMA |
983 NETIF_F_SG |
984 NETIF_F_FRAGLIST |
985 NETIF_F_HW_CSUM |
986 NETIF_F_LRO |
987 NETIF_F_TSO |
988 NETIF_F_HW_VLAN_CTAG_FILTER |
989 NETIF_F_HW_VLAN_STAG_FILTER;
990 dev->hw_features |= NETIF_F_HW_TC |
991 NETIF_F_SG |
992 NETIF_F_FRAGLIST |
993 NETIF_F_HW_CSUM |
994 NETIF_F_LRO |
995 NETIF_F_TSO |
996 NETIF_F_LOOPBACK |
997 NETIF_F_HW_VLAN_CTAG_FILTER |
998 NETIF_F_HW_VLAN_STAG_FILTER;
999 dev->pcpu_stat_type = NETDEV_PCPU_STAT_DSTATS;
1000 dev->max_mtu = ETH_MAX_MTU;
1001 dev->xdp_features = NETDEV_XDP_ACT_BASIC | NETDEV_XDP_ACT_HW_OFFLOAD;
1002}
1003
1004static int nsim_queue_init(struct netdevsim *ns)
1005{
1006 struct net_device *dev = ns->netdev;
1007 int i;
1008
1009 ns->rq = kzalloc_objs(*ns->rq, dev->num_rx_queues, GFP_KERNEL_ACCOUNT);
1010 if (!ns->rq)
1011 return -ENOMEM;
1012
1013 for (i = 0; i < dev->num_rx_queues; i++) {
1014 ns->rq[i] = nsim_queue_alloc();
1015 if (!ns->rq[i])
1016 goto err_free_prev;
1017 }
1018
1019 return 0;
1020
1021err_free_prev:
1022 while (i--)
1023 kfree(ns->rq[i]);
1024 kfree(ns->rq);
1025 return -ENOMEM;
1026}
1027
1028static void nsim_queue_uninit(struct netdevsim *ns)
1029{
1030 struct net_device *dev = ns->netdev;
1031 int i;
1032
1033 for (i = 0; i < dev->num_rx_queues; i++)
1034 nsim_queue_free(dev, ns->rq[i]);
1035
1036 kfree(ns->rq);
1037 ns->rq = NULL;
1038}
1039
1040static int nsim_init_netdevsim(struct netdevsim *ns)
1041{
1042 struct netdevsim *peer;
1043 struct mock_phc *phc;
1044 int err;
1045
1046 phc = mock_phc_create(&ns->nsim_bus_dev->dev);
1047 if (IS_ERR(phc))
1048 return PTR_ERR(phc);
1049
1050 ns->phc = phc;
1051 ns->netdev->netdev_ops = &nsim_netdev_ops;
1052 ns->netdev->stat_ops = &nsim_stat_ops;
1053 ns->netdev->queue_mgmt_ops = &nsim_queue_mgmt_ops;
1054 netdev_lockdep_set_classes(ns->netdev);
1055
1056 err = nsim_udp_tunnels_info_create(ns->nsim_dev, ns->netdev);
1057 if (err)
1058 goto err_phc_destroy;
1059
1060 rtnl_lock();
1061 err = nsim_queue_init(ns);
1062 if (err)
1063 goto err_utn_destroy;
1064
1065 err = nsim_bpf_init(ns);
1066 if (err)
1067 goto err_rq_destroy;
1068
1069 nsim_macsec_init(ns);
1070 nsim_ipsec_init(ns);
1071
1072 err = register_netdevice(ns->netdev);
1073 if (err)
1074 goto err_ipsec_teardown;
1075 rtnl_unlock();
1076
1077 err = nsim_psp_init(ns);
1078 if (err)
1079 goto err_unregister_netdev;
1080
1081 if (IS_ENABLED(CONFIG_DEBUG_NET)) {
1082 ns->nb.notifier_call = netdev_debug_event;
1083 if (register_netdevice_notifier_dev_net(ns->netdev, &ns->nb,
1084 &ns->nn))
1085 ns->nb.notifier_call = NULL;
1086 }
1087
1088 return 0;
1089
1090err_unregister_netdev:
1091 rtnl_lock();
1092 peer = rtnl_dereference(ns->peer);
1093 if (peer)
1094 RCU_INIT_POINTER(peer->peer, NULL);
1095 RCU_INIT_POINTER(ns->peer, NULL);
1096 unregister_netdevice(ns->netdev);
1097err_ipsec_teardown:
1098 nsim_ipsec_teardown(ns);
1099 nsim_macsec_teardown(ns);
1100 nsim_bpf_uninit(ns);
1101err_rq_destroy:
1102 nsim_queue_uninit(ns);
1103err_utn_destroy:
1104 rtnl_unlock();
1105 nsim_udp_tunnels_info_destroy(ns->netdev);
1106err_phc_destroy:
1107 mock_phc_destroy(ns->phc);
1108 return err;
1109}
1110
1111static int nsim_init_netdevsim_vf(struct netdevsim *ns)
1112{
1113 int err;
1114
1115 ns->netdev->netdev_ops = &nsim_vf_netdev_ops;
1116 rtnl_lock();
1117 err = register_netdevice(ns->netdev);
1118 rtnl_unlock();
1119 return err;
1120}
1121
1122static void nsim_exit_netdevsim(struct netdevsim *ns)
1123{
1124 nsim_udp_tunnels_info_destroy(ns->netdev);
1125 mock_phc_destroy(ns->phc);
1126}
1127
1128struct netdevsim *nsim_create(struct nsim_dev *nsim_dev,
1129 struct nsim_dev_port *nsim_dev_port,
1130 u8 perm_addr[ETH_ALEN])
1131{
1132 struct net_device *dev;
1133 struct netdevsim *ns;
1134 int err;
1135
1136 dev = alloc_netdev_mq(sizeof(*ns), "eth%d", NET_NAME_UNKNOWN, nsim_setup,
1137 nsim_dev->nsim_bus_dev->num_queues);
1138 if (!dev)
1139 return ERR_PTR(-ENOMEM);
1140
1141 if (perm_addr)
1142 memcpy(dev->perm_addr, perm_addr, ETH_ALEN);
1143
1144 dev_net_set(dev, nsim_dev_net(nsim_dev));
1145 ns = netdev_priv(dev);
1146 ns->netdev = dev;
1147 ns->nsim_dev = nsim_dev;
1148 ns->nsim_dev_port = nsim_dev_port;
1149 ns->nsim_bus_dev = nsim_dev->nsim_bus_dev;
1150 SET_NETDEV_DEV(dev, &ns->nsim_bus_dev->dev);
1151 SET_NETDEV_DEVLINK_PORT(dev, &nsim_dev_port->devlink_port);
1152 nsim_ethtool_init(ns);
1153 if (nsim_dev_port_is_pf(nsim_dev_port))
1154 err = nsim_init_netdevsim(ns);
1155 else
1156 err = nsim_init_netdevsim_vf(ns);
1157 if (err)
1158 goto err_free_netdev;
1159
1160 ns->pp_dfs = debugfs_create_file("pp_hold", 0600, nsim_dev_port->ddir,
1161 ns, &nsim_pp_hold_fops);
1162 ns->qr_dfs = debugfs_create_file("queue_reset", 0200,
1163 nsim_dev_port->ddir, ns,
1164 &nsim_qreset_fops);
1165 ns->vlan_dfs = debugfs_create_file("vlan", 0400, nsim_dev_port->ddir,
1166 ns, &nsim_vlan_fops);
1167 return ns;
1168
1169err_free_netdev:
1170 nsim_ethtool_fini(ns);
1171 free_netdev(dev);
1172 return ERR_PTR(err);
1173}
1174
1175void nsim_destroy(struct netdevsim *ns)
1176{
1177 struct net_device *dev = ns->netdev;
1178 struct netdevsim *peer;
1179 u16 vid;
1180
1181 debugfs_remove(ns->vlan_dfs);
1182 debugfs_remove(ns->qr_dfs);
1183 debugfs_remove(ns->pp_dfs);
1184 nsim_ethtool_fini(ns);
1185
1186 if (ns->nb.notifier_call)
1187 unregister_netdevice_notifier_dev_net(ns->netdev, &ns->nb,
1188 &ns->nn);
1189
1190 if (nsim_dev_port_is_pf(ns->nsim_dev_port))
1191 nsim_psp_uninit(ns);
1192
1193 rtnl_lock();
1194 peer = rtnl_dereference(ns->peer);
1195 if (peer)
1196 RCU_INIT_POINTER(peer->peer, NULL);
1197 RCU_INIT_POINTER(ns->peer, NULL);
1198 unregister_netdevice(dev);
1199 if (nsim_dev_port_is_pf(ns->nsim_dev_port)) {
1200 nsim_macsec_teardown(ns);
1201 nsim_ipsec_teardown(ns);
1202 nsim_bpf_uninit(ns);
1203 nsim_queue_uninit(ns);
1204 }
1205 rtnl_unlock();
1206 if (nsim_dev_port_is_pf(ns->nsim_dev_port))
1207 nsim_exit_netdevsim(ns);
1208
1209 for_each_set_bit(vid, ns->vlan.ctag, VLAN_N_VID)
1210 WARN_ON_ONCE(1);
1211 for_each_set_bit(vid, ns->vlan.stag, VLAN_N_VID)
1212 WARN_ON_ONCE(1);
1213
1214 /* Put this intentionally late to exercise the orphaning path */
1215 if (ns->page) {
1216 page_pool_put_full_page(pp_page_to_nmdesc(ns->page)->pp,
1217 ns->page, false);
1218 ns->page = NULL;
1219 }
1220
1221 free_netdev(dev);
1222}
1223
1224bool netdev_is_nsim(struct net_device *dev)
1225{
1226 return dev->netdev_ops == &nsim_netdev_ops;
1227}
1228
1229static int nsim_validate(struct nlattr *tb[], struct nlattr *data[],
1230 struct netlink_ext_ack *extack)
1231{
1232 NL_SET_ERR_MSG_MOD(extack,
1233 "Please use: echo \"[ID] [PORT_COUNT] [NUM_QUEUES]\" > /sys/bus/netdevsim/new_device");
1234 return -EOPNOTSUPP;
1235}
1236
1237static struct rtnl_link_ops nsim_link_ops __read_mostly = {
1238 .kind = DRV_NAME,
1239 .validate = nsim_validate,
1240};
1241
1242static int __init nsim_module_init(void)
1243{
1244 int err;
1245
1246 err = nsim_dev_init();
1247 if (err)
1248 return err;
1249
1250 err = nsim_bus_init();
1251 if (err)
1252 goto err_dev_exit;
1253
1254 err = rtnl_link_register(&nsim_link_ops);
1255 if (err)
1256 goto err_bus_exit;
1257
1258 return 0;
1259
1260err_bus_exit:
1261 nsim_bus_exit();
1262err_dev_exit:
1263 nsim_dev_exit();
1264 return err;
1265}
1266
1267static void __exit nsim_module_exit(void)
1268{
1269 rtnl_link_unregister(&nsim_link_ops);
1270 nsim_bus_exit();
1271 nsim_dev_exit();
1272}
1273
1274module_init(nsim_module_init);
1275module_exit(nsim_module_exit);
1276MODULE_LICENSE("GPL");
1277MODULE_DESCRIPTION("Simulated networking device for testing");
1278MODULE_ALIAS_RTNL_LINK(DRV_NAME);