linux - net/core/dev.c

13590 lines · 344.7 KiB

1	// SPDX-License-Identifier: GPL-2.0-or-later
2	/*
3	* NET3 Protocol independent device support routines.
4	*
5	* Derived from the non IP parts of dev.c 1.0.19
6	* Authors: Ross Biro
7	* Fred N. van Kempen, <[email protected]>
8	* Mark Evans, <[email protected]>
9	*
10	* Additional Authors:
11	* Florian la Roche <[email protected]>
12	* Alan Cox <[email protected]>
13	* David Hinds <[email protected]>
14	* Alexey Kuznetsov <[email protected]>
15	* Adam Sulmicki <[email protected]>
16	* Pekka Riikonen <[email protected]>
17	*
18	* Changes:
19	* D.J. Barrow : Fixed bug where dev->refcnt gets set
20	* to 2 if register_netdev gets called
21	* before net_dev_init & also removed a
22	* few lines of code in the process.
23	* Alan Cox : device private ioctl copies fields back.
24	* Alan Cox : Transmit queue code does relevant
25	* stunts to keep the queue safe.
26	* Alan Cox : Fixed double lock.
27	* Alan Cox : Fixed promisc NULL pointer trap
28	* ???????? : Support the full private ioctl range
29	* Alan Cox : Moved ioctl permission check into
30	* drivers
31	* Tim Kordas : SIOCADDMULTI/SIOCDELMULTI
32	* Alan Cox : 100 backlog just doesn't cut it when
33	* you start doing multicast video 8)
34	* Alan Cox : Rewrote net_bh and list manager.
35	* Alan Cox : Fix ETH_P_ALL echoback lengths.
36	* Alan Cox : Took out transmit every packet pass
37	* Saved a few bytes in the ioctl handler
38	* Alan Cox : Network driver sets packet type before
39	* calling netif_rx. Saves a function
40	* call a packet.
41	* Alan Cox : Hashed net_bh()
42	* Richard Kooijman: Timestamp fixes.
43	* Alan Cox : Wrong field in SIOCGIFDSTADDR
44	* Alan Cox : Device lock protection.
45	* Alan Cox : Fixed nasty side effect of device close
46	* changes.
47	* Rudi Cilibrasi : Pass the right thing to
48	* set_mac_address()
49	* Dave Miller : 32bit quantity for the device lock to
50	* make it work out on a Sparc.
51	* Bjorn Ekwall : Added KERNELD hack.
52	* Alan Cox : Cleaned up the backlog initialise.
53	* Craig Metz : SIOCGIFCONF fix if space for under
54	* 1 device.
55	* Thomas Bogendoerfer : Return ENODEV for dev_open, if there
56	* is no device open function.
57	* Andi Kleen : Fix error reporting for SIOCGIFCONF
58	* Michael Chastain : Fix signed/unsigned for SIOCGIFCONF
59	* Cyrus Durgin : Cleaned for KMOD
60	* Adam Sulmicki : Bug Fix : Network Device Unload
61	* A network device unload needs to purge
62	* the backlog queue.
63	* Paul Rusty Russell : SIOCSIFNAME
64	* Pekka Riikonen : Netdev boot-time settings code
65	* Andrew Morton : Make unregister_netdevice wait
66	* indefinitely on dev->refcnt
67	* J Hadi Salim : - Backlog queue sampling
68	* - netif_rx() feedback
69	*/
70
71	#include <linux/uaccess.h>
72	#include <linux/bitmap.h>
73	#include <linux/capability.h>
74	#include <linux/cpu.h>
75	#include <linux/types.h>
76	#include <linux/kernel.h>
77	#include <linux/hash.h>
78	#include <linux/slab.h>
79	#include <linux/sched.h>
80	#include <linux/sched/isolation.h>
81	#include <linux/sched/mm.h>
82	#include <linux/smpboot.h>
83	#include <linux/mutex.h>
84	#include <linux/rwsem.h>
85	#include <linux/string.h>
86	#include <linux/mm.h>
87	#include <linux/socket.h>
88	#include <linux/sockios.h>
89	#include <linux/errno.h>
90	#include <linux/interrupt.h>
91	#include <linux/if_ether.h>
92	#include <linux/netdevice.h>
93	#include <linux/etherdevice.h>
94	#include <linux/ethtool.h>
95	#include <linux/ethtool_netlink.h>
96	#include <linux/skbuff.h>
97	#include <linux/kthread.h>
98	#include <linux/bpf.h>
99	#include <linux/bpf_trace.h>
100	#include <net/net_namespace.h>
101	#include <net/sock.h>
102	#include <net/busy_poll.h>
103	#include <linux/rtnetlink.h>
104	#include <linux/stat.h>
105	#include <net/dsa.h>
106	#include <net/dst.h>
107	#include <net/dst_metadata.h>
108	#include <net/gro.h>
109	#include <net/netdev_queues.h>
110	#include <net/pkt_sched.h>
111	#include <net/pkt_cls.h>
112	#include <net/checksum.h>
113	#include <net/xfrm.h>
114	#include <net/tcx.h>
115	#include <linux/highmem.h>
116	#include <linux/init.h>
117	#include <linux/module.h>
118	#include <linux/netpoll.h>
119	#include <linux/rcupdate.h>
120	#include <linux/delay.h>
121	#include <net/iw_handler.h>
122	#include <asm/current.h>
123	#include <linux/audit.h>
124	#include <linux/dmaengine.h>
125	#include <linux/err.h>
126	#include <linux/ctype.h>
127	#include <linux/if_arp.h>
128	#include <linux/if_vlan.h>
129	#include <linux/ip.h>
130	#include <net/ip.h>
131	#include <net/mpls.h>
132	#include <linux/ipv6.h>
133	#include <linux/in.h>
134	#include <linux/jhash.h>
135	#include <linux/random.h>
136	#include <trace/events/napi.h>
137	#include <trace/events/net.h>
138	#include <trace/events/skb.h>
139	#include <trace/events/qdisc.h>
140	#include <trace/events/xdp.h>
141	#include <linux/inetdevice.h>
142	#include <linux/cpu_rmap.h>
143	#include <linux/static_key.h>
144	#include <linux/hashtable.h>
145	#include <linux/vmalloc.h>
146	#include <linux/if_macvlan.h>
147	#include <linux/errqueue.h>
148	#include <linux/hrtimer.h>
149	#include <linux/netfilter_netdev.h>
150	#include <linux/crash_dump.h>
151	#include <linux/sctp.h>
152	#include <net/udp_tunnel.h>
153	#include <linux/net_namespace.h>
154	#include <linux/indirect_call_wrapper.h>
155	#include <net/devlink.h>
156	#include <linux/pm_runtime.h>
157	#include <linux/prandom.h>
158	#include <linux/once_lite.h>
159	#include <net/netdev_lock.h>
160	#include <net/netdev_rx_queue.h>
161	#include <net/page_pool/types.h>
162	#include <net/page_pool/helpers.h>
163	#include <net/page_pool/memory_provider.h>
164	#include <net/rps.h>
165	#include <linux/phy_link_topology.h>
166
167	#include "dev.h"
168	#include "devmem.h"
169	#include "net-sysfs.h"
170
171	static DEFINE_SPINLOCK(ptype_lock);
172	struct list_head ptype_base[PTYPE_HASH_SIZE] __read_mostly;
173
174	static int netif_rx_internal(struct sk_buff *skb);
175	static int call_netdevice_notifiers_extack(unsigned long val,
176	struct net_device *dev,
177	struct netlink_ext_ack *extack);
178
179	static DEFINE_MUTEX(ifalias_mutex);
180
181	/* protects napi_hash addition/deletion and napi_gen_id */
182	static DEFINE_SPINLOCK(napi_hash_lock);
183
184	static unsigned int napi_gen_id = NR_CPUS;
185	static DEFINE_READ_MOSTLY_HASHTABLE(napi_hash, 8);
186
187	static inline void dev_base_seq_inc(struct net *net)
188	{
189	unsigned int val = net->dev_base_seq + 1;
190
191	WRITE_ONCE(net->dev_base_seq, val ?: 1);
192	}
193
194	static inline struct hlist_head dev_name_hash(struct net net, const char *name)
195	{
196	unsigned int hash = full_name_hash(net, name, strnlen(name, IFNAMSIZ));
197
198	return &net->dev_name_head[hash_32(hash, NETDEV_HASHBITS)];
199	}
200
201	static inline struct hlist_head dev_index_hash(struct net net, int ifindex)
202	{
203	return &net->dev_index_head[ifindex & (NETDEV_HASHENTRIES - 1)];
204	}
205
206	#ifndef CONFIG_PREEMPT_RT
207
208	static DEFINE_STATIC_KEY_FALSE(use_backlog_threads_key);
209
210	static int __init setup_backlog_napi_threads(char *arg)
211	{
212	static_branch_enable(&use_backlog_threads_key);
213	return 0;
214	}
215	early_param("thread_backlog_napi", setup_backlog_napi_threads);
216
217	static bool use_backlog_threads(void)
218	{
219	return static_branch_unlikely(&use_backlog_threads_key);
220	}
221
222	#else
223
224	static bool use_backlog_threads(void)
225	{
226	return true;
227	}
228
229	#endif
230
231	static inline void backlog_lock_irq_save(struct softnet_data *sd,
232	unsigned long *flags)
233	{
234	if (IS_ENABLED(CONFIG_PREEMPT_RT)) {
235	spin_lock_irqsave(&sd->input_pkt_queue.lock, *flags);
236	} else {
237	local_irq_save(*flags);
238	if (IS_ENABLED(CONFIG_RPS) \|\| use_backlog_threads())
239	spin_lock(&sd->input_pkt_queue.lock);
240	}
241	}
242
243	static inline void backlog_lock_irq_disable(struct softnet_data *sd)
244	{
245	if (IS_ENABLED(CONFIG_RPS) \|\| use_backlog_threads())
246	spin_lock_irq(&sd->input_pkt_queue.lock);
247	else
248	local_irq_disable();
249	}
250
251	static inline void backlog_unlock_irq_restore(struct softnet_data *sd,
252	unsigned long flags)
253	{
254	if (IS_ENABLED(CONFIG_PREEMPT_RT)) {
255	spin_unlock_irqrestore(&sd->input_pkt_queue.lock, flags);
256	} else {
257	if (IS_ENABLED(CONFIG_RPS) \|\| use_backlog_threads())
258	spin_unlock(&sd->input_pkt_queue.lock);
259	local_irq_restore(flags);
260	}
261	}
262
263	static inline void backlog_unlock_irq_enable(struct softnet_data *sd)
264	{
265	if (IS_ENABLED(CONFIG_RPS) \|\| use_backlog_threads())
266	spin_unlock_irq(&sd->input_pkt_queue.lock);
267	else
268	local_irq_enable();
269	}
270
271	static struct netdev_name_node netdev_name_node_alloc(struct net_device dev,
272	const char *name)
273	{
274	struct netdev_name_node *name_node;
275
276	name_node = kmalloc_obj(*name_node);
277	if (!name_node)
278	return NULL;
279	INIT_HLIST_NODE(&name_node->hlist);
280	name_node->dev = dev;
281	name_node->name = name;
282	return name_node;
283	}
284
285	static struct netdev_name_node *
286	netdev_name_node_head_alloc(struct net_device *dev)
287	{
288	struct netdev_name_node *name_node;
289
290	name_node = netdev_name_node_alloc(dev, dev->name);
291	if (!name_node)
292	return NULL;
293	INIT_LIST_HEAD(&name_node->list);
294	return name_node;
295	}
296
297	static void netdev_name_node_free(struct netdev_name_node *name_node)
298	{
299	kfree(name_node);
300	}
301
302	static void netdev_name_node_add(struct net *net,
303	struct netdev_name_node *name_node)
304	{
305	hlist_add_head_rcu(&name_node->hlist,
306	dev_name_hash(net, name_node->name));
307	}
308
309	static void netdev_name_node_del(struct netdev_name_node *name_node)
310	{
311	hlist_del_rcu(&name_node->hlist);
312	}
313
314	static struct netdev_name_node netdev_name_node_lookup(struct net net,
315	const char *name)
316	{
317	struct hlist_head *head = dev_name_hash(net, name);
318	struct netdev_name_node *name_node;
319
320	hlist_for_each_entry(name_node, head, hlist)
321	if (!strcmp(name_node->name, name))
322	return name_node;
323	return NULL;
324	}
325
326	static struct netdev_name_node netdev_name_node_lookup_rcu(struct net net,
327	const char *name)
328	{
329	struct hlist_head *head = dev_name_hash(net, name);
330	struct netdev_name_node *name_node;
331
332	hlist_for_each_entry_rcu(name_node, head, hlist)
333	if (!strcmp(name_node->name, name))
334	return name_node;
335	return NULL;
336	}
337
338	bool netdev_name_in_use(struct net net, const char name)
339	{
340	return netdev_name_node_lookup(net, name);
341	}
342	EXPORT_SYMBOL(netdev_name_in_use);
343
344	int netdev_name_node_alt_create(struct net_device dev, const char name)
345	{
346	struct netdev_name_node *name_node;
347	struct net *net = dev_net(dev);
348
349	name_node = netdev_name_node_lookup(net, name);
350	if (name_node)
351	return -EEXIST;
352	name_node = netdev_name_node_alloc(dev, name);
353	if (!name_node)
354	return -ENOMEM;
355	netdev_name_node_add(net, name_node);
356	/* The node that holds dev->name acts as a head of per-device list. */
357	list_add_tail_rcu(&name_node->list, &dev->name_node->list);
358
359	return 0;
360	}
361
362	static void netdev_name_node_alt_free(struct rcu_head *head)
363	{
364	struct netdev_name_node *name_node =
365	container_of(head, struct netdev_name_node, rcu);
366
367	kfree(name_node->name);
368	netdev_name_node_free(name_node);
369	}
370
371	static void __netdev_name_node_alt_destroy(struct netdev_name_node *name_node)
372	{
373	netdev_name_node_del(name_node);
374	list_del_rcu(&name_node->list);
375	call_rcu(&name_node->rcu, netdev_name_node_alt_free);
376	}
377
378	int netdev_name_node_alt_destroy(struct net_device dev, const char name)
379	{
380	struct netdev_name_node *name_node;
381	struct net *net = dev_net(dev);
382
383	name_node = netdev_name_node_lookup(net, name);
384	if (!name_node)
385	return -ENOENT;
386	/* lookup might have found our primary name or a name belonging
387	* to another device.
388	*/
389	if (name_node == dev->name_node \|\| name_node->dev != dev)
390	return -EINVAL;
391
392	__netdev_name_node_alt_destroy(name_node);
393	return 0;
394	}
395
396	static void netdev_name_node_alt_flush(struct net_device *dev)
397	{
398	struct netdev_name_node name_node, tmp;
399
400	list_for_each_entry_safe(name_node, tmp, &dev->name_node->list, list) {
401	list_del(&name_node->list);
402	netdev_name_node_alt_free(&name_node->rcu);
403	}
404	}
405
406	/* Device list insertion */
407	static void list_netdevice(struct net_device *dev)
408	{
409	struct netdev_name_node *name_node;
410	struct net *net = dev_net(dev);
411
412	ASSERT_RTNL();
413
414	list_add_tail_rcu(&dev->dev_list, &net->dev_base_head);
415	netdev_name_node_add(net, dev->name_node);
416	hlist_add_head_rcu(&dev->index_hlist,
417	dev_index_hash(net, dev->ifindex));
418
419	netdev_for_each_altname(dev, name_node)
420	netdev_name_node_add(net, name_node);
421
422	/* We reserved the ifindex, this can't fail */
423	WARN_ON(xa_store(&net->dev_by_index, dev->ifindex, dev, GFP_KERNEL));
424
425	dev_base_seq_inc(net);
426	}
427
428	/* Device list removal
429	* caller must respect a RCU grace period before freeing/reusing dev
430	*/
431	static void unlist_netdevice(struct net_device *dev)
432	{
433	struct netdev_name_node *name_node;
434	struct net *net = dev_net(dev);
435
436	ASSERT_RTNL();
437
438	xa_erase(&net->dev_by_index, dev->ifindex);
439
440	netdev_for_each_altname(dev, name_node)
441	netdev_name_node_del(name_node);
442
443	/* Unlink dev from the device chain */
444	list_del_rcu(&dev->dev_list);
445	netdev_name_node_del(dev->name_node);
446	hlist_del_rcu(&dev->index_hlist);
447
448	dev_base_seq_inc(dev_net(dev));
449	}
450
451	/*
452	* Our notifier list
453	*/
454
455	static RAW_NOTIFIER_HEAD(netdev_chain);
456
457	/*
458	* Device drivers call our routines to queue packets here. We empty the
459	* queue in the local softnet handler.
460	*/
461
462	DEFINE_PER_CPU_ALIGNED(struct softnet_data, softnet_data) = {
463	.process_queue_bh_lock = INIT_LOCAL_LOCK(process_queue_bh_lock),
464	};
465	EXPORT_PER_CPU_SYMBOL(softnet_data);
466
467	/* Page_pool has a lockless array/stack to alloc/recycle pages.
468	* PP consumers must pay attention to run APIs in the appropriate context
469	* (e.g. NAPI context).
470	*/
471	DEFINE_PER_CPU(struct page_pool_bh, system_page_pool) = {
472	.bh_lock = INIT_LOCAL_LOCK(bh_lock),
473	};
474
475	#ifdef CONFIG_LOCKDEP
476	/*
477	* register_netdevice() inits txq->_xmit_lock and sets lockdep class
478	* according to dev->type
479	*/
480	static const unsigned short netdev_lock_type[] = {
481	ARPHRD_NETROM, ARPHRD_ETHER, ARPHRD_EETHER, ARPHRD_AX25,
482	ARPHRD_PRONET, ARPHRD_CHAOS, ARPHRD_IEEE802, ARPHRD_ARCNET,
483	ARPHRD_APPLETLK, ARPHRD_DLCI, ARPHRD_ATM, ARPHRD_METRICOM,
484	ARPHRD_IEEE1394, ARPHRD_EUI64, ARPHRD_INFINIBAND, ARPHRD_SLIP,
485	ARPHRD_CSLIP, ARPHRD_SLIP6, ARPHRD_CSLIP6, ARPHRD_RSRVD,
486	ARPHRD_ADAPT, ARPHRD_ROSE, ARPHRD_X25, ARPHRD_HWX25,
487	ARPHRD_CAN, ARPHRD_MCTP,
488	ARPHRD_PPP, ARPHRD_CISCO, ARPHRD_LAPB, ARPHRD_DDCMP,
489	ARPHRD_RAWHDLC, ARPHRD_RAWIP,
490	ARPHRD_TUNNEL, ARPHRD_TUNNEL6, ARPHRD_FRAD,
491	ARPHRD_SKIP, ARPHRD_LOOPBACK, ARPHRD_LOCALTLK, ARPHRD_FDDI,
492	ARPHRD_BIF, ARPHRD_SIT, ARPHRD_IPDDP, ARPHRD_IPGRE,
493	ARPHRD_PIMREG, ARPHRD_HIPPI, ARPHRD_ASH, ARPHRD_ECONET,
494	ARPHRD_IRDA, ARPHRD_FCPP, ARPHRD_FCAL, ARPHRD_FCPL,
495	ARPHRD_FCFABRIC, ARPHRD_IEEE80211, ARPHRD_IEEE80211_PRISM,
496	ARPHRD_IEEE80211_RADIOTAP,
497	ARPHRD_IEEE802154, ARPHRD_IEEE802154_MONITOR,
498	ARPHRD_PHONET, ARPHRD_PHONET_PIPE,
499	ARPHRD_CAIF, ARPHRD_IP6GRE, ARPHRD_NETLINK, ARPHRD_6LOWPAN,
500	ARPHRD_VSOCKMON,
501	ARPHRD_VOID, ARPHRD_NONE};
502
503	static const char *const netdev_lock_name[] = {
504	"_xmit_NETROM", "_xmit_ETHER", "_xmit_EETHER", "_xmit_AX25",
505	"_xmit_PRONET", "_xmit_CHAOS", "_xmit_IEEE802", "_xmit_ARCNET",
506	"_xmit_APPLETLK", "_xmit_DLCI", "_xmit_ATM", "_xmit_METRICOM",
507	"_xmit_IEEE1394", "_xmit_EUI64", "_xmit_INFINIBAND", "_xmit_SLIP",
508	"_xmit_CSLIP", "_xmit_SLIP6", "_xmit_CSLIP6", "_xmit_RSRVD",
509	"_xmit_ADAPT", "_xmit_ROSE", "_xmit_X25", "_xmit_HWX25",
510	"_xmit_CAN", "_xmit_MCTP",
511	"_xmit_PPP", "_xmit_CISCO", "_xmit_LAPB", "_xmit_DDCMP",
512	"_xmit_RAWHDLC", "_xmit_RAWIP",
513	"_xmit_TUNNEL", "_xmit_TUNNEL6", "_xmit_FRAD",
514	"_xmit_SKIP", "_xmit_LOOPBACK", "_xmit_LOCALTLK", "_xmit_FDDI",
515	"_xmit_BIF", "_xmit_SIT", "_xmit_IPDDP", "_xmit_IPGRE",
516	"_xmit_PIMREG", "_xmit_HIPPI", "_xmit_ASH", "_xmit_ECONET",
517	"_xmit_IRDA", "_xmit_FCPP", "_xmit_FCAL", "_xmit_FCPL",
518	"_xmit_FCFABRIC", "_xmit_IEEE80211", "_xmit_IEEE80211_PRISM",
519	"_xmit_IEEE80211_RADIOTAP",
520	"_xmit_IEEE802154", "_xmit_IEEE802154_MONITOR",
521	"_xmit_PHONET", "_xmit_PHONET_PIPE",
522	"_xmit_CAIF", "_xmit_IP6GRE", "_xmit_NETLINK", "_xmit_6LOWPAN",
523	"_xmit_VSOCKMON",
524	"_xmit_VOID", "_xmit_NONE"};
525
526	static struct lock_class_key netdev_xmit_lock_key[ARRAY_SIZE(netdev_lock_type)];
527	static struct lock_class_key netdev_addr_lock_key[ARRAY_SIZE(netdev_lock_type)];
528
529	static inline unsigned short netdev_lock_pos(unsigned short dev_type)
530	{
531	int i;
532
533	for (i = 0; i < ARRAY_SIZE(netdev_lock_type); i++)
534	if (netdev_lock_type[i] == dev_type)
535	return i;
536	/* the last key is used by default */
537	WARN_ONCE(1, "netdev_lock_pos() could not find dev_type=%u\n", dev_type);
538	return ARRAY_SIZE(netdev_lock_type) - 1;
539	}
540
541	static inline void netdev_set_xmit_lockdep_class(spinlock_t *lock,
542	unsigned short dev_type)
543	{
544	int i;
545
546	i = netdev_lock_pos(dev_type);
547	lockdep_set_class_and_name(lock, &netdev_xmit_lock_key[i],
548	netdev_lock_name[i]);
549	}
550
551	static inline void netdev_set_addr_lockdep_class(struct net_device *dev)
552	{
553	int i;
554
555	i = netdev_lock_pos(dev->type);
556	lockdep_set_class_and_name(&dev->addr_list_lock,
557	&netdev_addr_lock_key[i],
558	netdev_lock_name[i]);
559	}
560	#else
561	static inline void netdev_set_xmit_lockdep_class(spinlock_t *lock,
562	unsigned short dev_type)
563	{
564	}
565
566	static inline void netdev_set_addr_lockdep_class(struct net_device *dev)
567	{
568	}
569	#endif
570
571	/*******************************************************************************
572	*
573	* Protocol management and registration routines
574	*
575	*******************************************************************************/
576
577
578	/*
579	* Add a protocol ID to the list. Now that the input handler is
580	* smarter we can dispense with all the messy stuff that used to be
581	* here.
582	*
583	* BEWARE!!! Protocol handlers, mangling input packets,
584	* MUST BE last in hash buckets and checking protocol handlers
585	* MUST start from promiscuous ptype_all chain in net_bh.
586	* It is true now, do not change it.
587	* Explanation follows: if protocol handler, mangling packet, will
588	* be the first on list, it is not able to sense, that packet
589	* is cloned and should be copied-on-write, so that it will
590	* change it and subsequent readers will get broken packet.
591	* --ANK (980803)
592	*/
593
594	static inline struct list_head ptype_head(const struct packet_type pt)
595	{
596	if (pt->type == htons(ETH_P_ALL)) {
597	if (!pt->af_packet_net && !pt->dev)
598	return NULL;
599
600	return pt->dev ? &pt->dev->ptype_all :
601	&pt->af_packet_net->ptype_all;
602	}
603
604	if (pt->dev)
605	return &pt->dev->ptype_specific;
606
607	return pt->af_packet_net ? &pt->af_packet_net->ptype_specific :
608	&ptype_base[ntohs(pt->type) & PTYPE_HASH_MASK];
609	}
610
611	/**
612	* dev_add_pack - add packet handler
613	* @pt: packet type declaration
614	*
615	* Add a protocol handler to the networking stack. The passed &packet_type
616	* is linked into kernel lists and may not be freed until it has been
617	* removed from the kernel lists.
618	*
619	* This call does not sleep therefore it can not
620	* guarantee all CPU's that are in middle of receiving packets
621	* will see the new packet type (until the next received packet).
622	*/
623
624	void dev_add_pack(struct packet_type *pt)
625	{
626	struct list_head *head = ptype_head(pt);
627
628	if (WARN_ON_ONCE(!head))
629	return;
630
631	spin_lock(&ptype_lock);
632	list_add_rcu(&pt->list, head);
633	spin_unlock(&ptype_lock);
634	}
635	EXPORT_SYMBOL(dev_add_pack);
636
637	/**
638	* __dev_remove_pack - remove packet handler
639	* @pt: packet type declaration
640	*
641	* Remove a protocol handler that was previously added to the kernel
642	* protocol handlers by dev_add_pack(). The passed &packet_type is removed
643	* from the kernel lists and can be freed or reused once this function
644	* returns.
645	*
646	* The packet type might still be in use by receivers
647	* and must not be freed until after all the CPU's have gone
648	* through a quiescent state.
649	*/
650	void __dev_remove_pack(struct packet_type *pt)
651	{
652	struct list_head *head = ptype_head(pt);
653	struct packet_type *pt1;
654
655	if (!head)
656	return;
657
658	spin_lock(&ptype_lock);
659
660	list_for_each_entry(pt1, head, list) {
661	if (pt == pt1) {
662	list_del_rcu(&pt->list);
663	goto out;
664	}
665	}
666
667	pr_warn("dev_remove_pack: %p not found\n", pt);
668	out:
669	spin_unlock(&ptype_lock);
670	}
671	EXPORT_SYMBOL(__dev_remove_pack);
672
673	/**
674	* dev_remove_pack - remove packet handler
675	* @pt: packet type declaration
676	*
677	* Remove a protocol handler that was previously added to the kernel
678	* protocol handlers by dev_add_pack(). The passed &packet_type is removed
679	* from the kernel lists and can be freed or reused once this function
680	* returns.
681	*
682	* This call sleeps to guarantee that no CPU is looking at the packet
683	* type after return.
684	*/
685	void dev_remove_pack(struct packet_type *pt)
686	{
687	__dev_remove_pack(pt);
688
689	synchronize_net();
690	}
691	EXPORT_SYMBOL(dev_remove_pack);
692
693
694	/*******************************************************************************
695	*
696	* Device Interface Subroutines
697	*
698	*******************************************************************************/
699
700	/**
701	* dev_get_iflink - get 'iflink' value of a interface
702	* @dev: targeted interface
703	*
704	* Indicates the ifindex the interface is linked to.
705	* Physical interfaces have the same 'ifindex' and 'iflink' values.
706	*/
707
708	int dev_get_iflink(const struct net_device *dev)
709	{
710	if (dev->netdev_ops && dev->netdev_ops->ndo_get_iflink)
711	return dev->netdev_ops->ndo_get_iflink(dev);
712
713	return READ_ONCE(dev->ifindex);
714	}
715	EXPORT_SYMBOL(dev_get_iflink);
716
717	/**
718	* dev_fill_metadata_dst - Retrieve tunnel egress information.
719	* @dev: targeted interface
720	* @skb: The packet.
721	*
722	* For better visibility of tunnel traffic OVS needs to retrieve
723	* egress tunnel information for a packet. Following API allows
724	* user to get this info.
725	*/
726	int dev_fill_metadata_dst(struct net_device dev, struct sk_buff skb)
727	{
728	struct ip_tunnel_info *info;
729
730	if (!dev->netdev_ops \|\| !dev->netdev_ops->ndo_fill_metadata_dst)
731	return -EINVAL;
732
733	info = skb_tunnel_info_unclone(skb);
734	if (!info)
735	return -ENOMEM;
736	if (unlikely(!(info->mode & IP_TUNNEL_INFO_TX)))
737	return -EINVAL;
738
739	return dev->netdev_ops->ndo_fill_metadata_dst(dev, skb);
740	}
741	EXPORT_SYMBOL_GPL(dev_fill_metadata_dst);
742
743	static struct net_device_path dev_fwd_path(struct net_device_path_stack stack)
744	{
745	if (stack->num_paths + 1 > NET_DEVICE_PATH_STACK_MAX)
746	return NULL;
747
748	return &stack->path[stack->num_paths];
749	}
750
751	void dev_fill_forward_path_release(struct net_device_path_stack *stack)
752	{
753	struct net_device_path *path;
754	int k;
755
756	if (stack->num_paths == 0)
757	return;
758
759	for (k = stack->num_paths - 1; k >= 0; k--) {
760	path = &stack->path[k];
761	switch (path->type) {
762	case DEV_PATH_TUN:
763	dst_release(path->tun.dst);
764	break;
765	default:
766	break;
767	}
768	}
769	}
770	EXPORT_SYMBOL_GPL(dev_fill_forward_path_release);
771
772	int dev_fill_forward_path(struct net_device_path_ctx *ctx,
773	struct net_device_path_stack *stack)
774	{
775	const struct net_device *last_dev;
776	struct net_device_path *path;
777	int ret = 0;
778
779	stack->num_paths = 0;
780	while (ctx->dev && ctx->dev->netdev_ops->ndo_fill_forward_path) {
781	last_dev = ctx->dev;
782	path = dev_fwd_path(stack);
783	if (!path)
784	goto err_out;
785
786	memset(path, 0, sizeof(struct net_device_path));
787	ret = ctx->dev->netdev_ops->ndo_fill_forward_path(ctx, path);
788	if (ret < 0)
789	goto err_out;
790
791	stack->num_paths++;
792	if (last_dev == ctx->dev)
793	goto err_out;
794	}
795
796	if (!ctx->dev)
797	return ret;
798
799	path = dev_fwd_path(stack);
800	if (!path)
801	goto err_out;
802
803	path->type = DEV_PATH_ETHERNET;
804	path->dev = ctx->dev;
805	stack->num_paths++;
806
807	return 0;
808	err_out:
809	dev_fill_forward_path_release(stack);
810
811	return -1;
812	}
813	EXPORT_SYMBOL_GPL(dev_fill_forward_path);
814
815	/* must be called under rcu_read_lock(), as we dont take a reference */
816	static struct napi_struct *napi_by_id(unsigned int napi_id)
817	{
818	unsigned int hash = napi_id % HASH_SIZE(napi_hash);
819	struct napi_struct *napi;
820
821	hlist_for_each_entry_rcu(napi, &napi_hash[hash], napi_hash_node)
822	if (napi->napi_id == napi_id)
823	return napi;
824
825	return NULL;
826	}
827
828	/* must be called under rcu_read_lock(), as we dont take a reference */
829	static struct napi_struct *
830	netdev_napi_by_id(struct net *net, unsigned int napi_id)
831	{
832	struct napi_struct *napi;
833
834	napi = napi_by_id(napi_id);
835	if (!napi)
836	return NULL;
837
838	if (WARN_ON_ONCE(!napi->dev))
839	return NULL;
840	if (!net_eq(net, dev_net(napi->dev)))
841	return NULL;
842
843	return napi;
844	}
845
846	/**
847	* netdev_napi_by_id_lock() - find a device by NAPI ID and lock it
848	* @net: the applicable net namespace
849	* @napi_id: ID of a NAPI of a target device
850	*
851	* Find a NAPI instance with @napi_id. Lock its device.
852	* The device must be in %NETREG_REGISTERED state for lookup to succeed.
853	* netdev_unlock() must be called to release it.
854	*
855	* Return: pointer to NAPI, its device with lock held, NULL if not found.
856	*/
857	struct napi_struct *
858	netdev_napi_by_id_lock(struct net *net, unsigned int napi_id)
859	{
860	struct napi_struct *napi;
861	struct net_device *dev;
862
863	rcu_read_lock();
864	napi = netdev_napi_by_id(net, napi_id);
865	if (!napi \|\| READ_ONCE(napi->dev->reg_state) != NETREG_REGISTERED) {
866	rcu_read_unlock();
867	return NULL;
868	}
869
870	dev = napi->dev;
871	dev_hold(dev);
872	rcu_read_unlock();
873
874	dev = __netdev_put_lock(dev, net);
875	if (!dev)
876	return NULL;
877
878	rcu_read_lock();
879	napi = netdev_napi_by_id(net, napi_id);
880	if (napi && napi->dev != dev)
881	napi = NULL;
882	rcu_read_unlock();
883
884	if (!napi)
885	netdev_unlock(dev);
886	return napi;
887	}
888
889	/**
890	* __dev_get_by_name - find a device by its name
891	* @net: the applicable net namespace
892	* @name: name to find
893	*
894	* Find an interface by name. Must be called under RTNL semaphore.
895	* If the name is found a pointer to the device is returned.
896	* If the name is not found then %NULL is returned. The
897	* reference counters are not incremented so the caller must be
898	* careful with locks.
899	*/
900
901	struct net_device __dev_get_by_name(struct net net, const char *name)
902	{
903	struct netdev_name_node *node_name;
904
905	node_name = netdev_name_node_lookup(net, name);
906	return node_name ? node_name->dev : NULL;
907	}
908	EXPORT_SYMBOL(__dev_get_by_name);
909
910	/**
911	* dev_get_by_name_rcu - find a device by its name
912	* @net: the applicable net namespace
913	* @name: name to find
914	*
915	* Find an interface by name.
916	* If the name is found a pointer to the device is returned.
917	* If the name is not found then %NULL is returned.
918	* The reference counters are not incremented so the caller must be
919	* careful with locks. The caller must hold RCU lock.
920	*/
921
922	struct net_device dev_get_by_name_rcu(struct net net, const char *name)
923	{
924	struct netdev_name_node *node_name;
925
926	node_name = netdev_name_node_lookup_rcu(net, name);
927	return node_name ? node_name->dev : NULL;
928	}
929	EXPORT_SYMBOL(dev_get_by_name_rcu);
930
931	/* Deprecated for new users, call netdev_get_by_name() instead */
932	struct net_device dev_get_by_name(struct net net, const char *name)
933	{
934	struct net_device *dev;
935
936	rcu_read_lock();
937	dev = dev_get_by_name_rcu(net, name);
938	dev_hold(dev);
939	rcu_read_unlock();
940	return dev;
941	}
942	EXPORT_SYMBOL(dev_get_by_name);
943
944	/**
945	* netdev_get_by_name() - find a device by its name
946	* @net: the applicable net namespace
947	* @name: name to find
948	* @tracker: tracking object for the acquired reference
949	* @gfp: allocation flags for the tracker
950	*
951	* Find an interface by name. This can be called from any
952	* context and does its own locking. The returned handle has
953	* the usage count incremented and the caller must use netdev_put() to
954	* release it when it is no longer needed. %NULL is returned if no
955	* matching device is found.
956	*/
957	struct net_device netdev_get_by_name(struct net net, const char *name,
958	netdevice_tracker *tracker, gfp_t gfp)
959	{
960	struct net_device *dev;
961
962	dev = dev_get_by_name(net, name);
963	if (dev)
964	netdev_tracker_alloc(dev, tracker, gfp);
965	return dev;
966	}
967	EXPORT_SYMBOL(netdev_get_by_name);
968
969	/**
970	* __dev_get_by_index - find a device by its ifindex
971	* @net: the applicable net namespace
972	* @ifindex: index of device
973	*
974	* Search for an interface by index. Returns %NULL if the device
975	* is not found or a pointer to the device. The device has not
976	* had its reference counter increased so the caller must be careful
977	* about locking. The caller must hold the RTNL semaphore.
978	*/
979
980	struct net_device __dev_get_by_index(struct net net, int ifindex)
981	{
982	struct net_device *dev;
983	struct hlist_head *head = dev_index_hash(net, ifindex);
984
985	hlist_for_each_entry(dev, head, index_hlist)
986	if (dev->ifindex == ifindex)
987	return dev;
988
989	return NULL;
990	}
991	EXPORT_SYMBOL(__dev_get_by_index);
992
993	/**
994	* dev_get_by_index_rcu - find a device by its ifindex
995	* @net: the applicable net namespace
996	* @ifindex: index of device
997	*
998	* Search for an interface by index. Returns %NULL if the device
999	* is not found or a pointer to the device. The device has not
1000	* had its reference counter increased so the caller must be careful
1001	* about locking. The caller must hold RCU lock.
1002	*/
1003
1004	struct net_device dev_get_by_index_rcu(struct net net, int ifindex)
1005	{
1006	struct net_device *dev;
1007	struct hlist_head *head = dev_index_hash(net, ifindex);
1008
1009	hlist_for_each_entry_rcu(dev, head, index_hlist)
1010	if (dev->ifindex == ifindex)
1011	return dev;
1012
1013	return NULL;
1014	}
1015	EXPORT_SYMBOL(dev_get_by_index_rcu);
1016
1017	/* Deprecated for new users, call netdev_get_by_index() instead */
1018	struct net_device dev_get_by_index(struct net net, int ifindex)
1019	{
1020	struct net_device *dev;
1021
1022	rcu_read_lock();
1023	dev = dev_get_by_index_rcu(net, ifindex);
1024	dev_hold(dev);
1025	rcu_read_unlock();
1026	return dev;
1027	}
1028	EXPORT_SYMBOL(dev_get_by_index);
1029
1030	/**
1031	* netdev_get_by_index() - find a device by its ifindex
1032	* @net: the applicable net namespace
1033	* @ifindex: index of device
1034	* @tracker: tracking object for the acquired reference
1035	* @gfp: allocation flags for the tracker
1036	*
1037	* Search for an interface by index. Returns NULL if the device
1038	* is not found or a pointer to the device. The device returned has
1039	* had a reference added and the pointer is safe until the user calls
1040	* netdev_put() to indicate they have finished with it.
1041	*/
1042	struct net_device netdev_get_by_index(struct net net, int ifindex,
1043	netdevice_tracker *tracker, gfp_t gfp)
1044	{
1045	struct net_device *dev;
1046
1047	dev = dev_get_by_index(net, ifindex);
1048	if (dev)
1049	netdev_tracker_alloc(dev, tracker, gfp);
1050	return dev;
1051	}
1052	EXPORT_SYMBOL(netdev_get_by_index);
1053
1054	/**
1055	* dev_get_by_napi_id - find a device by napi_id
1056	* @napi_id: ID of the NAPI struct
1057	*
1058	* Search for an interface by NAPI ID. Returns %NULL if the device
1059	* is not found or a pointer to the device. The device has not had
1060	* its reference counter increased so the caller must be careful
1061	* about locking. The caller must hold RCU lock.
1062	*/
1063	struct net_device *dev_get_by_napi_id(unsigned int napi_id)
1064	{
1065	struct napi_struct *napi;
1066
1067	WARN_ON_ONCE(!rcu_read_lock_held());
1068
1069	if (!napi_id_valid(napi_id))
1070	return NULL;
1071
1072	napi = napi_by_id(napi_id);
1073
1074	return napi ? napi->dev : NULL;
1075	}
1076
1077	/* Release the held reference on the net_device, and if the net_device
1078	* is still registered try to lock the instance lock. If device is being
1079	* unregistered NULL will be returned (but the reference has been released,
1080	* either way!)
1081	*
1082	* This helper is intended for locking net_device after it has been looked up
1083	* using a lockless lookup helper. Lock prevents the instance from going away.
1084	*/
1085	struct net_device *
1086	netdev_put_lock(struct net_device dev, struct net net,
1087	netdevice_tracker *tracker)
1088	{
1089	netdev_lock(dev);
1090	if (dev->reg_state > NETREG_REGISTERED \|\|
1091	dev->moving_ns \|\| !net_eq(dev_net(dev), net)) {
1092	netdev_unlock(dev);
1093	netdev_put(dev, tracker);
1094	return NULL;
1095	}
1096	netdev_put(dev, tracker);
1097	return dev;
1098	}
1099
1100	static struct net_device *
1101	__netdev_put_lock_ops_compat(struct net_device dev, struct net net)
1102	{
1103	netdev_lock_ops_compat(dev);
1104	if (dev->reg_state > NETREG_REGISTERED \|\|
1105	dev->moving_ns \|\| !net_eq(dev_net(dev), net)) {
1106	netdev_unlock_ops_compat(dev);
1107	dev_put(dev);
1108	return NULL;
1109	}
1110	dev_put(dev);
1111	return dev;
1112	}
1113
1114	/**
1115	* netdev_get_by_index_lock() - find a device by its ifindex
1116	* @net: the applicable net namespace
1117	* @ifindex: index of device
1118	*
1119	* Search for an interface by index. If a valid device
1120	* with @ifindex is found it will be returned with netdev->lock held.
1121	* netdev_unlock() must be called to release it.
1122	*
1123	* Return: pointer to a device with lock held, NULL if not found.
1124	*/
1125	struct net_device netdev_get_by_index_lock(struct net net, int ifindex)
1126	{
1127	struct net_device *dev;
1128
1129	dev = dev_get_by_index(net, ifindex);
1130	if (!dev)
1131	return NULL;
1132
1133	return __netdev_put_lock(dev, net);
1134	}
1135
1136	struct net_device *
1137	netdev_get_by_index_lock_ops_compat(struct net *net, int ifindex)
1138	{
1139	struct net_device *dev;
1140
1141	dev = dev_get_by_index(net, ifindex);
1142	if (!dev)
1143	return NULL;
1144
1145	return __netdev_put_lock_ops_compat(dev, net);
1146	}
1147
1148	struct net_device *
1149	netdev_xa_find_lock(struct net net, struct net_device dev,
1150	unsigned long *index)
1151	{
1152	if (dev)
1153	netdev_unlock(dev);
1154
1155	do {
1156	rcu_read_lock();
1157	dev = xa_find(&net->dev_by_index, index, ULONG_MAX, XA_PRESENT);
1158	if (!dev) {
1159	rcu_read_unlock();
1160	return NULL;
1161	}
1162	dev_hold(dev);
1163	rcu_read_unlock();
1164
1165	dev = __netdev_put_lock(dev, net);
1166	if (dev)
1167	return dev;
1168
1169	(*index)++;
1170	} while (true);
1171	}
1172
1173	struct net_device *
1174	netdev_xa_find_lock_ops_compat(struct net net, struct net_device dev,
1175	unsigned long *index)
1176	{
1177	if (dev)
1178	netdev_unlock_ops_compat(dev);
1179
1180	do {
1181	rcu_read_lock();
1182	dev = xa_find(&net->dev_by_index, index, ULONG_MAX, XA_PRESENT);
1183	if (!dev) {
1184	rcu_read_unlock();
1185	return NULL;
1186	}
1187	dev_hold(dev);
1188	rcu_read_unlock();
1189
1190	dev = __netdev_put_lock_ops_compat(dev, net);
1191	if (dev)
1192	return dev;
1193
1194	(*index)++;
1195	} while (true);
1196	}
1197
1198	static DEFINE_SEQLOCK(netdev_rename_lock);
1199
1200	void netdev_copy_name(struct net_device dev, char name)
1201	{
1202	unsigned int seq;
1203
1204	do {
1205	seq = read_seqbegin(&netdev_rename_lock);
1206	strscpy(name, dev->name, IFNAMSIZ);
1207	} while (read_seqretry(&netdev_rename_lock, seq));
1208	}
1209
1210	/**
1211	* netdev_get_name - get a netdevice name, knowing its ifindex.
1212	* @net: network namespace
1213	* @name: a pointer to the buffer where the name will be stored.
1214	* @ifindex: the ifindex of the interface to get the name from.
1215	*/
1216	int netdev_get_name(struct net net, char name, int ifindex)
1217	{
1218	struct net_device *dev;
1219	int ret;
1220
1221	rcu_read_lock();
1222
1223	dev = dev_get_by_index_rcu(net, ifindex);
1224	if (!dev) {
1225	ret = -ENODEV;
1226	goto out;
1227	}
1228
1229	netdev_copy_name(dev, name);
1230
1231	ret = 0;
1232	out:
1233	rcu_read_unlock();
1234	return ret;
1235	}
1236
1237	static bool dev_addr_cmp(struct net_device *dev, unsigned short type,
1238	const char *ha)
1239	{
1240	return dev->type == type && !memcmp(dev->dev_addr, ha, dev->addr_len);
1241	}
1242
1243	/**
1244	* dev_getbyhwaddr_rcu - find a device by its hardware address
1245	* @net: the applicable net namespace
1246	* @type: media type of device
1247	* @ha: hardware address
1248	*
1249	* Search for an interface by MAC address. Returns NULL if the device
1250	* is not found or a pointer to the device.
1251	* The caller must hold RCU.
1252	* The returned device has not had its ref count increased
1253	* and the caller must therefore be careful about locking
1254	*
1255	*/
1256
1257	struct net_device dev_getbyhwaddr_rcu(struct net net, unsigned short type,
1258	const char *ha)
1259	{
1260	struct net_device *dev;
1261
1262	for_each_netdev_rcu(net, dev)
1263	if (dev_addr_cmp(dev, type, ha))
1264	return dev;
1265
1266	return NULL;
1267	}
1268	EXPORT_SYMBOL(dev_getbyhwaddr_rcu);
1269
1270	/**
1271	* dev_getbyhwaddr() - find a device by its hardware address
1272	* @net: the applicable net namespace
1273	* @type: media type of device
1274	* @ha: hardware address
1275	*
1276	* Similar to dev_getbyhwaddr_rcu(), but the owner needs to hold
1277	* rtnl_lock.
1278	*
1279	* Context: rtnl_lock() must be held.
1280	* Return: pointer to the net_device, or NULL if not found
1281	*/
1282	struct net_device dev_getbyhwaddr(struct net net, unsigned short type,
1283	const char *ha)
1284	{
1285	struct net_device *dev;
1286
1287	ASSERT_RTNL();
1288	for_each_netdev(net, dev)
1289	if (dev_addr_cmp(dev, type, ha))
1290	return dev;
1291
1292	return NULL;
1293	}
1294	EXPORT_SYMBOL(dev_getbyhwaddr);
1295
1296	struct net_device dev_getfirstbyhwtype(struct net net, unsigned short type)
1297	{
1298	struct net_device dev, ret = NULL;
1299
1300	rcu_read_lock();
1301	for_each_netdev_rcu(net, dev)
1302	if (dev->type == type) {
1303	dev_hold(dev);
1304	ret = dev;
1305	break;
1306	}
1307	rcu_read_unlock();
1308	return ret;
1309	}
1310	EXPORT_SYMBOL(dev_getfirstbyhwtype);
1311
1312	/**
1313	* netdev_get_by_flags_rcu - find any device with given flags
1314	* @net: the applicable net namespace
1315	* @tracker: tracking object for the acquired reference
1316	* @if_flags: IFF_* values
1317	* @mask: bitmask of bits in if_flags to check
1318	*
1319	* Search for any interface with the given flags.
1320	*
1321	* Context: rcu_read_lock() must be held.
1322	* Returns: NULL if a device is not found or a pointer to the device.
1323	*/
1324	struct net_device netdev_get_by_flags_rcu(struct net net, netdevice_tracker *tracker,
1325	unsigned short if_flags, unsigned short mask)
1326	{
1327	struct net_device *dev;
1328
1329	for_each_netdev_rcu(net, dev) {
1330	if (((READ_ONCE(dev->flags) ^ if_flags) & mask) == 0) {
1331	netdev_hold(dev, tracker, GFP_ATOMIC);
1332	return dev;
1333	}
1334	}
1335
1336	return NULL;
1337	}
1338
1339	/**
1340	* dev_valid_name - check if name is okay for network device
1341	* @name: name string
1342	*
1343	* Network device names need to be valid file names to
1344	* allow sysfs to work. We also disallow any kind of
1345	* whitespace.
1346	*/
1347	bool dev_valid_name(const char *name)
1348	{
1349	if (*name == '\0')
1350	return false;
1351	if (strnlen(name, IFNAMSIZ) == IFNAMSIZ)
1352	return false;
1353	if (!strcmp(name, ".") \|\| !strcmp(name, ".."))
1354	return false;
1355
1356	while (*name) {
1357	if (name == '/' \|\| name == ':' \|\| isspace(*name))
1358	return false;
1359	name++;
1360	}
1361	return true;
1362	}
1363	EXPORT_SYMBOL(dev_valid_name);
1364
1365	/**
1366	* __dev_alloc_name - allocate a name for a device
1367	* @net: network namespace to allocate the device name in
1368	* @name: name format string
1369	* @res: result name string
1370	*
1371	* Passed a format string - eg "lt%d" it will try and find a suitable
1372	* id. It scans list of devices to build up a free map, then chooses
1373	* the first empty slot. The caller must hold the dev_base or rtnl lock
1374	* while allocating the name and adding the device in order to avoid
1375	* duplicates.
1376	* Limited to bits_per_byte * page size devices (ie 32K on most platforms).
1377	* Returns the number of the unit assigned or a negative errno code.
1378	*/
1379
1380	static int __dev_alloc_name(struct net net, const char name, char *res)
1381	{
1382	int i = 0;
1383	const char *p;
1384	const int max_netdevices = 8*PAGE_SIZE;
1385	unsigned long *inuse;
1386	struct net_device *d;
1387	char buf[IFNAMSIZ];
1388
1389	/* Verify the string as this thing may have come from the user.
1390	* There must be one "%d" and no other "%" characters.
1391	*/
1392	p = strchr(name, '%');
1393	if (!p \|\| p[1] != 'd' \|\| strchr(p + 2, '%'))
1394	return -EINVAL;
1395
1396	/* Use one page as a bit array of possible slots */
1397	inuse = bitmap_zalloc(max_netdevices, GFP_ATOMIC);
1398	if (!inuse)
1399	return -ENOMEM;
1400
1401	for_each_netdev(net, d) {
1402	struct netdev_name_node *name_node;
1403
1404	netdev_for_each_altname(d, name_node) {
1405	if (!sscanf(name_node->name, name, &i))
1406	continue;
1407	if (i < 0 \|\| i >= max_netdevices)
1408	continue;
1409
1410	/* avoid cases where sscanf is not exact inverse of printf */
1411	snprintf(buf, IFNAMSIZ, name, i);
1412	if (!strncmp(buf, name_node->name, IFNAMSIZ))
1413	__set_bit(i, inuse);
1414	}
1415	if (!sscanf(d->name, name, &i))
1416	continue;
1417	if (i < 0 \|\| i >= max_netdevices)
1418	continue;
1419
1420	/* avoid cases where sscanf is not exact inverse of printf */
1421	snprintf(buf, IFNAMSIZ, name, i);
1422	if (!strncmp(buf, d->name, IFNAMSIZ))
1423	__set_bit(i, inuse);
1424	}
1425
1426	i = find_first_zero_bit(inuse, max_netdevices);
1427	bitmap_free(inuse);
1428	if (i == max_netdevices)
1429	return -ENFILE;
1430
1431	/* 'res' and 'name' could overlap, use 'buf' as an intermediate buffer */
1432	strscpy(buf, name, IFNAMSIZ);
1433	snprintf(res, IFNAMSIZ, buf, i);
1434	return i;
1435	}
1436
1437	/* Returns negative errno or allocated unit id (see __dev_alloc_name()) */
1438	static int dev_prep_valid_name(struct net net, struct net_device dev,
1439	const char want_name, char out_name,
1440	int dup_errno)
1441	{
1442	if (!dev_valid_name(want_name))
1443	return -EINVAL;
1444
1445	if (strchr(want_name, '%'))
1446	return __dev_alloc_name(net, want_name, out_name);
1447
1448	if (netdev_name_in_use(net, want_name))
1449	return -dup_errno;
1450	if (out_name != want_name)
1451	strscpy(out_name, want_name, IFNAMSIZ);
1452	return 0;
1453	}
1454
1455	/**
1456	* dev_alloc_name - allocate a name for a device
1457	* @dev: device
1458	* @name: name format string
1459	*
1460	* Passed a format string - eg "lt%d" it will try and find a suitable
1461	* id. It scans list of devices to build up a free map, then chooses
1462	* the first empty slot. The caller must hold the dev_base or rtnl lock
1463	* while allocating the name and adding the device in order to avoid
1464	* duplicates.
1465	* Limited to bits_per_byte * page size devices (ie 32K on most platforms).
1466	* Returns the number of the unit assigned or a negative errno code.
1467	*/
1468
1469	int dev_alloc_name(struct net_device dev, const char name)
1470	{
1471	return dev_prep_valid_name(dev_net(dev), dev, name, dev->name, ENFILE);
1472	}
1473	EXPORT_SYMBOL(dev_alloc_name);
1474
1475	static int dev_get_valid_name(struct net net, struct net_device dev,
1476	const char *name)
1477	{
1478	int ret;
1479
1480	ret = dev_prep_valid_name(net, dev, name, dev->name, EEXIST);
1481	return ret < 0 ? ret : 0;
1482	}
1483
1484	int netif_change_name(struct net_device dev, const char newname)
1485	{
1486	struct net *net = dev_net(dev);
1487	unsigned char old_assign_type;
1488	char oldname[IFNAMSIZ];
1489	int err = 0;
1490	int ret;
1491
1492	ASSERT_RTNL_NET(net);
1493
1494	if (!strncmp(newname, dev->name, IFNAMSIZ))
1495	return 0;
1496
1497	memcpy(oldname, dev->name, IFNAMSIZ);
1498
1499	write_seqlock_bh(&netdev_rename_lock);
1500	err = dev_get_valid_name(net, dev, newname);
1501	write_sequnlock_bh(&netdev_rename_lock);
1502
1503	if (err < 0)
1504	return err;
1505
1506	if (oldname[0] && !strchr(oldname, '%'))
1507	netdev_info(dev, "renamed from %s%s\n", oldname,
1508	dev->flags & IFF_UP ? " (while UP)" : "");
1509
1510	old_assign_type = dev->name_assign_type;
1511	WRITE_ONCE(dev->name_assign_type, NET_NAME_RENAMED);
1512
1513	rollback:
1514	ret = device_rename(&dev->dev, dev->name);
1515	if (ret) {
1516	write_seqlock_bh(&netdev_rename_lock);
1517	memcpy(dev->name, oldname, IFNAMSIZ);
1518	write_sequnlock_bh(&netdev_rename_lock);
1519	WRITE_ONCE(dev->name_assign_type, old_assign_type);
1520	return ret;
1521	}
1522
1523	netdev_adjacent_rename_links(dev, oldname);
1524
1525	netdev_name_node_del(dev->name_node);
1526
1527	synchronize_net();
1528
1529	netdev_name_node_add(net, dev->name_node);
1530
1531	ret = call_netdevice_notifiers(NETDEV_CHANGENAME, dev);
1532	ret = notifier_to_errno(ret);
1533
1534	if (ret) {
1535	/* err >= 0 after dev_alloc_name() or stores the first errno */
1536	if (err >= 0) {
1537	err = ret;
1538	write_seqlock_bh(&netdev_rename_lock);
1539	memcpy(dev->name, oldname, IFNAMSIZ);
1540	write_sequnlock_bh(&netdev_rename_lock);
1541	memcpy(oldname, newname, IFNAMSIZ);
1542	WRITE_ONCE(dev->name_assign_type, old_assign_type);
1543	old_assign_type = NET_NAME_RENAMED;
1544	goto rollback;
1545	} else {
1546	netdev_err(dev, "name change rollback failed: %d\n",
1547	ret);
1548	}
1549	}
1550
1551	return err;
1552	}
1553
1554	int netif_set_alias(struct net_device dev, const char alias, size_t len)
1555	{
1556	struct dev_ifalias *new_alias = NULL;
1557
1558	if (len >= IFALIASZ)
1559	return -EINVAL;
1560
1561	if (len) {
1562	new_alias = kmalloc(sizeof(*new_alias) + len + 1, GFP_KERNEL);
1563	if (!new_alias)
1564	return -ENOMEM;
1565
1566	memcpy(new_alias->ifalias, alias, len);
1567	new_alias->ifalias[len] = 0;
1568	}
1569
1570	mutex_lock(&ifalias_mutex);
1571	new_alias = rcu_replace_pointer(dev->ifalias, new_alias,
1572	mutex_is_locked(&ifalias_mutex));
1573	mutex_unlock(&ifalias_mutex);
1574
1575	if (new_alias)
1576	kfree_rcu(new_alias, rcuhead);
1577
1578	return len;
1579	}
1580
1581	/**
1582	* dev_get_alias - get ifalias of a device
1583	* @dev: device
1584	* @name: buffer to store name of ifalias
1585	* @len: size of buffer
1586	*
1587	* get ifalias for a device. Caller must make sure dev cannot go
1588	* away, e.g. rcu read lock or own a reference count to device.
1589	*/
1590	int dev_get_alias(const struct net_device dev, char name, size_t len)
1591	{
1592	const struct dev_ifalias *alias;
1593	int ret = 0;
1594
1595	rcu_read_lock();
1596	alias = rcu_dereference(dev->ifalias);
1597	if (alias)
1598	ret = snprintf(name, len, "%s", alias->ifalias);
1599	rcu_read_unlock();
1600
1601	return ret;
1602	}
1603
1604	/**
1605	* netdev_features_change - device changes features
1606	* @dev: device to cause notification
1607	*
1608	* Called to indicate a device has changed features.
1609	*/
1610	void netdev_features_change(struct net_device *dev)
1611	{
1612	call_netdevice_notifiers(NETDEV_FEAT_CHANGE, dev);
1613	}
1614	EXPORT_SYMBOL(netdev_features_change);
1615
1616	void netif_state_change(struct net_device *dev)
1617	{
1618	netdev_assert_locked_ops_compat_or_invisible(dev);
1619
1620	if (dev->flags & IFF_UP) {
1621	struct netdev_notifier_change_info change_info = {
1622	.info.dev = dev,
1623	};
1624
1625	call_netdevice_notifiers_info(NETDEV_CHANGE,
1626	&change_info.info);
1627	rtmsg_ifinfo(RTM_NEWLINK, dev, 0, GFP_KERNEL, 0, NULL);
1628	}
1629	}
1630
1631	/**
1632	* __netdev_notify_peers - notify network peers about existence of @dev,
1633	* to be called when rtnl lock is already held.
1634	* @dev: network device
1635	*
1636	* Generate traffic such that interested network peers are aware of
1637	* @dev, such as by generating a gratuitous ARP. This may be used when
1638	* a device wants to inform the rest of the network about some sort of
1639	* reconfiguration such as a failover event or virtual machine
1640	* migration.
1641	*/
1642	void __netdev_notify_peers(struct net_device *dev)
1643	{
1644	ASSERT_RTNL();
1645	call_netdevice_notifiers(NETDEV_NOTIFY_PEERS, dev);
1646	call_netdevice_notifiers(NETDEV_RESEND_IGMP, dev);
1647	}
1648	EXPORT_SYMBOL(__netdev_notify_peers);
1649
1650	/**
1651	* netdev_notify_peers - notify network peers about existence of @dev
1652	* @dev: network device
1653	*
1654	* Generate traffic such that interested network peers are aware of
1655	* @dev, such as by generating a gratuitous ARP. This may be used when
1656	* a device wants to inform the rest of the network about some sort of
1657	* reconfiguration such as a failover event or virtual machine
1658	* migration.
1659	*/
1660	void netdev_notify_peers(struct net_device *dev)
1661	{
1662	rtnl_lock();
1663	__netdev_notify_peers(dev);
1664	rtnl_unlock();
1665	}
1666	EXPORT_SYMBOL(netdev_notify_peers);
1667
1668	static int napi_threaded_poll(void *data);
1669
1670	static int napi_kthread_create(struct napi_struct *n)
1671	{
1672	int err = 0;
1673
1674	/* Create and wake up the kthread once to put it in
1675	* TASK_INTERRUPTIBLE mode to avoid the blocked task
1676	* warning and work with loadavg.
1677	*/
1678	n->thread = kthread_run(napi_threaded_poll, n, "napi/%s-%d",
1679	n->dev->name, n->napi_id);
1680	if (IS_ERR(n->thread)) {
1681	err = PTR_ERR(n->thread);
1682	pr_err("kthread_run failed with err %d\n", err);
1683	n->thread = NULL;
1684	}
1685
1686	return err;
1687	}
1688
1689	static int __dev_open(struct net_device dev, struct netlink_ext_ack extack)
1690	{
1691	const struct net_device_ops *ops = dev->netdev_ops;
1692	int ret;
1693
1694	ASSERT_RTNL();
1695	dev_addr_check(dev);
1696
1697	if (!netif_device_present(dev)) {
1698	/* may be detached because parent is runtime-suspended */
1699	if (dev->dev.parent)
1700	pm_runtime_resume(dev->dev.parent);
1701	if (!netif_device_present(dev))
1702	return -ENODEV;
1703	}
1704
1705	/* Block netpoll from trying to do any rx path servicing.
1706	* If we don't do this there is a chance ndo_poll_controller
1707	* or ndo_poll may be running while we open the device
1708	*/
1709	netpoll_poll_disable(dev);
1710
1711	ret = call_netdevice_notifiers_extack(NETDEV_PRE_UP, dev, extack);
1712	ret = notifier_to_errno(ret);
1713	if (ret)
1714	return ret;
1715
1716	set_bit(__LINK_STATE_START, &dev->state);
1717
1718	netdev_assert_locked_ops_compat(dev);
1719
1720	if (ops->ndo_validate_addr)
1721	ret = ops->ndo_validate_addr(dev);
1722
1723	if (!ret && ops->ndo_open)
1724	ret = ops->ndo_open(dev);
1725
1726	netpoll_poll_enable(dev);
1727
1728	if (ret)
1729	clear_bit(__LINK_STATE_START, &dev->state);
1730	else {
1731	netif_set_up(dev, true);
1732	dev_set_rx_mode(dev);
1733	dev_activate(dev);
1734	add_device_randomness(dev->dev_addr, dev->addr_len);
1735	}
1736
1737	return ret;
1738	}
1739
1740	int netif_open(struct net_device dev, struct netlink_ext_ack extack)
1741	{
1742	int ret;
1743
1744	if (dev->flags & IFF_UP)
1745	return 0;
1746
1747	ret = __dev_open(dev, extack);
1748	if (ret < 0)
1749	return ret;
1750
1751	rtmsg_ifinfo(RTM_NEWLINK, dev, IFF_UP \| IFF_RUNNING, GFP_KERNEL, 0, NULL);
1752	call_netdevice_notifiers(NETDEV_UP, dev);
1753
1754	return ret;
1755	}
1756	EXPORT_SYMBOL(netif_open);
1757
1758	static void __dev_close_many(struct list_head *head)
1759	{
1760	struct net_device *dev;
1761
1762	ASSERT_RTNL();
1763	might_sleep();
1764
1765	list_for_each_entry(dev, head, close_list) {
1766	/* Temporarily disable netpoll until the interface is down */
1767	netpoll_poll_disable(dev);
1768
1769	call_netdevice_notifiers(NETDEV_GOING_DOWN, dev);
1770
1771	clear_bit(__LINK_STATE_START, &dev->state);
1772
1773	/* Synchronize to scheduled poll. We cannot touch poll list, it
1774	* can be even on different cpu. So just clear netif_running().
1775	*
1776	* dev->stop() will invoke napi_disable() on all of it's
1777	* napi_struct instances on this device.
1778	*/
1779	smp_mb__after_atomic(); /* Commit netif_running(). */
1780	}
1781
1782	dev_deactivate_many(head, true);
1783
1784	list_for_each_entry(dev, head, close_list) {
1785	const struct net_device_ops *ops = dev->netdev_ops;
1786
1787	/*
1788	* Call the device specific close. This cannot fail.
1789	* Only if device is UP
1790	*
1791	* We allow it to be called even after a DETACH hot-plug
1792	* event.
1793	*/
1794
1795	netdev_assert_locked_ops_compat(dev);
1796
1797	if (ops->ndo_stop)
1798	ops->ndo_stop(dev);
1799
1800	netif_rx_mode_cancel_retry(dev);
1801	netif_set_up(dev, false);
1802	netpoll_poll_enable(dev);
1803	}
1804	}
1805
1806	static void __dev_close(struct net_device *dev)
1807	{
1808	LIST_HEAD(single);
1809
1810	list_add(&dev->close_list, &single);
1811	__dev_close_many(&single);
1812	list_del(&single);
1813	}
1814
1815	void netif_close_many(struct list_head *head, bool unlink)
1816	{
1817	struct net_device dev, tmp;
1818
1819	/* Remove the devices that don't need to be closed */
1820	list_for_each_entry_safe(dev, tmp, head, close_list)
1821	if (!(dev->flags & IFF_UP))
1822	list_del_init(&dev->close_list);
1823
1824	__dev_close_many(head);
1825
1826	list_for_each_entry_safe(dev, tmp, head, close_list) {
1827	netdev_assert_locked_ops_compat(dev);
1828	rtmsg_ifinfo(RTM_NEWLINK, dev, IFF_UP \| IFF_RUNNING, GFP_KERNEL, 0, NULL);
1829	call_netdevice_notifiers(NETDEV_DOWN, dev);
1830	if (unlink)
1831	list_del_init(&dev->close_list);
1832	}
1833	}
1834	EXPORT_SYMBOL_NS_GPL(netif_close_many, "NETDEV_INTERNAL");
1835
1836	void netif_close(struct net_device *dev)
1837	{
1838	if (dev->flags & IFF_UP) {
1839	LIST_HEAD(single);
1840
1841	list_add(&dev->close_list, &single);
1842	netif_close_many(&single, true);
1843	list_del(&single);
1844	}
1845	}
1846	EXPORT_SYMBOL(netif_close);
1847
1848	void netif_disable_lro(struct net_device *dev)
1849	{
1850	struct net_device *lower_dev;
1851	struct list_head *iter;
1852
1853	dev->wanted_features &= ~NETIF_F_LRO;
1854	netdev_update_features(dev);
1855
1856	if (unlikely(dev->features & NETIF_F_LRO))
1857	netdev_WARN(dev, "failed to disable LRO!\n");
1858
1859	netdev_for_each_lower_dev(dev, lower_dev, iter) {
1860	netdev_lock_ops(lower_dev);
1861	netif_disable_lro(lower_dev);
1862	netdev_unlock_ops(lower_dev);
1863	}
1864	}
1865
1866	/**
1867	* dev_disable_gro_hw - disable HW Generic Receive Offload on a device
1868	* @dev: device
1869	*
1870	* Disable HW Generic Receive Offload (GRO_HW) on a net device. Must be
1871	* called under RTNL. This is needed if Generic XDP is installed on
1872	* the device.
1873	*/
1874	static void dev_disable_gro_hw(struct net_device *dev)
1875	{
1876	dev->wanted_features &= ~NETIF_F_GRO_HW;
1877	netdev_update_features(dev);
1878
1879	if (unlikely(dev->features & NETIF_F_GRO_HW))
1880	netdev_WARN(dev, "failed to disable GRO_HW!\n");
1881	}
1882
1883	const char *netdev_cmd_to_name(enum netdev_cmd cmd)
1884	{
1885	#define N(val) \
1886	case NETDEV_##val: \
1887	return "NETDEV_" __stringify(val);
1888	switch (cmd) {
1889	N(UP) N(DOWN) N(REBOOT) N(CHANGE) N(REGISTER) N(UNREGISTER)
1890	N(CHANGEMTU) N(CHANGEADDR) N(GOING_DOWN) N(CHANGENAME) N(FEAT_CHANGE)
1891	N(BONDING_FAILOVER) N(PRE_UP) N(PRE_TYPE_CHANGE) N(POST_TYPE_CHANGE)
1892	N(POST_INIT) N(PRE_UNINIT) N(RELEASE) N(NOTIFY_PEERS) N(JOIN)
1893	N(CHANGEUPPER) N(RESEND_IGMP) N(PRECHANGEMTU) N(CHANGEINFODATA)
1894	N(BONDING_INFO) N(PRECHANGEUPPER) N(CHANGELOWERSTATE)
1895	N(UDP_TUNNEL_PUSH_INFO) N(UDP_TUNNEL_DROP_INFO) N(CHANGE_TX_QUEUE_LEN)
1896	N(CVLAN_FILTER_PUSH_INFO) N(CVLAN_FILTER_DROP_INFO)
1897	N(SVLAN_FILTER_PUSH_INFO) N(SVLAN_FILTER_DROP_INFO)
1898	N(PRE_CHANGEADDR) N(OFFLOAD_XSTATS_ENABLE) N(OFFLOAD_XSTATS_DISABLE)
1899	N(OFFLOAD_XSTATS_REPORT_USED) N(OFFLOAD_XSTATS_REPORT_DELTA)
1900	N(XDP_FEAT_CHANGE)
1901	}
1902	#undef N
1903	return "UNKNOWN_NETDEV_EVENT";
1904	}
1905	EXPORT_SYMBOL_GPL(netdev_cmd_to_name);
1906
1907	static int call_netdevice_notifier(struct notifier_block *nb, unsigned long val,
1908	struct net_device *dev)
1909	{
1910	struct netdev_notifier_info info = {
1911	.dev = dev,
1912	};
1913
1914	return nb->notifier_call(nb, val, &info);
1915	}
1916
1917	static int call_netdevice_register_notifiers(struct notifier_block *nb,
1918	struct net_device *dev)
1919	{
1920	int err;
1921
1922	err = call_netdevice_notifier(nb, NETDEV_REGISTER, dev);
1923	err = notifier_to_errno(err);
1924	if (err)
1925	return err;
1926
1927	if (!(dev->flags & IFF_UP))
1928	return 0;
1929
1930	call_netdevice_notifier(nb, NETDEV_UP, dev);
1931	return 0;
1932	}
1933
1934	static void call_netdevice_unregister_notifiers(struct notifier_block *nb,
1935	struct net_device *dev)
1936	{
1937	if (dev->flags & IFF_UP) {
1938	netdev_lock_ops(dev);
1939	call_netdevice_notifier(nb, NETDEV_GOING_DOWN,
1940	dev);
1941	call_netdevice_notifier(nb, NETDEV_DOWN, dev);
1942	netdev_unlock_ops(dev);
1943	}
1944	call_netdevice_notifier(nb, NETDEV_UNREGISTER, dev);
1945	}
1946
1947	static int call_netdevice_register_net_notifiers(struct notifier_block *nb,
1948	struct net *net)
1949	{
1950	struct net_device *dev;
1951	int err;
1952
1953	for_each_netdev(net, dev) {
1954	netdev_lock_ops(dev);
1955	err = call_netdevice_register_notifiers(nb, dev);
1956	netdev_unlock_ops(dev);
1957	if (err)
1958	goto rollback;
1959	}
1960	return 0;
1961
1962	rollback:
1963	for_each_netdev_continue_reverse(net, dev)
1964	call_netdevice_unregister_notifiers(nb, dev);
1965	return err;
1966	}
1967
1968	static void call_netdevice_unregister_net_notifiers(struct notifier_block *nb,
1969	struct net *net)
1970	{
1971	struct net_device *dev;
1972
1973	for_each_netdev(net, dev)
1974	call_netdevice_unregister_notifiers(nb, dev);
1975	}
1976
1977	static int dev_boot_phase = 1;
1978
1979	/**
1980	* register_netdevice_notifier - register a network notifier block
1981	* @nb: notifier
1982	*
1983	* Register a notifier to be called when network device events occur.
1984	* The notifier passed is linked into the kernel structures and must
1985	* not be reused until it has been unregistered. A negative errno code
1986	* is returned on a failure.
1987	*
1988	* When registered all registration and up events are replayed
1989	* to the new notifier to allow device to have a race free
1990	* view of the network device list.
1991	*/
1992
1993	int register_netdevice_notifier(struct notifier_block *nb)
1994	{
1995	struct net *net;
1996	int err;
1997
1998	/* Close race with setup_net() and cleanup_net() */
1999	down_write(&pernet_ops_rwsem);
2000
2001	/* When RTNL is removed, we need protection for netdev_chain. */
2002	rtnl_lock();
2003
2004	err = raw_notifier_chain_register(&netdev_chain, nb);
2005	if (err)
2006	goto unlock;
2007	if (dev_boot_phase)
2008	goto unlock;
2009	for_each_net(net) {
2010	__rtnl_net_lock(net);
2011	err = call_netdevice_register_net_notifiers(nb, net);
2012	__rtnl_net_unlock(net);
2013	if (err)
2014	goto rollback;
2015	}
2016
2017	unlock:
2018	rtnl_unlock();
2019	up_write(&pernet_ops_rwsem);
2020	return err;
2021
2022	rollback:
2023	for_each_net_continue_reverse(net) {
2024	__rtnl_net_lock(net);
2025	call_netdevice_unregister_net_notifiers(nb, net);
2026	__rtnl_net_unlock(net);
2027	}
2028
2029	raw_notifier_chain_unregister(&netdev_chain, nb);
2030	goto unlock;
2031	}
2032	EXPORT_SYMBOL(register_netdevice_notifier);
2033
2034	/**
2035	* unregister_netdevice_notifier - unregister a network notifier block
2036	* @nb: notifier
2037	*
2038	* Unregister a notifier previously registered by
2039	* register_netdevice_notifier(). The notifier is unlinked into the
2040	* kernel structures and may then be reused. A negative errno code
2041	* is returned on a failure.
2042	*
2043	* After unregistering unregister and down device events are synthesized
2044	* for all devices on the device list to the removed notifier to remove
2045	* the need for special case cleanup code.
2046	*/
2047
2048	int unregister_netdevice_notifier(struct notifier_block *nb)
2049	{
2050	struct net *net;
2051	int err;
2052
2053	/* Close race with setup_net() and cleanup_net() */
2054	down_write(&pernet_ops_rwsem);
2055	rtnl_lock();
2056	err = raw_notifier_chain_unregister(&netdev_chain, nb);
2057	if (err)
2058	goto unlock;
2059
2060	for_each_net(net) {
2061	__rtnl_net_lock(net);
2062	call_netdevice_unregister_net_notifiers(nb, net);
2063	__rtnl_net_unlock(net);
2064	}
2065
2066	unlock:
2067	rtnl_unlock();
2068	up_write(&pernet_ops_rwsem);
2069	return err;
2070	}
2071	EXPORT_SYMBOL(unregister_netdevice_notifier);
2072
2073	static int __register_netdevice_notifier_net(struct net *net,
2074	struct notifier_block *nb,
2075	bool ignore_call_fail)
2076	{
2077	int err;
2078
2079	err = raw_notifier_chain_register(&net->netdev_chain, nb);
2080	if (err)
2081	return err;
2082	if (dev_boot_phase)
2083	return 0;
2084
2085	err = call_netdevice_register_net_notifiers(nb, net);
2086	if (err && !ignore_call_fail)
2087	goto chain_unregister;
2088
2089	return 0;
2090
2091	chain_unregister:
2092	raw_notifier_chain_unregister(&net->netdev_chain, nb);
2093	return err;
2094	}
2095
2096	static int __unregister_netdevice_notifier_net(struct net *net,
2097	struct notifier_block *nb)
2098	{
2099	int err;
2100
2101	err = raw_notifier_chain_unregister(&net->netdev_chain, nb);
2102	if (err)
2103	return err;
2104
2105	call_netdevice_unregister_net_notifiers(nb, net);
2106	return 0;
2107	}
2108
2109	/**
2110	* register_netdevice_notifier_net - register a per-netns network notifier block
2111	* @net: network namespace
2112	* @nb: notifier
2113	*
2114	* Register a notifier to be called when network device events occur.
2115	* The notifier passed is linked into the kernel structures and must
2116	* not be reused until it has been unregistered. A negative errno code
2117	* is returned on a failure.
2118	*
2119	* When registered all registration and up events are replayed
2120	* to the new notifier to allow device to have a race free
2121	* view of the network device list.
2122	*/
2123
2124	int register_netdevice_notifier_net(struct net net, struct notifier_block nb)
2125	{
2126	int err;
2127
2128	rtnl_net_lock(net);
2129	err = __register_netdevice_notifier_net(net, nb, false);
2130	rtnl_net_unlock(net);
2131
2132	return err;
2133	}
2134	EXPORT_SYMBOL(register_netdevice_notifier_net);
2135
2136	/**
2137	* unregister_netdevice_notifier_net - unregister a per-netns
2138	* network notifier block
2139	* @net: network namespace
2140	* @nb: notifier
2141	*
2142	* Unregister a notifier previously registered by
2143	* register_netdevice_notifier_net(). The notifier is unlinked from the
2144	* kernel structures and may then be reused. A negative errno code
2145	* is returned on a failure.
2146	*
2147	* After unregistering unregister and down device events are synthesized
2148	* for all devices on the device list to the removed notifier to remove
2149	* the need for special case cleanup code.
2150	*/
2151
2152	int unregister_netdevice_notifier_net(struct net *net,
2153	struct notifier_block *nb)
2154	{
2155	int err;
2156
2157	rtnl_net_lock(net);
2158	err = __unregister_netdevice_notifier_net(net, nb);
2159	rtnl_net_unlock(net);
2160
2161	return err;
2162	}
2163	EXPORT_SYMBOL(unregister_netdevice_notifier_net);
2164
2165	static void __move_netdevice_notifier_net(struct net *src_net,
2166	struct net *dst_net,
2167	struct notifier_block *nb)
2168	{
2169	__unregister_netdevice_notifier_net(src_net, nb);
2170	__register_netdevice_notifier_net(dst_net, nb, true);
2171	}
2172
2173	static void rtnl_net_dev_lock(struct net_device *dev)
2174	{
2175	bool again;
2176
2177	do {
2178	struct net *net;
2179
2180	again = false;
2181
2182	/* netns might be being dismantled. */
2183	rcu_read_lock();
2184	net = dev_net_rcu(dev);
2185	net_passive_inc(net);
2186	rcu_read_unlock();
2187
2188	rtnl_net_lock(net);
2189
2190	#ifdef CONFIG_NET_NS
2191	/* dev might have been moved to another netns. */
2192	if (!net_eq(net, rcu_access_pointer(dev->nd_net.net))) {
2193	rtnl_net_unlock(net);
2194	net_passive_dec(net);
2195	again = true;
2196	}
2197	#endif
2198	} while (again);
2199	}
2200
2201	static void rtnl_net_dev_unlock(struct net_device *dev)
2202	{
2203	struct net *net = dev_net(dev);
2204
2205	rtnl_net_unlock(net);
2206	net_passive_dec(net);
2207	}
2208
2209	int register_netdevice_notifier_dev_net(struct net_device *dev,
2210	struct notifier_block *nb,
2211	struct netdev_net_notifier *nn)
2212	{
2213	int err;
2214
2215	rtnl_net_dev_lock(dev);
2216	err = __register_netdevice_notifier_net(dev_net(dev), nb, false);
2217	if (!err) {
2218	nn->nb = nb;
2219	list_add(&nn->list, &dev->net_notifier_list);
2220	}
2221	rtnl_net_dev_unlock(dev);
2222
2223	return err;
2224	}
2225	EXPORT_SYMBOL(register_netdevice_notifier_dev_net);
2226
2227	int unregister_netdevice_notifier_dev_net(struct net_device *dev,
2228	struct notifier_block *nb,
2229	struct netdev_net_notifier *nn)
2230	{
2231	int err;
2232
2233	rtnl_net_dev_lock(dev);
2234	list_del(&nn->list);
2235	err = __unregister_netdevice_notifier_net(dev_net(dev), nb);
2236	rtnl_net_dev_unlock(dev);
2237
2238	return err;
2239	}
2240	EXPORT_SYMBOL(unregister_netdevice_notifier_dev_net);
2241
2242	static void move_netdevice_notifiers_dev_net(struct net_device *dev,
2243	struct net *net)
2244	{
2245	struct netdev_net_notifier *nn;
2246
2247	list_for_each_entry(nn, &dev->net_notifier_list, list)
2248	__move_netdevice_notifier_net(dev_net(dev), net, nn->nb);
2249	}
2250
2251	/**
2252	* call_netdevice_notifiers_info - call all network notifier blocks
2253	* @val: value passed unmodified to notifier function
2254	* @info: notifier information data
2255	*
2256	* Call all network notifier blocks. Parameters and return value
2257	* are as for raw_notifier_call_chain().
2258	*/
2259
2260	int call_netdevice_notifiers_info(unsigned long val,
2261	struct netdev_notifier_info *info)
2262	{
2263	struct net *net = dev_net(info->dev);
2264	int ret;
2265
2266	ASSERT_RTNL();
2267
2268	/* Run per-netns notifier block chain first, then run the global one.
2269	* Hopefully, one day, the global one is going to be removed after
2270	* all notifier block registrators get converted to be per-netns.
2271	*/
2272	ret = raw_notifier_call_chain(&net->netdev_chain, val, info);
2273	if (ret & NOTIFY_STOP_MASK)
2274	return ret;
2275	return raw_notifier_call_chain(&netdev_chain, val, info);
2276	}
2277
2278	/**
2279	* call_netdevice_notifiers_info_robust - call per-netns notifier blocks
2280	* for and rollback on error
2281	* @val_up: value passed unmodified to notifier function
2282	* @val_down: value passed unmodified to the notifier function when
2283	* recovering from an error on @val_up
2284	* @info: notifier information data
2285	*
2286	* Call all per-netns network notifier blocks, but not notifier blocks on
2287	* the global notifier chain. Parameters and return value are as for
2288	* raw_notifier_call_chain_robust().
2289	*/
2290
2291	static int
2292	call_netdevice_notifiers_info_robust(unsigned long val_up,
2293	unsigned long val_down,
2294	struct netdev_notifier_info *info)
2295	{
2296	struct net *net = dev_net(info->dev);
2297
2298	ASSERT_RTNL();
2299
2300	return raw_notifier_call_chain_robust(&net->netdev_chain,
2301	val_up, val_down, info);
2302	}
2303
2304	static int call_netdevice_notifiers_extack(unsigned long val,
2305	struct net_device *dev,
2306	struct netlink_ext_ack *extack)
2307	{
2308	struct netdev_notifier_info info = {
2309	.dev = dev,
2310	.extack = extack,
2311	};
2312
2313	return call_netdevice_notifiers_info(val, &info);
2314	}
2315
2316	/**
2317	* call_netdevice_notifiers - call all network notifier blocks
2318	* @val: value passed unmodified to notifier function
2319	* @dev: net_device pointer passed unmodified to notifier function
2320	*
2321	* Call all network notifier blocks. Parameters and return value
2322	* are as for raw_notifier_call_chain().
2323	*/
2324
2325	int call_netdevice_notifiers(unsigned long val, struct net_device *dev)
2326	{
2327	return call_netdevice_notifiers_extack(val, dev, NULL);
2328	}
2329	EXPORT_SYMBOL(call_netdevice_notifiers);
2330
2331	/**
2332	* call_netdevice_notifiers_mtu - call all network notifier blocks
2333	* @val: value passed unmodified to notifier function
2334	* @dev: net_device pointer passed unmodified to notifier function
2335	* @arg: additional u32 argument passed to the notifier function
2336	*
2337	* Call all network notifier blocks. Parameters and return value
2338	* are as for raw_notifier_call_chain().
2339	*/
2340	static int call_netdevice_notifiers_mtu(unsigned long val,
2341	struct net_device *dev, u32 arg)
2342	{
2343	struct netdev_notifier_info_ext info = {
2344	.info.dev = dev,
2345	.ext.mtu = arg,
2346	};
2347
2348	BUILD_BUG_ON(offsetof(struct netdev_notifier_info_ext, info) != 0);
2349
2350	return call_netdevice_notifiers_info(val, &info.info);
2351	}
2352
2353	#ifdef CONFIG_NET_INGRESS
2354	static DEFINE_STATIC_KEY_FALSE(ingress_needed_key);
2355
2356	void net_inc_ingress_queue(void)
2357	{
2358	static_branch_inc(&ingress_needed_key);
2359	}
2360	EXPORT_SYMBOL_GPL(net_inc_ingress_queue);
2361
2362	void net_dec_ingress_queue(void)
2363	{
2364	static_branch_dec(&ingress_needed_key);
2365	}
2366	EXPORT_SYMBOL_GPL(net_dec_ingress_queue);
2367	#endif
2368
2369	#ifdef CONFIG_NET_EGRESS
2370	static DEFINE_STATIC_KEY_FALSE(egress_needed_key);
2371
2372	void net_inc_egress_queue(void)
2373	{
2374	static_branch_inc(&egress_needed_key);
2375	}
2376	EXPORT_SYMBOL_GPL(net_inc_egress_queue);
2377
2378	void net_dec_egress_queue(void)
2379	{
2380	static_branch_dec(&egress_needed_key);
2381	}
2382	EXPORT_SYMBOL_GPL(net_dec_egress_queue);
2383	#endif
2384
2385	#ifdef CONFIG_NET_CLS_ACT
2386	DEFINE_STATIC_KEY_FALSE(tcf_sw_enabled_key);
2387	EXPORT_SYMBOL(tcf_sw_enabled_key);
2388	#endif
2389
2390	DEFINE_STATIC_KEY_FALSE(netstamp_needed_key);
2391	EXPORT_SYMBOL(netstamp_needed_key);
2392	#ifdef CONFIG_JUMP_LABEL
2393	static atomic_t netstamp_needed_deferred;
2394	static atomic_t netstamp_wanted;
2395	static void netstamp_clear(struct work_struct *work)
2396	{
2397	int deferred = atomic_xchg(&netstamp_needed_deferred, 0);
2398	int wanted;
2399
2400	wanted = atomic_add_return(deferred, &netstamp_wanted);
2401	if (wanted > 0)
2402	static_branch_enable(&netstamp_needed_key);
2403	else
2404	static_branch_disable(&netstamp_needed_key);
2405	}
2406	static DECLARE_WORK(netstamp_work, netstamp_clear);
2407	#endif
2408
2409	void net_enable_timestamp(void)
2410	{
2411	#ifdef CONFIG_JUMP_LABEL
2412	int wanted = atomic_read(&netstamp_wanted);
2413
2414	while (wanted > 0) {
2415	if (atomic_try_cmpxchg(&netstamp_wanted, &wanted, wanted + 1))
2416	return;
2417	}
2418	atomic_inc(&netstamp_needed_deferred);
2419	schedule_work(&netstamp_work);
2420	#else
2421	static_branch_inc(&netstamp_needed_key);
2422	#endif
2423	}
2424	EXPORT_SYMBOL(net_enable_timestamp);
2425
2426	void net_disable_timestamp(void)
2427	{
2428	#ifdef CONFIG_JUMP_LABEL
2429	int wanted = atomic_read(&netstamp_wanted);
2430
2431	while (wanted > 1) {
2432	if (atomic_try_cmpxchg(&netstamp_wanted, &wanted, wanted - 1))
2433	return;
2434	}
2435	atomic_dec(&netstamp_needed_deferred);
2436	schedule_work(&netstamp_work);
2437	#else
2438	static_branch_dec(&netstamp_needed_key);
2439	#endif
2440	}
2441	EXPORT_SYMBOL(net_disable_timestamp);
2442
2443	static inline void net_timestamp_set(struct sk_buff *skb)
2444	{
2445	skb->tstamp = 0;
2446	skb->tstamp_type = SKB_CLOCK_REALTIME;
2447	if (static_branch_unlikely(&netstamp_needed_key))
2448	skb->tstamp = ktime_get_real();
2449	}
2450
2451	#define net_timestamp_check(COND, SKB) \
2452	if (static_branch_unlikely(&netstamp_needed_key)) { \
2453	if ((COND) && !(SKB)->tstamp) \
2454	(SKB)->tstamp = ktime_get_real(); \
2455	} \
2456
2457	bool is_skb_forwardable(const struct net_device dev, const struct sk_buff skb)
2458	{
2459	return __is_skb_forwardable(dev, skb, true);
2460	}
2461	EXPORT_SYMBOL_GPL(is_skb_forwardable);
2462
2463	static int __dev_forward_skb2(struct net_device dev, struct sk_buff skb,
2464	bool check_mtu)
2465	{
2466	int ret = ____dev_forward_skb(dev, skb, check_mtu);
2467
2468	if (likely(!ret)) {
2469	skb->protocol = eth_type_trans(skb, dev);
2470	skb_postpull_rcsum(skb, eth_hdr(skb), ETH_HLEN);
2471	}
2472
2473	return ret;
2474	}
2475
2476	int __dev_forward_skb(struct net_device dev, struct sk_buff skb)
2477	{
2478	return __dev_forward_skb2(dev, skb, true);
2479	}
2480	EXPORT_SYMBOL_GPL(__dev_forward_skb);
2481
2482	/**
2483	* dev_forward_skb - loopback an skb to another netif
2484	*
2485	* @dev: destination network device
2486	* @skb: buffer to forward
2487	*
2488	* return values:
2489	* NET_RX_SUCCESS (no congestion)
2490	* NET_RX_DROP (packet was dropped, but freed)
2491	*
2492	* dev_forward_skb can be used for injecting an skb from the
2493	* start_xmit function of one device into the receive queue
2494	* of another device.
2495	*
2496	* The receiving device may be in another namespace, so
2497	* we have to clear all information in the skb that could
2498	* impact namespace isolation.
2499	*/
2500	int dev_forward_skb(struct net_device dev, struct sk_buff skb)
2501	{
2502	return __dev_forward_skb(dev, skb) ?: netif_rx_internal(skb);
2503	}
2504	EXPORT_SYMBOL_GPL(dev_forward_skb);
2505
2506	int dev_forward_skb_nomtu(struct net_device dev, struct sk_buff skb)
2507	{
2508	return __dev_forward_skb2(dev, skb, false) ?: netif_rx_internal(skb);
2509	}
2510
2511	static int deliver_skb(struct sk_buff *skb,
2512	struct packet_type *pt_prev,
2513	struct net_device *orig_dev)
2514	{
2515	if (unlikely(skb_orphan_frags_rx(skb, GFP_ATOMIC)))
2516	return -ENOMEM;
2517	refcount_inc(&skb->users);
2518	return pt_prev->func(skb, skb->dev, pt_prev, orig_dev);
2519	}
2520
2521	static inline void deliver_ptype_list_skb(struct sk_buff *skb,
2522	struct packet_type **pt,
2523	struct net_device *orig_dev,
2524	__be16 type,
2525	struct list_head *ptype_list)
2526	{
2527	struct packet_type ptype, pt_prev = *pt;
2528
2529	list_for_each_entry_rcu(ptype, ptype_list, list) {
2530	if (ptype->type != type)
2531	continue;
2532	if (unlikely(pt_prev))
2533	deliver_skb(skb, pt_prev, orig_dev);
2534	pt_prev = ptype;
2535	}
2536	*pt = pt_prev;
2537	}
2538
2539	static inline bool skb_loop_sk(struct packet_type ptype, struct sk_buff skb)
2540	{
2541	if (!ptype->af_packet_priv \|\| !skb->sk)
2542	return false;
2543
2544	if (ptype->id_match)
2545	return ptype->id_match(ptype, skb->sk);
2546	else if ((struct sock *)ptype->af_packet_priv == skb->sk)
2547	return true;
2548
2549	return false;
2550	}
2551
2552	/**
2553	* dev_nit_active_rcu - return true if any network interface taps are in use
2554	*
2555	* The caller must hold the RCU lock
2556	*
2557	* @dev: network device to check for the presence of taps
2558	*/
2559	bool dev_nit_active_rcu(const struct net_device *dev)
2560	{
2561	/* Callers may hold either RCU or RCU BH lock */
2562	WARN_ON_ONCE(!rcu_read_lock_held() && !rcu_read_lock_bh_held());
2563
2564	return !list_empty(&dev_net(dev)->ptype_all) \|\|
2565	!list_empty(&dev->ptype_all);
2566	}
2567	EXPORT_SYMBOL_GPL(dev_nit_active_rcu);
2568
2569	/*
2570	* Support routine. Sends outgoing frames to any network
2571	* taps currently in use.
2572	*/
2573
2574	void dev_queue_xmit_nit(struct sk_buff skb, struct net_device dev)
2575	{
2576	struct packet_type ptype, pt_prev = NULL;
2577	struct list_head *ptype_list;
2578	struct sk_buff *skb2 = NULL;
2579
2580	rcu_read_lock();
2581	ptype_list = &dev_net_rcu(dev)->ptype_all;
2582	again:
2583	list_for_each_entry_rcu(ptype, ptype_list, list) {
2584	if (READ_ONCE(ptype->ignore_outgoing))
2585	continue;
2586
2587	/* Never send packets back to the socket
2588	* they originated from - MvS ([email protected])
2589	*/
2590	if (skb_loop_sk(ptype, skb))
2591	continue;
2592
2593	if (unlikely(pt_prev)) {
2594	deliver_skb(skb2, pt_prev, skb->dev);
2595	pt_prev = ptype;
2596	continue;
2597	}
2598
2599	/* need to clone skb, done only once */
2600	skb2 = skb_clone(skb, GFP_ATOMIC);
2601	if (!skb2)
2602	goto out_unlock;
2603
2604	net_timestamp_set(skb2);
2605
2606	/* skb->nh should be correctly
2607	* set by sender, so that the second statement is
2608	* just protection against buggy protocols.
2609	*/
2610	skb_reset_mac_header(skb2);
2611
2612	if (skb_network_header(skb2) < skb2->data \|\|
2613	skb_network_header(skb2) > skb_tail_pointer(skb2)) {
2614	net_crit_ratelimited("protocol %04x is buggy, dev %s\n",
2615	ntohs(skb2->protocol),
2616	dev->name);
2617	skb_reset_network_header(skb2);
2618	}
2619
2620	skb2->transport_header = skb2->network_header;
2621	skb2->pkt_type = PACKET_OUTGOING;
2622	pt_prev = ptype;
2623	}
2624
2625	if (ptype_list != &dev->ptype_all) {
2626	ptype_list = &dev->ptype_all;
2627	goto again;
2628	}
2629	out_unlock:
2630	if (pt_prev) {
2631	if (!skb_orphan_frags_rx(skb2, GFP_ATOMIC))
2632	pt_prev->func(skb2, skb->dev, pt_prev, skb->dev);
2633	else
2634	kfree_skb(skb2);
2635	}
2636	rcu_read_unlock();
2637	}
2638	EXPORT_SYMBOL_GPL(dev_queue_xmit_nit);
2639
2640	/**
2641	* netif_setup_tc - Handle tc mappings on real_num_tx_queues change
2642	* @dev: Network device
2643	* @txq: number of queues available
2644	*
2645	* If real_num_tx_queues is changed the tc mappings may no longer be
2646	* valid. To resolve this verify the tc mapping remains valid and if
2647	* not NULL the mapping. With no priorities mapping to this
2648	* offset/count pair it will no longer be used. In the worst case TC0
2649	* is invalid nothing can be done so disable priority mappings. If is
2650	* expected that drivers will fix this mapping if they can before
2651	* calling netif_set_real_num_tx_queues.
2652	*/
2653	static void netif_setup_tc(struct net_device *dev, unsigned int txq)
2654	{
2655	struct netdev_tc_txq res;
2656	int i;
2657
2658	res.combined = READ_ONCE(dev->tc_to_txq[0].combined);
2659
2660	/* If TC0 is invalidated disable TC mapping */
2661	if (res.offset + res.count > txq) {
2662	netdev_warn(dev, "Number of in use tx queues changed invalidating tc mappings. Priority traffic classification disabled!\n");
2663	WRITE_ONCE(dev->num_tc, 0);
2664	return;
2665	}
2666
2667	/* Invalidated prio to tc mappings set to TC0 */
2668	for (i = 1; i < TC_BITMASK + 1; i++) {
2669	int q = netdev_get_prio_tc_map(dev, i);
2670
2671	res.combined = READ_ONCE(dev->tc_to_txq[q].combined);
2672	if (res.offset + res.count > txq) {
2673	netdev_warn(dev, "Number of in use tx queues changed. Priority %i to tc mapping %i is no longer valid. Setting map to 0\n",
2674	i, q);
2675	netdev_set_prio_tc_map(dev, i, 0);
2676	}
2677	}
2678	}
2679
2680	int netdev_txq_to_tc(struct net_device *dev, unsigned int txq)
2681	{
2682	if (READ_ONCE(dev->num_tc)) {
2683	struct netdev_tc_txq *tc = &dev->tc_to_txq[0];
2684	int i;
2685
2686	/* walk through the TCs and see if it falls into any of them */
2687	for (i = 0; i < TC_MAX_QUEUE; i++, tc++) {
2688	struct netdev_tc_txq res;
2689
2690	res.combined = READ_ONCE(tc->combined);
2691	if ((txq - res.offset) < res.count)
2692	return i;
2693	}
2694
2695	/* didn't find it, just return -1 to indicate no match */
2696	return -1;
2697	}
2698
2699	return 0;
2700	}
2701	EXPORT_SYMBOL(netdev_txq_to_tc);
2702
2703	#ifdef CONFIG_XPS
2704	static struct static_key xps_needed __read_mostly;
2705	static struct static_key xps_rxqs_needed __read_mostly;
2706	static DEFINE_MUTEX(xps_map_mutex);
2707	#define xmap_dereference(P) \
2708	rcu_dereference_protected((P), lockdep_is_held(&xps_map_mutex))
2709
2710	static bool remove_xps_queue(struct xps_dev_maps *dev_maps,
2711	struct xps_dev_maps *old_maps, int tci, u16 index)
2712	{
2713	struct xps_map *map = NULL;
2714	int pos;
2715
2716	map = xmap_dereference(dev_maps->attr_map[tci]);
2717	if (!map)
2718	return false;
2719
2720	for (pos = map->len; pos--;) {
2721	if (map->queues[pos] != index)
2722	continue;
2723
2724	if (map->len > 1) {
2725	map->queues[pos] = map->queues[--map->len];
2726	break;
2727	}
2728
2729	if (old_maps)
2730	RCU_INIT_POINTER(old_maps->attr_map[tci], NULL);
2731	RCU_INIT_POINTER(dev_maps->attr_map[tci], NULL);
2732	kfree_rcu(map, rcu);
2733	return false;
2734	}
2735
2736	return true;
2737	}
2738
2739	static bool remove_xps_queue_cpu(struct net_device *dev,
2740	struct xps_dev_maps *dev_maps,
2741	int cpu, u16 offset, u16 count)
2742	{
2743	int num_tc = dev_maps->num_tc;
2744	bool active = false;
2745	int tci;
2746
2747	for (tci = cpu * num_tc; num_tc--; tci++) {
2748	int i, j;
2749
2750	for (i = count, j = offset; i--; j++) {
2751	if (!remove_xps_queue(dev_maps, NULL, tci, j))
2752	break;
2753	}
2754
2755	active \|= i < 0;
2756	}
2757
2758	return active;
2759	}
2760
2761	static void reset_xps_maps(struct net_device *dev,
2762	struct xps_dev_maps *dev_maps,
2763	enum xps_map_type type)
2764	{
2765	static_key_slow_dec_cpuslocked(&xps_needed);
2766	if (type == XPS_RXQS)
2767	static_key_slow_dec_cpuslocked(&xps_rxqs_needed);
2768
2769	RCU_INIT_POINTER(dev->xps_maps[type], NULL);
2770
2771	kfree_rcu(dev_maps, rcu);
2772	}
2773
2774	static void clean_xps_maps(struct net_device *dev, enum xps_map_type type,
2775	u16 offset, u16 count)
2776	{
2777	struct xps_dev_maps *dev_maps;
2778	bool active = false;
2779	int i, j;
2780
2781	dev_maps = xmap_dereference(dev->xps_maps[type]);
2782	if (!dev_maps)
2783	return;
2784
2785	for (j = 0; j < dev_maps->nr_ids; j++)
2786	active \|= remove_xps_queue_cpu(dev, dev_maps, j, offset, count);
2787	if (!active)
2788	reset_xps_maps(dev, dev_maps, type);
2789
2790	if (type == XPS_CPUS) {
2791	for (i = offset + (count - 1); count--; i--)
2792	netdev_queue_numa_node_write(
2793	netdev_get_tx_queue(dev, i), NUMA_NO_NODE);
2794	}
2795	}
2796
2797	static void netif_reset_xps_queues(struct net_device *dev, u16 offset,
2798	u16 count)
2799	{
2800	if (!static_key_false(&xps_needed))
2801	return;
2802
2803	cpus_read_lock();
2804	mutex_lock(&xps_map_mutex);
2805
2806	if (static_key_false(&xps_rxqs_needed))
2807	clean_xps_maps(dev, XPS_RXQS, offset, count);
2808
2809	clean_xps_maps(dev, XPS_CPUS, offset, count);
2810
2811	mutex_unlock(&xps_map_mutex);
2812	cpus_read_unlock();
2813	}
2814
2815	static void netif_reset_xps_queues_gt(struct net_device *dev, u16 index)
2816	{
2817	netif_reset_xps_queues(dev, index, dev->num_tx_queues - index);
2818	}
2819
2820	static struct xps_map expand_xps_map(struct xps_map map, int attr_index,
2821	u16 index, bool is_rxqs_map)
2822	{
2823	struct xps_map *new_map;
2824	int alloc_len = XPS_MIN_MAP_ALLOC;
2825	int i, pos;
2826
2827	for (pos = 0; map && pos < map->len; pos++) {
2828	if (map->queues[pos] != index)
2829	continue;
2830	return map;
2831	}
2832
2833	/* Need to add tx-queue to this CPU's/rx-queue's existing map */
2834	if (map) {
2835	if (pos < map->alloc_len)
2836	return map;
2837
2838	alloc_len = map->alloc_len * 2;
2839	}
2840
2841	/* Need to allocate new map to store tx-queue on this CPU's/rx-queue's
2842	* map
2843	*/
2844	if (is_rxqs_map)
2845	new_map = kzalloc(XPS_MAP_SIZE(alloc_len), GFP_KERNEL);
2846	else
2847	new_map = kzalloc_node(XPS_MAP_SIZE(alloc_len), GFP_KERNEL,
2848	cpu_to_node(attr_index));
2849	if (!new_map)
2850	return NULL;
2851
2852	for (i = 0; i < pos; i++)
2853	new_map->queues[i] = map->queues[i];
2854	new_map->alloc_len = alloc_len;
2855	new_map->len = pos;
2856
2857	return new_map;
2858	}
2859
2860	/* Copy xps maps at a given index */
2861	static void xps_copy_dev_maps(struct xps_dev_maps *dev_maps,
2862	struct xps_dev_maps *new_dev_maps, int index,
2863	int tc, bool skip_tc)
2864	{
2865	int i, tci = index * dev_maps->num_tc;
2866	struct xps_map *map;
2867
2868	/* copy maps belonging to foreign traffic classes */
2869	for (i = 0; i < dev_maps->num_tc; i++, tci++) {
2870	if (i == tc && skip_tc)
2871	continue;
2872
2873	/* fill in the new device map from the old device map */
2874	map = xmap_dereference(dev_maps->attr_map[tci]);
2875	RCU_INIT_POINTER(new_dev_maps->attr_map[tci], map);
2876	}
2877	}
2878
2879	/* Must be called under cpus_read_lock */
2880	int __netif_set_xps_queue(struct net_device dev, const unsigned long mask,
2881	u16 index, enum xps_map_type type)
2882	{
2883	struct xps_dev_maps dev_maps, new_dev_maps = NULL, *old_dev_maps = NULL;
2884	int maps_sz, num_tc = 1, tc = 0, dev_num_tc;
2885	const unsigned long *online_mask = NULL;
2886	bool active = false, copy = false;
2887	int i, j, tci, numa_node_id = -2;
2888	struct xps_map map, new_map;
2889	unsigned int nr_ids;
2890
2891	WARN_ON_ONCE(index >= dev->num_tx_queues);
2892
2893	dev_num_tc = READ_ONCE(dev->num_tc);
2894	if (dev_num_tc) {
2895	/* Do not allow XPS on subordinate device directly */
2896	num_tc = dev_num_tc;
2897	if (num_tc < 0)
2898	return -EINVAL;
2899
2900	/* If queue belongs to subordinate dev use its map */
2901	dev = netdev_get_tx_queue(dev, index)->sb_dev ? : dev;
2902
2903	tc = netdev_txq_to_tc(dev, index);
2904	if (tc < 0 \|\| tc >= num_tc)
2905	return -EINVAL;
2906	}
2907
2908	mutex_lock(&xps_map_mutex);
2909
2910	dev_maps = xmap_dereference(dev->xps_maps[type]);
2911	if (type == XPS_RXQS) {
2912	maps_sz = XPS_RXQ_DEV_MAPS_SIZE(num_tc, dev->num_rx_queues);
2913	nr_ids = dev->num_rx_queues;
2914	} else {
2915	maps_sz = XPS_CPU_DEV_MAPS_SIZE(num_tc);
2916	if (num_possible_cpus() > 1)
2917	online_mask = cpumask_bits(cpu_online_mask);
2918	nr_ids = nr_cpu_ids;
2919	}
2920
2921	if (maps_sz < L1_CACHE_BYTES)
2922	maps_sz = L1_CACHE_BYTES;
2923
2924	/* The old dev_maps could be larger or smaller than the one we're
2925	* setting up now, as dev->num_tc or nr_ids could have been updated in
2926	* between. We could try to be smart, but let's be safe instead and only
2927	* copy foreign traffic classes if the two map sizes match.
2928	*/
2929	if (dev_maps &&
2930	dev_maps->num_tc == num_tc && dev_maps->nr_ids == nr_ids)
2931	copy = true;
2932
2933	/* allocate memory for queue storage */
2934	for (j = -1; j = netif_attrmask_next_and(j, online_mask, mask, nr_ids),
2935	j < nr_ids;) {
2936	if (!new_dev_maps) {
2937	new_dev_maps = kzalloc(maps_sz, GFP_KERNEL);
2938	if (!new_dev_maps) {
2939	mutex_unlock(&xps_map_mutex);
2940	return -ENOMEM;
2941	}
2942
2943	new_dev_maps->nr_ids = nr_ids;
2944	new_dev_maps->num_tc = num_tc;
2945	}
2946
2947	tci = j * num_tc + tc;
2948	map = copy ? xmap_dereference(dev_maps->attr_map[tci]) : NULL;
2949
2950	map = expand_xps_map(map, j, index, type == XPS_RXQS);
2951	if (!map)
2952	goto error;
2953
2954	RCU_INIT_POINTER(new_dev_maps->attr_map[tci], map);
2955	}
2956
2957	if (!new_dev_maps)
2958	goto out_no_new_maps;
2959
2960	if (!dev_maps) {
2961	/* Increment static keys at most once per type */
2962	static_key_slow_inc_cpuslocked(&xps_needed);
2963	if (type == XPS_RXQS)
2964	static_key_slow_inc_cpuslocked(&xps_rxqs_needed);
2965	}
2966
2967	for (j = 0; j < nr_ids; j++) {
2968	bool skip_tc = false;
2969
2970	tci = j * num_tc + tc;
2971	if (netif_attr_test_mask(j, mask, nr_ids) &&
2972	netif_attr_test_online(j, online_mask, nr_ids)) {
2973	/* add tx-queue to CPU/rx-queue maps */
2974	int pos = 0;
2975
2976	skip_tc = true;
2977
2978	map = xmap_dereference(new_dev_maps->attr_map[tci]);
2979	while ((pos < map->len) && (map->queues[pos] != index))
2980	pos++;
2981
2982	if (pos == map->len)
2983	map->queues[map->len++] = index;
2984	#ifdef CONFIG_NUMA
2985	if (type == XPS_CPUS) {
2986	if (numa_node_id == -2)
2987	numa_node_id = cpu_to_node(j);
2988	else if (numa_node_id != cpu_to_node(j))
2989	numa_node_id = -1;
2990	}
2991	#endif
2992	}
2993
2994	if (copy)
2995	xps_copy_dev_maps(dev_maps, new_dev_maps, j, tc,
2996	skip_tc);
2997	}
2998
2999	rcu_assign_pointer(dev->xps_maps[type], new_dev_maps);
3000
3001	/* Cleanup old maps */
3002	if (!dev_maps)
3003	goto out_no_old_maps;
3004
3005	for (j = 0; j < dev_maps->nr_ids; j++) {
3006	for (i = num_tc, tci = j * dev_maps->num_tc; i--; tci++) {
3007	map = xmap_dereference(dev_maps->attr_map[tci]);
3008	if (!map)
3009	continue;
3010
3011	if (copy) {
3012	new_map = xmap_dereference(new_dev_maps->attr_map[tci]);
3013	if (map == new_map)
3014	continue;
3015	}
3016
3017	RCU_INIT_POINTER(dev_maps->attr_map[tci], NULL);
3018	kfree_rcu(map, rcu);
3019	}
3020	}
3021
3022	old_dev_maps = dev_maps;
3023
3024	out_no_old_maps:
3025	dev_maps = new_dev_maps;
3026	active = true;
3027
3028	out_no_new_maps:
3029	if (type == XPS_CPUS)
3030	/* update Tx queue numa node */
3031	netdev_queue_numa_node_write(netdev_get_tx_queue(dev, index),
3032	(numa_node_id >= 0) ?
3033	numa_node_id : NUMA_NO_NODE);
3034
3035	if (!dev_maps)
3036	goto out_no_maps;
3037
3038	/* removes tx-queue from unused CPUs/rx-queues */
3039	for (j = 0; j < dev_maps->nr_ids; j++) {
3040	tci = j * dev_maps->num_tc;
3041
3042	for (i = 0; i < dev_maps->num_tc; i++, tci++) {
3043	if (i == tc &&
3044	netif_attr_test_mask(j, mask, dev_maps->nr_ids) &&
3045	netif_attr_test_online(j, online_mask, dev_maps->nr_ids))
3046	continue;
3047
3048	active \|= remove_xps_queue(dev_maps,
3049	copy ? old_dev_maps : NULL,
3050	tci, index);
3051	}
3052	}
3053
3054	if (old_dev_maps)
3055	kfree_rcu(old_dev_maps, rcu);
3056
3057	/* free map if not active */
3058	if (!active)
3059	reset_xps_maps(dev, dev_maps, type);
3060
3061	out_no_maps:
3062	mutex_unlock(&xps_map_mutex);
3063
3064	return 0;
3065	error:
3066	/* remove any maps that we added */
3067	for (j = 0; j < nr_ids; j++) {
3068	for (i = num_tc, tci = j * num_tc; i--; tci++) {
3069	new_map = xmap_dereference(new_dev_maps->attr_map[tci]);
3070	map = copy ?
3071	xmap_dereference(dev_maps->attr_map[tci]) :
3072	NULL;
3073	if (new_map && new_map != map)
3074	kfree(new_map);
3075	}
3076	}
3077
3078	mutex_unlock(&xps_map_mutex);
3079
3080	kfree(new_dev_maps);
3081	return -ENOMEM;
3082	}
3083	EXPORT_SYMBOL_GPL(__netif_set_xps_queue);
3084
3085	int netif_set_xps_queue(struct net_device dev, const struct cpumask mask,
3086	u16 index)
3087	{
3088	int ret;
3089
3090	cpus_read_lock();
3091	ret = __netif_set_xps_queue(dev, cpumask_bits(mask), index, XPS_CPUS);
3092	cpus_read_unlock();
3093
3094	return ret;
3095	}
3096	EXPORT_SYMBOL(netif_set_xps_queue);
3097
3098	#endif
3099	static void netdev_unbind_all_sb_channels(struct net_device *dev)
3100	{
3101	struct netdev_queue *txq = &dev->_tx[dev->num_tx_queues];
3102
3103	/* Unbind any subordinate channels */
3104	while (txq-- != &dev->_tx[0]) {
3105	if (txq->sb_dev)
3106	netdev_unbind_sb_channel(dev, txq->sb_dev);
3107	}
3108	}
3109
3110	void netdev_reset_tc(struct net_device *dev)
3111	{
3112	int i;
3113
3114	#ifdef CONFIG_XPS
3115	netif_reset_xps_queues_gt(dev, 0);
3116	#endif
3117	netdev_unbind_all_sb_channels(dev);
3118
3119	/* Reset TC configuration of device */
3120	WRITE_ONCE(dev->num_tc, 0);
3121	for (i = 0; i < TC_MAX_QUEUE; i++)
3122	WRITE_ONCE(dev->tc_to_txq[i].combined, 0);
3123	for (i = 0; i <= TC_BITMASK; i++)
3124	WRITE_ONCE(dev->prio_tc_map[i], 0);
3125	}
3126	EXPORT_SYMBOL(netdev_reset_tc);
3127
3128	int netdev_set_tc_queue(struct net_device *dev, u8 tc, u16 count, u16 offset)
3129	{
3130	struct netdev_tc_txq res = {
3131	.count = count,
3132	.offset = offset,
3133	};
3134
3135	if (tc >= READ_ONCE(dev->num_tc))
3136	return -EINVAL;
3137
3138	#ifdef CONFIG_XPS
3139	netif_reset_xps_queues(dev, offset, count);
3140	#endif
3141	WRITE_ONCE(dev->tc_to_txq[tc].combined, res.combined);
3142	return 0;
3143	}
3144	EXPORT_SYMBOL(netdev_set_tc_queue);
3145
3146	int netdev_set_num_tc(struct net_device *dev, u8 num_tc)
3147	{
3148	if (num_tc > TC_MAX_QUEUE)
3149	return -EINVAL;
3150
3151	#ifdef CONFIG_XPS
3152	netif_reset_xps_queues_gt(dev, 0);
3153	#endif
3154	netdev_unbind_all_sb_channels(dev);
3155
3156	WRITE_ONCE(dev->num_tc, num_tc);
3157	return 0;
3158	}
3159	EXPORT_SYMBOL(netdev_set_num_tc);
3160
3161	void netdev_unbind_sb_channel(struct net_device *dev,
3162	struct net_device *sb_dev)
3163	{
3164	struct netdev_queue *txq = &dev->_tx[dev->num_tx_queues];
3165	int i;
3166
3167	#ifdef CONFIG_XPS
3168	netif_reset_xps_queues_gt(sb_dev, 0);
3169	#endif
3170	for (i = 0; i < TC_MAX_QUEUE; i++)
3171	WRITE_ONCE(sb_dev->tc_to_txq[i].combined, 0);
3172	for (i = 0; i <= TC_BITMASK; i++)
3173	WRITE_ONCE(sb_dev->prio_tc_map[i], 0);
3174
3175	while (txq-- != &dev->_tx[0]) {
3176	if (txq->sb_dev == sb_dev)
3177	txq->sb_dev = NULL;
3178	}
3179	}
3180	EXPORT_SYMBOL(netdev_unbind_sb_channel);
3181
3182	int netdev_bind_sb_channel_queue(struct net_device *dev,
3183	struct net_device *sb_dev,
3184	u8 tc, u16 count, u16 offset)
3185	{
3186	/* Make certain the sb_dev and dev are already configured */
3187	if (READ_ONCE(sb_dev->num_tc) >= 0 \|\| tc >= READ_ONCE(dev->num_tc))
3188	return -EINVAL;
3189
3190	/* We cannot hand out queues we don't have */
3191	if ((offset + count) > dev->real_num_tx_queues)
3192	return -EINVAL;
3193
3194	/* Record the mapping */
3195	struct netdev_tc_txq res = {
3196	.count = count,
3197	.offset = offset,
3198	};
3199
3200	WRITE_ONCE(sb_dev->tc_to_txq[tc].combined, res.combined);
3201
3202	/* Provide a way for Tx queue to find the tc_to_txq map or
3203	* XPS map for itself.
3204	*/
3205	while (count--)
3206	netdev_get_tx_queue(dev, count + offset)->sb_dev = sb_dev;
3207
3208	return 0;
3209	}
3210	EXPORT_SYMBOL(netdev_bind_sb_channel_queue);
3211
3212	int netdev_set_sb_channel(struct net_device *dev, u16 channel)
3213	{
3214	/* Do not use a multiqueue device to represent a subordinate channel */
3215	if (netif_is_multiqueue(dev))
3216	return -ENODEV;
3217
3218	/* We allow channels 1 - 32767 to be used for subordinate channels.
3219	* Channel 0 is meant to be "native" mode and used only to represent
3220	* the main root device. We allow writing 0 to reset the device back
3221	* to normal mode after being used as a subordinate channel.
3222	*/
3223	if (channel > S16_MAX)
3224	return -EINVAL;
3225
3226	WRITE_ONCE(dev->num_tc, -channel);
3227
3228	return 0;
3229	}
3230	EXPORT_SYMBOL(netdev_set_sb_channel);
3231
3232	/*
3233	* Routine to help set real_num_tx_queues. To avoid skbs mapped to queues
3234	* greater than real_num_tx_queues stale skbs on the qdisc must be flushed.
3235	*/
3236	int netif_set_real_num_tx_queues(struct net_device *dev, unsigned int txq)
3237	{
3238	bool disabling;
3239	int rc;
3240
3241	disabling = txq < dev->real_num_tx_queues;
3242
3243	if (txq < 1 \|\| txq > dev->num_tx_queues)
3244	return -EINVAL;
3245
3246	if (dev->reg_state == NETREG_REGISTERED \|\|
3247	dev->reg_state == NETREG_UNREGISTERING) {
3248	netdev_assert_locked_ops_compat(dev);
3249
3250	rc = netdev_queue_update_kobjects(dev, dev->real_num_tx_queues,
3251	txq);
3252	if (rc)
3253	return rc;
3254
3255	if (READ_ONCE(dev->num_tc))
3256	netif_setup_tc(dev, txq);
3257
3258	net_shaper_set_real_num_tx_queues(dev, txq);
3259
3260	dev_qdisc_change_real_num_tx(dev, txq);
3261
3262	dev->real_num_tx_queues = txq;
3263
3264	if (disabling) {
3265	synchronize_net();
3266	qdisc_reset_all_tx_gt(dev, txq);
3267	#ifdef CONFIG_XPS
3268	netif_reset_xps_queues_gt(dev, txq);
3269	#endif
3270	}
3271	} else {
3272	dev->real_num_tx_queues = txq;
3273	}
3274
3275	return 0;
3276	}
3277	EXPORT_SYMBOL(netif_set_real_num_tx_queues);
3278
3279	/**
3280	* netif_set_real_num_rx_queues - set actual number of RX queues used
3281	* @dev: Network device
3282	* @rxq: Actual number of RX queues
3283	*
3284	* This must be called either with the rtnl_lock held or before
3285	* registration of the net device. Returns 0 on success, or a
3286	* negative error code. If called before registration, it always
3287	* succeeds.
3288	*/
3289	int netif_set_real_num_rx_queues(struct net_device *dev, unsigned int rxq)
3290	{
3291	int rc;
3292
3293	if (rxq < 1 \|\| rxq > dev->num_rx_queues)
3294	return -EINVAL;
3295
3296	if (dev->reg_state == NETREG_REGISTERED) {
3297	netdev_assert_locked_ops_compat(dev);
3298
3299	rc = net_rx_queue_update_kobjects(dev, dev->real_num_rx_queues,
3300	rxq);
3301	if (rc)
3302	return rc;
3303	}
3304
3305	dev->real_num_rx_queues = rxq;
3306	return 0;
3307	}
3308	EXPORT_SYMBOL(netif_set_real_num_rx_queues);
3309
3310	/**
3311	* netif_set_real_num_queues - set actual number of RX and TX queues used
3312	* @dev: Network device
3313	* @txq: Actual number of TX queues
3314	* @rxq: Actual number of RX queues
3315	*
3316	* Set the real number of both TX and RX queues.
3317	* Does nothing if the number of queues is already correct.
3318	*/
3319	int netif_set_real_num_queues(struct net_device *dev,
3320	unsigned int txq, unsigned int rxq)
3321	{
3322	unsigned int old_rxq = dev->real_num_rx_queues;
3323	int err;
3324
3325	if (txq < 1 \|\| txq > dev->num_tx_queues \|\|
3326	rxq < 1 \|\| rxq > dev->num_rx_queues)
3327	return -EINVAL;
3328
3329	/* Start from increases, so the error path only does decreases -
3330	* decreases can't fail.
3331	*/
3332	if (rxq > dev->real_num_rx_queues) {
3333	err = netif_set_real_num_rx_queues(dev, rxq);
3334	if (err)
3335	return err;
3336	}
3337	if (txq > dev->real_num_tx_queues) {
3338	err = netif_set_real_num_tx_queues(dev, txq);
3339	if (err)
3340	goto undo_rx;
3341	}
3342	if (rxq < dev->real_num_rx_queues)
3343	WARN_ON(netif_set_real_num_rx_queues(dev, rxq));
3344	if (txq < dev->real_num_tx_queues)
3345	WARN_ON(netif_set_real_num_tx_queues(dev, txq));
3346
3347	return 0;
3348	undo_rx:
3349	WARN_ON(netif_set_real_num_rx_queues(dev, old_rxq));
3350	return err;
3351	}
3352	EXPORT_SYMBOL(netif_set_real_num_queues);
3353
3354	/**
3355	* netif_set_tso_max_size() - set the max size of TSO frames supported
3356	* @dev: netdev to update
3357	* @size: max skb->len of a TSO frame
3358	*
3359	* Set the limit on the size of TSO super-frames the device can handle.
3360	* Unless explicitly set the stack will assume the value of
3361	* %GSO_LEGACY_MAX_SIZE.
3362	*/
3363	void netif_set_tso_max_size(struct net_device *dev, unsigned int size)
3364	{
3365	dev->tso_max_size = min(GSO_MAX_SIZE, size);
3366	if (size < READ_ONCE(dev->gso_max_size))
3367	netif_set_gso_max_size(dev, size);
3368	if (size < READ_ONCE(dev->gso_ipv4_max_size))
3369	netif_set_gso_ipv4_max_size(dev, size);
3370	}
3371	EXPORT_SYMBOL(netif_set_tso_max_size);
3372
3373	/**
3374	* netif_set_tso_max_segs() - set the max number of segs supported for TSO
3375	* @dev: netdev to update
3376	* @segs: max number of TCP segments
3377	*
3378	* Set the limit on the number of TCP segments the device can generate from
3379	* a single TSO super-frame.
3380	* Unless explicitly set the stack will assume the value of %GSO_MAX_SEGS.
3381	*/
3382	void netif_set_tso_max_segs(struct net_device *dev, unsigned int segs)
3383	{
3384	dev->tso_max_segs = segs;
3385	if (segs < READ_ONCE(dev->gso_max_segs))
3386	netif_set_gso_max_segs(dev, segs);
3387	}
3388	EXPORT_SYMBOL(netif_set_tso_max_segs);
3389
3390	/**
3391	* netif_inherit_tso_max() - copy all TSO limits from a lower device to an upper
3392	* @to: netdev to update
3393	* @from: netdev from which to copy the limits
3394	*/
3395	void netif_inherit_tso_max(struct net_device to, const struct net_device from)
3396	{
3397	netif_set_tso_max_size(to, from->tso_max_size);
3398	netif_set_tso_max_segs(to, from->tso_max_segs);
3399	}
3400	EXPORT_SYMBOL(netif_inherit_tso_max);
3401
3402	/**
3403	* netif_get_num_default_rss_queues - default number of RSS queues
3404	*
3405	* Default value is the number of physical cores if there are only 1 or 2, or
3406	* divided by 2 if there are more.
3407	*/
3408	int netif_get_num_default_rss_queues(void)
3409	{
3410	cpumask_var_t cpus;
3411	int cpu, count = 0;
3412
3413	if (unlikely(is_kdump_kernel() \|\| !zalloc_cpumask_var(&cpus, GFP_KERNEL)))
3414	return 1;
3415
3416	cpumask_copy(cpus, cpu_online_mask);
3417	for_each_cpu(cpu, cpus) {
3418	++count;
3419	cpumask_andnot(cpus, cpus, topology_sibling_cpumask(cpu));
3420	}
3421	free_cpumask_var(cpus);
3422
3423	return count > 2 ? DIV_ROUND_UP(count, 2) : count;
3424	}
3425	EXPORT_SYMBOL(netif_get_num_default_rss_queues);
3426
3427	static void __netif_reschedule(struct Qdisc *q)
3428	{
3429	struct softnet_data *sd;
3430	unsigned long flags;
3431
3432	local_irq_save(flags);
3433	sd = this_cpu_ptr(&softnet_data);
3434	q->next_sched = NULL;
3435	*sd->output_queue_tailp = q;
3436	sd->output_queue_tailp = &q->next_sched;
3437	raise_softirq_irqoff(NET_TX_SOFTIRQ);
3438	local_irq_restore(flags);
3439	}
3440
3441	void __netif_schedule(struct Qdisc *q)
3442	{
3443	/* If q->defer_list is not empty, at least one thread is
3444	* in __dev_xmit_skb() before llist_del_all(&q->defer_list).
3445	* This thread will attempt to run the queue.
3446	*/
3447	if (!llist_empty(&q->defer_list))
3448	return;
3449
3450	if (!test_and_set_bit(__QDISC_STATE_SCHED, &q->state))
3451	__netif_reschedule(q);
3452	}
3453	EXPORT_SYMBOL(__netif_schedule);
3454
3455	struct dev_kfree_skb_cb {
3456	enum skb_drop_reason reason;
3457	};
3458
3459	static struct dev_kfree_skb_cb get_kfree_skb_cb(const struct sk_buff skb)
3460	{
3461	return (struct dev_kfree_skb_cb *)skb->cb;
3462	}
3463
3464	void netif_schedule_queue(struct netdev_queue *txq)
3465	{
3466	rcu_read_lock();
3467	if (!netif_xmit_stopped(txq)) {
3468	struct Qdisc *q = rcu_dereference(txq->qdisc);
3469
3470	__netif_schedule(q);
3471	}
3472	rcu_read_unlock();
3473	}
3474	EXPORT_SYMBOL(netif_schedule_queue);
3475
3476	void netif_tx_wake_queue(struct netdev_queue *dev_queue)
3477	{
3478	if (test_and_clear_bit(__QUEUE_STATE_DRV_XOFF, &dev_queue->state)) {
3479	struct Qdisc *q;
3480
3481	rcu_read_lock();
3482	q = rcu_dereference(dev_queue->qdisc);
3483	__netif_schedule(q);
3484	rcu_read_unlock();
3485	}
3486	}
3487	EXPORT_SYMBOL(netif_tx_wake_queue);
3488
3489	void dev_kfree_skb_irq_reason(struct sk_buff *skb, enum skb_drop_reason reason)
3490	{
3491	unsigned long flags;
3492
3493	if (unlikely(!skb))
3494	return;
3495
3496	if (likely(refcount_read(&skb->users) == 1)) {
3497	smp_rmb();
3498	refcount_set(&skb->users, 0);
3499	} else if (likely(!refcount_dec_and_test(&skb->users))) {
3500	return;
3501	}
3502	get_kfree_skb_cb(skb)->reason = reason;
3503	local_irq_save(flags);
3504	skb->next = __this_cpu_read(softnet_data.completion_queue);
3505	__this_cpu_write(softnet_data.completion_queue, skb);
3506	raise_softirq_irqoff(NET_TX_SOFTIRQ);
3507	local_irq_restore(flags);
3508	}
3509	EXPORT_SYMBOL(dev_kfree_skb_irq_reason);
3510
3511	void dev_kfree_skb_any_reason(struct sk_buff *skb, enum skb_drop_reason reason)
3512	{
3513	if (in_hardirq() \|\| irqs_disabled())
3514	dev_kfree_skb_irq_reason(skb, reason);
3515	else
3516	kfree_skb_reason(skb, reason);
3517	}
3518	EXPORT_SYMBOL(dev_kfree_skb_any_reason);
3519
3520
3521	/**
3522	* netif_device_detach - mark device as removed
3523	* @dev: network device
3524	*
3525	* Mark device as removed from system and therefore no longer available.
3526	*/
3527	void netif_device_detach(struct net_device *dev)
3528	{
3529	if (test_and_clear_bit(__LINK_STATE_PRESENT, &dev->state) &&
3530	netif_running(dev)) {
3531	netif_tx_stop_all_queues(dev);
3532	}
3533	}
3534	EXPORT_SYMBOL(netif_device_detach);
3535
3536	/**
3537	* netif_device_attach - mark device as attached
3538	* @dev: network device
3539	*
3540	* Mark device as attached from system and restart if needed.
3541	*/
3542	void netif_device_attach(struct net_device *dev)
3543	{
3544	if (!test_and_set_bit(__LINK_STATE_PRESENT, &dev->state) &&
3545	netif_running(dev)) {
3546	netif_tx_wake_all_queues(dev);
3547	netdev_watchdog_up(dev);
3548	}
3549	}
3550	EXPORT_SYMBOL(netif_device_attach);
3551
3552	/*
3553	* Returns a Tx hash based on the given packet descriptor a Tx queues' number
3554	* to be used as a distribution range.
3555	*/
3556	static u16 skb_tx_hash(const struct net_device *dev,
3557	const struct net_device *sb_dev,
3558	struct sk_buff *skb)
3559	{
3560	u32 hash;
3561	u16 qoffset = 0;
3562	u16 qcount = dev->real_num_tx_queues;
3563
3564	if (READ_ONCE(dev->num_tc)) {
3565	u8 tc = netdev_get_prio_tc_map(dev, skb->priority);
3566	struct netdev_tc_txq res;
3567
3568	res.combined = READ_ONCE(sb_dev->tc_to_txq[tc].combined);
3569	qoffset = res.offset;
3570	qcount = res.count;
3571	if (unlikely(!qcount)) {
3572	net_warn_ratelimited("%s: invalid qcount, qoffset %u for tc %u\n",
3573	sb_dev->name, qoffset, tc);
3574	qoffset = 0;
3575	qcount = dev->real_num_tx_queues;
3576	}
3577	}
3578
3579	if (skb_rx_queue_recorded(skb)) {
3580	DEBUG_NET_WARN_ON_ONCE(qcount == 0);
3581	hash = skb_get_rx_queue(skb);
3582	if (hash >= qoffset)
3583	hash -= qoffset;
3584	while (unlikely(hash >= qcount))
3585	hash -= qcount;
3586	return hash + qoffset;
3587	}
3588
3589	return (u16) reciprocal_scale(skb_get_hash(skb), qcount) + qoffset;
3590	}
3591
3592	void skb_warn_bad_offload(const struct sk_buff *skb)
3593	{
3594	static const netdev_features_t null_features;
3595	struct net_device *dev = skb->dev;
3596	const char *name = "";
3597
3598	if (!net_ratelimit())
3599	return;
3600
3601	if (dev) {
3602	if (dev->dev.parent)
3603	name = dev_driver_string(dev->dev.parent);
3604	else
3605	name = netdev_name(dev);
3606	}
3607	skb_dump(KERN_WARNING, skb, false);
3608	WARN(1, "%s: caps=(%pNF, %pNF)\n",
3609	name, dev ? &dev->features : &null_features,
3610	skb->sk ? &skb->sk->sk_route_caps : &null_features);
3611	}
3612
3613	/*
3614	* Invalidate hardware checksum when packet is to be mangled, and
3615	* complete checksum manually on outgoing path.
3616	*/
3617	int skb_checksum_help(struct sk_buff *skb)
3618	{
3619	__wsum csum;
3620	int ret = 0, offset;
3621
3622	if (skb->ip_summed == CHECKSUM_COMPLETE)
3623	goto out_set_summed;
3624
3625	if (unlikely(skb_is_gso(skb))) {
3626	skb_warn_bad_offload(skb);
3627	return -EINVAL;
3628	}
3629
3630	if (!skb_frags_readable(skb)) {
3631	return -EFAULT;
3632	}
3633
3634	/* Before computing a checksum, we should make sure no frag could
3635	* be modified by an external entity : checksum could be wrong.
3636	*/
3637	if (skb_has_shared_frag(skb)) {
3638	ret = __skb_linearize(skb);
3639	if (ret)
3640	goto out;
3641	}
3642
3643	offset = skb_checksum_start_offset(skb);
3644	ret = -EINVAL;
3645	if (unlikely(offset >= skb_headlen(skb))) {
3646	DO_ONCE_LITE(skb_dump, KERN_ERR, skb, false);
3647	WARN_ONCE(true, "offset (%d) >= skb_headlen() (%u)\n",
3648	offset, skb_headlen(skb));
3649	goto out;
3650	}
3651	csum = skb_checksum(skb, offset, skb->len - offset, 0);
3652
3653	offset += skb->csum_offset;
3654	if (unlikely(offset + sizeof(__sum16) > skb_headlen(skb))) {
3655	DO_ONCE_LITE(skb_dump, KERN_ERR, skb, false);
3656	WARN_ONCE(true, "offset+2 (%zu) > skb_headlen() (%u)\n",
3657	offset + sizeof(__sum16), skb_headlen(skb));
3658	goto out;
3659	}
3660	ret = skb_ensure_writable(skb, offset + sizeof(__sum16));
3661	if (ret)
3662	goto out;
3663
3664	(__sum16 )(skb->data + offset) = csum_fold(csum) ?: CSUM_MANGLED_0;
3665	out_set_summed:
3666	skb->ip_summed = CHECKSUM_NONE;
3667	out:
3668	return ret;
3669	}
3670	EXPORT_SYMBOL(skb_checksum_help);
3671
3672	#ifdef CONFIG_NET_CRC32C
3673	int skb_crc32c_csum_help(struct sk_buff *skb)
3674	{
3675	u32 crc;
3676	int ret = 0, offset, start;
3677
3678	if (skb->ip_summed != CHECKSUM_PARTIAL)
3679	goto out;
3680
3681	if (unlikely(skb_is_gso(skb)))
3682	goto out;
3683
3684	/* Before computing a checksum, we should make sure no frag could
3685	* be modified by an external entity : checksum could be wrong.
3686	*/
3687	if (unlikely(skb_has_shared_frag(skb))) {
3688	ret = __skb_linearize(skb);
3689	if (ret)
3690	goto out;
3691	}
3692	start = skb_checksum_start_offset(skb);
3693	offset = start + offsetof(struct sctphdr, checksum);
3694	if (WARN_ON_ONCE(offset >= skb_headlen(skb))) {
3695	ret = -EINVAL;
3696	goto out;
3697	}
3698
3699	ret = skb_ensure_writable(skb, offset + sizeof(__le32));
3700	if (ret)
3701	goto out;
3702
3703	crc = ~skb_crc32c(skb, start, skb->len - start, ~0);
3704	(__le32 )(skb->data + offset) = cpu_to_le32(crc);
3705	skb_reset_csum_not_inet(skb);
3706	out:
3707	return ret;
3708	}
3709	EXPORT_SYMBOL(skb_crc32c_csum_help);
3710	#endif /* CONFIG_NET_CRC32C */
3711
3712	__be16 skb_network_protocol(struct sk_buff skb, int depth)
3713	{
3714	__be16 type = skb->protocol;
3715
3716	/* Tunnel gso handlers can set protocol to ethernet. */
3717	if (type == htons(ETH_P_TEB)) {
3718	struct ethhdr *eth;
3719
3720	if (unlikely(!pskb_may_pull(skb, sizeof(struct ethhdr))))
3721	return 0;
3722
3723	eth = (struct ethhdr *)skb->data;
3724	type = eth->h_proto;
3725	}
3726
3727	return vlan_get_protocol_and_depth(skb, type, depth);
3728	}
3729
3730
3731	/* Take action when hardware reception checksum errors are detected. */
3732	#ifdef CONFIG_BUG
3733	static void do_netdev_rx_csum_fault(struct net_device dev, struct sk_buff skb)
3734	{
3735	netdev_err(dev, "hw csum failure\n");
3736	skb_dump(KERN_ERR, skb, true);
3737	dump_stack();
3738	}
3739
3740	void netdev_rx_csum_fault(struct net_device dev, struct sk_buff skb)
3741	{
3742	DO_ONCE_LITE(do_netdev_rx_csum_fault, dev, skb);
3743	}
3744	EXPORT_SYMBOL(netdev_rx_csum_fault);
3745	#endif
3746
3747	/* XXX: check that highmem exists at all on the given machine. */
3748	static int illegal_highdma(struct net_device dev, struct sk_buff skb)
3749	{
3750	#ifdef CONFIG_HIGHMEM
3751	int i;
3752
3753	if (!(dev->features & NETIF_F_HIGHDMA)) {
3754	for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) {
3755	skb_frag_t *frag = &skb_shinfo(skb)->frags[i];
3756	struct page *page = skb_frag_page(frag);
3757
3758	if (page && PageHighMem(page))
3759	return 1;
3760	}
3761	}
3762	#endif
3763	return 0;
3764	}
3765
3766	/* If MPLS offload request, verify we are testing hardware MPLS features
3767	* instead of standard features for the netdev.
3768	*/
3769	#if IS_ENABLED(CONFIG_NET_MPLS_GSO)
3770	static netdev_features_t net_mpls_features(struct sk_buff *skb,
3771	netdev_features_t features,
3772	__be16 type)
3773	{
3774	if (eth_p_mpls(type))
3775	features &= skb->dev->mpls_features;
3776
3777	return features;
3778	}
3779	#else
3780	static netdev_features_t net_mpls_features(struct sk_buff *skb,
3781	netdev_features_t features,
3782	__be16 type)
3783	{
3784	return features;
3785	}
3786	#endif
3787
3788	static netdev_features_t harmonize_features(struct sk_buff *skb,
3789	netdev_features_t features)
3790	{
3791	__be16 type;
3792
3793	type = skb_network_protocol(skb, NULL);
3794	features = net_mpls_features(skb, features, type);
3795
3796	if (skb->ip_summed != CHECKSUM_NONE &&
3797	!can_checksum_protocol(features, type)) {
3798	features &= ~(NETIF_F_CSUM_MASK \| NETIF_F_GSO_MASK);
3799	}
3800	if (illegal_highdma(skb->dev, skb))
3801	features &= ~NETIF_F_SG;
3802
3803	return features;
3804	}
3805
3806	netdev_features_t passthru_features_check(struct sk_buff *skb,
3807	struct net_device *dev,
3808	netdev_features_t features)
3809	{
3810	return features;
3811	}
3812	EXPORT_SYMBOL(passthru_features_check);
3813
3814	static netdev_features_t dflt_features_check(struct sk_buff *skb,
3815	struct net_device *dev,
3816	netdev_features_t features)
3817	{
3818	return vlan_features_check(skb, features);
3819	}
3820
3821	static bool __skb_has_ipv6_ext_hdr(const struct sk_buff *skb, int nhoff)
3822	{
3823	const struct ipv6hdr *ip6h;
3824	struct ipv6hdr _ip6h;
3825
3826	ip6h = skb_header_pointer(skb, nhoff, sizeof(_ip6h), &_ip6h);
3827	return ip6h && ip6h->version == 6 && ipv6_ext_hdr(ip6h->nexthdr);
3828	}
3829
3830	static bool skb_has_ipv6_extension_hdr(const struct sk_buff *skb)
3831	{
3832	if (vlan_get_protocol(skb) == htons(ETH_P_IPV6)) {
3833	if (__skb_has_ipv6_ext_hdr(skb, skb_network_offset(skb)))
3834	return true;
3835
3836	/* Catch tunnels without skb->encapsulation (e.g., virtio). */
3837	if (!skb->encapsulation &&
3838	skb_transport_header_was_set(skb) &&
3839	skb_network_header_len(skb) != sizeof(struct ipv6hdr))
3840	return true;
3841	}
3842
3843	/* Tunnels without an inner network header, such as SCTP-in-UDP or
3844	* PSP, have no inner IP header and thus no inner extension header.
3845	*/
3846	if (skb->encapsulation && skb_inner_network_header_was_set(skb) &&
3847	__skb_has_ipv6_ext_hdr(skb, skb_inner_network_offset(skb)))
3848	return true;
3849
3850	return false;
3851	}
3852
3853	static netdev_features_t gso_features_check(const struct sk_buff *skb,
3854	struct net_device *dev,
3855	netdev_features_t features)
3856	{
3857	u16 gso_segs = skb_shinfo(skb)->gso_segs;
3858
3859	if (gso_segs > READ_ONCE(dev->gso_max_segs))
3860	return features & ~NETIF_F_GSO_MASK;
3861
3862	if (unlikely(skb->len >= netif_get_gso_max_size(dev, skb)))
3863	return features & ~NETIF_F_GSO_MASK;
3864
3865	if (!skb_shinfo(skb)->gso_type) {
3866	skb_warn_bad_offload(skb);
3867	return features & ~NETIF_F_GSO_MASK;
3868	}
3869
3870	/* Support for GSO partial features requires software
3871	* intervention before we can actually process the packets
3872	* so we need to strip support for any partial features now
3873	* and we can pull them back in after we have partially
3874	* segmented the frame.
3875	*/
3876	if (!(skb_shinfo(skb)->gso_type & SKB_GSO_PARTIAL))
3877	features &= ~dev->gso_partial_features;
3878
3879	/* Make sure to clear the IPv4 ID mangling feature if the IPv4 header
3880	* has the potential to be fragmented so that TSO does not generate
3881	* segments with the same ID. For encapsulated packets, the ID mangling
3882	* feature is guaranteed not to use the same ID for the outer IPv4
3883	* headers of the generated segments if the headers have the potential
3884	* to be fragmented, so there is no need to clear the IPv4 ID mangling
3885	* feature (see the section about NETIF_F_TSO_MANGLEID in
3886	* segmentation-offloads.rst).
3887	*/
3888	if (skb_shinfo(skb)->gso_type & SKB_GSO_TCPV4) {
3889	const struct iphdr *iph;
3890	struct iphdr _iph;
3891	int nhoff = skb->encapsulation ?
3892	skb_inner_network_offset(skb) :
3893	skb_network_offset(skb);
3894
3895	iph = skb_header_pointer(skb, nhoff, sizeof(_iph), &_iph);
3896
3897	if (!iph \|\| !(iph->frag_off & htons(IP_DF)))
3898	features &= ~dev->mangleid_features;
3899	}
3900
3901	/* NETIF_F_IPV6_CSUM does not support IPv6 extension headers,
3902	* so neither does TSO that depends on it.
3903	*/
3904	if (features & NETIF_F_IPV6_CSUM &&
3905	skb_has_ipv6_extension_hdr(skb))
3906	features &= ~(NETIF_F_IPV6_CSUM \| NETIF_F_TSO6 \| NETIF_F_GSO_UDP_L4);
3907
3908	return features;
3909	}
3910
3911	netdev_features_t netif_skb_features(struct sk_buff *skb)
3912	{
3913	struct net_device *dev = skb->dev;
3914	netdev_features_t features = dev->features;
3915
3916	if (skb_is_gso(skb))
3917	features = gso_features_check(skb, dev, features);
3918
3919	/* If encapsulation offload request, verify we are testing
3920	* hardware encapsulation features instead of standard
3921	* features for the netdev
3922	*/
3923	if (skb->encapsulation)
3924	features &= dev->hw_enc_features;
3925
3926	if (skb_vlan_tagged(skb))
3927	features = netdev_intersect_features(features,
3928	dev->vlan_features \|
3929	NETIF_F_HW_VLAN_CTAG_TX \|
3930	NETIF_F_HW_VLAN_STAG_TX);
3931
3932	if (dev->netdev_ops->ndo_features_check)
3933	features &= dev->netdev_ops->ndo_features_check(skb, dev,
3934	features);
3935	else
3936	features &= dflt_features_check(skb, dev, features);
3937
3938	return harmonize_features(skb, features);
3939	}
3940	EXPORT_SYMBOL(netif_skb_features);
3941
3942	static int xmit_one(struct sk_buff skb, struct net_device dev,
3943	struct netdev_queue *txq, bool more)
3944	{
3945	unsigned int len;
3946	int rc;
3947
3948	if (dev_nit_active_rcu(dev))
3949	dev_queue_xmit_nit(skb, dev);
3950
3951	len = skb->len;
3952	trace_net_dev_start_xmit(skb, dev);
3953	rc = netdev_start_xmit(skb, dev, txq, more);
3954	trace_net_dev_xmit(skb, rc, dev, len);
3955
3956	return rc;
3957	}
3958
3959	struct sk_buff dev_hard_start_xmit(struct sk_buff first, struct net_device *dev,
3960	struct netdev_queue txq, int ret)
3961	{
3962	struct sk_buff *skb = first;
3963	int rc = NETDEV_TX_OK;
3964
3965	while (skb) {
3966	struct sk_buff *next = skb->next;
3967
3968	skb_mark_not_on_list(skb);
3969	rc = xmit_one(skb, dev, txq, next != NULL);
3970	if (unlikely(!dev_xmit_complete(rc))) {
3971	skb->next = next;
3972	goto out;
3973	}
3974
3975	skb = next;
3976	if (netif_tx_queue_stopped(txq) && skb) {
3977	rc = NETDEV_TX_BUSY;
3978	break;
3979	}
3980	}
3981
3982	out:
3983	*ret = rc;
3984	return skb;
3985	}
3986
3987	static struct sk_buff validate_xmit_vlan(struct sk_buff skb,
3988	netdev_features_t features)
3989	{
3990	if (skb_vlan_tag_present(skb) &&
3991	!vlan_hw_offload_capable(features, skb->vlan_proto))
3992	skb = __vlan_hwaccel_push_inside(skb);
3993	return skb;
3994	}
3995
3996	int skb_csum_hwoffload_help(struct sk_buff *skb,
3997	const netdev_features_t features)
3998	{
3999	if (unlikely(skb_csum_is_sctp(skb)))
4000	return !!(features & NETIF_F_SCTP_CRC) ? 0 :
4001	skb_crc32c_csum_help(skb);
4002
4003	if (features & NETIF_F_HW_CSUM)
4004	return 0;
4005
4006	if (features & (NETIF_F_IP_CSUM \| NETIF_F_IPV6_CSUM)) {
4007	if (skb_has_ipv6_extension_hdr(skb))
4008	goto sw_checksum;
4009
4010	switch (skb->csum_offset) {
4011	case offsetof(struct tcphdr, check):
4012	case offsetof(struct udphdr, check):
4013	return 0;
4014	}
4015	}
4016
4017	sw_checksum:
4018	return skb_checksum_help(skb);
4019	}
4020	EXPORT_SYMBOL(skb_csum_hwoffload_help);
4021
4022	/* Checks if this SKB belongs to an HW offloaded socket
4023	* and whether any SW fallbacks are required based on dev.
4024	* Check decrypted mark in case skb_orphan() cleared socket.
4025	*/
4026	static struct sk_buff sk_validate_xmit_skb(struct sk_buff skb,
4027	struct net_device *dev)
4028	{
4029	#ifdef CONFIG_SOCK_VALIDATE_XMIT
4030	struct sk_buff (sk_validate)(struct sock sk, struct net_device dev,
4031	struct sk_buff *skb);
4032	struct sock *sk = skb->sk;
4033
4034	sk_validate = NULL;
4035	if (sk) {
4036	if (sk_fullsock(sk))
4037	sk_validate = sk->sk_validate_xmit_skb;
4038	else if (sk_is_inet(sk) && sk->sk_state == TCP_TIME_WAIT)
4039	sk_validate = inet_twsk(sk)->tw_validate_xmit_skb;
4040	}
4041
4042	if (sk_validate) {
4043	skb = sk_validate(sk, dev, skb);
4044	} else if (unlikely(skb_is_decrypted(skb))) {
4045	pr_warn_ratelimited("unencrypted skb with no associated socket - dropping\n");
4046	kfree_skb(skb);
4047	skb = NULL;
4048	}
4049	#endif
4050
4051	return skb;
4052	}
4053
4054	static struct sk_buff validate_xmit_unreadable_skb(struct sk_buff skb,
4055	struct net_device *dev)
4056	{
4057	struct skb_shared_info *shinfo;
4058	struct net_iov *niov;
4059
4060	if (likely(skb_frags_readable(skb) \|\|
4061	dev->netmem_tx == NETMEM_TX_NO_DMA))
4062	goto out;
4063
4064	if (dev->netmem_tx == NETMEM_TX_NONE)
4065	goto out_free;
4066
4067	shinfo = skb_shinfo(skb);
4068
4069	if (shinfo->nr_frags > 0) {
4070	niov = netmem_to_net_iov(skb_frag_netmem(&shinfo->frags[0]));
4071	if (net_is_devmem_iov(niov) &&
4072	READ_ONCE(net_devmem_iov_binding(niov)->dev) != dev)
4073	goto out_free;
4074	}
4075
4076	out:
4077	return skb;
4078
4079	out_free:
4080	kfree_skb(skb);
4081	return NULL;
4082	}
4083
4084	/* Returns the skb on success, NULL if dropped, or ERR_PTR(-EINPROGRESS)
4085	* if stolen by async xfrm crypto (delivered via xfrm_dev_resume()).
4086	*/
4087	static struct sk_buff validate_xmit_skb(struct sk_buff skb, struct net_device dev, bool again)
4088	{
4089	netdev_features_t features;
4090
4091	skb = validate_xmit_unreadable_skb(skb, dev);
4092	if (unlikely(!skb))
4093	goto out_null;
4094
4095	features = netif_skb_features(skb);
4096	skb = validate_xmit_vlan(skb, features);
4097	if (unlikely(!skb))
4098	goto out_null;
4099
4100	skb = sk_validate_xmit_skb(skb, dev);
4101	if (unlikely(!skb))
4102	goto out_null;
4103
4104	if (netif_needs_gso(skb, features)) {
4105	struct sk_buff *segs;
4106
4107	segs = skb_gso_segment(skb, features);
4108	if (IS_ERR(segs)) {
4109	goto out_kfree_skb;
4110	} else if (segs) {
4111	consume_skb(skb);
4112	skb = segs;
4113	}
4114	} else {
4115	if (skb_needs_linearize(skb, features) &&
4116	__skb_linearize(skb))
4117	goto out_kfree_skb;
4118
4119	/* If packet is not checksummed and device does not
4120	* support checksumming for this protocol, complete
4121	* checksumming here.
4122	*/
4123	if (skb->ip_summed == CHECKSUM_PARTIAL) {
4124	if (skb->encapsulation)
4125	skb_set_inner_transport_header(skb,
4126	skb_checksum_start_offset(skb));
4127	else
4128	skb_set_transport_header(skb,
4129	skb_checksum_start_offset(skb));
4130	if (skb_csum_hwoffload_help(skb, features))
4131	goto out_kfree_skb;
4132	}
4133	}
4134
4135	skb = validate_xmit_xfrm(skb, features, again);
4136
4137	return skb;
4138
4139	out_kfree_skb:
4140	kfree_skb(skb);
4141	out_null:
4142	dev_core_stats_tx_dropped_inc(dev);
4143	return NULL;
4144	}
4145
4146	struct sk_buff validate_xmit_skb_list(struct sk_buff skb, struct net_device dev, bool again)
4147	{
4148	struct sk_buff next, head = NULL, *tail;
4149
4150	for (; skb != NULL; skb = next) {
4151	next = skb->next;
4152	skb_mark_not_on_list(skb);
4153
4154	/* in case skb won't be segmented, point to itself */
4155	skb->prev = skb;
4156
4157	skb = validate_xmit_skb(skb, dev, again);
4158	if (IS_ERR_OR_NULL(skb))
4159	continue;
4160
4161	if (!head)
4162	head = skb;
4163	else
4164	tail->next = skb;
4165	/* If skb was segmented, skb->prev points to
4166	* the last segment. If not, it still contains skb.
4167	*/
4168	tail = skb->prev;
4169	}
4170	return head;
4171	}
4172	EXPORT_SYMBOL_GPL(validate_xmit_skb_list);
4173
4174	static enum skb_drop_reason qdisc_pkt_len_segs_init(struct sk_buff *skb)
4175	{
4176	struct skb_shared_info *shinfo = skb_shinfo(skb);
4177	unsigned int hdr_len, tlen;
4178	u16 gso_segs;
4179
4180	qdisc_skb_cb(skb)->pkt_len = skb->len;
4181	if (!shinfo->gso_size) {
4182	qdisc_skb_cb(skb)->pkt_segs = 1;
4183	return SKB_NOT_DROPPED_YET;
4184	}
4185
4186	qdisc_skb_cb(skb)->pkt_segs = gso_segs = shinfo->gso_segs;
4187
4188	/* To get more precise estimation of bytes sent on wire,
4189	* we add to pkt_len the headers size of all segments
4190	*/
4191
4192	/* mac layer + network layer */
4193	if (!skb->encapsulation) {
4194	if (unlikely(!skb_transport_header_was_set(skb)))
4195	return SKB_NOT_DROPPED_YET;
4196	hdr_len = skb_transport_offset(skb);
4197	} else {
4198	hdr_len = skb_inner_transport_offset(skb);
4199	}
4200	/* + transport layer */
4201	if (likely(shinfo->gso_type & (SKB_GSO_TCPV4 \| SKB_GSO_TCPV6))) {
4202	const struct tcphdr *th;
4203
4204	if (!pskb_may_pull(skb, hdr_len + sizeof(struct tcphdr)))
4205	return SKB_DROP_REASON_SKB_BAD_GSO;
4206
4207	th = (const struct tcphdr *)(skb->data + hdr_len);
4208	tlen = __tcp_hdrlen(th);
4209	if (tlen < sizeof(*th))
4210	return SKB_DROP_REASON_SKB_BAD_GSO;
4211	hdr_len += tlen;
4212	if (!pskb_may_pull(skb, hdr_len))
4213	return SKB_DROP_REASON_SKB_BAD_GSO;
4214	} else if (shinfo->gso_type & SKB_GSO_UDP_L4) {
4215	if (!pskb_may_pull(skb, hdr_len + sizeof(struct udphdr)))
4216	return SKB_DROP_REASON_SKB_BAD_GSO;
4217	hdr_len += sizeof(struct udphdr);
4218	}
4219
4220	/* prior pskb_may_pull() might have changed skb->head. */
4221	shinfo = skb_shinfo(skb);
4222	if (unlikely(shinfo->gso_type & SKB_GSO_DODGY)) {
4223	int payload = skb->len - hdr_len;
4224
4225	/* Malicious packet. */
4226	if (payload <= 0)
4227	return SKB_DROP_REASON_SKB_BAD_GSO;
4228	gso_segs = DIV_ROUND_UP(payload, shinfo->gso_size);
4229	shinfo->gso_segs = gso_segs;
4230	qdisc_skb_cb(skb)->pkt_segs = gso_segs;
4231	}
4232	qdisc_skb_cb(skb)->pkt_len += (gso_segs - 1) * hdr_len;
4233	return SKB_NOT_DROPPED_YET;
4234	}
4235
4236	static int dev_qdisc_enqueue(struct sk_buff skb, struct Qdisc q,
4237	struct sk_buff **to_free,
4238	struct netdev_queue *txq)
4239	{
4240	int rc;
4241
4242	rc = q->enqueue(skb, q, to_free) & NET_XMIT_MASK;
4243	if (rc == NET_XMIT_SUCCESS)
4244	trace_qdisc_enqueue(q, txq, skb);
4245	return rc;
4246	}
4247
4248	static inline int __dev_xmit_skb(struct sk_buff skb, struct Qdisc q,
4249	struct net_device *dev,
4250	struct netdev_queue *txq)
4251	{
4252	struct sk_buff next, to_free = NULL, *to_free2 = NULL;
4253	spinlock_t *root_lock = qdisc_lock(q);
4254	struct llist_node ll_list, first_n;
4255	unsigned long defer_count = 0;
4256	int rc;
4257
4258	qdisc_calculate_pkt_len(skb, q);
4259
4260	tcf_set_qdisc_drop_reason(skb, QDISC_DROP_GENERIC);
4261
4262	if (q->flags & TCQ_F_NOLOCK) {
4263	if (q->flags & TCQ_F_CAN_BYPASS && nolock_qdisc_is_empty(q) &&
4264	qdisc_run_begin(q)) {
4265	/* Retest nolock_qdisc_is_empty() within the protection
4266	* of q->seqlock to protect from racing with requeuing.
4267	*/
4268	if (unlikely(!nolock_qdisc_is_empty(q))) {
4269	rc = dev_qdisc_enqueue(skb, q, &to_free, txq);
4270	__qdisc_run(q);
4271	to_free2 = qdisc_run_end(q);
4272
4273	goto free_skbs;
4274	}
4275
4276	qdisc_bstats_cpu_update(q, skb);
4277	if (sch_direct_xmit(skb, q, dev, txq, NULL, true) &&
4278	!nolock_qdisc_is_empty(q))
4279	__qdisc_run(q);
4280
4281	to_free2 = qdisc_run_end(q);
4282	rc = NET_XMIT_SUCCESS;
4283	goto free_skbs;
4284	}
4285
4286	rc = dev_qdisc_enqueue(skb, q, &to_free, txq);
4287	to_free2 = qdisc_run(q);
4288	goto free_skbs;
4289	}
4290
4291	/* Open code llist_add(&skb->ll_node, &q->defer_list) + queue limit.
4292	* In the try_cmpxchg() loop, we want to increment q->defer_count
4293	* at most once to limit the number of skbs in defer_list.
4294	* We perform the defer_count increment only if the list is not empty,
4295	* because some arches have slow atomic_long_inc_return().
4296	*/
4297	first_n = READ_ONCE(q->defer_list.first);
4298	do {
4299	if (first_n && !defer_count) {
4300	defer_count = atomic_long_inc_return(&q->defer_count);
4301	if (unlikely(defer_count > READ_ONCE(net_hotdata.qdisc_max_burst))) {
4302	kfree_skb_reason(skb, SKB_DROP_REASON_QDISC_BURST_DROP);
4303	return NET_XMIT_DROP;
4304	}
4305	}
4306	skb->ll_node.next = first_n;
4307	} while (!try_cmpxchg(&q->defer_list.first, &first_n, &skb->ll_node));
4308
4309	/* If defer_list was not empty, we know the cpu which queued
4310	* the first skb will process the whole list for us.
4311	*/
4312	if (first_n)
4313	return NET_XMIT_SUCCESS;
4314
4315	spin_lock(root_lock);
4316
4317	ll_list = llist_del_all(&q->defer_list);
4318	/* There is a small race because we clear defer_count not atomically
4319	* with the prior llist_del_all(). This means defer_list could grow
4320	* over qdisc_max_burst.
4321	*/
4322	atomic_long_set(&q->defer_count, 0);
4323
4324	ll_list = llist_reverse_order(ll_list);
4325
4326	if (unlikely(test_bit(__QDISC_STATE_DEACTIVATED, &q->state))) {
4327	llist_for_each_entry_safe(skb, next, ll_list, ll_node)
4328	__qdisc_drop(skb, &to_free);
4329	rc = NET_XMIT_DROP;
4330	goto unlock;
4331	}
4332	if ((q->flags & TCQ_F_CAN_BYPASS) && !qdisc_qlen(q) &&
4333	!llist_next(ll_list) && qdisc_run_begin(q)) {
4334	/*
4335	* This is a work-conserving queue; there are no old skbs
4336	* waiting to be sent out; and the qdisc is not running -
4337	* xmit the skb directly.
4338	*/
4339
4340	DEBUG_NET_WARN_ON_ONCE(skb != llist_entry(ll_list,
4341	struct sk_buff,
4342	ll_node));
4343	qdisc_bstats_update(q, skb);
4344	if (sch_direct_xmit(skb, q, dev, txq, root_lock, true))
4345	__qdisc_run(q);
4346	to_free2 = qdisc_run_end(q);
4347	rc = NET_XMIT_SUCCESS;
4348	} else {
4349	int count = 0;
4350
4351	llist_for_each_entry_safe(skb, next, ll_list, ll_node) {
4352	if (next) {
4353	prefetch(next);
4354	prefetch(&next->priority);
4355	skb_mark_not_on_list(skb);
4356	}
4357	rc = dev_qdisc_enqueue(skb, q, &to_free, txq);
4358	count++;
4359	}
4360	to_free2 = qdisc_run(q);
4361	if (count != 1)
4362	rc = NET_XMIT_SUCCESS;
4363	}
4364	unlock:
4365	spin_unlock(root_lock);
4366
4367	free_skbs:
4368	tcf_kfree_skb_list(to_free, q, txq, dev);
4369	tcf_kfree_skb_list(to_free2, q, txq, dev);
4370	return rc;
4371	}
4372
4373	#if IS_ENABLED(CONFIG_CGROUP_NET_PRIO)
4374	static void skb_update_prio(struct sk_buff *skb)
4375	{
4376	const struct netprio_map *map;
4377	const struct sock *sk;
4378	unsigned int prioidx;
4379
4380	if (skb->priority)
4381	return;
4382	map = rcu_dereference_bh(skb->dev->priomap);
4383	if (!map)
4384	return;
4385	sk = skb_to_full_sk(skb);
4386	if (!sk)
4387	return;
4388
4389	prioidx = sock_cgroup_prioidx(&sk->sk_cgrp_data);
4390
4391	if (prioidx < map->priomap_len)
4392	skb->priority = map->priomap[prioidx];
4393	}
4394	#else
4395	#define skb_update_prio(skb)
4396	#endif
4397
4398	/**
4399	* dev_loopback_xmit - loop back @skb
4400	* @net: network namespace this loopback is happening in
4401	* @sk: sk needed to be a netfilter okfn
4402	* @skb: buffer to transmit
4403	*/
4404	int dev_loopback_xmit(struct net net, struct sock sk, struct sk_buff *skb)
4405	{
4406	skb_reset_mac_header(skb);
4407	__skb_pull(skb, skb_network_offset(skb));
4408	skb->pkt_type = PACKET_LOOPBACK;
4409	if (skb->ip_summed == CHECKSUM_NONE)
4410	skb->ip_summed = CHECKSUM_UNNECESSARY;
4411	DEBUG_NET_WARN_ON_ONCE(!skb_dst(skb));
4412	skb_dst_force(skb);
4413	netif_rx(skb);
4414	return 0;
4415	}
4416	EXPORT_SYMBOL(dev_loopback_xmit);
4417
4418	#ifdef CONFIG_NET_EGRESS
4419	static struct netdev_queue *
4420	netdev_tx_queue_mapping(struct net_device dev, struct sk_buff skb)
4421	{
4422	int queue = skb_get_queue_mapping(skb);
4423	int capped;
4424
4425	capped = netdev_cap_txqueue(dev, queue);
4426	if (unlikely(capped != queue))
4427	skb_set_queue_mapping(skb, capped);
4428
4429	return netdev_get_tx_queue(dev, capped);
4430	}
4431
4432	#ifndef CONFIG_PREEMPT_RT
4433	static bool netdev_xmit_txqueue_skipped(void)
4434	{
4435	return __this_cpu_read(softnet_data.xmit.skip_txqueue);
4436	}
4437
4438	bool netdev_xmit_skip_txqueue(bool skip)
4439	{
4440	bool prev = netdev_xmit_txqueue_skipped();
4441
4442	__this_cpu_write(softnet_data.xmit.skip_txqueue, skip);
4443
4444	return prev;
4445	}
4446	EXPORT_SYMBOL_GPL(netdev_xmit_skip_txqueue);
4447
4448	#else
4449	static bool netdev_xmit_txqueue_skipped(void)
4450	{
4451	return current->net_xmit.skip_txqueue;
4452	}
4453
4454	bool netdev_xmit_skip_txqueue(bool skip)
4455	{
4456	bool prev = netdev_xmit_txqueue_skipped();
4457
4458	current->net_xmit.skip_txqueue = skip;
4459
4460	return prev;
4461	}
4462	EXPORT_SYMBOL_GPL(netdev_xmit_skip_txqueue);
4463	#endif
4464	#endif /* CONFIG_NET_EGRESS */
4465
4466	#ifdef CONFIG_NET_XGRESS
4467	static int tc_run(struct tcx_entry entry, struct sk_buff skb,
4468	enum skb_drop_reason *drop_reason)
4469	{
4470	int ret = TC_ACT_UNSPEC;
4471	#ifdef CONFIG_NET_CLS_ACT
4472	struct mini_Qdisc *miniq = rcu_dereference_bh(entry->miniq);
4473	struct tcf_result res;
4474
4475	if (!miniq)
4476	return ret;
4477
4478	/* Global bypass */
4479	if (!static_branch_likely(&tcf_sw_enabled_key))
4480	return ret;
4481
4482	/* Block-wise bypass */
4483	if (tcf_block_bypass_sw(miniq->block))
4484	return ret;
4485
4486	tc_skb_cb(skb)->mru = 0;
4487	qdisc_skb_cb(skb)->post_ct = false;
4488	tcf_set_drop_reason(skb, *drop_reason);
4489
4490	mini_qdisc_bstats_cpu_update(miniq, skb);
4491	ret = tcf_classify(skb, miniq->block, miniq->filter_list, &res, false);
4492	/* Only tcf related quirks below. */
4493	switch (ret) {
4494	case TC_ACT_SHOT:
4495	*drop_reason = tcf_get_drop_reason(skb);
4496	mini_qdisc_qstats_cpu_drop(miniq);
4497	break;
4498	case TC_ACT_OK:
4499	case TC_ACT_RECLASSIFY:
4500	skb->tc_index = TC_H_MIN(res.classid);
4501	break;
4502	}
4503	#endif /* CONFIG_NET_CLS_ACT */
4504	return ret;
4505	}
4506
4507	static DEFINE_STATIC_KEY_FALSE(tcx_needed_key);
4508
4509	void tcx_inc(void)
4510	{
4511	static_branch_inc(&tcx_needed_key);
4512	}
4513
4514	void tcx_dec(void)
4515	{
4516	static_branch_dec(&tcx_needed_key);
4517	}
4518
4519	static __always_inline enum tcx_action_base
4520	tcx_run(const struct bpf_mprog_entry entry, struct sk_buff skb,
4521	const bool needs_mac)
4522	{
4523	const struct bpf_mprog_fp *fp;
4524	const struct bpf_prog *prog;
4525	int ret = TCX_NEXT;
4526
4527	if (needs_mac)
4528	__skb_push(skb, skb->mac_len);
4529	bpf_mprog_foreach_prog(entry, fp, prog) {
4530	bpf_compute_data_pointers(skb);
4531	ret = bpf_prog_run(prog, skb);
4532	if (ret != TCX_NEXT)
4533	break;
4534	}
4535	if (needs_mac)
4536	__skb_pull(skb, skb->mac_len);
4537	return tcx_action_code(skb, ret);
4538	}
4539
4540	static __always_inline struct sk_buff *
4541	sch_handle_ingress(struct sk_buff skb, struct packet_type pt_prev, int ret,
4542	struct net_device orig_dev, bool another)
4543	{
4544	struct bpf_mprog_entry *entry = rcu_dereference_bh(skb->dev->tcx_ingress);
4545	enum skb_drop_reason drop_reason = SKB_DROP_REASON_TC_INGRESS;
4546	struct bpf_net_context __bpf_net_ctx, *bpf_net_ctx;
4547	int sch_ret;
4548
4549	if (!entry)
4550	return skb;
4551
4552	bpf_net_ctx = bpf_net_ctx_set(&__bpf_net_ctx);
4553	if (unlikely(*pt_prev)) {
4554	ret = deliver_skb(skb, pt_prev, orig_dev);
4555	*pt_prev = NULL;
4556	}
4557
4558	qdisc_pkt_len_segs_init(skb);
4559	tcx_set_ingress(skb, true);
4560
4561	if (static_branch_unlikely(&tcx_needed_key)) {
4562	sch_ret = tcx_run(entry, skb, true);
4563	if (sch_ret != TC_ACT_UNSPEC)
4564	goto ingress_verdict;
4565	}
4566	sch_ret = tc_run(tcx_entry(entry), skb, &drop_reason);
4567	ingress_verdict:
4568	switch (sch_ret) {
4569	case TC_ACT_REDIRECT:
4570	/* skb_mac_header check was done by BPF, so we can safely
4571	* push the L2 header back before redirecting to another
4572	* netdev.
4573	*/
4574	__skb_push(skb, skb->mac_len);
4575	if (skb_do_redirect(skb) == -EAGAIN) {
4576	__skb_pull(skb, skb->mac_len);
4577	*another = true;
4578	break;
4579	}
4580	*ret = NET_RX_SUCCESS;
4581	bpf_net_ctx_clear(bpf_net_ctx);
4582	return NULL;
4583	case TC_ACT_SHOT:
4584	kfree_skb_reason(skb, drop_reason);
4585	*ret = NET_RX_DROP;
4586	bpf_net_ctx_clear(bpf_net_ctx);
4587	return NULL;
4588	/* used by tc_run */
4589	case TC_ACT_STOLEN:
4590	case TC_ACT_QUEUED:
4591	case TC_ACT_TRAP:
4592	consume_skb(skb);
4593	fallthrough;
4594	case TC_ACT_CONSUMED:
4595	*ret = NET_RX_SUCCESS;
4596	bpf_net_ctx_clear(bpf_net_ctx);
4597	return NULL;
4598	}
4599	bpf_net_ctx_clear(bpf_net_ctx);
4600
4601	return skb;
4602	}
4603
4604	static __always_inline struct sk_buff *
4605	sch_handle_egress(struct sk_buff skb, int ret, struct net_device *dev)
4606	{
4607	struct bpf_mprog_entry *entry = rcu_dereference_bh(dev->tcx_egress);
4608	enum skb_drop_reason drop_reason = SKB_DROP_REASON_TC_EGRESS;
4609	struct bpf_net_context __bpf_net_ctx, *bpf_net_ctx;
4610	int sch_ret;
4611
4612	if (!entry)
4613	return skb;
4614
4615	bpf_net_ctx = bpf_net_ctx_set(&__bpf_net_ctx);
4616
4617	/* qdisc_skb_cb(skb)->pkt_len & tcx_set_ingress() was
4618	* already set by the caller.
4619	*/
4620	if (static_branch_unlikely(&tcx_needed_key)) {
4621	sch_ret = tcx_run(entry, skb, false);
4622	if (sch_ret != TC_ACT_UNSPEC)
4623	goto egress_verdict;
4624	}
4625	sch_ret = tc_run(tcx_entry(entry), skb, &drop_reason);
4626	egress_verdict:
4627	switch (sch_ret) {
4628	case TC_ACT_REDIRECT:
4629	/* No need to push/pop skb's mac_header here on egress! */
4630	skb_do_redirect(skb);
4631	*ret = NET_XMIT_SUCCESS;
4632	bpf_net_ctx_clear(bpf_net_ctx);
4633	return NULL;
4634	case TC_ACT_SHOT:
4635	kfree_skb_reason(skb, drop_reason);
4636	*ret = NET_XMIT_DROP;
4637	bpf_net_ctx_clear(bpf_net_ctx);
4638	return NULL;
4639	/* used by tc_run */
4640	case TC_ACT_STOLEN:
4641	case TC_ACT_QUEUED:
4642	case TC_ACT_TRAP:
4643	consume_skb(skb);
4644	fallthrough;
4645	case TC_ACT_CONSUMED:
4646	*ret = NET_XMIT_SUCCESS;
4647	bpf_net_ctx_clear(bpf_net_ctx);
4648	return NULL;
4649	}
4650	bpf_net_ctx_clear(bpf_net_ctx);
4651
4652	return skb;
4653	}
4654	#else
4655	static __always_inline struct sk_buff *
4656	sch_handle_ingress(struct sk_buff skb, struct packet_type pt_prev, int ret,
4657	struct net_device orig_dev, bool another)
4658	{
4659	return skb;
4660	}
4661
4662	static __always_inline struct sk_buff *
4663	sch_handle_egress(struct sk_buff skb, int ret, struct net_device *dev)
4664	{
4665	return skb;
4666	}
4667	#endif /* CONFIG_NET_XGRESS */
4668
4669	#ifdef CONFIG_XPS
4670	static int __get_xps_queue_idx(struct net_device dev, struct sk_buff skb,
4671	struct xps_dev_maps *dev_maps, unsigned int tci)
4672	{
4673	int tc = netdev_get_prio_tc_map(dev, skb->priority);
4674	struct xps_map *map;
4675	int queue_index = -1;
4676
4677	if (tc >= dev_maps->num_tc \|\| tci >= dev_maps->nr_ids)
4678	return queue_index;
4679
4680	tci *= dev_maps->num_tc;
4681	tci += tc;
4682
4683	map = rcu_dereference(dev_maps->attr_map[tci]);
4684	if (map) {
4685	if (map->len == 1)
4686	queue_index = map->queues[0];
4687	else
4688	queue_index = map->queues[reciprocal_scale(
4689	skb_get_hash(skb), map->len)];
4690	if (unlikely(queue_index >= dev->real_num_tx_queues))
4691	queue_index = -1;
4692	}
4693	return queue_index;
4694	}
4695	#endif
4696
4697	static int get_xps_queue(struct net_device dev, struct net_device sb_dev,
4698	struct sk_buff *skb)
4699	{
4700	#ifdef CONFIG_XPS
4701	struct xps_dev_maps *dev_maps;
4702	struct sock *sk = skb->sk;
4703	int queue_index = -1;
4704
4705	if (!static_key_false(&xps_needed))
4706	return -1;
4707
4708	rcu_read_lock();
4709	if (!static_key_false(&xps_rxqs_needed))
4710	goto get_cpus_map;
4711
4712	dev_maps = rcu_dereference(sb_dev->xps_maps[XPS_RXQS]);
4713	if (dev_maps) {
4714	int tci = sk_rx_queue_get(sk);
4715
4716	if (tci >= 0)
4717	queue_index = __get_xps_queue_idx(dev, skb, dev_maps,
4718	tci);
4719	}
4720
4721	get_cpus_map:
4722	if (queue_index < 0) {
4723	dev_maps = rcu_dereference(sb_dev->xps_maps[XPS_CPUS]);
4724	if (dev_maps) {
4725	unsigned int tci = skb->sender_cpu - 1;
4726
4727	queue_index = __get_xps_queue_idx(dev, skb, dev_maps,
4728	tci);
4729	}
4730	}
4731	rcu_read_unlock();
4732
4733	return queue_index;
4734	#else
4735	return -1;
4736	#endif
4737	}
4738
4739	u16 dev_pick_tx_zero(struct net_device dev, struct sk_buff skb,
4740	struct net_device *sb_dev)
4741	{
4742	return 0;
4743	}
4744	EXPORT_SYMBOL(dev_pick_tx_zero);
4745
4746	int sk_tx_queue_get(const struct sock *sk)
4747	{
4748	int resel, val;
4749
4750	if (!sk)
4751	return -1;
4752	/* Paired with WRITE_ONCE() in sk_tx_queue_clear()
4753	* and sk_tx_queue_set().
4754	*/
4755	val = READ_ONCE(sk->sk_tx_queue_mapping);
4756
4757	if (val == NO_QUEUE_MAPPING)
4758	return -1;
4759
4760	if (!sk_fullsock(sk))
4761	return val;
4762
4763	resel = READ_ONCE(sock_net(sk)->core.sysctl_txq_reselection);
4764	if (resel && time_is_before_jiffies(
4765	READ_ONCE(sk->sk_tx_queue_mapping_jiffies) + resel))
4766	return -1;
4767
4768	return val;
4769	}
4770	EXPORT_SYMBOL(sk_tx_queue_get);
4771
4772	u16 netdev_pick_tx(struct net_device dev, struct sk_buff skb,
4773	struct net_device *sb_dev)
4774	{
4775	struct sock *sk = skb->sk;
4776	int queue_index = sk_tx_queue_get(sk);
4777
4778	sb_dev = sb_dev ? : dev;
4779
4780	if (queue_index < 0 \|\| skb->ooo_okay \|\|
4781	queue_index >= dev->real_num_tx_queues) {
4782	int new_index = get_xps_queue(dev, sb_dev, skb);
4783
4784	if (new_index < 0)
4785	new_index = skb_tx_hash(dev, sb_dev, skb);
4786
4787	if (sk && sk_fullsock(sk) &&
4788	rcu_access_pointer(sk->sk_dst_cache))
4789	sk_tx_queue_set(sk, new_index);
4790
4791	queue_index = new_index;
4792	}
4793
4794	return queue_index;
4795	}
4796	EXPORT_SYMBOL(netdev_pick_tx);
4797
4798	struct netdev_queue netdev_core_pick_tx(struct net_device dev,
4799	struct sk_buff *skb,
4800	struct net_device *sb_dev)
4801	{
4802	int queue_index = 0;
4803
4804	#ifdef CONFIG_XPS
4805	u32 sender_cpu = skb->sender_cpu - 1;
4806
4807	if (sender_cpu >= (u32)NR_CPUS)
4808	skb->sender_cpu = raw_smp_processor_id() + 1;
4809	#endif
4810
4811	if (dev->real_num_tx_queues != 1) {
4812	const struct net_device_ops *ops = dev->netdev_ops;
4813
4814	if (ops->ndo_select_queue)
4815	queue_index = ops->ndo_select_queue(dev, skb, sb_dev);
4816	else
4817	queue_index = netdev_pick_tx(dev, skb, sb_dev);
4818
4819	queue_index = netdev_cap_txqueue(dev, queue_index);
4820	}
4821
4822	skb_set_queue_mapping(skb, queue_index);
4823	return netdev_get_tx_queue(dev, queue_index);
4824	}
4825
4826	/**
4827	* __dev_queue_xmit() - transmit a buffer
4828	* @skb: buffer to transmit
4829	* @sb_dev: suboordinate device used for L2 forwarding offload
4830	*
4831	* Queue a buffer for transmission to a network device. The caller must
4832	* have set the device and priority and built the buffer before calling
4833	* this function. The function can be called from an interrupt.
4834	*
4835	* When calling this method, interrupts MUST be enabled. This is because
4836	* the BH enable code must have IRQs enabled so that it will not deadlock.
4837	*
4838	* Regardless of the return value, the skb is consumed, so it is currently
4839	* difficult to retry a send to this method. (You can bump the ref count
4840	* before sending to hold a reference for retry if you are careful.)
4841	*
4842	* Return:
4843	* * 0 - buffer successfully transmitted
4844	* * positive qdisc return code - NET_XMIT_DROP etc.
4845	* * negative errno - other errors
4846	*/
4847	int __dev_queue_xmit(struct sk_buff skb, struct net_device sb_dev)
4848	{
4849	struct net_device *dev = skb->dev;
4850	struct netdev_queue *txq = NULL;
4851	enum skb_drop_reason reason;
4852	int cpu, rc = -ENOMEM;
4853	bool again = false;
4854	struct Qdisc *q;
4855
4856	skb_reset_mac_header(skb);
4857	skb_assert_len(skb);
4858
4859	if (unlikely(skb_shinfo(skb)->tx_flags &
4860	(SKBTX_SCHED_TSTAMP \| SKBTX_BPF)))
4861	__skb_tstamp_tx(skb, NULL, NULL, skb->sk, SCM_TSTAMP_SCHED);
4862
4863	reason = qdisc_pkt_len_segs_init(skb);
4864	if (unlikely(reason)) {
4865	dev_core_stats_tx_dropped_inc(dev);
4866	kfree_skb_reason(skb, reason);
4867	return -EINVAL;
4868	}
4869	/* Disable soft irqs for various locks below. Also
4870	* stops preemption for RCU.
4871	*/
4872	rcu_read_lock_bh();
4873
4874	skb_update_prio(skb);
4875
4876	tcx_set_ingress(skb, false);
4877	#ifdef CONFIG_NET_EGRESS
4878	if (static_branch_unlikely(&egress_needed_key)) {
4879	bool skip_txq;
4880
4881	if (nf_hook_egress_active()) {
4882	skb = nf_hook_egress(skb, &rc, dev);
4883	if (!skb)
4884	goto out;
4885	}
4886
4887	skip_txq = netdev_xmit_skip_txqueue(false);
4888
4889	nf_skip_egress(skb, true);
4890	skb = sch_handle_egress(skb, &rc, dev);
4891	if (!skb) {
4892	netdev_xmit_skip_txqueue(skip_txq);
4893	goto out;
4894	}
4895	nf_skip_egress(skb, false);
4896
4897	if (netdev_xmit_skip_txqueue(skip_txq))
4898	txq = netdev_tx_queue_mapping(dev, skb);
4899	}
4900	#endif
4901	/* If device/qdisc don't need skb->dst, release it right now while
4902	* its hot in this cpu cache.
4903	*/
4904	if (dev->priv_flags & IFF_XMIT_DST_RELEASE)
4905	skb_dst_drop(skb);
4906	else
4907	skb_dst_force(skb);
4908
4909	if (!txq)
4910	txq = netdev_core_pick_tx(dev, skb, sb_dev);
4911
4912	q = rcu_dereference_bh(txq->qdisc);
4913
4914	trace_net_dev_queue(skb);
4915	if (q->enqueue) {
4916	rc = __dev_xmit_skb(skb, q, dev, txq);
4917	goto out;
4918	}
4919
4920	/* The device has no queue. Common case for software devices:
4921	* loopback, all the sorts of tunnels...
4922
4923	* Really, it is unlikely that netif_tx_lock protection is necessary
4924	* here. (f.e. loopback and IP tunnels are clean ignoring statistics
4925	* counters.)
4926	* However, it is possible, that they rely on protection
4927	* made by us here.
4928
4929	* Check this and shot the lock. It is not prone from deadlocks.
4930	*Either shot noqueue qdisc, it is even simpler 8)
4931	*/
4932	if (unlikely(!(dev->flags & IFF_UP))) {
4933	reason = SKB_DROP_REASON_DEV_READY;
4934	goto drop;
4935	}
4936
4937	cpu = smp_processor_id(); /* ok because BHs are off */
4938
4939	if (likely(!netif_tx_owned(txq, cpu))) {
4940	bool is_list = false;
4941
4942	if (dev_xmit_recursion())
4943	goto recursion_alert;
4944
4945	skb = validate_xmit_skb(skb, dev, &again);
4946	if (IS_ERR_OR_NULL(skb)) {
4947	if (PTR_ERR(skb) == -EINPROGRESS)
4948	rc = NET_XMIT_SUCCESS;
4949	goto out;
4950	}
4951
4952	HARD_TX_LOCK(dev, txq, cpu);
4953
4954	if (!netif_xmit_stopped(txq)) {
4955	is_list = !!skb->next;
4956
4957	dev_xmit_recursion_inc();
4958	skb = dev_hard_start_xmit(skb, dev, txq, &rc);
4959	dev_xmit_recursion_dec();
4960
4961	/* GSO segments a single SKB into a list of frames.
4962	* TCP expects error to mean none of the data was sent.
4963	*/
4964	if (is_list)
4965	rc = NETDEV_TX_OK;
4966	}
4967	HARD_TX_UNLOCK(dev, txq);
4968	if (!skb) /* xmit completed */
4969	goto out;
4970
4971	net_crit_ratelimited("Virtual device %s asks to queue packet!\n",
4972	dev->name);
4973	/* NETDEV_TX_BUSY or queue was stopped */
4974	if (!is_list)
4975	rc = -ENETDOWN;
4976	} else {
4977	/* Recursion is detected! It is possible unfortunately. */
4978	recursion_alert:
4979	net_crit_ratelimited("Dead loop on virtual device %s (net %llu), fix it urgently!\n",
4980	dev->name, dev_net(dev)->net_cookie);
4981
4982	rc = -ENETDOWN;
4983	}
4984
4985	reason = SKB_DROP_REASON_RECURSION_LIMIT;
4986	drop:
4987	rcu_read_unlock_bh();
4988
4989	dev_core_stats_tx_dropped_inc(dev);
4990	kfree_skb_list_reason(skb, reason);
4991	return rc;
4992	out:
4993	rcu_read_unlock_bh();
4994	return rc;
4995	}
4996	EXPORT_SYMBOL(__dev_queue_xmit);
4997
4998	int __dev_direct_xmit(struct sk_buff *skb, u16 queue_id)
4999	{
5000	struct net_device *dev = skb->dev;
5001	struct sk_buff *orig_skb = skb;
5002	struct netdev_queue *txq;
5003	int ret = NETDEV_TX_BUSY;
5004	bool again = false;
5005
5006	if (unlikely(!netif_running(dev) \|\|
5007	!netif_carrier_ok(dev)))
5008	goto drop;
5009
5010	skb = validate_xmit_skb_list(skb, dev, &again);
5011	if (skb != orig_skb)
5012	goto drop;
5013
5014	skb_set_queue_mapping(skb, queue_id);
5015	txq = skb_get_tx_queue(dev, skb);
5016
5017	local_bh_disable();
5018
5019	dev_xmit_recursion_inc();
5020	HARD_TX_LOCK(dev, txq, smp_processor_id());
5021	if (!netif_xmit_frozen_or_drv_stopped(txq))
5022	ret = netdev_start_xmit(skb, dev, txq, false);
5023	HARD_TX_UNLOCK(dev, txq);
5024	dev_xmit_recursion_dec();
5025
5026	local_bh_enable();
5027	return ret;
5028	drop:
5029	dev_core_stats_tx_dropped_inc(dev);
5030	kfree_skb_list(skb);
5031	return NET_XMIT_DROP;
5032	}
5033	EXPORT_SYMBOL(__dev_direct_xmit);
5034
5035	/*************************************************************************
5036	* Receiver routines
5037	*************************************************************************/
5038	static DEFINE_PER_CPU(struct task_struct *, backlog_napi);
5039
5040	int weight_p __read_mostly = 64; /* old backlog weight */
5041	int dev_weight_rx_bias __read_mostly = 1; /* bias for backlog weight */
5042	int dev_weight_tx_bias __read_mostly = 1; /* bias for output_queue quota */
5043
5044	/* Called with irq disabled */
5045	static inline void ____napi_schedule(struct softnet_data *sd,
5046	struct napi_struct *napi)
5047	{
5048	struct task_struct *thread;
5049
5050	lockdep_assert_irqs_disabled();
5051
5052	if (test_bit(NAPI_STATE_THREADED, &napi->state)) {
5053	/* Paired with smp_mb__before_atomic() in
5054	* napi_enable()/netif_set_threaded().
5055	* Use READ_ONCE() to guarantee a complete
5056	* read on napi->thread. Only call
5057	* wake_up_process() when it's not NULL.
5058	*/
5059	thread = READ_ONCE(napi->thread);
5060	if (thread) {
5061	if (use_backlog_threads() && thread == raw_cpu_read(backlog_napi))
5062	goto use_local_napi;
5063
5064	set_bit(NAPI_STATE_SCHED_THREADED, &napi->state);
5065	wake_up_process(thread);
5066	return;
5067	}
5068	}
5069
5070	use_local_napi:
5071	DEBUG_NET_WARN_ON_ONCE(!list_empty(&napi->poll_list));
5072	list_add_tail(&napi->poll_list, &sd->poll_list);
5073	WRITE_ONCE(napi->list_owner, smp_processor_id());
5074	/* If not called from net_rx_action()
5075	* we have to raise NET_RX_SOFTIRQ.
5076	*/
5077	if (!sd->in_net_rx_action)
5078	raise_softirq_irqoff(NET_RX_SOFTIRQ);
5079	}
5080
5081	#ifdef CONFIG_RPS
5082
5083	struct static_key_false rps_needed __read_mostly;
5084	EXPORT_SYMBOL(rps_needed);
5085	struct static_key_false rfs_needed __read_mostly;
5086	EXPORT_SYMBOL(rfs_needed);
5087
5088	static u32 rfs_slot(u32 hash, rps_tag_ptr tag_ptr)
5089	{
5090	return hash_32(hash, rps_tag_to_log(tag_ptr));
5091	}
5092
5093	#ifdef CONFIG_RFS_ACCEL
5094	/**
5095	* rps_flow_is_active - check whether the flow is recently active.
5096	* @rflow: Specific flow to check activity.
5097	* @log: ilog2(hashsize).
5098	* @cpu: CPU saved in @rflow.
5099	*
5100	* If the CPU has processed many packets since the flow's last activity
5101	* (beyond 10 times the table size), the flow is considered stale.
5102	*
5103	* Return: true if flow was recently active.
5104	*/
5105	static bool rps_flow_is_active(struct rps_dev_flow *rflow,
5106	u8 log,
5107	unsigned int cpu)
5108	{
5109	unsigned int flow_last_active;
5110	unsigned int sd_input_head;
5111
5112	if (cpu >= nr_cpu_ids)
5113	return false;
5114
5115	sd_input_head = READ_ONCE(per_cpu(softnet_data, cpu).input_queue_head);
5116	flow_last_active = READ_ONCE(rflow->last_qtail);
5117
5118	return (int)(sd_input_head - flow_last_active) <
5119	(int)(10 << log);
5120	}
5121	#endif
5122
5123	static struct rps_dev_flow *
5124	set_rps_cpu(struct net_device dev, struct sk_buff skb,
5125	struct rps_dev_flow *rflow, u16 next_cpu, u32 hash)
5126	{
5127	if (next_cpu < nr_cpu_ids) {
5128	u32 head;
5129	#ifdef CONFIG_RFS_ACCEL
5130	struct netdev_rx_queue *rxqueue;
5131	struct rps_dev_flow *flow_table;
5132	struct rps_dev_flow *old_rflow;
5133	struct rps_dev_flow *tmp_rflow;
5134	rps_tag_ptr q_tag_ptr;
5135	unsigned int tmp_cpu;
5136	u16 rxq_index;
5137	u32 flow_id;
5138	int rc;
5139
5140	/* Should we steer this flow to a different hardware queue? */
5141	if (!skb_rx_queue_recorded(skb) \|\| !dev->rx_cpu_rmap \|\|
5142	!(dev->features & NETIF_F_NTUPLE))
5143	goto out;
5144	rxq_index = cpu_rmap_lookup_index(dev->rx_cpu_rmap, next_cpu);
5145	if (rxq_index == skb_get_rx_queue(skb))
5146	goto out;
5147
5148	rxqueue = dev->_rx + rxq_index;
5149	q_tag_ptr = READ_ONCE(rxqueue->rps_flow_table);
5150	if (!q_tag_ptr)
5151	goto out;
5152
5153	flow_id = rfs_slot(hash, q_tag_ptr);
5154	flow_table = rps_tag_to_table(q_tag_ptr);
5155	tmp_rflow = flow_table + flow_id;
5156	tmp_cpu = READ_ONCE(tmp_rflow->cpu);
5157
5158	if (READ_ONCE(tmp_rflow->filter) != RPS_NO_FILTER) {
5159	if (rps_flow_is_active(tmp_rflow,
5160	rps_tag_to_log(q_tag_ptr),
5161	tmp_cpu)) {
5162	if (hash != READ_ONCE(tmp_rflow->hash) \|\|
5163	next_cpu == tmp_cpu)
5164	goto out;
5165	}
5166	}
5167
5168	rc = dev->netdev_ops->ndo_rx_flow_steer(dev, skb,
5169	rxq_index, flow_id);
5170	if (rc < 0)
5171	goto out;
5172
5173	old_rflow = rflow;
5174	rflow = tmp_rflow;
5175	WRITE_ONCE(rflow->filter, rc);
5176	WRITE_ONCE(rflow->hash, hash);
5177
5178	if (old_rflow->filter == rc)
5179	WRITE_ONCE(old_rflow->filter, RPS_NO_FILTER);
5180	out:
5181	#endif
5182	head = READ_ONCE(per_cpu(softnet_data, next_cpu).input_queue_head);
5183	rps_input_queue_tail_save(&rflow->last_qtail, head);
5184	}
5185
5186	WRITE_ONCE(rflow->cpu, next_cpu);
5187	return rflow;
5188	}
5189
5190	/*
5191	* get_rps_cpu is called from netif_receive_skb and returns the target
5192	* CPU from the RPS map of the receiving queue for a given skb.
5193	* rcu_read_lock must be held on entry.
5194	*/
5195	static int get_rps_cpu(struct net_device dev, struct sk_buff skb,
5196	struct rps_dev_flow **rflowp)
5197	{
5198	struct netdev_rx_queue *rxqueue = dev->_rx;
5199	rps_tag_ptr global_tag_ptr, q_tag_ptr;
5200	struct rps_map *map;
5201	int cpu = -1;
5202	u32 tcpu;
5203	u32 hash;
5204
5205	if (skb_rx_queue_recorded(skb)) {
5206	u16 index = skb_get_rx_queue(skb);
5207
5208	if (unlikely(index >= dev->real_num_rx_queues)) {
5209	WARN_ONCE(dev->real_num_rx_queues > 1,
5210	"%s received packet on queue %u, but number "
5211	"of RX queues is %u\n",
5212	dev->name, index, dev->real_num_rx_queues);
5213	goto done;
5214	}
5215	rxqueue += index;
5216	}
5217
5218	/* Avoid computing hash if RFS/RPS is not active for this rxqueue */
5219
5220	q_tag_ptr = READ_ONCE(rxqueue->rps_flow_table);
5221	map = rcu_dereference(rxqueue->rps_map);
5222	if (!q_tag_ptr && !map)
5223	goto done;
5224
5225	skb_reset_network_header(skb);
5226	hash = skb_get_hash(skb);
5227	if (!hash)
5228	goto done;
5229
5230	global_tag_ptr = READ_ONCE(net_hotdata.rps_sock_flow_table);
5231	if (q_tag_ptr && global_tag_ptr) {
5232	struct rps_sock_flow_table *sock_flow_table;
5233	struct rps_dev_flow *flow_table;
5234	struct rps_dev_flow *rflow;
5235	u32 next_cpu;
5236	u32 flow_id;
5237	u32 ident;
5238
5239	/* First check into global flow table if there is a match.
5240	* This READ_ONCE() pairs with WRITE_ONCE() from rps_record_sock_flow().
5241	*/
5242	flow_id = hash & rps_tag_to_mask(global_tag_ptr);
5243	sock_flow_table = rps_tag_to_table(global_tag_ptr);
5244	ident = READ_ONCE(sock_flow_table[flow_id].ent);
5245	if ((ident ^ hash) & ~net_hotdata.rps_cpu_mask)
5246	goto try_rps;
5247
5248	next_cpu = ident & net_hotdata.rps_cpu_mask;
5249
5250	/* OK, now we know there is a match,
5251	* we can look at the local (per receive queue) flow table
5252	*/
5253	flow_id = rfs_slot(hash, q_tag_ptr);
5254	flow_table = rps_tag_to_table(q_tag_ptr);
5255	rflow = flow_table + flow_id;
5256	tcpu = rflow->cpu;
5257
5258	/*
5259	* If the desired CPU (where last recvmsg was done) is
5260	* different from current CPU (one in the rx-queue flow
5261	* table entry), switch if one of the following holds:
5262	* - Current CPU is unset (>= nr_cpu_ids).
5263	* - Current CPU is offline.
5264	* - The current CPU's queue tail has advanced beyond the
5265	* last packet that was enqueued using this table entry.
5266	* This guarantees that all previous packets for the flow
5267	* have been dequeued, thus preserving in order delivery.
5268	*/
5269	if (unlikely(tcpu != next_cpu) &&
5270	(tcpu >= nr_cpu_ids \|\| !cpu_online(tcpu) \|\|
5271	((int)(READ_ONCE(per_cpu(softnet_data, tcpu).input_queue_head) -
5272	rflow->last_qtail)) >= 0)) {
5273	tcpu = next_cpu;
5274	rflow = set_rps_cpu(dev, skb, rflow, next_cpu, hash);
5275	}
5276
5277	if (tcpu < nr_cpu_ids && cpu_online(tcpu)) {
5278	*rflowp = rflow;
5279	cpu = tcpu;
5280	goto done;
5281	}
5282	}
5283
5284	try_rps:
5285
5286	if (map) {
5287	tcpu = map->cpus[reciprocal_scale(hash, map->len)];
5288	if (cpu_online(tcpu)) {
5289	cpu = tcpu;
5290	goto done;
5291	}
5292	}
5293
5294	done:
5295	return cpu;
5296	}
5297
5298	#ifdef CONFIG_RFS_ACCEL
5299
5300	/**
5301	* rps_may_expire_flow - check whether an RFS hardware filter may be removed
5302	* @dev: Device on which the filter was set
5303	* @rxq_index: RX queue index
5304	* @flow_id: Flow ID passed to ndo_rx_flow_steer()
5305	* @filter_id: Filter ID returned by ndo_rx_flow_steer()
5306	*
5307	* Drivers that implement ndo_rx_flow_steer() should periodically call
5308	* this function for each installed filter and remove the filters for
5309	* which it returns %true.
5310	*/
5311	bool rps_may_expire_flow(struct net_device *dev, u16 rxq_index,
5312	u32 flow_id, u16 filter_id)
5313	{
5314	struct netdev_rx_queue *rxqueue = dev->_rx + rxq_index;
5315	struct rps_dev_flow *flow_table;
5316	struct rps_dev_flow *rflow;
5317	rps_tag_ptr q_tag_ptr;
5318	bool expire = true;
5319	u8 log;
5320
5321	rcu_read_lock();
5322	q_tag_ptr = READ_ONCE(rxqueue->rps_flow_table);
5323	log = rps_tag_to_log(q_tag_ptr);
5324	if (q_tag_ptr && flow_id < (1UL << log)) {
5325	unsigned int cpu;
5326
5327	flow_table = rps_tag_to_table(q_tag_ptr);
5328	rflow = flow_table + flow_id;
5329	cpu = READ_ONCE(rflow->cpu);
5330	if (READ_ONCE(rflow->filter) == filter_id &&
5331	rps_flow_is_active(rflow, log, cpu))
5332	expire = false;
5333	}
5334	rcu_read_unlock();
5335	return expire;
5336	}
5337	EXPORT_SYMBOL(rps_may_expire_flow);
5338
5339	#endif /* CONFIG_RFS_ACCEL */
5340
5341	/* Called from hardirq (IPI) context */
5342	static void rps_trigger_softirq(void *data)
5343	{
5344	struct softnet_data *sd = data;
5345
5346	____napi_schedule(sd, &sd->backlog);
5347	/* Pairs with READ_ONCE() in softnet_seq_show() */
5348	WRITE_ONCE(sd->received_rps, sd->received_rps + 1);
5349	}
5350
5351	#endif /* CONFIG_RPS */
5352
5353	/* Called from hardirq (IPI) context */
5354	static void trigger_rx_softirq(void *data)
5355	{
5356	struct softnet_data *sd = data;
5357
5358	__raise_softirq_irqoff(NET_RX_SOFTIRQ);
5359	smp_store_release(&sd->defer_ipi_scheduled, 0);
5360	}
5361
5362	/*
5363	* After we queued a packet into sd->input_pkt_queue,
5364	* we need to make sure this queue is serviced soon.
5365	*
5366	* - If this is another cpu queue, link it to our rps_ipi_list,
5367	* and make sure we will process rps_ipi_list from net_rx_action().
5368	*
5369	* - If this is our own queue, NAPI schedule our backlog.
5370	* Note that this also raises NET_RX_SOFTIRQ.
5371	*/
5372	static void napi_schedule_rps(struct softnet_data *sd)
5373	{
5374	struct softnet_data *mysd = this_cpu_ptr(&softnet_data);
5375
5376	#ifdef CONFIG_RPS
5377	if (sd != mysd) {
5378	if (use_backlog_threads()) {
5379	__napi_schedule_irqoff(&sd->backlog);
5380	return;
5381	}
5382
5383	sd->rps_ipi_next = mysd->rps_ipi_list;
5384	mysd->rps_ipi_list = sd;
5385
5386	/* If not called from net_rx_action() or napi_threaded_poll()
5387	* we have to raise NET_RX_SOFTIRQ.
5388	*/
5389	if (!mysd->in_net_rx_action && !mysd->in_napi_threaded_poll)
5390	__raise_softirq_irqoff(NET_RX_SOFTIRQ);
5391	return;
5392	}
5393	#endif /* CONFIG_RPS */
5394	__napi_schedule_irqoff(&mysd->backlog);
5395	}
5396
5397	void kick_defer_list_purge(unsigned int cpu)
5398	{
5399	struct softnet_data *sd = &per_cpu(softnet_data, cpu);
5400	unsigned long flags;
5401
5402	if (use_backlog_threads()) {
5403	backlog_lock_irq_save(sd, &flags);
5404
5405	if (!__test_and_set_bit(NAPI_STATE_SCHED, &sd->backlog.state))
5406	__napi_schedule_irqoff(&sd->backlog);
5407
5408	backlog_unlock_irq_restore(sd, flags);
5409
5410	} else if (!cmpxchg(&sd->defer_ipi_scheduled, 0, 1)) {
5411	if (smp_call_function_single_async(cpu, &sd->defer_csd))
5412	WRITE_ONCE(sd->defer_ipi_scheduled, 0);
5413	}
5414	}
5415
5416	#ifdef CONFIG_NET_FLOW_LIMIT
5417	int netdev_flow_limit_table_len __read_mostly = (1 << 12);
5418	#endif
5419
5420	static bool skb_flow_limit(struct sk_buff *skb, unsigned int qlen,
5421	int max_backlog)
5422	{
5423	#ifdef CONFIG_NET_FLOW_LIMIT
5424	unsigned int old_flow, new_flow;
5425	const struct softnet_data *sd;
5426	struct sd_flow_limit *fl;
5427
5428	if (likely(qlen < (max_backlog >> 1)))
5429	return false;
5430
5431	sd = this_cpu_ptr(&softnet_data);
5432
5433	rcu_read_lock();
5434	fl = rcu_dereference(sd->flow_limit);
5435	if (fl) {
5436	new_flow = hash_32(skb_get_hash(skb), fl->log_buckets);
5437	old_flow = fl->history[fl->history_head];
5438	fl->history[fl->history_head] = new_flow;
5439
5440	fl->history_head++;
5441	fl->history_head &= FLOW_LIMIT_HISTORY - 1;
5442
5443	if (likely(fl->buckets[old_flow]))
5444	fl->buckets[old_flow]--;
5445
5446	if (++fl->buckets[new_flow] > (FLOW_LIMIT_HISTORY >> 1)) {
5447	/* Pairs with READ_ONCE() in softnet_seq_show() */
5448	WRITE_ONCE(fl->count, fl->count + 1);
5449	rcu_read_unlock();
5450	return true;
5451	}
5452	}
5453	rcu_read_unlock();
5454	#endif
5455	return false;
5456	}
5457
5458	/*
5459	* enqueue_to_backlog is called to queue an skb to a per CPU backlog
5460	* queue (may be a remote CPU queue).
5461	*/
5462	static int enqueue_to_backlog(struct sk_buff *skb, int cpu,
5463	unsigned int *qtail)
5464	{
5465	enum skb_drop_reason reason;
5466	struct softnet_data *sd;
5467	unsigned long flags;
5468	unsigned int qlen;
5469	int max_backlog;
5470	u32 tail;
5471
5472	reason = SKB_DROP_REASON_DEV_READY;
5473
5474	sd = &per_cpu(softnet_data, cpu);
5475
5476	qlen = skb_queue_len_lockless(&sd->input_pkt_queue);
5477	max_backlog = READ_ONCE(net_hotdata.max_backlog);
5478	if (unlikely(qlen > max_backlog) \|\|
5479	skb_flow_limit(skb, qlen, max_backlog))
5480	goto cpu_backlog_drop;
5481	backlog_lock_irq_save(sd, &flags);
5482	qlen = skb_queue_len(&sd->input_pkt_queue);
5483	if (likely(qlen <= max_backlog)) {
5484	if (unlikely(!netif_running(skb->dev))) {
5485	backlog_unlock_irq_restore(sd, flags);
5486	goto bad_dev;
5487	}
5488	if (!qlen) {
5489	/* Schedule NAPI for backlog device. We can use
5490	* non atomic operation as we own the queue lock.
5491	*/
5492	if (!__test_and_set_bit(NAPI_STATE_SCHED,
5493	&sd->backlog.state))
5494	napi_schedule_rps(sd);
5495	}
5496	__skb_queue_tail(&sd->input_pkt_queue, skb);
5497	tail = rps_input_queue_tail_incr(sd);
5498	backlog_unlock_irq_restore(sd, flags);
5499
5500	/* save the tail outside of the critical section */
5501	rps_input_queue_tail_save(qtail, tail);
5502	return NET_RX_SUCCESS;
5503	}
5504
5505	backlog_unlock_irq_restore(sd, flags);
5506
5507	cpu_backlog_drop:
5508	reason = SKB_DROP_REASON_CPU_BACKLOG;
5509	numa_drop_add(&sd->drop_counters, 1);
5510	bad_dev:
5511	dev_core_stats_rx_dropped_inc(skb->dev);
5512	kfree_skb_reason(skb, reason);
5513	return NET_RX_DROP;
5514	}
5515
5516	static struct netdev_rx_queue netif_get_rxqueue(struct sk_buff skb)
5517	{
5518	struct net_device *dev = skb->dev;
5519	struct netdev_rx_queue *rxqueue;
5520
5521	rxqueue = dev->_rx;
5522
5523	if (skb_rx_queue_recorded(skb)) {
5524	u16 index = skb_get_rx_queue(skb);
5525
5526	if (unlikely(index >= dev->real_num_rx_queues)) {
5527	WARN_ONCE(dev->real_num_rx_queues > 1,
5528	"%s received packet on queue %u, but number "
5529	"of RX queues is %u\n",
5530	dev->name, index, dev->real_num_rx_queues);
5531
5532	return rxqueue; /* Return first rxqueue */
5533	}
5534	rxqueue += index;
5535	}
5536	return rxqueue;
5537	}
5538
5539	u32 bpf_prog_run_generic_xdp(struct sk_buff skb, struct xdp_buff xdp,
5540	const struct bpf_prog *xdp_prog)
5541	{
5542	void orig_data, orig_data_end, *hard_start;
5543	struct netdev_rx_queue *rxqueue;
5544	bool orig_bcast, orig_host;
5545	u32 mac_len, frame_sz;
5546	__be16 orig_eth_type;
5547	struct ethhdr *eth;
5548	u32 metalen, act;
5549	int off;
5550
5551	/* The XDP program wants to see the packet starting at the MAC
5552	* header.
5553	*/
5554	mac_len = skb->data - skb_mac_header(skb);
5555	hard_start = skb->data - skb_headroom(skb);
5556
5557	/* SKB "head" area always have tailroom for skb_shared_info */
5558	frame_sz = (void *)skb_end_pointer(skb) - hard_start;
5559	frame_sz += SKB_DATA_ALIGN(sizeof(struct skb_shared_info));
5560
5561	rxqueue = netif_get_rxqueue(skb);
5562	xdp_init_buff(xdp, frame_sz, &rxqueue->xdp_rxq);
5563	xdp_prepare_buff(xdp, hard_start, skb_headroom(skb) - mac_len,
5564	skb_headlen(skb) + mac_len, true);
5565	if (skb_is_nonlinear(skb)) {
5566	skb_shinfo(skb)->xdp_frags_size = skb->data_len;
5567	xdp_buff_set_frags_flag(xdp);
5568	} else {
5569	xdp_buff_clear_frags_flag(xdp);
5570	}
5571
5572	orig_data_end = xdp->data_end;
5573	orig_data = xdp->data;
5574	eth = (struct ethhdr *)xdp->data;
5575	orig_host = ether_addr_equal_64bits(eth->h_dest, skb->dev->dev_addr);
5576	orig_bcast = is_multicast_ether_addr_64bits(eth->h_dest);
5577	orig_eth_type = eth->h_proto;
5578
5579	act = bpf_prog_run_xdp(xdp_prog, xdp);
5580
5581	/* check if bpf_xdp_adjust_head was used */
5582	off = xdp->data - orig_data;
5583	if (off) {
5584	if (off > 0)
5585	__skb_pull(skb, off);
5586	else if (off < 0)
5587	__skb_push(skb, -off);
5588
5589	skb->mac_header += off;
5590	skb_reset_network_header(skb);
5591	}
5592
5593	/* check if bpf_xdp_adjust_tail was used */
5594	off = xdp->data_end - orig_data_end;
5595	if (off != 0) {
5596	skb_set_tail_pointer(skb, xdp->data_end - xdp->data);
5597	skb->len += off; /* positive on grow, negative on shrink */
5598	}
5599
5600	/* XDP frag metadata (e.g. nr_frags) are updated in eBPF helpers
5601	* (e.g. bpf_xdp_adjust_tail). Remove the old fragment contribution
5602	* from skb->len before updating data_len, then add the new one back.
5603	*/
5604	skb->len -= skb->data_len;
5605	if (xdp_buff_has_frags(xdp)) {
5606	skb->data_len = skb_shinfo(skb)->xdp_frags_size;
5607	skb->len += skb->data_len;
5608	} else {
5609	skb->data_len = 0;
5610	}
5611
5612	/* check if XDP changed eth hdr such SKB needs update */
5613	eth = (struct ethhdr *)xdp->data;
5614	if ((orig_eth_type != eth->h_proto) \|\|
5615	(orig_host != ether_addr_equal_64bits(eth->h_dest,
5616	skb->dev->dev_addr)) \|\|
5617	(orig_bcast != is_multicast_ether_addr_64bits(eth->h_dest))) {
5618	__skb_push(skb, ETH_HLEN);
5619	skb->pkt_type = PACKET_HOST;
5620	skb->protocol = eth_type_trans(skb, skb->dev);
5621	}
5622
5623	/* Redirect/Tx gives L2 packet, code that will reuse skb must __skb_pull
5624	* before calling us again on redirect path. We do not call do_redirect
5625	* as we leave that up to the caller.
5626	*
5627	* Caller is responsible for managing lifetime of skb (i.e. calling
5628	* kfree_skb in response to actions it cannot handle/XDP_DROP).
5629	*/
5630	switch (act) {
5631	case XDP_REDIRECT:
5632	case XDP_TX:
5633	__skb_push(skb, mac_len);
5634	break;
5635	case XDP_PASS:
5636	metalen = xdp->data - xdp->data_meta;
5637	if (metalen)
5638	skb_metadata_set(skb, metalen);
5639	break;
5640	}
5641
5642	return act;
5643	}
5644
5645	static int
5646	netif_skb_check_for_xdp(struct sk_buff *pskb, const struct bpf_prog prog)
5647	{
5648	struct sk_buff skb = pskb;
5649	int err, hroom, troom;
5650
5651	local_lock_nested_bh(&system_page_pool.bh_lock);
5652	err = skb_cow_data_for_xdp(this_cpu_read(system_page_pool.pool), pskb, prog);
5653	local_unlock_nested_bh(&system_page_pool.bh_lock);
5654	if (!err)
5655	return 0;
5656
5657	/* In case we have to go down the path and also linearize,
5658	* then lets do the pskb_expand_head() work just once here.
5659	*/
5660	hroom = XDP_PACKET_HEADROOM - skb_headroom(skb);
5661	troom = skb->tail + skb->data_len - skb->end;
5662	err = pskb_expand_head(skb,
5663	hroom > 0 ? ALIGN(hroom, NET_SKB_PAD) : 0,
5664	troom > 0 ? troom + 128 : 0, GFP_ATOMIC);
5665	if (err)
5666	return err;
5667
5668	return skb_linearize(skb);
5669	}
5670
5671	static u32 netif_receive_generic_xdp(struct sk_buff **pskb,
5672	struct xdp_buff *xdp,
5673	const struct bpf_prog *xdp_prog)
5674	{
5675	struct sk_buff skb = pskb;
5676	u32 mac_len, act = XDP_DROP;
5677
5678	/* Reinjected packets coming from act_mirred or similar should
5679	* not get XDP generic processing.
5680	*/
5681	if (skb_is_redirected(skb))
5682	return XDP_PASS;
5683
5684	/* XDP packets must have sufficient headroom of XDP_PACKET_HEADROOM
5685	* bytes. This is the guarantee that also native XDP provides,
5686	* thus we need to do it here as well.
5687	*/
5688	mac_len = skb->data - skb_mac_header(skb);
5689	__skb_push(skb, mac_len);
5690
5691	if (skb_cloned(skb) \|\| skb_is_nonlinear(skb) \|\|
5692	skb_headroom(skb) < XDP_PACKET_HEADROOM) {
5693	if (netif_skb_check_for_xdp(pskb, xdp_prog))
5694	goto do_drop;
5695	}
5696
5697	__skb_pull(*pskb, mac_len);
5698
5699	act = bpf_prog_run_generic_xdp(*pskb, xdp, xdp_prog);
5700	switch (act) {
5701	case XDP_REDIRECT:
5702	case XDP_TX:
5703	case XDP_PASS:
5704	break;
5705	default:
5706	bpf_warn_invalid_xdp_action((*pskb)->dev, xdp_prog, act);
5707	fallthrough;
5708	case XDP_ABORTED:
5709	trace_xdp_exception((*pskb)->dev, xdp_prog, act);
5710	fallthrough;
5711	case XDP_DROP:
5712	do_drop:
5713	kfree_skb(*pskb);
5714	break;
5715	}
5716
5717	return act;
5718	}
5719
5720	/* When doing generic XDP we have to bypass the qdisc layer and the
5721	* network taps in order to match in-driver-XDP behavior. This also means
5722	* that XDP packets are able to starve other packets going through a qdisc,
5723	* and DDOS attacks will be more effective. In-driver-XDP use dedicated TX
5724	* queues, so they do not have this starvation issue.
5725	*/
5726	void generic_xdp_tx(struct sk_buff skb, const struct bpf_prog xdp_prog)
5727	{
5728	struct net_device *dev = skb->dev;
5729	struct netdev_queue *txq;
5730	bool free_skb = true;
5731	int cpu, rc;
5732
5733	txq = netdev_core_pick_tx(dev, skb, NULL);
5734	cpu = smp_processor_id();
5735	HARD_TX_LOCK(dev, txq, cpu);
5736	if (!netif_xmit_frozen_or_drv_stopped(txq)) {
5737	rc = netdev_start_xmit(skb, dev, txq, 0);
5738	if (dev_xmit_complete(rc))
5739	free_skb = false;
5740	}
5741	HARD_TX_UNLOCK(dev, txq);
5742	if (free_skb) {
5743	trace_xdp_exception(dev, xdp_prog, XDP_TX);
5744	dev_core_stats_tx_dropped_inc(dev);
5745	kfree_skb(skb);
5746	}
5747	}
5748
5749	static DEFINE_STATIC_KEY_FALSE(generic_xdp_needed_key);
5750
5751	int do_xdp_generic(const struct bpf_prog xdp_prog, struct sk_buff *pskb)
5752	{
5753	struct bpf_net_context __bpf_net_ctx, *bpf_net_ctx;
5754
5755	if (xdp_prog) {
5756	struct xdp_buff xdp;
5757	u32 act;
5758	int err;
5759
5760	bpf_net_ctx = bpf_net_ctx_set(&__bpf_net_ctx);
5761	act = netif_receive_generic_xdp(pskb, &xdp, xdp_prog);
5762	if (act != XDP_PASS) {
5763	switch (act) {
5764	case XDP_REDIRECT:
5765	err = xdp_do_generic_redirect((pskb)->dev, pskb,
5766	&xdp, xdp_prog);
5767	if (err)
5768	goto out_redir;
5769	break;
5770	case XDP_TX:
5771	generic_xdp_tx(*pskb, xdp_prog);
5772	break;
5773	}
5774	bpf_net_ctx_clear(bpf_net_ctx);
5775	return XDP_DROP;
5776	}
5777	bpf_net_ctx_clear(bpf_net_ctx);
5778	}
5779	return XDP_PASS;
5780	out_redir:
5781	bpf_net_ctx_clear(bpf_net_ctx);
5782	kfree_skb_reason(*pskb, SKB_DROP_REASON_XDP);
5783	return XDP_DROP;
5784	}
5785	EXPORT_SYMBOL_GPL(do_xdp_generic);
5786
5787	static int netif_rx_internal(struct sk_buff *skb)
5788	{
5789	int ret;
5790
5791	net_timestamp_check(READ_ONCE(net_hotdata.tstamp_prequeue), skb);
5792
5793	trace_netif_rx(skb);
5794
5795	#ifdef CONFIG_RPS
5796	if (static_branch_unlikely(&rps_needed)) {
5797	struct rps_dev_flow voidflow, *rflow = &voidflow;
5798	int cpu;
5799
5800	rcu_read_lock();
5801
5802	cpu = get_rps_cpu(skb->dev, skb, &rflow);
5803	if (cpu < 0)
5804	cpu = smp_processor_id();
5805
5806	ret = enqueue_to_backlog(skb, cpu, &rflow->last_qtail);
5807
5808	rcu_read_unlock();
5809	} else
5810	#endif
5811	{
5812	unsigned int qtail;
5813
5814	ret = enqueue_to_backlog(skb, smp_processor_id(), &qtail);
5815	}
5816	return ret;
5817	}
5818
5819	/**
5820	* __netif_rx - Slightly optimized version of netif_rx
5821	* @skb: buffer to post
5822	*
5823	* This behaves as netif_rx except that it does not disable bottom halves.
5824	* As a result this function may only be invoked from the interrupt context
5825	* (either hard or soft interrupt).
5826	*/
5827	int __netif_rx(struct sk_buff *skb)
5828	{
5829	int ret;
5830
5831	lockdep_assert_once(hardirq_count() \| softirq_count());
5832
5833	trace_netif_rx_entry(skb);
5834	ret = netif_rx_internal(skb);
5835	trace_netif_rx_exit(ret);
5836	return ret;
5837	}
5838	EXPORT_SYMBOL(__netif_rx);
5839
5840	/**
5841	* netif_rx - post buffer to the network code
5842	* @skb: buffer to post
5843	*
5844	* This function receives a packet from a device driver and queues it for
5845	* the upper (protocol) levels to process via the backlog NAPI device. It
5846	* always succeeds. The buffer may be dropped during processing for
5847	* congestion control or by the protocol layers.
5848	* The network buffer is passed via the backlog NAPI device. Modern NIC
5849	* driver should use NAPI and GRO.
5850	* This function can used from interrupt and from process context. The
5851	* caller from process context must not disable interrupts before invoking
5852	* this function.
5853	*
5854	* return values:
5855	* NET_RX_SUCCESS (no congestion)
5856	* NET_RX_DROP (packet was dropped)
5857	*
5858	*/
5859	int netif_rx(struct sk_buff *skb)
5860	{
5861	bool need_bh_off = !(hardirq_count() \| softirq_count());
5862	int ret;
5863
5864	if (need_bh_off)
5865	local_bh_disable();
5866	trace_netif_rx_entry(skb);
5867	ret = netif_rx_internal(skb);
5868	trace_netif_rx_exit(ret);
5869	if (need_bh_off)
5870	local_bh_enable();
5871	return ret;
5872	}
5873	EXPORT_SYMBOL(netif_rx);
5874
5875	static __latent_entropy void net_tx_action(void)
5876	{
5877	struct softnet_data *sd = this_cpu_ptr(&softnet_data);
5878
5879	if (sd->completion_queue) {
5880	struct sk_buff *clist;
5881
5882	local_irq_disable();
5883	clist = sd->completion_queue;
5884	sd->completion_queue = NULL;
5885	local_irq_enable();
5886
5887	while (clist) {
5888	struct sk_buff *skb = clist;
5889
5890	clist = clist->next;
5891
5892	WARN_ON(refcount_read(&skb->users));
5893	if (likely(get_kfree_skb_cb(skb)->reason == SKB_CONSUMED))
5894	trace_consume_skb(skb, net_tx_action);
5895	else
5896	trace_kfree_skb(skb, net_tx_action,
5897	get_kfree_skb_cb(skb)->reason, NULL);
5898
5899	if (skb->fclone != SKB_FCLONE_UNAVAILABLE)
5900	__kfree_skb(skb);
5901	else
5902	__napi_kfree_skb(skb,
5903	get_kfree_skb_cb(skb)->reason);
5904	}
5905	}
5906
5907	if (sd->output_queue) {
5908	struct Qdisc *head;
5909
5910	local_irq_disable();
5911	head = sd->output_queue;
5912	sd->output_queue = NULL;
5913	sd->output_queue_tailp = &sd->output_queue;
5914	local_irq_enable();
5915
5916	rcu_read_lock();
5917
5918	while (head) {
5919	spinlock_t *root_lock = NULL;
5920	struct sk_buff *to_free;
5921	struct Qdisc *q = head;
5922
5923	head = head->next_sched;
5924
5925	/* We need to make sure head->next_sched is read
5926	* before clearing __QDISC_STATE_SCHED
5927	*/
5928	smp_mb__before_atomic();
5929
5930	if (!(q->flags & TCQ_F_NOLOCK)) {
5931	root_lock = qdisc_lock(q);
5932	spin_lock(root_lock);
5933	} else if (unlikely(test_bit(__QDISC_STATE_DEACTIVATED,
5934	&q->state))) {
5935	/* There is a synchronize_net() between
5936	* STATE_DEACTIVATED flag being set and
5937	* qdisc_reset()/some_qdisc_is_busy() in
5938	* dev_deactivate(), so we can safely bail out
5939	* early here to avoid data race between
5940	* qdisc_deactivate() and some_qdisc_is_busy()
5941	* for lockless qdisc.
5942	*/
5943	clear_bit(__QDISC_STATE_SCHED, &q->state);
5944	continue;
5945	}
5946
5947	clear_bit(__QDISC_STATE_SCHED, &q->state);
5948	to_free = qdisc_run(q);
5949	if (root_lock)
5950	spin_unlock(root_lock);
5951	tcf_kfree_skb_list(to_free, q, NULL, qdisc_dev(q));
5952	}
5953
5954	rcu_read_unlock();
5955	}
5956
5957	xfrm_dev_backlog(sd);
5958	}
5959
5960	/**
5961	* netdev_is_rx_handler_busy - check if receive handler is registered
5962	* @dev: device to check
5963	*
5964	* Check if a receive handler is already registered for a given device.
5965	* Return true if there one.
5966	*
5967	* The caller must hold the rtnl_mutex.
5968	*/
5969	bool netdev_is_rx_handler_busy(struct net_device *dev)
5970	{
5971	ASSERT_RTNL();
5972	return dev && rtnl_dereference(dev->rx_handler);
5973	}
5974	EXPORT_SYMBOL_GPL(netdev_is_rx_handler_busy);
5975
5976	/**
5977	* netdev_rx_handler_register - register receive handler
5978	* @dev: device to register a handler for
5979	* @rx_handler: receive handler to register
5980	* @rx_handler_data: data pointer that is used by rx handler
5981	*
5982	* Register a receive handler for a device. This handler will then be
5983	* called from __netif_receive_skb. A negative errno code is returned
5984	* on a failure.
5985	*
5986	* The caller must hold the rtnl_mutex.
5987	*
5988	* For a general description of rx_handler, see enum rx_handler_result.
5989	*/
5990	int netdev_rx_handler_register(struct net_device *dev,
5991	rx_handler_func_t *rx_handler,
5992	void *rx_handler_data)
5993	{
5994	if (netdev_is_rx_handler_busy(dev))
5995	return -EBUSY;
5996
5997	if (dev->priv_flags & IFF_NO_RX_HANDLER)
5998	return -EINVAL;
5999
6000	/* Note: rx_handler_data must be set before rx_handler */
6001	rcu_assign_pointer(dev->rx_handler_data, rx_handler_data);
6002	rcu_assign_pointer(dev->rx_handler, rx_handler);
6003
6004	return 0;
6005	}
6006	EXPORT_SYMBOL_GPL(netdev_rx_handler_register);
6007
6008	/**
6009	* netdev_rx_handler_unregister - unregister receive handler
6010	* @dev: device to unregister a handler from
6011	*
6012	* Unregister a receive handler from a device.
6013	*
6014	* The caller must hold the rtnl_mutex.
6015	*/
6016	void netdev_rx_handler_unregister(struct net_device *dev)
6017	{
6018
6019	ASSERT_RTNL();
6020	RCU_INIT_POINTER(dev->rx_handler, NULL);
6021	/* a reader seeing a non NULL rx_handler in a rcu_read_lock()
6022	* section has a guarantee to see a non NULL rx_handler_data
6023	* as well.
6024	*/
6025	synchronize_net();
6026	RCU_INIT_POINTER(dev->rx_handler_data, NULL);
6027	}
6028	EXPORT_SYMBOL_GPL(netdev_rx_handler_unregister);
6029
6030	/*
6031	* Limit the use of PFMEMALLOC reserves to those protocols that implement
6032	* the special handling of PFMEMALLOC skbs.
6033	*/
6034	static bool skb_pfmemalloc_protocol(struct sk_buff *skb)
6035	{
6036	switch (skb->protocol) {
6037	case htons(ETH_P_ARP):
6038	case htons(ETH_P_IP):
6039	case htons(ETH_P_IPV6):
6040	case htons(ETH_P_8021Q):
6041	case htons(ETH_P_8021AD):
6042	return true;
6043	default:
6044	return false;
6045	}
6046	}
6047
6048	static inline int nf_ingress(struct sk_buff skb, struct packet_type *pt_prev,
6049	int ret, struct net_device orig_dev)
6050	{
6051	if (nf_hook_ingress_active(skb)) {
6052	int ingress_retval;
6053
6054	if (unlikely(*pt_prev)) {
6055	ret = deliver_skb(skb, pt_prev, orig_dev);
6056	*pt_prev = NULL;
6057	}
6058
6059	rcu_read_lock();
6060	ingress_retval = nf_hook_ingress(skb);
6061	rcu_read_unlock();
6062	return ingress_retval;
6063	}
6064	return 0;
6065	}
6066
6067	static int __netif_receive_skb_core(struct sk_buff **pskb, bool pfmemalloc,
6068	struct packet_type **ppt_prev)
6069	{
6070	enum skb_drop_reason drop_reason = SKB_DROP_REASON_UNHANDLED_PROTO;
6071	struct packet_type ptype, pt_prev;
6072	rx_handler_func_t *rx_handler;
6073	struct sk_buff skb = pskb;
6074	struct net_device *orig_dev;
6075	bool deliver_exact = false;
6076	int ret = NET_RX_DROP;
6077	__be16 type;
6078
6079	net_timestamp_check(!READ_ONCE(net_hotdata.tstamp_prequeue), skb);
6080
6081	trace_netif_receive_skb(skb);
6082
6083	orig_dev = skb->dev;
6084
6085	skb_reset_network_header(skb);
6086	#if !defined(CONFIG_DEBUG_NET)
6087	/* We plan to no longer reset the transport header here.
6088	* Give some time to fuzzers and dev build to catch bugs
6089	* in network stacks.
6090	*/
6091	if (!skb_transport_header_was_set(skb))
6092	skb_reset_transport_header(skb);
6093	#endif
6094	skb_reset_mac_len(skb);
6095
6096	pt_prev = NULL;
6097
6098	another_round:
6099	skb->skb_iif = skb->dev->ifindex;
6100
6101	__this_cpu_inc(softnet_data.processed);
6102
6103	if (static_branch_unlikely(&generic_xdp_needed_key)) {
6104	int ret2;
6105
6106	migrate_disable();
6107	ret2 = do_xdp_generic(rcu_dereference(skb->dev->xdp_prog),
6108	&skb);
6109	migrate_enable();
6110
6111	if (ret2 != XDP_PASS) {
6112	ret = NET_RX_DROP;
6113	goto out;
6114	}
6115	}
6116
6117	if (eth_type_vlan(skb->protocol)) {
6118	skb = skb_vlan_untag(skb);
6119	if (unlikely(!skb))
6120	goto out;
6121	}
6122
6123	if (skb_skip_tc_classify(skb))
6124	goto skip_classify;
6125
6126	if (pfmemalloc)
6127	goto skip_taps;
6128
6129	list_for_each_entry_rcu(ptype, &dev_net_rcu(skb->dev)->ptype_all,
6130	list) {
6131	if (unlikely(pt_prev))
6132	ret = deliver_skb(skb, pt_prev, orig_dev);
6133	pt_prev = ptype;
6134	}
6135
6136	list_for_each_entry_rcu(ptype, &skb->dev->ptype_all, list) {
6137	if (unlikely(pt_prev))
6138	ret = deliver_skb(skb, pt_prev, orig_dev);
6139	pt_prev = ptype;
6140	}
6141
6142	skip_taps:
6143	#ifdef CONFIG_NET_INGRESS
6144	if (static_branch_unlikely(&ingress_needed_key)) {
6145	bool another = false;
6146
6147	nf_skip_egress(skb, true);
6148	skb = sch_handle_ingress(skb, &pt_prev, &ret, orig_dev,
6149	&another);
6150	if (another)
6151	goto another_round;
6152	if (!skb)
6153	goto out;
6154
6155	nf_skip_egress(skb, false);
6156	if (nf_ingress(skb, &pt_prev, &ret, orig_dev) < 0)
6157	goto out;
6158	}
6159	#endif
6160	skb_reset_redirect(skb);
6161	skip_classify:
6162	if (pfmemalloc && !skb_pfmemalloc_protocol(skb)) {
6163	drop_reason = SKB_DROP_REASON_PFMEMALLOC;
6164	goto drop;
6165	}
6166
6167	if (skb_vlan_tag_present(skb)) {
6168	if (unlikely(pt_prev)) {
6169	ret = deliver_skb(skb, pt_prev, orig_dev);
6170	pt_prev = NULL;
6171	}
6172	if (vlan_do_receive(&skb))
6173	goto another_round;
6174	else if (unlikely(!skb))
6175	goto out;
6176	}
6177
6178	rx_handler = rcu_dereference(skb->dev->rx_handler);
6179	if (rx_handler) {
6180	if (unlikely(pt_prev)) {
6181	ret = deliver_skb(skb, pt_prev, orig_dev);
6182	pt_prev = NULL;
6183	}
6184	switch (rx_handler(&skb)) {
6185	case RX_HANDLER_CONSUMED:
6186	ret = NET_RX_SUCCESS;
6187	goto out;
6188	case RX_HANDLER_ANOTHER:
6189	goto another_round;
6190	case RX_HANDLER_EXACT:
6191	deliver_exact = true;
6192	break;
6193	case RX_HANDLER_PASS:
6194	break;
6195	default:
6196	BUG();
6197	}
6198	}
6199
6200	if (unlikely(skb_vlan_tag_present(skb)) && !netdev_uses_dsa(skb->dev)) {
6201	check_vlan_id:
6202	if (skb_vlan_tag_get_id(skb)) {
6203	/* Vlan id is non 0 and vlan_do_receive() above couldn't
6204	* find vlan device.
6205	*/
6206	skb->pkt_type = PACKET_OTHERHOST;
6207	} else if (eth_type_vlan(skb->protocol)) {
6208	/* Outer header is 802.1P with vlan 0, inner header is
6209	* 802.1Q or 802.1AD and vlan_do_receive() above could
6210	* not find vlan dev for vlan id 0.
6211	*/
6212	__vlan_hwaccel_clear_tag(skb);
6213	skb = skb_vlan_untag(skb);
6214	if (unlikely(!skb))
6215	goto out;
6216	if (vlan_do_receive(&skb))
6217	/* After stripping off 802.1P header with vlan 0
6218	* vlan dev is found for inner header.
6219	*/
6220	goto another_round;
6221	else if (unlikely(!skb))
6222	goto out;
6223	else
6224	/* We have stripped outer 802.1P vlan 0 header.
6225	* But could not find vlan dev.
6226	* check again for vlan id to set OTHERHOST.
6227	*/
6228	goto check_vlan_id;
6229	}
6230	/* Note: we might in the future use prio bits
6231	* and set skb->priority like in vlan_do_receive()
6232	* For the time being, just ignore Priority Code Point
6233	*/
6234	__vlan_hwaccel_clear_tag(skb);
6235	}
6236
6237	type = skb->protocol;
6238
6239	/* deliver only exact match when indicated */
6240	if (likely(!deliver_exact)) {
6241	deliver_ptype_list_skb(skb, &pt_prev, orig_dev, type,
6242	&ptype_base[ntohs(type) &
6243	PTYPE_HASH_MASK]);
6244
6245	/* orig_dev and skb->dev could belong to different netns;
6246	* Even in such case we need to traverse only the list
6247	* coming from skb->dev, as the ptype owner (packet socket)
6248	* will use dev_net(skb->dev) to do namespace filtering.
6249	*/
6250	deliver_ptype_list_skb(skb, &pt_prev, orig_dev, type,
6251	&dev_net_rcu(skb->dev)->ptype_specific);
6252	}
6253
6254	deliver_ptype_list_skb(skb, &pt_prev, orig_dev, type,
6255	&orig_dev->ptype_specific);
6256
6257	if (unlikely(skb->dev != orig_dev)) {
6258	deliver_ptype_list_skb(skb, &pt_prev, orig_dev, type,
6259	&skb->dev->ptype_specific);
6260	}
6261
6262	if (pt_prev) {
6263	*ppt_prev = pt_prev;
6264	} else {
6265	drop:
6266	if (!deliver_exact)
6267	dev_core_stats_rx_dropped_inc(skb->dev);
6268	else
6269	dev_core_stats_rx_nohandler_inc(skb->dev);
6270
6271	kfree_skb_reason(skb, drop_reason);
6272	/* Jamal, now you will not able to escape explaining
6273	* me how you were going to use this. :-)
6274	*/
6275	ret = NET_RX_DROP;
6276	}
6277
6278	out:
6279	/* The invariant here is that if *ppt_prev is not NULL
6280	* then skb should also be non-NULL.
6281	*
6282	* Apparently *ppt_prev assignment above holds this invariant due to
6283	* skb dereferencing near it.
6284	*/
6285	*pskb = skb;
6286	return ret;
6287	}
6288
6289	static int __netif_receive_skb_one_core(struct sk_buff *skb, bool pfmemalloc)
6290	{
6291	struct net_device *orig_dev = skb->dev;
6292	struct packet_type *pt_prev = NULL;
6293	int ret;
6294
6295	ret = __netif_receive_skb_core(&skb, pfmemalloc, &pt_prev);
6296	if (pt_prev)
6297	ret = INDIRECT_CALL_INET(pt_prev->func, ipv6_rcv, ip_rcv, skb,
6298	skb->dev, pt_prev, orig_dev);
6299	return ret;
6300	}
6301
6302	/**
6303	* netif_receive_skb_core - special purpose version of netif_receive_skb
6304	* @skb: buffer to process
6305	*
6306	* More direct receive version of netif_receive_skb(). It should
6307	* only be used by callers that have a need to skip RPS and Generic XDP.
6308	* Caller must also take care of handling if ``(page_is_)pfmemalloc``.
6309	*
6310	* This function may only be called from softirq context and interrupts
6311	* should be enabled.
6312	*
6313	* Return values (usually ignored):
6314	* NET_RX_SUCCESS: no congestion
6315	* NET_RX_DROP: packet was dropped
6316	*/
6317	int netif_receive_skb_core(struct sk_buff *skb)
6318	{
6319	int ret;
6320
6321	rcu_read_lock();
6322	ret = __netif_receive_skb_one_core(skb, false);
6323	rcu_read_unlock();
6324
6325	return ret;
6326	}
6327	EXPORT_SYMBOL(netif_receive_skb_core);
6328
6329	static inline void __netif_receive_skb_list_ptype(struct list_head *head,
6330	struct packet_type *pt_prev,
6331	struct net_device *orig_dev)
6332	{
6333	struct sk_buff skb, next;
6334
6335	if (!pt_prev)
6336	return;
6337	if (list_empty(head))
6338	return;
6339	if (pt_prev->list_func != NULL)
6340	INDIRECT_CALL_INET(pt_prev->list_func, ipv6_list_rcv,
6341	ip_list_rcv, head, pt_prev, orig_dev);
6342	else
6343	list_for_each_entry_safe(skb, next, head, list) {
6344	skb_list_del_init(skb);
6345	pt_prev->func(skb, skb->dev, pt_prev, orig_dev);
6346	}
6347	}
6348
6349	static void __netif_receive_skb_list_core(struct list_head *head, bool pfmemalloc)
6350	{
6351	/* Fast-path assumptions:
6352	* - There is no RX handler.
6353	* - Only one packet_type matches.
6354	* If either of these fails, we will end up doing some per-packet
6355	* processing in-line, then handling the 'last ptype' for the whole
6356	* sublist. This can't cause out-of-order delivery to any single ptype,
6357	* because the 'last ptype' must be constant across the sublist, and all
6358	* other ptypes are handled per-packet.
6359	*/
6360	/* Current (common) ptype of sublist */
6361	struct packet_type *pt_curr = NULL;
6362	/* Current (common) orig_dev of sublist */
6363	struct net_device *od_curr = NULL;
6364	struct sk_buff skb, next;
6365	LIST_HEAD(sublist);
6366
6367	list_for_each_entry_safe(skb, next, head, list) {
6368	struct net_device *orig_dev = skb->dev;
6369	struct packet_type *pt_prev = NULL;
6370
6371	skb_list_del_init(skb);
6372	__netif_receive_skb_core(&skb, pfmemalloc, &pt_prev);
6373	if (!pt_prev)
6374	continue;
6375	if (pt_curr != pt_prev \|\| od_curr != orig_dev) {
6376	/* dispatch old sublist */
6377	__netif_receive_skb_list_ptype(&sublist, pt_curr, od_curr);
6378	/* start new sublist */
6379	INIT_LIST_HEAD(&sublist);
6380	pt_curr = pt_prev;
6381	od_curr = orig_dev;
6382	}
6383	list_add_tail(&skb->list, &sublist);
6384	}
6385
6386	/* dispatch final sublist */
6387	__netif_receive_skb_list_ptype(&sublist, pt_curr, od_curr);
6388	}
6389
6390	static int __netif_receive_skb(struct sk_buff *skb)
6391	{
6392	int ret;
6393
6394	if (sk_memalloc_socks() && skb_pfmemalloc(skb)) {
6395	unsigned int noreclaim_flag;
6396
6397	/*
6398	* PFMEMALLOC skbs are special, they should
6399	* - be delivered to SOCK_MEMALLOC sockets only
6400	* - stay away from userspace
6401	* - have bounded memory usage
6402	*
6403	* Use PF_MEMALLOC as this saves us from propagating the allocation
6404	* context down to all allocation sites.
6405	*/
6406	noreclaim_flag = memalloc_noreclaim_save();
6407	ret = __netif_receive_skb_one_core(skb, true);
6408	memalloc_noreclaim_restore(noreclaim_flag);
6409	} else
6410	ret = __netif_receive_skb_one_core(skb, false);
6411
6412	return ret;
6413	}
6414
6415	static void __netif_receive_skb_list(struct list_head *head)
6416	{
6417	unsigned long noreclaim_flag = 0;
6418	struct sk_buff skb, next;
6419	bool pfmemalloc = false; /* Is current sublist PF_MEMALLOC? */
6420
6421	list_for_each_entry_safe(skb, next, head, list) {
6422	if ((sk_memalloc_socks() && skb_pfmemalloc(skb)) != pfmemalloc) {
6423	struct list_head sublist;
6424
6425	/* Handle the previous sublist */
6426	list_cut_before(&sublist, head, &skb->list);
6427	if (!list_empty(&sublist))
6428	__netif_receive_skb_list_core(&sublist, pfmemalloc);
6429	pfmemalloc = !pfmemalloc;
6430	/* See comments in __netif_receive_skb */
6431	if (pfmemalloc)
6432	noreclaim_flag = memalloc_noreclaim_save();
6433	else
6434	memalloc_noreclaim_restore(noreclaim_flag);
6435	}
6436	}
6437	/* Handle the remaining sublist */
6438	if (!list_empty(head))
6439	__netif_receive_skb_list_core(head, pfmemalloc);
6440	/* Restore pflags */
6441	if (pfmemalloc)
6442	memalloc_noreclaim_restore(noreclaim_flag);
6443	}
6444
6445	static int generic_xdp_install(struct net_device dev, struct netdev_bpf xdp)
6446	{
6447	struct bpf_prog *old = rtnl_dereference(dev->xdp_prog);
6448	struct bpf_prog *new = xdp->prog;
6449	int ret = 0;
6450
6451	switch (xdp->command) {
6452	case XDP_SETUP_PROG:
6453	rcu_assign_pointer(dev->xdp_prog, new);
6454	if (old)
6455	bpf_prog_put(old);
6456
6457	if (old && !new) {
6458	static_branch_dec(&generic_xdp_needed_key);
6459	} else if (new && !old) {
6460	static_branch_inc(&generic_xdp_needed_key);
6461	netif_disable_lro(dev);
6462	dev_disable_gro_hw(dev);
6463	}
6464	break;
6465
6466	default:
6467	ret = -EINVAL;
6468	break;
6469	}
6470
6471	return ret;
6472	}
6473
6474	static int netif_receive_skb_internal(struct sk_buff *skb)
6475	{
6476	int ret;
6477
6478	net_timestamp_check(READ_ONCE(net_hotdata.tstamp_prequeue), skb);
6479
6480	if (skb_defer_rx_timestamp(skb))
6481	return NET_RX_SUCCESS;
6482
6483	rcu_read_lock();
6484	#ifdef CONFIG_RPS
6485	if (static_branch_unlikely(&rps_needed)) {
6486	struct rps_dev_flow voidflow, *rflow = &voidflow;
6487	int cpu = get_rps_cpu(skb->dev, skb, &rflow);
6488
6489	if (cpu >= 0) {
6490	ret = enqueue_to_backlog(skb, cpu, &rflow->last_qtail);
6491	rcu_read_unlock();
6492	return ret;
6493	}
6494	}
6495	#endif
6496	ret = __netif_receive_skb(skb);
6497	rcu_read_unlock();
6498	return ret;
6499	}
6500
6501	void netif_receive_skb_list_internal(struct list_head *head)
6502	{
6503	struct sk_buff skb, next;
6504	LIST_HEAD(sublist);
6505
6506	list_for_each_entry_safe(skb, next, head, list) {
6507	net_timestamp_check(READ_ONCE(net_hotdata.tstamp_prequeue),
6508	skb);
6509	skb_list_del_init(skb);
6510	if (!skb_defer_rx_timestamp(skb))
6511	list_add_tail(&skb->list, &sublist);
6512	}
6513	list_splice_init(&sublist, head);
6514
6515	rcu_read_lock();
6516	#ifdef CONFIG_RPS
6517	if (static_branch_unlikely(&rps_needed)) {
6518	list_for_each_entry_safe(skb, next, head, list) {
6519	struct rps_dev_flow voidflow, *rflow = &voidflow;
6520	int cpu = get_rps_cpu(skb->dev, skb, &rflow);
6521
6522	if (cpu >= 0) {
6523	/* Will be handled, remove from list */
6524	skb_list_del_init(skb);
6525	enqueue_to_backlog(skb, cpu, &rflow->last_qtail);
6526	}
6527	}
6528	}
6529	#endif
6530	__netif_receive_skb_list(head);
6531	rcu_read_unlock();
6532	}
6533
6534	/**
6535	* netif_receive_skb - process receive buffer from network
6536	* @skb: buffer to process
6537	*
6538	* netif_receive_skb() is the main receive data processing function.
6539	* It always succeeds. The buffer may be dropped during processing
6540	* for congestion control or by the protocol layers.
6541	*
6542	* This function may only be called from softirq context and interrupts
6543	* should be enabled.
6544	*
6545	* Return values (usually ignored):
6546	* NET_RX_SUCCESS: no congestion
6547	* NET_RX_DROP: packet was dropped
6548	*/
6549	int netif_receive_skb(struct sk_buff *skb)
6550	{
6551	int ret;
6552
6553	trace_netif_receive_skb_entry(skb);
6554
6555	ret = netif_receive_skb_internal(skb);
6556	trace_netif_receive_skb_exit(ret);
6557
6558	return ret;
6559	}
6560	EXPORT_SYMBOL(netif_receive_skb);
6561
6562	/**
6563	* netif_receive_skb_list - process many receive buffers from network
6564	* @head: list of skbs to process.
6565	*
6566	* Since return value of netif_receive_skb() is normally ignored, and
6567	* wouldn't be meaningful for a list, this function returns void.
6568	*
6569	* This function may only be called from softirq context and interrupts
6570	* should be enabled.
6571	*/
6572	void netif_receive_skb_list(struct list_head *head)
6573	{
6574	struct sk_buff *skb;
6575
6576	if (list_empty(head))
6577	return;
6578	if (trace_netif_receive_skb_list_entry_enabled()) {
6579	list_for_each_entry(skb, head, list)
6580	trace_netif_receive_skb_list_entry(skb);
6581	}
6582	netif_receive_skb_list_internal(head);
6583	trace_netif_receive_skb_list_exit(0);
6584	}
6585	EXPORT_SYMBOL(netif_receive_skb_list);
6586
6587	/* Network device is going away, flush any packets still pending */
6588	static void flush_backlog(struct work_struct *work)
6589	{
6590	struct sk_buff skb, tmp;
6591	struct sk_buff_head list;
6592	struct softnet_data *sd;
6593
6594	__skb_queue_head_init(&list);
6595	local_bh_disable();
6596	sd = this_cpu_ptr(&softnet_data);
6597
6598	backlog_lock_irq_disable(sd);
6599	skb_queue_walk_safe(&sd->input_pkt_queue, skb, tmp) {
6600	if (READ_ONCE(skb->dev->reg_state) == NETREG_UNREGISTERING) {
6601	__skb_unlink(skb, &sd->input_pkt_queue);
6602	__skb_queue_tail(&list, skb);
6603	rps_input_queue_head_incr(sd);
6604	}
6605	}
6606	backlog_unlock_irq_enable(sd);
6607
6608	local_lock_nested_bh(&softnet_data.process_queue_bh_lock);
6609	skb_queue_walk_safe(&sd->process_queue, skb, tmp) {
6610	if (READ_ONCE(skb->dev->reg_state) == NETREG_UNREGISTERING) {
6611	__skb_unlink(skb, &sd->process_queue);
6612	__skb_queue_tail(&list, skb);
6613	rps_input_queue_head_incr(sd);
6614	}
6615	}
6616	local_unlock_nested_bh(&softnet_data.process_queue_bh_lock);
6617	local_bh_enable();
6618
6619	__skb_queue_purge_reason(&list, SKB_DROP_REASON_DEV_READY);
6620	}
6621
6622	static bool flush_required(int cpu)
6623	{
6624	#if IS_ENABLED(CONFIG_RPS)
6625	struct softnet_data *sd = &per_cpu(softnet_data, cpu);
6626	bool do_flush;
6627
6628	backlog_lock_irq_disable(sd);
6629
6630	/* as insertion into process_queue happens with the rps lock held,
6631	* process_queue access may race only with dequeue
6632	*/
6633	do_flush = !skb_queue_empty(&sd->input_pkt_queue) \|\|
6634	!skb_queue_empty_lockless(&sd->process_queue);
6635	backlog_unlock_irq_enable(sd);
6636
6637	return do_flush;
6638	#endif
6639	/* without RPS we can't safely check input_pkt_queue: during a
6640	* concurrent remote skb_queue_splice() we can detect as empty both
6641	* input_pkt_queue and process_queue even if the latter could end-up
6642	* containing a lot of packets.
6643	*/
6644	return true;
6645	}
6646
6647	struct flush_backlogs {
6648	cpumask_t flush_cpus;
6649	struct work_struct w[];
6650	};
6651
6652	static struct flush_backlogs *flush_backlogs_alloc(void)
6653	{
6654	return kmalloc_flex(struct flush_backlogs, w, nr_cpu_ids);
6655	}
6656
6657	static struct flush_backlogs *flush_backlogs_fallback;
6658	static DEFINE_MUTEX(flush_backlogs_mutex);
6659
6660	static void flush_all_backlogs(void)
6661	{
6662	struct flush_backlogs *ptr = flush_backlogs_alloc();
6663	unsigned int cpu;
6664
6665	if (!ptr) {
6666	mutex_lock(&flush_backlogs_mutex);
6667	ptr = flush_backlogs_fallback;
6668	}
6669	cpumask_clear(&ptr->flush_cpus);
6670
6671	cpus_read_lock();
6672
6673	for_each_online_cpu(cpu) {
6674	if (flush_required(cpu)) {
6675	INIT_WORK(&ptr->w[cpu], flush_backlog);
6676	queue_work_on(cpu, system_highpri_wq, &ptr->w[cpu]);
6677	__cpumask_set_cpu(cpu, &ptr->flush_cpus);
6678	}
6679	}
6680
6681	/* we can have in flight packet[s] on the cpus we are not flushing,
6682	* synchronize_net() in unregister_netdevice_many() will take care of
6683	* them.
6684	*/
6685	for_each_cpu(cpu, &ptr->flush_cpus)
6686	flush_work(&ptr->w[cpu]);
6687
6688	cpus_read_unlock();
6689
6690	if (ptr != flush_backlogs_fallback)
6691	kfree(ptr);
6692	else
6693	mutex_unlock(&flush_backlogs_mutex);
6694	}
6695
6696	static void net_rps_send_ipi(struct softnet_data *remsd)
6697	{
6698	#ifdef CONFIG_RPS
6699	while (remsd) {
6700	struct softnet_data *next = remsd->rps_ipi_next;
6701
6702	if (cpu_online(remsd->cpu))
6703	smp_call_function_single_async(remsd->cpu, &remsd->csd);
6704	remsd = next;
6705	}
6706	#endif
6707	}
6708
6709	/*
6710	* net_rps_action_and_irq_enable sends any pending IPI's for rps.
6711	* Note: called with local irq disabled, but exits with local irq enabled.
6712	*/
6713	static void net_rps_action_and_irq_enable(struct softnet_data *sd)
6714	{
6715	#ifdef CONFIG_RPS
6716	struct softnet_data *remsd = sd->rps_ipi_list;
6717
6718	if (!use_backlog_threads() && remsd) {
6719	sd->rps_ipi_list = NULL;
6720
6721	local_irq_enable();
6722
6723	/* Send pending IPI's to kick RPS processing on remote cpus. */
6724	net_rps_send_ipi(remsd);
6725	} else
6726	#endif
6727	local_irq_enable();
6728	}
6729
6730	static bool sd_has_rps_ipi_waiting(struct softnet_data *sd)
6731	{
6732	#ifdef CONFIG_RPS
6733	return !use_backlog_threads() && sd->rps_ipi_list;
6734	#else
6735	return false;
6736	#endif
6737	}
6738
6739	static int process_backlog(struct napi_struct *napi, int quota)
6740	{
6741	struct softnet_data *sd = container_of(napi, struct softnet_data, backlog);
6742	bool again = true;
6743	int work = 0;
6744
6745	/* Check if we have pending ipi, its better to send them now,
6746	* not waiting net_rx_action() end.
6747	*/
6748	if (sd_has_rps_ipi_waiting(sd)) {
6749	local_irq_disable();
6750	net_rps_action_and_irq_enable(sd);
6751	}
6752
6753	napi->weight = READ_ONCE(net_hotdata.dev_rx_weight);
6754	while (again) {
6755	struct sk_buff *skb;
6756
6757	local_lock_nested_bh(&softnet_data.process_queue_bh_lock);
6758	while ((skb = __skb_dequeue(&sd->process_queue))) {
6759	local_unlock_nested_bh(&softnet_data.process_queue_bh_lock);
6760	rcu_read_lock();
6761	__netif_receive_skb(skb);
6762	rcu_read_unlock();
6763	if (++work >= quota) {
6764	rps_input_queue_head_add(sd, work);
6765	return work;
6766	}
6767
6768	local_lock_nested_bh(&softnet_data.process_queue_bh_lock);
6769	}
6770	local_unlock_nested_bh(&softnet_data.process_queue_bh_lock);
6771
6772	backlog_lock_irq_disable(sd);
6773	if (skb_queue_empty(&sd->input_pkt_queue)) {
6774	/*
6775	* Inline a custom version of __napi_complete().
6776	* only current cpu owns and manipulates this napi,
6777	* and NAPI_STATE_SCHED is the only possible flag set
6778	* on backlog.
6779	* We can use a plain write instead of clear_bit(),
6780	* and we dont need an smp_mb() memory barrier.
6781	*/
6782	napi->state &= NAPIF_STATE_THREADED;
6783	again = false;
6784	} else {
6785	local_lock_nested_bh(&softnet_data.process_queue_bh_lock);
6786	skb_queue_splice_tail_init(&sd->input_pkt_queue,
6787	&sd->process_queue);
6788	local_unlock_nested_bh(&softnet_data.process_queue_bh_lock);
6789	}
6790	backlog_unlock_irq_enable(sd);
6791	}
6792
6793	if (work)
6794	rps_input_queue_head_add(sd, work);
6795	return work;
6796	}
6797
6798	/**
6799	* __napi_schedule - schedule for receive
6800	* @n: entry to schedule
6801	*
6802	* The entry's receive function will be scheduled to run.
6803	* Consider using __napi_schedule_irqoff() if hard irqs are masked.
6804	*/
6805	void __napi_schedule(struct napi_struct *n)
6806	{
6807	unsigned long flags;
6808
6809	local_irq_save(flags);
6810	____napi_schedule(this_cpu_ptr(&softnet_data), n);
6811	local_irq_restore(flags);
6812	}
6813	EXPORT_SYMBOL(__napi_schedule);
6814
6815	/**
6816	* napi_schedule_prep - check if napi can be scheduled
6817	* @n: napi context
6818	*
6819	* Test if NAPI routine is already running, and if not mark
6820	* it as running. This is used as a condition variable to
6821	* insure only one NAPI poll instance runs. We also make
6822	* sure there is no pending NAPI disable.
6823	*/
6824	bool napi_schedule_prep(struct napi_struct *n)
6825	{
6826	unsigned long new, val = READ_ONCE(n->state);
6827
6828	do {
6829	if (unlikely(val & NAPIF_STATE_DISABLE))
6830	return false;
6831	new = val \| NAPIF_STATE_SCHED;
6832
6833	/* Sets STATE_MISSED bit if STATE_SCHED was already set
6834	* This was suggested by Alexander Duyck, as compiler
6835	* emits better code than :
6836	* if (val & NAPIF_STATE_SCHED)
6837	* new \|= NAPIF_STATE_MISSED;
6838	*/
6839	new \|= (val & NAPIF_STATE_SCHED) / NAPIF_STATE_SCHED *
6840	NAPIF_STATE_MISSED;
6841	} while (!try_cmpxchg(&n->state, &val, new));
6842
6843	return !(val & NAPIF_STATE_SCHED);
6844	}
6845	EXPORT_SYMBOL(napi_schedule_prep);
6846
6847	/**
6848	* __napi_schedule_irqoff - schedule for receive
6849	* @n: entry to schedule
6850	*
6851	* Variant of __napi_schedule() assuming hard irqs are masked.
6852	*
6853	* On PREEMPT_RT enabled kernels this maps to __napi_schedule()
6854	* because the interrupt disabled assumption might not be true
6855	* due to force-threaded interrupts and spinlock substitution.
6856	*/
6857	void __napi_schedule_irqoff(struct napi_struct *n)
6858	{
6859	if (!IS_ENABLED(CONFIG_PREEMPT_RT))
6860	____napi_schedule(this_cpu_ptr(&softnet_data), n);
6861	else
6862	__napi_schedule(n);
6863	}
6864	EXPORT_SYMBOL(__napi_schedule_irqoff);
6865
6866	bool napi_complete_done(struct napi_struct *n, int work_done)
6867	{
6868	unsigned long flags, val, new, timeout = 0;
6869	bool ret = true;
6870
6871	/*
6872	* 1) Don't let napi dequeue from the cpu poll list
6873	* just in case its running on a different cpu.
6874	* 2) If we are busy polling, do nothing here, we have
6875	* the guarantee we will be called later.
6876	*/
6877	if (unlikely(n->state & (NAPIF_STATE_NPSVC \|
6878	NAPIF_STATE_IN_BUSY_POLL)))
6879	return false;
6880
6881	if (work_done) {
6882	if (n->gro.bitmask)
6883	timeout = napi_get_gro_flush_timeout(n);
6884	n->defer_hard_irqs_count = napi_get_defer_hard_irqs(n);
6885	}
6886	if (n->defer_hard_irqs_count > 0) {
6887	n->defer_hard_irqs_count--;
6888	timeout = napi_get_gro_flush_timeout(n);
6889	if (timeout)
6890	ret = false;
6891	}
6892
6893	/*
6894	* When the NAPI instance uses a timeout and keeps postponing
6895	* it, we need to bound somehow the time packets are kept in
6896	* the GRO layer.
6897	*/
6898	gro_flush_normal(&n->gro, !!timeout);
6899
6900	if (unlikely(!list_empty(&n->poll_list))) {
6901	/* If n->poll_list is not empty, we need to mask irqs */
6902	local_irq_save(flags);
6903	list_del_init(&n->poll_list);
6904	local_irq_restore(flags);
6905	}
6906	WRITE_ONCE(n->list_owner, -1);
6907
6908	val = READ_ONCE(n->state);
6909	do {
6910	WARN_ON_ONCE(!(val & NAPIF_STATE_SCHED));
6911
6912	new = val & ~(NAPIF_STATE_MISSED \| NAPIF_STATE_SCHED \|
6913	NAPIF_STATE_SCHED_THREADED \|
6914	NAPIF_STATE_PREFER_BUSY_POLL);
6915
6916	/* If STATE_MISSED was set, leave STATE_SCHED set,
6917	* because we will call napi->poll() one more time.
6918	* This C code was suggested by Alexander Duyck to help gcc.
6919	*/
6920	new \|= (val & NAPIF_STATE_MISSED) / NAPIF_STATE_MISSED *
6921	NAPIF_STATE_SCHED;
6922	} while (!try_cmpxchg(&n->state, &val, new));
6923
6924	if (unlikely(val & NAPIF_STATE_MISSED)) {
6925	__napi_schedule(n);
6926	return false;
6927	}
6928
6929	if (timeout)
6930	hrtimer_start(&n->timer, ns_to_ktime(timeout),
6931	HRTIMER_MODE_REL_PINNED);
6932	return ret;
6933	}
6934	EXPORT_SYMBOL(napi_complete_done);
6935
6936	static void __skb_defer_free_flush(struct skb_defer_node *sdn, int budget)
6937	{
6938	struct llist_node *free_list;
6939	struct sk_buff skb, next;
6940
6941	if (llist_empty(&sdn->defer_list))
6942	return;
6943	atomic_long_set(&sdn->defer_count, 0);
6944	free_list = llist_del_all(&sdn->defer_list);
6945
6946	llist_for_each_entry_safe(skb, next, free_list, ll_node) {
6947	prefetch(next);
6948	napi_consume_skb(skb, budget);
6949	}
6950	}
6951
6952	void skb_defer_node_flush(struct skb_defer_node *sdn)
6953	{
6954	__skb_defer_free_flush(sdn, 0);
6955	}
6956
6957	static void skb_defer_free_flush(void)
6958	{
6959	struct skb_defer_node *sdn;
6960	int node;
6961
6962	for_each_node(node) {
6963	sdn = this_cpu_ptr(net_hotdata.skb_defer_nodes) + node;
6964	__skb_defer_free_flush(sdn, 1);
6965	}
6966	}
6967
6968	#if defined(CONFIG_NET_RX_BUSY_POLL)
6969
6970	enum {
6971	NAPI_F_PREFER_BUSY_POLL = 1,
6972	NAPI_F_END_ON_RESCHED = 2,
6973	};
6974
6975	static void busy_poll_stop(struct napi_struct napi, void have_poll_lock,
6976	unsigned flags, u16 budget)
6977	{
6978	struct bpf_net_context __bpf_net_ctx, *bpf_net_ctx;
6979	unsigned long timeout = 0;
6980	int rc;
6981
6982	/* Busy polling means there is a high chance device driver hard irq
6983	* could not grab NAPI_STATE_SCHED, and that NAPI_STATE_MISSED was
6984	* set in napi_schedule_prep().
6985	* Since we either call napi->poll() once more or start the timer,
6986	* we can safely clear NAPI_STATE_MISSED.
6987	*
6988	* Note: x86 could use a single "lock and ..." instruction
6989	* to perform these two clear_bit()
6990	*/
6991	clear_bit(NAPI_STATE_MISSED, &napi->state);
6992	clear_bit(NAPI_STATE_IN_BUSY_POLL, &napi->state);
6993
6994	local_bh_disable();
6995	bpf_net_ctx = bpf_net_ctx_set(&__bpf_net_ctx);
6996
6997	if (flags & NAPI_F_PREFER_BUSY_POLL) {
6998	napi->defer_hard_irqs_count = napi_get_defer_hard_irqs(napi);
6999	if (napi->defer_hard_irqs_count)
7000	timeout = napi_get_gro_flush_timeout(napi);
7001	}
7002	if (timeout) {
7003	netpoll_poll_unlock(have_poll_lock);
7004
7005	/* Drain aged GRO packets before clearing SCHED since the NAPI
7006	* won't run again until after the timer fires. When HZ < 1000,
7007	* GRO age comparison is too coarse, so flush everything.
7008	*/
7009	gro_flush_normal(&napi->gro, HZ >= 1000);
7010
7011	clear_bit(NAPI_STATE_SCHED, &napi->state);
7012	hrtimer_start(&napi->timer, ns_to_ktime(timeout),
7013	HRTIMER_MODE_REL_PINNED);
7014	} else {
7015	/* Use driver poll to re-enable device interrupts. */
7016	rc = napi->poll(napi, budget);
7017	/* Unless rc == budget we no longer own the NAPI instance,
7018	* IRQ may fire on another CPU, poll this NAPI, and enter GRO.
7019	*/
7020	trace_napi_poll(napi, rc, budget);
7021	netpoll_poll_unlock(have_poll_lock);
7022	if (rc == budget) {
7023	gro_normal_list(&napi->gro);
7024	__napi_schedule(napi);
7025	}
7026	}
7027
7028	bpf_net_ctx_clear(bpf_net_ctx);
7029	local_bh_enable();
7030	}
7031
7032	static void __napi_busy_loop(unsigned int napi_id,
7033	bool (loop_end)(void , unsigned long),
7034	void *loop_end_arg, unsigned flags, u16 budget)
7035	{
7036	unsigned long start_time = loop_end ? busy_loop_current_time() : 0;
7037	int (napi_poll)(struct napi_struct napi, int budget);
7038	struct bpf_net_context __bpf_net_ctx, *bpf_net_ctx;
7039	void *have_poll_lock = NULL;
7040	struct napi_struct *napi;
7041
7042	WARN_ON_ONCE(!rcu_read_lock_held());
7043
7044	restart:
7045	napi_poll = NULL;
7046
7047	napi = napi_by_id(napi_id);
7048	if (!napi)
7049	return;
7050
7051	if (!IS_ENABLED(CONFIG_PREEMPT_RT))
7052	preempt_disable();
7053	for (;;) {
7054	int work = 0;
7055
7056	local_bh_disable();
7057	bpf_net_ctx = bpf_net_ctx_set(&__bpf_net_ctx);
7058	if (!napi_poll) {
7059	unsigned long val = READ_ONCE(napi->state);
7060
7061	/* If multiple threads are competing for this napi,
7062	* we avoid dirtying napi->state as much as we can.
7063	*/
7064	if (val & (NAPIF_STATE_DISABLE \| NAPIF_STATE_SCHED \|
7065	NAPIF_STATE_IN_BUSY_POLL)) {
7066	if (flags & NAPI_F_PREFER_BUSY_POLL)
7067	set_bit(NAPI_STATE_PREFER_BUSY_POLL, &napi->state);
7068	goto count;
7069	}
7070	if (cmpxchg(&napi->state, val,
7071	val \| NAPIF_STATE_IN_BUSY_POLL \|
7072	NAPIF_STATE_SCHED) != val) {
7073	if (flags & NAPI_F_PREFER_BUSY_POLL)
7074	set_bit(NAPI_STATE_PREFER_BUSY_POLL, &napi->state);
7075	goto count;
7076	}
7077	have_poll_lock = netpoll_poll_lock(napi);
7078	napi_poll = napi->poll;
7079	}
7080	work = napi_poll(napi, budget);
7081	trace_napi_poll(napi, work, budget);
7082	gro_normal_list(&napi->gro);
7083	count:
7084	if (work > 0)
7085	__NET_ADD_STATS(dev_net(napi->dev),
7086	LINUX_MIB_BUSYPOLLRXPACKETS, work);
7087	skb_defer_free_flush();
7088	bpf_net_ctx_clear(bpf_net_ctx);
7089	local_bh_enable();
7090
7091	if (!loop_end \|\| loop_end(loop_end_arg, start_time))
7092	break;
7093
7094	if (unlikely(need_resched())) {
7095	if (flags & NAPI_F_END_ON_RESCHED)
7096	break;
7097	if (napi_poll)
7098	busy_poll_stop(napi, have_poll_lock, flags, budget);
7099	if (!IS_ENABLED(CONFIG_PREEMPT_RT))
7100	preempt_enable();
7101	rcu_read_unlock();
7102	cond_resched();
7103	rcu_read_lock();
7104	if (loop_end(loop_end_arg, start_time))
7105	return;
7106	goto restart;
7107	}
7108	cpu_relax();
7109	}
7110	if (napi_poll)
7111	busy_poll_stop(napi, have_poll_lock, flags, budget);
7112	if (!IS_ENABLED(CONFIG_PREEMPT_RT))
7113	preempt_enable();
7114	}
7115
7116	void napi_busy_loop_rcu(unsigned int napi_id,
7117	bool (loop_end)(void , unsigned long),
7118	void *loop_end_arg, bool prefer_busy_poll, u16 budget)
7119	{
7120	unsigned flags = NAPI_F_END_ON_RESCHED;
7121
7122	if (prefer_busy_poll)
7123	flags \|= NAPI_F_PREFER_BUSY_POLL;
7124
7125	__napi_busy_loop(napi_id, loop_end, loop_end_arg, flags, budget);
7126	}
7127
7128	void napi_busy_loop(unsigned int napi_id,
7129	bool (loop_end)(void , unsigned long),
7130	void *loop_end_arg, bool prefer_busy_poll, u16 budget)
7131	{
7132	unsigned flags = prefer_busy_poll ? NAPI_F_PREFER_BUSY_POLL : 0;
7133
7134	rcu_read_lock();
7135	__napi_busy_loop(napi_id, loop_end, loop_end_arg, flags, budget);
7136	rcu_read_unlock();
7137	}
7138	EXPORT_SYMBOL(napi_busy_loop);
7139
7140	void napi_suspend_irqs(unsigned int napi_id)
7141	{
7142	struct napi_struct *napi;
7143
7144	rcu_read_lock();
7145	napi = napi_by_id(napi_id);
7146	if (napi) {
7147	unsigned long timeout = napi_get_irq_suspend_timeout(napi);
7148
7149	if (timeout)
7150	hrtimer_start(&napi->timer, ns_to_ktime(timeout),
7151	HRTIMER_MODE_REL_PINNED);
7152	}
7153	rcu_read_unlock();
7154	}
7155
7156	void napi_resume_irqs(unsigned int napi_id)
7157	{
7158	struct napi_struct *napi;
7159
7160	rcu_read_lock();
7161	napi = napi_by_id(napi_id);
7162	if (napi) {
7163	/* If irq_suspend_timeout is set to 0 between the call to
7164	* napi_suspend_irqs and now, the original value still
7165	* determines the safety timeout as intended and napi_watchdog
7166	* will resume irq processing.
7167	*/
7168	if (napi_get_irq_suspend_timeout(napi)) {
7169	local_bh_disable();
7170	napi_schedule(napi);
7171	local_bh_enable();
7172	}
7173	}
7174	rcu_read_unlock();
7175	}
7176
7177	#endif /* CONFIG_NET_RX_BUSY_POLL */
7178
7179	static void __napi_hash_add_with_id(struct napi_struct *napi,
7180	unsigned int napi_id)
7181	{
7182	napi->gro.cached_napi_id = napi_id;
7183
7184	WRITE_ONCE(napi->napi_id, napi_id);
7185	hlist_add_head_rcu(&napi->napi_hash_node,
7186	&napi_hash[napi->napi_id % HASH_SIZE(napi_hash)]);
7187	}
7188
7189	static void napi_hash_add_with_id(struct napi_struct *napi,
7190	unsigned int napi_id)
7191	{
7192	unsigned long flags;
7193
7194	spin_lock_irqsave(&napi_hash_lock, flags);
7195	WARN_ON_ONCE(napi_by_id(napi_id));
7196	__napi_hash_add_with_id(napi, napi_id);
7197	spin_unlock_irqrestore(&napi_hash_lock, flags);
7198	}
7199
7200	static void napi_hash_add(struct napi_struct *napi)
7201	{
7202	unsigned long flags;
7203
7204	if (test_bit(NAPI_STATE_NO_BUSY_POLL, &napi->state))
7205	return;
7206
7207	spin_lock_irqsave(&napi_hash_lock, flags);
7208
7209	/* 0..NR_CPUS range is reserved for sender_cpu use */
7210	do {
7211	if (unlikely(!napi_id_valid(++napi_gen_id)))
7212	napi_gen_id = MIN_NAPI_ID;
7213	} while (napi_by_id(napi_gen_id));
7214
7215	__napi_hash_add_with_id(napi, napi_gen_id);
7216
7217	spin_unlock_irqrestore(&napi_hash_lock, flags);
7218	}
7219
7220	/* Warning : caller is responsible to make sure rcu grace period
7221	* is respected before freeing memory containing @napi
7222	*/
7223	static void napi_hash_del(struct napi_struct *napi)
7224	{
7225	unsigned long flags;
7226
7227	spin_lock_irqsave(&napi_hash_lock, flags);
7228
7229	hlist_del_init_rcu(&napi->napi_hash_node);
7230
7231	spin_unlock_irqrestore(&napi_hash_lock, flags);
7232	}
7233
7234	static enum hrtimer_restart napi_watchdog(struct hrtimer *timer)
7235	{
7236	struct napi_struct *napi;
7237
7238	napi = container_of(timer, struct napi_struct, timer);
7239
7240	/* Note : we use a relaxed variant of napi_schedule_prep() not setting
7241	* NAPI_STATE_MISSED, since we do not react to a device IRQ.
7242	*/
7243	if (!napi_disable_pending(napi) &&
7244	!test_and_set_bit(NAPI_STATE_SCHED, &napi->state)) {
7245	clear_bit(NAPI_STATE_PREFER_BUSY_POLL, &napi->state);
7246	__napi_schedule_irqoff(napi);
7247	}
7248
7249	return HRTIMER_NORESTART;
7250	}
7251
7252	static void napi_stop_kthread(struct napi_struct *napi)
7253	{
7254	unsigned long val, new;
7255
7256	/* Wait until the napi STATE_THREADED is unset. */
7257	while (true) {
7258	val = READ_ONCE(napi->state);
7259
7260	/* If napi kthread own this napi or the napi is idle,
7261	* STATE_THREADED can be unset here.
7262	*/
7263	if ((val & NAPIF_STATE_SCHED_THREADED) \|\|
7264	!(val & NAPIF_STATE_SCHED)) {
7265	new = val & (~(NAPIF_STATE_THREADED \|
7266	NAPIF_STATE_THREADED_BUSY_POLL));
7267	} else {
7268	msleep(20);
7269	continue;
7270	}
7271
7272	if (try_cmpxchg(&napi->state, &val, new))
7273	break;
7274	}
7275
7276	/* Once STATE_THREADED is unset, wait for SCHED_THREADED to be unset by
7277	* the kthread.
7278	*/
7279	while (true) {
7280	if (!test_bit(NAPI_STATE_SCHED_THREADED, &napi->state))
7281	break;
7282
7283	msleep(20);
7284	}
7285
7286	kthread_stop(napi->thread);
7287	napi->thread = NULL;
7288	}
7289
7290	static void napi_set_threaded_state(struct napi_struct *napi,
7291	enum netdev_napi_threaded threaded_mode)
7292	{
7293	bool threaded = threaded_mode != NETDEV_NAPI_THREADED_DISABLED;
7294	bool busy_poll = threaded_mode == NETDEV_NAPI_THREADED_BUSY_POLL;
7295
7296	assign_bit(NAPI_STATE_THREADED, &napi->state, threaded);
7297	assign_bit(NAPI_STATE_THREADED_BUSY_POLL, &napi->state, busy_poll);
7298	}
7299
7300	int napi_set_threaded(struct napi_struct *napi,
7301	enum netdev_napi_threaded threaded)
7302	{
7303	if (threaded) {
7304	if (!napi->thread) {
7305	int err = napi_kthread_create(napi);
7306
7307	if (err)
7308	return err;
7309	}
7310	}
7311
7312	if (napi->config)
7313	napi->config->threaded = threaded;
7314
7315	/* Setting/unsetting threaded mode on a napi might not immediately
7316	* take effect, if the current napi instance is actively being
7317	* polled. In this case, the switch between threaded mode and
7318	* softirq mode will happen in the next round of napi_schedule().
7319	* This should not cause hiccups/stalls to the live traffic.
7320	*/
7321	if (!threaded && napi->thread) {
7322	napi_stop_kthread(napi);
7323	} else {
7324	/* Make sure kthread is created before THREADED bit is set. */
7325	smp_mb__before_atomic();
7326	napi_set_threaded_state(napi, threaded);
7327	}
7328
7329	return 0;
7330	}
7331
7332	int netif_set_threaded(struct net_device *dev,
7333	enum netdev_napi_threaded threaded)
7334	{
7335	struct napi_struct *napi;
7336	int i, err = 0;
7337
7338	netdev_assert_locked_or_invisible(dev);
7339
7340	if (threaded) {
7341	list_for_each_entry(napi, &dev->napi_list, dev_list) {
7342	if (!napi->thread) {
7343	err = napi_kthread_create(napi);
7344	if (err) {
7345	threaded = NETDEV_NAPI_THREADED_DISABLED;
7346	break;
7347	}
7348	}
7349	}
7350	}
7351
7352	WRITE_ONCE(dev->threaded, threaded);
7353
7354	/* The error should not occur as the kthreads are already created. */
7355	list_for_each_entry(napi, &dev->napi_list, dev_list)
7356	WARN_ON_ONCE(napi_set_threaded(napi, threaded));
7357
7358	/* Override the config for all NAPIs even if currently not listed */
7359	for (i = 0; i < dev->num_napi_configs; i++)
7360	dev->napi_config[i].threaded = threaded;
7361
7362	return err;
7363	}
7364
7365	/**
7366	* netif_threaded_enable() - enable threaded NAPIs
7367	* @dev: net_device instance
7368	*
7369	* Enable threaded mode for the NAPI instances of the device. This may be useful
7370	* for devices where multiple NAPI instances get scheduled by a single
7371	* interrupt. Threaded NAPI allows moving the NAPI processing to cores other
7372	* than the core where IRQ is mapped.
7373	*
7374	* This function should be called before @dev is registered.
7375	*/
7376	void netif_threaded_enable(struct net_device *dev)
7377	{
7378	WARN_ON_ONCE(netif_set_threaded(dev, NETDEV_NAPI_THREADED_ENABLED));
7379	}
7380	EXPORT_SYMBOL(netif_threaded_enable);
7381
7382	/**
7383	* netif_queue_set_napi - Associate queue with the napi
7384	* @dev: device to which NAPI and queue belong
7385	* @queue_index: Index of queue
7386	* @type: queue type as RX or TX
7387	* @napi: NAPI context, pass NULL to clear previously set NAPI
7388	*
7389	* Set queue with its corresponding napi context. This should be done after
7390	* registering the NAPI handler for the queue-vector and the queues have been
7391	* mapped to the corresponding interrupt vector.
7392	*/
7393	void netif_queue_set_napi(struct net_device *dev, unsigned int queue_index,
7394	enum netdev_queue_type type, struct napi_struct *napi)
7395	{
7396	struct netdev_rx_queue *rxq;
7397	struct netdev_queue *txq;
7398
7399	if (WARN_ON_ONCE(napi && !napi->dev))
7400	return;
7401	netdev_assert_locked_ops_compat_or_invisible(dev);
7402
7403	switch (type) {
7404	case NETDEV_QUEUE_TYPE_RX:
7405	rxq = __netif_get_rx_queue(dev, queue_index);
7406	rxq->napi = napi;
7407	return;
7408	case NETDEV_QUEUE_TYPE_TX:
7409	txq = netdev_get_tx_queue(dev, queue_index);
7410	txq->napi = napi;
7411	return;
7412	default:
7413	return;
7414	}
7415	}
7416	EXPORT_SYMBOL(netif_queue_set_napi);
7417
7418	static void
7419	netif_napi_irq_notify(struct irq_affinity_notify *notify,
7420	const cpumask_t *mask)
7421	{
7422	struct napi_struct *napi =
7423	container_of(notify, struct napi_struct, notify);
7424	#ifdef CONFIG_RFS_ACCEL
7425	struct cpu_rmap *rmap = napi->dev->rx_cpu_rmap;
7426	int err;
7427	#endif
7428
7429	if (napi->config && napi->dev->irq_affinity_auto)
7430	cpumask_copy(&napi->config->affinity_mask, mask);
7431
7432	#ifdef CONFIG_RFS_ACCEL
7433	if (napi->dev->rx_cpu_rmap_auto) {
7434	err = cpu_rmap_update(rmap, napi->napi_rmap_idx, mask);
7435	if (err)
7436	netdev_warn(napi->dev, "RMAP update failed (%d)\n",
7437	err);
7438	}
7439	#endif
7440	}
7441
7442	#ifdef CONFIG_RFS_ACCEL
7443	static void netif_napi_affinity_release(struct kref *ref)
7444	{
7445	struct napi_struct *napi =
7446	container_of(ref, struct napi_struct, notify.kref);
7447	struct cpu_rmap *rmap = napi->dev->rx_cpu_rmap;
7448
7449	netdev_assert_locked(napi->dev);
7450	WARN_ON(test_and_clear_bit(NAPI_STATE_HAS_NOTIFIER,
7451	&napi->state));
7452
7453	if (!napi->dev->rx_cpu_rmap_auto)
7454	return;
7455	rmap->obj[napi->napi_rmap_idx] = NULL;
7456	napi->napi_rmap_idx = -1;
7457	cpu_rmap_put(rmap);
7458	}
7459
7460	int netif_enable_cpu_rmap(struct net_device *dev, unsigned int num_irqs)
7461	{
7462	if (dev->rx_cpu_rmap_auto)
7463	return 0;
7464
7465	dev->rx_cpu_rmap = alloc_irq_cpu_rmap(num_irqs);
7466	if (!dev->rx_cpu_rmap)
7467	return -ENOMEM;
7468
7469	dev->rx_cpu_rmap_auto = true;
7470	return 0;
7471	}
7472	EXPORT_SYMBOL(netif_enable_cpu_rmap);
7473
7474	static void netif_del_cpu_rmap(struct net_device *dev)
7475	{
7476	struct cpu_rmap *rmap = dev->rx_cpu_rmap;
7477
7478	if (!dev->rx_cpu_rmap_auto)
7479	return;
7480
7481	/* Free the rmap */
7482	cpu_rmap_put(rmap);
7483	dev->rx_cpu_rmap = NULL;
7484	dev->rx_cpu_rmap_auto = false;
7485	}
7486
7487	#else
7488	static void netif_napi_affinity_release(struct kref *ref)
7489	{
7490	}
7491
7492	int netif_enable_cpu_rmap(struct net_device *dev, unsigned int num_irqs)
7493	{
7494	return 0;
7495	}
7496	EXPORT_SYMBOL(netif_enable_cpu_rmap);
7497
7498	static void netif_del_cpu_rmap(struct net_device *dev)
7499	{
7500	}
7501	#endif
7502
7503	void netif_set_affinity_auto(struct net_device *dev)
7504	{
7505	unsigned int i, maxqs, numa;
7506
7507	maxqs = max(dev->num_tx_queues, dev->num_rx_queues);
7508	numa = dev_to_node(&dev->dev);
7509
7510	for (i = 0; i < maxqs; i++)
7511	cpumask_set_cpu(cpumask_local_spread(i, numa),
7512	&dev->napi_config[i].affinity_mask);
7513
7514	dev->irq_affinity_auto = true;
7515	}
7516	EXPORT_SYMBOL(netif_set_affinity_auto);
7517
7518	void netif_napi_set_irq_locked(struct napi_struct *napi, int irq)
7519	{
7520	int rc;
7521
7522	netdev_assert_locked_or_invisible(napi->dev);
7523
7524	if (napi->irq == irq)
7525	return;
7526
7527	/* Remove existing resources */
7528	if (test_and_clear_bit(NAPI_STATE_HAS_NOTIFIER, &napi->state))
7529	irq_set_affinity_notifier(napi->irq, NULL);
7530
7531	napi->irq = irq;
7532	if (irq < 0 \|\|
7533	(!napi->dev->rx_cpu_rmap_auto && !napi->dev->irq_affinity_auto))
7534	return;
7535
7536	/* Abort for buggy drivers */
7537	if (napi->dev->irq_affinity_auto && WARN_ON_ONCE(!napi->config))
7538	return;
7539
7540	#ifdef CONFIG_RFS_ACCEL
7541	if (napi->dev->rx_cpu_rmap_auto) {
7542	rc = cpu_rmap_add(napi->dev->rx_cpu_rmap, napi);
7543	if (rc < 0)
7544	return;
7545
7546	cpu_rmap_get(napi->dev->rx_cpu_rmap);
7547	napi->napi_rmap_idx = rc;
7548	}
7549	#endif
7550
7551	/* Use core IRQ notifier */
7552	napi->notify.notify = netif_napi_irq_notify;
7553	napi->notify.release = netif_napi_affinity_release;
7554	rc = irq_set_affinity_notifier(irq, &napi->notify);
7555	if (rc) {
7556	netdev_warn(napi->dev, "Unable to set IRQ notifier (%d)\n",
7557	rc);
7558	goto put_rmap;
7559	}
7560
7561	set_bit(NAPI_STATE_HAS_NOTIFIER, &napi->state);
7562	return;
7563
7564	put_rmap:
7565	#ifdef CONFIG_RFS_ACCEL
7566	if (napi->dev->rx_cpu_rmap_auto) {
7567	napi->dev->rx_cpu_rmap->obj[napi->napi_rmap_idx] = NULL;
7568	cpu_rmap_put(napi->dev->rx_cpu_rmap);
7569	napi->napi_rmap_idx = -1;
7570	}
7571	#endif
7572	napi->notify.notify = NULL;
7573	napi->notify.release = NULL;
7574	}
7575	EXPORT_SYMBOL(netif_napi_set_irq_locked);
7576
7577	static void napi_restore_config(struct napi_struct *n)
7578	{
7579	n->defer_hard_irqs = n->config->defer_hard_irqs;
7580	n->gro_flush_timeout = n->config->gro_flush_timeout;
7581	n->irq_suspend_timeout = n->config->irq_suspend_timeout;
7582
7583	if (n->dev->irq_affinity_auto &&
7584	test_bit(NAPI_STATE_HAS_NOTIFIER, &n->state))
7585	irq_set_affinity(n->irq, &n->config->affinity_mask);
7586
7587	/* a NAPI ID might be stored in the config, if so use it. if not, use
7588	* napi_hash_add to generate one for us.
7589	*/
7590	if (n->config->napi_id) {
7591	napi_hash_add_with_id(n, n->config->napi_id);
7592	} else {
7593	napi_hash_add(n);
7594	n->config->napi_id = n->napi_id;
7595	}
7596
7597	WARN_ON_ONCE(napi_set_threaded(n, n->config->threaded));
7598	}
7599
7600	static void napi_save_config(struct napi_struct *n)
7601	{
7602	n->config->defer_hard_irqs = n->defer_hard_irqs;
7603	n->config->gro_flush_timeout = n->gro_flush_timeout;
7604	n->config->irq_suspend_timeout = n->irq_suspend_timeout;
7605	napi_hash_del(n);
7606	}
7607
7608	/* Netlink wants the NAPI list to be sorted by ID, if adding a NAPI which will
7609	* inherit an existing ID try to insert it at the right position.
7610	*/
7611	static void
7612	netif_napi_dev_list_add(struct net_device dev, struct napi_struct napi)
7613	{
7614	unsigned int new_id, pos_id;
7615	struct list_head *higher;
7616	struct napi_struct *pos;
7617
7618	new_id = UINT_MAX;
7619	if (napi->config && napi->config->napi_id)
7620	new_id = napi->config->napi_id;
7621
7622	higher = &dev->napi_list;
7623	list_for_each_entry(pos, &dev->napi_list, dev_list) {
7624	if (napi_id_valid(pos->napi_id))
7625	pos_id = pos->napi_id;
7626	else if (pos->config)
7627	pos_id = pos->config->napi_id;
7628	else
7629	pos_id = UINT_MAX;
7630
7631	if (pos_id <= new_id)
7632	break;
7633	higher = &pos->dev_list;
7634	}
7635	list_add_rcu(&napi->dev_list, higher); /* adds after higher */
7636	}
7637
7638	/* Double check that napi_get_frags() allocates skbs with
7639	* skb->head being backed by slab, not a page fragment.
7640	* This is to make sure bug fixed in 3226b158e67c
7641	* ("net: avoid 32 x truesize under-estimation for tiny skbs")
7642	* does not accidentally come back.
7643	*/
7644	static void napi_get_frags_check(struct napi_struct *napi)
7645	{
7646	struct sk_buff *skb;
7647
7648	local_bh_disable();
7649	skb = napi_get_frags(napi);
7650	WARN_ON_ONCE(skb && skb->head_frag);
7651	napi_free_frags(napi);
7652	local_bh_enable();
7653	}
7654
7655	void netif_napi_add_weight_locked(struct net_device *dev,
7656	struct napi_struct *napi,
7657	int (poll)(struct napi_struct , int),
7658	int weight)
7659	{
7660	netdev_assert_locked(dev);
7661	if (WARN_ON(test_and_set_bit(NAPI_STATE_LISTED, &napi->state)))
7662	return;
7663
7664	INIT_LIST_HEAD(&napi->poll_list);
7665	INIT_HLIST_NODE(&napi->napi_hash_node);
7666	hrtimer_setup(&napi->timer, napi_watchdog, CLOCK_MONOTONIC, HRTIMER_MODE_REL_PINNED);
7667	gro_init(&napi->gro);
7668	napi->skb = NULL;
7669	napi->poll = poll;
7670	if (weight > NAPI_POLL_WEIGHT)
7671	netdev_err_once(dev, "%s() called with weight %d\n", __func__,
7672	weight);
7673	napi->weight = weight;
7674	napi->dev = dev;
7675	#ifdef CONFIG_NETPOLL
7676	napi->poll_owner = -1;
7677	#endif
7678	napi->list_owner = -1;
7679	set_bit(NAPI_STATE_SCHED, &napi->state);
7680	set_bit(NAPI_STATE_NPSVC, &napi->state);
7681	netif_napi_dev_list_add(dev, napi);
7682
7683	/* default settings from sysfs are applied to all NAPIs. any per-NAPI
7684	* configuration will be loaded in napi_enable
7685	*/
7686	napi_set_defer_hard_irqs(napi, READ_ONCE(dev->napi_defer_hard_irqs));
7687	napi_set_gro_flush_timeout(napi, READ_ONCE(dev->gro_flush_timeout));
7688
7689	napi_get_frags_check(napi);
7690	/* Create kthread for this napi if dev->threaded is set.
7691	* Clear dev->threaded if kthread creation failed so that
7692	* threaded mode will not be enabled in napi_enable().
7693	*/
7694	if (napi_get_threaded_config(dev, napi))
7695	if (napi_kthread_create(napi))
7696	dev->threaded = NETDEV_NAPI_THREADED_DISABLED;
7697	netif_napi_set_irq_locked(napi, -1);
7698	}
7699	EXPORT_SYMBOL(netif_napi_add_weight_locked);
7700
7701	void napi_disable_locked(struct napi_struct *n)
7702	{
7703	unsigned long val, new;
7704
7705	might_sleep();
7706	netdev_assert_locked(n->dev);
7707
7708	set_bit(NAPI_STATE_DISABLE, &n->state);
7709
7710	val = READ_ONCE(n->state);
7711	do {
7712	while (val & (NAPIF_STATE_SCHED \| NAPIF_STATE_NPSVC)) {
7713	usleep_range(20, 200);
7714	val = READ_ONCE(n->state);
7715	}
7716
7717	new = val \| NAPIF_STATE_SCHED \| NAPIF_STATE_NPSVC;
7718	new &= ~(NAPIF_STATE_THREADED \|
7719	NAPIF_STATE_THREADED_BUSY_POLL \|
7720	NAPIF_STATE_PREFER_BUSY_POLL);
7721	} while (!try_cmpxchg(&n->state, &val, new));
7722
7723	hrtimer_cancel(&n->timer);
7724
7725	if (n->config)
7726	napi_save_config(n);
7727	else
7728	napi_hash_del(n);
7729
7730	clear_bit(NAPI_STATE_DISABLE, &n->state);
7731	}
7732	EXPORT_SYMBOL(napi_disable_locked);
7733
7734	/**
7735	* napi_disable() - prevent NAPI from scheduling
7736	* @n: NAPI context
7737	*
7738	* Stop NAPI from being scheduled on this context.
7739	* Waits till any outstanding processing completes.
7740	* Takes netdev_lock() for associated net_device.
7741	*/
7742	void napi_disable(struct napi_struct *n)
7743	{
7744	netdev_lock(n->dev);
7745	napi_disable_locked(n);
7746	netdev_unlock(n->dev);
7747	}
7748	EXPORT_SYMBOL(napi_disable);
7749
7750	void napi_enable_locked(struct napi_struct *n)
7751	{
7752	unsigned long new, val = READ_ONCE(n->state);
7753
7754	if (n->config)
7755	napi_restore_config(n);
7756	else
7757	napi_hash_add(n);
7758
7759	do {
7760	BUG_ON(!test_bit(NAPI_STATE_SCHED, &val));
7761
7762	new = val & ~(NAPIF_STATE_SCHED \| NAPIF_STATE_NPSVC);
7763	if (n->dev->threaded && n->thread)
7764	new \|= NAPIF_STATE_THREADED;
7765	} while (!try_cmpxchg(&n->state, &val, new));
7766	}
7767	EXPORT_SYMBOL(napi_enable_locked);
7768
7769	/**
7770	* napi_enable() - enable NAPI scheduling
7771	* @n: NAPI context
7772	*
7773	* Enable scheduling of a NAPI instance.
7774	* Must be paired with napi_disable().
7775	* Takes netdev_lock() for associated net_device.
7776	*/
7777	void napi_enable(struct napi_struct *n)
7778	{
7779	netdev_lock(n->dev);
7780	napi_enable_locked(n);
7781	netdev_unlock(n->dev);
7782	}
7783	EXPORT_SYMBOL(napi_enable);
7784
7785	/* Must be called in process context */
7786	void __netif_napi_del_locked(struct napi_struct *napi)
7787	{
7788	netdev_assert_locked(napi->dev);
7789
7790	if (!test_and_clear_bit(NAPI_STATE_LISTED, &napi->state))
7791	return;
7792
7793	/* Make sure NAPI is disabled (or was never enabled). */
7794	WARN_ON(!test_bit(NAPI_STATE_SCHED, &napi->state));
7795
7796	if (test_and_clear_bit(NAPI_STATE_HAS_NOTIFIER, &napi->state))
7797	irq_set_affinity_notifier(napi->irq, NULL);
7798
7799	if (napi->config) {
7800	napi->index = -1;
7801	napi->config = NULL;
7802	}
7803
7804	list_del_rcu(&napi->dev_list);
7805	napi_free_frags(napi);
7806
7807	gro_cleanup(&napi->gro);
7808
7809	if (napi->thread) {
7810	kthread_stop(napi->thread);
7811	napi->thread = NULL;
7812	}
7813	}
7814	EXPORT_SYMBOL(__netif_napi_del_locked);
7815
7816	static int __napi_poll(struct napi_struct n, bool repoll)
7817	{
7818	int work, weight;
7819
7820	weight = n->weight;
7821
7822	/* This NAPI_STATE_SCHED test is for avoiding a race
7823	* with netpoll's poll_napi(). Only the entity which
7824	* obtains the lock and sees NAPI_STATE_SCHED set will
7825	* actually make the ->poll() call. Therefore we avoid
7826	* accidentally calling ->poll() when NAPI is not scheduled.
7827	*/
7828	work = 0;
7829	if (napi_is_scheduled(n)) {
7830	work = n->poll(n, weight);
7831	trace_napi_poll(n, work, weight);
7832
7833	xdp_do_check_flushed(n);
7834	}
7835
7836	if (unlikely(work > weight))
7837	netdev_err_once(n->dev, "NAPI poll function %pS returned %d, exceeding its budget of %d.\n",
7838	n->poll, work, weight);
7839
7840	if (likely(work < weight))
7841	return work;
7842
7843	/* Drivers must not modify the NAPI state if they
7844	* consume the entire weight. In such cases this code
7845	* still "owns" the NAPI instance and therefore can
7846	* move the instance around on the list at-will.
7847	*/
7848	if (unlikely(napi_disable_pending(n))) {
7849	napi_complete(n);
7850	return work;
7851	}
7852
7853	/* The NAPI context has more processing work, but busy-polling
7854	* is preferred. Exit early.
7855	*/
7856	if (napi_prefer_busy_poll(n)) {
7857	if (napi_complete_done(n, work)) {
7858	/* If timeout is not set, we need to make sure
7859	* that the NAPI is re-scheduled.
7860	*/
7861	napi_schedule(n);
7862	}
7863	return work;
7864	}
7865
7866	/* Flush too old packets. If HZ < 1000, flush all packets */
7867	gro_flush_normal(&n->gro, HZ >= 1000);
7868
7869	/* Some drivers may have called napi_schedule
7870	* prior to exhausting their budget.
7871	*/
7872	if (unlikely(!list_empty(&n->poll_list))) {
7873	pr_warn_once("%s: Budget exhausted after napi rescheduled\n",
7874	n->dev ? n->dev->name : "backlog");
7875	return work;
7876	}
7877
7878	*repoll = true;
7879
7880	return work;
7881	}
7882
7883	static int napi_poll(struct napi_struct n, struct list_head repoll)
7884	{
7885	bool do_repoll = false;
7886	void *have;
7887	int work;
7888
7889	list_del_init(&n->poll_list);
7890
7891	have = netpoll_poll_lock(n);
7892
7893	work = __napi_poll(n, &do_repoll);
7894
7895	if (do_repoll) {
7896	#if defined(CONFIG_DEBUG_NET)
7897	if (unlikely(!napi_is_scheduled(n)))
7898	pr_crit("repoll requested for device %s %ps but napi is not scheduled.\n",
7899	n->dev->name, n->poll);
7900	#endif
7901	list_add_tail(&n->poll_list, repoll);
7902	}
7903	netpoll_poll_unlock(have);
7904
7905	return work;
7906	}
7907
7908	static int napi_thread_wait(struct napi_struct *napi)
7909	{
7910	set_current_state(TASK_INTERRUPTIBLE);
7911
7912	while (!kthread_should_stop()) {
7913	/* Testing SCHED_THREADED bit here to make sure the current
7914	* kthread owns this napi and could poll on this napi.
7915	* Testing SCHED bit is not enough because SCHED bit might be
7916	* set by some other busy poll thread or by napi_disable().
7917	*/
7918	if (test_bit(NAPI_STATE_SCHED_THREADED, &napi->state)) {
7919	WARN_ON(!list_empty(&napi->poll_list));
7920	__set_current_state(TASK_RUNNING);
7921	return 0;
7922	}
7923
7924	schedule();
7925	set_current_state(TASK_INTERRUPTIBLE);
7926	}
7927	__set_current_state(TASK_RUNNING);
7928
7929	return -1;
7930	}
7931
7932	static void napi_threaded_poll_loop(struct napi_struct *napi,
7933	unsigned long *busy_poll_last_qs)
7934	{
7935	unsigned long last_qs = busy_poll_last_qs ? *busy_poll_last_qs : jiffies;
7936	struct bpf_net_context __bpf_net_ctx, *bpf_net_ctx;
7937	struct softnet_data *sd;
7938
7939	for (;;) {
7940	bool repoll = false;
7941	void *have;
7942
7943	local_bh_disable();
7944	bpf_net_ctx = bpf_net_ctx_set(&__bpf_net_ctx);
7945
7946	sd = this_cpu_ptr(&softnet_data);
7947	sd->in_napi_threaded_poll = true;
7948
7949	have = netpoll_poll_lock(napi);
7950	__napi_poll(napi, &repoll);
7951	netpoll_poll_unlock(have);
7952
7953	sd->in_napi_threaded_poll = false;
7954	barrier();
7955
7956	if (sd_has_rps_ipi_waiting(sd)) {
7957	local_irq_disable();
7958	net_rps_action_and_irq_enable(sd);
7959	}
7960	skb_defer_free_flush();
7961	bpf_net_ctx_clear(bpf_net_ctx);
7962
7963	/* When busy poll is enabled, the old packets are not flushed in
7964	* napi_complete_done. So flush them here.
7965	*/
7966	if (busy_poll_last_qs)
7967	gro_flush_normal(&napi->gro, HZ >= 1000);
7968	local_bh_enable();
7969
7970	/* Call cond_resched here to avoid watchdog warnings. */
7971	if (repoll \|\| busy_poll_last_qs) {
7972	rcu_softirq_qs_periodic(last_qs);
7973	cond_resched();
7974	}
7975
7976	if (!repoll)
7977	break;
7978	}
7979
7980	if (busy_poll_last_qs)
7981	*busy_poll_last_qs = last_qs;
7982	}
7983
7984	static int napi_threaded_poll(void *data)
7985	{
7986	struct napi_struct *napi = data;
7987	unsigned long last_qs = jiffies;
7988	bool want_busy_poll;
7989	bool in_busy_poll;
7990	unsigned long val;
7991
7992	while (!napi_thread_wait(napi)) {
7993	val = READ_ONCE(napi->state);
7994
7995	want_busy_poll = val & NAPIF_STATE_THREADED_BUSY_POLL;
7996	in_busy_poll = val & NAPIF_STATE_IN_BUSY_POLL;
7997
7998	if (unlikely(val & NAPIF_STATE_DISABLE))
7999	want_busy_poll = false;
8000
8001	if (want_busy_poll != in_busy_poll)
8002	assign_bit(NAPI_STATE_IN_BUSY_POLL, &napi->state,
8003	want_busy_poll);
8004
8005	napi_threaded_poll_loop(napi, want_busy_poll ? &last_qs : NULL);
8006	}
8007
8008	return 0;
8009	}
8010
8011	static __latent_entropy void net_rx_action(void)
8012	{
8013	struct softnet_data *sd = this_cpu_ptr(&softnet_data);
8014	unsigned long time_limit = jiffies +
8015	usecs_to_jiffies(READ_ONCE(net_hotdata.netdev_budget_usecs));
8016	struct bpf_net_context __bpf_net_ctx, *bpf_net_ctx;
8017	int budget = READ_ONCE(net_hotdata.netdev_budget);
8018	LIST_HEAD(list);
8019	LIST_HEAD(repoll);
8020
8021	bpf_net_ctx = bpf_net_ctx_set(&__bpf_net_ctx);
8022	start:
8023	sd->in_net_rx_action = true;
8024	local_irq_disable();
8025	list_splice_init(&sd->poll_list, &list);
8026	local_irq_enable();
8027
8028	for (;;) {
8029	struct napi_struct *n;
8030
8031	skb_defer_free_flush();
8032
8033	if (list_empty(&list)) {
8034	if (list_empty(&repoll)) {
8035	sd->in_net_rx_action = false;
8036	barrier();
8037	/* We need to check if ____napi_schedule()
8038	* had refilled poll_list while
8039	* sd->in_net_rx_action was true.
8040	*/
8041	if (!list_empty(&sd->poll_list))
8042	goto start;
8043	if (!sd_has_rps_ipi_waiting(sd))
8044	goto end;
8045	}
8046	break;
8047	}
8048
8049	n = list_first_entry(&list, struct napi_struct, poll_list);
8050	budget -= napi_poll(n, &repoll);
8051
8052	/* If softirq window is exhausted then punt.
8053	* Allow this to run for 2 jiffies since which will allow
8054	* an average latency of 1.5/HZ.
8055	*/
8056	if (unlikely(budget <= 0 \|\|
8057	time_after_eq(jiffies, time_limit))) {
8058	/* Pairs with READ_ONCE() in softnet_seq_show() */
8059	WRITE_ONCE(sd->time_squeeze, sd->time_squeeze + 1);
8060	break;
8061	}
8062	}
8063
8064	local_irq_disable();
8065
8066	list_splice_tail_init(&sd->poll_list, &list);
8067	list_splice_tail(&repoll, &list);
8068	list_splice(&list, &sd->poll_list);
8069	if (!list_empty(&sd->poll_list))
8070	__raise_softirq_irqoff(NET_RX_SOFTIRQ);
8071	else
8072	sd->in_net_rx_action = false;
8073
8074	net_rps_action_and_irq_enable(sd);
8075	end:
8076	bpf_net_ctx_clear(bpf_net_ctx);
8077	}
8078
8079	struct netdev_adjacent {
8080	struct net_device *dev;
8081	netdevice_tracker dev_tracker;
8082
8083	/* upper master flag, there can only be one master device per list */
8084	bool master;
8085
8086	/* lookup ignore flag */
8087	bool ignore;
8088
8089	/* counter for the number of times this device was added to us */
8090	u16 ref_nr;
8091
8092	/* private field for the users */
8093	void *private;
8094
8095	struct list_head list;
8096	struct rcu_head rcu;
8097	};
8098
8099	static struct netdev_adjacent __netdev_find_adj(struct net_device adj_dev,
8100	struct list_head *adj_list)
8101	{
8102	struct netdev_adjacent *adj;
8103
8104	list_for_each_entry(adj, adj_list, list) {
8105	if (adj->dev == adj_dev)
8106	return adj;
8107	}
8108	return NULL;
8109	}
8110
8111	static int ____netdev_has_upper_dev(struct net_device *upper_dev,
8112	struct netdev_nested_priv *priv)
8113	{
8114	struct net_device dev = (struct net_device )priv->data;
8115
8116	return upper_dev == dev;
8117	}
8118
8119	/**
8120	* netdev_has_upper_dev - Check if device is linked to an upper device
8121	* @dev: device
8122	* @upper_dev: upper device to check
8123	*
8124	* Find out if a device is linked to specified upper device and return true
8125	* in case it is. Note that this checks only immediate upper device,
8126	* not through a complete stack of devices. The caller must hold the RTNL lock.
8127	*/
8128	bool netdev_has_upper_dev(struct net_device *dev,
8129	struct net_device *upper_dev)
8130	{
8131	struct netdev_nested_priv priv = {
8132	.data = (void *)upper_dev,
8133	};
8134
8135	ASSERT_RTNL();
8136
8137	return netdev_walk_all_upper_dev_rcu(dev, ____netdev_has_upper_dev,
8138	&priv);
8139	}
8140	EXPORT_SYMBOL(netdev_has_upper_dev);
8141
8142	/**
8143	* netdev_has_upper_dev_all_rcu - Check if device is linked to an upper device
8144	* @dev: device
8145	* @upper_dev: upper device to check
8146	*
8147	* Find out if a device is linked to specified upper device and return true
8148	* in case it is. Note that this checks the entire upper device chain.
8149	* The caller must hold rcu lock.
8150	*/
8151
8152	bool netdev_has_upper_dev_all_rcu(struct net_device *dev,
8153	struct net_device *upper_dev)
8154	{
8155	struct netdev_nested_priv priv = {
8156	.data = (void *)upper_dev,
8157	};
8158
8159	return !!netdev_walk_all_upper_dev_rcu(dev, ____netdev_has_upper_dev,
8160	&priv);
8161	}
8162	EXPORT_SYMBOL(netdev_has_upper_dev_all_rcu);
8163
8164	/**
8165	* netdev_has_any_upper_dev - Check if device is linked to some device
8166	* @dev: device
8167	*
8168	* Find out if a device is linked to an upper device and return true in case
8169	* it is. The caller must hold the RTNL lock.
8170	*/
8171	bool netdev_has_any_upper_dev(struct net_device *dev)
8172	{
8173	ASSERT_RTNL();
8174
8175	return !list_empty(&dev->adj_list.upper);
8176	}
8177	EXPORT_SYMBOL(netdev_has_any_upper_dev);
8178
8179	/**
8180	* netdev_master_upper_dev_get - Get master upper device
8181	* @dev: device
8182	*
8183	* Find a master upper device and return pointer to it or NULL in case
8184	* it's not there. The caller must hold the RTNL lock.
8185	*/
8186	struct net_device netdev_master_upper_dev_get(struct net_device dev)
8187	{
8188	struct netdev_adjacent *upper;
8189
8190	ASSERT_RTNL();
8191
8192	if (list_empty(&dev->adj_list.upper))
8193	return NULL;
8194
8195	upper = list_first_entry(&dev->adj_list.upper,
8196	struct netdev_adjacent, list);
8197	if (likely(upper->master))
8198	return upper->dev;
8199	return NULL;
8200	}
8201	EXPORT_SYMBOL(netdev_master_upper_dev_get);
8202
8203	static struct net_device __netdev_master_upper_dev_get(struct net_device dev)
8204	{
8205	struct netdev_adjacent *upper;
8206
8207	ASSERT_RTNL();
8208
8209	if (list_empty(&dev->adj_list.upper))
8210	return NULL;
8211
8212	upper = list_first_entry(&dev->adj_list.upper,
8213	struct netdev_adjacent, list);
8214	if (likely(upper->master) && !upper->ignore)
8215	return upper->dev;
8216	return NULL;
8217	}
8218
8219	/**
8220	* netdev_has_any_lower_dev - Check if device is linked to some device
8221	* @dev: device
8222	*
8223	* Find out if a device is linked to a lower device and return true in case
8224	* it is. The caller must hold the RTNL lock.
8225	*/
8226	static bool netdev_has_any_lower_dev(struct net_device *dev)
8227	{
8228	ASSERT_RTNL();
8229
8230	return !list_empty(&dev->adj_list.lower);
8231	}
8232
8233	void netdev_adjacent_get_private(struct list_head adj_list)
8234	{
8235	struct netdev_adjacent *adj;
8236
8237	adj = list_entry(adj_list, struct netdev_adjacent, list);
8238
8239	return adj->private;
8240	}
8241	EXPORT_SYMBOL(netdev_adjacent_get_private);
8242
8243	/**
8244	* netdev_upper_get_next_dev_rcu - Get the next dev from upper list
8245	* @dev: device
8246	* @iter: list_head ** of the current position
8247	*
8248	* Gets the next device from the dev's upper list, starting from iter
8249	* position. The caller must hold RCU read lock.
8250	*/
8251	struct net_device netdev_upper_get_next_dev_rcu(struct net_device dev,
8252	struct list_head **iter)
8253	{
8254	struct netdev_adjacent *upper;
8255
8256	WARN_ON_ONCE(!rcu_read_lock_held() && !rcu_read_lock_bh_held() &&
8257	!lockdep_rtnl_is_held());
8258
8259	upper = list_entry_rcu((*iter)->next, struct netdev_adjacent, list);
8260
8261	if (&upper->list == &dev->adj_list.upper)
8262	return NULL;
8263
8264	*iter = &upper->list;
8265
8266	return upper->dev;
8267	}
8268	EXPORT_SYMBOL(netdev_upper_get_next_dev_rcu);
8269
8270	static struct net_device __netdev_next_upper_dev(struct net_device dev,
8271	struct list_head **iter,
8272	bool *ignore)
8273	{
8274	struct netdev_adjacent *upper;
8275
8276	upper = list_entry((*iter)->next, struct netdev_adjacent, list);
8277
8278	if (&upper->list == &dev->adj_list.upper)
8279	return NULL;
8280
8281	*iter = &upper->list;
8282	*ignore = upper->ignore;
8283
8284	return upper->dev;
8285	}
8286
8287	static struct net_device netdev_next_upper_dev_rcu(struct net_device dev,
8288	struct list_head **iter)
8289	{
8290	struct netdev_adjacent *upper;
8291
8292	WARN_ON_ONCE(!rcu_read_lock_held() && !lockdep_rtnl_is_held());
8293
8294	upper = list_entry_rcu((*iter)->next, struct netdev_adjacent, list);
8295
8296	if (&upper->list == &dev->adj_list.upper)
8297	return NULL;
8298
8299	*iter = &upper->list;
8300
8301	return upper->dev;
8302	}
8303
8304	static int __netdev_walk_all_upper_dev(struct net_device *dev,
8305	int (fn)(struct net_device dev,
8306	struct netdev_nested_priv *priv),
8307	struct netdev_nested_priv *priv)
8308	{
8309	struct net_device udev, next, now, dev_stack[MAX_NEST_DEV + 1];
8310	struct list_head niter, iter, *iter_stack[MAX_NEST_DEV + 1];
8311	int ret, cur = 0;
8312	bool ignore;
8313
8314	now = dev;
8315	iter = &dev->adj_list.upper;
8316
8317	while (1) {
8318	if (now != dev) {
8319	ret = fn(now, priv);
8320	if (ret)
8321	return ret;
8322	}
8323
8324	next = NULL;
8325	while (1) {
8326	udev = __netdev_next_upper_dev(now, &iter, &ignore);
8327	if (!udev)
8328	break;
8329	if (ignore)
8330	continue;
8331
8332	next = udev;
8333	niter = &udev->adj_list.upper;
8334	dev_stack[cur] = now;
8335	iter_stack[cur++] = iter;
8336	break;
8337	}
8338
8339	if (!next) {
8340	if (!cur)
8341	return 0;
8342	next = dev_stack[--cur];
8343	niter = iter_stack[cur];
8344	}
8345
8346	now = next;
8347	iter = niter;
8348	}
8349
8350	return 0;
8351	}
8352
8353	int netdev_walk_all_upper_dev_rcu(struct net_device *dev,
8354	int (fn)(struct net_device dev,
8355	struct netdev_nested_priv *priv),
8356	struct netdev_nested_priv *priv)
8357	{
8358	struct net_device udev, next, now, dev_stack[MAX_NEST_DEV + 1];
8359	struct list_head niter, iter, *iter_stack[MAX_NEST_DEV + 1];
8360	int ret, cur = 0;
8361
8362	now = dev;
8363	iter = &dev->adj_list.upper;
8364
8365	while (1) {
8366	if (now != dev) {
8367	ret = fn(now, priv);
8368	if (ret)
8369	return ret;
8370	}
8371
8372	next = NULL;
8373	while (1) {
8374	udev = netdev_next_upper_dev_rcu(now, &iter);
8375	if (!udev)
8376	break;
8377
8378	next = udev;
8379	niter = &udev->adj_list.upper;
8380	dev_stack[cur] = now;
8381	iter_stack[cur++] = iter;
8382	break;
8383	}
8384
8385	if (!next) {
8386	if (!cur)
8387	return 0;
8388	next = dev_stack[--cur];
8389	niter = iter_stack[cur];
8390	}
8391
8392	now = next;
8393	iter = niter;
8394	}
8395
8396	return 0;
8397	}
8398	EXPORT_SYMBOL_GPL(netdev_walk_all_upper_dev_rcu);
8399
8400	static bool __netdev_has_upper_dev(struct net_device *dev,
8401	struct net_device *upper_dev)
8402	{
8403	struct netdev_nested_priv priv = {
8404	.flags = 0,
8405	.data = (void *)upper_dev,
8406	};
8407
8408	ASSERT_RTNL();
8409
8410	return __netdev_walk_all_upper_dev(dev, ____netdev_has_upper_dev,
8411	&priv);
8412	}
8413
8414	/**
8415	* netdev_lower_get_next_private - Get the next ->private from the
8416	* lower neighbour list
8417	* @dev: device
8418	* @iter: list_head ** of the current position
8419	*
8420	* Gets the next netdev_adjacent->private from the dev's lower neighbour
8421	* list, starting from iter position. The caller must hold either hold the
8422	* RTNL lock or its own locking that guarantees that the neighbour lower
8423	* list will remain unchanged.
8424	*/
8425	void netdev_lower_get_next_private(struct net_device dev,
8426	struct list_head **iter)
8427	{
8428	struct netdev_adjacent *lower;
8429
8430	lower = list_entry(*iter, struct netdev_adjacent, list);
8431
8432	if (&lower->list == &dev->adj_list.lower)
8433	return NULL;
8434
8435	*iter = lower->list.next;
8436
8437	return lower->private;
8438	}
8439	EXPORT_SYMBOL(netdev_lower_get_next_private);
8440
8441	/**
8442	* netdev_lower_get_next_private_rcu - Get the next ->private from the
8443	* lower neighbour list, RCU
8444	* variant
8445	* @dev: device
8446	* @iter: list_head ** of the current position
8447	*
8448	* Gets the next netdev_adjacent->private from the dev's lower neighbour
8449	* list, starting from iter position. The caller must hold RCU read lock.
8450	*/
8451	void netdev_lower_get_next_private_rcu(struct net_device dev,
8452	struct list_head **iter)
8453	{
8454	struct netdev_adjacent *lower;
8455
8456	WARN_ON_ONCE(!rcu_read_lock_held() && !rcu_read_lock_bh_held());
8457
8458	lower = list_entry_rcu((*iter)->next, struct netdev_adjacent, list);
8459
8460	if (&lower->list == &dev->adj_list.lower)
8461	return NULL;
8462
8463	*iter = &lower->list;
8464
8465	return lower->private;
8466	}
8467	EXPORT_SYMBOL(netdev_lower_get_next_private_rcu);
8468
8469	/**
8470	* netdev_lower_get_next - Get the next device from the lower neighbour
8471	* list
8472	* @dev: device
8473	* @iter: list_head ** of the current position
8474	*
8475	* Gets the next netdev_adjacent from the dev's lower neighbour
8476	* list, starting from iter position. The caller must hold RTNL lock or
8477	* its own locking that guarantees that the neighbour lower
8478	* list will remain unchanged.
8479	*/
8480	void netdev_lower_get_next(struct net_device dev, struct list_head **iter)
8481	{
8482	struct netdev_adjacent *lower;
8483
8484	lower = list_entry(*iter, struct netdev_adjacent, list);
8485
8486	if (&lower->list == &dev->adj_list.lower)
8487	return NULL;
8488
8489	*iter = lower->list.next;
8490
8491	return lower->dev;
8492	}
8493	EXPORT_SYMBOL(netdev_lower_get_next);
8494
8495	static struct net_device netdev_next_lower_dev(struct net_device dev,
8496	struct list_head **iter)
8497	{
8498	struct netdev_adjacent *lower;
8499
8500	lower = list_entry((*iter)->next, struct netdev_adjacent, list);
8501
8502	if (&lower->list == &dev->adj_list.lower)
8503	return NULL;
8504
8505	*iter = &lower->list;
8506
8507	return lower->dev;
8508	}
8509
8510	static struct net_device __netdev_next_lower_dev(struct net_device dev,
8511	struct list_head **iter,
8512	bool *ignore)
8513	{
8514	struct netdev_adjacent *lower;
8515
8516	lower = list_entry((*iter)->next, struct netdev_adjacent, list);
8517
8518	if (&lower->list == &dev->adj_list.lower)
8519	return NULL;
8520
8521	*iter = &lower->list;
8522	*ignore = lower->ignore;
8523
8524	return lower->dev;
8525	}
8526
8527	int netdev_walk_all_lower_dev(struct net_device *dev,
8528	int (fn)(struct net_device dev,
8529	struct netdev_nested_priv *priv),
8530	struct netdev_nested_priv *priv)
8531	{
8532	struct net_device ldev, next, now, dev_stack[MAX_NEST_DEV + 1];
8533	struct list_head niter, iter, *iter_stack[MAX_NEST_DEV + 1];
8534	int ret, cur = 0;
8535
8536	now = dev;
8537	iter = &dev->adj_list.lower;
8538
8539	while (1) {
8540	if (now != dev) {
8541	ret = fn(now, priv);
8542	if (ret)
8543	return ret;
8544	}
8545
8546	next = NULL;
8547	while (1) {
8548	ldev = netdev_next_lower_dev(now, &iter);
8549	if (!ldev)
8550	break;
8551
8552	next = ldev;
8553	niter = &ldev->adj_list.lower;
8554	dev_stack[cur] = now;
8555	iter_stack[cur++] = iter;
8556	break;
8557	}
8558
8559	if (!next) {
8560	if (!cur)
8561	return 0;
8562	next = dev_stack[--cur];
8563	niter = iter_stack[cur];
8564	}
8565
8566	now = next;
8567	iter = niter;
8568	}
8569
8570	return 0;
8571	}
8572	EXPORT_SYMBOL_GPL(netdev_walk_all_lower_dev);
8573
8574	static int __netdev_walk_all_lower_dev(struct net_device *dev,
8575	int (fn)(struct net_device dev,
8576	struct netdev_nested_priv *priv),
8577	struct netdev_nested_priv *priv)
8578	{
8579	struct net_device ldev, next, now, dev_stack[MAX_NEST_DEV + 1];
8580	struct list_head niter, iter, *iter_stack[MAX_NEST_DEV + 1];
8581	int ret, cur = 0;
8582	bool ignore;
8583
8584	now = dev;
8585	iter = &dev->adj_list.lower;
8586
8587	while (1) {
8588	if (now != dev) {
8589	ret = fn(now, priv);
8590	if (ret)
8591	return ret;
8592	}
8593
8594	next = NULL;
8595	while (1) {
8596	ldev = __netdev_next_lower_dev(now, &iter, &ignore);
8597	if (!ldev)
8598	break;
8599	if (ignore)
8600	continue;
8601
8602	next = ldev;
8603	niter = &ldev->adj_list.lower;
8604	dev_stack[cur] = now;
8605	iter_stack[cur++] = iter;
8606	break;
8607	}
8608
8609	if (!next) {
8610	if (!cur)
8611	return 0;
8612	next = dev_stack[--cur];
8613	niter = iter_stack[cur];
8614	}
8615
8616	now = next;
8617	iter = niter;
8618	}
8619
8620	return 0;
8621	}
8622
8623	struct net_device netdev_next_lower_dev_rcu(struct net_device dev,
8624	struct list_head **iter)
8625	{
8626	struct netdev_adjacent *lower;
8627
8628	lower = list_entry_rcu((*iter)->next, struct netdev_adjacent, list);
8629	if (&lower->list == &dev->adj_list.lower)
8630	return NULL;
8631
8632	*iter = &lower->list;
8633
8634	return lower->dev;
8635	}
8636	EXPORT_SYMBOL(netdev_next_lower_dev_rcu);
8637
8638	static u8 __netdev_upper_depth(struct net_device *dev)
8639	{
8640	struct net_device *udev;
8641	struct list_head *iter;
8642	u8 max_depth = 0;
8643	bool ignore;
8644
8645	for (iter = &dev->adj_list.upper,
8646	udev = __netdev_next_upper_dev(dev, &iter, &ignore);
8647	udev;
8648	udev = __netdev_next_upper_dev(dev, &iter, &ignore)) {
8649	if (ignore)
8650	continue;
8651	if (max_depth < udev->upper_level)
8652	max_depth = udev->upper_level;
8653	}
8654
8655	return max_depth;
8656	}
8657
8658	static u8 __netdev_lower_depth(struct net_device *dev)
8659	{
8660	struct net_device *ldev;
8661	struct list_head *iter;
8662	u8 max_depth = 0;
8663	bool ignore;
8664
8665	for (iter = &dev->adj_list.lower,
8666	ldev = __netdev_next_lower_dev(dev, &iter, &ignore);
8667	ldev;
8668	ldev = __netdev_next_lower_dev(dev, &iter, &ignore)) {
8669	if (ignore)
8670	continue;
8671	if (max_depth < ldev->lower_level)
8672	max_depth = ldev->lower_level;
8673	}
8674
8675	return max_depth;
8676	}
8677
8678	static int __netdev_update_upper_level(struct net_device *dev,
8679	struct netdev_nested_priv *__unused)
8680	{
8681	dev->upper_level = __netdev_upper_depth(dev) + 1;
8682	return 0;
8683	}
8684
8685	#ifdef CONFIG_LOCKDEP
8686	static LIST_HEAD(net_unlink_list);
8687
8688	static void net_unlink_todo(struct net_device *dev)
8689	{
8690	if (list_empty(&dev->unlink_list))
8691	list_add_tail(&dev->unlink_list, &net_unlink_list);
8692	}
8693	#endif
8694
8695	static int __netdev_update_lower_level(struct net_device *dev,
8696	struct netdev_nested_priv *priv)
8697	{
8698	dev->lower_level = __netdev_lower_depth(dev) + 1;
8699
8700	#ifdef CONFIG_LOCKDEP
8701	if (!priv)
8702	return 0;
8703
8704	if (priv->flags & NESTED_SYNC_IMM)
8705	dev->nested_level = dev->lower_level - 1;
8706	if (priv->flags & NESTED_SYNC_TODO)
8707	net_unlink_todo(dev);
8708	#endif
8709	return 0;
8710	}
8711
8712	int netdev_walk_all_lower_dev_rcu(struct net_device *dev,
8713	int (fn)(struct net_device dev,
8714	struct netdev_nested_priv *priv),
8715	struct netdev_nested_priv *priv)
8716	{
8717	struct net_device ldev, next, now, dev_stack[MAX_NEST_DEV + 1];
8718	struct list_head niter, iter, *iter_stack[MAX_NEST_DEV + 1];
8719	int ret, cur = 0;
8720
8721	now = dev;
8722	iter = &dev->adj_list.lower;
8723
8724	while (1) {
8725	if (now != dev) {
8726	ret = fn(now, priv);
8727	if (ret)
8728	return ret;
8729	}
8730
8731	next = NULL;
8732	while (1) {
8733	ldev = netdev_next_lower_dev_rcu(now, &iter);
8734	if (!ldev)
8735	break;
8736
8737	next = ldev;
8738	niter = &ldev->adj_list.lower;
8739	dev_stack[cur] = now;
8740	iter_stack[cur++] = iter;
8741	break;
8742	}
8743
8744	if (!next) {
8745	if (!cur)
8746	return 0;
8747	next = dev_stack[--cur];
8748	niter = iter_stack[cur];
8749	}
8750
8751	now = next;
8752	iter = niter;
8753	}
8754
8755	return 0;
8756	}
8757	EXPORT_SYMBOL_GPL(netdev_walk_all_lower_dev_rcu);
8758
8759	/**
8760	* netdev_lower_get_first_private_rcu - Get the first ->private from the
8761	* lower neighbour list, RCU
8762	* variant
8763	* @dev: device
8764	*
8765	* Gets the first netdev_adjacent->private from the dev's lower neighbour
8766	* list. The caller must hold RCU read lock.
8767	*/
8768	void netdev_lower_get_first_private_rcu(struct net_device dev)
8769	{
8770	struct netdev_adjacent *lower;
8771
8772	lower = list_first_or_null_rcu(&dev->adj_list.lower,
8773	struct netdev_adjacent, list);
8774	if (lower)
8775	return lower->private;
8776	return NULL;
8777	}
8778	EXPORT_SYMBOL(netdev_lower_get_first_private_rcu);
8779
8780	/**
8781	* netdev_master_upper_dev_get_rcu - Get master upper device
8782	* @dev: device
8783	*
8784	* Find a master upper device and return pointer to it or NULL in case
8785	* it's not there. The caller must hold the RCU read lock.
8786	*/
8787	struct net_device netdev_master_upper_dev_get_rcu(struct net_device dev)
8788	{
8789	struct netdev_adjacent *upper;
8790
8791	upper = list_first_or_null_rcu(&dev->adj_list.upper,
8792	struct netdev_adjacent, list);
8793	if (upper && likely(upper->master))
8794	return upper->dev;
8795	return NULL;
8796	}
8797	EXPORT_SYMBOL(netdev_master_upper_dev_get_rcu);
8798
8799	static int netdev_adjacent_sysfs_add(struct net_device *dev,
8800	struct net_device *adj_dev,
8801	struct list_head *dev_list)
8802	{
8803	char linkname[IFNAMSIZ+7];
8804
8805	sprintf(linkname, dev_list == &dev->adj_list.upper ?
8806	"upper_%s" : "lower_%s", adj_dev->name);
8807	return sysfs_create_link(&(dev->dev.kobj), &(adj_dev->dev.kobj),
8808	linkname);
8809	}
8810	static void netdev_adjacent_sysfs_del(struct net_device *dev,
8811	char *name,
8812	struct list_head *dev_list)
8813	{
8814	char linkname[IFNAMSIZ+7];
8815
8816	sprintf(linkname, dev_list == &dev->adj_list.upper ?
8817	"upper_%s" : "lower_%s", name);
8818	sysfs_remove_link(&(dev->dev.kobj), linkname);
8819	}
8820
8821	static inline bool netdev_adjacent_is_neigh_list(struct net_device *dev,
8822	struct net_device *adj_dev,
8823	struct list_head *dev_list)
8824	{
8825	return (dev_list == &dev->adj_list.upper \|\|
8826	dev_list == &dev->adj_list.lower) &&
8827	net_eq(dev_net(dev), dev_net(adj_dev));
8828	}
8829
8830	static int __netdev_adjacent_dev_insert(struct net_device *dev,
8831	struct net_device *adj_dev,
8832	struct list_head *dev_list,
8833	void *private, bool master)
8834	{
8835	struct netdev_adjacent *adj;
8836	int ret;
8837
8838	adj = __netdev_find_adj(adj_dev, dev_list);
8839
8840	if (adj) {
8841	adj->ref_nr += 1;
8842	pr_debug("Insert adjacency: dev %s adj_dev %s adj->ref_nr %d\n",
8843	dev->name, adj_dev->name, adj->ref_nr);
8844
8845	return 0;
8846	}
8847
8848	adj = kmalloc_obj(*adj);
8849	if (!adj)
8850	return -ENOMEM;
8851
8852	adj->dev = adj_dev;
8853	adj->master = master;
8854	adj->ref_nr = 1;
8855	adj->private = private;
8856	adj->ignore = false;
8857	netdev_hold(adj_dev, &adj->dev_tracker, GFP_KERNEL);
8858
8859	pr_debug("Insert adjacency: dev %s adj_dev %s adj->ref_nr %d; dev_hold on %s\n",
8860	dev->name, adj_dev->name, adj->ref_nr, adj_dev->name);
8861
8862	if (netdev_adjacent_is_neigh_list(dev, adj_dev, dev_list)) {
8863	ret = netdev_adjacent_sysfs_add(dev, adj_dev, dev_list);
8864	if (ret)
8865	goto free_adj;
8866	}
8867
8868	/* Ensure that master link is always the first item in list. */
8869	if (master) {
8870	ret = sysfs_create_link(&(dev->dev.kobj),
8871	&(adj_dev->dev.kobj), "master");
8872	if (ret)
8873	goto remove_symlinks;
8874
8875	list_add_rcu(&adj->list, dev_list);
8876	} else {
8877	list_add_tail_rcu(&adj->list, dev_list);
8878	}
8879
8880	return 0;
8881
8882	remove_symlinks:
8883	if (netdev_adjacent_is_neigh_list(dev, adj_dev, dev_list))
8884	netdev_adjacent_sysfs_del(dev, adj_dev->name, dev_list);
8885	free_adj:
8886	netdev_put(adj_dev, &adj->dev_tracker);
8887	kfree(adj);
8888
8889	return ret;
8890	}
8891
8892	static void __netdev_adjacent_dev_remove(struct net_device *dev,
8893	struct net_device *adj_dev,
8894	u16 ref_nr,
8895	struct list_head *dev_list)
8896	{
8897	struct netdev_adjacent *adj;
8898
8899	pr_debug("Remove adjacency: dev %s adj_dev %s ref_nr %d\n",
8900	dev->name, adj_dev->name, ref_nr);
8901
8902	adj = __netdev_find_adj(adj_dev, dev_list);
8903
8904	if (!adj) {
8905	pr_err("Adjacency does not exist for device %s from %s\n",
8906	dev->name, adj_dev->name);
8907	WARN_ON(1);
8908	return;
8909	}
8910
8911	if (adj->ref_nr > ref_nr) {
8912	pr_debug("adjacency: %s to %s ref_nr - %d = %d\n",
8913	dev->name, adj_dev->name, ref_nr,
8914	adj->ref_nr - ref_nr);
8915	adj->ref_nr -= ref_nr;
8916	return;
8917	}
8918
8919	if (adj->master)
8920	sysfs_remove_link(&(dev->dev.kobj), "master");
8921
8922	if (netdev_adjacent_is_neigh_list(dev, adj_dev, dev_list))
8923	netdev_adjacent_sysfs_del(dev, adj_dev->name, dev_list);
8924
8925	list_del_rcu(&adj->list);
8926	pr_debug("adjacency: dev_put for %s, because link removed from %s to %s\n",
8927	adj_dev->name, dev->name, adj_dev->name);
8928	netdev_put(adj_dev, &adj->dev_tracker);
8929	kfree_rcu(adj, rcu);
8930	}
8931
8932	static int __netdev_adjacent_dev_link_lists(struct net_device *dev,
8933	struct net_device *upper_dev,
8934	struct list_head *up_list,
8935	struct list_head *down_list,
8936	void *private, bool master)
8937	{
8938	int ret;
8939
8940	ret = __netdev_adjacent_dev_insert(dev, upper_dev, up_list,
8941	private, master);
8942	if (ret)
8943	return ret;
8944
8945	ret = __netdev_adjacent_dev_insert(upper_dev, dev, down_list,
8946	private, false);
8947	if (ret) {
8948	__netdev_adjacent_dev_remove(dev, upper_dev, 1, up_list);
8949	return ret;
8950	}
8951
8952	return 0;
8953	}
8954
8955	static void __netdev_adjacent_dev_unlink_lists(struct net_device *dev,
8956	struct net_device *upper_dev,
8957	u16 ref_nr,
8958	struct list_head *up_list,
8959	struct list_head *down_list)
8960	{
8961	__netdev_adjacent_dev_remove(dev, upper_dev, ref_nr, up_list);
8962	__netdev_adjacent_dev_remove(upper_dev, dev, ref_nr, down_list);
8963	}
8964
8965	static int __netdev_adjacent_dev_link_neighbour(struct net_device *dev,
8966	struct net_device *upper_dev,
8967	void *private, bool master)
8968	{
8969	return __netdev_adjacent_dev_link_lists(dev, upper_dev,
8970	&dev->adj_list.upper,
8971	&upper_dev->adj_list.lower,
8972	private, master);
8973	}
8974
8975	static void __netdev_adjacent_dev_unlink_neighbour(struct net_device *dev,
8976	struct net_device *upper_dev)
8977	{
8978	__netdev_adjacent_dev_unlink_lists(dev, upper_dev, 1,
8979	&dev->adj_list.upper,
8980	&upper_dev->adj_list.lower);
8981	}
8982
8983	static int __netdev_upper_dev_link(struct net_device *dev,
8984	struct net_device *upper_dev, bool master,
8985	void upper_priv, void upper_info,
8986	struct netdev_nested_priv *priv,
8987	struct netlink_ext_ack *extack)
8988	{
8989	struct netdev_notifier_changeupper_info changeupper_info = {
8990	.info = {
8991	.dev = dev,
8992	.extack = extack,
8993	},
8994	.upper_dev = upper_dev,
8995	.master = master,
8996	.linking = true,
8997	.upper_info = upper_info,
8998	};
8999	struct net_device *master_dev;
9000	int ret = 0;
9001
9002	ASSERT_RTNL();
9003
9004	if (dev == upper_dev)
9005	return -EBUSY;
9006
9007	/* To prevent loops, check if dev is not upper device to upper_dev. */
9008	if (__netdev_has_upper_dev(upper_dev, dev))
9009	return -EBUSY;
9010
9011	if ((dev->lower_level + upper_dev->upper_level) > MAX_NEST_DEV)
9012	return -EMLINK;
9013
9014	if (!master) {
9015	if (__netdev_has_upper_dev(dev, upper_dev))
9016	return -EEXIST;
9017	} else {
9018	master_dev = __netdev_master_upper_dev_get(dev);
9019	if (master_dev)
9020	return master_dev == upper_dev ? -EEXIST : -EBUSY;
9021	}
9022
9023	ret = call_netdevice_notifiers_info(NETDEV_PRECHANGEUPPER,
9024	&changeupper_info.info);
9025	ret = notifier_to_errno(ret);
9026	if (ret)
9027	return ret;
9028
9029	ret = __netdev_adjacent_dev_link_neighbour(dev, upper_dev, upper_priv,
9030	master);
9031	if (ret)
9032	return ret;
9033
9034	ret = call_netdevice_notifiers_info(NETDEV_CHANGEUPPER,
9035	&changeupper_info.info);
9036	ret = notifier_to_errno(ret);
9037	if (ret)
9038	goto rollback;
9039
9040	__netdev_update_upper_level(dev, NULL);
9041	__netdev_walk_all_lower_dev(dev, __netdev_update_upper_level, NULL);
9042
9043	__netdev_update_lower_level(upper_dev, priv);
9044	__netdev_walk_all_upper_dev(upper_dev, __netdev_update_lower_level,
9045	priv);
9046
9047	return 0;
9048
9049	rollback:
9050	__netdev_adjacent_dev_unlink_neighbour(dev, upper_dev);
9051
9052	return ret;
9053	}
9054
9055	/**
9056	* netdev_upper_dev_link - Add a link to the upper device
9057	* @dev: device
9058	* @upper_dev: new upper device
9059	* @extack: netlink extended ack
9060	*
9061	* Adds a link to device which is upper to this one. The caller must hold
9062	* the RTNL lock. On a failure a negative errno code is returned.
9063	* On success the reference counts are adjusted and the function
9064	* returns zero.
9065	*/
9066	int netdev_upper_dev_link(struct net_device *dev,
9067	struct net_device *upper_dev,
9068	struct netlink_ext_ack *extack)
9069	{
9070	struct netdev_nested_priv priv = {
9071	.flags = NESTED_SYNC_IMM \| NESTED_SYNC_TODO,
9072	.data = NULL,
9073	};
9074
9075	return __netdev_upper_dev_link(dev, upper_dev, false,
9076	NULL, NULL, &priv, extack);
9077	}
9078	EXPORT_SYMBOL(netdev_upper_dev_link);
9079
9080	/**
9081	* netdev_master_upper_dev_link - Add a master link to the upper device
9082	* @dev: device
9083	* @upper_dev: new upper device
9084	* @upper_priv: upper device private
9085	* @upper_info: upper info to be passed down via notifier
9086	* @extack: netlink extended ack
9087	*
9088	* Adds a link to device which is upper to this one. In this case, only
9089	* one master upper device can be linked, although other non-master devices
9090	* might be linked as well. The caller must hold the RTNL lock.
9091	* On a failure a negative errno code is returned. On success the reference
9092	* counts are adjusted and the function returns zero.
9093	*/
9094	int netdev_master_upper_dev_link(struct net_device *dev,
9095	struct net_device *upper_dev,
9096	void upper_priv, void upper_info,
9097	struct netlink_ext_ack *extack)
9098	{
9099	struct netdev_nested_priv priv = {
9100	.flags = NESTED_SYNC_IMM \| NESTED_SYNC_TODO,
9101	.data = NULL,
9102	};
9103
9104	return __netdev_upper_dev_link(dev, upper_dev, true,
9105	upper_priv, upper_info, &priv, extack);
9106	}
9107	EXPORT_SYMBOL(netdev_master_upper_dev_link);
9108
9109	static void __netdev_upper_dev_unlink(struct net_device *dev,
9110	struct net_device *upper_dev,
9111	struct netdev_nested_priv *priv)
9112	{
9113	struct netdev_notifier_changeupper_info changeupper_info = {
9114	.info = {
9115	.dev = dev,
9116	},
9117	.upper_dev = upper_dev,
9118	.linking = false,
9119	};
9120
9121	ASSERT_RTNL();
9122
9123	changeupper_info.master = netdev_master_upper_dev_get(dev) == upper_dev;
9124
9125	call_netdevice_notifiers_info(NETDEV_PRECHANGEUPPER,
9126	&changeupper_info.info);
9127
9128	__netdev_adjacent_dev_unlink_neighbour(dev, upper_dev);
9129
9130	call_netdevice_notifiers_info(NETDEV_CHANGEUPPER,
9131	&changeupper_info.info);
9132
9133	__netdev_update_upper_level(dev, NULL);
9134	__netdev_walk_all_lower_dev(dev, __netdev_update_upper_level, NULL);
9135
9136	__netdev_update_lower_level(upper_dev, priv);
9137	__netdev_walk_all_upper_dev(upper_dev, __netdev_update_lower_level,
9138	priv);
9139	}
9140
9141	/**
9142	* netdev_upper_dev_unlink - Removes a link to upper device
9143	* @dev: device
9144	* @upper_dev: new upper device
9145	*
9146	* Removes a link to device which is upper to this one. The caller must hold
9147	* the RTNL lock.
9148	*/
9149	void netdev_upper_dev_unlink(struct net_device *dev,
9150	struct net_device *upper_dev)
9151	{
9152	struct netdev_nested_priv priv = {
9153	.flags = NESTED_SYNC_TODO,
9154	.data = NULL,
9155	};
9156
9157	__netdev_upper_dev_unlink(dev, upper_dev, &priv);
9158	}
9159	EXPORT_SYMBOL(netdev_upper_dev_unlink);
9160
9161	static void __netdev_adjacent_dev_set(struct net_device *upper_dev,
9162	struct net_device *lower_dev,
9163	bool val)
9164	{
9165	struct netdev_adjacent *adj;
9166
9167	adj = __netdev_find_adj(lower_dev, &upper_dev->adj_list.lower);
9168	if (adj)
9169	adj->ignore = val;
9170
9171	adj = __netdev_find_adj(upper_dev, &lower_dev->adj_list.upper);
9172	if (adj)
9173	adj->ignore = val;
9174	}
9175
9176	static void netdev_adjacent_dev_disable(struct net_device *upper_dev,
9177	struct net_device *lower_dev)
9178	{
9179	__netdev_adjacent_dev_set(upper_dev, lower_dev, true);
9180	}
9181
9182	static void netdev_adjacent_dev_enable(struct net_device *upper_dev,
9183	struct net_device *lower_dev)
9184	{
9185	__netdev_adjacent_dev_set(upper_dev, lower_dev, false);
9186	}
9187
9188	int netdev_adjacent_change_prepare(struct net_device *old_dev,
9189	struct net_device *new_dev,
9190	struct net_device *dev,
9191	struct netlink_ext_ack *extack)
9192	{
9193	struct netdev_nested_priv priv = {
9194	.flags = 0,
9195	.data = NULL,
9196	};
9197	int err;
9198
9199	if (!new_dev)
9200	return 0;
9201
9202	if (old_dev && new_dev != old_dev)
9203	netdev_adjacent_dev_disable(dev, old_dev);
9204	err = __netdev_upper_dev_link(new_dev, dev, false, NULL, NULL, &priv,
9205	extack);
9206	if (err) {
9207	if (old_dev && new_dev != old_dev)
9208	netdev_adjacent_dev_enable(dev, old_dev);
9209	return err;
9210	}
9211
9212	return 0;
9213	}
9214	EXPORT_SYMBOL(netdev_adjacent_change_prepare);
9215
9216	void netdev_adjacent_change_commit(struct net_device *old_dev,
9217	struct net_device *new_dev,
9218	struct net_device *dev)
9219	{
9220	struct netdev_nested_priv priv = {
9221	.flags = NESTED_SYNC_IMM \| NESTED_SYNC_TODO,
9222	.data = NULL,
9223	};
9224
9225	if (!new_dev \|\| !old_dev)
9226	return;
9227
9228	if (new_dev == old_dev)
9229	return;
9230
9231	netdev_adjacent_dev_enable(dev, old_dev);
9232	__netdev_upper_dev_unlink(old_dev, dev, &priv);
9233	}
9234	EXPORT_SYMBOL(netdev_adjacent_change_commit);
9235
9236	void netdev_adjacent_change_abort(struct net_device *old_dev,
9237	struct net_device *new_dev,
9238	struct net_device *dev)
9239	{
9240	struct netdev_nested_priv priv = {
9241	.flags = 0,
9242	.data = NULL,
9243	};
9244
9245	if (!new_dev)
9246	return;
9247
9248	if (old_dev && new_dev != old_dev)
9249	netdev_adjacent_dev_enable(dev, old_dev);
9250
9251	__netdev_upper_dev_unlink(new_dev, dev, &priv);
9252	}
9253	EXPORT_SYMBOL(netdev_adjacent_change_abort);
9254
9255	/**
9256	* netdev_bonding_info_change - Dispatch event about slave change
9257	* @dev: device
9258	* @bonding_info: info to dispatch
9259	*
9260	* Send NETDEV_BONDING_INFO to netdev notifiers with info.
9261	* The caller must hold the RTNL lock.
9262	*/
9263	void netdev_bonding_info_change(struct net_device *dev,
9264	struct netdev_bonding_info *bonding_info)
9265	{
9266	struct netdev_notifier_bonding_info info = {
9267	.info.dev = dev,
9268	};
9269
9270	memcpy(&info.bonding_info, bonding_info,
9271	sizeof(struct netdev_bonding_info));
9272	call_netdevice_notifiers_info(NETDEV_BONDING_INFO,
9273	&info.info);
9274	}
9275	EXPORT_SYMBOL(netdev_bonding_info_change);
9276
9277	static int netdev_offload_xstats_enable_l3(struct net_device *dev,
9278	struct netlink_ext_ack *extack)
9279	{
9280	struct netdev_notifier_offload_xstats_info info = {
9281	.info.dev = dev,
9282	.info.extack = extack,
9283	.type = NETDEV_OFFLOAD_XSTATS_TYPE_L3,
9284	};
9285	int err;
9286	int rc;
9287
9288	dev->offload_xstats_l3 = kzalloc_obj(*dev->offload_xstats_l3);
9289	if (!dev->offload_xstats_l3)
9290	return -ENOMEM;
9291
9292	rc = call_netdevice_notifiers_info_robust(NETDEV_OFFLOAD_XSTATS_ENABLE,
9293	NETDEV_OFFLOAD_XSTATS_DISABLE,
9294	&info.info);
9295	err = notifier_to_errno(rc);
9296	if (err)
9297	goto free_stats;
9298
9299	return 0;
9300
9301	free_stats:
9302	kfree(dev->offload_xstats_l3);
9303	dev->offload_xstats_l3 = NULL;
9304	return err;
9305	}
9306
9307	int netdev_offload_xstats_enable(struct net_device *dev,
9308	enum netdev_offload_xstats_type type,
9309	struct netlink_ext_ack *extack)
9310	{
9311	ASSERT_RTNL();
9312
9313	if (netdev_offload_xstats_enabled(dev, type))
9314	return -EALREADY;
9315
9316	switch (type) {
9317	case NETDEV_OFFLOAD_XSTATS_TYPE_L3:
9318	return netdev_offload_xstats_enable_l3(dev, extack);
9319	}
9320
9321	WARN_ON(1);
9322	return -EINVAL;
9323	}
9324	EXPORT_SYMBOL(netdev_offload_xstats_enable);
9325
9326	static void netdev_offload_xstats_disable_l3(struct net_device *dev)
9327	{
9328	struct netdev_notifier_offload_xstats_info info = {
9329	.info.dev = dev,
9330	.type = NETDEV_OFFLOAD_XSTATS_TYPE_L3,
9331	};
9332
9333	call_netdevice_notifiers_info(NETDEV_OFFLOAD_XSTATS_DISABLE,
9334	&info.info);
9335	kfree(dev->offload_xstats_l3);
9336	dev->offload_xstats_l3 = NULL;
9337	}
9338
9339	int netdev_offload_xstats_disable(struct net_device *dev,
9340	enum netdev_offload_xstats_type type)
9341	{
9342	ASSERT_RTNL();
9343
9344	if (!netdev_offload_xstats_enabled(dev, type))
9345	return -EALREADY;
9346
9347	switch (type) {
9348	case NETDEV_OFFLOAD_XSTATS_TYPE_L3:
9349	netdev_offload_xstats_disable_l3(dev);
9350	return 0;
9351	}
9352
9353	WARN_ON(1);
9354	return -EINVAL;
9355	}
9356	EXPORT_SYMBOL(netdev_offload_xstats_disable);
9357
9358	static void netdev_offload_xstats_disable_all(struct net_device *dev)
9359	{
9360	netdev_offload_xstats_disable(dev, NETDEV_OFFLOAD_XSTATS_TYPE_L3);
9361	}
9362
9363	static struct rtnl_hw_stats64 *
9364	netdev_offload_xstats_get_ptr(const struct net_device *dev,
9365	enum netdev_offload_xstats_type type)
9366	{
9367	switch (type) {
9368	case NETDEV_OFFLOAD_XSTATS_TYPE_L3:
9369	return dev->offload_xstats_l3;
9370	}
9371
9372	WARN_ON(1);
9373	return NULL;
9374	}
9375
9376	bool netdev_offload_xstats_enabled(const struct net_device *dev,
9377	enum netdev_offload_xstats_type type)
9378	{
9379	ASSERT_RTNL();
9380
9381	return netdev_offload_xstats_get_ptr(dev, type);
9382	}
9383	EXPORT_SYMBOL(netdev_offload_xstats_enabled);
9384
9385	struct netdev_notifier_offload_xstats_ru {
9386	bool used;
9387	};
9388
9389	struct netdev_notifier_offload_xstats_rd {
9390	struct rtnl_hw_stats64 stats;
9391	bool used;
9392	};
9393
9394	static void netdev_hw_stats64_add(struct rtnl_hw_stats64 *dest,
9395	const struct rtnl_hw_stats64 *src)
9396	{
9397	dest->rx_packets += src->rx_packets;
9398	dest->tx_packets += src->tx_packets;
9399	dest->rx_bytes += src->rx_bytes;
9400	dest->tx_bytes += src->tx_bytes;
9401	dest->rx_errors += src->rx_errors;
9402	dest->tx_errors += src->tx_errors;
9403	dest->rx_dropped += src->rx_dropped;
9404	dest->tx_dropped += src->tx_dropped;
9405	dest->multicast += src->multicast;
9406	}
9407
9408	static int netdev_offload_xstats_get_used(struct net_device *dev,
9409	enum netdev_offload_xstats_type type,
9410	bool *p_used,
9411	struct netlink_ext_ack *extack)
9412	{
9413	struct netdev_notifier_offload_xstats_ru report_used = {};
9414	struct netdev_notifier_offload_xstats_info info = {
9415	.info.dev = dev,
9416	.info.extack = extack,
9417	.type = type,
9418	.report_used = &report_used,
9419	};
9420	int rc;
9421
9422	WARN_ON(!netdev_offload_xstats_enabled(dev, type));
9423	rc = call_netdevice_notifiers_info(NETDEV_OFFLOAD_XSTATS_REPORT_USED,
9424	&info.info);
9425	*p_used = report_used.used;
9426	return notifier_to_errno(rc);
9427	}
9428
9429	static int netdev_offload_xstats_get_stats(struct net_device *dev,
9430	enum netdev_offload_xstats_type type,
9431	struct rtnl_hw_stats64 *p_stats,
9432	bool *p_used,
9433	struct netlink_ext_ack *extack)
9434	{
9435	struct netdev_notifier_offload_xstats_rd report_delta = {};
9436	struct netdev_notifier_offload_xstats_info info = {
9437	.info.dev = dev,
9438	.info.extack = extack,
9439	.type = type,
9440	.report_delta = &report_delta,
9441	};
9442	struct rtnl_hw_stats64 *stats;
9443	int rc;
9444
9445	stats = netdev_offload_xstats_get_ptr(dev, type);
9446	if (WARN_ON(!stats))
9447	return -EINVAL;
9448
9449	rc = call_netdevice_notifiers_info(NETDEV_OFFLOAD_XSTATS_REPORT_DELTA,
9450	&info.info);
9451
9452	/* Cache whatever we got, even if there was an error, otherwise the
9453	* successful stats retrievals would get lost.
9454	*/
9455	netdev_hw_stats64_add(stats, &report_delta.stats);
9456
9457	if (p_stats)
9458	p_stats = stats;
9459	*p_used = report_delta.used;
9460
9461	return notifier_to_errno(rc);
9462	}
9463
9464	int netdev_offload_xstats_get(struct net_device *dev,
9465	enum netdev_offload_xstats_type type,
9466	struct rtnl_hw_stats64 p_stats, bool p_used,
9467	struct netlink_ext_ack *extack)
9468	{
9469	ASSERT_RTNL();
9470
9471	if (p_stats)
9472	return netdev_offload_xstats_get_stats(dev, type, p_stats,
9473	p_used, extack);
9474	else
9475	return netdev_offload_xstats_get_used(dev, type, p_used,
9476	extack);
9477	}
9478	EXPORT_SYMBOL(netdev_offload_xstats_get);
9479
9480	void
9481	netdev_offload_xstats_report_delta(struct netdev_notifier_offload_xstats_rd *report_delta,
9482	const struct rtnl_hw_stats64 *stats)
9483	{
9484	report_delta->used = true;
9485	netdev_hw_stats64_add(&report_delta->stats, stats);
9486	}
9487	EXPORT_SYMBOL(netdev_offload_xstats_report_delta);
9488
9489	void
9490	netdev_offload_xstats_report_used(struct netdev_notifier_offload_xstats_ru *report_used)
9491	{
9492	report_used->used = true;
9493	}
9494	EXPORT_SYMBOL(netdev_offload_xstats_report_used);
9495
9496	void netdev_offload_xstats_push_delta(struct net_device *dev,
9497	enum netdev_offload_xstats_type type,
9498	const struct rtnl_hw_stats64 *p_stats)
9499	{
9500	struct rtnl_hw_stats64 *stats;
9501
9502	ASSERT_RTNL();
9503
9504	stats = netdev_offload_xstats_get_ptr(dev, type);
9505	if (WARN_ON(!stats))
9506	return;
9507
9508	netdev_hw_stats64_add(stats, p_stats);
9509	}
9510	EXPORT_SYMBOL(netdev_offload_xstats_push_delta);
9511
9512	/**
9513	* netdev_get_xmit_slave - Get the xmit slave of master device
9514	* @dev: device
9515	* @skb: The packet
9516	* @all_slaves: assume all the slaves are active
9517	*
9518	* The reference counters are not incremented so the caller must be
9519	* careful with locks. The caller must hold RCU lock.
9520	* %NULL is returned if no slave is found.
9521	*/
9522
9523	struct net_device netdev_get_xmit_slave(struct net_device dev,
9524	struct sk_buff *skb,
9525	bool all_slaves)
9526	{
9527	const struct net_device_ops *ops = dev->netdev_ops;
9528
9529	if (!ops->ndo_get_xmit_slave)
9530	return NULL;
9531	return ops->ndo_get_xmit_slave(dev, skb, all_slaves);
9532	}
9533	EXPORT_SYMBOL(netdev_get_xmit_slave);
9534
9535	static struct net_device netdev_sk_get_lower_dev(struct net_device dev,
9536	struct sock *sk)
9537	{
9538	const struct net_device_ops *ops = dev->netdev_ops;
9539
9540	if (!ops->ndo_sk_get_lower_dev)
9541	return NULL;
9542	return ops->ndo_sk_get_lower_dev(dev, sk);
9543	}
9544
9545	/**
9546	* netdev_sk_get_lowest_dev - Get the lowest device in chain given device and socket
9547	* @dev: device
9548	* @sk: the socket
9549	*
9550	* %NULL is returned if no lower device is found.
9551	*/
9552
9553	struct net_device netdev_sk_get_lowest_dev(struct net_device dev,
9554	struct sock *sk)
9555	{
9556	struct net_device *lower;
9557
9558	lower = netdev_sk_get_lower_dev(dev, sk);
9559	while (lower) {
9560	dev = lower;
9561	lower = netdev_sk_get_lower_dev(dev, sk);
9562	}
9563
9564	return dev;
9565	}
9566	EXPORT_SYMBOL(netdev_sk_get_lowest_dev);
9567
9568	static void netdev_adjacent_add_links(struct net_device *dev)
9569	{
9570	struct netdev_adjacent *iter;
9571
9572	struct net *net = dev_net(dev);
9573
9574	list_for_each_entry(iter, &dev->adj_list.upper, list) {
9575	if (!net_eq(net, dev_net(iter->dev)))
9576	continue;
9577	netdev_adjacent_sysfs_add(iter->dev, dev,
9578	&iter->dev->adj_list.lower);
9579	netdev_adjacent_sysfs_add(dev, iter->dev,
9580	&dev->adj_list.upper);
9581	}
9582
9583	list_for_each_entry(iter, &dev->adj_list.lower, list) {
9584	if (!net_eq(net, dev_net(iter->dev)))
9585	continue;
9586	netdev_adjacent_sysfs_add(iter->dev, dev,
9587	&iter->dev->adj_list.upper);
9588	netdev_adjacent_sysfs_add(dev, iter->dev,
9589	&dev->adj_list.lower);
9590	}
9591	}
9592
9593	static void netdev_adjacent_del_links(struct net_device *dev)
9594	{
9595	struct netdev_adjacent *iter;
9596
9597	struct net *net = dev_net(dev);
9598
9599	list_for_each_entry(iter, &dev->adj_list.upper, list) {
9600	if (!net_eq(net, dev_net(iter->dev)))
9601	continue;
9602	netdev_adjacent_sysfs_del(iter->dev, dev->name,
9603	&iter->dev->adj_list.lower);
9604	netdev_adjacent_sysfs_del(dev, iter->dev->name,
9605	&dev->adj_list.upper);
9606	}
9607
9608	list_for_each_entry(iter, &dev->adj_list.lower, list) {
9609	if (!net_eq(net, dev_net(iter->dev)))
9610	continue;
9611	netdev_adjacent_sysfs_del(iter->dev, dev->name,
9612	&iter->dev->adj_list.upper);
9613	netdev_adjacent_sysfs_del(dev, iter->dev->name,
9614	&dev->adj_list.lower);
9615	}
9616	}
9617
9618	void netdev_adjacent_rename_links(struct net_device dev, char oldname)
9619	{
9620	struct netdev_adjacent *iter;
9621
9622	struct net *net = dev_net(dev);
9623
9624	list_for_each_entry(iter, &dev->adj_list.upper, list) {
9625	if (!net_eq(net, dev_net(iter->dev)))
9626	continue;
9627	netdev_adjacent_sysfs_del(iter->dev, oldname,
9628	&iter->dev->adj_list.lower);
9629	netdev_adjacent_sysfs_add(iter->dev, dev,
9630	&iter->dev->adj_list.lower);
9631	}
9632
9633	list_for_each_entry(iter, &dev->adj_list.lower, list) {
9634	if (!net_eq(net, dev_net(iter->dev)))
9635	continue;
9636	netdev_adjacent_sysfs_del(iter->dev, oldname,
9637	&iter->dev->adj_list.upper);
9638	netdev_adjacent_sysfs_add(iter->dev, dev,
9639	&iter->dev->adj_list.upper);
9640	}
9641	}
9642
9643	void netdev_lower_dev_get_private(struct net_device dev,
9644	struct net_device *lower_dev)
9645	{
9646	struct netdev_adjacent *lower;
9647
9648	if (!lower_dev)
9649	return NULL;
9650	lower = __netdev_find_adj(lower_dev, &dev->adj_list.lower);
9651	if (!lower)
9652	return NULL;
9653
9654	return lower->private;
9655	}
9656	EXPORT_SYMBOL(netdev_lower_dev_get_private);
9657
9658
9659	/**
9660	* netdev_lower_state_changed - Dispatch event about lower device state change
9661	* @lower_dev: device
9662	* @lower_state_info: state to dispatch
9663	*
9664	* Send NETDEV_CHANGELOWERSTATE to netdev notifiers with info.
9665	* The caller must hold the RTNL lock.
9666	*/
9667	void netdev_lower_state_changed(struct net_device *lower_dev,
9668	void *lower_state_info)
9669	{
9670	struct netdev_notifier_changelowerstate_info changelowerstate_info = {
9671	.info.dev = lower_dev,
9672	};
9673
9674	ASSERT_RTNL();
9675	changelowerstate_info.lower_state_info = lower_state_info;
9676	call_netdevice_notifiers_info(NETDEV_CHANGELOWERSTATE,
9677	&changelowerstate_info.info);
9678	}
9679	EXPORT_SYMBOL(netdev_lower_state_changed);
9680
9681	static void dev_change_rx_flags(struct net_device *dev, int flags)
9682	{
9683	const struct net_device_ops *ops = dev->netdev_ops;
9684
9685	if (ops->ndo_change_rx_flags)
9686	ops->ndo_change_rx_flags(dev, flags);
9687	}
9688
9689	int __dev_set_promiscuity(struct net_device *dev, int inc, bool notify)
9690	{
9691	unsigned int old_flags = dev->flags;
9692	unsigned int promiscuity, flags;
9693	kuid_t uid;
9694	kgid_t gid;
9695
9696	netdev_assert_locked_ops_compat(dev);
9697
9698	promiscuity = dev->promiscuity + inc;
9699	if (promiscuity == 0) {
9700	/*
9701	* Avoid overflow.
9702	* If inc causes overflow, untouch promisc and return error.
9703	*/
9704	if (unlikely(inc > 0)) {
9705	netdev_warn(dev, "promiscuity touches roof, set promiscuity failed. promiscuity feature of device might be broken.\n");
9706	return -EOVERFLOW;
9707	}
9708	flags = old_flags & ~IFF_PROMISC;
9709	} else {
9710	flags = old_flags \| IFF_PROMISC;
9711	}
9712	WRITE_ONCE(dev->promiscuity, promiscuity);
9713	if (flags != old_flags) {
9714	WRITE_ONCE(dev->flags, flags);
9715	netdev_info(dev, "%s promiscuous mode\n",
9716	dev->flags & IFF_PROMISC ? "entered" : "left");
9717	if (audit_enabled) {
9718	current_uid_gid(&uid, &gid);
9719	audit_log(audit_context(), GFP_ATOMIC,
9720	AUDIT_ANOM_PROMISCUOUS,
9721	"dev=%s prom=%d old_prom=%d auid=%u uid=%u gid=%u ses=%u",
9722	dev->name, (dev->flags & IFF_PROMISC),
9723	(old_flags & IFF_PROMISC),
9724	from_kuid(&init_user_ns, audit_get_loginuid(current)),
9725	from_kuid(&init_user_ns, uid),
9726	from_kgid(&init_user_ns, gid),
9727	audit_get_sessionid(current));
9728	}
9729
9730	dev_change_rx_flags(dev, IFF_PROMISC);
9731	}
9732	if (notify)
9733	__dev_notify_flags(dev, old_flags, IFF_PROMISC, 0, NULL);
9734	return 0;
9735	}
9736
9737	int netif_set_promiscuity(struct net_device *dev, int inc)
9738	{
9739	unsigned int old_flags = dev->flags;
9740	int err;
9741
9742	err = __dev_set_promiscuity(dev, inc, true);
9743	if (err < 0)
9744	return err;
9745	if (dev->flags != old_flags)
9746	dev_set_rx_mode(dev);
9747	return err;
9748	}
9749
9750	int netif_set_allmulti(struct net_device *dev, int inc, bool notify)
9751	{
9752	unsigned int old_flags = dev->flags, old_gflags = dev->gflags;
9753	unsigned int allmulti, flags;
9754
9755	netdev_assert_locked_ops_compat(dev);
9756
9757	allmulti = dev->allmulti + inc;
9758	if (allmulti == 0) {
9759	/*
9760	* Avoid overflow.
9761	* If inc causes overflow, untouch allmulti and return error.
9762	*/
9763	if (unlikely(inc > 0)) {
9764	netdev_warn(dev, "allmulti touches roof, set allmulti failed. allmulti feature of device might be broken.\n");
9765	return -EOVERFLOW;
9766	}
9767	flags = old_flags & ~IFF_ALLMULTI;
9768	} else {
9769	flags = old_flags \| IFF_ALLMULTI;
9770	}
9771	WRITE_ONCE(dev->allmulti, allmulti);
9772	if (flags != old_flags) {
9773	WRITE_ONCE(dev->flags, flags);
9774	netdev_info(dev, "%s allmulticast mode\n",
9775	dev->flags & IFF_ALLMULTI ? "entered" : "left");
9776	dev_change_rx_flags(dev, IFF_ALLMULTI);
9777	dev_set_rx_mode(dev);
9778	if (notify)
9779	__dev_notify_flags(dev, old_flags,
9780	dev->gflags ^ old_gflags, 0, NULL);
9781	}
9782	return 0;
9783	}
9784
9785
9786	/**
9787	* netif_get_flags() - get flags reported to userspace
9788	* @dev: device
9789	*
9790	* Get the combination of flag bits exported through APIs to userspace.
9791	*/
9792	unsigned int netif_get_flags(const struct net_device *dev)
9793	{
9794	unsigned int flags;
9795
9796	flags = (READ_ONCE(dev->flags) & ~(IFF_PROMISC \|
9797	IFF_ALLMULTI \|
9798	IFF_RUNNING \|
9799	IFF_LOWER_UP \|
9800	IFF_DORMANT)) \|
9801	(READ_ONCE(dev->gflags) & (IFF_PROMISC \|
9802	IFF_ALLMULTI));
9803
9804	if (netif_running(dev)) {
9805	if (netif_oper_up(dev))
9806	flags \|= IFF_RUNNING;
9807	if (netif_carrier_ok(dev))
9808	flags \|= IFF_LOWER_UP;
9809	if (netif_dormant(dev))
9810	flags \|= IFF_DORMANT;
9811	}
9812
9813	return flags;
9814	}
9815	EXPORT_SYMBOL(netif_get_flags);
9816
9817	int __dev_change_flags(struct net_device *dev, unsigned int flags,
9818	struct netlink_ext_ack *extack)
9819	{
9820	unsigned int old_flags = dev->flags;
9821	int ret;
9822
9823	netdev_assert_locked_ops_compat(dev);
9824
9825	/*
9826	* Set the flags on our device.
9827	*/
9828
9829	dev->flags = (flags & (IFF_DEBUG \| IFF_NOTRAILERS \| IFF_NOARP \|
9830	IFF_DYNAMIC \| IFF_MULTICAST \| IFF_PORTSEL \|
9831	IFF_AUTOMEDIA)) \|
9832	(dev->flags & (IFF_UP \| IFF_VOLATILE \| IFF_PROMISC \|
9833	IFF_ALLMULTI));
9834
9835	/*
9836	* Load in the correct multicast list now the flags have changed.
9837	*/
9838
9839	if ((old_flags ^ flags) & IFF_MULTICAST)
9840	dev_change_rx_flags(dev, IFF_MULTICAST);
9841
9842	dev_set_rx_mode(dev);
9843
9844	/*
9845	* Have we downed the interface. We handle IFF_UP ourselves
9846	* according to user attempts to set it, rather than blindly
9847	* setting it.
9848	*/
9849
9850	ret = 0;
9851	if ((old_flags ^ flags) & IFF_UP) {
9852	if (old_flags & IFF_UP)
9853	__dev_close(dev);
9854	else
9855	ret = __dev_open(dev, extack);
9856	}
9857
9858	if ((flags ^ dev->gflags) & IFF_PROMISC) {
9859	int inc = (flags & IFF_PROMISC) ? 1 : -1;
9860	old_flags = dev->flags;
9861
9862	dev->gflags ^= IFF_PROMISC;
9863
9864	if (__dev_set_promiscuity(dev, inc, false) >= 0)
9865	if (dev->flags != old_flags)
9866	dev_set_rx_mode(dev);
9867	}
9868
9869	/* NOTE: order of synchronization of IFF_PROMISC and IFF_ALLMULTI
9870	* is important. Some (broken) drivers set IFF_PROMISC, when
9871	* IFF_ALLMULTI is requested not asking us and not reporting.
9872	*/
9873	if ((flags ^ dev->gflags) & IFF_ALLMULTI) {
9874	int inc = (flags & IFF_ALLMULTI) ? 1 : -1;
9875
9876	dev->gflags ^= IFF_ALLMULTI;
9877	netif_set_allmulti(dev, inc, false);
9878	}
9879
9880	return ret;
9881	}
9882
9883	void __dev_notify_flags(struct net_device *dev, unsigned int old_flags,
9884	unsigned int gchanges, u32 portid,
9885	const struct nlmsghdr *nlh)
9886	{
9887	unsigned int changes = dev->flags ^ old_flags;
9888
9889	netdev_assert_locked_ops_compat(dev);
9890
9891	if (gchanges)
9892	rtmsg_ifinfo(RTM_NEWLINK, dev, gchanges, GFP_ATOMIC, portid, nlh);
9893
9894	if (changes & IFF_UP) {
9895	if (dev->flags & IFF_UP)
9896	call_netdevice_notifiers(NETDEV_UP, dev);
9897	else
9898	call_netdevice_notifiers(NETDEV_DOWN, dev);
9899	}
9900
9901	if (dev->flags & IFF_UP &&
9902	(changes & ~(IFF_UP \| IFF_PROMISC \| IFF_ALLMULTI \| IFF_VOLATILE))) {
9903	struct netdev_notifier_change_info change_info = {
9904	.info = {
9905	.dev = dev,
9906	},
9907	.flags_changed = changes,
9908	};
9909
9910	call_netdevice_notifiers_info(NETDEV_CHANGE, &change_info.info);
9911	}
9912	}
9913
9914	int netif_change_flags(struct net_device *dev, unsigned int flags,
9915	struct netlink_ext_ack *extack)
9916	{
9917	int ret;
9918	unsigned int changes, old_flags = dev->flags, old_gflags = dev->gflags;
9919
9920	ret = __dev_change_flags(dev, flags, extack);
9921	if (ret < 0)
9922	return ret;
9923
9924	changes = (old_flags ^ dev->flags) \| (old_gflags ^ dev->gflags);
9925	__dev_notify_flags(dev, old_flags, changes, 0, NULL);
9926	return ret;
9927	}
9928	EXPORT_SYMBOL(netif_change_flags);
9929
9930	int __netif_set_mtu(struct net_device *dev, int new_mtu)
9931	{
9932	const struct net_device_ops *ops = dev->netdev_ops;
9933
9934	if (ops->ndo_change_mtu)
9935	return ops->ndo_change_mtu(dev, new_mtu);
9936
9937	/* Pairs with all the lockless reads of dev->mtu in the stack */
9938	WRITE_ONCE(dev->mtu, new_mtu);
9939	return 0;
9940	}
9941	EXPORT_SYMBOL_NS_GPL(__netif_set_mtu, "NETDEV_INTERNAL");
9942
9943	int dev_validate_mtu(struct net_device *dev, int new_mtu,
9944	struct netlink_ext_ack *extack)
9945	{
9946	/* MTU must be positive, and in range */
9947	if (new_mtu < 0 \|\| new_mtu < dev->min_mtu) {
9948	NL_SET_ERR_MSG(extack, "mtu less than device minimum");
9949	return -EINVAL;
9950	}
9951
9952	if (dev->max_mtu > 0 && new_mtu > dev->max_mtu) {
9953	NL_SET_ERR_MSG(extack, "mtu greater than device maximum");
9954	return -EINVAL;
9955	}
9956	return 0;
9957	}
9958
9959	/**
9960	* netif_set_mtu_ext() - Change maximum transfer unit
9961	* @dev: device
9962	* @new_mtu: new transfer unit
9963	* @extack: netlink extended ack
9964	*
9965	* Change the maximum transfer size of the network device.
9966	*
9967	* Return: 0 on success, -errno on failure.
9968	*/
9969	int netif_set_mtu_ext(struct net_device *dev, int new_mtu,
9970	struct netlink_ext_ack *extack)
9971	{
9972	int err, orig_mtu;
9973
9974	netdev_assert_locked_ops_compat(dev);
9975
9976	if (new_mtu == dev->mtu)
9977	return 0;
9978
9979	err = dev_validate_mtu(dev, new_mtu, extack);
9980	if (err)
9981	return err;
9982
9983	if (!netif_device_present(dev))
9984	return -ENODEV;
9985
9986	err = call_netdevice_notifiers(NETDEV_PRECHANGEMTU, dev);
9987	err = notifier_to_errno(err);
9988	if (err)
9989	return err;
9990
9991	orig_mtu = dev->mtu;
9992	err = __netif_set_mtu(dev, new_mtu);
9993
9994	if (!err) {
9995	err = call_netdevice_notifiers_mtu(NETDEV_CHANGEMTU, dev,
9996	orig_mtu);
9997	err = notifier_to_errno(err);
9998	if (err) {
9999	/* setting mtu back and notifying everyone again,
10000	* so that they have a chance to revert changes.
10001	*/
10002	__netif_set_mtu(dev, orig_mtu);
10003	call_netdevice_notifiers_mtu(NETDEV_CHANGEMTU, dev,
10004	new_mtu);
10005	}
10006	}
10007	return err;
10008	}
10009
10010	int netif_set_mtu(struct net_device *dev, int new_mtu)
10011	{
10012	struct netlink_ext_ack extack;
10013	int err;
10014
10015	memset(&extack, 0, sizeof(extack));
10016	err = netif_set_mtu_ext(dev, new_mtu, &extack);
10017	if (err && extack._msg)
10018	net_err_ratelimited("%s: %s\n", dev->name, extack._msg);
10019	return err;
10020	}
10021	EXPORT_SYMBOL(netif_set_mtu);
10022
10023	int netif_change_tx_queue_len(struct net_device *dev, unsigned long new_len)
10024	{
10025	unsigned int orig_len = dev->tx_queue_len;
10026	int res;
10027
10028	if (new_len > S16_MAX)
10029	return -ERANGE;
10030
10031	if (new_len != orig_len) {
10032	WRITE_ONCE(dev->tx_queue_len, new_len);
10033	res = call_netdevice_notifiers(NETDEV_CHANGE_TX_QUEUE_LEN, dev);
10034	res = notifier_to_errno(res);
10035	if (res)
10036	goto err_rollback;
10037	res = dev_qdisc_change_tx_queue_len(dev);
10038	if (res)
10039	goto err_rollback;
10040	}
10041
10042	return 0;
10043
10044	err_rollback:
10045	netdev_err(dev, "refused to change device tx_queue_len\n");
10046	WRITE_ONCE(dev->tx_queue_len, orig_len);
10047	return res;
10048	}
10049
10050	void netif_set_group(struct net_device *dev, int new_group)
10051	{
10052	dev->group = new_group;
10053	}
10054
10055	/**
10056	* netif_pre_changeaddr_notify() - Call NETDEV_PRE_CHANGEADDR.
10057	* @dev: device
10058	* @addr: new address
10059	* @extack: netlink extended ack
10060	*
10061	* Return: 0 on success, -errno on failure.
10062	*/
10063	int netif_pre_changeaddr_notify(struct net_device dev, const char addr,
10064	struct netlink_ext_ack *extack)
10065	{
10066	struct netdev_notifier_pre_changeaddr_info info = {
10067	.info.dev = dev,
10068	.info.extack = extack,
10069	.dev_addr = addr,
10070	};
10071	int rc;
10072
10073	rc = call_netdevice_notifiers_info(NETDEV_PRE_CHANGEADDR, &info.info);
10074	return notifier_to_errno(rc);
10075	}
10076	EXPORT_SYMBOL_NS_GPL(netif_pre_changeaddr_notify, "NETDEV_INTERNAL");
10077
10078	int netif_set_mac_address(struct net_device dev, struct sockaddr_storage ss,
10079	struct netlink_ext_ack *extack)
10080	{
10081	const struct net_device_ops *ops = dev->netdev_ops;
10082	int err;
10083
10084	if (!ops->ndo_set_mac_address)
10085	return -EOPNOTSUPP;
10086	if (ss->ss_family != dev->type)
10087	return -EINVAL;
10088	if (!netif_device_present(dev))
10089	return -ENODEV;
10090	err = netif_pre_changeaddr_notify(dev, ss->__data, extack);
10091	if (err)
10092	return err;
10093	if (memcmp(dev->dev_addr, ss->__data, dev->addr_len)) {
10094	err = ops->ndo_set_mac_address(dev, ss);
10095	if (err)
10096	return err;
10097	}
10098	dev->addr_assign_type = NET_ADDR_SET;
10099	call_netdevice_notifiers(NETDEV_CHANGEADDR, dev);
10100	add_device_randomness(dev->dev_addr, dev->addr_len);
10101	return 0;
10102	}
10103
10104	DECLARE_RWSEM(dev_addr_sem);
10105
10106	/* "sa" is a true struct sockaddr with limited "sa_data" member. */
10107	int netif_get_mac_address(struct sockaddr sa, struct net net, char *dev_name)
10108	{
10109	size_t size = sizeof(sa->sa_data);
10110	struct net_device *dev;
10111	int ret = 0;
10112
10113	down_read(&dev_addr_sem);
10114	rcu_read_lock();
10115
10116	dev = dev_get_by_name_rcu(net, dev_name);
10117	if (!dev) {
10118	ret = -ENODEV;
10119	goto unlock;
10120	}
10121	if (!dev->addr_len)
10122	memset(sa->sa_data, 0, size);
10123	else
10124	memcpy(sa->sa_data, dev->dev_addr,
10125	min_t(size_t, size, dev->addr_len));
10126	sa->sa_family = dev->type;
10127
10128	unlock:
10129	rcu_read_unlock();
10130	up_read(&dev_addr_sem);
10131	return ret;
10132	}
10133	EXPORT_SYMBOL_NS_GPL(netif_get_mac_address, "NETDEV_INTERNAL");
10134
10135	int netif_change_carrier(struct net_device *dev, bool new_carrier)
10136	{
10137	const struct net_device_ops *ops = dev->netdev_ops;
10138
10139	if (!ops->ndo_change_carrier)
10140	return -EOPNOTSUPP;
10141	if (!netif_device_present(dev))
10142	return -ENODEV;
10143	return ops->ndo_change_carrier(dev, new_carrier);
10144	}
10145
10146	/**
10147	* dev_get_phys_port_id - Get device physical port ID
10148	* @dev: device
10149	* @ppid: port ID
10150	*
10151	* Get device physical port ID
10152	*/
10153	int dev_get_phys_port_id(struct net_device *dev,
10154	struct netdev_phys_item_id *ppid)
10155	{
10156	const struct net_device_ops *ops = dev->netdev_ops;
10157
10158	if (!ops->ndo_get_phys_port_id)
10159	return -EOPNOTSUPP;
10160	return ops->ndo_get_phys_port_id(dev, ppid);
10161	}
10162
10163	/**
10164	* dev_get_phys_port_name - Get device physical port name
10165	* @dev: device
10166	* @name: port name
10167	* @len: limit of bytes to copy to name
10168	*
10169	* Get device physical port name
10170	*/
10171	int dev_get_phys_port_name(struct net_device *dev,
10172	char *name, size_t len)
10173	{
10174	const struct net_device_ops *ops = dev->netdev_ops;
10175	int err;
10176
10177	if (ops->ndo_get_phys_port_name) {
10178	err = ops->ndo_get_phys_port_name(dev, name, len);
10179	if (err != -EOPNOTSUPP)
10180	return err;
10181	}
10182	return devlink_compat_phys_port_name_get(dev, name, len);
10183	}
10184
10185	/**
10186	* netif_get_port_parent_id() - Get the device's port parent identifier
10187	* @dev: network device
10188	* @ppid: pointer to a storage for the port's parent identifier
10189	* @recurse: allow/disallow recursion to lower devices
10190	*
10191	* Get the devices's port parent identifier.
10192	*
10193	* Return: 0 on success, -errno on failure.
10194	*/
10195	int netif_get_port_parent_id(struct net_device *dev,
10196	struct netdev_phys_item_id *ppid, bool recurse)
10197	{
10198	const struct net_device_ops *ops = dev->netdev_ops;
10199	struct netdev_phys_item_id first = { };
10200	struct net_device *lower_dev;
10201	struct list_head *iter;
10202	int err;
10203
10204	if (ops->ndo_get_port_parent_id) {
10205	err = ops->ndo_get_port_parent_id(dev, ppid);
10206	if (err != -EOPNOTSUPP)
10207	return err;
10208	}
10209
10210	err = devlink_compat_switch_id_get(dev, ppid);
10211	if (!recurse \|\| err != -EOPNOTSUPP)
10212	return err;
10213
10214	netdev_for_each_lower_dev(dev, lower_dev, iter) {
10215	err = netif_get_port_parent_id(lower_dev, ppid, true);
10216	if (err)
10217	break;
10218	if (!first.id_len)
10219	first = *ppid;
10220	else if (memcmp(&first, ppid, sizeof(*ppid)))
10221	return -EOPNOTSUPP;
10222	}
10223
10224	return err;
10225	}
10226	EXPORT_SYMBOL(netif_get_port_parent_id);
10227
10228	/**
10229	* netdev_port_same_parent_id - Indicate if two network devices have
10230	* the same port parent identifier
10231	* @a: first network device
10232	* @b: second network device
10233	*/
10234	bool netdev_port_same_parent_id(struct net_device a, struct net_device b)
10235	{
10236	struct netdev_phys_item_id a_id = { };
10237	struct netdev_phys_item_id b_id = { };
10238
10239	if (netif_get_port_parent_id(a, &a_id, true) \|\|
10240	netif_get_port_parent_id(b, &b_id, true))
10241	return false;
10242
10243	return netdev_phys_item_id_same(&a_id, &b_id);
10244	}
10245	EXPORT_SYMBOL(netdev_port_same_parent_id);
10246
10247	static struct net_device dev_get_iflink_dev(struct net_device dev)
10248	{
10249	struct net *net;
10250
10251	ASSERT_RTNL();
10252
10253	if (!dev->netdev_ops->ndo_get_iflink \|\| !dev->rtnl_link_ops \|\|
10254	!dev->rtnl_link_ops->get_link_net)
10255	return dev;
10256
10257	net = dev->rtnl_link_ops->get_link_net(dev);
10258	return __dev_get_by_index(net, dev_get_iflink(dev));
10259	}
10260
10261	int netif_change_proto_down(struct net_device *dev, bool proto_down)
10262	{
10263	struct net_device *iflink_dev;
10264
10265	if (!dev->change_proto_down)
10266	return -EOPNOTSUPP;
10267	if (!netif_device_present(dev))
10268	return -ENODEV;
10269	iflink_dev = dev_get_iflink_dev(dev);
10270	if (!iflink_dev)
10271	return -ENODEV;
10272	WRITE_ONCE(dev->proto_down, proto_down);
10273	if (proto_down)
10274	netif_carrier_off(dev);
10275	else if (dev == iflink_dev \|\| netif_carrier_ok(iflink_dev))
10276	netif_carrier_on(dev);
10277	return 0;
10278	}
10279
10280	/**
10281	* netdev_change_proto_down_reason_locked - proto down reason
10282	*
10283	* @dev: device
10284	* @mask: proto down mask
10285	* @value: proto down value
10286	*/
10287	void netdev_change_proto_down_reason_locked(struct net_device *dev,
10288	unsigned long mask, u32 value)
10289	{
10290	u32 proto_down_reason;
10291	int b;
10292
10293	if (!mask) {
10294	proto_down_reason = value;
10295	} else {
10296	proto_down_reason = dev->proto_down_reason;
10297	for_each_set_bit(b, &mask, 32) {
10298	if (value & (1 << b))
10299	proto_down_reason \|= BIT(b);
10300	else
10301	proto_down_reason &= ~BIT(b);
10302	}
10303	}
10304	WRITE_ONCE(dev->proto_down_reason, proto_down_reason);
10305	}
10306
10307	struct bpf_xdp_link {
10308	struct bpf_link link;
10309	struct net_device dev; / protected by rtnl_lock, no refcnt held */
10310	int flags;
10311	};
10312
10313	static enum bpf_xdp_mode dev_xdp_mode(struct net_device *dev, u32 flags)
10314	{
10315	if (flags & XDP_FLAGS_HW_MODE)
10316	return XDP_MODE_HW;
10317	if (flags & XDP_FLAGS_DRV_MODE)
10318	return XDP_MODE_DRV;
10319	if (flags & XDP_FLAGS_SKB_MODE)
10320	return XDP_MODE_SKB;
10321	return dev->netdev_ops->ndo_bpf ? XDP_MODE_DRV : XDP_MODE_SKB;
10322	}
10323
10324	static bpf_op_t dev_xdp_bpf_op(struct net_device *dev, enum bpf_xdp_mode mode)
10325	{
10326	switch (mode) {
10327	case XDP_MODE_SKB:
10328	return generic_xdp_install;
10329	case XDP_MODE_DRV:
10330	case XDP_MODE_HW:
10331	return dev->netdev_ops->ndo_bpf;
10332	default:
10333	return NULL;
10334	}
10335	}
10336
10337	static struct bpf_xdp_link dev_xdp_link(struct net_device dev,
10338	enum bpf_xdp_mode mode)
10339	{
10340	return dev->xdp_state[mode].link;
10341	}
10342
10343	static struct bpf_prog dev_xdp_prog(struct net_device dev,
10344	enum bpf_xdp_mode mode)
10345	{
10346	struct bpf_xdp_link *link = dev_xdp_link(dev, mode);
10347
10348	if (link)
10349	return link->link.prog;
10350	return dev->xdp_state[mode].prog;
10351	}
10352
10353	u8 dev_xdp_prog_count(struct net_device *dev)
10354	{
10355	u8 count = 0;
10356	int i;
10357
10358	for (i = 0; i < __MAX_XDP_MODE; i++)
10359	if (dev->xdp_state[i].prog \|\| dev->xdp_state[i].link)
10360	count++;
10361	return count;
10362	}
10363	EXPORT_SYMBOL_GPL(dev_xdp_prog_count);
10364
10365	u8 dev_xdp_sb_prog_count(struct net_device *dev)
10366	{
10367	struct bpf_prog *prog;
10368	u8 count = 0;
10369	int i;
10370
10371	for (i = 0; i < __MAX_XDP_MODE; i++) {
10372	prog = dev_xdp_prog(dev, i);
10373	if (prog && !prog->aux->xdp_has_frags)
10374	count++;
10375	}
10376	return count;
10377	}
10378
10379	int netif_xdp_propagate(struct net_device dev, struct netdev_bpf bpf)
10380	{
10381	if (!dev->netdev_ops->ndo_bpf)
10382	return -EOPNOTSUPP;
10383
10384	if (dev->cfg->hds_config == ETHTOOL_TCP_DATA_SPLIT_ENABLED &&
10385	bpf->command == XDP_SETUP_PROG &&
10386	bpf->prog && !bpf->prog->aux->xdp_has_frags) {
10387	NL_SET_ERR_MSG(bpf->extack,
10388	"unable to propagate XDP to device using tcp-data-split");
10389	return -EBUSY;
10390	}
10391
10392	if (dev_get_min_mp_channel_count(dev)) {
10393	NL_SET_ERR_MSG(bpf->extack, "unable to propagate XDP to device using memory provider");
10394	return -EBUSY;
10395	}
10396
10397	return dev->netdev_ops->ndo_bpf(dev, bpf);
10398	}
10399	EXPORT_SYMBOL_GPL(netif_xdp_propagate);
10400
10401	u32 dev_xdp_prog_id(struct net_device *dev, enum bpf_xdp_mode mode)
10402	{
10403	struct bpf_prog *prog = dev_xdp_prog(dev, mode);
10404
10405	return prog ? prog->aux->id : 0;
10406	}
10407
10408	static void dev_xdp_set_link(struct net_device *dev, enum bpf_xdp_mode mode,
10409	struct bpf_xdp_link *link)
10410	{
10411	dev->xdp_state[mode].link = link;
10412	dev->xdp_state[mode].prog = NULL;
10413	}
10414
10415	static void dev_xdp_set_prog(struct net_device *dev, enum bpf_xdp_mode mode,
10416	struct bpf_prog *prog)
10417	{
10418	dev->xdp_state[mode].link = NULL;
10419	dev->xdp_state[mode].prog = prog;
10420	}
10421
10422	static int dev_xdp_install(struct net_device *dev, enum bpf_xdp_mode mode,
10423	bpf_op_t bpf_op, struct netlink_ext_ack *extack,
10424	u32 flags, struct bpf_prog *prog)
10425	{
10426	struct netdev_bpf xdp;
10427	int err;
10428
10429	netdev_assert_locked_ops_compat(dev);
10430
10431	if (prog) {
10432	enum bpf_xdp_mode other_mode = mode == XDP_MODE_SKB
10433	? XDP_MODE_DRV : XDP_MODE_SKB;
10434	bool offload = mode == XDP_MODE_HW;
10435
10436	if (!offload && dev_xdp_prog(dev, other_mode)) {
10437	NL_SET_ERR_MSG(extack, "Native and generic XDP can't be active at the same time");
10438	return -EEXIST;
10439	}
10440	if (!offload && bpf_prog_is_offloaded(prog->aux)) {
10441	NL_SET_ERR_MSG(extack, "Using offloaded program without HW_MODE flag is not supported");
10442	return -EINVAL;
10443	}
10444	if (bpf_prog_is_dev_bound(prog->aux) && !bpf_offload_dev_match(prog, dev)) {
10445	NL_SET_ERR_MSG(extack, "Program bound to different device");
10446	return -EINVAL;
10447	}
10448	if (bpf_prog_is_dev_bound(prog->aux) && mode == XDP_MODE_SKB) {
10449	NL_SET_ERR_MSG(extack, "Can't attach device-bound programs in generic mode");
10450	return -EINVAL;
10451	}
10452	if (prog->expected_attach_type == BPF_XDP_DEVMAP) {
10453	NL_SET_ERR_MSG(extack, "BPF_XDP_DEVMAP programs can not be attached to a device");
10454	return -EINVAL;
10455	}
10456	if (prog->expected_attach_type == BPF_XDP_CPUMAP) {
10457	NL_SET_ERR_MSG(extack, "BPF_XDP_CPUMAP programs can not be attached to a device");
10458	return -EINVAL;
10459	}
10460	}
10461
10462	if (dev->cfg->hds_config == ETHTOOL_TCP_DATA_SPLIT_ENABLED &&
10463	prog && !prog->aux->xdp_has_frags) {
10464	NL_SET_ERR_MSG(extack, "unable to install XDP to device using tcp-data-split");
10465	return -EBUSY;
10466	}
10467
10468	if (dev_get_min_mp_channel_count(dev)) {
10469	NL_SET_ERR_MSG(extack, "unable to install XDP to device using memory provider");
10470	return -EBUSY;
10471	}
10472
10473	memset(&xdp, 0, sizeof(xdp));
10474	xdp.command = mode == XDP_MODE_HW ? XDP_SETUP_PROG_HW : XDP_SETUP_PROG;
10475	xdp.extack = extack;
10476	xdp.flags = flags;
10477	xdp.prog = prog;
10478
10479	/* Drivers assume refcnt is already incremented (i.e, prog pointer is
10480	* "moved" into driver), so they don't increment it on their own, but
10481	* they do decrement refcnt when program is detached or replaced.
10482	* Given net_device also owns link/prog, we need to bump refcnt here
10483	* to prevent drivers from underflowing it.
10484	*/
10485	if (prog)
10486	bpf_prog_inc(prog);
10487	err = bpf_op(dev, &xdp);
10488	if (err) {
10489	if (prog)
10490	bpf_prog_put(prog);
10491	return err;
10492	}
10493
10494	if (mode != XDP_MODE_HW)
10495	bpf_prog_change_xdp(dev_xdp_prog(dev, mode), prog);
10496
10497	return 0;
10498	}
10499
10500	static void dev_xdp_uninstall(struct net_device *dev)
10501	{
10502	struct bpf_xdp_link *link;
10503	struct bpf_prog *prog;
10504	enum bpf_xdp_mode mode;
10505	bpf_op_t bpf_op;
10506
10507	ASSERT_RTNL();
10508
10509	for (mode = XDP_MODE_SKB; mode < __MAX_XDP_MODE; mode++) {
10510	prog = dev_xdp_prog(dev, mode);
10511	if (!prog)
10512	continue;
10513
10514	bpf_op = dev_xdp_bpf_op(dev, mode);
10515	if (!bpf_op)
10516	continue;
10517
10518	WARN_ON(dev_xdp_install(dev, mode, bpf_op, NULL, 0, NULL));
10519
10520	/* auto-detach link from net device */
10521	link = dev_xdp_link(dev, mode);
10522	if (link)
10523	link->dev = NULL;
10524	else
10525	bpf_prog_put(prog);
10526
10527	dev_xdp_set_link(dev, mode, NULL);
10528	}
10529	}
10530
10531	static int dev_xdp_attach(struct net_device dev, struct netlink_ext_ack extack,
10532	struct bpf_xdp_link link, struct bpf_prog new_prog,
10533	struct bpf_prog *old_prog, u32 flags)
10534	{
10535	unsigned int num_modes = hweight32(flags & XDP_FLAGS_MODES);
10536	struct bpf_prog *cur_prog;
10537	struct net_device *upper;
10538	struct list_head *iter;
10539	enum bpf_xdp_mode mode;
10540	bpf_op_t bpf_op;
10541	int err;
10542
10543	ASSERT_RTNL();
10544
10545	/* either link or prog attachment, never both */
10546	if (link && (new_prog \|\| old_prog))
10547	return -EINVAL;
10548	/* link supports only XDP mode flags */
10549	if (link && (flags & ~XDP_FLAGS_MODES)) {
10550	NL_SET_ERR_MSG(extack, "Invalid XDP flags for BPF link attachment");
10551	return -EINVAL;
10552	}
10553	/* just one XDP mode bit should be set, zero defaults to drv/skb mode */
10554	if (num_modes > 1) {
10555	NL_SET_ERR_MSG(extack, "Only one XDP mode flag can be set");
10556	return -EINVAL;
10557	}
10558	/* avoid ambiguity if offload + drv/skb mode progs are both loaded */
10559	if (!num_modes && dev_xdp_prog_count(dev) > 1) {
10560	NL_SET_ERR_MSG(extack,
10561	"More than one program loaded, unset mode is ambiguous");
10562	return -EINVAL;
10563	}
10564	/* old_prog != NULL implies XDP_FLAGS_REPLACE is set */
10565	if (old_prog && !(flags & XDP_FLAGS_REPLACE)) {
10566	NL_SET_ERR_MSG(extack, "XDP_FLAGS_REPLACE is not specified");
10567	return -EINVAL;
10568	}
10569
10570	mode = dev_xdp_mode(dev, flags);
10571	/* can't replace attached link */
10572	if (dev_xdp_link(dev, mode)) {
10573	NL_SET_ERR_MSG(extack, "Can't replace active BPF XDP link");
10574	return -EBUSY;
10575	}
10576
10577	/* don't allow if an upper device already has a program */
10578	netdev_for_each_upper_dev_rcu(dev, upper, iter) {
10579	if (dev_xdp_prog_count(upper) > 0) {
10580	NL_SET_ERR_MSG(extack, "Cannot attach when an upper device already has a program");
10581	return -EEXIST;
10582	}
10583	}
10584
10585	cur_prog = dev_xdp_prog(dev, mode);
10586	/* can't replace attached prog with link */
10587	if (link && cur_prog) {
10588	NL_SET_ERR_MSG(extack, "Can't replace active XDP program with BPF link");
10589	return -EBUSY;
10590	}
10591	if ((flags & XDP_FLAGS_REPLACE) && cur_prog != old_prog) {
10592	NL_SET_ERR_MSG(extack, "Active program does not match expected");
10593	return -EEXIST;
10594	}
10595
10596	/* put effective new program into new_prog */
10597	if (link)
10598	new_prog = link->link.prog;
10599
10600	if (new_prog) {
10601	if ((flags & XDP_FLAGS_UPDATE_IF_NOEXIST) && cur_prog) {
10602	NL_SET_ERR_MSG(extack, "XDP program already attached");
10603	return -EBUSY;
10604	}
10605	}
10606
10607	/* don't call drivers if the effective program didn't change */
10608	if (new_prog != cur_prog) {
10609	bpf_op = dev_xdp_bpf_op(dev, mode);
10610	if (!bpf_op) {
10611	NL_SET_ERR_MSG(extack, "Underlying driver does not support XDP in native mode");
10612	return -EOPNOTSUPP;
10613	}
10614
10615	err = dev_xdp_install(dev, mode, bpf_op, extack, flags, new_prog);
10616	if (err)
10617	return err;
10618	}
10619
10620	if (link)
10621	dev_xdp_set_link(dev, mode, link);
10622	else
10623	dev_xdp_set_prog(dev, mode, new_prog);
10624	if (cur_prog)
10625	bpf_prog_put(cur_prog);
10626
10627	return 0;
10628	}
10629
10630	static int dev_xdp_attach_link(struct net_device *dev,
10631	struct netlink_ext_ack *extack,
10632	struct bpf_xdp_link *link)
10633	{
10634	return dev_xdp_attach(dev, extack, link, NULL, NULL, link->flags);
10635	}
10636
10637	static int dev_xdp_detach_link(struct net_device *dev,
10638	struct netlink_ext_ack *extack,
10639	struct bpf_xdp_link *link)
10640	{
10641	enum bpf_xdp_mode mode;
10642	bpf_op_t bpf_op;
10643
10644	ASSERT_RTNL();
10645
10646	mode = dev_xdp_mode(dev, link->flags);
10647	if (dev_xdp_link(dev, mode) != link)
10648	return -EINVAL;
10649
10650	bpf_op = dev_xdp_bpf_op(dev, mode);
10651	WARN_ON(dev_xdp_install(dev, mode, bpf_op, NULL, 0, NULL));
10652	dev_xdp_set_link(dev, mode, NULL);
10653	return 0;
10654	}
10655
10656	static void bpf_xdp_link_release(struct bpf_link *link)
10657	{
10658	struct bpf_xdp_link *xdp_link = container_of(link, struct bpf_xdp_link, link);
10659
10660	rtnl_lock();
10661
10662	/* if racing with net_device's tear down, xdp_link->dev might be
10663	* already NULL, in which case link was already auto-detached
10664	*/
10665	if (xdp_link->dev) {
10666	netdev_lock_ops(xdp_link->dev);
10667	WARN_ON(dev_xdp_detach_link(xdp_link->dev, NULL, xdp_link));
10668	netdev_unlock_ops(xdp_link->dev);
10669	xdp_link->dev = NULL;
10670	}
10671
10672	rtnl_unlock();
10673	}
10674
10675	static int bpf_xdp_link_detach(struct bpf_link *link)
10676	{
10677	bpf_xdp_link_release(link);
10678	return 0;
10679	}
10680
10681	static void bpf_xdp_link_dealloc(struct bpf_link *link)
10682	{
10683	struct bpf_xdp_link *xdp_link = container_of(link, struct bpf_xdp_link, link);
10684
10685	kfree(xdp_link);
10686	}
10687
10688	static void bpf_xdp_link_show_fdinfo(const struct bpf_link *link,
10689	struct seq_file *seq)
10690	{
10691	struct bpf_xdp_link *xdp_link = container_of(link, struct bpf_xdp_link, link);
10692	u32 ifindex = 0;
10693
10694	rtnl_lock();
10695	if (xdp_link->dev)
10696	ifindex = xdp_link->dev->ifindex;
10697	rtnl_unlock();
10698
10699	seq_printf(seq, "ifindex:\t%u\n", ifindex);
10700	}
10701
10702	static int bpf_xdp_link_fill_link_info(const struct bpf_link *link,
10703	struct bpf_link_info *info)
10704	{
10705	struct bpf_xdp_link *xdp_link = container_of(link, struct bpf_xdp_link, link);
10706	u32 ifindex = 0;
10707
10708	rtnl_lock();
10709	if (xdp_link->dev)
10710	ifindex = xdp_link->dev->ifindex;
10711	rtnl_unlock();
10712
10713	info->xdp.ifindex = ifindex;
10714	return 0;
10715	}
10716
10717	static int bpf_xdp_link_update(struct bpf_link link, struct bpf_prog new_prog,
10718	struct bpf_prog *old_prog)
10719	{
10720	struct bpf_xdp_link *xdp_link = container_of(link, struct bpf_xdp_link, link);
10721	enum bpf_xdp_mode mode;
10722	bpf_op_t bpf_op;
10723	int err = 0;
10724
10725	rtnl_lock();
10726
10727	/* link might have been auto-released already, so fail */
10728	if (!xdp_link->dev) {
10729	err = -ENOLINK;
10730	goto out_unlock;
10731	}
10732
10733	if (old_prog && link->prog != old_prog) {
10734	err = -EPERM;
10735	goto out_unlock;
10736	}
10737	old_prog = link->prog;
10738	if (old_prog->type != new_prog->type \|\|
10739	old_prog->expected_attach_type != new_prog->expected_attach_type) {
10740	err = -EINVAL;
10741	goto out_unlock;
10742	}
10743
10744	if (old_prog == new_prog) {
10745	/* no-op, don't disturb drivers */
10746	bpf_prog_put(new_prog);
10747	goto out_unlock;
10748	}
10749
10750	netdev_lock_ops(xdp_link->dev);
10751	mode = dev_xdp_mode(xdp_link->dev, xdp_link->flags);
10752	bpf_op = dev_xdp_bpf_op(xdp_link->dev, mode);
10753	err = dev_xdp_install(xdp_link->dev, mode, bpf_op, NULL,
10754	xdp_link->flags, new_prog);
10755	if (!err)
10756	old_prog = xchg(&link->prog, new_prog);
10757	netdev_unlock_ops(xdp_link->dev);
10758	if (err)
10759	goto out_unlock;
10760
10761	bpf_prog_put(old_prog);
10762
10763	out_unlock:
10764	rtnl_unlock();
10765	return err;
10766	}
10767
10768	static const struct bpf_link_ops bpf_xdp_link_lops = {
10769	.release = bpf_xdp_link_release,
10770	.dealloc = bpf_xdp_link_dealloc,
10771	.detach = bpf_xdp_link_detach,
10772	.show_fdinfo = bpf_xdp_link_show_fdinfo,
10773	.fill_link_info = bpf_xdp_link_fill_link_info,
10774	.update_prog = bpf_xdp_link_update,
10775	};
10776
10777	int bpf_xdp_link_attach(const union bpf_attr attr, struct bpf_prog prog)
10778	{
10779	struct net *net = current->nsproxy->net_ns;
10780	struct bpf_link_primer link_primer;
10781	struct netlink_ext_ack extack = {};
10782	struct bpf_xdp_link *link;
10783	struct net_device *dev;
10784	int err, fd;
10785
10786	rtnl_lock();
10787	dev = dev_get_by_index(net, attr->link_create.target_ifindex);
10788	if (!dev) {
10789	rtnl_unlock();
10790	return -EINVAL;
10791	}
10792
10793	link = kzalloc_obj(*link, GFP_USER);
10794	if (!link) {
10795	err = -ENOMEM;
10796	goto unlock;
10797	}
10798
10799	bpf_link_init(&link->link, BPF_LINK_TYPE_XDP, &bpf_xdp_link_lops, prog,
10800	attr->link_create.attach_type);
10801	link->dev = dev;
10802	link->flags = attr->link_create.flags;
10803
10804	err = bpf_link_prime(&link->link, &link_primer);
10805	if (err) {
10806	kfree(link);
10807	goto unlock;
10808	}
10809
10810	netdev_lock_ops(dev);
10811	err = dev_xdp_attach_link(dev, &extack, link);
10812	netdev_unlock_ops(dev);
10813	rtnl_unlock();
10814
10815	if (err) {
10816	link->dev = NULL;
10817	bpf_link_cleanup(&link_primer);
10818	trace_bpf_xdp_link_attach_failed(extack._msg);
10819	goto out_put_dev;
10820	}
10821
10822	fd = bpf_link_settle(&link_primer);
10823	/* link itself doesn't hold dev's refcnt to not complicate shutdown */
10824	dev_put(dev);
10825	return fd;
10826
10827	unlock:
10828	rtnl_unlock();
10829
10830	out_put_dev:
10831	dev_put(dev);
10832	return err;
10833	}
10834
10835	/**
10836	* dev_change_xdp_fd - set or clear a bpf program for a device rx path
10837	* @dev: device
10838	* @extack: netlink extended ack
10839	* @fd: new program fd or negative value to clear
10840	* @expected_fd: old program fd that userspace expects to replace or clear
10841	* @flags: xdp-related flags
10842	*
10843	* Set or clear a bpf program for a device
10844	*/
10845	int dev_change_xdp_fd(struct net_device dev, struct netlink_ext_ack extack,
10846	int fd, int expected_fd, u32 flags)
10847	{
10848	enum bpf_xdp_mode mode = dev_xdp_mode(dev, flags);
10849	struct bpf_prog new_prog = NULL, old_prog = NULL;
10850	int err;
10851
10852	ASSERT_RTNL();
10853
10854	if (fd >= 0) {
10855	new_prog = bpf_prog_get_type_dev(fd, BPF_PROG_TYPE_XDP,
10856	mode != XDP_MODE_SKB);
10857	if (IS_ERR(new_prog))
10858	return PTR_ERR(new_prog);
10859	}
10860
10861	if (expected_fd >= 0) {
10862	old_prog = bpf_prog_get_type_dev(expected_fd, BPF_PROG_TYPE_XDP,
10863	mode != XDP_MODE_SKB);
10864	if (IS_ERR(old_prog)) {
10865	err = PTR_ERR(old_prog);
10866	old_prog = NULL;
10867	goto err_out;
10868	}
10869	}
10870
10871	err = dev_xdp_attach(dev, extack, NULL, new_prog, old_prog, flags);
10872
10873	err_out:
10874	if (err && new_prog)
10875	bpf_prog_put(new_prog);
10876	if (old_prog)
10877	bpf_prog_put(old_prog);
10878	return err;
10879	}
10880
10881	u32 dev_get_min_mp_channel_count(const struct net_device *dev)
10882	{
10883	int i;
10884
10885	netdev_assert_locked_ops_compat(dev);
10886
10887	for (i = dev->real_num_rx_queues - 1; i >= 0; i--)
10888	if (dev->_rx[i].mp_params.mp_priv)
10889	/* The channel count is the idx plus 1. */
10890	return i + 1;
10891
10892	return 0;
10893	}
10894
10895	/**
10896	* dev_index_reserve() - allocate an ifindex in a namespace
10897	* @net: the applicable net namespace
10898	* @ifindex: requested ifindex, pass %0 to get one allocated
10899	*
10900	* Allocate a ifindex for a new device. Caller must either use the ifindex
10901	* to store the device (via list_netdevice()) or call dev_index_release()
10902	* to give the index up.
10903	*
10904	* Return: a suitable unique value for a new device interface number or -errno.
10905	*/
10906	static int dev_index_reserve(struct net *net, u32 ifindex)
10907	{
10908	int err;
10909
10910	if (ifindex > INT_MAX) {
10911	DEBUG_NET_WARN_ON_ONCE(1);
10912	return -EINVAL;
10913	}
10914
10915	if (!ifindex)
10916	err = xa_alloc_cyclic(&net->dev_by_index, &ifindex, NULL,
10917	xa_limit_31b, &net->ifindex, GFP_KERNEL);
10918	else
10919	err = xa_insert(&net->dev_by_index, ifindex, NULL, GFP_KERNEL);
10920	if (err < 0)
10921	return err;
10922
10923	return ifindex;
10924	}
10925
10926	static void dev_index_release(struct net *net, int ifindex)
10927	{
10928	/* Expect only unused indexes, unlist_netdevice() removes the used */
10929	WARN_ON(xa_erase(&net->dev_by_index, ifindex));
10930	}
10931
10932	static bool from_cleanup_net(void)
10933	{
10934	#ifdef CONFIG_NET_NS
10935	return current == READ_ONCE(cleanup_net_task);
10936	#else
10937	return false;
10938	#endif
10939	}
10940
10941	/* Delayed registration/unregisteration */
10942	LIST_HEAD(net_todo_list);
10943	DECLARE_WAIT_QUEUE_HEAD(netdev_unregistering_wq);
10944	atomic_t dev_unreg_count = ATOMIC_INIT(0);
10945
10946	static void net_set_todo(struct net_device *dev)
10947	{
10948	list_add_tail(&dev->todo_list, &net_todo_list);
10949	}
10950
10951	static netdev_features_t netdev_sync_upper_features(struct net_device *lower,
10952	struct net_device *upper, netdev_features_t features)
10953	{
10954	netdev_features_t upper_disables = NETIF_F_UPPER_DISABLES;
10955	netdev_features_t feature;
10956	int feature_bit;
10957
10958	for_each_netdev_feature(upper_disables, feature_bit) {
10959	feature = __NETIF_F_BIT(feature_bit);
10960	if (!(upper->wanted_features & feature)
10961	&& (features & feature)) {
10962	netdev_dbg(lower, "Dropping feature %pNF, upper dev %s has it off.\n",
10963	&feature, upper->name);
10964	features &= ~feature;
10965	}
10966	}
10967
10968	return features;
10969	}
10970
10971	static void netdev_sync_lower_features(struct net_device *upper,
10972	struct net_device *lower, netdev_features_t features)
10973	{
10974	netdev_features_t upper_disables = NETIF_F_UPPER_DISABLES;
10975	netdev_features_t feature;
10976	int feature_bit;
10977
10978	for_each_netdev_feature(upper_disables, feature_bit) {
10979	feature = __NETIF_F_BIT(feature_bit);
10980	if (!(features & feature) && (lower->features & feature)) {
10981	netdev_dbg(upper, "Disabling feature %pNF on lower dev %s.\n",
10982	&feature, lower->name);
10983	netdev_lock_ops(lower);
10984	lower->wanted_features &= ~feature;
10985	__netdev_update_features(lower);
10986
10987	if (unlikely(lower->features & feature))
10988	netdev_WARN(upper, "failed to disable %pNF on %s!\n",
10989	&feature, lower->name);
10990	else
10991	netdev_features_change(lower);
10992	netdev_unlock_ops(lower);
10993	}
10994	}
10995	}
10996
10997	static bool netdev_has_ip_or_hw_csum(netdev_features_t features)
10998	{
10999	netdev_features_t ip_csum_mask = NETIF_F_IP_CSUM \| NETIF_F_IPV6_CSUM;
11000	bool ip_csum = (features & ip_csum_mask) == ip_csum_mask;
11001	bool hw_csum = features & NETIF_F_HW_CSUM;
11002
11003	return ip_csum \|\| hw_csum;
11004	}
11005
11006	static netdev_features_t netdev_fix_features(struct net_device *dev,
11007	netdev_features_t features)
11008	{
11009	/* Fix illegal checksum combinations */
11010	if ((features & NETIF_F_HW_CSUM) &&
11011	(features & (NETIF_F_IP_CSUM\|NETIF_F_IPV6_CSUM))) {
11012	netdev_warn(dev, "mixed HW and IP checksum settings.\n");
11013	features &= ~(NETIF_F_IP_CSUM\|NETIF_F_IPV6_CSUM);
11014	}
11015
11016	/* TSO requires that SG is present as well. */
11017	if ((features & NETIF_F_ALL_TSO) && !(features & NETIF_F_SG)) {
11018	netdev_dbg(dev, "Dropping TSO features since no SG feature.\n");
11019	features &= ~NETIF_F_ALL_TSO;
11020	}
11021
11022	if ((features & NETIF_F_TSO) && !(features & NETIF_F_HW_CSUM) &&
11023	!(features & NETIF_F_IP_CSUM)) {
11024	netdev_dbg(dev, "Dropping TSO features since no CSUM feature.\n");
11025	features &= ~NETIF_F_TSO;
11026	features &= ~NETIF_F_TSO_ECN;
11027	}
11028
11029	if ((features & NETIF_F_TSO6) && !(features & NETIF_F_HW_CSUM) &&
11030	!(features & NETIF_F_IPV6_CSUM)) {
11031	netdev_dbg(dev, "Dropping TSO6 features since no CSUM feature.\n");
11032	features &= ~NETIF_F_TSO6;
11033	}
11034
11035	/* TSO with IPv4 ID mangling requires IPv4 TSO be enabled */
11036	if ((features & NETIF_F_TSO_MANGLEID) && !(features & NETIF_F_TSO))
11037	features &= ~NETIF_F_TSO_MANGLEID;
11038
11039	/* TSO ECN requires that TSO is present as well. */
11040	if ((features & NETIF_F_ALL_TSO) == NETIF_F_TSO_ECN)
11041	features &= ~NETIF_F_TSO_ECN;
11042
11043	/* Software GSO depends on SG. */
11044	if ((features & NETIF_F_GSO) && !(features & NETIF_F_SG)) {
11045	netdev_dbg(dev, "Dropping NETIF_F_GSO since no SG feature.\n");
11046	features &= ~NETIF_F_GSO;
11047	}
11048
11049	/* GSO partial features require GSO partial be set */
11050	if ((features & dev->gso_partial_features) &&
11051	!(features & NETIF_F_GSO_PARTIAL)) {
11052	netdev_dbg(dev,
11053	"Dropping partially supported GSO features since no GSO partial.\n");
11054	features &= ~dev->gso_partial_features;
11055	}
11056
11057	if (!(features & NETIF_F_RXCSUM)) {
11058	/* NETIF_F_GRO_HW implies doing RXCSUM since every packet
11059	* successfully merged by hardware must also have the
11060	* checksum verified by hardware. If the user does not
11061	* want to enable RXCSUM, logically, we should disable GRO_HW.
11062	*/
11063	if (features & NETIF_F_GRO_HW) {
11064	netdev_dbg(dev, "Dropping NETIF_F_GRO_HW since no RXCSUM feature.\n");
11065	features &= ~NETIF_F_GRO_HW;
11066	}
11067	}
11068
11069	/* LRO/HW-GRO features cannot be combined with RX-FCS */
11070	if (features & NETIF_F_RXFCS) {
11071	if (features & NETIF_F_LRO) {
11072	netdev_dbg(dev, "Dropping LRO feature since RX-FCS is requested.\n");
11073	features &= ~NETIF_F_LRO;
11074	}
11075
11076	if (features & NETIF_F_GRO_HW) {
11077	netdev_dbg(dev, "Dropping HW-GRO feature since RX-FCS is requested.\n");
11078	features &= ~NETIF_F_GRO_HW;
11079	}
11080	}
11081
11082	if ((features & NETIF_F_GRO_HW) && (features & NETIF_F_LRO)) {
11083	netdev_dbg(dev, "Dropping LRO feature since HW-GRO is requested.\n");
11084	features &= ~NETIF_F_LRO;
11085	}
11086
11087	if ((features & NETIF_F_HW_TLS_TX) && !netdev_has_ip_or_hw_csum(features)) {
11088	netdev_dbg(dev, "Dropping TLS TX HW offload feature since no CSUM feature.\n");
11089	features &= ~NETIF_F_HW_TLS_TX;
11090	}
11091
11092	if ((features & NETIF_F_HW_TLS_RX) && !(features & NETIF_F_RXCSUM)) {
11093	netdev_dbg(dev, "Dropping TLS RX HW offload feature since no RXCSUM feature.\n");
11094	features &= ~NETIF_F_HW_TLS_RX;
11095	}
11096
11097	if ((features & NETIF_F_GSO_UDP_L4) && !netdev_has_ip_or_hw_csum(features)) {
11098	netdev_dbg(dev, "Dropping USO feature since no CSUM feature.\n");
11099	features &= ~NETIF_F_GSO_UDP_L4;
11100	}
11101
11102	return features;
11103	}
11104
11105	int __netdev_update_features(struct net_device *dev)
11106	{
11107	struct net_device upper, lower;
11108	netdev_features_t features;
11109	struct list_head *iter;
11110	int err = -1;
11111
11112	ASSERT_RTNL();
11113	netdev_assert_locked_ops_compat(dev);
11114
11115	features = netdev_get_wanted_features(dev);
11116
11117	if (dev->netdev_ops->ndo_fix_features)
11118	features = dev->netdev_ops->ndo_fix_features(dev, features);
11119
11120	/* driver might be less strict about feature dependencies */
11121	features = netdev_fix_features(dev, features);
11122
11123	/* some features can't be enabled if they're off on an upper device */
11124	netdev_for_each_upper_dev_rcu(dev, upper, iter)
11125	features = netdev_sync_upper_features(dev, upper, features);
11126
11127	if (dev->features == features)
11128	goto sync_lower;
11129
11130	netdev_dbg(dev, "Features changed: %pNF -> %pNF\n",
11131	&dev->features, &features);
11132
11133	if (dev->netdev_ops->ndo_set_features)
11134	err = dev->netdev_ops->ndo_set_features(dev, features);
11135	else
11136	err = 0;
11137
11138	if (unlikely(err < 0)) {
11139	netdev_err(dev,
11140	"set_features() failed (%d); wanted %pNF, left %pNF\n",
11141	err, &features, &dev->features);
11142	/* return non-0 since some features might have changed and
11143	* it's better to fire a spurious notification than miss it
11144	*/
11145	return -1;
11146	}
11147
11148	sync_lower:
11149	/* some features must be disabled on lower devices when disabled
11150	* on an upper device (think: bonding master or bridge)
11151	*/
11152	netdev_for_each_lower_dev(dev, lower, iter)
11153	netdev_sync_lower_features(dev, lower, features);
11154
11155	if (!err) {
11156	netdev_features_t diff = features ^ dev->features;
11157
11158	if (diff & NETIF_F_RX_UDP_TUNNEL_PORT) {
11159	/* udp_tunnel_{get,drop}_rx_info both need
11160	* NETIF_F_RX_UDP_TUNNEL_PORT enabled on the
11161	* device, or they won't do anything.
11162	* Thus we need to update dev->features
11163	* before calling udp_tunnel_get_rx_info,
11164	* but after calling udp_tunnel_drop_rx_info.
11165	*/
11166	udp_tunnel_nic_lock(dev);
11167	if (features & NETIF_F_RX_UDP_TUNNEL_PORT) {
11168	dev->features = features;
11169	udp_tunnel_get_rx_info(dev);
11170	} else {
11171	udp_tunnel_drop_rx_info(dev);
11172	}
11173	udp_tunnel_nic_unlock(dev);
11174	}
11175
11176	if (diff & NETIF_F_HW_VLAN_CTAG_FILTER) {
11177	if (features & NETIF_F_HW_VLAN_CTAG_FILTER) {
11178	dev->features = features;
11179	err \|= vlan_get_rx_ctag_filter_info(dev);
11180	} else {
11181	vlan_drop_rx_ctag_filter_info(dev);
11182	}
11183	}
11184
11185	if (diff & NETIF_F_HW_VLAN_STAG_FILTER) {
11186	if (features & NETIF_F_HW_VLAN_STAG_FILTER) {
11187	dev->features = features;
11188	err \|= vlan_get_rx_stag_filter_info(dev);
11189	} else {
11190	vlan_drop_rx_stag_filter_info(dev);
11191	}
11192	}
11193
11194	dev->features = features;
11195	}
11196
11197	return err < 0 ? 0 : 1;
11198	}
11199
11200	/**
11201	* netdev_update_features - recalculate device features
11202	* @dev: the device to check
11203	*
11204	* Recalculate dev->features set and send notifications if it
11205	* has changed. Should be called after driver or hardware dependent
11206	* conditions might have changed that influence the features.
11207	*/
11208	void netdev_update_features(struct net_device *dev)
11209	{
11210	if (__netdev_update_features(dev))
11211	netdev_features_change(dev);
11212	}
11213	EXPORT_SYMBOL(netdev_update_features);
11214
11215	/**
11216	* netdev_change_features - recalculate device features
11217	* @dev: the device to check
11218	*
11219	* Recalculate dev->features set and send notifications even
11220	* if they have not changed. Should be called instead of
11221	* netdev_update_features() if also dev->vlan_features might
11222	* have changed to allow the changes to be propagated to stacked
11223	* VLAN devices.
11224	*/
11225	void netdev_change_features(struct net_device *dev)
11226	{
11227	__netdev_update_features(dev);
11228	netdev_features_change(dev);
11229	}
11230	EXPORT_SYMBOL(netdev_change_features);
11231
11232	/**
11233	* netif_stacked_transfer_operstate - transfer operstate
11234	* @rootdev: the root or lower level device to transfer state from
11235	* @dev: the device to transfer operstate to
11236	*
11237	* Transfer operational state from root to device. This is normally
11238	* called when a stacking relationship exists between the root
11239	* device and the device(a leaf device).
11240	*/
11241	void netif_stacked_transfer_operstate(const struct net_device *rootdev,
11242	struct net_device *dev)
11243	{
11244	if (rootdev->operstate == IF_OPER_DORMANT)
11245	netif_dormant_on(dev);
11246	else
11247	netif_dormant_off(dev);
11248
11249	if (rootdev->operstate == IF_OPER_TESTING)
11250	netif_testing_on(dev);
11251	else
11252	netif_testing_off(dev);
11253
11254	if (netif_carrier_ok(rootdev))
11255	netif_carrier_on(dev);
11256	else
11257	netif_carrier_off(dev);
11258	}
11259	EXPORT_SYMBOL(netif_stacked_transfer_operstate);
11260
11261	static int netif_alloc_rx_queues(struct net_device *dev)
11262	{
11263	unsigned int i, count = dev->num_rx_queues;
11264	struct netdev_rx_queue *rx;
11265	size_t sz = count * sizeof(*rx);
11266	int err = 0;
11267
11268	BUG_ON(count < 1);
11269
11270	rx = kvzalloc(sz, GFP_KERNEL_ACCOUNT \| __GFP_RETRY_MAYFAIL);
11271	if (!rx)
11272	return -ENOMEM;
11273
11274	dev->_rx = rx;
11275
11276	for (i = 0; i < count; i++) {
11277	rx[i].dev = dev;
11278
11279	/* XDP RX-queue setup */
11280	err = xdp_rxq_info_reg(&rx[i].xdp_rxq, dev, i, 0);
11281	if (err < 0)
11282	goto err_rxq_info;
11283	}
11284	return 0;
11285
11286	err_rxq_info:
11287	/* Rollback successful reg's and free other resources */
11288	while (i--)
11289	xdp_rxq_info_unreg(&rx[i].xdp_rxq);
11290	kvfree(dev->_rx);
11291	dev->_rx = NULL;
11292	return err;
11293	}
11294
11295	static void netif_free_rx_queues(struct net_device *dev)
11296	{
11297	unsigned int i, count = dev->num_rx_queues;
11298
11299	/* netif_alloc_rx_queues alloc failed, resources have been unreg'ed */
11300	if (!dev->_rx)
11301	return;
11302
11303	for (i = 0; i < count; i++)
11304	xdp_rxq_info_unreg(&dev->_rx[i].xdp_rxq);
11305
11306	kvfree(dev->_rx);
11307	}
11308
11309	static void netdev_init_one_queue(struct net_device *dev,
11310	struct netdev_queue queue, void _unused)
11311	{
11312	/* Initialize queue lock */
11313	spin_lock_init(&queue->_xmit_lock);
11314	netdev_set_xmit_lockdep_class(&queue->_xmit_lock, dev->type);
11315	queue->xmit_lock_owner = -1;
11316	netdev_queue_numa_node_write(queue, NUMA_NO_NODE);
11317	queue->dev = dev;
11318	#ifdef CONFIG_BQL
11319	dql_init(&queue->dql, HZ);
11320	#endif
11321	}
11322
11323	static void netif_free_tx_queues(struct net_device *dev)
11324	{
11325	kvfree(dev->_tx);
11326	}
11327
11328	static int netif_alloc_netdev_queues(struct net_device *dev)
11329	{
11330	unsigned int count = dev->num_tx_queues;
11331	struct netdev_queue *tx;
11332	size_t sz = count * sizeof(*tx);
11333
11334	if (count < 1 \|\| count > 0xffff)
11335	return -EINVAL;
11336
11337	tx = kvzalloc(sz, GFP_KERNEL_ACCOUNT \| __GFP_RETRY_MAYFAIL);
11338	if (!tx)
11339	return -ENOMEM;
11340
11341	dev->_tx = tx;
11342
11343	netdev_for_each_tx_queue(dev, netdev_init_one_queue, NULL);
11344	spin_lock_init(&dev->tx_global_lock);
11345	spin_lock_init(&dev->watchdog_lock);
11346	dev->watchdog_ref_held = false;
11347	return 0;
11348	}
11349
11350	void netif_tx_stop_all_queues(struct net_device *dev)
11351	{
11352	unsigned int i;
11353
11354	for (i = 0; i < dev->num_tx_queues; i++) {
11355	struct netdev_queue *txq = netdev_get_tx_queue(dev, i);
11356
11357	netif_tx_stop_queue(txq);
11358	}
11359	}
11360	EXPORT_SYMBOL(netif_tx_stop_all_queues);
11361
11362	static int netdev_do_alloc_pcpu_stats(struct net_device *dev)
11363	{
11364	void __percpu *v;
11365
11366	/* Drivers implementing ndo_get_peer_dev must support tstat
11367	* accounting, so that skb_do_redirect() can bump the dev's
11368	* RX stats upon network namespace switch.
11369	*/
11370	if (dev->netdev_ops->ndo_get_peer_dev &&
11371	dev->pcpu_stat_type != NETDEV_PCPU_STAT_TSTATS)
11372	return -EOPNOTSUPP;
11373
11374	switch (dev->pcpu_stat_type) {
11375	case NETDEV_PCPU_STAT_NONE:
11376	return 0;
11377	case NETDEV_PCPU_STAT_LSTATS:
11378	v = dev->lstats = netdev_alloc_pcpu_stats(struct pcpu_lstats);
11379	break;
11380	case NETDEV_PCPU_STAT_TSTATS:
11381	v = dev->tstats = netdev_alloc_pcpu_stats(struct pcpu_sw_netstats);
11382	break;
11383	case NETDEV_PCPU_STAT_DSTATS:
11384	v = dev->dstats = netdev_alloc_pcpu_stats(struct pcpu_dstats);
11385	break;
11386	default:
11387	return -EINVAL;
11388	}
11389
11390	return v ? 0 : -ENOMEM;
11391	}
11392
11393	static void netdev_do_free_pcpu_stats(struct net_device *dev)
11394	{
11395	switch (dev->pcpu_stat_type) {
11396	case NETDEV_PCPU_STAT_NONE:
11397	return;
11398	case NETDEV_PCPU_STAT_LSTATS:
11399	free_percpu(dev->lstats);
11400	break;
11401	case NETDEV_PCPU_STAT_TSTATS:
11402	free_percpu(dev->tstats);
11403	break;
11404	case NETDEV_PCPU_STAT_DSTATS:
11405	free_percpu(dev->dstats);
11406	break;
11407	}
11408	}
11409
11410	static void netdev_free_phy_link_topology(struct net_device *dev)
11411	{
11412	struct phy_link_topology *topo = dev->link_topo;
11413
11414	if (IS_ENABLED(CONFIG_PHYLIB) && topo) {
11415	xa_destroy(&topo->phys);
11416	kfree(topo);
11417	dev->link_topo = NULL;
11418	}
11419	}
11420
11421	/**
11422	* register_netdevice() - register a network device
11423	* @dev: device to register
11424	*
11425	* Take a prepared network device structure and make it externally accessible.
11426	* A %NETDEV_REGISTER message is sent to the netdev notifier chain.
11427	* Callers must hold the rtnl lock - you may want register_netdev()
11428	* instead of this.
11429	*/
11430	int register_netdevice(struct net_device *dev)
11431	{
11432	int ret;
11433	struct net *net = dev_net(dev);
11434
11435	BUILD_BUG_ON(sizeof(netdev_features_t) * BITS_PER_BYTE <
11436	NETDEV_FEATURE_COUNT);
11437	BUG_ON(dev_boot_phase);
11438	ASSERT_RTNL();
11439
11440	might_sleep();
11441
11442	/* When net_device's are persistent, this will be fatal. */
11443	BUG_ON(dev->reg_state != NETREG_UNINITIALIZED);
11444	BUG_ON(!net);
11445
11446	ret = ethtool_check_ops(dev->ethtool_ops);
11447	if (ret)
11448	return ret;
11449
11450	/* rss ctx ID 0 is reserved for the default context, start from 1 */
11451	xa_init_flags(&dev->ethtool->rss_ctx, XA_FLAGS_ALLOC1);
11452	mutex_init(&dev->ethtool->rss_lock);
11453
11454	spin_lock_init(&dev->addr_list_lock);
11455	netdev_set_addr_lockdep_class(dev);
11456
11457	ret = dev_get_valid_name(net, dev, dev->name);
11458	if (ret < 0)
11459	goto out;
11460
11461	ret = -ENOMEM;
11462	dev->name_node = netdev_name_node_head_alloc(dev);
11463	if (!dev->name_node)
11464	goto out;
11465
11466	/* Init, if this function is available */
11467	if (dev->netdev_ops->ndo_init) {
11468	ret = dev->netdev_ops->ndo_init(dev);
11469	if (ret) {
11470	if (ret > 0)
11471	ret = -EIO;
11472	goto err_free_name;
11473	}
11474	}
11475
11476	if (((dev->hw_features \| dev->features) &
11477	NETIF_F_HW_VLAN_CTAG_FILTER) &&
11478	(!dev->netdev_ops->ndo_vlan_rx_add_vid \|\|
11479	!dev->netdev_ops->ndo_vlan_rx_kill_vid)) {
11480	netdev_WARN(dev, "Buggy VLAN acceleration in driver!\n");
11481	ret = -EINVAL;
11482	goto err_uninit;
11483	}
11484
11485	if (netdev_need_ops_lock(dev) &&
11486	dev->netdev_ops->ndo_set_rx_mode &&
11487	!dev->netdev_ops->ndo_set_rx_mode_async)
11488	netdev_WARN(dev, "ops-locked drivers should use ndo_set_rx_mode_async\n");
11489
11490	ret = netdev_do_alloc_pcpu_stats(dev);
11491	if (ret)
11492	goto err_uninit;
11493
11494	ret = dev_index_reserve(net, dev->ifindex);
11495	if (ret < 0)
11496	goto err_free_pcpu;
11497	dev->ifindex = ret;
11498
11499	/* Transfer changeable features to wanted_features and enable
11500	* software offloads (GSO and GRO).
11501	*/
11502	dev->hw_features \|= (NETIF_F_SOFT_FEATURES \| NETIF_F_SOFT_FEATURES_OFF);
11503	dev->features \|= NETIF_F_SOFT_FEATURES;
11504
11505	if (dev->udp_tunnel_nic_info) {
11506	dev->features \|= NETIF_F_RX_UDP_TUNNEL_PORT;
11507	dev->hw_features \|= NETIF_F_RX_UDP_TUNNEL_PORT;
11508	}
11509
11510	dev->wanted_features = dev->features & dev->hw_features;
11511
11512	if (!(dev->flags & IFF_LOOPBACK))
11513	dev->hw_features \|= NETIF_F_NOCACHE_COPY;
11514
11515	/* If IPv4 TCP segmentation offload is supported we should also
11516	* allow the device to enable segmenting the frame with the option
11517	* of ignoring a static IP ID value. This doesn't enable the
11518	* feature itself but allows the user to enable it later.
11519	*/
11520	if (dev->hw_features & NETIF_F_TSO)
11521	dev->hw_features \|= NETIF_F_TSO_MANGLEID;
11522	if (dev->vlan_features & NETIF_F_TSO)
11523	dev->vlan_features \|= NETIF_F_TSO_MANGLEID;
11524	if (dev->mpls_features & NETIF_F_TSO)
11525	dev->mpls_features \|= NETIF_F_TSO_MANGLEID;
11526	if (dev->hw_enc_features & NETIF_F_TSO)
11527	dev->hw_enc_features \|= NETIF_F_TSO_MANGLEID;
11528
11529	/* TSO_MANGLEID belongs in mangleid_features by definition */
11530	dev->mangleid_features \|= NETIF_F_TSO_MANGLEID;
11531
11532	/* Make NETIF_F_HIGHDMA inheritable to VLAN devices.
11533	*/
11534	dev->vlan_features \|= NETIF_F_HIGHDMA;
11535
11536	/* Make NETIF_F_SG inheritable to tunnel devices.
11537	*/
11538	dev->hw_enc_features \|= NETIF_F_SG \| NETIF_F_GSO_PARTIAL;
11539
11540	/* Make NETIF_F_SG inheritable to MPLS.
11541	*/
11542	dev->mpls_features \|= NETIF_F_SG;
11543
11544	ret = call_netdevice_notifiers(NETDEV_POST_INIT, dev);
11545	ret = notifier_to_errno(ret);
11546	if (ret)
11547	goto err_ifindex_release;
11548
11549	ret = netdev_register_kobject(dev);
11550
11551	netdev_lock(dev);
11552	WRITE_ONCE(dev->reg_state, ret ? NETREG_UNREGISTERED : NETREG_REGISTERED);
11553	netdev_unlock(dev);
11554
11555	if (ret)
11556	goto err_uninit_notify;
11557
11558	netdev_lock_ops(dev);
11559	__netdev_update_features(dev);
11560	netdev_unlock_ops(dev);
11561
11562	/*
11563	* Default initial state at registry is that the
11564	* device is present.
11565	*/
11566
11567	set_bit(__LINK_STATE_PRESENT, &dev->state);
11568
11569	linkwatch_init_dev(dev);
11570
11571	dev_init_scheduler(dev);
11572
11573	netdev_hold(dev, &dev->dev_registered_tracker, GFP_KERNEL);
11574	list_netdevice(dev);
11575
11576	add_device_randomness(dev->dev_addr, dev->addr_len);
11577
11578	/* If the device has permanent device address, driver should
11579	* set dev_addr and also addr_assign_type should be set to
11580	* NET_ADDR_PERM (default value).
11581	*/
11582	if (dev->addr_assign_type == NET_ADDR_PERM)
11583	memcpy(dev->perm_addr, dev->dev_addr, dev->addr_len);
11584
11585	/* Notify protocols, that a new device appeared. */
11586	netdev_lock_ops(dev);
11587	ret = call_netdevice_notifiers(NETDEV_REGISTER, dev);
11588	netdev_unlock_ops(dev);
11589	ret = notifier_to_errno(ret);
11590	if (ret) {
11591	/* Expect explicit free_netdev() on failure */
11592	dev->needs_free_netdev = false;
11593	unregister_netdevice_queue(dev, NULL);
11594	goto out;
11595	}
11596	/*
11597	* Prevent userspace races by waiting until the network
11598	* device is fully setup before sending notifications.
11599	*/
11600	netdev_uevent_add(dev);
11601	if (!(dev->rtnl_link_ops && dev->rtnl_link_initializing))
11602	rtmsg_ifinfo(RTM_NEWLINK, dev, ~0U, GFP_KERNEL, 0, NULL);
11603
11604	out:
11605	return ret;
11606
11607	err_uninit_notify:
11608	call_netdevice_notifiers(NETDEV_PRE_UNINIT, dev);
11609	err_ifindex_release:
11610	dev_index_release(net, dev->ifindex);
11611	err_free_pcpu:
11612	netdev_do_free_pcpu_stats(dev);
11613	err_uninit:
11614	if (dev->netdev_ops->ndo_uninit)
11615	dev->netdev_ops->ndo_uninit(dev);
11616	if (dev->priv_destructor)
11617	dev->priv_destructor(dev);
11618	err_free_name:
11619	netdev_name_node_free(dev->name_node);
11620	goto out;
11621	}
11622	EXPORT_SYMBOL(register_netdevice);
11623
11624	/* Initialize the core of a dummy net device.
11625	* The setup steps dummy netdevs need which normal netdevs get by going
11626	* through register_netdevice().
11627	*/
11628	static void init_dummy_netdev(struct net_device *dev)
11629	{
11630	/* make sure we BUG if trying to hit standard
11631	* register/unregister code path
11632	*/
11633	dev->reg_state = NETREG_DUMMY;
11634
11635	/* a dummy interface is started by default */
11636	set_bit(__LINK_STATE_PRESENT, &dev->state);
11637	set_bit(__LINK_STATE_START, &dev->state);
11638
11639	/* Note : We dont allocate pcpu_refcnt for dummy devices,
11640	* because users of this 'device' dont need to change
11641	* its refcount.
11642	*/
11643	}
11644
11645	/**
11646	* register_netdev - register a network device
11647	* @dev: device to register
11648	*
11649	* Take a completed network device structure and add it to the kernel
11650	* interfaces. A %NETDEV_REGISTER message is sent to the netdev notifier
11651	* chain. 0 is returned on success. A negative errno code is returned
11652	* on a failure to set up the device, or if the name is a duplicate.
11653	*
11654	* This is a wrapper around register_netdevice that takes the rtnl semaphore
11655	* and expands the device name if you passed a format string to
11656	* alloc_netdev.
11657	*/
11658	int register_netdev(struct net_device *dev)
11659	{
11660	struct net *net = dev_net(dev);
11661	int err;
11662
11663	if (rtnl_net_lock_killable(net))
11664	return -EINTR;
11665
11666	err = register_netdevice(dev);
11667
11668	rtnl_net_unlock(net);
11669
11670	return err;
11671	}
11672	EXPORT_SYMBOL(register_netdev);
11673
11674	int netdev_refcnt_read(const struct net_device *dev)
11675	{
11676	#ifdef CONFIG_PCPU_DEV_REFCNT
11677	int i, refcnt = 0;
11678
11679	for_each_possible_cpu(i)
11680	refcnt += *per_cpu_ptr(dev->pcpu_refcnt, i);
11681	return refcnt;
11682	#else
11683	return refcount_read(&dev->dev_refcnt);
11684	#endif
11685	}
11686	EXPORT_SYMBOL(netdev_refcnt_read);
11687
11688	int netdev_unregister_timeout_secs __read_mostly = 10;
11689
11690	#define WAIT_REFS_MIN_MSECS 1
11691	#define WAIT_REFS_MAX_MSECS 250
11692	/**
11693	* netdev_wait_allrefs_any - wait until all references are gone.
11694	* @list: list of net_devices to wait on
11695	*
11696	* This is called when unregistering network devices.
11697	*
11698	* Any protocol or device that holds a reference should register
11699	* for netdevice notification, and cleanup and put back the
11700	* reference if they receive an UNREGISTER event.
11701	* We can get stuck here if buggy protocols don't correctly
11702	* call dev_put.
11703	*/
11704	static struct net_device netdev_wait_allrefs_any(struct list_head list)
11705	{
11706	unsigned long rebroadcast_time, warning_time;
11707	struct net_device *dev;
11708	int wait = 0;
11709
11710	rebroadcast_time = warning_time = jiffies;
11711
11712	list_for_each_entry(dev, list, todo_list)
11713	if (netdev_refcnt_read(dev) == 1)
11714	return dev;
11715
11716	while (true) {
11717	if (time_after(jiffies, rebroadcast_time + 1 * HZ)) {
11718	rtnl_lock();
11719
11720	/* Rebroadcast unregister notification */
11721	list_for_each_entry(dev, list, todo_list) {
11722	struct net *net = dev_net(dev);
11723
11724	__rtnl_net_lock(net);
11725	call_netdevice_notifiers(NETDEV_UNREGISTER, dev);
11726	__rtnl_net_unlock(net);
11727	}
11728
11729	__rtnl_unlock();
11730	rcu_barrier();
11731	rtnl_lock();
11732
11733	list_for_each_entry(dev, list, todo_list)
11734	if (test_bit(__LINK_STATE_LINKWATCH_PENDING,
11735	&dev->state)) {
11736	/* We must not have linkwatch events
11737	* pending on unregister. If this
11738	* happens, we simply run the queue
11739	* unscheduled, resulting in a noop
11740	* for this device.
11741	*/
11742	linkwatch_run_queue();
11743	break;
11744	}
11745
11746	__rtnl_unlock();
11747
11748	rebroadcast_time = jiffies;
11749	}
11750
11751	rcu_barrier();
11752
11753	if (!wait) {
11754	wait = WAIT_REFS_MIN_MSECS;
11755	} else {
11756	msleep(wait);
11757	wait = min(wait << 1, WAIT_REFS_MAX_MSECS);
11758	}
11759
11760	list_for_each_entry(dev, list, todo_list)
11761	if (netdev_refcnt_read(dev) == 1)
11762	return dev;
11763
11764	if (time_after(jiffies, warning_time +
11765	READ_ONCE(netdev_unregister_timeout_secs) * HZ)) {
11766	list_for_each_entry(dev, list, todo_list) {
11767	pr_emerg("unregister_netdevice: waiting for %s to become free. Usage count = %d\n",
11768	dev->name, netdev_refcnt_read(dev));
11769	ref_tracker_dir_print(&dev->refcnt_tracker, 10);
11770	}
11771
11772	warning_time = jiffies;
11773	}
11774	}
11775	}
11776
11777	/* The sequence is:
11778	*
11779	* rtnl_lock();
11780	* ...
11781	* register_netdevice(x1);
11782	* register_netdevice(x2);
11783	* ...
11784	* unregister_netdevice(y1);
11785	* unregister_netdevice(y2);
11786	* ...
11787	* rtnl_unlock();
11788	* free_netdev(y1);
11789	* free_netdev(y2);
11790	*
11791	* We are invoked by rtnl_unlock().
11792	* This allows us to deal with problems:
11793	* 1) We can delete sysfs objects which invoke hotplug
11794	* without deadlocking with linkwatch via keventd.
11795	* 2) Since we run with the RTNL semaphore not held, we can sleep
11796	* safely in order to wait for the netdev refcnt to drop to zero.
11797	*
11798	* We must not return until all unregister events added during
11799	* the interval the lock was held have been completed.
11800	*/
11801	void netdev_run_todo(void)
11802	{
11803	struct net_device dev, tmp;
11804	struct list_head list;
11805	int cnt;
11806	#ifdef CONFIG_LOCKDEP
11807	struct list_head unlink_list;
11808
11809	list_replace_init(&net_unlink_list, &unlink_list);
11810
11811	while (!list_empty(&unlink_list)) {
11812	dev = list_first_entry(&unlink_list, struct net_device,
11813	unlink_list);
11814	list_del_init(&dev->unlink_list);
11815	dev->nested_level = dev->lower_level - 1;
11816	}
11817	#endif
11818
11819	/* Snapshot list, allow later requests */
11820	list_replace_init(&net_todo_list, &list);
11821
11822	__rtnl_unlock();
11823
11824	/* Wait for rcu callbacks to finish before next phase */
11825	if (!list_empty(&list))
11826	rcu_barrier();
11827
11828	list_for_each_entry_safe(dev, tmp, &list, todo_list) {
11829	if (unlikely(dev->reg_state != NETREG_UNREGISTERING)) {
11830	netdev_WARN(dev, "run_todo but not unregistering\n");
11831	list_del(&dev->todo_list);
11832	continue;
11833	}
11834
11835	netdev_lock(dev);
11836	WRITE_ONCE(dev->reg_state, NETREG_UNREGISTERED);
11837	netdev_unlock(dev);
11838	linkwatch_sync_dev(dev);
11839	}
11840
11841	cnt = 0;
11842	while (!list_empty(&list)) {
11843	dev = netdev_wait_allrefs_any(&list);
11844	list_del(&dev->todo_list);
11845
11846	/* paranoia */
11847	BUG_ON(netdev_refcnt_read(dev) != 1);
11848	BUG_ON(!list_empty(&dev->ptype_all));
11849	BUG_ON(!list_empty(&dev->ptype_specific));
11850	WARN_ON(rcu_access_pointer(dev->ip_ptr));
11851	WARN_ON(rcu_access_pointer(dev->ip6_ptr));
11852
11853	netdev_name_node_alt_flush(dev);
11854	netdev_name_node_free(dev->name_node);
11855	netdev_do_free_pcpu_stats(dev);
11856	if (dev->priv_destructor)
11857	dev->priv_destructor(dev);
11858	if (dev->needs_free_netdev)
11859	free_netdev(dev);
11860
11861	cnt++;
11862
11863	/* Free network device */
11864	kobject_put(&dev->dev.kobj);
11865	}
11866	if (cnt && atomic_sub_and_test(cnt, &dev_unreg_count))
11867	wake_up(&netdev_unregistering_wq);
11868	}
11869
11870	/* Collate per-cpu network dstats statistics
11871	*
11872	* Read per-cpu network statistics from dev->dstats and populate the related
11873	* fields in @s.
11874	*/
11875	static void dev_fetch_dstats(struct rtnl_link_stats64 *s,
11876	const struct pcpu_dstats __percpu *dstats)
11877	{
11878	int cpu;
11879
11880	for_each_possible_cpu(cpu) {
11881	u64 rx_packets, rx_bytes, rx_drops;
11882	u64 tx_packets, tx_bytes, tx_drops;
11883	const struct pcpu_dstats *stats;
11884	unsigned int start;
11885
11886	stats = per_cpu_ptr(dstats, cpu);
11887	do {
11888	start = u64_stats_fetch_begin(&stats->syncp);
11889	rx_packets = u64_stats_read(&stats->rx_packets);
11890	rx_bytes = u64_stats_read(&stats->rx_bytes);
11891	rx_drops = u64_stats_read(&stats->rx_drops);
11892	tx_packets = u64_stats_read(&stats->tx_packets);
11893	tx_bytes = u64_stats_read(&stats->tx_bytes);
11894	tx_drops = u64_stats_read(&stats->tx_drops);
11895	} while (u64_stats_fetch_retry(&stats->syncp, start));
11896
11897	s->rx_packets += rx_packets;
11898	s->rx_bytes += rx_bytes;
11899	s->rx_dropped += rx_drops;
11900	s->tx_packets += tx_packets;
11901	s->tx_bytes += tx_bytes;
11902	s->tx_dropped += tx_drops;
11903	}
11904	}
11905
11906	/* ndo_get_stats64 implementation for dtstats-based accounting.
11907	*
11908	* Populate @s from dev->stats and dev->dstats. This is used internally by the
11909	* core for NETDEV_PCPU_STAT_DSTAT-type stats collection.
11910	*/
11911	static void dev_get_dstats64(const struct net_device *dev,
11912	struct rtnl_link_stats64 *s)
11913	{
11914	netdev_stats_to_stats64(s, &dev->stats);
11915	dev_fetch_dstats(s, dev->dstats);
11916	}
11917
11918	/* Convert net_device_stats to rtnl_link_stats64. rtnl_link_stats64 has
11919	* all the same fields in the same order as net_device_stats, with only
11920	* the type differing, but rtnl_link_stats64 may have additional fields
11921	* at the end for newer counters.
11922	*/
11923	void netdev_stats_to_stats64(struct rtnl_link_stats64 *stats64,
11924	const struct net_device_stats *netdev_stats)
11925	{
11926	size_t i, n = sizeof(*netdev_stats) / sizeof(atomic_long_t);
11927	const atomic_long_t src = (atomic_long_t )netdev_stats;
11928	u64 dst = (u64 )stats64;
11929
11930	BUILD_BUG_ON(n > sizeof(*stats64) / sizeof(u64));
11931	for (i = 0; i < n; i++)
11932	dst[i] = (unsigned long)atomic_long_read(&src[i]);
11933	/* zero out counters that only exist in rtnl_link_stats64 */
11934	memset((char )stats64 + n sizeof(u64), 0,
11935	sizeof(stats64) - n sizeof(u64));
11936	}
11937	EXPORT_SYMBOL(netdev_stats_to_stats64);
11938
11939	static __cold struct net_device_core_stats __percpu *netdev_core_stats_alloc(
11940	struct net_device *dev)
11941	{
11942	struct net_device_core_stats __percpu *p;
11943
11944	p = alloc_percpu_gfp(struct net_device_core_stats,
11945	GFP_ATOMIC \| __GFP_NOWARN);
11946
11947	if (p && cmpxchg(&dev->core_stats, NULL, p))
11948	free_percpu(p);
11949
11950	/* This READ_ONCE() pairs with the cmpxchg() above */
11951	return READ_ONCE(dev->core_stats);
11952	}
11953
11954	noinline void netdev_core_stats_inc(struct net_device *dev, u32 offset)
11955	{
11956	/* This READ_ONCE() pairs with the write in netdev_core_stats_alloc() */
11957	struct net_device_core_stats __percpu *p = READ_ONCE(dev->core_stats);
11958	unsigned long __percpu *field;
11959
11960	if (unlikely(!p)) {
11961	p = netdev_core_stats_alloc(dev);
11962	if (!p)
11963	return;
11964	}
11965
11966	field = (unsigned long __percpu )((void __percpu )p + offset);
11967	this_cpu_inc(*field);
11968	}
11969	EXPORT_SYMBOL_GPL(netdev_core_stats_inc);
11970
11971	/**
11972	* dev_get_stats - get network device statistics
11973	* @dev: device to get statistics from
11974	* @storage: place to store stats
11975	*
11976	* Get network statistics from device. Return @storage.
11977	* The device driver may provide its own method by setting
11978	* dev->netdev_ops->get_stats64 or dev->netdev_ops->get_stats;
11979	* otherwise the internal statistics structure is used.
11980	*/
11981	struct rtnl_link_stats64 dev_get_stats(struct net_device dev,
11982	struct rtnl_link_stats64 *storage)
11983	{
11984	const struct net_device_ops *ops = dev->netdev_ops;
11985	const struct net_device_core_stats __percpu *p;
11986
11987	/*
11988	* IPv{4,6} and udp tunnels share common stat helpers and use
11989	* different stat type (NETDEV_PCPU_STAT_TSTATS vs
11990	* NETDEV_PCPU_STAT_DSTATS). Ensure the accounting is consistent.
11991	*/
11992	BUILD_BUG_ON(offsetof(struct pcpu_sw_netstats, rx_bytes) !=
11993	offsetof(struct pcpu_dstats, rx_bytes));
11994	BUILD_BUG_ON(offsetof(struct pcpu_sw_netstats, rx_packets) !=
11995	offsetof(struct pcpu_dstats, rx_packets));
11996	BUILD_BUG_ON(offsetof(struct pcpu_sw_netstats, tx_bytes) !=
11997	offsetof(struct pcpu_dstats, tx_bytes));
11998	BUILD_BUG_ON(offsetof(struct pcpu_sw_netstats, tx_packets) !=
11999	offsetof(struct pcpu_dstats, tx_packets));
12000
12001	if (ops->ndo_get_stats64) {
12002	memset(storage, 0, sizeof(*storage));
12003	ops->ndo_get_stats64(dev, storage);
12004	} else if (ops->ndo_get_stats) {
12005	netdev_stats_to_stats64(storage, ops->ndo_get_stats(dev));
12006	} else if (dev->pcpu_stat_type == NETDEV_PCPU_STAT_TSTATS) {
12007	dev_get_tstats64(dev, storage);
12008	} else if (dev->pcpu_stat_type == NETDEV_PCPU_STAT_DSTATS) {
12009	dev_get_dstats64(dev, storage);
12010	} else {
12011	netdev_stats_to_stats64(storage, &dev->stats);
12012	}
12013
12014	/* This READ_ONCE() pairs with the write in netdev_core_stats_alloc() */
12015	p = READ_ONCE(dev->core_stats);
12016	if (p) {
12017	const struct net_device_core_stats *core_stats;
12018	int i;
12019
12020	for_each_possible_cpu(i) {
12021	core_stats = per_cpu_ptr(p, i);
12022	storage->rx_dropped += READ_ONCE(core_stats->rx_dropped);
12023	storage->tx_dropped += READ_ONCE(core_stats->tx_dropped);
12024	storage->rx_nohandler += READ_ONCE(core_stats->rx_nohandler);
12025	storage->rx_otherhost_dropped += READ_ONCE(core_stats->rx_otherhost_dropped);
12026	}
12027	}
12028	return storage;
12029	}
12030	EXPORT_SYMBOL(dev_get_stats);
12031
12032	/**
12033	* dev_fetch_sw_netstats - get per-cpu network device statistics
12034	* @s: place to store stats
12035	* @netstats: per-cpu network stats to read from
12036	*
12037	* Read per-cpu network statistics and populate the related fields in @s.
12038	*/
12039	void dev_fetch_sw_netstats(struct rtnl_link_stats64 *s,
12040	const struct pcpu_sw_netstats __percpu *netstats)
12041	{
12042	int cpu;
12043
12044	for_each_possible_cpu(cpu) {
12045	u64 rx_packets, rx_bytes, tx_packets, tx_bytes;
12046	const struct pcpu_sw_netstats *stats;
12047	unsigned int start;
12048
12049	stats = per_cpu_ptr(netstats, cpu);
12050	do {
12051	start = u64_stats_fetch_begin(&stats->syncp);
12052	rx_packets = u64_stats_read(&stats->rx_packets);
12053	rx_bytes = u64_stats_read(&stats->rx_bytes);
12054	tx_packets = u64_stats_read(&stats->tx_packets);
12055	tx_bytes = u64_stats_read(&stats->tx_bytes);
12056	} while (u64_stats_fetch_retry(&stats->syncp, start));
12057
12058	s->rx_packets += rx_packets;
12059	s->rx_bytes += rx_bytes;
12060	s->tx_packets += tx_packets;
12061	s->tx_bytes += tx_bytes;
12062	}
12063	}
12064	EXPORT_SYMBOL_GPL(dev_fetch_sw_netstats);
12065
12066	/**
12067	* dev_get_tstats64 - ndo_get_stats64 implementation
12068	* @dev: device to get statistics from
12069	* @s: place to store stats
12070	*
12071	* Populate @s from dev->stats and dev->tstats. Can be used as
12072	* ndo_get_stats64() callback.
12073	*/
12074	void dev_get_tstats64(struct net_device dev, struct rtnl_link_stats64 s)
12075	{
12076	netdev_stats_to_stats64(s, &dev->stats);
12077	dev_fetch_sw_netstats(s, dev->tstats);
12078	}
12079	EXPORT_SYMBOL_GPL(dev_get_tstats64);
12080
12081	struct netdev_queue dev_ingress_queue_create(struct net_device dev)
12082	{
12083	struct netdev_queue *queue = dev_ingress_queue(dev);
12084
12085	#ifdef CONFIG_NET_CLS_ACT
12086	if (queue)
12087	return queue;
12088	queue = kzalloc_obj(*queue);
12089	if (!queue)
12090	return NULL;
12091	netdev_init_one_queue(dev, queue, NULL);
12092	RCU_INIT_POINTER(queue->qdisc, &noop_qdisc);
12093	RCU_INIT_POINTER(queue->qdisc_sleeping, &noop_qdisc);
12094	rcu_assign_pointer(dev->ingress_queue, queue);
12095	#endif
12096	return queue;
12097	}
12098
12099	static const struct ethtool_ops default_ethtool_ops;
12100
12101	void netdev_set_default_ethtool_ops(struct net_device *dev,
12102	const struct ethtool_ops *ops)
12103	{
12104	if (dev->ethtool_ops == &default_ethtool_ops)
12105	dev->ethtool_ops = ops;
12106	}
12107	EXPORT_SYMBOL_GPL(netdev_set_default_ethtool_ops);
12108
12109	/**
12110	* netdev_sw_irq_coalesce_default_on() - enable SW IRQ coalescing by default
12111	* @dev: netdev to enable the IRQ coalescing on
12112	*
12113	* Sets a conservative default for SW IRQ coalescing. Users can use
12114	* sysfs attributes to override the default values.
12115	*/
12116	void netdev_sw_irq_coalesce_default_on(struct net_device *dev)
12117	{
12118	WARN_ON(dev->reg_state == NETREG_REGISTERED);
12119
12120	if (!IS_ENABLED(CONFIG_PREEMPT_RT)) {
12121	netdev_set_gro_flush_timeout(dev, 20000);
12122	netdev_set_defer_hard_irqs(dev, 1);
12123	}
12124	}
12125	EXPORT_SYMBOL_GPL(netdev_sw_irq_coalesce_default_on);
12126
12127	/**
12128	* alloc_netdev_mqs - allocate network device
12129	* @sizeof_priv: size of private data to allocate space for
12130	* @name: device name format string
12131	* @name_assign_type: origin of device name
12132	* @setup: callback to initialize device
12133	* @txqs: the number of TX subqueues to allocate
12134	* @rxqs: the number of RX subqueues to allocate
12135	*
12136	* Allocates a struct net_device with private data area for driver use
12137	* and performs basic initialization. Also allocates subqueue structs
12138	* for each queue on the device.
12139	*/
12140	struct net_device alloc_netdev_mqs(int sizeof_priv, const char name,
12141	unsigned char name_assign_type,
12142	void (setup)(struct net_device ),
12143	unsigned int txqs, unsigned int rxqs)
12144	{
12145	struct net_device *dev;
12146	size_t napi_config_sz;
12147	unsigned int maxqs;
12148
12149	BUG_ON(strlen(name) >= sizeof(dev->name));
12150
12151	if (txqs < 1) {
12152	pr_err("alloc_netdev: Unable to allocate device with zero queues\n");
12153	return NULL;
12154	}
12155
12156	if (rxqs < 1) {
12157	pr_err("alloc_netdev: Unable to allocate device with zero RX queues\n");
12158	return NULL;
12159	}
12160
12161	maxqs = max(txqs, rxqs);
12162
12163	dev = kvzalloc_flex(*dev, priv, sizeof_priv,
12164	GFP_KERNEL_ACCOUNT \| __GFP_RETRY_MAYFAIL);
12165	if (!dev)
12166	return NULL;
12167
12168	dev->priv_len = sizeof_priv;
12169
12170	ref_tracker_dir_init(&dev->refcnt_tracker, 128, "netdev");
12171	#ifdef CONFIG_PCPU_DEV_REFCNT
12172	dev->pcpu_refcnt = alloc_percpu(int);
12173	if (!dev->pcpu_refcnt)
12174	goto free_dev;
12175	__dev_hold(dev);
12176	#else
12177	refcount_set(&dev->dev_refcnt, 1);
12178	#endif
12179
12180	if (dev_addr_init(dev))
12181	goto free_pcpu;
12182
12183	dev_mc_init(dev);
12184	dev_uc_init(dev);
12185
12186	dev_net_set(dev, &init_net);
12187
12188	dev->gso_max_size = GSO_LEGACY_MAX_SIZE;
12189	dev->xdp_zc_max_segs = 1;
12190	dev->gso_max_segs = GSO_MAX_SEGS;
12191	dev->gro_max_size = GRO_LEGACY_MAX_SIZE;
12192	dev->gso_ipv4_max_size = GSO_LEGACY_MAX_SIZE;
12193	dev->gro_ipv4_max_size = GRO_LEGACY_MAX_SIZE;
12194	dev->tso_max_size = TSO_LEGACY_MAX_SIZE;
12195	dev->tso_max_segs = TSO_MAX_SEGS;
12196	dev->upper_level = 1;
12197	dev->lower_level = 1;
12198	#ifdef CONFIG_LOCKDEP
12199	dev->nested_level = 0;
12200	INIT_LIST_HEAD(&dev->unlink_list);
12201	#endif
12202
12203	INIT_LIST_HEAD(&dev->napi_list);
12204	INIT_LIST_HEAD(&dev->unreg_list);
12205	#ifdef CONFIG_DEBUG_NET_SMALL_RTNL
12206	INIT_LIST_HEAD(&dev->unreg_list_net);
12207	#endif
12208	INIT_LIST_HEAD(&dev->close_list);
12209	INIT_LIST_HEAD(&dev->link_watch_list);
12210	INIT_LIST_HEAD(&dev->adj_list.upper);
12211	INIT_LIST_HEAD(&dev->adj_list.lower);
12212	INIT_LIST_HEAD(&dev->ptype_all);
12213	INIT_LIST_HEAD(&dev->ptype_specific);
12214	INIT_LIST_HEAD(&dev->net_notifier_list);
12215	INIT_LIST_HEAD(&dev->work_node);
12216	#ifdef CONFIG_NET_SCHED
12217	hash_init(dev->qdisc_hash);
12218	#endif
12219
12220	mutex_init(&dev->lock);
12221	netif_rx_mode_init(dev);
12222
12223	dev->priv_flags = IFF_XMIT_DST_RELEASE \| IFF_XMIT_DST_RELEASE_PERM;
12224	setup(dev);
12225
12226	if (!dev->tx_queue_len) {
12227	dev->priv_flags \|= IFF_NO_QUEUE;
12228	dev->tx_queue_len = DEFAULT_TX_QUEUE_LEN;
12229	}
12230
12231	dev->num_tx_queues = txqs;
12232	dev->real_num_tx_queues = txqs;
12233	if (netif_alloc_netdev_queues(dev))
12234	goto free_all;
12235
12236	dev->num_rx_queues = rxqs;
12237	dev->real_num_rx_queues = rxqs;
12238	if (netif_alloc_rx_queues(dev))
12239	goto free_all;
12240	dev->ethtool = kzalloc_obj(*dev->ethtool, GFP_KERNEL_ACCOUNT);
12241	if (!dev->ethtool)
12242	goto free_all;
12243
12244	dev->cfg = kzalloc_obj(*dev->cfg, GFP_KERNEL_ACCOUNT);
12245	if (!dev->cfg)
12246	goto free_all;
12247	dev->cfg_pending = dev->cfg;
12248
12249	dev->num_napi_configs = maxqs;
12250	napi_config_sz = array_size(maxqs, sizeof(*dev->napi_config));
12251	dev->napi_config = kvzalloc(napi_config_sz, GFP_KERNEL_ACCOUNT);
12252	if (!dev->napi_config)
12253	goto free_all;
12254
12255	strscpy(dev->name, name);
12256	dev->name_assign_type = name_assign_type;
12257	dev->group = INIT_NETDEV_GROUP;
12258	if (!dev->ethtool_ops)
12259	dev->ethtool_ops = &default_ethtool_ops;
12260
12261	nf_hook_netdev_init(dev);
12262
12263	return dev;
12264
12265	free_all:
12266	free_netdev(dev);
12267	return NULL;
12268
12269	free_pcpu:
12270	#ifdef CONFIG_PCPU_DEV_REFCNT
12271	free_percpu(dev->pcpu_refcnt);
12272	free_dev:
12273	#endif
12274	ref_tracker_dir_exit(&dev->refcnt_tracker);
12275	kvfree(dev);
12276	return NULL;
12277	}
12278	EXPORT_SYMBOL(alloc_netdev_mqs);
12279
12280	static void netdev_napi_exit(struct net_device *dev)
12281	{
12282	if (!list_empty(&dev->napi_list)) {
12283	struct napi_struct p, n;
12284
12285	netdev_lock(dev);
12286	list_for_each_entry_safe(p, n, &dev->napi_list, dev_list)
12287	__netif_napi_del_locked(p);
12288	netdev_unlock(dev);
12289
12290	synchronize_net();
12291	}
12292
12293	kvfree(dev->napi_config);
12294	}
12295
12296	/**
12297	* free_netdev - free network device
12298	* @dev: device
12299	*
12300	* This function does the last stage of destroying an allocated device
12301	* interface. The reference to the device object is released. If this
12302	* is the last reference then it will be freed.Must be called in process
12303	* context.
12304	*/
12305	void free_netdev(struct net_device *dev)
12306	{
12307	might_sleep();
12308
12309	/* When called immediately after register_netdevice() failed the unwind
12310	* handling may still be dismantling the device. Handle that case by
12311	* deferring the free.
12312	*/
12313	if (dev->reg_state == NETREG_UNREGISTERING) {
12314	ASSERT_RTNL();
12315	dev->needs_free_netdev = true;
12316	return;
12317	}
12318
12319	WARN_ON(dev->cfg != dev->cfg_pending);
12320	kfree(dev->cfg);
12321	kfree(dev->ethtool);
12322	netif_free_tx_queues(dev);
12323	netif_free_rx_queues(dev);
12324
12325	kfree(rcu_dereference_protected(dev->ingress_queue, 1));
12326
12327	__hw_addr_flush(&dev->rx_mode_addr_cache);
12328
12329	/* Flush device addresses */
12330	dev_addr_flush(dev);
12331
12332	netdev_napi_exit(dev);
12333
12334	netif_del_cpu_rmap(dev);
12335
12336	ref_tracker_dir_exit(&dev->refcnt_tracker);
12337	#ifdef CONFIG_PCPU_DEV_REFCNT
12338	free_percpu(dev->pcpu_refcnt);
12339	dev->pcpu_refcnt = NULL;
12340	#endif
12341	free_percpu(dev->core_stats);
12342	dev->core_stats = NULL;
12343	free_percpu(dev->xdp_bulkq);
12344	dev->xdp_bulkq = NULL;
12345
12346	netdev_free_phy_link_topology(dev);
12347
12348	mutex_destroy(&dev->lock);
12349
12350	/* Compatibility with error handling in drivers */
12351	if (dev->reg_state == NETREG_UNINITIALIZED \|\|
12352	dev->reg_state == NETREG_DUMMY) {
12353	kvfree(dev);
12354	return;
12355	}
12356
12357	BUG_ON(dev->reg_state != NETREG_UNREGISTERED);
12358	WRITE_ONCE(dev->reg_state, NETREG_RELEASED);
12359
12360	/* will free via device release */
12361	put_device(&dev->dev);
12362	}
12363	EXPORT_SYMBOL(free_netdev);
12364
12365	/**
12366	* alloc_netdev_dummy - Allocate and initialize a dummy net device.
12367	* @sizeof_priv: size of private data to allocate space for
12368	*
12369	* Return: the allocated net_device on success, NULL otherwise
12370	*/
12371	struct net_device *alloc_netdev_dummy(int sizeof_priv)
12372	{
12373	return alloc_netdev(sizeof_priv, "dummy#", NET_NAME_UNKNOWN,
12374	init_dummy_netdev);
12375	}
12376	EXPORT_SYMBOL_GPL(alloc_netdev_dummy);
12377
12378	/**
12379	* synchronize_net - Synchronize with packet receive processing
12380	*
12381	* Wait for packets currently being received to be done.
12382	* Does not block later packets from starting.
12383	*/
12384	void synchronize_net(void)
12385	{
12386	might_sleep();
12387	if (from_cleanup_net() \|\| rtnl_is_locked())
12388	synchronize_rcu_expedited();
12389	else
12390	synchronize_rcu();
12391	}
12392	EXPORT_SYMBOL(synchronize_net);
12393
12394	static void netdev_rss_contexts_free(struct net_device *dev)
12395	{
12396	struct ethtool_rxfh_context *ctx;
12397	unsigned long context;
12398
12399	mutex_lock(&dev->ethtool->rss_lock);
12400	xa_for_each(&dev->ethtool->rss_ctx, context, ctx) {
12401	xa_erase(&dev->ethtool->rss_ctx, context);
12402	dev->ethtool_ops->remove_rxfh_context(dev, ctx, context, NULL);
12403	kfree(ctx);
12404	}
12405	xa_destroy(&dev->ethtool->rss_ctx);
12406	mutex_unlock(&dev->ethtool->rss_lock);
12407	}
12408
12409	/**
12410	* unregister_netdevice_queue - remove device from the kernel
12411	* @dev: device
12412	* @head: list
12413	*
12414	* This function shuts down a device interface and removes it
12415	* from the kernel tables.
12416	* If head not NULL, device is queued to be unregistered later.
12417	*
12418	* Callers must hold the rtnl semaphore. You may want
12419	* unregister_netdev() instead of this.
12420	*/
12421
12422	void unregister_netdevice_queue(struct net_device dev, struct list_head head)
12423	{
12424	ASSERT_RTNL();
12425
12426	#ifdef CONFIG_DEBUG_NET_SMALL_RTNL
12427	DEBUG_NET_WARN_ON_ONCE(!list_empty(&dev->unreg_list_net));
12428	#endif
12429
12430	if (head) {
12431	list_move_tail(&dev->unreg_list, head);
12432	} else {
12433	LIST_HEAD(single);
12434
12435	list_add(&dev->unreg_list, &single);
12436	unregister_netdevice_many(&single);
12437	}
12438	}
12439	EXPORT_SYMBOL(unregister_netdevice_queue);
12440
12441	static void dev_memory_provider_uninstall(struct net_device *dev)
12442	{
12443	unsigned int i;
12444
12445	for (i = 0; i < dev->real_num_rx_queues; i++) {
12446	struct netdev_rx_queue *rxq = &dev->_rx[i];
12447
12448	__netif_mp_uninstall_rxq(rxq, &rxq->mp_params);
12449	}
12450	}
12451
12452	/* devices must be UP and netdev_lock()'d */
12453	static void netif_close_many_and_unlock(struct list_head *close_head)
12454	{
12455	struct net_device dev, tmp;
12456
12457	netif_close_many(close_head, false);
12458
12459	/* ... now unlock them */
12460	list_for_each_entry_safe(dev, tmp, close_head, close_list) {
12461	netdev_unlock(dev);
12462	list_del_init(&dev->close_list);
12463	}
12464	}
12465
12466	static void netif_close_many_and_unlock_cond(struct list_head *close_head)
12467	{
12468	#ifdef CONFIG_LOCKDEP
12469	/* We can only track up to MAX_LOCK_DEPTH locks per task.
12470	*
12471	* Reserve half the available slots for additional locks possibly
12472	* taken by notifiers and (soft)irqs.
12473	*/
12474	unsigned int limit = MAX_LOCK_DEPTH / 2;
12475
12476	if (lockdep_depth(current) > limit)
12477	netif_close_many_and_unlock(close_head);
12478	#endif
12479	}
12480
12481	bool unregister_netdevice_queued(const struct net_device *dev)
12482	{
12483	ASSERT_RTNL();
12484	return !list_empty(&dev->unreg_list);
12485	}
12486
12487	void unregister_netdevice_many_notify(struct list_head *head,
12488	u32 portid, const struct nlmsghdr *nlh)
12489	{
12490	struct net_device dev, tmp;
12491	LIST_HEAD(close_head);
12492	int cnt = 0;
12493
12494	BUG_ON(dev_boot_phase);
12495	ASSERT_RTNL();
12496
12497	if (list_empty(head))
12498	return;
12499
12500	list_for_each_entry_safe(dev, tmp, head, unreg_list) {
12501	/* Some devices call without registering
12502	* for initialization unwind. Remove those
12503	* devices and proceed with the remaining.
12504	*/
12505	if (dev->reg_state == NETREG_UNINITIALIZED) {
12506	pr_debug("unregister_netdevice: device %s/%p never was registered\n",
12507	dev->name, dev);
12508
12509	WARN_ON(1);
12510	list_del(&dev->unreg_list);
12511	continue;
12512	}
12513	dev->dismantle = true;
12514	BUG_ON(dev->reg_state != NETREG_REGISTERED);
12515	}
12516
12517	/* If device is running, close it first. Start with ops locked... */
12518	list_for_each_entry(dev, head, unreg_list) {
12519	if (!(dev->flags & IFF_UP))
12520	continue;
12521	if (netdev_need_ops_lock(dev)) {
12522	list_add_tail(&dev->close_list, &close_head);
12523	netdev_lock(dev);
12524	}
12525	netif_close_many_and_unlock_cond(&close_head);
12526	}
12527	netif_close_many_and_unlock(&close_head);
12528	/* ... now go over the rest. */
12529	list_for_each_entry(dev, head, unreg_list) {
12530	if (!netdev_need_ops_lock(dev))
12531	list_add_tail(&dev->close_list, &close_head);
12532	}
12533	netif_close_many(&close_head, true);
12534
12535	list_for_each_entry(dev, head, unreg_list) {
12536	/* And unlink it from device chain. */
12537	unlist_netdevice(dev);
12538	netdev_lock(dev);
12539	WRITE_ONCE(dev->reg_state, NETREG_UNREGISTERING);
12540	netdev_unlock(dev);
12541	}
12542	flush_all_backlogs();
12543
12544	synchronize_net();
12545
12546	list_for_each_entry(dev, head, unreg_list) {
12547	struct sk_buff *skb = NULL;
12548
12549	/* Shutdown queueing discipline. */
12550	netdev_lock_ops(dev);
12551	dev_shutdown(dev);
12552	dev_tcx_uninstall(dev);
12553	dev_xdp_uninstall(dev);
12554	dev_memory_provider_uninstall(dev);
12555	netdev_work_cancel_all(dev);
12556	netdev_unlock_ops(dev);
12557	bpf_dev_bound_netdev_unregister(dev);
12558
12559	netdev_offload_xstats_disable_all(dev);
12560
12561	/* Notify protocols, that we are about to destroy
12562	* this device. They should clean all the things.
12563	*/
12564	call_netdevice_notifiers(NETDEV_UNREGISTER, dev);
12565
12566	if (!(dev->rtnl_link_ops && dev->rtnl_link_initializing))
12567	skb = rtmsg_ifinfo_build_skb(RTM_DELLINK, dev, ~0U, 0,
12568	GFP_KERNEL, NULL, 0,
12569	portid, nlh);
12570
12571	/*
12572	* Flush the unicast and multicast chains
12573	*/
12574	dev_uc_flush(dev);
12575	dev_mc_flush(dev);
12576
12577
12578	netdev_rss_contexts_free(dev);
12579
12580	call_netdevice_notifiers(NETDEV_PRE_UNINIT, dev);
12581
12582	if (dev->netdev_ops->ndo_uninit)
12583	dev->netdev_ops->ndo_uninit(dev);
12584
12585	mutex_destroy(&dev->ethtool->rss_lock);
12586
12587	net_shaper_flush_netdev(dev);
12588
12589	if (skb)
12590	rtmsg_ifinfo_send(skb, dev, GFP_KERNEL, portid, nlh);
12591
12592	/* Notifier chain MUST detach us all upper devices. */
12593	WARN_ON(netdev_has_any_upper_dev(dev));
12594	WARN_ON(netdev_has_any_lower_dev(dev));
12595
12596	/* Remove entries from kobject tree */
12597	netdev_unregister_kobject(dev);
12598	#ifdef CONFIG_XPS
12599	/* Remove XPS queueing entries */
12600	netif_reset_xps_queues_gt(dev, 0);
12601	#endif
12602	}
12603
12604	synchronize_net();
12605
12606	list_for_each_entry(dev, head, unreg_list) {
12607	#ifdef CONFIG_DEBUG_NET_SMALL_RTNL
12608	struct net *net = dev_net(dev);
12609
12610	/* spin_lock() can be moved outside of the loop
12611	* once the per-netns RTNL conversion completes.
12612	*/
12613	spin_lock(&net->dev_unreg_lock);
12614	list_del(&dev->unreg_list_net);
12615	spin_unlock(&net->dev_unreg_lock);
12616	#endif
12617	netdev_put(dev, &dev->dev_registered_tracker);
12618	net_set_todo(dev);
12619	cnt++;
12620	}
12621	atomic_add(cnt, &dev_unreg_count);
12622
12623	list_del(head);
12624	}
12625
12626	/**
12627	* unregister_netdevice_many - unregister many devices
12628	* @head: list of devices
12629	*
12630	* Note: As most callers use a stack allocated list_head,
12631	* we force a list_del() to make sure stack won't be corrupted later.
12632	*/
12633	void unregister_netdevice_many(struct list_head *head)
12634	{
12635	unregister_netdevice_many_notify(head, 0, NULL);
12636	}
12637	EXPORT_SYMBOL(unregister_netdevice_many);
12638
12639	#ifdef CONFIG_DEBUG_NET_SMALL_RTNL
12640	void unregister_netdevice_queue_net(struct net net, struct net_device dev,
12641	struct list_head *head)
12642	{
12643	netdev_lock(dev);
12644
12645	if (net_eq(dev_net(dev), net)) {
12646	netdev_unlock(dev);
12647	unregister_netdevice_queue(dev, head);
12648	return;
12649	}
12650
12651	net = dev_net(dev);
12652
12653	spin_lock(&net->dev_unreg_lock);
12654
12655	DEBUG_NET_WARN_ON_ONCE(!list_empty(&dev->unreg_list));
12656	DEBUG_NET_WARN_ON_ONCE(!list_empty(&dev->unreg_list_net));
12657
12658	list_add_tail(&dev->unreg_list_net, &net->dev_unreg_head);
12659	rtnl_net_queue_work(net);
12660
12661	spin_unlock(&net->dev_unreg_lock);
12662
12663	netdev_unlock(dev);
12664	}
12665	EXPORT_SYMBOL(unregister_netdevice_queue_net);
12666
12667	void unregister_netdevice_queue_many_net(struct net net, struct list_head head)
12668	{
12669	struct net_device dev, tmp;
12670
12671	spin_lock(&net->dev_unreg_lock);
12672	list_for_each_entry_safe(dev, tmp, head, unreg_list) {
12673	/* Once all cross-netns unregister_netdevice_queue() is
12674	* converted to _net() (or for debugging), remove this check.
12675	*/
12676	if (!net_eq(dev_net(dev), net))
12677	continue;
12678
12679	DEBUG_NET_WARN_ONCE(!net_eq(dev_net(dev), net),
12680	"%s was unregistered from a different netns.\n",
12681	dev->name);
12682
12683	list_del_init(&dev->unreg_list);
12684	list_move_tail(&dev->unreg_list_net, &net->dev_unreg_head);
12685	}
12686	spin_unlock(&net->dev_unreg_lock);
12687	}
12688
12689	static void unregister_netdevice_move_net(struct net *net_old,
12690	struct net *net,
12691	struct net_device *dev)
12692	{
12693	if (net_old > net) {
12694	spin_lock(&net->dev_unreg_lock);
12695	spin_lock_nested(&net_old->dev_unreg_lock, SINGLE_DEPTH_NESTING);
12696	} else {
12697	spin_lock(&net_old->dev_unreg_lock);
12698	spin_lock_nested(&net->dev_unreg_lock, SINGLE_DEPTH_NESTING);
12699	}
12700
12701	if (!list_empty(&dev->unreg_list_net)) {
12702	list_del(&dev->unreg_list_net);
12703	list_add_tail(&dev->unreg_list_net, &net->dev_unreg_head);
12704	}
12705
12706	spin_unlock(&net_old->dev_unreg_lock);
12707	spin_unlock(&net->dev_unreg_lock);
12708	}
12709
12710	void unregister_netdevice_many_net(struct net *net)
12711	{
12712	struct net_device dev, tmp;
12713	LIST_HEAD(unreg_head_net);
12714	LIST_HEAD(unreg_head);
12715
12716	spin_lock(&net->dev_unreg_lock);
12717	list_splice_init(&net->dev_unreg_head, &unreg_head_net);
12718	spin_unlock(&net->dev_unreg_lock);
12719
12720	list_for_each_entry_safe(dev, tmp, &unreg_head_net, unreg_list_net) {
12721	list_del_init(&dev->unreg_list_net);
12722	list_add_tail(&dev->unreg_list, &unreg_head);
12723	}
12724
12725	unregister_netdevice_many(&unreg_head);
12726	}
12727	#endif
12728
12729	/**
12730	* unregister_netdev - remove device from the kernel
12731	* @dev: device
12732	*
12733	* This function shuts down a device interface and removes it
12734	* from the kernel tables.
12735	*
12736	* This is just a wrapper for unregister_netdevice that takes
12737	* the rtnl semaphore. In general you want to use this and not
12738	* unregister_netdevice.
12739	*/
12740	void unregister_netdev(struct net_device *dev)
12741	{
12742	rtnl_net_dev_lock(dev);
12743	unregister_netdevice(dev);
12744	rtnl_net_dev_unlock(dev);
12745	}
12746	EXPORT_SYMBOL(unregister_netdev);
12747
12748	int __dev_change_net_namespace(struct net_device dev, struct net net,
12749	const char *pat, int new_ifindex,
12750	struct netlink_ext_ack *extack)
12751	{
12752	struct netdev_name_node name_node, tmp;
12753	struct net *net_old = dev_net(dev);
12754	char new_name[IFNAMSIZ] = {};
12755	int err, new_nsid;
12756
12757	ASSERT_RTNL();
12758
12759	/* Don't allow namespace local devices to be moved. */
12760	err = -EINVAL;
12761	if (dev->netns_immutable) {
12762	NL_SET_ERR_MSG(extack, "The interface netns is immutable");
12763	goto out;
12764	}
12765
12766	/* Ensure the device has been registered */
12767	if (dev->reg_state != NETREG_REGISTERED) {
12768	NL_SET_ERR_MSG(extack, "The interface isn't registered");
12769	goto out;
12770	}
12771
12772	/* Get out if there is nothing todo */
12773	err = 0;
12774	if (net_eq(net_old, net))
12775	goto out;
12776
12777	/* Pick the destination device name, and ensure
12778	* we can use it in the destination network namespace.
12779	*/
12780	err = -EEXIST;
12781	if (netdev_name_in_use(net, dev->name)) {
12782	/* We get here if we can't use the current device name */
12783	if (!pat) {
12784	NL_SET_ERR_MSG(extack,
12785	"An interface with the same name exists in the target netns");
12786	goto out;
12787	}
12788	err = dev_prep_valid_name(net, dev, pat, new_name, EEXIST);
12789	if (err < 0) {
12790	NL_SET_ERR_MSG_FMT(extack,
12791	"Unable to use '%s' for the new interface name in the target netns",
12792	pat);
12793	goto out;
12794	}
12795	}
12796	/* Check that none of the altnames conflicts. */
12797	err = -EEXIST;
12798	netdev_for_each_altname_safe(dev, name_node, tmp) {
12799	if (!netdev_name_in_use(net, name_node->name))
12800	continue;
12801
12802	if (!check_net(net_old)) {
12803	__netdev_name_node_alt_destroy(name_node);
12804	continue;
12805	}
12806
12807	NL_SET_ERR_MSG_FMT(extack,
12808	"An interface with the altname %s exists in the target netns",
12809	name_node->name);
12810	goto out;
12811	}
12812
12813	/* Check that new_ifindex isn't used yet. */
12814	if (new_ifindex) {
12815	err = dev_index_reserve(net, new_ifindex);
12816	if (err < 0) {
12817	NL_SET_ERR_MSG_FMT(extack,
12818	"The ifindex %d is not available in the target netns",
12819	new_ifindex);
12820	goto out;
12821	}
12822	} else {
12823	/* If there is an ifindex conflict assign a new one */
12824	err = dev_index_reserve(net, dev->ifindex);
12825	if (err == -EBUSY)
12826	err = dev_index_reserve(net, 0);
12827	if (err < 0) {
12828	NL_SET_ERR_MSG(extack,
12829	"Unable to allocate a new ifindex in the target netns");
12830	goto out;
12831	}
12832	new_ifindex = err;
12833	}
12834
12835	/*
12836	* And now a mini version of register_netdevice unregister_netdevice.
12837	*/
12838
12839	netdev_lock_ops(dev);
12840	/* If device is running close it first. */
12841	netif_close(dev);
12842	/* And unlink it from device chain */
12843	unlist_netdevice(dev);
12844
12845	if (!netdev_need_ops_lock(dev))
12846	netdev_lock(dev);
12847	dev->moving_ns = true;
12848	netdev_unlock(dev);
12849
12850	synchronize_net();
12851
12852	/* Shutdown queueing discipline. */
12853	netdev_lock_ops(dev);
12854	dev_shutdown(dev);
12855	netdev_unlock_ops(dev);
12856
12857	/* Notify protocols, that we are about to destroy
12858	* this device. They should clean all the things.
12859	*
12860	* Note that dev->reg_state stays at NETREG_REGISTERED.
12861	* This is wanted because this way 8021q and macvlan know
12862	* the device is just moving and can keep their slaves up.
12863	*/
12864	call_netdevice_notifiers(NETDEV_UNREGISTER, dev);
12865	rcu_barrier();
12866
12867	new_nsid = peernet2id_alloc(dev_net(dev), net, GFP_KERNEL);
12868
12869	rtmsg_ifinfo_newnet(RTM_DELLINK, dev, ~0U, GFP_KERNEL, &new_nsid,
12870	new_ifindex);
12871
12872	/*
12873	* Flush the unicast and multicast chains
12874	*/
12875	dev_uc_flush(dev);
12876	dev_mc_flush(dev);
12877
12878	/* Send a netdev-removed uevent to the old namespace */
12879	kobject_uevent(&dev->dev.kobj, KOBJ_REMOVE);
12880	netdev_adjacent_del_links(dev);
12881
12882	/* Move per-net netdevice notifiers that are following the netdevice */
12883	move_netdevice_notifiers_dev_net(dev, net);
12884
12885	/* Actually switch the network namespace */
12886	netdev_lock(dev);
12887	dev_net_set(dev, net);
12888	netdev_unlock(dev);
12889	dev->ifindex = new_ifindex;
12890
12891	#ifdef CONFIG_DEBUG_NET_SMALL_RTNL
12892	unregister_netdevice_move_net(net_old, net, dev);
12893	#endif
12894
12895	if (new_name[0]) {
12896	/* Rename the netdev to prepared name */
12897	write_seqlock_bh(&netdev_rename_lock);
12898	strscpy(dev->name, new_name, IFNAMSIZ);
12899	write_sequnlock_bh(&netdev_rename_lock);
12900	}
12901
12902	/* Fixup kobjects */
12903	dev_set_uevent_suppress(&dev->dev, 1);
12904	err = device_rename(&dev->dev, dev->name);
12905	dev_set_uevent_suppress(&dev->dev, 0);
12906	WARN_ON(err);
12907
12908	/* Send a netdev-add uevent to the new namespace */
12909	kobject_uevent(&dev->dev.kobj, KOBJ_ADD);
12910	netdev_adjacent_add_links(dev);
12911
12912	/* Adapt owner in case owning user namespace of target network
12913	* namespace is different from the original one.
12914	*/
12915	err = netdev_change_owner(dev, net_old, net);
12916	WARN_ON(err);
12917
12918	netdev_lock(dev);
12919	dev->moving_ns = false;
12920	if (!netdev_need_ops_lock(dev))
12921	netdev_unlock(dev);
12922
12923	/* Add the device back in the hashes */
12924	list_netdevice(dev);
12925	/* Notify protocols, that a new device appeared. */
12926	call_netdevice_notifiers(NETDEV_REGISTER, dev);
12927	netdev_unlock_ops(dev);
12928
12929	/*
12930	* Prevent userspace races by waiting until the network
12931	* device is fully setup before sending notifications.
12932	*/
12933	rtmsg_ifinfo(RTM_NEWLINK, dev, ~0U, GFP_KERNEL, 0, NULL);
12934
12935	synchronize_net();
12936	err = 0;
12937	out:
12938	return err;
12939	}
12940
12941	static int dev_cpu_dead(unsigned int oldcpu)
12942	{
12943	struct sk_buff **list_skb;
12944	struct sk_buff *skb;
12945	unsigned int cpu;
12946	int node;
12947	struct softnet_data sd, oldsd, *remsd = NULL;
12948
12949	local_irq_disable();
12950	cpu = smp_processor_id();
12951	sd = &per_cpu(softnet_data, cpu);
12952	oldsd = &per_cpu(softnet_data, oldcpu);
12953
12954	/* Find end of our completion_queue. */
12955	list_skb = &sd->completion_queue;
12956	while (*list_skb)
12957	list_skb = &(*list_skb)->next;
12958	/* Append completion queue from offline CPU. */
12959	*list_skb = oldsd->completion_queue;
12960	oldsd->completion_queue = NULL;
12961
12962	/* Append output queue from offline CPU. */
12963	if (oldsd->output_queue) {
12964	*sd->output_queue_tailp = oldsd->output_queue;
12965	sd->output_queue_tailp = oldsd->output_queue_tailp;
12966	oldsd->output_queue = NULL;
12967	oldsd->output_queue_tailp = &oldsd->output_queue;
12968	}
12969	/* Append NAPI poll list from offline CPU, with one exception :
12970	* process_backlog() must be called by cpu owning percpu backlog.
12971	* We properly handle process_queue & input_pkt_queue later.
12972	*/
12973	while (!list_empty(&oldsd->poll_list)) {
12974	struct napi_struct *napi = list_first_entry(&oldsd->poll_list,
12975	struct napi_struct,
12976	poll_list);
12977
12978	list_del_init(&napi->poll_list);
12979	if (napi->poll == process_backlog)
12980	napi->state &= NAPIF_STATE_THREADED;
12981	else
12982	____napi_schedule(sd, napi);
12983	}
12984
12985	raise_softirq_irqoff(NET_TX_SOFTIRQ);
12986	local_irq_enable();
12987
12988	if (!use_backlog_threads()) {
12989	#ifdef CONFIG_RPS
12990	remsd = oldsd->rps_ipi_list;
12991	oldsd->rps_ipi_list = NULL;
12992	#endif
12993	/* send out pending IPI's on offline CPU */
12994	net_rps_send_ipi(remsd);
12995	}
12996
12997	/* Process offline CPU's input_pkt_queue */
12998	while ((skb = __skb_dequeue(&oldsd->process_queue))) {
12999	netif_rx(skb);
13000	rps_input_queue_head_incr(oldsd);
13001	}
13002	while ((skb = skb_dequeue(&oldsd->input_pkt_queue))) {
13003	netif_rx(skb);
13004	rps_input_queue_head_incr(oldsd);
13005	}
13006
13007	for_each_node(node)
13008	skb_defer_node_flush(per_cpu_ptr(net_hotdata.skb_defer_nodes,
13009	oldcpu) + node);
13010	node = cpu_to_node(oldcpu);
13011	if (node_possible(node) &&
13012	!cpumask_intersects(cpumask_of_node(node), cpu_online_mask)) {
13013	for_each_possible_cpu(cpu)
13014	skb_defer_node_flush(per_cpu_ptr(net_hotdata.skb_defer_nodes,
13015	cpu) + node);
13016	}
13017
13018	return 0;
13019	}
13020
13021	/**
13022	* netdev_increment_features - increment feature set by one
13023	* @all: current feature set
13024	* @one: new feature set
13025	* @mask: mask feature set
13026	*
13027	* Computes a new feature set after adding a device with feature set
13028	* @one to the master device with current feature set @all. Will not
13029	* enable anything that is off in @mask. Returns the new feature set.
13030	*/
13031	netdev_features_t netdev_increment_features(netdev_features_t all,
13032	netdev_features_t one, netdev_features_t mask)
13033	{
13034	if (mask & NETIF_F_HW_CSUM)
13035	mask \|= NETIF_F_CSUM_MASK;
13036	mask \|= NETIF_F_VLAN_CHALLENGED;
13037
13038	all \|= one & (NETIF_F_ONE_FOR_ALL \| NETIF_F_CSUM_MASK) & mask;
13039	all &= one \| ~NETIF_F_ALL_FOR_ALL;
13040
13041	/* If one device supports hw checksumming, set for all. */
13042	if (all & NETIF_F_HW_CSUM)
13043	all &= ~(NETIF_F_CSUM_MASK & ~NETIF_F_HW_CSUM);
13044
13045	return all;
13046	}
13047	EXPORT_SYMBOL(netdev_increment_features);
13048
13049	/**
13050	* netdev_compute_master_upper_features - compute feature from lowers
13051	* @dev: the upper device
13052	* @update_header: whether to update upper device's header_len/headroom/tailroom
13053	*
13054	* Recompute the upper device's feature based on all lower devices.
13055	*/
13056	void netdev_compute_master_upper_features(struct net_device *dev, bool update_header)
13057	{
13058	unsigned int dst_release_flag = IFF_XMIT_DST_RELEASE \| IFF_XMIT_DST_RELEASE_PERM;
13059	netdev_features_t gso_partial_features = MASTER_UPPER_DEV_GSO_PARTIAL_FEATURES;
13060	netdev_features_t xfrm_features = MASTER_UPPER_DEV_XFRM_FEATURES;
13061	netdev_features_t mpls_features = MASTER_UPPER_DEV_MPLS_FEATURES;
13062	netdev_features_t vlan_features = MASTER_UPPER_DEV_VLAN_FEATURES;
13063	netdev_features_t enc_features = MASTER_UPPER_DEV_ENC_FEATURES;
13064	unsigned short max_header_len = ETH_HLEN;
13065	unsigned int tso_max_size = TSO_MAX_SIZE;
13066	unsigned short max_headroom = 0;
13067	unsigned short max_tailroom = 0;
13068	u16 tso_max_segs = TSO_MAX_SEGS;
13069	struct net_device *lower_dev;
13070	struct list_head *iter;
13071
13072	mpls_features = netdev_base_features(mpls_features);
13073	vlan_features = netdev_base_features(vlan_features);
13074	enc_features = netdev_base_features(enc_features);
13075
13076	netdev_for_each_lower_dev(dev, lower_dev, iter) {
13077	gso_partial_features = netdev_increment_features(gso_partial_features,
13078	lower_dev->gso_partial_features,
13079	MASTER_UPPER_DEV_GSO_PARTIAL_FEATURES);
13080
13081	vlan_features = netdev_increment_features(vlan_features,
13082	lower_dev->vlan_features,
13083	MASTER_UPPER_DEV_VLAN_FEATURES);
13084
13085	enc_features = netdev_increment_features(enc_features,
13086	lower_dev->hw_enc_features,
13087	MASTER_UPPER_DEV_ENC_FEATURES);
13088
13089	if (IS_ENABLED(CONFIG_XFRM_OFFLOAD))
13090	xfrm_features = netdev_increment_features(xfrm_features,
13091	lower_dev->hw_enc_features,
13092	MASTER_UPPER_DEV_XFRM_FEATURES);
13093
13094	mpls_features = netdev_increment_features(mpls_features,
13095	lower_dev->mpls_features,
13096	MASTER_UPPER_DEV_MPLS_FEATURES);
13097
13098	dst_release_flag &= lower_dev->priv_flags;
13099
13100	if (update_header) {
13101	max_header_len = max(max_header_len, lower_dev->hard_header_len);
13102	max_headroom = max(max_headroom, lower_dev->needed_headroom);
13103	max_tailroom = max(max_tailroom, lower_dev->needed_tailroom);
13104	}
13105
13106	tso_max_size = min(tso_max_size, lower_dev->tso_max_size);
13107	tso_max_segs = min(tso_max_segs, lower_dev->tso_max_segs);
13108	}
13109
13110	dev->gso_partial_features = gso_partial_features;
13111	dev->vlan_features = vlan_features;
13112	dev->hw_enc_features = enc_features \| NETIF_F_GSO_ENCAP_ALL \|
13113	NETIF_F_HW_VLAN_CTAG_TX \|
13114	NETIF_F_HW_VLAN_STAG_TX;
13115	if (IS_ENABLED(CONFIG_XFRM_OFFLOAD))
13116	dev->hw_enc_features \|= xfrm_features;
13117	dev->mpls_features = mpls_features;
13118
13119	dev->priv_flags &= ~IFF_XMIT_DST_RELEASE;
13120	if ((dev->priv_flags & IFF_XMIT_DST_RELEASE_PERM) &&
13121	dst_release_flag == (IFF_XMIT_DST_RELEASE \| IFF_XMIT_DST_RELEASE_PERM))
13122	dev->priv_flags \|= IFF_XMIT_DST_RELEASE;
13123
13124	if (update_header) {
13125	dev->hard_header_len = max_header_len;
13126	dev->needed_headroom = max_headroom;
13127	dev->needed_tailroom = max_tailroom;
13128	}
13129
13130	netif_set_tso_max_segs(dev, tso_max_segs);
13131	netif_set_tso_max_size(dev, tso_max_size);
13132
13133	netdev_change_features(dev);
13134	}
13135	EXPORT_SYMBOL(netdev_compute_master_upper_features);
13136
13137	static struct hlist_head * __net_init netdev_create_hash(void)
13138	{
13139	int i;
13140	struct hlist_head *hash;
13141
13142	hash = kmalloc_objs(*hash, NETDEV_HASHENTRIES);
13143	if (hash != NULL)
13144	for (i = 0; i < NETDEV_HASHENTRIES; i++)
13145	INIT_HLIST_HEAD(&hash[i]);
13146
13147	return hash;
13148	}
13149
13150	/* Initialize per network namespace state */
13151	static int __net_init netdev_init(struct net *net)
13152	{
13153	BUILD_BUG_ON(GRO_HASH_BUCKETS >
13154	BITS_PER_BYTE * sizeof_field(struct gro_node, bitmask));
13155
13156	INIT_LIST_HEAD(&net->dev_base_head);
13157
13158	net->dev_name_head = netdev_create_hash();
13159	if (net->dev_name_head == NULL)
13160	goto err_name;
13161
13162	net->dev_index_head = netdev_create_hash();
13163	if (net->dev_index_head == NULL)
13164	goto err_idx;
13165
13166	xa_init_flags(&net->dev_by_index, XA_FLAGS_ALLOC1);
13167
13168	RAW_INIT_NOTIFIER_HEAD(&net->netdev_chain);
13169
13170	return 0;
13171
13172	err_idx:
13173	kfree(net->dev_name_head);
13174	err_name:
13175	return -ENOMEM;
13176	}
13177
13178	/**
13179	* netdev_drivername - network driver for the device
13180	* @dev: network device
13181	*
13182	* Determine network driver for device.
13183	*/
13184	const char netdev_drivername(const struct net_device dev)
13185	{
13186	const struct device_driver *driver;
13187	const struct device *parent;
13188	const char *empty = "";
13189
13190	parent = dev->dev.parent;
13191	if (!parent)
13192	return empty;
13193
13194	driver = parent->driver;
13195	if (driver && driver->name)
13196	return driver->name;
13197	return empty;
13198	}
13199
13200	static void __netdev_printk(const char level, const struct net_device dev,
13201	struct va_format *vaf)
13202	{
13203	if (dev && dev->dev.parent) {
13204	dev_printk_emit(level[1] - '0',
13205	dev->dev.parent,
13206	"%s %s %s%s: %pV",
13207	dev_driver_string(dev->dev.parent),
13208	dev_name(dev->dev.parent),
13209	netdev_name(dev), netdev_reg_state(dev),
13210	vaf);
13211	} else if (dev) {
13212	printk("%s%s%s: %pV",
13213	level, netdev_name(dev), netdev_reg_state(dev), vaf);
13214	} else {
13215	printk("%s(NULL net_device): %pV", level, vaf);
13216	}
13217	}
13218
13219	void netdev_printk(const char level, const struct net_device dev,
13220	const char *format, ...)
13221	{
13222	struct va_format vaf;
13223	va_list args;
13224
13225	va_start(args, format);
13226
13227	vaf.fmt = format;
13228	vaf.va = &args;
13229
13230	__netdev_printk(level, dev, &vaf);
13231
13232	va_end(args);
13233	}
13234	EXPORT_SYMBOL(netdev_printk);
13235
13236	#define define_netdev_printk_level(func, level) \
13237	void func(const struct net_device dev, const char fmt, ...) \
13238	{ \
13239	struct va_format vaf; \
13240	va_list args; \
13241	\
13242	va_start(args, fmt); \
13243	\
13244	vaf.fmt = fmt; \
13245	vaf.va = &args; \
13246	\
13247	__netdev_printk(level, dev, &vaf); \
13248	\
13249	va_end(args); \
13250	} \
13251	EXPORT_SYMBOL(func);
13252
13253	define_netdev_printk_level(netdev_emerg, KERN_EMERG);
13254	define_netdev_printk_level(netdev_alert, KERN_ALERT);
13255	define_netdev_printk_level(netdev_crit, KERN_CRIT);
13256	define_netdev_printk_level(netdev_err, KERN_ERR);
13257	define_netdev_printk_level(netdev_warn, KERN_WARNING);
13258	define_netdev_printk_level(netdev_notice, KERN_NOTICE);
13259	define_netdev_printk_level(netdev_info, KERN_INFO);
13260
13261	static void __net_exit netdev_exit(struct net *net)
13262	{
13263	kfree(net->dev_name_head);
13264	kfree(net->dev_index_head);
13265	xa_destroy(&net->dev_by_index);
13266	if (net != &init_net)
13267	WARN_ON_ONCE(!list_empty(&net->dev_base_head));
13268	}
13269
13270	static struct pernet_operations __net_initdata netdev_net_ops = {
13271	.init = netdev_init,
13272	.exit = netdev_exit,
13273	};
13274
13275	static void __net_exit default_device_exit_net(struct net *net)
13276	{
13277	struct net_device dev, aux;
13278	/*
13279	* Push all migratable network devices back to the
13280	* initial network namespace
13281	*/
13282
13283	for_each_netdev_safe(net, dev, aux) {
13284	int err;
13285	char fb_name[IFNAMSIZ];
13286
13287	/* Ignore unmoveable devices (i.e. loopback) */
13288	if (dev->netns_immutable)
13289	continue;
13290
13291	/* Leave virtual devices for the generic cleanup */
13292	if (dev->rtnl_link_ops && !dev->rtnl_link_ops->netns_refund)
13293	continue;
13294
13295	/* Push remaining network devices to init_net */
13296	snprintf(fb_name, IFNAMSIZ, "dev%d", dev->ifindex);
13297	if (netdev_name_in_use(&init_net, fb_name))
13298	snprintf(fb_name, IFNAMSIZ, "dev%%d");
13299
13300	err = dev_change_net_namespace(dev, &init_net, fb_name);
13301	if (err) {
13302	pr_emerg("%s: failed to move %s to init_net: %d\n",
13303	__func__, dev->name, err);
13304	BUG();
13305	}
13306	}
13307	}
13308
13309	static void __net_exit default_device_exit_batch(struct list_head *net_list)
13310	{
13311	/* At exit all network devices most be removed from a network
13312	* namespace. Do this in the reverse order of registration.
13313	* Do this across as many network namespaces as possible to
13314	* improve batching efficiency.
13315	*/
13316	struct net_device *dev;
13317	struct net *net;
13318	LIST_HEAD(dev_kill_list);
13319
13320	rtnl_lock();
13321
13322	__rtnl_net_lock(&init_net);
13323
13324	list_for_each_entry(net, net_list, exit_list) {
13325	__rtnl_net_lock(net);
13326	default_device_exit_net(net);
13327	__rtnl_net_unlock(net);
13328
13329	cond_resched();
13330	}
13331
13332	__rtnl_net_unlock(&init_net);
13333
13334	list_for_each_entry(net, net_list, exit_list) {
13335	__rtnl_net_lock(net);
13336
13337	for_each_netdev_reverse(net, dev) {
13338	if (dev->rtnl_link_ops && dev->rtnl_link_ops->dellink)
13339	dev->rtnl_link_ops->dellink(dev, &dev_kill_list);
13340	else
13341	unregister_netdevice_queue(dev, &dev_kill_list);
13342	}
13343
13344	unregister_netdevice_queue_many_net(net, &dev_kill_list);
13345	__rtnl_net_unlock(net);
13346	}
13347	unregister_netdevice_many(&dev_kill_list);
13348	rtnl_unlock();
13349
13350	rtnl_net_flush_workqueue();
13351	}
13352
13353	static struct pernet_operations __net_initdata default_device_ops = {
13354	.exit_batch = default_device_exit_batch,
13355	};
13356
13357	static void __init net_dev_struct_check(void)
13358	{
13359	/* TX read-mostly hotpath */
13360	CACHELINE_ASSERT_GROUP_MEMBER(struct net_device, net_device_read_tx, priv_flags_fast);
13361	CACHELINE_ASSERT_GROUP_MEMBER(struct net_device, net_device_read_tx, netdev_ops);
13362	CACHELINE_ASSERT_GROUP_MEMBER(struct net_device, net_device_read_tx, header_ops);
13363	CACHELINE_ASSERT_GROUP_MEMBER(struct net_device, net_device_read_tx, _tx);
13364	CACHELINE_ASSERT_GROUP_MEMBER(struct net_device, net_device_read_tx, real_num_tx_queues);
13365	CACHELINE_ASSERT_GROUP_MEMBER(struct net_device, net_device_read_tx, gso_max_size);
13366	CACHELINE_ASSERT_GROUP_MEMBER(struct net_device, net_device_read_tx, gso_ipv4_max_size);
13367	CACHELINE_ASSERT_GROUP_MEMBER(struct net_device, net_device_read_tx, gso_max_segs);
13368	CACHELINE_ASSERT_GROUP_MEMBER(struct net_device, net_device_read_tx, gso_partial_features);
13369	CACHELINE_ASSERT_GROUP_MEMBER(struct net_device, net_device_read_tx, num_tc);
13370	CACHELINE_ASSERT_GROUP_MEMBER(struct net_device, net_device_read_tx, mtu);
13371	CACHELINE_ASSERT_GROUP_MEMBER(struct net_device, net_device_read_tx, needed_headroom);
13372	CACHELINE_ASSERT_GROUP_MEMBER(struct net_device, net_device_read_tx, tc_to_txq);
13373	#ifdef CONFIG_XPS
13374	CACHELINE_ASSERT_GROUP_MEMBER(struct net_device, net_device_read_tx, xps_maps);
13375	#endif
13376	#ifdef CONFIG_NETFILTER_EGRESS
13377	CACHELINE_ASSERT_GROUP_MEMBER(struct net_device, net_device_read_tx, nf_hooks_egress);
13378	#endif
13379	#ifdef CONFIG_NET_XGRESS
13380	CACHELINE_ASSERT_GROUP_MEMBER(struct net_device, net_device_read_tx, tcx_egress);
13381	#endif
13382	CACHELINE_ASSERT_GROUP_SIZE(struct net_device, net_device_read_tx, 160);
13383
13384	/* TXRX read-mostly hotpath */
13385	CACHELINE_ASSERT_GROUP_MEMBER(struct net_device, net_device_read_txrx, lstats);
13386	CACHELINE_ASSERT_GROUP_MEMBER(struct net_device, net_device_read_txrx, state);
13387	CACHELINE_ASSERT_GROUP_MEMBER(struct net_device, net_device_read_txrx, flags);
13388	CACHELINE_ASSERT_GROUP_MEMBER(struct net_device, net_device_read_txrx, hard_header_len);
13389	CACHELINE_ASSERT_GROUP_MEMBER(struct net_device, net_device_read_txrx, features);
13390	CACHELINE_ASSERT_GROUP_MEMBER(struct net_device, net_device_read_txrx, ip6_ptr);
13391	CACHELINE_ASSERT_GROUP_SIZE(struct net_device, net_device_read_txrx, 46);
13392
13393	/* RX read-mostly hotpath */
13394	CACHELINE_ASSERT_GROUP_MEMBER(struct net_device, net_device_read_rx, ptype_specific);
13395	CACHELINE_ASSERT_GROUP_MEMBER(struct net_device, net_device_read_rx, ifindex);
13396	CACHELINE_ASSERT_GROUP_MEMBER(struct net_device, net_device_read_rx, real_num_rx_queues);
13397	CACHELINE_ASSERT_GROUP_MEMBER(struct net_device, net_device_read_rx, _rx);
13398	CACHELINE_ASSERT_GROUP_MEMBER(struct net_device, net_device_read_rx, gro_max_size);
13399	CACHELINE_ASSERT_GROUP_MEMBER(struct net_device, net_device_read_rx, gro_ipv4_max_size);
13400	CACHELINE_ASSERT_GROUP_MEMBER(struct net_device, net_device_read_rx, rx_handler);
13401	CACHELINE_ASSERT_GROUP_MEMBER(struct net_device, net_device_read_rx, rx_handler_data);
13402	CACHELINE_ASSERT_GROUP_MEMBER(struct net_device, net_device_read_rx, nd_net);
13403	#ifdef CONFIG_NETPOLL
13404	CACHELINE_ASSERT_GROUP_MEMBER(struct net_device, net_device_read_rx, npinfo);
13405	#endif
13406	#ifdef CONFIG_NET_XGRESS
13407	CACHELINE_ASSERT_GROUP_MEMBER(struct net_device, net_device_read_rx, tcx_ingress);
13408	#endif
13409	CACHELINE_ASSERT_GROUP_SIZE(struct net_device, net_device_read_rx, 92);
13410	}
13411
13412	/*
13413	* Initialize the DEV module. At boot time this walks the device list and
13414	* unhooks any devices that fail to initialise (normally hardware not
13415	* present) and leaves us with a valid list of present and active devices.
13416	*
13417	*/
13418
13419	/* We allocate 256 pages for each CPU if PAGE_SHIFT is 12 */
13420	#define SYSTEM_PERCPU_PAGE_POOL_SIZE ((1 << 20) / PAGE_SIZE)
13421
13422	static int net_page_pool_create(int cpuid)
13423	{
13424	#if IS_ENABLED(CONFIG_PAGE_POOL)
13425	struct page_pool_params page_pool_params = {
13426	.pool_size = SYSTEM_PERCPU_PAGE_POOL_SIZE,
13427	.flags = PP_FLAG_SYSTEM_POOL,
13428	.nid = cpu_to_mem(cpuid),
13429	};
13430	struct page_pool *pp_ptr;
13431	int err;
13432
13433	pp_ptr = page_pool_create_percpu(&page_pool_params, cpuid);
13434	if (IS_ERR(pp_ptr))
13435	return -ENOMEM;
13436
13437	err = xdp_reg_page_pool(pp_ptr);
13438	if (err) {
13439	page_pool_destroy(pp_ptr);
13440	return err;
13441	}
13442
13443	per_cpu(system_page_pool.pool, cpuid) = pp_ptr;
13444	#endif
13445	return 0;
13446	}
13447
13448	static int backlog_napi_should_run(unsigned int cpu)
13449	{
13450	struct softnet_data *sd = per_cpu_ptr(&softnet_data, cpu);
13451	struct napi_struct *napi = &sd->backlog;
13452
13453	return test_bit(NAPI_STATE_SCHED_THREADED, &napi->state);
13454	}
13455
13456	static void run_backlog_napi(unsigned int cpu)
13457	{
13458	struct softnet_data *sd = per_cpu_ptr(&softnet_data, cpu);
13459
13460	napi_threaded_poll_loop(&sd->backlog, NULL);
13461	}
13462
13463	static void backlog_napi_setup(unsigned int cpu)
13464	{
13465	struct softnet_data *sd = per_cpu_ptr(&softnet_data, cpu);
13466	struct napi_struct *napi = &sd->backlog;
13467
13468	napi->thread = this_cpu_read(backlog_napi);
13469	set_bit(NAPI_STATE_THREADED, &napi->state);
13470	}
13471
13472	static struct smp_hotplug_thread backlog_threads = {
13473	.store = &backlog_napi,
13474	.thread_should_run = backlog_napi_should_run,
13475	.thread_fn = run_backlog_napi,
13476	.thread_comm = "backlog_napi/%u",
13477	.setup = backlog_napi_setup,
13478	};
13479
13480	/*
13481	* This is called single threaded during boot, so no need
13482	* to take the rtnl semaphore.
13483	*/
13484	static int __init net_dev_init(void)
13485	{
13486	int i, rc = -ENOMEM;
13487
13488	BUG_ON(!dev_boot_phase);
13489
13490	net_dev_struct_check();
13491
13492	if (dev_proc_init())
13493	goto out;
13494
13495	if (netdev_kobject_init())
13496	goto out;
13497
13498	for (i = 0; i < PTYPE_HASH_SIZE; i++)
13499	INIT_LIST_HEAD(&ptype_base[i]);
13500
13501	if (register_pernet_subsys(&netdev_net_ops))
13502	goto out;
13503
13504	/*
13505	* Initialise the packet receive queues.
13506	*/
13507
13508	flush_backlogs_fallback = flush_backlogs_alloc();
13509	if (!flush_backlogs_fallback)
13510	goto out;
13511
13512	for_each_possible_cpu(i) {
13513	struct softnet_data *sd = &per_cpu(softnet_data, i);
13514
13515	skb_queue_head_init(&sd->input_pkt_queue);
13516	skb_queue_head_init(&sd->process_queue);
13517	#ifdef CONFIG_XFRM_OFFLOAD
13518	skb_queue_head_init(&sd->xfrm_backlog);
13519	#endif
13520	INIT_LIST_HEAD(&sd->poll_list);
13521	sd->output_queue_tailp = &sd->output_queue;
13522	#ifdef CONFIG_RPS
13523	INIT_CSD(&sd->csd, rps_trigger_softirq, sd);
13524	sd->cpu = i;
13525	#endif
13526	INIT_CSD(&sd->defer_csd, trigger_rx_softirq, sd);
13527
13528	gro_init(&sd->backlog.gro);
13529	sd->backlog.poll = process_backlog;
13530	sd->backlog.weight = weight_p;
13531	INIT_LIST_HEAD(&sd->backlog.poll_list);
13532
13533	if (net_page_pool_create(i))
13534	goto out;
13535	}
13536	net_hotdata.skb_defer_nodes =
13537	__alloc_percpu(sizeof(struct skb_defer_node) * nr_node_ids,
13538	__alignof__(struct skb_defer_node));
13539	if (!net_hotdata.skb_defer_nodes)
13540	goto out;
13541	if (use_backlog_threads())
13542	smpboot_register_percpu_thread(&backlog_threads);
13543
13544	dev_boot_phase = 0;
13545
13546	/* The loopback device is special if any other network devices
13547	* is present in a network namespace the loopback device must
13548	* be present. Since we now dynamically allocate and free the
13549	* loopback device ensure this invariant is maintained by
13550	* keeping the loopback device as the first device on the
13551	* list of network devices. Ensuring the loopback devices
13552	* is the first device that appears and the last network device
13553	* that disappears.
13554	*/
13555	if (register_pernet_device(&loopback_net_ops))
13556	goto out;
13557
13558	if (register_pernet_device(&default_device_ops))
13559	goto out;
13560
13561	open_softirq(NET_TX_SOFTIRQ, net_tx_action);
13562	open_softirq(NET_RX_SOFTIRQ, net_rx_action);
13563
13564	rc = cpuhp_setup_state_nocalls(CPUHP_NET_DEV_DEAD, "net/dev:dead",
13565	NULL, dev_cpu_dead);
13566	WARN_ON(rc < 0);
13567	rc = 0;
13568
13569	/* avoid static key IPIs to isolated CPUs */
13570	if (housekeeping_enabled(HK_TYPE_MISC))
13571	net_enable_timestamp();
13572	out:
13573	if (rc < 0) {
13574	for_each_possible_cpu(i) {
13575	struct page_pool *pp_ptr;
13576
13577	pp_ptr = per_cpu(system_page_pool.pool, i);
13578	if (!pp_ptr)
13579	continue;
13580
13581	xdp_unreg_page_pool(pp_ptr);
13582	page_pool_destroy(pp_ptr);
13583	per_cpu(system_page_pool.pool, i) = NULL;
13584	}
13585	}
13586
13587	return rc;
13588	}
13589
13590	subsys_initcall(net_dev_init);