projects/linux·blob master · refs

419 lines · 13.1 KiB

1/* SPDX-License-Identifier: GPL-2.0-or-later */
2#ifndef _NET_CORE_DEV_H
3#define _NET_CORE_DEV_H
4
5#include <linux/cleanup.h>
6#include <linux/types.h>
7#include <linux/rwsem.h>
8#include <linux/netdevice.h>
9#include <net/netdev_lock.h>
10
11struct net;
12struct netlink_ext_ack;
13struct netdev_queue_config;
14struct cpumask;
15struct pp_memory_provider_params;
16
17/* Random bits of netdevice that don't need to be exposed */
18#define FLOW_LIMIT_HISTORY (1 << 7) /* must be ^2 and !overflow buckets */
19struct sd_flow_limit {
20 struct rcu_head rcu;
21 unsigned int count;
22 u8 log_buckets;
23 unsigned int history_head;
24 u16 history[FLOW_LIMIT_HISTORY];
25 u8 buckets[];
26};
27
28extern int netdev_flow_limit_table_len;
29
30struct napi_struct *
31netdev_napi_by_id_lock(struct net *net, unsigned int napi_id);
32struct net_device *dev_get_by_napi_id(unsigned int napi_id);
33
34struct net_device *netdev_put_lock(struct net_device *dev, struct net *net,
35 netdevice_tracker *tracker);
36
37static inline struct net_device *
38__netdev_put_lock(struct net_device *dev, struct net *net)
39{
40 return netdev_put_lock(dev, net, NULL);
41}
42
43struct net_device *
44netdev_xa_find_lock(struct net *net, struct net_device *dev,
45 unsigned long *index);
46
47DEFINE_FREE(netdev_unlock, struct net_device *, if (_T) netdev_unlock(_T));
48
49#define for_each_netdev_lock_scoped(net, var_name, ifindex) \
50 for (struct net_device *var_name __free(netdev_unlock) = NULL; \
51 (var_name = netdev_xa_find_lock(net, var_name, &ifindex)); \
52 ifindex++)
53
54struct net_device *
55netdev_get_by_index_lock_ops_compat(struct net *net, int ifindex);
56struct net_device *
57netdev_xa_find_lock_ops_compat(struct net *net, struct net_device *dev,
58 unsigned long *index);
59
60DEFINE_FREE(netdev_unlock_ops_compat, struct net_device *,
61 if (_T) netdev_unlock_ops_compat(_T));
62
63#define for_each_netdev_lock_ops_compat_scoped(net, var_name, ifindex) \
64 for (struct net_device *var_name __free(netdev_unlock_ops_compat) = NULL; \
65 (var_name = netdev_xa_find_lock_ops_compat(net, var_name, \
66 &ifindex)); \
67 ifindex++)
68
69#ifdef CONFIG_PROC_FS
70int __init dev_proc_init(void);
71#else
72#define dev_proc_init() 0
73#endif
74
75void linkwatch_init_dev(struct net_device *dev);
76void linkwatch_run_queue(void);
77
78void dev_addr_flush(struct net_device *dev);
79int dev_addr_init(struct net_device *dev);
80void dev_addr_check(struct net_device *dev);
81void __hw_addr_flush(struct netdev_hw_addr_list *list);
82
83#if IS_ENABLED(CONFIG_NET_SHAPER)
84void net_shaper_flush_netdev(struct net_device *dev);
85void net_shaper_set_real_num_tx_queues(struct net_device *dev,
86 unsigned int txq);
87#else
88static inline void net_shaper_flush_netdev(struct net_device *dev) {}
89static inline void net_shaper_set_real_num_tx_queues(struct net_device *dev,
90 unsigned int txq) {}
91#endif
92
93/* sysctls not referred to from outside net/core/ */
94extern int netdev_unregister_timeout_secs;
95extern int weight_p;
96extern int dev_weight_rx_bias;
97extern int dev_weight_tx_bias;
98
99extern struct rw_semaphore dev_addr_sem;
100
101/* rtnl helpers */
102extern struct list_head net_todo_list;
103void netdev_run_todo(void);
104
105int netdev_queue_config_validate(struct net_device *dev, int rxq_idx,
106 struct netdev_queue_config *qcfg,
107 struct netlink_ext_ack *extack);
108
109bool netif_rxq_has_mp(struct net_device *dev, unsigned int rxq_idx);
110bool netif_rxq_is_leased(struct net_device *dev, unsigned int rxq_idx);
111bool netif_is_queue_leasee(const struct net_device *dev);
112
113void __netif_mp_uninstall_rxq(struct netdev_rx_queue *rxq,
114 const struct pp_memory_provider_params *p);
115
116void netif_rxq_cleanup_unlease(struct netdev_rx_queue *phys_rxq,
117 struct netdev_rx_queue *virt_rxq);
118
119/* netdev management, shared between various uAPI entry points */
120struct netdev_name_node {
121 struct hlist_node hlist;
122 struct list_head list;
123 struct net_device *dev;
124 const char *name;
125 struct rcu_head rcu;
126};
127
128int netdev_get_name(struct net *net, char *name, int ifindex);
129int netif_change_name(struct net_device *dev, const char *newname);
130int dev_change_name(struct net_device *dev, const char *newname);
131
132#define netdev_for_each_altname(dev, namenode) \
133 list_for_each_entry((namenode), &(dev)->name_node->list, list)
134#define netdev_for_each_altname_safe(dev, namenode, next) \
135 list_for_each_entry_safe((namenode), (next), &(dev)->name_node->list, \
136 list)
137
138int netdev_name_node_alt_create(struct net_device *dev, const char *name);
139int netdev_name_node_alt_destroy(struct net_device *dev, const char *name);
140
141int dev_validate_mtu(struct net_device *dev, int mtu,
142 struct netlink_ext_ack *extack);
143int netif_set_mtu_ext(struct net_device *dev, int new_mtu,
144 struct netlink_ext_ack *extack);
145
146int dev_get_phys_port_id(struct net_device *dev,
147 struct netdev_phys_item_id *ppid);
148int dev_get_phys_port_name(struct net_device *dev,
149 char *name, size_t len);
150
151int netif_change_proto_down(struct net_device *dev, bool proto_down);
152int dev_change_proto_down(struct net_device *dev, bool proto_down);
153void netdev_change_proto_down_reason_locked(struct net_device *dev,
154 unsigned long mask, u32 value);
155
156typedef int (*bpf_op_t)(struct net_device *dev, struct netdev_bpf *bpf);
157int dev_change_xdp_fd(struct net_device *dev, struct netlink_ext_ack *extack,
158 int fd, int expected_fd, u32 flags);
159
160int netif_change_tx_queue_len(struct net_device *dev, unsigned long new_len);
161int dev_change_tx_queue_len(struct net_device *dev, unsigned long new_len);
162void netif_set_group(struct net_device *dev, int new_group);
163void dev_set_group(struct net_device *dev, int new_group);
164int netif_change_carrier(struct net_device *dev, bool new_carrier);
165int dev_change_carrier(struct net_device *dev, bool new_carrier);
166
167void __dev_set_rx_mode(struct net_device *dev);
168int __dev_set_promiscuity(struct net_device *dev, int inc, bool notify);
169void netif_rx_mode_init(struct net_device *dev);
170void netif_rx_mode_run(struct net_device *dev);
171void netif_rx_mode_sync(struct net_device *dev);
172void netif_rx_mode_cancel_retry(struct net_device *dev);
173
174/* Events for the async netdev work, tracked in netdev->work_core_pending. */
175enum netdev_work_core {
176 NETDEV_WORK_RX_MODE = BIT(0), /* run the rx_mode update */
177};
178
179void __netdev_work_core_sched(struct net_device *dev, unsigned long event);
180unsigned long
181__netdev_work_core_cancel(struct net_device *dev, unsigned long mask);
182void netdev_work_cancel_all(struct net_device *dev);
183
184void __dev_notify_flags(struct net_device *dev, unsigned int old_flags,
185 unsigned int gchanges, u32 portid,
186 const struct nlmsghdr *nlh);
187
188void unregister_netdevice_many_notify(struct list_head *head,
189 u32 portid, const struct nlmsghdr *nlh);
190
191static inline void netif_set_up(struct net_device *dev, bool value)
192{
193 if (value)
194 dev->flags |= IFF_UP;
195 else
196 dev->flags &= ~IFF_UP;
197
198 if (!netdev_need_ops_lock(dev))
199 netdev_lock(dev);
200 dev->up = value;
201 if (!netdev_need_ops_lock(dev))
202 netdev_unlock(dev);
203}
204
205static inline void netif_set_gso_max_size(struct net_device *dev,
206 unsigned int size)
207{
208 /* dev->gso_max_size is read locklessly from sk_setup_caps() */
209 WRITE_ONCE(dev->gso_max_size, size);
210 if (size <= GSO_LEGACY_MAX_SIZE)
211 WRITE_ONCE(dev->gso_ipv4_max_size, size);
212}
213
214static inline void netif_set_gso_max_segs(struct net_device *dev,
215 unsigned int segs)
216{
217 /* dev->gso_max_segs is read locklessly from sk_setup_caps() */
218 WRITE_ONCE(dev->gso_max_segs, segs);
219}
220
221static inline void netif_set_gro_max_size(struct net_device *dev,
222 unsigned int size)
223{
224 /* This pairs with the READ_ONCE() in skb_gro_receive() */
225 WRITE_ONCE(dev->gro_max_size, size);
226 if (size <= GRO_LEGACY_MAX_SIZE)
227 WRITE_ONCE(dev->gro_ipv4_max_size, size);
228}
229
230static inline void netif_set_gso_ipv4_max_size(struct net_device *dev,
231 unsigned int size)
232{
233 /* dev->gso_ipv4_max_size is read locklessly from sk_setup_caps() */
234 WRITE_ONCE(dev->gso_ipv4_max_size, size);
235}
236
237static inline void netif_set_gro_ipv4_max_size(struct net_device *dev,
238 unsigned int size)
239{
240 /* This pairs with the READ_ONCE() in skb_gro_receive() */
241 WRITE_ONCE(dev->gro_ipv4_max_size, size);
242}
243
244/**
245 * napi_get_defer_hard_irqs - get the NAPI's defer_hard_irqs
246 * @n: napi struct to get the defer_hard_irqs field from
247 *
248 * Return: the per-NAPI value of the defar_hard_irqs field.
249 */
250static inline u32 napi_get_defer_hard_irqs(const struct napi_struct *n)
251{
252 return READ_ONCE(n->defer_hard_irqs);
253}
254
255/**
256 * napi_set_defer_hard_irqs - set the defer_hard_irqs for a napi
257 * @n: napi_struct to set the defer_hard_irqs field
258 * @defer: the value the field should be set to
259 */
260static inline void napi_set_defer_hard_irqs(struct napi_struct *n, u32 defer)
261{
262 WRITE_ONCE(n->defer_hard_irqs, defer);
263}
264
265/**
266 * netdev_set_defer_hard_irqs - set defer_hard_irqs for all NAPIs of a netdev
267 * @netdev: the net_device for which all NAPIs will have defer_hard_irqs set
268 * @defer: the defer_hard_irqs value to set
269 */
270static inline void netdev_set_defer_hard_irqs(struct net_device *netdev,
271 u32 defer)
272{
273 unsigned int count = max(netdev->num_rx_queues,
274 netdev->num_tx_queues);
275 struct napi_struct *napi;
276 int i;
277
278 WRITE_ONCE(netdev->napi_defer_hard_irqs, defer);
279 list_for_each_entry(napi, &netdev->napi_list, dev_list)
280 napi_set_defer_hard_irqs(napi, defer);
281
282 for (i = 0; i < count; i++)
283 netdev->napi_config[i].defer_hard_irqs = defer;
284}
285
286/**
287 * napi_get_gro_flush_timeout - get the gro_flush_timeout
288 * @n: napi struct to get the gro_flush_timeout from
289 *
290 * Return: the per-NAPI value of the gro_flush_timeout field.
291 */
292static inline unsigned long
293napi_get_gro_flush_timeout(const struct napi_struct *n)
294{
295 return READ_ONCE(n->gro_flush_timeout);
296}
297
298/**
299 * napi_set_gro_flush_timeout - set the gro_flush_timeout for a napi
300 * @n: napi struct to set the gro_flush_timeout
301 * @timeout: timeout value to set
302 *
303 * napi_set_gro_flush_timeout sets the per-NAPI gro_flush_timeout
304 */
305static inline void napi_set_gro_flush_timeout(struct napi_struct *n,
306 unsigned long timeout)
307{
308 WRITE_ONCE(n->gro_flush_timeout, timeout);
309}
310
311/**
312 * netdev_set_gro_flush_timeout - set gro_flush_timeout of a netdev's NAPIs
313 * @netdev: the net_device for which all NAPIs will have gro_flush_timeout set
314 * @timeout: the timeout value to set
315 */
316static inline void netdev_set_gro_flush_timeout(struct net_device *netdev,
317 unsigned long timeout)
318{
319 unsigned int count = max(netdev->num_rx_queues,
320 netdev->num_tx_queues);
321 struct napi_struct *napi;
322 int i;
323
324 WRITE_ONCE(netdev->gro_flush_timeout, timeout);
325 list_for_each_entry(napi, &netdev->napi_list, dev_list)
326 napi_set_gro_flush_timeout(napi, timeout);
327
328 for (i = 0; i < count; i++)
329 netdev->napi_config[i].gro_flush_timeout = timeout;
330}
331
332/**
333 * napi_get_irq_suspend_timeout - get the irq_suspend_timeout
334 * @n: napi struct to get the irq_suspend_timeout from
335 *
336 * Return: the per-NAPI value of the irq_suspend_timeout field.
337 */
338static inline unsigned long
339napi_get_irq_suspend_timeout(const struct napi_struct *n)
340{
341 return READ_ONCE(n->irq_suspend_timeout);
342}
343
344/**
345 * napi_set_irq_suspend_timeout - set the irq_suspend_timeout for a napi
346 * @n: napi struct to set the irq_suspend_timeout
347 * @timeout: timeout value to set
348 *
349 * napi_set_irq_suspend_timeout sets the per-NAPI irq_suspend_timeout
350 */
351static inline void napi_set_irq_suspend_timeout(struct napi_struct *n,
352 unsigned long timeout)
353{
354 WRITE_ONCE(n->irq_suspend_timeout, timeout);
355}
356
357static inline enum netdev_napi_threaded napi_get_threaded(struct napi_struct *n)
358{
359 if (test_bit(NAPI_STATE_THREADED_BUSY_POLL, &n->state))
360 return NETDEV_NAPI_THREADED_BUSY_POLL;
361
362 if (test_bit(NAPI_STATE_THREADED, &n->state))
363 return NETDEV_NAPI_THREADED_ENABLED;
364
365 return NETDEV_NAPI_THREADED_DISABLED;
366}
367
368static inline enum netdev_napi_threaded
369napi_get_threaded_config(struct net_device *dev, struct napi_struct *n)
370{
371 if (n->config)
372 return n->config->threaded;
373 return dev->threaded;
374}
375
376int napi_set_threaded(struct napi_struct *n,
377 enum netdev_napi_threaded threaded);
378
379int netif_set_threaded(struct net_device *dev,
380 enum netdev_napi_threaded threaded);
381
382int rps_cpumask_housekeeping(struct cpumask *mask);
383
384#if defined(CONFIG_DEBUG_NET) && defined(CONFIG_BPF_SYSCALL)
385void xdp_do_check_flushed(struct napi_struct *napi);
386#else
387static inline void xdp_do_check_flushed(struct napi_struct *napi) { }
388#endif
389
390/* Best effort check that NAPI is not idle (can't be scheduled to run) */
391static inline void napi_assert_will_not_race(const struct napi_struct *napi)
392{
393 /* uninitialized instance, can't race */
394 if (!napi->poll_list.next)
395 return;
396
397 /* SCHED bit is set on disabled instances */
398 WARN_ON(!test_bit(NAPI_STATE_SCHED, &napi->state));
399 WARN_ON(READ_ONCE(napi->list_owner) != -1);
400}
401
402struct skb_defer_node;
403void skb_defer_node_flush(struct skb_defer_node *sdn);
404void kick_defer_list_purge(unsigned int cpu);
405
406int dev_set_hwtstamp_phylib(struct net_device *dev,
407 struct kernel_hwtstamp_config *cfg,
408 struct netlink_ext_ack *extack);
409int dev_get_hwtstamp_phylib(struct net_device *dev,
410 struct kernel_hwtstamp_config *cfg);
411int net_hwtstamp_validate(const struct kernel_hwtstamp_config *cfg);
412
413/* Caller holds RTNL, netdev->lock or RCU */
414static inline bool dev_isalive(const struct net_device *dev)
415{
416 return READ_ONCE(dev->reg_state) <= NETREG_REGISTERED;
417}
418
419#endif