projects/linux·blob master · refs

457 lines · 10.4 KiB

1// SPDX-License-Identifier: GPL-2.0-only
2/*
3 * Common framework for low-level network console, dump, and debugger code
4 *
5 * Sep 8 2003 Matt Mackall <[email protected]>
6 *
7 * based on the netconsole code from:
8 *
9 * Copyright (C) 2001 Ingo Molnar <[email protected]>
10 * Copyright (C) 2002 Red Hat, Inc.
11 */
12
13#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
14
15#include <linux/moduleparam.h>
16#include <linux/kernel.h>
17#include <linux/netdevice.h>
18#include <linux/etherdevice.h>
19#include <linux/string.h>
20#include <linux/if_arp.h>
21#include <linux/inetdevice.h>
22#include <linux/inet.h>
23#include <linux/interrupt.h>
24#include <linux/netpoll.h>
25#include <linux/sched.h>
26#include <linux/delay.h>
27#include <linux/rcupdate.h>
28#include <linux/workqueue.h>
29#include <linux/slab.h>
30#include <linux/export.h>
31#include <linux/if_vlan.h>
32#include <linux/udp.h>
33#include <net/tcp.h>
34#include <net/addrconf.h>
35#include <net/ndisc.h>
36#include <trace/events/napi.h>
37#include <linux/kconfig.h>
38
39#define USEC_PER_POLL 50
40
41/*
42 * carrier_timeout is netconsole-specific and only kept here to preserve the
43 * netpoll.carrier_timeout module-parameter ABI. Its value is exposed to
44 * netconsole through netpoll_get_carrier_timeout().
45 */
46static unsigned int carrier_timeout = 4;
47module_param(carrier_timeout, uint, 0644);
48
49unsigned int netpoll_get_carrier_timeout(void)
50{
51 return carrier_timeout;
52}
53EXPORT_SYMBOL_GPL(netpoll_get_carrier_timeout);
54
55static netdev_tx_t netpoll_start_xmit(struct sk_buff *skb,
56 struct net_device *dev,
57 struct netdev_queue *txq)
58{
59 netdev_tx_t status = NETDEV_TX_OK;
60 netdev_features_t features;
61
62 features = netif_skb_features(skb);
63
64 if (skb_vlan_tag_present(skb) &&
65 !vlan_hw_offload_capable(features, skb->vlan_proto)) {
66 skb = __vlan_hwaccel_push_inside(skb);
67 if (unlikely(!skb)) {
68 /* This is actually a packet drop, but we
69 * don't want the code that calls this
70 * function to try and operate on a NULL skb.
71 */
72 goto out;
73 }
74 }
75
76 status = netdev_start_xmit(skb, dev, txq, false);
77
78out:
79 return status;
80}
81
82static void queue_process(struct work_struct *work)
83{
84 struct netpoll_info *npinfo =
85 container_of(work, struct netpoll_info, tx_work.work);
86 struct sk_buff *skb;
87 unsigned long flags;
88
89 while ((skb = skb_dequeue(&npinfo->txq))) {
90 struct net_device *dev = skb->dev;
91 struct netdev_queue *txq;
92 unsigned int q_index;
93
94 if (!netif_device_present(dev) || !netif_running(dev)) {
95 kfree_skb(skb);
96 continue;
97 }
98
99 local_irq_save(flags);
100 /* check if skb->queue_mapping is still valid */
101 q_index = skb_get_queue_mapping(skb);
102 if (unlikely(q_index >= dev->real_num_tx_queues)) {
103 q_index = q_index % dev->real_num_tx_queues;
104 skb_set_queue_mapping(skb, q_index);
105 }
106 txq = netdev_get_tx_queue(dev, q_index);
107 HARD_TX_LOCK(dev, txq, smp_processor_id());
108 if (netif_xmit_frozen_or_stopped(txq) ||
109 !dev_xmit_complete(netpoll_start_xmit(skb, dev, txq))) {
110 skb_queue_head(&npinfo->txq, skb);
111 HARD_TX_UNLOCK(dev, txq);
112 local_irq_restore(flags);
113
114 schedule_delayed_work(&npinfo->tx_work, HZ/10);
115 return;
116 }
117 HARD_TX_UNLOCK(dev, txq);
118 local_irq_restore(flags);
119 }
120}
121
122static int netif_local_xmit_active(struct net_device *dev)
123{
124 int i;
125
126 for (i = 0; i < dev->num_tx_queues; i++) {
127 struct netdev_queue *txq = netdev_get_tx_queue(dev, i);
128
129 if (netif_tx_owned(txq, smp_processor_id()))
130 return 1;
131 }
132
133 return 0;
134}
135
136static void poll_one_napi(struct napi_struct *napi)
137{
138 int work;
139
140 /* If we set this bit but see that it has already been set,
141 * that indicates that napi has been disabled and we need
142 * to abort this operation
143 */
144 if (test_and_set_bit(NAPI_STATE_NPSVC, &napi->state))
145 return;
146
147 /* We explicitly pass the polling call a budget of 0 to
148 * indicate that we are clearing the Tx path only.
149 */
150 work = napi->poll(napi, 0);
151 WARN_ONCE(work, "%pS exceeded budget in poll\n", napi->poll);
152 trace_napi_poll(napi, work, 0);
153
154 clear_bit(NAPI_STATE_NPSVC, &napi->state);
155}
156
157static void poll_napi(struct net_device *dev)
158{
159 struct napi_struct *napi;
160 int cpu = smp_processor_id();
161
162 list_for_each_entry_rcu(napi, &dev->napi_list, dev_list) {
163 if (cmpxchg(&napi->poll_owner, -1, cpu) == -1) {
164 poll_one_napi(napi);
165 smp_store_release(&napi->poll_owner, -1);
166 }
167 }
168}
169
170void netpoll_poll_dev(struct net_device *dev)
171{
172 struct netpoll_info *ni = rcu_dereference_bh(dev->npinfo);
173 const struct net_device_ops *ops;
174
175 /* Don't do any rx activity if the dev_lock mutex is held
176 * the dev_open/close paths use this to block netpoll activity
177 * while changing device state
178 */
179 if (!ni || down_trylock(&ni->dev_lock))
180 return;
181
182 /* Some drivers will take the same locks in poll and xmit,
183 * we can't poll if local CPU is already in xmit.
184 */
185 if (!netif_running(dev) || netif_local_xmit_active(dev)) {
186 up(&ni->dev_lock);
187 return;
188 }
189
190 ops = dev->netdev_ops;
191 if (ops->ndo_poll_controller)
192 ops->ndo_poll_controller(dev);
193
194 poll_napi(dev);
195
196 up(&ni->dev_lock);
197
198 netpoll_zap_completion_queue();
199}
200EXPORT_SYMBOL(netpoll_poll_dev);
201
202void netpoll_poll_disable(struct net_device *dev)
203{
204 struct netpoll_info *ni;
205
206 might_sleep();
207 ni = rtnl_dereference(dev->npinfo);
208 if (ni)
209 down(&ni->dev_lock);
210}
211
212void netpoll_poll_enable(struct net_device *dev)
213{
214 struct netpoll_info *ni;
215
216 ni = rtnl_dereference(dev->npinfo);
217 if (ni)
218 up(&ni->dev_lock);
219}
220
221void netpoll_zap_completion_queue(void)
222{
223 unsigned long flags;
224 struct softnet_data *sd = &get_cpu_var(softnet_data);
225
226 if (sd->completion_queue) {
227 struct sk_buff *clist;
228
229 local_irq_save(flags);
230 clist = sd->completion_queue;
231 sd->completion_queue = NULL;
232 local_irq_restore(flags);
233
234 while (clist != NULL) {
235 struct sk_buff *skb = clist;
236 clist = clist->next;
237 if (!skb_irq_freeable(skb)) {
238 refcount_set(&skb->users, 1);
239 dev_kfree_skb_any(skb); /* put this one back */
240 } else {
241 __kfree_skb(skb);
242 }
243 }
244 }
245
246 put_cpu_var(softnet_data);
247}
248EXPORT_SYMBOL_NS_GPL(netpoll_zap_completion_queue, "NETDEV_INTERNAL");
249
250static int netpoll_owner_active(struct net_device *dev)
251{
252 struct napi_struct *napi;
253
254 list_for_each_entry_rcu(napi, &dev->napi_list, dev_list) {
255 if (READ_ONCE(napi->poll_owner) == smp_processor_id())
256 return 1;
257 }
258 return 0;
259}
260
261/* call with IRQ disabled */
262static netdev_tx_t __netpoll_send_skb(struct netpoll *np, struct sk_buff *skb)
263{
264 netdev_tx_t status = NETDEV_TX_BUSY;
265 netdev_tx_t ret = NET_XMIT_DROP;
266 struct net_device *dev;
267 unsigned long tries;
268 /* It is up to the caller to keep npinfo alive. */
269 struct netpoll_info *npinfo;
270
271 lockdep_assert_irqs_disabled();
272
273 dev = np->dev;
274 /* npinfo->txq belongs to np->dev, so retries must stay bound to it. */
275 skb->dev = dev;
276 rcu_read_lock();
277 npinfo = rcu_dereference_bh(dev->npinfo);
278
279 if (!npinfo || !netif_running(dev) || !netif_device_present(dev)) {
280 dev_kfree_skb_irq(skb);
281 goto out;
282 }
283
284 /* don't get messages out of order, and no recursion */
285 if (skb_queue_len(&npinfo->txq) == 0 && !netpoll_owner_active(dev)) {
286 struct netdev_queue *txq;
287
288 txq = netdev_core_pick_tx(dev, skb, NULL);
289
290 /* try until next clock tick */
291 for (tries = jiffies_to_usecs(1)/USEC_PER_POLL;
292 tries > 0; --tries) {
293 if (HARD_TX_TRYLOCK(dev, txq)) {
294 if (!netif_xmit_stopped(txq))
295 status = netpoll_start_xmit(skb, dev, txq);
296
297 HARD_TX_UNLOCK(dev, txq);
298
299 if (dev_xmit_complete(status))
300 break;
301
302 }
303
304 /* tickle device maybe there is some cleanup */
305 netpoll_poll_dev(np->dev);
306
307 udelay(USEC_PER_POLL);
308 }
309
310 WARN_ONCE(!irqs_disabled(),
311 "netpoll_send_skb_on_dev(): %s enabled interrupts in poll (%pS)\n",
312 dev->name, dev->netdev_ops->ndo_start_xmit);
313
314 }
315
316 if (!dev_xmit_complete(status)) {
317 skb_queue_tail(&npinfo->txq, skb);
318 schedule_delayed_work(&npinfo->tx_work,0);
319 }
320 ret = NETDEV_TX_OK;
321out:
322 rcu_read_unlock();
323 return ret;
324}
325
326netdev_tx_t netpoll_send_skb(struct netpoll *np, struct sk_buff *skb)
327{
328 unsigned long flags;
329 netdev_tx_t ret;
330
331 if (unlikely(!np)) {
332 dev_kfree_skb_irq(skb);
333 ret = NET_XMIT_DROP;
334 } else {
335 local_irq_save(flags);
336 ret = __netpoll_send_skb(np, skb);
337 local_irq_restore(flags);
338 }
339 return ret;
340}
341EXPORT_SYMBOL(netpoll_send_skb);
342
343int __netpoll_setup(struct netpoll *np, struct net_device *ndev)
344{
345 struct netpoll_info *npinfo;
346 const struct net_device_ops *ops;
347 int err;
348
349 if (ndev->priv_flags & IFF_DISABLE_NETPOLL) {
350 np_err(np, "%s doesn't support polling, aborting\n",
351 ndev->name);
352 err = -ENOTSUPP;
353 goto out;
354 }
355
356 npinfo = rtnl_dereference(ndev->npinfo);
357 if (!npinfo) {
358 npinfo = kmalloc_obj(*npinfo);
359 if (!npinfo) {
360 err = -ENOMEM;
361 goto out;
362 }
363
364 sema_init(&npinfo->dev_lock, 1);
365 skb_queue_head_init(&npinfo->txq);
366 INIT_DELAYED_WORK(&npinfo->tx_work, queue_process);
367
368 refcount_set(&npinfo->refcnt, 1);
369
370 ops = ndev->netdev_ops;
371 if (ops->ndo_netpoll_setup) {
372 err = ops->ndo_netpoll_setup(ndev);
373 if (err)
374 goto free_npinfo;
375 }
376 } else {
377 refcount_inc(&npinfo->refcnt);
378 }
379
380 np->dev = ndev;
381 strscpy(np->dev_name, ndev->name, IFNAMSIZ);
382
383 /* last thing to do is link it to the net device structure */
384 rcu_assign_pointer(ndev->npinfo, npinfo);
385
386 return 0;
387
388free_npinfo:
389 kfree(npinfo);
390out:
391 return err;
392}
393EXPORT_SYMBOL_GPL(__netpoll_setup);
394
395static void rcu_cleanup_netpoll_info(struct rcu_head *rcu_head)
396{
397 struct netpoll_info *npinfo =
398 container_of(rcu_head, struct netpoll_info, rcu);
399
400 skb_queue_purge(&npinfo->txq);
401 kfree(npinfo);
402}
403
404static void __netpoll_cleanup(struct netpoll *np)
405{
406 struct netpoll_info *npinfo;
407
408 npinfo = rtnl_dereference(np->dev->npinfo);
409 if (!npinfo)
410 return;
411
412 /* At this point, there is a single npinfo instance per netdevice, and
413 * its refcnt tracks how many netpoll structures are linked to it. We
414 * only perform npinfo cleanup when the refcnt decrements to zero.
415 */
416 if (refcount_dec_and_test(&npinfo->refcnt)) {
417 const struct net_device_ops *ops;
418
419 ops = np->dev->netdev_ops;
420 if (ops->ndo_netpoll_cleanup)
421 ops->ndo_netpoll_cleanup(np->dev);
422
423 RCU_INIT_POINTER(np->dev->npinfo, NULL);
424 disable_delayed_work_sync(&npinfo->tx_work);
425 call_rcu(&npinfo->rcu, rcu_cleanup_netpoll_info);
426 }
427}
428
429void __netpoll_free(struct netpoll *np)
430{
431 ASSERT_RTNL();
432
433 /* Wait for transmitting packets to finish before freeing. */
434 synchronize_net();
435 __netpoll_cleanup(np);
436 kfree(np);
437}
438EXPORT_SYMBOL_GPL(__netpoll_free);
439
440void do_netpoll_cleanup(struct netpoll *np)
441{
442 __netpoll_cleanup(np);
443 netdev_put(np->dev, &np->dev_tracker);
444 np->dev = NULL;
445}
446EXPORT_SYMBOL(do_netpoll_cleanup);
447
448void netpoll_cleanup(struct netpoll *np)
449{
450 rtnl_lock();
451 if (!np->dev)
452 goto out;
453 do_netpoll_cleanup(np);
454out:
455 rtnl_unlock();
456}
457EXPORT_SYMBOL(netpoll_cleanup);