projects/linux·blob master · refs

349 lines · 9.5 KiB

1// SPDX-License-Identifier: GPL-2.0-or-later
2
3#include <linux/ethtool_netlink.h>
4#include <linux/netdevice.h>
5#include <net/netdev_lock.h>
6#include <net/netdev_queues.h>
7#include <net/netdev_rx_queue.h>
8#include <net/page_pool/memory_provider.h>
9
10#include "dev.h"
11#include "page_pool_priv.h"
12
13void netdev_rx_queue_lease(struct netdev_rx_queue *rxq_dst,
14 struct netdev_rx_queue *rxq_src)
15{
16 netdev_assert_locked(rxq_src->dev);
17 netdev_assert_locked(rxq_dst->dev);
18
19 netdev_hold(rxq_src->dev, &rxq_src->lease_tracker, GFP_KERNEL);
20
21 WRITE_ONCE(rxq_src->lease, rxq_dst);
22 WRITE_ONCE(rxq_dst->lease, rxq_src);
23}
24
25void netdev_rx_queue_unlease(struct netdev_rx_queue *rxq_dst,
26 struct netdev_rx_queue *rxq_src)
27{
28 netdev_assert_locked(rxq_dst->dev);
29 netdev_assert_locked(rxq_src->dev);
30
31 netif_rxq_cleanup_unlease(rxq_src, rxq_dst);
32
33 WRITE_ONCE(rxq_src->lease, NULL);
34 WRITE_ONCE(rxq_dst->lease, NULL);
35
36 netdev_put(rxq_src->dev, &rxq_src->lease_tracker);
37}
38
39bool netif_rxq_is_leased(struct net_device *dev, unsigned int rxq_idx)
40{
41 if (rxq_idx < dev->real_num_rx_queues)
42 return READ_ONCE(__netif_get_rx_queue(dev, rxq_idx)->lease);
43 return false;
44}
45
46/* Virtual devices eligible for leasing have no dev->dev.parent, while
47 * physical devices always have one. Use this to enforce the correct
48 * lease traversal direction.
49 */
50static bool netif_lease_dir_ok(const struct net_device *dev,
51 enum netif_lease_dir dir)
52{
53 if (dir == NETIF_VIRT_TO_PHYS && !dev->dev.parent)
54 return true;
55 if (dir == NETIF_PHYS_TO_VIRT && dev->dev.parent)
56 return true;
57 return false;
58}
59
60bool netif_is_queue_leasee(const struct net_device *dev)
61{
62 return netif_lease_dir_ok(dev, NETIF_VIRT_TO_PHYS);
63}
64
65struct netdev_rx_queue *
66__netif_get_rx_queue_lease(struct net_device **dev, unsigned int *rxq_idx,
67 enum netif_lease_dir dir)
68{
69 struct net_device *orig_dev = *dev;
70 struct netdev_rx_queue *rxq = __netif_get_rx_queue(orig_dev, *rxq_idx);
71
72 if (rxq->lease) {
73 if (!netif_lease_dir_ok(orig_dev, dir))
74 return NULL;
75 rxq = rxq->lease;
76 *rxq_idx = get_netdev_rx_queue_index(rxq);
77 *dev = rxq->dev;
78 }
79 return rxq;
80}
81
82/* See also page_pool_is_unreadable() */
83bool netif_rxq_has_unreadable_mp(struct net_device *dev, unsigned int rxq_idx)
84{
85 if (rxq_idx < dev->real_num_rx_queues)
86 return __netif_get_rx_queue(dev, rxq_idx)->mp_params.mp_ops;
87 return false;
88}
89EXPORT_SYMBOL(netif_rxq_has_unreadable_mp);
90
91bool netif_rxq_has_mp(struct net_device *dev, unsigned int rxq_idx)
92{
93 if (rxq_idx < dev->real_num_rx_queues)
94 return __netif_get_rx_queue(dev, rxq_idx)->mp_params.mp_priv;
95 return false;
96}
97
98static int netdev_rx_queue_reconfig(struct net_device *dev,
99 unsigned int rxq_idx,
100 struct netdev_queue_config *qcfg_old,
101 struct netdev_queue_config *qcfg_new)
102{
103 struct netdev_rx_queue *rxq = __netif_get_rx_queue(dev, rxq_idx);
104 const struct netdev_queue_mgmt_ops *qops = dev->queue_mgmt_ops;
105 void *new_mem, *old_mem;
106 int err;
107
108 if (!qops || !qops->ndo_queue_stop || !qops->ndo_queue_mem_free ||
109 !qops->ndo_queue_mem_alloc || !qops->ndo_queue_start)
110 return -EOPNOTSUPP;
111
112 netdev_assert_locked(dev);
113
114 new_mem = kvzalloc(qops->ndo_queue_mem_size, GFP_KERNEL);
115 if (!new_mem)
116 return -ENOMEM;
117
118 old_mem = kvzalloc(qops->ndo_queue_mem_size, GFP_KERNEL);
119 if (!old_mem) {
120 err = -ENOMEM;
121 goto err_free_new_mem;
122 }
123
124 err = qops->ndo_queue_mem_alloc(dev, qcfg_new, new_mem, rxq_idx);
125 if (err)
126 goto err_free_old_mem;
127
128 err = page_pool_check_memory_provider(dev, rxq);
129 if (err)
130 goto err_free_new_queue_mem;
131
132 if (netif_running(dev)) {
133 err = qops->ndo_queue_stop(dev, old_mem, rxq_idx);
134 if (err)
135 goto err_free_new_queue_mem;
136
137 err = qops->ndo_queue_start(dev, qcfg_new, new_mem, rxq_idx);
138 if (err)
139 goto err_start_queue;
140 } else {
141 swap(new_mem, old_mem);
142 }
143
144 qops->ndo_queue_mem_free(dev, old_mem);
145
146 kvfree(old_mem);
147 kvfree(new_mem);
148
149 return 0;
150
151err_start_queue:
152 /* Restarting the queue with old_mem should be successful as we haven't
153 * changed any of the queue configuration, and there is not much we can
154 * do to recover from a failure here.
155 *
156 * WARN if we fail to recover the old rx queue, and at least free
157 * old_mem so we don't also leak that.
158 */
159 if (qops->ndo_queue_start(dev, qcfg_old, old_mem, rxq_idx)) {
160 WARN(1,
161 "Failed to restart old queue in error path. RX queue %d may be unhealthy.",
162 rxq_idx);
163 qops->ndo_queue_mem_free(dev, old_mem);
164 }
165
166err_free_new_queue_mem:
167 qops->ndo_queue_mem_free(dev, new_mem);
168
169err_free_old_mem:
170 kvfree(old_mem);
171
172err_free_new_mem:
173 kvfree(new_mem);
174
175 return err;
176}
177
178int netdev_rx_queue_restart(struct net_device *dev, unsigned int rxq_idx)
179{
180 struct netdev_queue_config qcfg;
181
182 netdev_queue_config(dev, rxq_idx, &qcfg);
183 return netdev_rx_queue_reconfig(dev, rxq_idx, &qcfg, &qcfg);
184}
185EXPORT_SYMBOL_NS_GPL(netdev_rx_queue_restart, "NETDEV_INTERNAL");
186
187static int __netif_mp_open_rxq(struct net_device *dev, unsigned int rxq_idx,
188 const struct pp_memory_provider_params *p,
189 struct netlink_ext_ack *extack)
190{
191 const struct netdev_queue_mgmt_ops *qops = dev->queue_mgmt_ops;
192 struct netdev_queue_config qcfg[2];
193 struct netdev_rx_queue *rxq;
194 int ret;
195
196 if (!qops)
197 return -EOPNOTSUPP;
198
199 if (dev->cfg->hds_config != ETHTOOL_TCP_DATA_SPLIT_ENABLED) {
200 NL_SET_ERR_MSG(extack, "tcp-data-split is disabled");
201 return -EINVAL;
202 }
203 if (dev->cfg->hds_thresh) {
204 NL_SET_ERR_MSG(extack, "hds-thresh is not zero");
205 return -EINVAL;
206 }
207 if (dev_xdp_prog_count(dev)) {
208 NL_SET_ERR_MSG(extack, "unable to custom memory provider to device with XDP program attached");
209 return -EEXIST;
210 }
211 if (p->rx_page_size && !(qops->supported_params & QCFG_RX_PAGE_SIZE)) {
212 NL_SET_ERR_MSG(extack, "device does not support: rx_page_size");
213 return -EOPNOTSUPP;
214 }
215
216 rxq = __netif_get_rx_queue(dev, rxq_idx);
217 if (rxq->mp_params.mp_ops) {
218 NL_SET_ERR_MSG(extack, "designated queue already memory provider bound");
219 return -EEXIST;
220 }
221#ifdef CONFIG_XDP_SOCKETS
222 if (rxq->pool) {
223 NL_SET_ERR_MSG(extack, "designated queue already in use by AF_XDP");
224 return -EBUSY;
225 }
226#endif
227
228 netdev_queue_config(dev, rxq_idx, &qcfg[0]);
229 rxq->mp_params = *p;
230 ret = netdev_queue_config_validate(dev, rxq_idx, &qcfg[1], extack);
231 if (ret)
232 goto err_clear_mp;
233
234 ret = netdev_rx_queue_reconfig(dev, rxq_idx, &qcfg[0], &qcfg[1]);
235 if (ret)
236 goto err_clear_mp;
237
238 return 0;
239
240err_clear_mp:
241 memset(&rxq->mp_params, 0, sizeof(rxq->mp_params));
242 return ret;
243}
244
245int netif_mp_open_rxq(struct net_device *dev, unsigned int rxq_idx,
246 const struct pp_memory_provider_params *p,
247 struct netlink_ext_ack *extack)
248{
249 int ret;
250
251 if (!netdev_need_ops_lock(dev))
252 return -EOPNOTSUPP;
253
254 if (rxq_idx >= dev->real_num_rx_queues) {
255 NL_SET_ERR_MSG(extack, "rx queue index out of range");
256 return -ERANGE;
257 }
258 rxq_idx = array_index_nospec(rxq_idx, dev->real_num_rx_queues);
259
260 if (!netif_rxq_is_leased(dev, rxq_idx))
261 return __netif_mp_open_rxq(dev, rxq_idx, p, extack);
262
263 if (!__netif_get_rx_queue_lease(&dev, &rxq_idx, NETIF_VIRT_TO_PHYS)) {
264 NL_SET_ERR_MSG(extack, "rx queue leased to a virtual netdev");
265 return -EBUSY;
266 }
267 if (!dev->dev.parent) {
268 NL_SET_ERR_MSG(extack, "rx queue belongs to a virtual netdev");
269 return -EOPNOTSUPP;
270 }
271
272 netdev_lock(dev);
273 ret = __netif_mp_open_rxq(dev, rxq_idx, p, extack);
274 netdev_unlock(dev);
275 return ret;
276}
277
278static void __netif_mp_close_rxq(struct net_device *dev, unsigned int rxq_idx,
279 const struct pp_memory_provider_params *old_p)
280{
281 struct netdev_queue_config qcfg[2];
282 struct netdev_rx_queue *rxq;
283 int err;
284
285 rxq = __netif_get_rx_queue(dev, rxq_idx);
286
287 /* Callers holding a netdev ref may get here after we already
288 * went thru shutdown via dev_memory_provider_uninstall().
289 */
290 if (dev->reg_state > NETREG_REGISTERED &&
291 !rxq->mp_params.mp_ops)
292 return;
293
294 if (WARN_ON_ONCE(rxq->mp_params.mp_ops != old_p->mp_ops ||
295 rxq->mp_params.mp_priv != old_p->mp_priv))
296 return;
297
298 netdev_queue_config(dev, rxq_idx, &qcfg[0]);
299 memset(&rxq->mp_params, 0, sizeof(rxq->mp_params));
300 netdev_queue_config(dev, rxq_idx, &qcfg[1]);
301
302 err = netdev_rx_queue_reconfig(dev, rxq_idx, &qcfg[0], &qcfg[1]);
303 WARN_ON(err && err != -ENETDOWN);
304}
305
306void netif_mp_close_rxq(struct net_device *dev, unsigned int rxq_idx,
307 const struct pp_memory_provider_params *old_p)
308{
309 if (WARN_ON_ONCE(rxq_idx >= dev->real_num_rx_queues))
310 return;
311 if (!netif_rxq_is_leased(dev, rxq_idx))
312 return __netif_mp_close_rxq(dev, rxq_idx, old_p);
313
314 if (!__netif_get_rx_queue_lease(&dev, &rxq_idx, NETIF_VIRT_TO_PHYS)) {
315 WARN_ON_ONCE(1);
316 return;
317 }
318 netdev_lock(dev);
319 __netif_mp_close_rxq(dev, rxq_idx, old_p);
320 netdev_unlock(dev);
321}
322
323void __netif_mp_uninstall_rxq(struct netdev_rx_queue *rxq,
324 const struct pp_memory_provider_params *p)
325{
326 if (p->mp_ops && p->mp_ops->uninstall)
327 p->mp_ops->uninstall(p->mp_priv, rxq);
328}
329
330/* Clean up memory provider state when a queue lease is torn down. If
331 * a memory provider was installed on the physical queue via the lease,
332 * close it now. The memory provider is a property of the queue itself,
333 * and it was _guaranteed_ to be installed on the physical queue via
334 * the lease redirection. The extra __netif_mp_close_rxq is needed
335 * since the physical queue can outlive the virtual queue in the lease
336 * case, so it needs to be reconfigured to clear the memory provider.
337 */
338void netif_rxq_cleanup_unlease(struct netdev_rx_queue *phys_rxq,
339 struct netdev_rx_queue *virt_rxq)
340{
341 unsigned int rxq_idx = get_netdev_rx_queue_index(phys_rxq);
342 struct pp_memory_provider_params p = phys_rxq->mp_params;
343
344 if (!p.mp_ops)
345 return;
346
347 __netif_mp_close_rxq(phys_rxq->dev, rxq_idx, &p);
348 __netif_mp_uninstall_rxq(virt_rxq, &p);
349}