projects/linux·blob uaccess-cleanups · refs

552 lines · 13.6 KiB

1// SPDX-License-Identifier: GPL-2.0-or-later
2/*
3 * Devmem TCP
4 *
5 * Authors: Mina Almasry <[email protected]>
6 * Willem de Bruijn <[email protected]>
7 * Kaiyuan Zhang <[email protected]
8 */
9
10#include <linux/dma-buf.h>
11#include <linux/genalloc.h>
12#include <linux/mm.h>
13#include <linux/netdevice.h>
14#include <linux/types.h>
15#include <net/netdev_queues.h>
16#include <net/netdev_rx_queue.h>
17#include <net/page_pool/helpers.h>
18#include <net/page_pool/memory_provider.h>
19#include <net/sock.h>
20#include <net/tcp.h>
21#include <trace/events/page_pool.h>
22
23#include "devmem.h"
24#include "mp_dmabuf_devmem.h"
25#include "page_pool_priv.h"
26
27/* Device memory support */
28
29static DEFINE_XARRAY_FLAGS(net_devmem_dmabuf_bindings, XA_FLAGS_ALLOC1);
30
31static const struct memory_provider_ops dmabuf_devmem_ops;
32
33static void net_devmem_dmabuf_free_chunk_owner(struct gen_pool *genpool,
34 struct gen_pool_chunk *chunk,
35 void *not_used)
36{
37 struct dmabuf_genpool_chunk_owner *owner = chunk->owner;
38
39 kvfree(owner->area.niovs);
40 kfree(owner);
41}
42
43static dma_addr_t net_devmem_get_dma_addr(const struct net_iov *niov)
44{
45 struct dmabuf_genpool_chunk_owner *owner;
46
47 owner = net_devmem_iov_to_chunk_owner(niov);
48 return owner->base_dma_addr +
49 ((dma_addr_t)net_iov_idx(niov) << owner->binding->niov_shift);
50}
51
52static void net_devmem_dmabuf_binding_release(struct percpu_ref *ref)
53{
54 struct net_devmem_dmabuf_binding *binding =
55 container_of(ref, struct net_devmem_dmabuf_binding, ref);
56
57 INIT_WORK(&binding->unbind_w, __net_devmem_dmabuf_binding_free);
58 schedule_work(&binding->unbind_w);
59}
60
61void __net_devmem_dmabuf_binding_free(struct work_struct *wq)
62{
63 struct net_devmem_dmabuf_binding *binding = container_of(wq, typeof(*binding), unbind_w);
64
65 size_t size, avail;
66
67 gen_pool_for_each_chunk(binding->chunk_pool,
68 net_devmem_dmabuf_free_chunk_owner, NULL);
69
70 size = gen_pool_size(binding->chunk_pool);
71 avail = gen_pool_avail(binding->chunk_pool);
72
73 if (!WARN(size != avail, "can't destroy genpool. size=%zu, avail=%zu",
74 size, avail))
75 gen_pool_destroy(binding->chunk_pool);
76
77 dma_buf_unmap_attachment_unlocked(binding->attachment, binding->sgt,
78 binding->direction);
79 dma_buf_detach(binding->dmabuf, binding->attachment);
80 dma_buf_put(binding->dmabuf);
81 xa_destroy(&binding->bound_rxqs);
82 percpu_ref_exit(&binding->ref);
83 kvfree(binding->tx_vec);
84 kfree(binding);
85}
86
87struct net_iov *
88net_devmem_alloc_dmabuf(struct net_devmem_dmabuf_binding *binding)
89{
90 struct dmabuf_genpool_chunk_owner *owner;
91 unsigned long dma_addr;
92 struct net_iov *niov;
93 ssize_t offset;
94 ssize_t index;
95
96 dma_addr = gen_pool_alloc_owner(binding->chunk_pool,
97 1UL << binding->niov_shift,
98 (void **)&owner);
99 if (!dma_addr)
100 return NULL;
101
102 offset = dma_addr - owner->base_dma_addr;
103 index = offset >> binding->niov_shift;
104 niov = &owner->area.niovs[index];
105
106 niov->desc.pp_magic = 0;
107 niov->desc.pp = NULL;
108 atomic_long_set(&niov->desc.pp_ref_count, 0);
109
110 return niov;
111}
112
113void net_devmem_free_dmabuf(struct net_iov *niov)
114{
115 struct net_devmem_dmabuf_binding *binding = net_devmem_iov_binding(niov);
116 unsigned long dma_addr = net_devmem_get_dma_addr(niov);
117 size_t niov_size = 1UL << binding->niov_shift;
118
119 if (WARN_ON(!gen_pool_has_addr(binding->chunk_pool, dma_addr,
120 niov_size)))
121 return;
122
123 gen_pool_free(binding->chunk_pool, dma_addr, niov_size);
124}
125
126void net_devmem_unbind_dmabuf(struct net_devmem_dmabuf_binding *binding)
127{
128 struct netdev_rx_queue *rxq;
129 unsigned long xa_idx;
130 unsigned int rxq_idx;
131
132 xa_erase(&net_devmem_dmabuf_bindings, binding->id);
133
134 /* Ensure no tx net_devmem_lookup_dmabuf() are in flight after the
135 * erase.
136 */
137 synchronize_net();
138
139 if (binding->list.next)
140 list_del(&binding->list);
141
142 xa_for_each(&binding->bound_rxqs, xa_idx, rxq) {
143 const struct pp_memory_provider_params mp_params = {
144 .mp_priv = binding,
145 .mp_ops = &dmabuf_devmem_ops,
146 };
147
148 rxq_idx = get_netdev_rx_queue_index(rxq);
149
150 netif_mp_close_rxq(binding->dev, rxq_idx, &mp_params);
151 }
152
153 percpu_ref_kill(&binding->ref);
154}
155
156int net_devmem_bind_dmabuf_to_queue(struct net_device *dev, u32 rxq_idx,
157 struct net_devmem_dmabuf_binding *binding,
158 struct netlink_ext_ack *extack)
159{
160 struct pp_memory_provider_params mp_params = {
161 .mp_priv = binding,
162 .mp_ops = &dmabuf_devmem_ops,
163 };
164 struct netdev_rx_queue *rxq;
165 u32 xa_idx;
166 int err;
167
168 if (binding->niov_shift != PAGE_SHIFT)
169 mp_params.rx_page_size = 1U << binding->niov_shift;
170
171 err = netif_mp_open_rxq(dev, rxq_idx, &mp_params, extack);
172 if (err)
173 return err;
174
175 rxq = __netif_get_rx_queue(dev, rxq_idx);
176 err = xa_alloc(&binding->bound_rxqs, &xa_idx, rxq, xa_limit_32b,
177 GFP_KERNEL);
178 if (err)
179 goto err_close_rxq;
180
181 return 0;
182
183err_close_rxq:
184 netif_mp_close_rxq(dev, rxq_idx, &mp_params);
185 return err;
186}
187
188struct net_devmem_dmabuf_binding *
189net_devmem_bind_dmabuf(struct net_device *dev, void *vdev,
190 struct device *dma_dev,
191 enum dma_data_direction direction,
192 unsigned int dmabuf_fd, unsigned int niov_shift,
193 struct netdev_nl_sock *priv,
194 struct netlink_ext_ack *extack)
195{
196 struct net_devmem_dmabuf_binding *binding;
197 size_t niov_size = 1UL << niov_shift;
198 static u32 id_alloc_next;
199 struct scatterlist *sg;
200 struct dma_buf *dmabuf;
201 unsigned int sg_idx, i;
202 unsigned long virtual;
203 int err;
204
205 if (!dma_dev) {
206 NL_SET_ERR_MSG(extack, "Device doesn't support DMA");
207 return ERR_PTR(-EOPNOTSUPP);
208 }
209
210 dmabuf = dma_buf_get(dmabuf_fd);
211 if (IS_ERR(dmabuf))
212 return ERR_CAST(dmabuf);
213
214 binding = kzalloc_node(sizeof(*binding), GFP_KERNEL,
215 dev_to_node(&dev->dev));
216 if (!binding) {
217 err = -ENOMEM;
218 goto err_put_dmabuf;
219 }
220
221 binding->dev = dev;
222 binding->vdev = vdev;
223 binding->niov_shift = niov_shift;
224 xa_init_flags(&binding->bound_rxqs, XA_FLAGS_ALLOC);
225
226 err = percpu_ref_init(&binding->ref,
227 net_devmem_dmabuf_binding_release,
228 0, GFP_KERNEL);
229 if (err < 0)
230 goto err_free_binding;
231
232 mutex_init(&binding->lock);
233
234 binding->dmabuf = dmabuf;
235 binding->direction = direction;
236
237 binding->attachment = dma_buf_attach(binding->dmabuf, dma_dev);
238 if (IS_ERR(binding->attachment)) {
239 err = PTR_ERR(binding->attachment);
240 NL_SET_ERR_MSG(extack, "Failed to bind dmabuf to device");
241 goto err_exit_ref;
242 }
243
244 binding->sgt = dma_buf_map_attachment_unlocked(binding->attachment,
245 direction);
246 if (IS_ERR(binding->sgt)) {
247 err = PTR_ERR(binding->sgt);
248 NL_SET_ERR_MSG(extack, "Failed to map dmabuf attachment");
249 goto err_detach;
250 }
251
252 if (direction == DMA_TO_DEVICE) {
253 if (!IS_ALIGNED(dmabuf->size, PAGE_SIZE)) {
254 err = -EINVAL;
255 NL_SET_ERR_MSG(extack, "TX dma-buf size must be a multiple of PAGE_SIZE");
256 goto err_unmap;
257 }
258 binding->tx_vec = kvmalloc_objs(struct net_iov *,
259 dmabuf->size / PAGE_SIZE);
260 if (!binding->tx_vec) {
261 err = -ENOMEM;
262 goto err_unmap;
263 }
264 }
265
266 binding->chunk_pool = gen_pool_create(niov_shift,
267 dev_to_node(&dev->dev));
268 if (!binding->chunk_pool) {
269 err = -ENOMEM;
270 goto err_tx_vec;
271 }
272
273 virtual = 0;
274 for_each_sgtable_dma_sg(binding->sgt, sg, sg_idx) {
275 dma_addr_t dma_addr = sg_dma_address(sg);
276 struct dmabuf_genpool_chunk_owner *owner;
277 size_t len = sg_dma_len(sg);
278 struct net_iov *niov;
279
280 if (!IS_ALIGNED(dma_addr, niov_size) ||
281 !IS_ALIGNED(len, niov_size)) {
282 err = -EINVAL;
283 NL_SET_ERR_MSG_FMT(extack,
284 "dmabuf sg entry (addr=%pad, len=%zu) not aligned to niov size %zu",
285 &dma_addr, len, niov_size);
286 goto err_free_chunks;
287 }
288
289 owner = kzalloc_node(sizeof(*owner), GFP_KERNEL,
290 dev_to_node(&dev->dev));
291 if (!owner) {
292 err = -ENOMEM;
293 goto err_free_chunks;
294 }
295
296 owner->area.base_virtual = virtual;
297 owner->base_dma_addr = dma_addr;
298 owner->area.num_niovs = len >> niov_shift;
299 owner->binding = binding;
300
301 err = gen_pool_add_owner(binding->chunk_pool, dma_addr,
302 dma_addr, len, dev_to_node(&dev->dev),
303 owner);
304 if (err) {
305 kfree(owner);
306 err = -EINVAL;
307 goto err_free_chunks;
308 }
309
310 owner->area.niovs = kvmalloc_objs(*owner->area.niovs,
311 owner->area.num_niovs);
312 if (!owner->area.niovs) {
313 err = -ENOMEM;
314 goto err_free_chunks;
315 }
316
317 for (i = 0; i < owner->area.num_niovs; i++) {
318 niov = &owner->area.niovs[i];
319 net_iov_init(niov, &owner->area, NET_IOV_DMABUF);
320 page_pool_set_dma_addr_netmem(net_iov_to_netmem(niov),
321 net_devmem_get_dma_addr(niov));
322 if (direction == DMA_TO_DEVICE)
323 binding->tx_vec[owner->area.base_virtual / PAGE_SIZE + i] = niov;
324 }
325
326 virtual += len;
327 }
328
329 err = xa_alloc_cyclic(&net_devmem_dmabuf_bindings, &binding->id,
330 binding, xa_limit_32b, &id_alloc_next,
331 GFP_KERNEL);
332 if (err < 0)
333 goto err_free_chunks;
334
335 list_add(&binding->list, &priv->bindings);
336
337 return binding;
338
339err_free_chunks:
340 gen_pool_for_each_chunk(binding->chunk_pool,
341 net_devmem_dmabuf_free_chunk_owner, NULL);
342 gen_pool_destroy(binding->chunk_pool);
343err_tx_vec:
344 kvfree(binding->tx_vec);
345err_unmap:
346 dma_buf_unmap_attachment_unlocked(binding->attachment, binding->sgt,
347 direction);
348err_detach:
349 dma_buf_detach(dmabuf, binding->attachment);
350err_exit_ref:
351 percpu_ref_exit(&binding->ref);
352err_free_binding:
353 kfree(binding);
354err_put_dmabuf:
355 dma_buf_put(dmabuf);
356 return ERR_PTR(err);
357}
358
359struct net_devmem_dmabuf_binding *net_devmem_lookup_dmabuf(u32 id)
360{
361 struct net_devmem_dmabuf_binding *binding;
362
363 rcu_read_lock();
364 binding = xa_load(&net_devmem_dmabuf_bindings, id);
365 if (binding) {
366 if (!net_devmem_dmabuf_binding_get(binding))
367 binding = NULL;
368 }
369 rcu_read_unlock();
370
371 return binding;
372}
373
374void net_devmem_get_net_iov(struct net_iov *niov)
375{
376 net_devmem_dmabuf_binding_get(net_devmem_iov_binding(niov));
377}
378
379void net_devmem_put_net_iov(struct net_iov *niov)
380{
381 net_devmem_dmabuf_binding_put(net_devmem_iov_binding(niov));
382}
383
384struct net_devmem_dmabuf_binding *net_devmem_get_binding(struct sock *sk,
385 unsigned int dmabuf_id)
386{
387 struct net_devmem_dmabuf_binding *binding;
388 struct net_device *dst_dev;
389 struct dst_entry *dst;
390 int err = 0;
391
392 binding = net_devmem_lookup_dmabuf(dmabuf_id);
393 if (!binding || !binding->tx_vec) {
394 err = -EINVAL;
395 goto out_err;
396 }
397
398 rcu_read_lock();
399 dst = __sk_dst_get(sk);
400 /* If dst is NULL (route expired), attempt to rebuild it. */
401 if (unlikely(!dst)) {
402 if (inet_csk(sk)->icsk_af_ops->rebuild_header(sk)) {
403 err = -EHOSTUNREACH;
404 goto out_unlock;
405 }
406 dst = __sk_dst_get(sk);
407 if (unlikely(!dst)) {
408 err = -ENODEV;
409 goto out_unlock;
410 }
411 }
412
413 /* The dma-addrs in this binding are only reachable to the corresponding
414 * net_device.
415 */
416 dst_dev = dst_dev_rcu(dst);
417 if (unlikely(!dst_dev) ||
418 unlikely(dst_dev != READ_ONCE(binding->dev) &&
419 dst_dev != READ_ONCE(binding->vdev))) {
420 err = -ENODEV;
421 goto out_unlock;
422 }
423
424 rcu_read_unlock();
425 return binding;
426
427out_unlock:
428 rcu_read_unlock();
429out_err:
430 if (binding)
431 net_devmem_dmabuf_binding_put(binding);
432
433 return ERR_PTR(err);
434}
435
436struct net_iov *
437net_devmem_get_niov_at(struct net_devmem_dmabuf_binding *binding,
438 size_t virt_addr, size_t *off, size_t *size)
439{
440 if (virt_addr >= binding->dmabuf->size)
441 return NULL;
442
443 *off = virt_addr % PAGE_SIZE;
444 *size = PAGE_SIZE - *off;
445
446 return binding->tx_vec[virt_addr / PAGE_SIZE];
447}
448
449/*** "Dmabuf devmem memory provider" ***/
450
451int mp_dmabuf_devmem_init(struct page_pool *pool)
452{
453 struct net_devmem_dmabuf_binding *binding = pool->mp_priv;
454
455 if (!binding)
456 return -EINVAL;
457
458 /* dma-buf dma addresses do not need and should not be used with
459 * dma_sync_for_cpu/device. Force disable dma_sync.
460 */
461 pool->dma_sync = false;
462 pool->dma_sync_for_cpu = false;
463
464 if (pool->p.order != binding->niov_shift - PAGE_SHIFT)
465 return -E2BIG;
466
467 net_devmem_dmabuf_binding_get(binding);
468 return 0;
469}
470
471netmem_ref mp_dmabuf_devmem_alloc_netmems(struct page_pool *pool, gfp_t gfp)
472{
473 struct net_devmem_dmabuf_binding *binding = pool->mp_priv;
474 struct net_iov *niov;
475 netmem_ref netmem;
476
477 niov = net_devmem_alloc_dmabuf(binding);
478 if (!niov)
479 return 0;
480
481 netmem = net_iov_to_netmem(niov);
482
483 page_pool_set_pp_info(pool, netmem);
484
485 pool->pages_state_hold_cnt++;
486 trace_page_pool_state_hold(pool, netmem, pool->pages_state_hold_cnt);
487 return netmem;
488}
489
490void mp_dmabuf_devmem_destroy(struct page_pool *pool)
491{
492 struct net_devmem_dmabuf_binding *binding = pool->mp_priv;
493
494 net_devmem_dmabuf_binding_put(binding);
495}
496
497bool mp_dmabuf_devmem_release_page(struct page_pool *pool, netmem_ref netmem)
498{
499 long refcount = atomic_long_read(netmem_get_pp_ref_count_ref(netmem));
500
501 if (WARN_ON_ONCE(!netmem_is_net_iov(netmem)))
502 return false;
503
504 if (WARN_ON_ONCE(refcount != 1))
505 return false;
506
507 page_pool_clear_pp_info(netmem);
508
509 net_devmem_free_dmabuf(netmem_to_net_iov(netmem));
510
511 /* We don't want the page pool put_page()ing our net_iovs. */
512 return false;
513}
514
515static int mp_dmabuf_devmem_nl_fill(void *mp_priv, struct sk_buff *rsp,
516 struct netdev_rx_queue *rxq)
517{
518 const struct net_devmem_dmabuf_binding *binding = mp_priv;
519 int type = rxq ? NETDEV_A_QUEUE_DMABUF : NETDEV_A_PAGE_POOL_DMABUF;
520
521 return nla_put_u32(rsp, type, binding->id);
522}
523
524static void mp_dmabuf_devmem_uninstall(void *mp_priv,
525 struct netdev_rx_queue *rxq)
526{
527 struct net_devmem_dmabuf_binding *binding = mp_priv;
528 struct netdev_rx_queue *bound_rxq;
529 unsigned long xa_idx;
530
531 xa_for_each(&binding->bound_rxqs, xa_idx, bound_rxq) {
532 if (bound_rxq == rxq) {
533 xa_erase(&binding->bound_rxqs, xa_idx);
534 if (xa_empty(&binding->bound_rxqs)) {
535 mutex_lock(&binding->lock);
536 ASSERT_EXCLUSIVE_WRITER(binding->dev);
537 WRITE_ONCE(binding->dev, NULL);
538 mutex_unlock(&binding->lock);
539 }
540 break;
541 }
542 }
543}
544
545static const struct memory_provider_ops dmabuf_devmem_ops = {
546 .init = mp_dmabuf_devmem_init,
547 .destroy = mp_dmabuf_devmem_destroy,
548 .alloc_netmems = mp_dmabuf_devmem_alloc_netmems,
549 .release_netmem = mp_dmabuf_devmem_release_page,
550 .nl_fill = mp_dmabuf_devmem_nl_fill,
551 .uninstall = mp_dmabuf_devmem_uninstall,
552};