projects/linux·blob master · refs

1970 lines · 45.6 KiB

1// SPDX-License-Identifier: GPL-2.0
2#include <linux/kernel.h>
3#include <linux/errno.h>
4#include <linux/dma-map-ops.h>
5#include <linux/mm.h>
6#include <linux/nospec.h>
7#include <linux/io_uring.h>
8#include <linux/netdevice.h>
9#include <linux/rtnetlink.h>
10#include <linux/skbuff_ref.h>
11#include <linux/anon_inodes.h>
12
13#include <net/page_pool/helpers.h>
14#include <net/page_pool/memory_provider.h>
15#include <net/netlink.h>
16#include <net/netdev_queues.h>
17#include <net/netdev_rx_queue.h>
18#include <net/tcp.h>
19#include <net/rps.h>
20
21#include <trace/events/page_pool.h>
22
23#include <uapi/linux/io_uring.h>
24
25#include "io_uring.h"
26#include "kbuf.h"
27#include "memmap.h"
28#include "zcrx.h"
29#include "rsrc.h"
30
31#define ZCRX_MAX_FRAGS_PER_PAGE MAX(PAGE_SIZE / 1024, 1)
32/*
33 * We need a reasonable limit to be able to fill in 64 entries on average
34 * for 1500 byte MTU. Over-estimate it to keep it pow2.
35 */
36#define ZCRX_REFILL_CAP MIN(64 * ZCRX_MAX_FRAGS_PER_PAGE, 1024)
37
38#define IO_ZCRX_AREA_SUPPORTED_FLAGS (IORING_ZCRX_AREA_DMABUF)
39#define ZCRX_MAX_AREAS 1024
40
41#define IO_DMA_ATTR (DMA_ATTR_SKIP_CPU_SYNC | DMA_ATTR_WEAK_ORDERING)
42
43static inline u32 zcrx_next_area_id(struct io_zcrx_ifq *zcrx)
44{
45 return zcrx->nr_areas;
46}
47
48static inline u64 zcrx_area_id_to_token(u32 area_id)
49{
50 return (u64)area_id << IORING_ZCRX_AREA_SHIFT;
51}
52
53static inline struct io_zcrx_ifq *io_pp_to_ifq(struct page_pool *pp)
54{
55 return pp->mp_priv;
56}
57
58static inline struct io_zcrx_area *io_zcrx_iov_to_area(const struct net_iov *niov)
59{
60 struct net_iov_area *owner = net_iov_owner(niov);
61
62 return container_of(owner, struct io_zcrx_area, nia);
63}
64
65static bool zcrx_set_ring_ctx(struct io_zcrx_ifq *zcrx,
66 struct io_ring_ctx *ctx)
67{
68 guard(spinlock_bh)(&zcrx->ctx_lock);
69 if (zcrx->master_ctx)
70 return false;
71 percpu_ref_get(&ctx->refs);
72 zcrx->master_ctx = ctx;
73 return true;
74}
75
76static inline struct page *io_zcrx_iov_page(const struct net_iov *niov)
77{
78 struct io_zcrx_area *area = io_zcrx_iov_to_area(niov);
79 unsigned niov_pages_shift;
80
81 lockdep_assert(!area->mem.is_dmabuf);
82
83 niov_pages_shift = area->ifq->niov_shift - PAGE_SHIFT;
84 return area->mem.pages[net_iov_idx(niov) << niov_pages_shift];
85}
86
87static int io_area_max_shift(struct io_zcrx_mem *mem)
88{
89 struct sg_table *sgt = mem->sgt;
90 struct scatterlist *sg;
91 unsigned shift = -1U;
92 unsigned i;
93
94 for_each_sgtable_dma_sg(sgt, sg, i)
95 shift = min(shift, __ffs(sg_dma_len(sg)));
96 return shift;
97}
98
99static int io_populate_area_dma(struct io_zcrx_ifq *ifq,
100 struct io_zcrx_area *area)
101{
102 unsigned niov_size = 1U << ifq->niov_shift;
103 struct sg_table *sgt = area->mem.sgt;
104 struct scatterlist *sg;
105 unsigned i, niov_idx = 0;
106
107 for_each_sgtable_dma_sg(sgt, sg, i) {
108 dma_addr_t dma = sg_dma_address(sg);
109 unsigned long sg_len = sg_dma_len(sg);
110
111 if (WARN_ON_ONCE(sg_len % niov_size))
112 return -EINVAL;
113
114 while (sg_len && niov_idx < area->nia.num_niovs) {
115 struct net_iov *niov = &area->nia.niovs[niov_idx];
116
117 if (net_mp_niov_set_dma_addr(niov, dma))
118 return -EFAULT;
119 sg_len -= niov_size;
120 dma += niov_size;
121 niov_idx++;
122 }
123 }
124
125 if (WARN_ON_ONCE(niov_idx != area->nia.num_niovs))
126 return -EFAULT;
127 return 0;
128}
129
130static void io_unmap_dmabuf(struct io_zcrx_mem *mem)
131{
132 if (!IS_ENABLED(CONFIG_DMA_SHARED_BUFFER))
133 return;
134 if (mem->sgt)
135 dma_buf_unmap_attachment_unlocked(mem->attach, mem->sgt,
136 DMA_FROM_DEVICE);
137 if (mem->attach)
138 dma_buf_detach(mem->dmabuf, mem->attach);
139 mem->sgt = NULL;
140 mem->attach = NULL;
141}
142
143static void io_release_dmabuf(struct io_zcrx_mem *mem)
144{
145 if (!IS_ENABLED(CONFIG_DMA_SHARED_BUFFER))
146 return;
147 if (mem->dmabuf)
148 dma_buf_put(mem->dmabuf);
149 mem->dmabuf = NULL;
150}
151
152static int io_import_dmabuf(struct io_zcrx_ifq *ifq,
153 struct io_zcrx_mem *mem,
154 const struct io_uring_zcrx_area_reg *area_reg)
155{
156 unsigned long off = (unsigned long)area_reg->addr;
157 unsigned long len = (unsigned long)area_reg->len;
158 unsigned long total_size = 0;
159 struct scatterlist *sg;
160 int dmabuf_fd = area_reg->dmabuf_fd;
161 int i, ret;
162
163 if (!ifq->dev)
164 return -EINVAL;
165 if (off)
166 return -EINVAL;
167 if (!IS_ENABLED(CONFIG_DMA_SHARED_BUFFER))
168 return -EINVAL;
169
170 mem->is_dmabuf = true;
171 mem->dmabuf = dma_buf_get(dmabuf_fd);
172 if (IS_ERR(mem->dmabuf)) {
173 ret = PTR_ERR(mem->dmabuf);
174 mem->dmabuf = NULL;
175 goto err;
176 }
177
178 mem->attach = dma_buf_attach(mem->dmabuf, ifq->dev);
179 if (IS_ERR(mem->attach)) {
180 ret = PTR_ERR(mem->attach);
181 mem->attach = NULL;
182 goto err;
183 }
184
185 mem->sgt = dma_buf_map_attachment_unlocked(mem->attach, DMA_FROM_DEVICE);
186 if (IS_ERR(mem->sgt)) {
187 ret = PTR_ERR(mem->sgt);
188 mem->sgt = NULL;
189 goto err;
190 }
191
192 for_each_sgtable_dma_sg(mem->sgt, sg, i)
193 total_size += sg_dma_len(sg);
194
195 if (total_size != len) {
196 ret = -EINVAL;
197 goto err;
198 }
199
200 mem->size = len;
201 return 0;
202err:
203 io_unmap_dmabuf(mem);
204 io_release_dmabuf(mem);
205 return ret;
206}
207
208static unsigned long io_count_account_pages(struct page **pages, unsigned nr_pages)
209{
210 struct folio *last_folio = NULL;
211 unsigned long res = 0;
212 int i;
213
214 for (i = 0; i < nr_pages; i++) {
215 struct folio *folio = page_folio(pages[i]);
216
217 if (folio == last_folio)
218 continue;
219 last_folio = folio;
220 res += folio_nr_pages(folio);
221 }
222 return res;
223}
224
225static int io_import_umem(struct io_zcrx_ifq *ifq,
226 struct io_zcrx_mem *mem,
227 const struct io_uring_zcrx_area_reg *area_reg)
228{
229 struct page **pages;
230 int nr_pages, ret;
231 bool mapped = false;
232
233 if (area_reg->dmabuf_fd)
234 return -EINVAL;
235 if (!area_reg->addr)
236 return -EFAULT;
237 pages = io_pin_pages((unsigned long)area_reg->addr, area_reg->len,
238 &nr_pages);
239 if (IS_ERR(pages))
240 return PTR_ERR(pages);
241
242 ret = sg_alloc_table_from_pages(&mem->page_sg_table, pages, nr_pages,
243 0, (unsigned long)nr_pages << PAGE_SHIFT,
244 GFP_KERNEL_ACCOUNT);
245 if (ret)
246 goto out_err;
247
248 if (ifq->dev) {
249 ret = dma_map_sgtable(ifq->dev, &mem->page_sg_table,
250 DMA_FROM_DEVICE, IO_DMA_ATTR);
251 if (ret < 0)
252 goto out_err;
253 mapped = true;
254 }
255
256 mem->account_pages = io_count_account_pages(pages, nr_pages);
257 ret = io_account_mem(ifq->user, ifq->mm_account, mem->account_pages);
258 if (ret < 0) {
259 mem->account_pages = 0;
260 goto out_err;
261 }
262
263 mem->sgt = &mem->page_sg_table;
264 mem->pages = pages;
265 mem->nr_folios = nr_pages;
266 mem->size = area_reg->len;
267 return ret;
268out_err:
269 if (mapped)
270 dma_unmap_sgtable(ifq->dev, &mem->page_sg_table,
271 DMA_FROM_DEVICE, IO_DMA_ATTR);
272 sg_free_table(&mem->page_sg_table);
273 unpin_user_pages(pages, nr_pages);
274 kvfree(pages);
275 return ret;
276}
277
278static void io_release_area_mem(struct io_zcrx_mem *mem)
279{
280 if (mem->is_dmabuf) {
281 io_release_dmabuf(mem);
282 } else if (mem->pages) {
283 unpin_user_pages(mem->pages, mem->nr_folios);
284 sg_free_table(mem->sgt);
285 kvfree(mem->pages);
286 }
287 mem->pages = IO_URING_PTR_POISON;
288 mem->sgt = IO_URING_PTR_POISON;
289}
290
291static int io_import_area(struct io_zcrx_ifq *ifq,
292 struct io_zcrx_mem *mem,
293 const struct io_uring_zcrx_area_reg *area_reg)
294{
295 int ret;
296
297 if (area_reg->flags & ~IO_ZCRX_AREA_SUPPORTED_FLAGS)
298 return -EINVAL;
299 if (area_reg->rq_area_token)
300 return -EINVAL;
301 if (area_reg->__resv2[0] || area_reg->__resv2[1])
302 return -EINVAL;
303
304 ret = io_validate_user_buf_range(area_reg->addr, area_reg->len);
305 if (ret)
306 return ret;
307 if (area_reg->addr & ~PAGE_MASK || area_reg->len & ~PAGE_MASK)
308 return -EINVAL;
309
310 if (area_reg->flags & IORING_ZCRX_AREA_DMABUF)
311 return io_import_dmabuf(ifq, mem, area_reg);
312 return io_import_umem(ifq, mem, area_reg);
313}
314
315static void __io_zcrx_unmap_area(struct io_zcrx_ifq *ifq,
316 struct io_zcrx_area *area)
317{
318 int i;
319
320 lockdep_assert_held(&ifq->pp_lock);
321
322 if (!area || !area->is_mapped)
323 return;
324 area->is_mapped = false;
325
326 if (area->nia.niovs) {
327 for (i = 0; i < area->nia.num_niovs; i++)
328 net_mp_niov_set_dma_addr(&area->nia.niovs[i], 0);
329 }
330
331 if (area->mem.is_dmabuf) {
332 io_unmap_dmabuf(&area->mem);
333 } else {
334 dma_unmap_sgtable(ifq->dev, &area->mem.page_sg_table,
335 DMA_FROM_DEVICE, IO_DMA_ATTR);
336 }
337}
338
339static void io_zcrx_unmap_areas(struct io_zcrx_ifq *ifq)
340{
341 unsigned area_idx;
342
343 lockdep_assert_held(&ifq->pp_lock);
344
345 for (area_idx = 0; area_idx < ifq->nr_areas; area_idx++)
346 __io_zcrx_unmap_area(ifq, ifq->areas[area_idx]);
347}
348
349static void zcrx_sync_for_device(struct page_pool *pp, struct io_zcrx_ifq *zcrx,
350 netmem_ref *netmems, unsigned nr)
351{
352#if defined(CONFIG_HAS_DMA) && defined(CONFIG_DMA_NEED_SYNC)
353 struct device *dev = pp->p.dev;
354 unsigned i, niov_size;
355 dma_addr_t dma_addr;
356
357 if (!dma_dev_need_sync(dev))
358 return;
359 niov_size = 1U << zcrx->niov_shift;
360
361 for (i = 0; i < nr; i++) {
362 dma_addr = page_pool_get_dma_addr_netmem(netmems[i]);
363 __dma_sync_single_for_device(dev, dma_addr + pp->p.offset,
364 niov_size, pp->p.dma_dir);
365 }
366#endif
367}
368
369#define IO_RQ_MAX_ENTRIES 32768
370
371#define IO_SKBS_PER_CALL_LIMIT 20
372
373struct io_zcrx_args {
374 struct io_kiocb *req;
375 struct io_zcrx_ifq *ifq;
376 unsigned nr_skbs;
377 bool stopped_early;
378};
379
380static const struct memory_provider_ops io_uring_pp_zc_ops;
381
382static inline atomic_t *io_get_user_counter(struct net_iov *niov)
383{
384 struct io_zcrx_area *area = io_zcrx_iov_to_area(niov);
385
386 return &area->user_refs[net_iov_idx(niov)];
387}
388
389static bool io_zcrx_put_niov_uref(struct net_iov *niov, unsigned refs)
390{
391 atomic_t *uref = io_get_user_counter(niov);
392 int old;
393
394 old = atomic_read(uref);
395 do {
396 if (unlikely(old < refs))
397 return false;
398 } while (!atomic_try_cmpxchg(uref, &old, old - refs));
399
400 return true;
401}
402
403static void io_zcrx_get_niov_uref(struct net_iov *niov)
404{
405 atomic_inc(io_get_user_counter(niov));
406}
407
408static void io_fill_zcrx_offsets(struct io_uring_zcrx_offsets *offsets)
409{
410 offsets->head = offsetof(struct zcrx_rq_hdr, head);
411 offsets->tail = offsetof(struct zcrx_rq_hdr, tail);
412 offsets->rqes = ALIGN(sizeof(struct zcrx_rq_hdr), L1_CACHE_BYTES);
413}
414
415static int io_allocate_rbuf_ring(struct io_ring_ctx *ctx,
416 struct io_zcrx_ifq *ifq,
417 struct io_uring_zcrx_ifq_reg *reg,
418 struct io_uring_region_desc *rd,
419 u32 id)
420{
421 u64 mmap_offset;
422 size_t off, size;
423 void *ptr;
424 int ret;
425
426 io_fill_zcrx_offsets(&reg->offsets);
427 off = reg->offsets.rqes;
428 size = off + sizeof(struct io_uring_zcrx_rqe) * reg->rq_entries;
429 if (size > rd->size)
430 return -EINVAL;
431
432 mmap_offset = IORING_MAP_OFF_ZCRX_REGION;
433 mmap_offset += (u64)id << IORING_OFF_ZCRX_SHIFT;
434
435 ret = io_create_region(ctx->user, &ifq->rq_region, rd, mmap_offset);
436 if (ret < 0)
437 return ret;
438
439 ptr = io_region_get_ptr(&ifq->rq_region);
440 ifq->rq.ring = (struct zcrx_rq_hdr *)ptr;
441 ifq->rq.rqes = (struct io_uring_zcrx_rqe *)(ptr + off);
442
443 memset(ifq->rq.ring, 0, sizeof(*ifq->rq.ring));
444 return 0;
445}
446
447static void io_free_rbuf_ring(struct io_zcrx_ifq *ifq)
448{
449 io_free_region(ifq->user, &ifq->rq_region);
450 ifq->rq.ring = IO_URING_PTR_POISON;
451 ifq->rq.rqes = IO_URING_PTR_POISON;
452 ifq->notif_stats = IO_URING_PTR_POISON;
453}
454
455static void io_zcrx_free_area(struct io_zcrx_ifq *ifq,
456 struct io_zcrx_area *area)
457{
458 if (WARN_ON_ONCE(area->is_mapped))
459 return;
460 io_release_area_mem(&area->mem);
461
462 if (area->mem.account_pages)
463 io_unaccount_mem(ifq->user, ifq->mm_account,
464 area->mem.account_pages);
465
466 kvfree(area->freelist);
467 kvfree(area->nia.niovs);
468 kvfree(area->user_refs);
469 kfree(area);
470}
471
472static int io_zcrx_append_area(struct io_zcrx_ifq *ifq,
473 struct io_zcrx_area *area)
474{
475 bool kern_readable = !area->mem.is_dmabuf;
476 struct io_zcrx_area **areas, **old_areas;
477 unsigned old_nr;
478
479 if (ifq->kern_readable != kern_readable)
480 return -EINVAL;
481 if (ifq->nr_areas + 1 > ZCRX_MAX_AREAS)
482 return -EINVAL;
483
484 old_areas = ifq->areas;
485 old_nr = ifq->nr_areas;
486
487 areas = kmalloc_objs(areas[0], old_nr + 1,
488 GFP_KERNEL_ACCOUNT | __GFP_ZERO);
489 if (!areas)
490 return -ENOMEM;
491 if (old_areas)
492 memcpy(areas, old_areas, old_nr * sizeof(areas[0]));
493 areas[old_nr] = area;
494
495 scoped_guard(spinlock_bh, &ifq->rq.lock) {
496 guard(spinlock_bh)(&ifq->alloc_lock);
497 ifq->areas = areas;
498 ifq->nr_areas = old_nr + 1;
499 }
500 kfree(old_areas);
501 return 0;
502}
503
504static int __zcrx_create_area(struct io_zcrx_ifq *ifq,
505 struct io_uring_zcrx_area_reg *area_reg,
506 u32 rx_buf_len)
507{
508 int buf_size_shift = PAGE_SHIFT;
509 struct io_zcrx_area *area;
510 unsigned nr_iovs;
511 int i, ret;
512
513 lockdep_assert_held(&ifq->pp_lock);
514
515 if (rx_buf_len) {
516 if (!is_power_of_2(rx_buf_len) || rx_buf_len < PAGE_SIZE)
517 return -EINVAL;
518 buf_size_shift = ilog2(rx_buf_len);
519 }
520 if (ifq->niov_shift && ifq->niov_shift != buf_size_shift)
521 return -EINVAL;
522 if (!ifq->dev && buf_size_shift != PAGE_SHIFT)
523 return -EOPNOTSUPP;
524
525 ret = -ENOMEM;
526 area = kzalloc_obj(*area);
527 if (!area)
528 goto err;
529 area->ifq = ifq;
530
531 ret = io_import_area(ifq, &area->mem, area_reg);
532 if (ret)
533 goto err;
534 if (ifq->dev)
535 area->is_mapped = true;
536
537 if (ifq->dev && buf_size_shift > io_area_max_shift(&area->mem)) {
538 ret = -ERANGE;
539 goto err;
540 }
541
542 ifq->niov_shift = buf_size_shift;
543 nr_iovs = area->mem.size >> ifq->niov_shift;
544 area->nia.num_niovs = nr_iovs;
545
546 ret = -ENOMEM;
547 area->nia.niovs = kvmalloc_objs(area->nia.niovs[0], nr_iovs,
548 GFP_KERNEL_ACCOUNT | __GFP_ZERO);
549 if (!area->nia.niovs)
550 goto err;
551
552 area->freelist = kvmalloc_array(nr_iovs, sizeof(area->freelist[0]),
553 GFP_KERNEL_ACCOUNT | __GFP_ZERO);
554 if (!area->freelist)
555 goto err;
556
557 area->user_refs = kvmalloc_objs(area->user_refs[0], nr_iovs,
558 GFP_KERNEL_ACCOUNT | __GFP_ZERO);
559 if (!area->user_refs)
560 goto err;
561
562 for (i = 0; i < nr_iovs; i++) {
563 struct net_iov *niov = &area->nia.niovs[i];
564
565 net_iov_init(niov, &area->nia, NET_IOV_IOURING);
566 area->freelist[i] = i;
567 atomic_set(&area->user_refs[i], 0);
568 }
569
570 if (ifq->dev) {
571 ret = io_populate_area_dma(ifq, area);
572 if (ret)
573 goto err;
574 }
575
576 area->free_count = nr_iovs;
577 /* we're only supporting one area per ifq for now */
578 area->area_id = zcrx_next_area_id(ifq);
579 area_reg->rq_area_token = zcrx_area_id_to_token(area->area_id);
580
581 ret = io_zcrx_append_area(ifq, area);
582 if (!ret)
583 return 0;
584err:
585 if (area) {
586 __io_zcrx_unmap_area(ifq, area);
587 io_zcrx_free_area(ifq, area);
588 }
589 return ret;
590}
591
592static int io_zcrx_create_area(struct io_zcrx_ifq *ifq,
593 struct io_uring_zcrx_area_reg *area_reg,
594 struct io_uring_zcrx_ifq_reg *reg)
595{
596 guard(mutex)(&ifq->pp_lock);
597 return __zcrx_create_area(ifq, area_reg, reg->rx_buf_len);
598}
599
600static struct io_zcrx_ifq *io_zcrx_ifq_alloc(struct io_ring_ctx *ctx)
601{
602 struct io_zcrx_ifq *ifq;
603
604 ifq = kzalloc_obj(*ifq);
605 if (!ifq)
606 return NULL;
607
608 ifq->if_rxq = -1;
609 spin_lock_init(&ifq->ctx_lock);
610 spin_lock_init(&ifq->rq.lock);
611 spin_lock_init(&ifq->alloc_lock);
612 mutex_init(&ifq->pp_lock);
613 refcount_set(&ifq->refs, 1);
614 refcount_set(&ifq->user_refs, 1);
615 return ifq;
616}
617
618static void io_zcrx_drop_netdev(struct io_zcrx_ifq *ifq)
619{
620 lockdep_assert_held(&ifq->pp_lock);
621
622 if (!ifq->netdev)
623 return;
624 netdev_put(ifq->netdev, &ifq->netdev_tracker);
625 ifq->netdev = NULL;
626}
627
628static void io_close_queue(struct io_zcrx_ifq *ifq)
629{
630 struct net_device *netdev;
631 netdevice_tracker netdev_tracker;
632 struct pp_memory_provider_params p = {
633 .mp_ops = &io_uring_pp_zc_ops,
634 .mp_priv = ifq,
635 };
636
637 scoped_guard(mutex, &ifq->pp_lock) {
638 netdev = ifq->netdev;
639 netdev_tracker = ifq->netdev_tracker;
640 ifq->netdev = NULL;
641 }
642
643 if (netdev) {
644 netdev_lock(netdev);
645 if (ifq->if_rxq != -1)
646 netif_mp_close_rxq(netdev, ifq->if_rxq, &p);
647
648 scoped_guard(mutex, &ifq->pp_lock)
649 io_zcrx_unmap_areas(ifq);
650 netdev_unlock(netdev);
651 netdev_put(netdev, &netdev_tracker);
652 }
653 ifq->if_rxq = -1;
654}
655
656static void io_zcrx_ifq_free(struct io_zcrx_ifq *ifq)
657{
658 int i;
659
660 if (WARN_ON_ONCE(ifq->if_rxq != -1))
661 return;
662 if (WARN_ON_ONCE(ifq->netdev != NULL))
663 return;
664 if (WARN_ON_ONCE(ifq->master_ctx))
665 return;
666
667 for (i = 0; i < ifq->nr_areas; i++)
668 io_zcrx_free_area(ifq, ifq->areas[i]);
669 if (ifq->mm_account)
670 mmdrop(ifq->mm_account);
671 if (ifq->dev)
672 put_device(ifq->dev);
673
674 io_free_rbuf_ring(ifq);
675 free_uid(ifq->user);
676 mutex_destroy(&ifq->pp_lock);
677 kfree(ifq->areas);
678 kfree(ifq);
679}
680
681static void io_put_zcrx_ifq(struct io_zcrx_ifq *ifq)
682{
683 if (refcount_dec_and_test(&ifq->refs))
684 io_zcrx_ifq_free(ifq);
685}
686
687static void io_zcrx_return_niov_freelist(struct net_iov *niov)
688{
689 struct io_zcrx_area *area = io_zcrx_iov_to_area(niov);
690 struct io_zcrx_ifq *ifq = area->ifq;
691
692 guard(spinlock_bh)(&ifq->alloc_lock);
693 if (WARN_ON_ONCE(area->free_count >= area->nia.num_niovs))
694 return;
695 area->freelist[area->free_count++] = net_iov_idx(niov);
696}
697
698static struct net_iov *zcrx_get_free_niov(struct io_zcrx_area *area)
699{
700 unsigned niov_idx;
701
702 lockdep_assert_held(&area->ifq->alloc_lock);
703
704 if (unlikely(!area->free_count))
705 return NULL;
706
707 niov_idx = area->freelist[--area->free_count];
708 return &area->nia.niovs[niov_idx];
709}
710
711static void io_zcrx_return_niov(struct net_iov *niov)
712{
713 netmem_ref netmem = net_iov_to_netmem(niov);
714
715 if (!niov->desc.pp) {
716 /* copy fallback allocated niovs */
717 io_zcrx_return_niov_freelist(niov);
718 return;
719 }
720 page_pool_put_unrefed_netmem(niov->desc.pp, netmem, -1, false);
721}
722
723static void io_zcrx_scrub_area(struct io_zcrx_ifq *ifq, struct io_zcrx_area *area)
724{
725 int i;
726
727 /* Reclaim back all buffers given to the user space. */
728 for (i = 0; i < area->nia.num_niovs; i++) {
729 struct net_iov *niov = &area->nia.niovs[i];
730 int nr;
731
732 if (!atomic_read(io_get_user_counter(niov)))
733 continue;
734 nr = atomic_xchg(io_get_user_counter(niov), 0);
735 if (nr && !page_pool_unref_netmem(net_iov_to_netmem(niov), nr))
736 io_zcrx_return_niov(niov);
737 }
738}
739
740static void io_zcrx_scrub(struct io_zcrx_ifq *ifq)
741{
742 int i;
743
744 guard(mutex)(&ifq->pp_lock);
745 for (i = 0; i < ifq->nr_areas; i++)
746 io_zcrx_scrub_area(ifq, ifq->areas[i]);
747}
748
749static void zcrx_unregister_user(struct io_zcrx_ifq *ifq, struct io_ring_ctx *ctx)
750{
751 scoped_guard(spinlock_bh, &ifq->ctx_lock) {
752 if (ctx && ifq->master_ctx == ctx) {
753 ifq->master_ctx = NULL;
754 percpu_ref_put(&ctx->refs);
755 }
756 }
757
758 if (refcount_dec_and_test(&ifq->user_refs)) {
759 io_close_queue(ifq);
760 io_zcrx_scrub(ifq);
761 }
762}
763
764static void zcrx_unregister(struct io_zcrx_ifq *ifq, struct io_ring_ctx *ctx)
765{
766 zcrx_unregister_user(ifq, ctx);
767 io_put_zcrx_ifq(ifq);
768}
769
770struct io_mapped_region *io_zcrx_get_region(struct io_ring_ctx *ctx,
771 unsigned int id)
772{
773 struct io_zcrx_ifq *ifq = xa_load(&ctx->zcrx_ctxs, id);
774
775 lockdep_assert_held(&ctx->mmap_lock);
776
777 return ifq ? &ifq->rq_region : NULL;
778}
779
780static int zcrx_box_release(struct inode *inode, struct file *file)
781{
782 struct io_zcrx_ifq *ifq = file->private_data;
783
784 if (WARN_ON_ONCE(!ifq))
785 return -EFAULT;
786 zcrx_unregister(ifq, NULL);
787 return 0;
788}
789
790static const struct file_operations zcrx_box_fops = {
791 .owner = THIS_MODULE,
792 .release = zcrx_box_release,
793};
794
795static int zcrx_export(struct io_ring_ctx *ctx, struct io_zcrx_ifq *ifq,
796 struct zcrx_ctrl *ctrl, void __user *arg)
797{
798 struct zcrx_ctrl_export *ce = &ctrl->zc_export;
799 struct file *file;
800 int fd;
801
802 if (!mem_is_zero(ce, sizeof(*ce)))
803 return -EINVAL;
804
805 refcount_inc(&ifq->refs);
806 refcount_inc(&ifq->user_refs);
807
808 file = anon_inode_create_getfile("[zcrx]", &zcrx_box_fops,
809 ifq, O_CLOEXEC, NULL);
810 if (IS_ERR(file)) {
811 zcrx_unregister(ifq, NULL);
812 return PTR_ERR(file);
813 }
814
815 fd = get_unused_fd_flags(O_CLOEXEC);
816 if (fd < 0) {
817 fput(file);
818 return fd;
819 }
820
821 ce->zcrx_fd = fd;
822 if (copy_to_user(arg, ctrl, sizeof(*ctrl))) {
823 fput(file);
824 put_unused_fd(fd);
825 return -EFAULT;
826 }
827
828 fd_install(fd, file);
829 return 0;
830}
831
832static int import_zcrx(struct io_ring_ctx *ctx,
833 struct io_uring_zcrx_ifq_reg __user *arg,
834 struct io_uring_zcrx_ifq_reg *reg)
835{
836 struct io_zcrx_ifq *ifq;
837 struct file *file;
838 int fd, ret;
839 u32 id;
840
841 if (!(ctx->flags & IORING_SETUP_DEFER_TASKRUN))
842 return -EINVAL;
843 if (!(ctx->flags & (IORING_SETUP_CQE32|IORING_SETUP_CQE_MIXED)))
844 return -EINVAL;
845 if (reg->if_rxq || reg->rq_entries || reg->area_ptr || reg->region_ptr)
846 return -EINVAL;
847 if (reg->event_desc)
848 return -EINVAL;
849 if (reg->flags & ~ZCRX_REG_IMPORT)
850 return -EINVAL;
851
852 fd = reg->if_idx;
853 CLASS(fd, f)(fd);
854 if (fd_empty(f))
855 return -EBADF;
856
857 file = fd_file(f);
858 if (file->f_op != &zcrx_box_fops || !file->private_data)
859 return -EBADF;
860
861 ifq = file->private_data;
862 refcount_inc(&ifq->refs);
863 refcount_inc(&ifq->user_refs);
864
865 scoped_guard(mutex, &ctx->mmap_lock) {
866 ret = xa_alloc(&ctx->zcrx_ctxs, &id, NULL, xa_limit_31b, GFP_KERNEL);
867 if (ret)
868 goto err;
869 }
870
871 reg->zcrx_id = id;
872 io_fill_zcrx_offsets(&reg->offsets);
873 if (copy_to_user(arg, reg, sizeof(*reg))) {
874 ret = -EFAULT;
875 goto err_xa_erase;
876 }
877
878 scoped_guard(mutex, &ctx->mmap_lock) {
879 ret = -ENOMEM;
880 if (xa_store(&ctx->zcrx_ctxs, id, ifq, GFP_KERNEL))
881 goto err_xa_erase;
882 }
883
884 return 0;
885err_xa_erase:
886 scoped_guard(mutex, &ctx->mmap_lock)
887 xa_erase(&ctx->zcrx_ctxs, id);
888err:
889 /* the import path never set the ->master_ctx ref, don't drop it */
890 zcrx_unregister(ifq, NULL);
891 return ret;
892}
893
894static int zcrx_register_netdev(struct io_zcrx_ifq *ifq,
895 struct io_uring_zcrx_ifq_reg *reg,
896 struct io_uring_zcrx_area_reg *area)
897{
898 struct pp_memory_provider_params mp_param = {};
899 unsigned if_rxq = reg->if_rxq;
900 int ret;
901
902 ifq->netdev = netdev_get_by_index_lock(current->nsproxy->net_ns,
903 reg->if_idx);
904 if (!ifq->netdev)
905 return -ENODEV;
906
907 netdev_hold(ifq->netdev, &ifq->netdev_tracker, GFP_KERNEL);
908
909 ifq->dev = netdev_queue_get_dma_dev(ifq->netdev, if_rxq, NETDEV_QUEUE_TYPE_RX);
910 if (!ifq->dev) {
911 ret = -EOPNOTSUPP;
912 goto netdev_put_unlock;
913 }
914 get_device(ifq->dev);
915
916 ret = io_zcrx_create_area(ifq, area, reg);
917 if (ret)
918 goto netdev_put_unlock;
919
920 if (reg->rx_buf_len)
921 mp_param.rx_page_size = 1U << ifq->niov_shift;
922 mp_param.mp_ops = &io_uring_pp_zc_ops;
923 mp_param.mp_priv = ifq;
924 ret = netif_mp_open_rxq(ifq->netdev, if_rxq, &mp_param, NULL);
925 if (ret)
926 goto netdev_put_unlock;
927
928 ifq->if_rxq = if_rxq;
929 ret = 0;
930netdev_put_unlock:
931 netdev_unlock(ifq->netdev);
932 return ret;
933}
934
935static int zcrx_validate_notif_stats(struct io_zcrx_ifq *ifq,
936 const struct io_uring_zcrx_ifq_reg *reg,
937 const struct zcrx_event_desc *notif)
938{
939 size_t stats_off = notif->stats_offset;
940 size_t used, end;
941
942 used = reg->offsets.rqes +
943 sizeof(struct io_uring_zcrx_rqe) * reg->rq_entries;
944
945 if (!IS_ALIGNED(stats_off, __alignof__(struct zcrx_stats)))
946 return -EINVAL;
947 if (stats_off < used)
948 return -ERANGE;
949 if (check_add_overflow(stats_off,
950 sizeof(struct zcrx_stats),
951 &end))
952 return -ERANGE;
953 if (end > io_region_size(&ifq->rq_region))
954 return -ERANGE;
955
956 ifq->notif_stats = io_region_get_ptr(&ifq->rq_region) + stats_off;
957 memset(ifq->notif_stats, 0, sizeof(*ifq->notif_stats));
958
959 return 0;
960}
961
962int io_register_zcrx(struct io_ring_ctx *ctx,
963 struct io_uring_zcrx_ifq_reg __user *arg)
964{
965 struct zcrx_event_desc notif;
966 struct io_uring_zcrx_area_reg area;
967 struct io_uring_zcrx_ifq_reg reg;
968 struct io_uring_region_desc rd;
969 struct io_zcrx_ifq *ifq;
970 int ret;
971 u32 id;
972
973 /*
974 * 1. Interface queue allocation.
975 * 2. It can observe data destined for sockets of other tasks.
976 */
977 if (!capable(CAP_NET_ADMIN))
978 return -EPERM;
979
980 /* mandatory io_uring features for zc rx */
981 if (!(ctx->flags & IORING_SETUP_DEFER_TASKRUN))
982 return -EINVAL;
983 if (!(ctx->flags & (IORING_SETUP_CQE32|IORING_SETUP_CQE_MIXED)))
984 return -EINVAL;
985 if (copy_from_user(&reg, arg, sizeof(reg)))
986 return -EFAULT;
987 if (!mem_is_zero(&reg.__resv, sizeof(reg.__resv)) || reg.zcrx_id)
988 return -EINVAL;
989 if (reg.flags & ~ZCRX_SUPPORTED_REG_FLAGS)
990 return -EINVAL;
991 if (reg.flags & ZCRX_REG_IMPORT)
992 return import_zcrx(ctx, arg, &reg);
993 if (copy_from_user(&rd, u64_to_user_ptr(reg.region_ptr), sizeof(rd)))
994 return -EFAULT;
995 if (reg.if_rxq == -1 || !reg.rq_entries)
996 return -EINVAL;
997 if ((reg.if_rxq || reg.if_idx) && (reg.flags & ZCRX_REG_NODEV))
998 return -EINVAL;
999 if (reg.rq_entries > IO_RQ_MAX_ENTRIES) {
1000 if (!(ctx->flags & IORING_SETUP_CLAMP))
1001 return -EINVAL;
1002 reg.rq_entries = IO_RQ_MAX_ENTRIES;
1003 }
1004 reg.rq_entries = roundup_pow_of_two(reg.rq_entries);
1005
1006 if (copy_from_user(&area, u64_to_user_ptr(reg.area_ptr), sizeof(area)))
1007 return -EFAULT;
1008 if (area.rq_area_token)
1009 return -EINVAL;
1010
1011 memset(&notif, 0, sizeof(notif));
1012 if (reg.event_desc && copy_from_user(&notif, u64_to_user_ptr(reg.event_desc),
1013 sizeof(notif)))
1014 return -EFAULT;
1015 if (notif.type_mask & ~ZCRX_EVENT_TYPE_MASK)
1016 return -EINVAL;
1017 if (notif.flags & ~ZCRX_EVENT_DESC_FLAG_STATS)
1018 return -EINVAL;
1019 if (!(notif.flags & ZCRX_EVENT_DESC_FLAG_STATS)) {
1020 if (notif.stats_offset)
1021 return -EINVAL;
1022 }
1023 if (!mem_is_zero(&notif.__resv2, sizeof(notif.__resv2)))
1024 return -EINVAL;
1025
1026 ifq = io_zcrx_ifq_alloc(ctx);
1027 if (!ifq)
1028 return -ENOMEM;
1029
1030 ifq->notif_data = notif.user_data;
1031 ifq->allowed_notif_mask = notif.type_mask;
1032
1033 if (ctx->user) {
1034 get_uid(ctx->user);
1035 ifq->user = ctx->user;
1036 }
1037 if (ctx->mm_account) {
1038 mmgrab(ctx->mm_account);
1039 ifq->mm_account = ctx->mm_account;
1040 }
1041 ifq->rq.nr_entries = reg.rq_entries;
1042
1043 scoped_guard(mutex, &ctx->mmap_lock) {
1044 /* preallocate id */
1045 ret = xa_alloc(&ctx->zcrx_ctxs, &id, NULL, xa_limit_31b, GFP_KERNEL);
1046 if (ret)
1047 goto ifq_free;
1048 }
1049
1050 ret = io_allocate_rbuf_ring(ctx, ifq, &reg, &rd, id);
1051 if (ret)
1052 goto err;
1053
1054 if (notif.flags & ZCRX_EVENT_DESC_FLAG_STATS) {
1055 ret = zcrx_validate_notif_stats(ifq, &reg, &notif);
1056 if (ret)
1057 goto err;
1058 }
1059
1060 ifq->kern_readable = !(area.flags & IORING_ZCRX_AREA_DMABUF);
1061
1062 if (!(reg.flags & ZCRX_REG_NODEV)) {
1063 ret = zcrx_register_netdev(ifq, &reg, &area);
1064 if (ret)
1065 goto err;
1066 } else {
1067 ret = io_zcrx_create_area(ifq, &area, &reg);
1068 if (ret)
1069 goto err;
1070 }
1071
1072 WARN_ON_ONCE(!ifq->niov_shift);
1073
1074 reg.zcrx_id = id;
1075
1076 scoped_guard(mutex, &ctx->mmap_lock) {
1077 /* publish ifq */
1078 ret = -ENOMEM;
1079 if (xa_store(&ctx->zcrx_ctxs, id, ifq, GFP_KERNEL))
1080 goto err;
1081 }
1082
1083 reg.rx_buf_len = 1U << ifq->niov_shift;
1084
1085 if (copy_to_user(arg, &reg, sizeof(reg)) ||
1086 copy_to_user(u64_to_user_ptr(reg.region_ptr), &rd, sizeof(rd)) ||
1087 copy_to_user(u64_to_user_ptr(reg.area_ptr), &area, sizeof(area))) {
1088 ret = -EFAULT;
1089 goto err;
1090 }
1091
1092 if (notif.type_mask)
1093 zcrx_set_ring_ctx(ifq, ctx);
1094 return 0;
1095err:
1096 scoped_guard(mutex, &ctx->mmap_lock)
1097 xa_erase(&ctx->zcrx_ctxs, id);
1098ifq_free:
1099 zcrx_unregister(ifq, ctx);
1100 return ret;
1101}
1102
1103static inline bool is_zcrx_entry_marked(struct io_ring_ctx *ctx, unsigned long id)
1104{
1105 return xa_get_mark(&ctx->zcrx_ctxs, id, XA_MARK_1);
1106}
1107
1108static inline void set_zcrx_entry_mark(struct io_ring_ctx *ctx, unsigned long id)
1109{
1110 xa_set_mark(&ctx->zcrx_ctxs, id, XA_MARK_1);
1111}
1112
1113void io_terminate_zcrx(struct io_ring_ctx *ctx)
1114{
1115 struct io_zcrx_ifq *ifq;
1116 unsigned long id = 0;
1117
1118 lockdep_assert_held(&ctx->uring_lock);
1119
1120 while (1) {
1121 scoped_guard(mutex, &ctx->mmap_lock)
1122 ifq = xa_find(&ctx->zcrx_ctxs, &id, ULONG_MAX, XA_PRESENT);
1123 if (!ifq)
1124 break;
1125 if (WARN_ON_ONCE(is_zcrx_entry_marked(ctx, id)))
1126 break;
1127 set_zcrx_entry_mark(ctx, id);
1128 id++;
1129 zcrx_unregister_user(ifq, ctx);
1130 }
1131}
1132
1133void io_unregister_zcrx(struct io_ring_ctx *ctx)
1134{
1135 struct io_zcrx_ifq *ifq;
1136
1137 lockdep_assert_held(&ctx->uring_lock);
1138
1139 while (1) {
1140 scoped_guard(mutex, &ctx->mmap_lock) {
1141 unsigned long id = 0;
1142
1143 ifq = xa_find(&ctx->zcrx_ctxs, &id, ULONG_MAX, XA_PRESENT);
1144 if (ifq) {
1145 if (WARN_ON_ONCE(!is_zcrx_entry_marked(ctx, id))) {
1146 ifq = NULL;
1147 break;
1148 }
1149 xa_erase(&ctx->zcrx_ctxs, id);
1150 }
1151 }
1152 if (!ifq)
1153 break;
1154 /*
1155 * io_uring can run requests and return buffers to the user
1156 * after termination, scrub it again.
1157 */
1158 if (refcount_read(&ifq->user_refs) == 0)
1159 io_zcrx_scrub(ifq);
1160 io_put_zcrx_ifq(ifq);
1161 }
1162
1163 xa_destroy(&ctx->zcrx_ctxs);
1164}
1165
1166struct zcrx_rq_iter {
1167 int rqes_left;
1168 bool flushed;
1169};
1170
1171static inline u32 __zcrx_rq_entries(struct zcrx_rq *rq)
1172{
1173 u32 entries = rq->cached_tail - rq->cached_head;
1174
1175 return min(entries, rq->nr_entries);
1176}
1177
1178static inline u32 zcrx_rq_entries(struct zcrx_rq *rq)
1179{
1180 rq->cached_tail = smp_load_acquire(&rq->ring->tail);
1181 return __zcrx_rq_entries(rq);
1182}
1183
1184static struct io_uring_zcrx_rqe *zcrx_next_rqe(struct zcrx_rq *rq, unsigned mask)
1185{
1186 unsigned int idx = rq->cached_head++ & mask;
1187
1188 return &rq->rqes[idx];
1189}
1190
1191static inline void zcrx_rq_iter_init(struct zcrx_rq_iter *it,
1192 struct zcrx_rq *rq)
1193{
1194 it->rqes_left = min_t(unsigned, __zcrx_rq_entries(rq), ZCRX_REFILL_CAP);
1195 it->flushed = false;
1196}
1197
1198static inline bool zcrx_rq_iter_next(struct zcrx_rq_iter *it,
1199 struct zcrx_rq *rq,
1200 struct io_uring_zcrx_rqe **rqe)
1201{
1202 it->rqes_left--;
1203 if (unlikely(it->rqes_left < 0)) {
1204 if (it->flushed)
1205 return false;
1206 rq->cached_tail = smp_load_acquire(&rq->ring->tail);
1207 it->rqes_left = min_t(unsigned, __zcrx_rq_entries(rq),
1208 ZCRX_REFILL_CAP);
1209 it->flushed = true;
1210 if (--it->rqes_left < 0)
1211 return false;
1212 }
1213
1214 *rqe = zcrx_next_rqe(rq, rq->nr_entries - 1);
1215 return true;
1216}
1217
1218static inline bool io_parse_rqe(struct io_uring_zcrx_rqe *rqe,
1219 struct io_zcrx_ifq *ifq,
1220 struct net_iov **ret_niov)
1221{
1222 __u64 off = READ_ONCE(rqe->off);
1223 unsigned niov_idx, area_idx;
1224 struct io_zcrx_area *area;
1225
1226 lockdep_assert_held(&ifq->rq.lock);
1227
1228 area_idx = off >> IORING_ZCRX_AREA_SHIFT;
1229 niov_idx = (off & ~IORING_ZCRX_AREA_MASK) >> ifq->niov_shift;
1230
1231 if (unlikely(rqe->__pad || area_idx >= ifq->nr_areas))
1232 return false;
1233 area_idx = array_index_nospec(area_idx, ifq->nr_areas);
1234 area = ifq->areas[area_idx];
1235
1236 if (unlikely(niov_idx >= area->nia.num_niovs))
1237 return false;
1238 niov_idx = array_index_nospec(niov_idx, area->nia.num_niovs);
1239
1240 *ret_niov = &area->nia.niovs[niov_idx];
1241 return true;
1242}
1243
1244static bool zcrx_put_refill_niov(struct net_iov *niov, struct page_pool *pp,
1245 unsigned refs)
1246{
1247 netmem_ref netmem = net_iov_to_netmem(niov);
1248
1249 if (!io_zcrx_put_niov_uref(niov, refs))
1250 return false;
1251 if (page_pool_unref_netmem(netmem, refs) != 0)
1252 return false;
1253 if (unlikely(niov->desc.pp != pp)) {
1254 io_zcrx_return_niov(niov);
1255 return false;
1256 }
1257 return true;
1258}
1259
1260static unsigned io_zcrx_ring_refill(struct page_pool *pp,
1261 struct io_zcrx_ifq *ifq,
1262 netmem_ref *netmems, unsigned to_alloc)
1263{
1264 struct zcrx_rq *rq = &ifq->rq;
1265 struct io_uring_zcrx_rqe *rqe;
1266 struct zcrx_rq_iter it;
1267 struct net_iov *niov = NULL;
1268 unsigned niov_refs = 0;
1269 unsigned allocated = 0;
1270
1271 guard(spinlock_bh)(&rq->lock);
1272
1273 zcrx_rq_iter_init(&it, rq);
1274
1275 while (allocated < to_alloc - 1 && zcrx_rq_iter_next(&it, rq, &rqe)) {
1276 struct net_iov *next_niov;
1277
1278 if (!io_parse_rqe(rqe, ifq, &next_niov))
1279 continue;
1280 if (niov == next_niov) {
1281 niov_refs++;
1282 continue;
1283 }
1284 if (niov && zcrx_put_refill_niov(niov, pp, niov_refs)) {
1285 netmems[allocated] = net_iov_to_netmem(niov);
1286 allocated++;
1287 }
1288 niov = next_niov;
1289 niov_refs = 1;
1290 }
1291
1292 if (niov && zcrx_put_refill_niov(niov, pp, niov_refs)) {
1293 netmems[allocated] = net_iov_to_netmem(niov);
1294 allocated++;
1295 }
1296
1297 smp_store_release(&rq->ring->head, rq->cached_head);
1298 return allocated;
1299}
1300
1301static unsigned io_zcrx_refill_slow(struct page_pool *pp, struct io_zcrx_ifq *ifq,
1302 netmem_ref *netmems, unsigned to_alloc)
1303{
1304 unsigned area_idx = 0;
1305 unsigned allocated = 0;
1306
1307 guard(spinlock_bh)(&ifq->alloc_lock);
1308
1309 while (allocated < to_alloc) {
1310 struct net_iov *niov = zcrx_get_free_niov(ifq->areas[area_idx]);
1311
1312 if (!niov) {
1313 area_idx++;
1314 if (area_idx >= ifq->nr_areas)
1315 break;
1316 continue;
1317 }
1318
1319 net_mp_niov_set_page_pool(pp, niov);
1320 netmems[allocated] = net_iov_to_netmem(niov);
1321 allocated++;
1322 }
1323 return allocated;
1324}
1325
1326static void zcrx_notif_tw(struct io_tw_req tw_req, io_tw_token_t tw)
1327{
1328 struct io_kiocb *req = tw_req.req;
1329 struct io_ring_ctx *ctx = req->ctx;
1330
1331 io_post_aux_cqe(ctx, req->cqe.user_data, req->cqe.res, 0);
1332 percpu_ref_put(&ctx->refs);
1333 io_poison_req(req);
1334 kmem_cache_free(req_cachep, req);
1335}
1336
1337static void zcrx_stat_add(__u64 *p, s64 v)
1338{
1339 WRITE_ONCE(*p, READ_ONCE(*p) + v);
1340}
1341
1342static void zcrx_send_notif(struct io_zcrx_ifq *ifq, unsigned type)
1343{
1344 gfp_t gfp = GFP_ATOMIC | __GFP_NOWARN | __GFP_ZERO;
1345 u32 type_mask = 1 << type;
1346 struct io_kiocb *req;
1347
1348 if (!(type_mask & ifq->allowed_notif_mask))
1349 return;
1350
1351 guard(spinlock_bh)(&ifq->ctx_lock);
1352 if (!ifq->master_ctx)
1353 return;
1354 if (type_mask & ifq->fired_notifs)
1355 return;
1356
1357 req = kmem_cache_alloc(req_cachep, gfp);
1358 if (unlikely(!req))
1359 return;
1360
1361 ifq->fired_notifs |= type_mask;
1362
1363 req->opcode = IORING_OP_NOP;
1364 req->cqe.user_data = ifq->notif_data;
1365 req->cqe.res = type;
1366 req->ctx = ifq->master_ctx;
1367 percpu_ref_get(&req->ctx->refs);
1368 req->tctx = NULL;
1369 req->io_task_work.func = zcrx_notif_tw;
1370 io_req_task_work_add(req);
1371}
1372
1373static netmem_ref io_pp_zc_alloc_netmems(struct page_pool *pp, gfp_t gfp)
1374{
1375 struct io_zcrx_ifq *ifq = io_pp_to_ifq(pp);
1376 netmem_ref *netmems = pp->alloc.cache;
1377 unsigned to_alloc = PP_ALLOC_CACHE_REFILL;
1378 unsigned allocated;
1379
1380 /* pp should already be ensuring that */
1381 if (WARN_ON_ONCE(pp->alloc.count))
1382 return 0;
1383
1384 allocated = io_zcrx_ring_refill(pp, ifq, netmems, to_alloc);
1385 if (likely(allocated))
1386 goto out_return;
1387
1388 allocated = io_zcrx_refill_slow(pp, ifq, netmems, to_alloc);
1389 if (!allocated) {
1390 zcrx_send_notif(ifq, ZCRX_EVENT_ALLOC_FAIL);
1391 return 0;
1392 }
1393out_return:
1394 zcrx_sync_for_device(pp, ifq, netmems, allocated);
1395 allocated--;
1396 pp->alloc.count += allocated;
1397 return netmems[allocated];
1398}
1399
1400static bool io_pp_zc_release_netmem(struct page_pool *pp, netmem_ref netmem)
1401{
1402 struct net_iov *niov;
1403
1404 if (WARN_ON_ONCE(!netmem_is_net_iov(netmem)))
1405 return false;
1406
1407 niov = netmem_to_net_iov(netmem);
1408 net_mp_niov_clear_page_pool(niov);
1409 io_zcrx_return_niov_freelist(niov);
1410 return false;
1411}
1412
1413static int io_pp_zc_init(struct page_pool *pp)
1414{
1415 struct io_zcrx_ifq *ifq = io_pp_to_ifq(pp);
1416
1417 if (WARN_ON_ONCE(!ifq))
1418 return -EINVAL;
1419 if (WARN_ON_ONCE(ifq->dev != pp->p.dev))
1420 return -EINVAL;
1421 if (WARN_ON_ONCE(!pp->dma_map))
1422 return -EOPNOTSUPP;
1423 if (pp->p.order + PAGE_SHIFT != ifq->niov_shift)
1424 return -EINVAL;
1425 if (pp->p.dma_dir != DMA_FROM_DEVICE)
1426 return -EOPNOTSUPP;
1427
1428 refcount_inc(&ifq->refs);
1429 return 0;
1430}
1431
1432static void io_pp_zc_destroy(struct page_pool *pp)
1433{
1434 io_put_zcrx_ifq(io_pp_to_ifq(pp));
1435}
1436
1437static int io_pp_nl_fill(void *mp_priv, struct sk_buff *rsp,
1438 struct netdev_rx_queue *rxq)
1439{
1440 struct io_zcrx_ifq *ifq = mp_priv;
1441 struct nlattr *nest;
1442 int type;
1443
1444 type = rxq ? NETDEV_A_QUEUE_IO_URING : NETDEV_A_PAGE_POOL_IO_URING;
1445 nest = nla_nest_start(rsp, type);
1446 if (!nest)
1447 return -EMSGSIZE;
1448
1449 if (nla_put_uint(rsp, NETDEV_A_IO_URING_PROVIDER_INFO_RX_BUF_LEN,
1450 1ULL << ifq->niov_shift)) {
1451 nla_nest_cancel(rsp, nest);
1452 return -EMSGSIZE;
1453 }
1454
1455 nla_nest_end(rsp, nest);
1456
1457 return 0;
1458}
1459
1460static void io_pp_uninstall(void *mp_priv, struct netdev_rx_queue *rxq)
1461{
1462 struct pp_memory_provider_params *p = &rxq->mp_params;
1463 struct io_zcrx_ifq *ifq = mp_priv;
1464
1465 guard(mutex)(&ifq->pp_lock);
1466 io_zcrx_unmap_areas(ifq);
1467 io_zcrx_drop_netdev(ifq);
1468
1469 p->mp_ops = NULL;
1470 p->mp_priv = NULL;
1471}
1472
1473static const struct memory_provider_ops io_uring_pp_zc_ops = {
1474 .alloc_netmems = io_pp_zc_alloc_netmems,
1475 .release_netmem = io_pp_zc_release_netmem,
1476 .init = io_pp_zc_init,
1477 .destroy = io_pp_zc_destroy,
1478 .nl_fill = io_pp_nl_fill,
1479 .uninstall = io_pp_uninstall,
1480};
1481
1482static unsigned zcrx_parse_rq(netmem_ref *netmem_array, unsigned nr,
1483 struct io_zcrx_ifq *zcrx, struct zcrx_rq *rq)
1484{
1485 unsigned int mask = rq->nr_entries - 1;
1486 unsigned int i;
1487
1488 nr = min(nr, zcrx_rq_entries(rq));
1489 for (i = 0; i < nr; i++) {
1490 struct io_uring_zcrx_rqe *rqe = zcrx_next_rqe(rq, mask);
1491 struct net_iov *niov;
1492
1493 if (!io_parse_rqe(rqe, zcrx, &niov))
1494 break;
1495 netmem_array[i] = net_iov_to_netmem(niov);
1496 }
1497
1498 smp_store_release(&rq->ring->head, rq->cached_head);
1499 return i;
1500}
1501
1502#define ZCRX_FLUSH_BATCH 32
1503
1504static void zcrx_return_buffers(netmem_ref *netmems, unsigned nr)
1505{
1506 unsigned i;
1507
1508 for (i = 0; i < nr; i++) {
1509 netmem_ref netmem = netmems[i];
1510 struct net_iov *niov = netmem_to_net_iov(netmem);
1511
1512 if (!io_zcrx_put_niov_uref(niov, 1))
1513 continue;
1514 if (!page_pool_unref_and_test(netmem))
1515 continue;
1516 io_zcrx_return_niov(niov);
1517 }
1518}
1519
1520static int zcrx_flush_rq(struct io_ring_ctx *ctx, struct io_zcrx_ifq *zcrx,
1521 struct zcrx_ctrl *ctrl)
1522{
1523 struct zcrx_ctrl_flush_rq *frq = &ctrl->zc_flush;
1524 netmem_ref netmems[ZCRX_FLUSH_BATCH];
1525 unsigned total = 0;
1526 unsigned nr;
1527
1528 if (!mem_is_zero(&frq->__resv, sizeof(frq->__resv)))
1529 return -EINVAL;
1530
1531 do {
1532 struct zcrx_rq *rq = &zcrx->rq;
1533
1534 scoped_guard(spinlock_bh, &rq->lock) {
1535 nr = zcrx_parse_rq(netmems, ZCRX_FLUSH_BATCH, zcrx, rq);
1536 zcrx_return_buffers(netmems, nr);
1537 }
1538
1539 total += nr;
1540
1541 if (fatal_signal_pending(current))
1542 break;
1543 cond_resched();
1544 } while (nr == ZCRX_FLUSH_BATCH && total < zcrx->rq.nr_entries);
1545
1546 return 0;
1547}
1548
1549static int zcrx_arm_notif(struct io_ring_ctx *ctx, struct io_zcrx_ifq *zcrx,
1550 struct zcrx_ctrl *ctrl)
1551{
1552 const struct zcrx_ctrl_arm_event *an = &ctrl->zc_arm_event;
1553 unsigned type_mask;
1554
1555 if (an->event_type >= __ZCRX_EVENT_TYPE_LAST)
1556 return -EINVAL;
1557 if (!mem_is_zero(&an->__resv, sizeof(an->__resv)))
1558 return -EINVAL;
1559
1560 guard(spinlock_bh)(&zcrx->ctx_lock);
1561 type_mask = 1U << an->event_type;
1562 if (type_mask & ~zcrx->fired_notifs)
1563 return -EINVAL;
1564 zcrx->fired_notifs &= ~type_mask;
1565 return 0;
1566}
1567
1568static int zcrx_ctrl_add_area(struct io_ring_ctx *ctx, struct io_zcrx_ifq *ifq,
1569 struct zcrx_ctrl *ctrl)
1570{
1571 struct zcrx_ctrl_add_area *ctrl_add = &ctrl->zc_area;
1572 struct io_uring_zcrx_area_reg __user *area_uptr;
1573 struct io_uring_zcrx_area_reg area_reg;
1574
1575 area_uptr = u64_to_user_ptr(ctrl_add->area_ptr);
1576 if (copy_from_user(&area_reg, area_uptr, sizeof(area_reg)))
1577 return -EFAULT;
1578 if (!mem_is_zero(&ctrl_add->__resv, sizeof(ctrl_add->__resv)))
1579 return -EINVAL;
1580 if (area_reg.rq_area_token)
1581 return -EINVAL;
1582
1583 guard(mutex)(&ifq->pp_lock);
1584 if (ifq->dev && !ifq->netdev)
1585 return -EFAULT;
1586
1587 /* we can't safely roll back area append, copy it out first */
1588 area_reg.rq_area_token = zcrx_area_id_to_token(zcrx_next_area_id(ifq));
1589 if (copy_to_user(area_uptr, &area_reg, sizeof(area_reg)))
1590 return -EFAULT;
1591 area_reg.rq_area_token = 0;
1592
1593 return __zcrx_create_area(ifq, &area_reg, 1U << ifq->niov_shift);
1594}
1595
1596int io_zcrx_ctrl(struct io_ring_ctx *ctx, void __user *arg, unsigned nr_args)
1597{
1598 struct zcrx_ctrl ctrl;
1599 struct io_zcrx_ifq *zcrx;
1600
1601 BUILD_BUG_ON(sizeof(ctrl.zc_export) != sizeof(ctrl.zc_flush));
1602 BUILD_BUG_ON(sizeof(ctrl.zc_export) != sizeof(ctrl.zc_arm_event));
1603
1604 if (nr_args)
1605 return -EINVAL;
1606 if (copy_from_user(&ctrl, arg, sizeof(ctrl)))
1607 return -EFAULT;
1608 if (!mem_is_zero(&ctrl.__resv, sizeof(ctrl.__resv)))
1609 return -EFAULT;
1610
1611 zcrx = xa_load(&ctx->zcrx_ctxs, ctrl.zcrx_id);
1612 if (!zcrx)
1613 return -ENXIO;
1614
1615 switch (ctrl.op) {
1616 case ZCRX_CTRL_FLUSH_RQ:
1617 return zcrx_flush_rq(ctx, zcrx, &ctrl);
1618 case ZCRX_CTRL_EXPORT:
1619 return zcrx_export(ctx, zcrx, &ctrl, arg);
1620 case ZCRX_CTRL_ARM_EVENT:
1621 return zcrx_arm_notif(ctx, zcrx, &ctrl);
1622 case ZCRX_CTRL_ADD_AREA:
1623 return zcrx_ctrl_add_area(ctx, zcrx, &ctrl);
1624 }
1625
1626 return -EOPNOTSUPP;
1627}
1628
1629static bool io_zcrx_queue_cqe(struct io_kiocb *req, struct net_iov *niov,
1630 struct io_zcrx_ifq *ifq, int off, int len)
1631{
1632 struct io_ring_ctx *ctx = req->ctx;
1633 struct io_uring_zcrx_cqe *rcqe;
1634 struct io_zcrx_area *area;
1635 struct io_uring_cqe *cqe;
1636 u64 offset;
1637
1638 if (!io_defer_get_uncommited_cqe(ctx, &cqe))
1639 return false;
1640
1641 cqe->user_data = req->cqe.user_data;
1642 cqe->res = len;
1643 cqe->flags = IORING_CQE_F_MORE;
1644 if (ctx->flags & IORING_SETUP_CQE_MIXED)
1645 cqe->flags |= IORING_CQE_F_32;
1646
1647 area = io_zcrx_iov_to_area(niov);
1648 offset = off + (net_iov_idx(niov) << ifq->niov_shift);
1649 rcqe = (struct io_uring_zcrx_cqe *)(cqe + 1);
1650 rcqe->off = offset + zcrx_area_id_to_token(area->area_id);
1651 rcqe->__pad = 0;
1652 return true;
1653}
1654
1655static struct net_iov *io_alloc_fallback_niov(struct io_zcrx_ifq *ifq)
1656{
1657 struct net_iov *niov = NULL;
1658 unsigned area_idx;
1659
1660 if (!ifq->kern_readable)
1661 return NULL;
1662
1663 guard(spinlock_bh)(&ifq->alloc_lock);
1664
1665 for (area_idx = 0; area_idx < ifq->nr_areas; area_idx++) {
1666 niov = zcrx_get_free_niov(ifq->areas[area_idx]);
1667 if (niov) {
1668 page_pool_fragment_netmem(net_iov_to_netmem(niov), 1);
1669 return niov;
1670 }
1671 }
1672
1673 return NULL;
1674}
1675
1676struct io_copy_cache {
1677 struct page *page;
1678 unsigned long offset;
1679 size_t size;
1680};
1681
1682static ssize_t io_copy_page(struct io_copy_cache *cc, struct page *src_page,
1683 unsigned int src_offset, size_t len)
1684{
1685 size_t copied = 0;
1686
1687 len = min(len, cc->size);
1688
1689 while (len) {
1690 void *src_addr, *dst_addr;
1691 struct page *dst_page = cc->page;
1692 unsigned dst_offset = cc->offset;
1693 size_t n = len;
1694
1695 if (folio_test_partial_kmap(page_folio(dst_page)) ||
1696 folio_test_partial_kmap(page_folio(src_page))) {
1697 dst_page += dst_offset / PAGE_SIZE;
1698 dst_offset = offset_in_page(dst_offset);
1699 src_page += src_offset / PAGE_SIZE;
1700 src_offset = offset_in_page(src_offset);
1701 n = min(PAGE_SIZE - src_offset, PAGE_SIZE - dst_offset);
1702 n = min(n, len);
1703 }
1704
1705 dst_addr = kmap_local_page(dst_page) + dst_offset;
1706 src_addr = kmap_local_page(src_page) + src_offset;
1707
1708 memcpy(dst_addr, src_addr, n);
1709
1710 kunmap_local(src_addr);
1711 kunmap_local(dst_addr);
1712
1713 cc->size -= n;
1714 cc->offset += n;
1715 src_offset += n;
1716 len -= n;
1717 copied += n;
1718 }
1719 return copied;
1720}
1721
1722static ssize_t io_zcrx_copy_chunk(struct io_kiocb *req, struct io_zcrx_ifq *ifq,
1723 struct page *src_page, unsigned int src_offset,
1724 size_t len)
1725{
1726 size_t copied = 0;
1727 int ret = 0;
1728
1729 while (len) {
1730 struct io_copy_cache cc;
1731 struct net_iov *niov;
1732 size_t n;
1733
1734 niov = io_alloc_fallback_niov(ifq);
1735 if (!niov) {
1736 ret = -ENOMEM;
1737 break;
1738 }
1739
1740 cc.page = io_zcrx_iov_page(niov);
1741 cc.offset = 0;
1742 cc.size = PAGE_SIZE;
1743
1744 n = io_copy_page(&cc, src_page, src_offset, len);
1745
1746 if (!io_zcrx_queue_cqe(req, niov, ifq, 0, n)) {
1747 io_zcrx_return_niov(niov);
1748 ret = -ENOSPC;
1749 break;
1750 }
1751
1752 io_zcrx_get_niov_uref(niov);
1753 src_offset += n;
1754 len -= n;
1755 copied += n;
1756 }
1757
1758 return copied ? copied : ret;
1759}
1760
1761static int io_zcrx_copy_frag(struct io_kiocb *req, struct io_zcrx_ifq *ifq,
1762 const skb_frag_t *frag, int off, int len)
1763{
1764 struct page *page = skb_frag_page(frag);
1765 int ret;
1766
1767 ret = io_zcrx_copy_chunk(req, ifq, page, off + skb_frag_off(frag), len);
1768 if (ret > 0) {
1769 if (ifq->notif_stats) {
1770 zcrx_stat_add(&ifq->notif_stats->copy_count, 1);
1771 zcrx_stat_add(&ifq->notif_stats->copy_bytes, ret);
1772 }
1773 zcrx_send_notif(ifq, ZCRX_EVENT_COPY);
1774 }
1775
1776 return ret;
1777}
1778
1779static int io_zcrx_recv_frag(struct io_kiocb *req, struct io_zcrx_ifq *ifq,
1780 const skb_frag_t *frag, int off, int len)
1781{
1782 struct net_iov *niov;
1783 struct page_pool *pp;
1784
1785 if (unlikely(!skb_frag_is_net_iov(frag)))
1786 return io_zcrx_copy_frag(req, ifq, frag, off, len);
1787
1788 niov = netmem_to_net_iov(frag->netmem);
1789 pp = niov->desc.pp;
1790
1791 if (!pp || pp->mp_ops != &io_uring_pp_zc_ops || io_pp_to_ifq(pp) != ifq)
1792 return -EFAULT;
1793
1794 if (!io_zcrx_queue_cqe(req, niov, ifq, off + skb_frag_off(frag), len))
1795 return -ENOSPC;
1796
1797 /*
1798 * Prevent it from being recycled while user is accessing it.
1799 * It has to be done before grabbing a user reference.
1800 */
1801 page_pool_ref_netmem(net_iov_to_netmem(niov));
1802 io_zcrx_get_niov_uref(niov);
1803 return len;
1804}
1805
1806static int
1807io_zcrx_recv_skb(read_descriptor_t *desc, struct sk_buff *skb,
1808 unsigned int offset, size_t len)
1809{
1810 struct io_zcrx_args *args = desc->arg.data;
1811 struct io_zcrx_ifq *ifq = args->ifq;
1812 struct io_kiocb *req = args->req;
1813 struct sk_buff *frag_iter;
1814 unsigned start, start_off = offset;
1815 struct skb_shared_info *shi;
1816 int i, ret = 0;
1817
1818 len = min_t(size_t, len, desc->count);
1819 /*
1820 * __tcp_read_sock() always calls io_zcrx_recv_skb one last time, even
1821 * if desc->count is already 0. This is caused by the if (offset + 1 !=
1822 * skb->len) check. Return early in this case to break out of
1823 * __tcp_read_sock().
1824 */
1825 if (!len)
1826 return 0;
1827 if (unlikely(args->nr_skbs++ > IO_SKBS_PER_CALL_LIMIT))
1828 return -EAGAIN;
1829
1830 if (unlikely(offset < skb_headlen(skb))) {
1831 ssize_t copied;
1832 size_t to_copy;
1833
1834 to_copy = min_t(size_t, skb_headlen(skb) - offset, len);
1835 copied = io_zcrx_copy_chunk(req, ifq, virt_to_page(skb->data),
1836 offset_in_page(skb->data) + offset,
1837 to_copy);
1838 if (copied < 0) {
1839 ret = copied;
1840 goto out;
1841 }
1842 offset += copied;
1843 len -= copied;
1844 if (!len)
1845 goto out;
1846 if (offset != skb_headlen(skb))
1847 goto out;
1848 }
1849
1850 start = skb_headlen(skb);
1851 shi = skb_shinfo(skb);
1852
1853 for (i = 0; i < shi->nr_frags; i++) {
1854 const skb_frag_t *frag = &shi->frags[i];
1855 unsigned frag_end = start + skb_frag_size(frag);
1856
1857 if (WARN_ON(start > offset + len))
1858 return -EFAULT;
1859
1860 if (offset < frag_end) {
1861 unsigned copy = min(frag_end - offset, len);
1862 unsigned frag_off = offset - start;
1863
1864 ret = io_zcrx_recv_frag(req, ifq, frag, frag_off, copy);
1865 if (ret < 0)
1866 goto out;
1867
1868 offset += ret;
1869 len -= ret;
1870 if (len == 0 || ret != copy)
1871 goto out;
1872 }
1873 start = frag_end;
1874 }
1875
1876 skb_walk_frags(skb, frag_iter) {
1877 unsigned frag_end;
1878
1879 if (WARN_ON(start > offset + len))
1880 return -EFAULT;
1881
1882 frag_end = start + frag_iter->len;
1883 if (offset < frag_end) {
1884 unsigned copy = min(frag_end - offset, len);
1885 unsigned frag_off = offset - start;
1886 size_t count;
1887
1888 count = desc->count;
1889 ret = io_zcrx_recv_skb(desc, frag_iter, frag_off, copy);
1890 desc->count = count;
1891 if (ret < 0)
1892 goto out;
1893
1894 offset += ret;
1895 len -= ret;
1896 if (len == 0 || ret != copy)
1897 goto out;
1898 }
1899 start = frag_end;
1900 }
1901
1902out:
1903 /* Bytes left in len mean an error stopped the walk early. */
1904 if (len)
1905 args->stopped_early = true;
1906 if (offset == start_off)
1907 return ret;
1908 desc->count -= (offset - start_off);
1909 return offset - start_off;
1910}
1911
1912static int io_zcrx_tcp_recvmsg(struct io_kiocb *req, struct io_zcrx_ifq *ifq,
1913 struct sock *sk, int flags,
1914 unsigned issue_flags, unsigned int *outlen)
1915{
1916 unsigned int len = *outlen;
1917 struct io_zcrx_args args = {
1918 .req = req,
1919 .ifq = ifq,
1920 };
1921 read_descriptor_t rd_desc = {
1922 .count = len ? len : UINT_MAX,
1923 .arg.data = &args,
1924 };
1925 int ret;
1926
1927 lock_sock(sk);
1928 ret = tcp_read_sock(sk, &rd_desc, io_zcrx_recv_skb);
1929 if (len && ret > 0)
1930 *outlen = len - ret;
1931 if (ret <= 0) {
1932 if (ret < 0 || sock_flag(sk, SOCK_DONE))
1933 goto out;
1934 if (sk->sk_err)
1935 ret = sock_error(sk);
1936 else if (sk->sk_shutdown & RCV_SHUTDOWN)
1937 goto out;
1938 else if (sk->sk_state == TCP_CLOSE)
1939 ret = -ENOTCONN;
1940 else
1941 ret = -EAGAIN;
1942 } else if ((issue_flags & IO_URING_F_MULTISHOT) &&
1943 (unlikely(args.nr_skbs > IO_SKBS_PER_CALL_LIMIT) ||
1944 args.stopped_early)) {
1945 ret = IOU_REQUEUE;
1946 } else if (sock_flag(sk, SOCK_DONE)) {
1947 /* Make it to retry until it finally gets 0. */
1948 if (issue_flags & IO_URING_F_MULTISHOT)
1949 ret = IOU_REQUEUE;
1950 else
1951 ret = -EAGAIN;
1952 }
1953out:
1954 release_sock(sk);
1955 return ret;
1956}
1957
1958int io_zcrx_recv(struct io_kiocb *req, struct io_zcrx_ifq *ifq,
1959 struct socket *sock, unsigned int flags,
1960 unsigned issue_flags, unsigned int *len)
1961{
1962 struct sock *sk = sock->sk;
1963 const struct proto *prot = READ_ONCE(sk->sk_prot);
1964
1965 if (prot->recvmsg != tcp_recvmsg)
1966 return -EPROTONOSUPPORT;
1967
1968 sock_rps_record_flow(sk);
1969 return io_zcrx_tcp_recvmsg(req, ifq, sk, flags, issue_flags, len);
1970}