| 1 | |
| 2 | |
| 3 | * Copyright(c) 2018 Intel Corporation.
|
| 4 |
|
| 5 |
|
| 6 | #include <linux/log2.h>
|
| 7 | #include <linux/slab.h>
|
| 8 | #include <linux/overflow.h>
|
| 9 | #include <linux/vmalloc.h>
|
| 10 | #include <net/xdp_sock_drv.h>
|
| 11 |
|
| 12 | #include <asm/shmparam.h>
|
| 13 |
|
| 14 | #include "xsk_queue.h"
|
| 15 |
|
| 16 | static size_t xskq_get_ring_size(struct xsk_queue *q, bool umem_queue)
|
| 17 | {
|
| 18 | struct xdp_umem_ring *umem_ring;
|
| 19 | struct xdp_rxtx_ring *rxtx_ring;
|
| 20 |
|
| 21 | if (umem_queue)
|
| 22 | return struct_size(umem_ring, desc, q->nentries);
|
| 23 | return struct_size(rxtx_ring, desc, q->nentries);
|
| 24 | }
|
| 25 |
|
| 26 | static void *xskq_vmalloc_user(unsigned long size)
|
| 27 | {
|
| 28 | return __vmalloc_node_range(size, SHMLBA, VMALLOC_START, VMALLOC_END,
|
| 29 | GFP_KERNEL_ACCOUNT | __GFP_ZERO, PAGE_KERNEL,
|
| 30 | VM_USERMAP, NUMA_NO_NODE,
|
| 31 | __builtin_return_address(0));
|
| 32 | }
|
| 33 |
|
| 34 | struct xsk_queue *xskq_create(u32 nentries, bool umem_queue)
|
| 35 | {
|
| 36 | struct xsk_queue *q;
|
| 37 | size_t size;
|
| 38 |
|
| 39 | q = kzalloc_obj(*q);
|
| 40 | if (!q)
|
| 41 | return NULL;
|
| 42 |
|
| 43 | q->nentries = nentries;
|
| 44 | q->ring_mask = nentries - 1;
|
| 45 |
|
| 46 | size = xskq_get_ring_size(q, umem_queue);
|
| 47 |
|
| 48 | |
| 49 | * PAGE_ALIGN(), checking SIZE_MAX is enough due to the previous
|
| 50 | * is_power_of_2(), the rest will be handled by vmalloc_user()
|
| 51 |
|
| 52 | if (unlikely(size == SIZE_MAX)) {
|
| 53 | kfree(q);
|
| 54 | return NULL;
|
| 55 | }
|
| 56 |
|
| 57 | size = PAGE_ALIGN(size);
|
| 58 |
|
| 59 | q->ring = xskq_vmalloc_user(size);
|
| 60 | if (!q->ring) {
|
| 61 | kfree(q);
|
| 62 | return NULL;
|
| 63 | }
|
| 64 |
|
| 65 | q->ring_vmalloc_size = size;
|
| 66 | return q;
|
| 67 | }
|
| 68 |
|
| 69 | void xskq_destroy(struct xsk_queue *q)
|
| 70 | {
|
| 71 | if (!q)
|
| 72 | return;
|
| 73 |
|
| 74 | vfree(q->ring);
|
| 75 | kfree(q);
|
| 76 | }
|