projects/linux·blob assoc-array-fixes · refs

462 lines · 11.7 KiB

1// SPDX-License-Identifier: GPL-2.0
2#include <linux/kernel.h>
3#include <linux/init.h>
4#include <linux/errno.h>
5#include <linux/mm.h>
6#include <linux/mman.h>
7#include <linux/slab.h>
8#include <linux/vmalloc.h>
9#include <linux/io_uring.h>
10#include <linux/io_uring_types.h>
11#include <asm/shmparam.h>
12
13#include "memmap.h"
14#include "kbuf.h"
15#include "rsrc.h"
16#include "zcrx.h"
17
18static bool io_mem_alloc_compound(struct page **pages, int nr_pages,
19 size_t size, gfp_t gfp,
20 struct user_struct *user)
21{
22 unsigned long nr_compound, extra;
23 struct page *page;
24 int i, order;
25
26 order = get_order(size);
27 if (order > MAX_PAGE_ORDER)
28 return false;
29 else if (order)
30 gfp |= __GFP_COMP;
31
32 /*
33 * get_order() rounds a non power of two size up, so the allocation
34 * can hold more pages than the region exposes. Account those too,
35 * and leave the compound allocation alone if they do not fit.
36 */
37 nr_compound = 1UL << order;
38 extra = nr_compound - nr_pages;
39 if (extra && user && __io_account_mem(user, extra))
40 return false;
41
42 page = alloc_pages(gfp, order);
43 if (!page) {
44 if (extra && user)
45 __io_unaccount_mem(user, extra);
46 return false;
47 }
48
49 for (i = 0; i < nr_pages; i++)
50 pages[i] = page + i;
51
52 return true;
53}
54
55struct page **io_pin_pages(unsigned long uaddr, unsigned long len, int *npages)
56{
57 unsigned long start, end, nr_pages;
58 struct page **pages;
59 int ret;
60
61 if (check_add_overflow(uaddr, len, &end))
62 return ERR_PTR(-EOVERFLOW);
63 if (check_add_overflow(end, PAGE_SIZE - 1, &end))
64 return ERR_PTR(-EOVERFLOW);
65
66 end = end >> PAGE_SHIFT;
67 start = uaddr >> PAGE_SHIFT;
68 nr_pages = end - start;
69 if (WARN_ON_ONCE(!nr_pages))
70 return ERR_PTR(-EINVAL);
71 if (nr_pages > INT_MAX / sizeof(struct page *))
72 return ERR_PTR(-EOVERFLOW);
73
74 pages = kvmalloc_objs(struct page *, nr_pages, GFP_KERNEL_ACCOUNT);
75 if (!pages)
76 return ERR_PTR(-ENOMEM);
77
78 ret = pin_user_pages_fast(uaddr, nr_pages, FOLL_WRITE | FOLL_LONGTERM,
79 pages);
80 /* success, mapped all pages */
81 if (ret == nr_pages) {
82 *npages = nr_pages;
83 return pages;
84 }
85
86 /* partial map, or didn't map anything */
87 if (ret >= 0) {
88 /* if we did partial map, release any pages we did get */
89 if (ret)
90 unpin_user_pages(pages, ret);
91 ret = -EFAULT;
92 }
93 kvfree(pages);
94 return ERR_PTR(ret);
95}
96
97enum {
98 /* memory was vmap'ed for the kernel, freeing the region vunmap's it */
99 IO_REGION_F_VMAP = 1,
100 /* memory is provided by user and pinned by the kernel */
101 IO_REGION_F_USER_PROVIDED = 2,
102 /* only the first page in the array is ref'ed */
103 IO_REGION_F_SINGLE_REF = 4,
104};
105
106void io_free_region(struct user_struct *user, struct io_mapped_region *mr)
107{
108 if (mr->pages) {
109 long nr_refs = mr->nr_pages;
110
111 if (mr->flags & IO_REGION_F_SINGLE_REF)
112 nr_refs = 1;
113
114 if (mr->flags & IO_REGION_F_USER_PROVIDED)
115 unpin_user_pages(mr->pages, nr_refs);
116 else
117 release_pages(mr->pages, nr_refs);
118
119 kvfree(mr->pages);
120 }
121 if ((mr->flags & IO_REGION_F_VMAP) && mr->ptr)
122 vunmap(mr->ptr);
123 if (mr->nr_pages && user) {
124 unsigned long nr_accounted = mr->nr_pages;
125
126 /* a compound region was accounted for the whole allocation */
127 if (mr->flags & IO_REGION_F_SINGLE_REF)
128 nr_accounted = 1UL << get_order(io_region_size(mr));
129
130 __io_unaccount_mem(user, nr_accounted);
131 }
132
133 memset(mr, 0, sizeof(*mr));
134}
135
136static int io_region_init_ptr(struct io_mapped_region *mr)
137{
138 struct io_imu_folio_data ifd;
139 void *ptr;
140
141 if (io_check_coalesce_buffer(mr->pages, mr->nr_pages, &ifd)) {
142 if (ifd.nr_folios == 1 && !PageHighMem(mr->pages[0])) {
143 mr->ptr = page_address(mr->pages[0]);
144 return 0;
145 }
146 }
147 ptr = vmap(mr->pages, mr->nr_pages, VM_MAP, PAGE_KERNEL);
148 if (!ptr)
149 return -ENOMEM;
150
151 mr->ptr = ptr;
152 mr->flags |= IO_REGION_F_VMAP;
153 return 0;
154}
155
156static int io_region_pin_pages(struct io_mapped_region *mr,
157 struct io_uring_region_desc *reg)
158{
159 size_t size = io_region_size(mr);
160 struct page **pages;
161 int nr_pages;
162
163 pages = io_pin_pages(reg->user_addr, size, &nr_pages);
164 if (IS_ERR(pages))
165 return PTR_ERR(pages);
166 if (WARN_ON_ONCE(nr_pages != mr->nr_pages))
167 return -EFAULT;
168
169 mr->pages = pages;
170 mr->flags |= IO_REGION_F_USER_PROVIDED;
171 return 0;
172}
173
174static int io_region_allocate_pages(struct io_mapped_region *mr,
175 struct io_uring_region_desc *reg,
176 unsigned long mmap_offset,
177 struct user_struct *user)
178{
179 gfp_t gfp = GFP_KERNEL_ACCOUNT | __GFP_ZERO | __GFP_NOWARN;
180 size_t size = io_region_size(mr);
181 unsigned long nr_allocated;
182 struct page **pages;
183
184 pages = kvmalloc_objs(*pages, mr->nr_pages, gfp);
185 if (!pages)
186 return -ENOMEM;
187
188 if (io_mem_alloc_compound(pages, mr->nr_pages, size, gfp, user)) {
189 mr->flags |= IO_REGION_F_SINGLE_REF;
190 goto done;
191 }
192
193 nr_allocated = alloc_pages_bulk_node(gfp, NUMA_NO_NODE,
194 mr->nr_pages, pages);
195 if (nr_allocated != mr->nr_pages) {
196 if (nr_allocated)
197 release_pages(pages, nr_allocated);
198 kvfree(pages);
199 return -ENOMEM;
200 }
201done:
202 reg->mmap_offset = mmap_offset;
203 mr->pages = pages;
204 return 0;
205}
206
207int io_create_region(struct io_ring_ctx *ctx, struct io_mapped_region *mr,
208 struct io_uring_region_desc *reg,
209 unsigned long mmap_offset)
210{
211 int nr_pages, ret;
212 u64 end;
213
214 if (WARN_ON_ONCE(mr->pages || mr->ptr || mr->nr_pages))
215 return -EFAULT;
216 if (memchr_inv(&reg->__resv, 0, sizeof(reg->__resv)))
217 return -EINVAL;
218 if (reg->flags & ~IORING_MEM_REGION_TYPE_USER)
219 return -EINVAL;
220 /* user_addr should be set IFF it's a user memory backed region */
221 if ((reg->flags & IORING_MEM_REGION_TYPE_USER) != !!reg->user_addr)
222 return -EFAULT;
223 if (!reg->size || reg->mmap_offset || reg->id)
224 return -EINVAL;
225 if ((reg->size >> PAGE_SHIFT) > INT_MAX)
226 return -E2BIG;
227 if ((reg->user_addr | reg->size) & ~PAGE_MASK)
228 return -EINVAL;
229 if (check_add_overflow(reg->user_addr, reg->size, &end))
230 return -EOVERFLOW;
231
232 nr_pages = reg->size >> PAGE_SHIFT;
233 if (ctx->user) {
234 ret = __io_account_mem(ctx->user, nr_pages);
235 if (ret)
236 return ret;
237 }
238 mr->nr_pages = nr_pages;
239
240 if (reg->flags & IORING_MEM_REGION_TYPE_USER)
241 ret = io_region_pin_pages(mr, reg);
242 else
243 ret = io_region_allocate_pages(mr, reg, mmap_offset, ctx->user);
244 if (ret)
245 goto out_free;
246
247 ret = io_region_init_ptr(mr);
248 if (ret)
249 goto out_free;
250 return 0;
251out_free:
252 io_free_region(ctx->user, mr);
253 return ret;
254}
255
256static struct io_mapped_region *io_mmap_get_region(struct io_ring_ctx *ctx,
257 loff_t pgoff)
258{
259 loff_t offset = pgoff << PAGE_SHIFT;
260 unsigned int id;
261
262
263 switch (offset & IORING_OFF_MMAP_MASK) {
264 case IORING_OFF_SQ_RING:
265 case IORING_OFF_CQ_RING:
266 return &ctx->ring_region;
267 case IORING_OFF_SQES:
268 return &ctx->sq_region;
269 case IORING_OFF_PBUF_RING:
270 id = (offset & ~IORING_OFF_MMAP_MASK) >> IORING_OFF_PBUF_SHIFT;
271 return io_pbuf_get_region(ctx, id);
272 case IORING_MAP_OFF_PARAM_REGION:
273 return &ctx->param_region;
274 case IORING_MAP_OFF_ZCRX_REGION:
275 id = (offset & ~IORING_OFF_MMAP_MASK) >> IORING_OFF_ZCRX_SHIFT;
276 return io_zcrx_get_region(ctx, id);
277 }
278 return NULL;
279}
280
281static void *io_region_validate_mmap(struct io_ring_ctx *ctx,
282 struct io_mapped_region *mr)
283{
284 lockdep_assert_held(&ctx->mmap_lock);
285
286 if (!io_region_is_set(mr))
287 return ERR_PTR(-EINVAL);
288 if (mr->flags & IO_REGION_F_USER_PROVIDED)
289 return ERR_PTR(-EINVAL);
290
291 return io_region_get_ptr(mr);
292}
293
294static void *io_uring_validate_mmap_request(struct file *file, loff_t pgoff)
295{
296 struct io_ring_ctx *ctx = file->private_data;
297 struct io_mapped_region *region;
298
299 region = io_mmap_get_region(ctx, pgoff);
300 if (!region)
301 return ERR_PTR(-EINVAL);
302 return io_region_validate_mmap(ctx, region);
303}
304
305#ifdef CONFIG_MMU
306
307static int io_region_mmap(struct io_ring_ctx *ctx,
308 struct io_mapped_region *mr,
309 struct vm_area_struct *vma,
310 unsigned max_pages)
311{
312 unsigned long nr_pages = min(mr->nr_pages, max_pages);
313
314 vm_flags_set(vma, VM_DONTEXPAND);
315 return vm_insert_pages(vma, vma->vm_start, mr->pages, &nr_pages);
316}
317
318__cold int io_uring_mmap(struct file *file, struct vm_area_struct *vma)
319{
320 struct io_ring_ctx *ctx = file->private_data;
321 size_t sz = vma->vm_end - vma->vm_start;
322 long offset = vma->vm_pgoff << PAGE_SHIFT;
323 unsigned int page_limit = UINT_MAX;
324 struct io_mapped_region *region;
325 void *ptr;
326
327 guard(mutex)(&ctx->mmap_lock);
328
329 ptr = io_uring_validate_mmap_request(file, vma->vm_pgoff);
330 if (IS_ERR(ptr))
331 return PTR_ERR(ptr);
332
333 switch (offset & IORING_OFF_MMAP_MASK) {
334 case IORING_OFF_SQ_RING:
335 case IORING_OFF_CQ_RING:
336 page_limit = (sz + PAGE_SIZE - 1) >> PAGE_SHIFT;
337 break;
338 }
339
340 region = io_mmap_get_region(ctx, vma->vm_pgoff);
341 return io_region_mmap(ctx, region, vma, page_limit);
342}
343
344unsigned long io_uring_get_unmapped_area(struct file *filp, unsigned long addr,
345 unsigned long len, unsigned long pgoff,
346 unsigned long flags)
347{
348 struct io_ring_ctx *ctx = filp->private_data;
349 void *ptr;
350
351 /*
352 * Do not allow to map to user-provided address to avoid breaking the
353 * aliasing rules. Userspace is not able to guess the offset address of
354 * kernel kmalloc()ed memory area.
355 */
356 if (addr)
357 return -EINVAL;
358
359 guard(mutex)(&ctx->mmap_lock);
360
361 ptr = io_uring_validate_mmap_request(filp, pgoff);
362 if (IS_ERR(ptr))
363 return PTR_ERR(ptr);
364
365 /*
366 * Some architectures have strong cache aliasing requirements.
367 * For such architectures we need a coherent mapping which aliases
368 * kernel memory *and* userspace memory. To achieve that:
369 * - use a NULL file pointer to reference physical memory, and
370 * - use the kernel virtual address of the shared io_uring context
371 * (instead of the userspace-provided address, which has to be 0UL
372 * anyway).
373 * - use the same pgoff which the get_unmapped_area() uses to
374 * calculate the page colouring.
375 * For architectures without such aliasing requirements, the
376 * architecture will return any suitable mapping because addr is 0.
377 */
378 filp = NULL;
379 flags |= MAP_SHARED;
380 pgoff = 0; /* has been translated to ptr above */
381#ifdef SHM_COLOUR
382 addr = (uintptr_t) ptr;
383 pgoff = addr >> PAGE_SHIFT;
384#else
385 addr = 0UL;
386#endif
387 return mm_get_unmapped_area(filp, addr, len, pgoff, flags);
388}
389
390#else /* !CONFIG_MMU */
391
392/*
393 * Drop the pages that were initially referenced and added in
394 * io_uring_mmap(). We cannot have had a mremap() as that isn't supported,
395 * hence the vma should be identical to the one we initially referenced and
396 * mapped, and partial unmaps and splitting isn't possible on a file backed
397 * mapping.
398 */
399static void io_uring_nommu_vm_close(struct vm_area_struct *vma)
400{
401 unsigned long index;
402
403 for (index = vma->vm_start; index < vma->vm_end; index += PAGE_SIZE)
404 put_page(virt_to_page((void *) index));
405}
406
407static const struct vm_operations_struct io_uring_nommu_vm_ops = {
408 .close = io_uring_nommu_vm_close,
409};
410
411int io_uring_mmap(struct file *file, struct vm_area_struct *vma)
412{
413 struct io_ring_ctx *ctx = file->private_data;
414 struct io_mapped_region *region;
415 unsigned long i;
416
417 if (!is_nommu_shared_mapping(vma->vm_flags))
418 return -EINVAL;
419
420 guard(mutex)(&ctx->mmap_lock);
421 region = io_mmap_get_region(ctx, vma->vm_pgoff);
422 if (!region || !io_region_is_set(region))
423 return -EINVAL;
424
425 if ((vma->vm_end - vma->vm_start) !=
426 (unsigned long) region->nr_pages << PAGE_SHIFT)
427 return -EINVAL;
428
429 /*
430 * Pin the pages so io_free_region()'s release_pages() does not
431 * drop the last reference while this VMA exists. delete_vma()
432 * in mm/nommu.c calls vma_close() which runs ->close above.
433 */
434 for (i = 0; i < region->nr_pages; i++)
435 get_page(region->pages[i]);
436
437 vma->vm_ops = &io_uring_nommu_vm_ops;
438 return 0;
439}
440
441unsigned int io_uring_nommu_mmap_capabilities(struct file *file)
442{
443 return NOMMU_MAP_DIRECT | NOMMU_MAP_READ | NOMMU_MAP_WRITE;
444}
445
446unsigned long io_uring_get_unmapped_area(struct file *file, unsigned long addr,
447 unsigned long len, unsigned long pgoff,
448 unsigned long flags)
449{
450 struct io_ring_ctx *ctx = file->private_data;
451 void *ptr;
452
453 guard(mutex)(&ctx->mmap_lock);
454
455 ptr = io_uring_validate_mmap_request(file, pgoff);
456 if (IS_ERR(ptr))
457 return PTR_ERR(ptr);
458
459 return (unsigned long) ptr;
460}
461
462#endif /* !CONFIG_MMU */