projects/linux·blob master · refs

1058 lines · 26.2 KiB

1// SPDX-License-Identifier: GPL-2.0
2/*
3 * Code related to the io_uring_register() syscall
4 *
5 * Copyright (C) 2023 Jens Axboe
6 */
7#include <linux/kernel.h>
8#include <linux/errno.h>
9#include <linux/syscalls.h>
10#include <linux/refcount.h>
11#include <linux/bits.h>
12#include <linux/fs.h>
13#include <linux/file.h>
14#include <linux/slab.h>
15#include <linux/uaccess.h>
16#include <linux/nospec.h>
17#include <linux/compat.h>
18#include <linux/io_uring.h>
19#include <linux/io_uring_types.h>
20
21#include "filetable.h"
22#include "io_uring.h"
23#include "opdef.h"
24#include "tctx.h"
25#include "rsrc.h"
26#include "sqpoll.h"
27#include "register.h"
28#include "cancel.h"
29#include "kbuf.h"
30#include "napi.h"
31#include "eventfd.h"
32#include "msg_ring.h"
33#include "memmap.h"
34#include "zcrx.h"
35#include "query.h"
36#include "bpf_filter.h"
37
38#define IORING_MAX_RESTRICTIONS (IORING_RESTRICTION_LAST + \
39 IORING_REGISTER_LAST + IORING_OP_LAST)
40
41static __cold int io_probe(struct io_ring_ctx *ctx, void __user *arg,
42 unsigned nr_args)
43{
44 struct io_uring_probe *p;
45 size_t size;
46 int i, ret;
47
48 if (nr_args > IORING_OP_LAST)
49 nr_args = IORING_OP_LAST;
50
51 size = struct_size(p, ops, nr_args);
52 p = memdup_user(arg, size);
53 if (IS_ERR(p))
54 return PTR_ERR(p);
55 ret = -EINVAL;
56 if (memchr_inv(p, 0, size))
57 goto out;
58
59 p->last_op = IORING_OP_LAST - 1;
60
61 for (i = 0; i < nr_args; i++) {
62 p->ops[i].op = i;
63 if (io_uring_op_supported(i))
64 p->ops[i].flags = IO_URING_OP_SUPPORTED;
65 }
66 p->ops_len = i;
67
68 ret = 0;
69 if (copy_to_user(arg, p, size))
70 ret = -EFAULT;
71out:
72 kfree(p);
73 return ret;
74}
75
76int io_unregister_personality(struct io_ring_ctx *ctx, unsigned id)
77{
78 const struct cred *creds;
79
80 creds = xa_erase(&ctx->personalities, id);
81 if (creds) {
82 put_cred(creds);
83 return 0;
84 }
85
86 return -EINVAL;
87}
88
89
90static int io_register_personality(struct io_ring_ctx *ctx)
91{
92 const struct cred *creds;
93 u32 id;
94 int ret;
95
96 creds = get_current_cred();
97
98 ret = xa_alloc_cyclic(&ctx->personalities, &id, (void *)creds,
99 XA_LIMIT(0, USHRT_MAX), &ctx->pers_next, GFP_KERNEL);
100 if (ret < 0) {
101 put_cred(creds);
102 return ret;
103 }
104 return id;
105}
106
107/*
108 * Returns number of restrictions parsed and added on success, or < 0 for
109 * an error.
110 */
111static __cold int io_parse_restrictions(void __user *arg, unsigned int nr_args,
112 struct io_restriction *restrictions)
113{
114 struct io_uring_restriction *res;
115 size_t size;
116 int i, ret;
117
118 if (!arg || nr_args > IORING_MAX_RESTRICTIONS)
119 return -EINVAL;
120
121 size = array_size(nr_args, sizeof(*res));
122 if (size == SIZE_MAX)
123 return -EOVERFLOW;
124
125 res = memdup_user(arg, size);
126 if (IS_ERR(res))
127 return PTR_ERR(res);
128
129 ret = -EINVAL;
130
131 for (i = 0; i < nr_args; i++) {
132 switch (res[i].opcode) {
133 case IORING_RESTRICTION_REGISTER_OP:
134 if (res[i].register_op >= IORING_REGISTER_LAST)
135 goto err;
136 __set_bit(res[i].register_op, restrictions->register_op);
137 restrictions->reg_registered = true;
138 break;
139 case IORING_RESTRICTION_SQE_OP:
140 if (res[i].sqe_op >= IORING_OP_LAST)
141 goto err;
142 __set_bit(res[i].sqe_op, restrictions->sqe_op);
143 restrictions->op_registered = true;
144 break;
145 case IORING_RESTRICTION_SQE_FLAGS_ALLOWED:
146 restrictions->sqe_flags_allowed = res[i].sqe_flags;
147 restrictions->op_registered = true;
148 break;
149 case IORING_RESTRICTION_SQE_FLAGS_REQUIRED:
150 restrictions->sqe_flags_required = res[i].sqe_flags;
151 restrictions->op_registered = true;
152 break;
153 default:
154 goto err;
155 }
156 }
157 ret = nr_args;
158 if (!nr_args) {
159 restrictions->op_registered = true;
160 restrictions->reg_registered = true;
161 }
162err:
163 kfree(res);
164 return ret;
165}
166
167static __cold int io_register_restrictions(struct io_ring_ctx *ctx,
168 void __user *arg, unsigned int nr_args)
169{
170 int ret;
171
172 /* Restrictions allowed only if rings started disabled */
173 if (!(ctx->flags & IORING_SETUP_R_DISABLED))
174 return -EBADFD;
175
176 /* We allow only a single restrictions registration */
177 if (ctx->restrictions.op_registered || ctx->restrictions.reg_registered)
178 return -EBUSY;
179
180 ret = io_parse_restrictions(arg, nr_args, &ctx->restrictions);
181 /*
182 * Reset all restrictions if an error happened, but retain any COW'ed
183 * settings.
184 */
185 if (ret < 0) {
186 struct io_bpf_filters *bpf = ctx->restrictions.bpf_filters;
187 bool cowed = ctx->restrictions.bpf_filters_cow;
188
189 memset(&ctx->restrictions, 0, sizeof(ctx->restrictions));
190 ctx->restrictions.bpf_filters = bpf;
191 ctx->restrictions.bpf_filters_cow = cowed;
192 return ret;
193 }
194 if (ctx->restrictions.op_registered)
195 ctx->int_flags |= IO_RING_F_OP_RESTRICTED;
196 if (ctx->restrictions.reg_registered)
197 ctx->int_flags |= IO_RING_F_REG_RESTRICTED;
198 return 0;
199}
200
201static int io_register_restrictions_task(void __user *arg, unsigned int nr_args)
202{
203 struct io_uring_task_restriction __user *ures = arg;
204 struct io_uring_task_restriction tres;
205 struct io_restriction *res;
206 int ret;
207
208 /* Disallow if task already has registered restrictions */
209 if (current->io_uring_restrict)
210 return -EPERM;
211 /*
212 * Similar to seccomp, disallow setting a filter if task_no_new_privs
213 * is false and we're not CAP_SYS_ADMIN.
214 */
215 if (!task_no_new_privs(current) &&
216 !ns_capable_noaudit(current_user_ns(), CAP_SYS_ADMIN))
217 return -EACCES;
218 if (nr_args != 1)
219 return -EINVAL;
220
221 if (copy_from_user(&tres, arg, sizeof(tres)))
222 return -EFAULT;
223
224 if (tres.flags)
225 return -EINVAL;
226 if (!mem_is_zero(tres.resv, sizeof(tres.resv)))
227 return -EINVAL;
228
229 res = kzalloc_obj(*res, GFP_KERNEL_ACCOUNT);
230 if (!res)
231 return -ENOMEM;
232
233 ret = io_parse_restrictions(ures->restrictions, tres.nr_res, res);
234 if (ret < 0) {
235 kfree(res);
236 return ret;
237 }
238 current->io_uring_restrict = res;
239 return 0;
240}
241
242static int io_register_bpf_filter_task(void __user *arg, unsigned int nr_args)
243{
244 struct io_restriction *res;
245 int ret;
246
247 /*
248 * Similar to seccomp, disallow setting a filter if task_no_new_privs
249 * is false and we're not CAP_SYS_ADMIN.
250 */
251 if (!task_no_new_privs(current) &&
252 !ns_capable_noaudit(current_user_ns(), CAP_SYS_ADMIN))
253 return -EACCES;
254
255 if (nr_args != 1)
256 return -EINVAL;
257
258 /* If no task restrictions exist, setup a new set */
259 res = current->io_uring_restrict;
260 if (!res) {
261 res = kzalloc_obj(*res, GFP_KERNEL_ACCOUNT);
262 if (!res)
263 return -ENOMEM;
264 }
265
266 ret = io_register_bpf_filter(res, arg);
267 if (ret) {
268 if (res != current->io_uring_restrict)
269 kfree(res);
270 return ret;
271 }
272 if (!current->io_uring_restrict)
273 current->io_uring_restrict = res;
274 return 0;
275}
276
277static int io_register_enable_rings(struct io_ring_ctx *ctx)
278{
279 if (!(ctx->flags & IORING_SETUP_R_DISABLED))
280 return -EBADFD;
281
282 if (ctx->flags & IORING_SETUP_SINGLE_ISSUER) {
283 ctx->submitter_task = get_task_struct(current);
284 /*
285 * Lazy activation attempts would fail if it was polled before
286 * submitter_task is set.
287 */
288 if (wq_has_sleeper(&ctx->poll_wq))
289 io_activate_pollwq(ctx);
290 }
291
292 /* Keep submitter_task store before clearing IORING_SETUP_R_DISABLED */
293 smp_store_release(&ctx->flags, ctx->flags & ~IORING_SETUP_R_DISABLED);
294 if (ctx->sq_data && wq_has_sleeper(&ctx->sq_data->wait))
295 wake_up(&ctx->sq_data->wait);
296 return 0;
297}
298
299static __cold int __io_register_iowq_aff(struct io_ring_ctx *ctx,
300 cpumask_var_t new_mask)
301{
302 int ret;
303
304 if (!(ctx->flags & IORING_SETUP_SQPOLL)) {
305 ret = io_wq_cpu_affinity(current->io_uring, new_mask);
306 } else {
307 mutex_unlock(&ctx->uring_lock);
308 ret = io_sqpoll_wq_cpu_affinity(ctx, new_mask);
309 mutex_lock(&ctx->uring_lock);
310 }
311
312 return ret;
313}
314
315static __cold int io_register_iowq_aff(struct io_ring_ctx *ctx,
316 void __user *arg, unsigned len)
317{
318 cpumask_var_t new_mask;
319 int ret;
320
321 if (!alloc_cpumask_var(&new_mask, GFP_KERNEL))
322 return -ENOMEM;
323
324 cpumask_clear(new_mask);
325 if (len > cpumask_size())
326 len = cpumask_size();
327
328#ifdef CONFIG_COMPAT
329 if (in_compat_syscall())
330 ret = compat_get_bitmap(cpumask_bits(new_mask),
331 (const compat_ulong_t __user *)arg,
332 len * 8 /* CHAR_BIT */);
333 else
334#endif
335 ret = copy_from_user(new_mask, arg, len);
336
337 if (ret) {
338 free_cpumask_var(new_mask);
339 return -EFAULT;
340 }
341
342 ret = __io_register_iowq_aff(ctx, new_mask);
343 free_cpumask_var(new_mask);
344 return ret;
345}
346
347static __cold int io_unregister_iowq_aff(struct io_ring_ctx *ctx)
348{
349 return __io_register_iowq_aff(ctx, NULL);
350}
351
352static __cold int io_register_iowq_max_workers(struct io_ring_ctx *ctx,
353 void __user *arg)
354 __must_hold(&ctx->uring_lock)
355{
356 struct io_tctx_node *node;
357 struct io_uring_task *tctx = NULL;
358 struct io_sq_data *sqd = NULL;
359 __u32 new_count[2];
360 int i, ret;
361
362 if (copy_from_user(new_count, arg, sizeof(new_count)))
363 return -EFAULT;
364 for (i = 0; i < ARRAY_SIZE(new_count); i++)
365 if (new_count[i] > INT_MAX)
366 return -EINVAL;
367
368 if (ctx->flags & IORING_SETUP_SQPOLL) {
369 sqd = ctx->sq_data;
370 if (sqd) {
371 struct task_struct *tsk;
372
373 /*
374 * Observe the correct sqd->lock -> ctx->uring_lock
375 * ordering. Fine to drop uring_lock here, we hold
376 * a ref to the ctx.
377 */
378 refcount_inc(&sqd->refs);
379 mutex_unlock(&ctx->uring_lock);
380 mutex_lock(&sqd->lock);
381 mutex_lock(&ctx->uring_lock);
382 tsk = sqpoll_task_locked(sqd);
383 if (tsk)
384 tctx = tsk->io_uring;
385 }
386 } else {
387 tctx = current->io_uring;
388 }
389
390 BUILD_BUG_ON(sizeof(new_count) != sizeof(ctx->iowq_limits));
391
392 for (i = 0; i < ARRAY_SIZE(new_count); i++)
393 if (new_count[i])
394 ctx->iowq_limits[i] = new_count[i];
395 ctx->int_flags |= IO_RING_F_IOWQ_LIMITS_SET;
396
397 if (tctx && tctx->io_wq) {
398 ret = io_wq_max_workers(tctx->io_wq, new_count);
399 if (ret)
400 goto err;
401 } else {
402 memset(new_count, 0, sizeof(new_count));
403 }
404
405 if (sqd) {
406 mutex_unlock(&ctx->uring_lock);
407 mutex_unlock(&sqd->lock);
408 io_put_sq_data(sqd);
409 mutex_lock(&ctx->uring_lock);
410 }
411
412 if (copy_to_user(arg, new_count, sizeof(new_count)))
413 return -EFAULT;
414
415 /* that's it for SQPOLL, only the SQPOLL task creates requests */
416 if (sqd)
417 return 0;
418
419 /* now propagate the restriction to all registered users */
420 mutex_lock(&ctx->tctx_lock);
421 list_for_each_entry(node, &ctx->tctx_list, ctx_node) {
422 tctx = node->task->io_uring;
423 if (WARN_ON_ONCE(!tctx->io_wq))
424 continue;
425
426 for (i = 0; i < ARRAY_SIZE(new_count); i++)
427 new_count[i] = ctx->iowq_limits[i];
428 /* ignore errors, it always returns zero anyway */
429 (void)io_wq_max_workers(tctx->io_wq, new_count);
430 }
431 mutex_unlock(&ctx->tctx_lock);
432 return 0;
433err:
434 if (sqd) {
435 mutex_unlock(&ctx->uring_lock);
436 mutex_unlock(&sqd->lock);
437 io_put_sq_data(sqd);
438 mutex_lock(&ctx->uring_lock);
439 }
440 return ret;
441}
442
443static int io_register_clock(struct io_ring_ctx *ctx,
444 struct io_uring_clock_register __user *arg)
445{
446 struct io_uring_clock_register reg;
447
448 if (copy_from_user(&reg, arg, sizeof(reg)))
449 return -EFAULT;
450 if (memchr_inv(&reg.__resv, 0, sizeof(reg.__resv)))
451 return -EINVAL;
452
453 switch (reg.clockid) {
454 case CLOCK_MONOTONIC:
455 ctx->clock_offset = 0;
456 break;
457 case CLOCK_BOOTTIME:
458 ctx->clock_offset = TK_OFFS_BOOT;
459 break;
460 default:
461 return -EINVAL;
462 }
463
464 ctx->clockid = reg.clockid;
465 return 0;
466}
467
468/*
469 * State to maintain until we can swap. Both new and old state, used for
470 * either mapping or freeing.
471 */
472struct io_ring_ctx_rings {
473 struct io_rings *rings;
474 struct io_uring_sqe *sq_sqes;
475
476 struct io_mapped_region sq_region;
477 struct io_mapped_region ring_region;
478};
479
480static void io_register_free_rings(struct io_ring_ctx *ctx,
481 struct io_ring_ctx_rings *r)
482{
483 /* ring memory is not charged to RLIMIT_MEMLOCK, hence no user */
484 io_free_region(NULL, &r->sq_region);
485 io_free_region(NULL, &r->ring_region);
486}
487
488#define swap_old(ctx, o, n, field) \
489 do { \
490 (o).field = (ctx)->field; \
491 (ctx)->field = (n).field; \
492 } while (0)
493
494#define RESIZE_FLAGS (IORING_SETUP_CQSIZE | IORING_SETUP_CLAMP)
495#define COPY_FLAGS (IORING_SETUP_NO_SQARRAY | IORING_SETUP_SQE128 | \
496 IORING_SETUP_CQE32 | IORING_SETUP_NO_MMAP | \
497 IORING_SETUP_CQE_MIXED | IORING_SETUP_SQE_MIXED)
498
499static int io_register_resize_rings(struct io_ring_ctx *ctx, void __user *arg)
500{
501 struct io_ctx_config config;
502 struct io_uring_region_desc rd;
503 struct io_ring_ctx_rings o = { }, n = { }, *to_free = NULL;
504 unsigned i, tail, old_head;
505 struct io_uring_params *p = &config.p;
506 struct io_rings_layout *rl = &config.layout;
507 u32 *o_sq_array, *n_sq_array = NULL;
508 int ret;
509
510 memset(&config, 0, sizeof(config));
511
512 /* limited to DEFER_TASKRUN for now */
513 if (!(ctx->flags & IORING_SETUP_DEFER_TASKRUN))
514 return -EINVAL;
515 if (copy_from_user(p, arg, sizeof(*p)))
516 return -EFAULT;
517 if (p->flags & ~RESIZE_FLAGS)
518 return -EINVAL;
519
520 /* properties that are always inherited */
521 p->flags |= (ctx->flags & COPY_FLAGS);
522
523 ret = io_prepare_config(&config);
524 if (unlikely(ret))
525 return ret;
526
527 memset(&rd, 0, sizeof(rd));
528 rd.size = PAGE_ALIGN(rl->rings_size);
529 if (p->flags & IORING_SETUP_NO_MMAP) {
530 rd.user_addr = p->cq_off.user_addr;
531 rd.flags |= IORING_MEM_REGION_TYPE_USER;
532 }
533 ret = io_create_region(NULL, &n.ring_region, &rd, IORING_OFF_CQ_RING);
534 if (ret)
535 return ret;
536
537 n.rings = io_region_get_ptr(&n.ring_region);
538
539 /*
540 * At this point n.rings is shared with userspace, just like o.rings
541 * is as well. While we don't expect userspace to modify it while
542 * a resize is in progress, and it's most likely that userspace will
543 * shoot itself in the foot if it does, we can't always assume good
544 * intent... Use read/write once helpers from here on to indicate the
545 * shared nature of it.
546 */
547 WRITE_ONCE(n.rings->sq_ring_mask, p->sq_entries - 1);
548 WRITE_ONCE(n.rings->cq_ring_mask, p->cq_entries - 1);
549 WRITE_ONCE(n.rings->sq_ring_entries, p->sq_entries);
550 WRITE_ONCE(n.rings->cq_ring_entries, p->cq_entries);
551
552 if (copy_to_user(arg, p, sizeof(*p))) {
553 io_register_free_rings(ctx, &n);
554 return -EFAULT;
555 }
556
557 memset(&rd, 0, sizeof(rd));
558 rd.size = PAGE_ALIGN(rl->sq_size);
559 if (p->flags & IORING_SETUP_NO_MMAP) {
560 rd.user_addr = p->sq_off.user_addr;
561 rd.flags |= IORING_MEM_REGION_TYPE_USER;
562 }
563 ret = io_create_region(NULL, &n.sq_region, &rd, IORING_OFF_SQES);
564 if (ret) {
565 io_register_free_rings(ctx, &n);
566 return ret;
567 }
568 n.sq_sqes = io_region_get_ptr(&n.sq_region);
569
570 /*
571 * If using SQPOLL, park the thread
572 */
573 if (ctx->sq_data) {
574 mutex_unlock(&ctx->uring_lock);
575 io_sq_thread_park(ctx->sq_data);
576 mutex_lock(&ctx->uring_lock);
577 }
578
579 /*
580 * We'll do the swap. Grab the ctx->mmap_lock, which will exclude
581 * any new mmap's on the ring fd. Clear out existing mappings to prevent
582 * mmap from seeing them, as we'll unmap them. Any attempt to mmap
583 * existing rings beyond this point will fail. Not that it could proceed
584 * at this point anyway, as the io_uring mmap side needs go grab the
585 * ctx->mmap_lock as well. Likewise, hold the completion lock over the
586 * duration of the actual swap.
587 */
588 mutex_lock(&ctx->mmap_lock);
589 spin_lock(&ctx->completion_lock);
590 o.rings = ctx->rings;
591 ctx->rings = NULL;
592 o.sq_sqes = ctx->sq_sqes;
593 ctx->sq_sqes = NULL;
594 o_sq_array = ctx->sq_array;
595 if (!(ctx->flags & IORING_SETUP_NO_SQARRAY))
596 n_sq_array = (u32 *)((char *)n.rings + rl->sq_array_offset);
597
598 /*
599 * Now copy SQ and CQ entries, if any. If either of the destination
600 * rings can't hold what is already there, then fail the operation.
601 */
602 tail = READ_ONCE(o.rings->sq.tail);
603 old_head = READ_ONCE(o.rings->sq.head);
604 if (tail - old_head > p->sq_entries)
605 goto overflow;
606 for (i = old_head; i < tail; i++) {
607 unsigned int dst, src;
608 size_t sq_size;
609
610 dst = i & (p->sq_entries - 1);
611 src = i & (ctx->sq_entries - 1);
612 if (n_sq_array) {
613 src = READ_ONCE(o_sq_array[src]);
614 if (unlikely(src >= ctx->sq_entries)) {
615 WRITE_ONCE(n_sq_array[dst], UINT_MAX);
616 continue;
617 }
618 WRITE_ONCE(n_sq_array[dst], dst);
619 }
620
621 sq_size = sizeof(struct io_uring_sqe);
622 if (ctx->flags & IORING_SETUP_SQE128) {
623 dst <<= 1;
624 src <<= 1;
625 sq_size <<= 1;
626 }
627 memcpy(&n.sq_sqes[dst], &o.sq_sqes[src], sq_size);
628 }
629 WRITE_ONCE(n.rings->sq.head, old_head);
630 WRITE_ONCE(n.rings->sq.tail, tail);
631
632 tail = READ_ONCE(o.rings->cq.tail);
633 old_head = READ_ONCE(o.rings->cq.head);
634 if (tail - old_head > p->cq_entries) {
635overflow:
636 /* restore old rings, and return -EOVERFLOW via cleanup path */
637 ctx->rings = o.rings;
638 ctx->sq_sqes = o.sq_sqes;
639 to_free = &n;
640 ret = -EOVERFLOW;
641 goto out;
642 }
643 for (i = old_head; i < tail; i++) {
644 unsigned index, dst_mask, src_mask;
645 size_t cq_size;
646
647 index = i;
648 cq_size = sizeof(struct io_uring_cqe);
649 src_mask = ctx->cq_entries - 1;
650 dst_mask = p->cq_entries - 1;
651 if (ctx->flags & IORING_SETUP_CQE32) {
652 index <<= 1;
653 cq_size <<= 1;
654 src_mask = (ctx->cq_entries << 1) - 1;
655 dst_mask = (p->cq_entries << 1) - 1;
656 }
657 memcpy(&n.rings->cqes[index & dst_mask], &o.rings->cqes[index & src_mask], cq_size);
658 }
659 WRITE_ONCE(n.rings->cq.head, old_head);
660 WRITE_ONCE(n.rings->cq.tail, tail);
661 /* invalidate cached cqe refill */
662 ctx->cqe_cached = ctx->cqe_sentinel = NULL;
663
664 WRITE_ONCE(n.rings->sq_dropped, READ_ONCE(o.rings->sq_dropped));
665 atomic_set(&n.rings->sq_flags, atomic_read(&o.rings->sq_flags));
666 WRITE_ONCE(n.rings->cq_flags, READ_ONCE(o.rings->cq_flags));
667 WRITE_ONCE(n.rings->cq_overflow, READ_ONCE(o.rings->cq_overflow));
668
669 /* all done, store old pointers and assign new ones */
670 if (n_sq_array)
671 ctx->sq_array = n_sq_array;
672
673 ctx->sq_entries = p->sq_entries;
674 ctx->cq_entries = p->cq_entries;
675
676 /*
677 * Just mark any flag we may have missed and that the application
678 * should act on unconditionally. Worst case it'll be an extra
679 * syscall.
680 */
681 atomic_or(IORING_SQ_TASKRUN | IORING_SQ_NEED_WAKEUP, &n.rings->sq_flags);
682 ctx->rings = n.rings;
683 rcu_assign_pointer(ctx->rings_rcu, n.rings);
684
685 ctx->sq_sqes = n.sq_sqes;
686 swap_old(ctx, o, n, ring_region);
687 swap_old(ctx, o, n, sq_region);
688 to_free = &o;
689 ret = 0;
690out:
691 spin_unlock(&ctx->completion_lock);
692 mutex_unlock(&ctx->mmap_lock);
693 /* Wait for concurrent io_ctx_mark_taskrun() */
694 if (to_free == &o)
695 synchronize_rcu_expedited();
696 io_register_free_rings(ctx, to_free);
697
698 if (ctx->sq_data)
699 io_sq_thread_unpark(ctx->sq_data);
700
701 return ret;
702}
703
704static int io_register_mem_region(struct io_ring_ctx *ctx, void __user *uarg)
705{
706 struct io_uring_mem_region_reg __user *reg_uptr = uarg;
707 struct io_uring_mem_region_reg reg;
708 struct io_uring_region_desc __user *rd_uptr;
709 struct io_uring_region_desc rd;
710 struct io_mapped_region region = {};
711 int ret;
712
713 if (io_region_is_set(&ctx->param_region))
714 return -EBUSY;
715 if (copy_from_user(&reg, reg_uptr, sizeof(reg)))
716 return -EFAULT;
717 rd_uptr = u64_to_user_ptr(reg.region_uptr);
718 if (copy_from_user(&rd, rd_uptr, sizeof(rd)))
719 return -EFAULT;
720 if (memchr_inv(&reg.__resv, 0, sizeof(reg.__resv)))
721 return -EINVAL;
722 if (reg.flags & ~IORING_MEM_REGION_REG_WAIT_ARG)
723 return -EINVAL;
724
725 /*
726 * This ensures there are no waiters. Waiters are unlocked and it's
727 * hard to synchronise with them, especially if we need to initialise
728 * the region.
729 */
730 if ((reg.flags & IORING_MEM_REGION_REG_WAIT_ARG) &&
731 !(ctx->flags & IORING_SETUP_R_DISABLED))
732 return -EINVAL;
733
734 ret = io_create_region(ctx->user, &region, &rd,
735 IORING_MAP_OFF_PARAM_REGION);
736 if (ret)
737 return ret;
738 if (copy_to_user(rd_uptr, &rd, sizeof(rd))) {
739 io_free_region(ctx->user, &region);
740 return -EFAULT;
741 }
742
743 if (reg.flags & IORING_MEM_REGION_REG_WAIT_ARG) {
744 ctx->cq_wait_arg = io_region_get_ptr(&region);
745 ctx->cq_wait_size = rd.size;
746 }
747
748 io_region_publish(ctx, &region, &ctx->param_region);
749 return 0;
750}
751
752static int __io_uring_register(struct io_ring_ctx *ctx, unsigned opcode,
753 void __user *arg, unsigned nr_args)
754 __releases(ctx->uring_lock)
755 __acquires(ctx->uring_lock)
756{
757 int ret;
758
759 /*
760 * We don't quiesce the refs for register anymore and so it can't be
761 * dying as we're holding a file ref here.
762 */
763 if (WARN_ON_ONCE(percpu_ref_is_dying(&ctx->refs)))
764 return -ENXIO;
765
766 if (ctx->submitter_task && ctx->submitter_task != current)
767 return -EEXIST;
768
769 /*
770 * A disabled ring is exempt from its own register restrictions, so
771 * that the creator can set them up and then enable the ring. That
772 * does not apply to restrictions inherited from the task.
773 */
774 if ((ctx->int_flags & IO_RING_F_REG_RESTRICTED) &&
775 (!(ctx->flags & IORING_SETUP_R_DISABLED) ||
776 (ctx->int_flags & IO_RING_F_RESTRICT_INHERITED))) {
777 opcode = array_index_nospec(opcode, IORING_REGISTER_LAST);
778 if (!test_bit(opcode, ctx->restrictions.register_op))
779 return -EACCES;
780 }
781
782 switch (opcode) {
783 case IORING_REGISTER_BUFFERS:
784 ret = -EFAULT;
785 if (!arg)
786 break;
787 ret = io_sqe_buffers_register(ctx, arg, nr_args, NULL);
788 break;
789 case IORING_UNREGISTER_BUFFERS:
790 ret = -EINVAL;
791 if (arg || nr_args)
792 break;
793 ret = io_sqe_buffers_unregister(ctx);
794 break;
795 case IORING_REGISTER_FILES:
796 ret = -EFAULT;
797 if (!arg)
798 break;
799 ret = io_sqe_files_register(ctx, arg, nr_args, NULL);
800 break;
801 case IORING_UNREGISTER_FILES:
802 ret = -EINVAL;
803 if (arg || nr_args)
804 break;
805 ret = io_sqe_files_unregister(ctx);
806 break;
807 case IORING_REGISTER_FILES_UPDATE:
808 ret = io_register_files_update(ctx, arg, nr_args);
809 break;
810 case IORING_REGISTER_EVENTFD:
811 ret = -EINVAL;
812 if (nr_args != 1)
813 break;
814 ret = io_eventfd_register(ctx, arg, 0);
815 break;
816 case IORING_REGISTER_EVENTFD_ASYNC:
817 ret = -EINVAL;
818 if (nr_args != 1)
819 break;
820 ret = io_eventfd_register(ctx, arg, 1);
821 break;
822 case IORING_UNREGISTER_EVENTFD:
823 ret = -EINVAL;
824 if (arg || nr_args)
825 break;
826 ret = io_eventfd_unregister(ctx);
827 break;
828 case IORING_REGISTER_PROBE:
829 ret = -EINVAL;
830 if (!arg || nr_args > 256)
831 break;
832 ret = io_probe(ctx, arg, nr_args);
833 break;
834 case IORING_REGISTER_PERSONALITY:
835 ret = -EINVAL;
836 if (arg || nr_args)
837 break;
838 ret = io_register_personality(ctx);
839 break;
840 case IORING_UNREGISTER_PERSONALITY:
841 ret = -EINVAL;
842 if (arg)
843 break;
844 ret = io_unregister_personality(ctx, nr_args);
845 break;
846 case IORING_REGISTER_ENABLE_RINGS:
847 ret = -EINVAL;
848 if (arg || nr_args)
849 break;
850 ret = io_register_enable_rings(ctx);
851 break;
852 case IORING_REGISTER_RESTRICTIONS:
853 ret = io_register_restrictions(ctx, arg, nr_args);
854 break;
855 case IORING_REGISTER_FILES2:
856 ret = io_register_rsrc(ctx, arg, nr_args, IORING_RSRC_FILE);
857 break;
858 case IORING_REGISTER_FILES_UPDATE2:
859 ret = io_register_rsrc_update(ctx, arg, nr_args,
860 IORING_RSRC_FILE);
861 break;
862 case IORING_REGISTER_BUFFERS2:
863 ret = io_register_rsrc(ctx, arg, nr_args, IORING_RSRC_BUFFER);
864 break;
865 case IORING_REGISTER_BUFFERS_UPDATE:
866 ret = io_register_rsrc_update(ctx, arg, nr_args,
867 IORING_RSRC_BUFFER);
868 break;
869 case IORING_REGISTER_IOWQ_AFF:
870 ret = -EINVAL;
871 if (!arg || !nr_args)
872 break;
873 ret = io_register_iowq_aff(ctx, arg, nr_args);
874 break;
875 case IORING_UNREGISTER_IOWQ_AFF:
876 ret = -EINVAL;
877 if (arg || nr_args)
878 break;
879 ret = io_unregister_iowq_aff(ctx);
880 break;
881 case IORING_REGISTER_IOWQ_MAX_WORKERS:
882 ret = -EINVAL;
883 if (!arg || nr_args != 2)
884 break;
885 ret = io_register_iowq_max_workers(ctx, arg);
886 break;
887 case IORING_REGISTER_RING_FDS:
888 ret = io_ringfd_register(ctx, arg, nr_args);
889 break;
890 case IORING_UNREGISTER_RING_FDS:
891 ret = io_ringfd_unregister(ctx, arg, nr_args);
892 break;
893 case IORING_REGISTER_PBUF_RING:
894 ret = -EINVAL;
895 if (!arg || nr_args != 1)
896 break;
897 ret = io_register_pbuf_ring(ctx, arg);
898 break;
899 case IORING_UNREGISTER_PBUF_RING:
900 ret = -EINVAL;
901 if (!arg || nr_args != 1)
902 break;
903 ret = io_unregister_pbuf_ring(ctx, arg);
904 break;
905 case IORING_REGISTER_SYNC_CANCEL:
906 ret = -EINVAL;
907 if (!arg || nr_args != 1)
908 break;
909 ret = io_sync_cancel(ctx, arg);
910 break;
911 case IORING_REGISTER_FILE_ALLOC_RANGE:
912 ret = -EINVAL;
913 if (!arg || nr_args)
914 break;
915 ret = io_register_file_alloc_range(ctx, arg);
916 break;
917 case IORING_REGISTER_PBUF_STATUS:
918 ret = -EINVAL;
919 if (!arg || nr_args != 1)
920 break;
921 ret = io_register_pbuf_status(ctx, arg);
922 break;
923 case IORING_REGISTER_NAPI:
924 ret = -EINVAL;
925 if (!arg || nr_args != 1)
926 break;
927 ret = io_register_napi(ctx, arg);
928 break;
929 case IORING_UNREGISTER_NAPI:
930 ret = -EINVAL;
931 if (nr_args != 1)
932 break;
933 ret = io_unregister_napi(ctx, arg);
934 break;
935 case IORING_REGISTER_CLOCK:
936 ret = -EINVAL;
937 if (!arg || nr_args)
938 break;
939 ret = io_register_clock(ctx, arg);
940 break;
941 case IORING_REGISTER_CLONE_BUFFERS:
942 ret = -EINVAL;
943 if (!arg || nr_args != 1)
944 break;
945 ret = io_register_clone_buffers(ctx, arg);
946 break;
947 case IORING_REGISTER_ZCRX_IFQ:
948 ret = -EINVAL;
949 if (!arg || nr_args != 1)
950 break;
951 ret = io_register_zcrx(ctx, arg);
952 break;
953 case IORING_REGISTER_RESIZE_RINGS:
954 ret = -EINVAL;
955 if (!arg || nr_args != 1)
956 break;
957 ret = io_register_resize_rings(ctx, arg);
958 break;
959 case IORING_REGISTER_MEM_REGION:
960 ret = -EINVAL;
961 if (!arg || nr_args != 1)
962 break;
963 ret = io_register_mem_region(ctx, arg);
964 break;
965 case IORING_REGISTER_QUERY:
966 ret = io_query(arg, nr_args);
967 break;
968 case IORING_REGISTER_ZCRX_CTRL:
969 ret = io_zcrx_ctrl(ctx, arg, nr_args);
970 break;
971 case IORING_REGISTER_BPF_FILTER:
972 ret = -EINVAL;
973
974 if (nr_args != 1)
975 break;
976 ret = io_register_bpf_filter(&ctx->restrictions, arg);
977 if (!ret)
978 WRITE_ONCE(ctx->bpf_filters,
979 ctx->restrictions.bpf_filters->filters);
980 break;
981 default:
982 ret = -EINVAL;
983 break;
984 }
985
986 return ret;
987}
988
989static int io_uring_register_send_msg_ring(void __user *arg, unsigned int nr_args)
990{
991 struct io_uring_sqe sqe;
992
993 if (!arg || nr_args != 1)
994 return -EINVAL;
995 if (copy_from_user(&sqe, arg, sizeof(sqe)))
996 return -EFAULT;
997 /* no flags supported */
998 if (sqe.flags)
999 return -EINVAL;
1000 if (sqe.opcode != IORING_OP_MSG_RING)
1001 return -EINVAL;
1002
1003 return io_uring_sync_msg_ring(&sqe);
1004}
1005
1006/*
1007 * "blind" registration opcodes are ones where there's no ring given, and
1008 * hence the source fd must be -1.
1009 */
1010static int io_uring_register_blind(unsigned int opcode, void __user *arg,
1011 unsigned int nr_args)
1012{
1013 switch (opcode) {
1014 case IORING_REGISTER_SEND_MSG_RING:
1015 return io_uring_register_send_msg_ring(arg, nr_args);
1016 case IORING_REGISTER_QUERY:
1017 return io_query(arg, nr_args);
1018 case IORING_REGISTER_RESTRICTIONS:
1019 return io_register_restrictions_task(arg, nr_args);
1020 case IORING_REGISTER_BPF_FILTER:
1021 return io_register_bpf_filter_task(arg, nr_args);
1022 }
1023 return -EINVAL;
1024}
1025
1026SYSCALL_DEFINE4(io_uring_register, unsigned int, fd, unsigned int, opcode,
1027 void __user *, arg, unsigned int, nr_args)
1028{
1029 struct io_ring_ctx *ctx;
1030 long ret = -EBADF;
1031 struct file *file;
1032 bool use_registered_ring;
1033
1034 use_registered_ring = !!(opcode & IORING_REGISTER_USE_REGISTERED_RING);
1035 opcode &= ~IORING_REGISTER_USE_REGISTERED_RING;
1036
1037 if (opcode >= IORING_REGISTER_LAST)
1038 return -EINVAL;
1039
1040 if (fd == -1)
1041 return io_uring_register_blind(opcode, arg, nr_args);
1042
1043 file = io_uring_ctx_get_file(fd, use_registered_ring);
1044 if (IS_ERR(file))
1045 return PTR_ERR(file);
1046 ctx = file->private_data;
1047
1048 mutex_lock(&ctx->uring_lock);
1049 ret = __io_uring_register(ctx, opcode, arg, nr_args);
1050
1051 trace_io_uring_register(ctx, opcode, ctx->file_table.data.nr,
1052 ctx->buf_table.nr, ret);
1053 mutex_unlock(&ctx->uring_lock);
1054
1055 if (!use_registered_ring)
1056 fput(file);
1057 return ret;
1058}