| 1 | |
| 2 | |
| 3 | * Shared application/kernel submission and completion ring pairs, for
|
| 4 | * supporting fast/efficient IO.
|
| 5 | *
|
| 6 | * A note on the read/write ordering memory barriers that are matched between
|
| 7 | * the application and kernel side.
|
| 8 | *
|
| 9 | * After the application reads the CQ ring tail, it must use an
|
| 10 | * appropriate smp_rmb() to pair with the smp_wmb() the kernel uses
|
| 11 | * before writing the tail (using smp_load_acquire to read the tail will
|
| 12 | * do). It also needs a smp_mb() before updating CQ head (ordering the
|
| 13 | * entry load(s) with the head store), pairing with an implicit barrier
|
| 14 | * through a control-dependency in io_get_cqe (smp_store_release to
|
| 15 | * store head will do). Failure to do so could lead to reading invalid
|
| 16 | * CQ entries.
|
| 17 | *
|
| 18 | * Likewise, the application must use an appropriate smp_wmb() before
|
| 19 | * writing the SQ tail (ordering SQ entry stores with the tail store),
|
| 20 | * which pairs with smp_load_acquire in io_get_sqring (smp_store_release
|
| 21 | * to store the tail will do). And it needs a barrier ordering the SQ
|
| 22 | * head load before writing new SQ entries (smp_load_acquire to read
|
| 23 | * head will do).
|
| 24 | *
|
| 25 | * When using the SQ poll thread (IORING_SETUP_SQPOLL), the application
|
| 26 | * needs to check the SQ flags for IORING_SQ_NEED_WAKEUP *after*
|
| 27 | * updating the SQ tail; a full memory barrier smp_mb() is needed
|
| 28 | * between.
|
| 29 | *
|
| 30 | * Also see the examples in the liburing library:
|
| 31 | *
|
| 32 | * git://git.kernel.org/pub/scm/linux/kernel/git/axboe/liburing.git
|
| 33 | *
|
| 34 | * io_uring also uses READ/WRITE_ONCE() for _any_ store or load that happens
|
| 35 | * from data shared between the kernel and application. This is done both
|
| 36 | * for ordering purposes, but also to ensure that once a value is loaded from
|
| 37 | * data that the application could potentially modify, it remains stable.
|
| 38 | *
|
| 39 | * Copyright (C) 2018-2019 Jens Axboe
|
| 40 | * Copyright (c) 2018-2019 Christoph Hellwig
|
| 41 |
|
| 42 | #include <linux/kernel.h>
|
| 43 | #include <linux/errno.h>
|
| 44 | #include <linux/syscalls.h>
|
| 45 | #include <linux/refcount.h>
|
| 46 | #include <linux/bits.h>
|
| 47 |
|
| 48 | #include <linux/sched/signal.h>
|
| 49 | #include <linux/fs.h>
|
| 50 | #include <linux/mm.h>
|
| 51 | #include <linux/percpu.h>
|
| 52 | #include <linux/slab.h>
|
| 53 | #include <linux/anon_inodes.h>
|
| 54 | #include <linux/uaccess.h>
|
| 55 | #include <linux/nospec.h>
|
| 56 | #include <linux/task_work.h>
|
| 57 | #include <linux/io_uring.h>
|
| 58 | #include <linux/io_uring/cmd.h>
|
| 59 | #include <linux/audit.h>
|
| 60 | #include <linux/security.h>
|
| 61 | #include <linux/jump_label.h>
|
| 62 | #include <linux/kcov.h>
|
| 63 |
|
| 64 | #define CREATE_TRACE_POINTS
|
| 65 | #include <trace/events/io_uring.h>
|
| 66 |
|
| 67 | #include <uapi/linux/io_uring.h>
|
| 68 |
|
| 69 | #include "io-wq.h"
|
| 70 |
|
| 71 | #include "filetable.h"
|
| 72 | #include "io_uring.h"
|
| 73 | #include "opdef.h"
|
| 74 | #include "refs.h"
|
| 75 | #include "tctx.h"
|
| 76 | #include "register.h"
|
| 77 | #include "sqpoll.h"
|
| 78 | #include "fdinfo.h"
|
| 79 | #include "kbuf.h"
|
| 80 | #include "rsrc.h"
|
| 81 | #include "cancel.h"
|
| 82 | #include "net.h"
|
| 83 | #include "notif.h"
|
| 84 | #include "waitid.h"
|
| 85 | #include "futex.h"
|
| 86 | #include "napi.h"
|
| 87 | #include "uring_cmd.h"
|
| 88 | #include "msg_ring.h"
|
| 89 | #include "memmap.h"
|
| 90 | #include "zcrx.h"
|
| 91 | #include "bpf-ops.h"
|
| 92 |
|
| 93 | #include "timeout.h"
|
| 94 | #include "poll.h"
|
| 95 | #include "rw.h"
|
| 96 | #include "alloc_cache.h"
|
| 97 | #include "eventfd.h"
|
| 98 | #include "wait.h"
|
| 99 | #include "bpf_filter.h"
|
| 100 | #include "loop.h"
|
| 101 |
|
| 102 | #define SQE_COMMON_FLAGS (IOSQE_FIXED_FILE | IOSQE_IO_LINK | \
|
| 103 | IOSQE_IO_HARDLINK | IOSQE_ASYNC)
|
| 104 |
|
| 105 | #define IO_REQ_LINK_FLAGS (REQ_F_LINK | REQ_F_HARDLINK)
|
| 106 |
|
| 107 | #define IO_REQ_CLEAN_FLAGS (REQ_F_BUFFER_SELECTED | REQ_F_NEED_CLEANUP | \
|
| 108 | REQ_F_INFLIGHT | REQ_F_CREDS | REQ_F_ASYNC_DATA)
|
| 109 |
|
| 110 | #define IO_REQ_CLEAN_SLOW_FLAGS (REQ_F_REFCOUNT | IO_REQ_LINK_FLAGS | \
|
| 111 | REQ_F_REISSUE | REQ_F_POLLED | \
|
| 112 | IO_REQ_CLEAN_FLAGS)
|
| 113 |
|
| 114 | #define IO_TCTX_REFS_CACHE_NR (1U << 10)
|
| 115 |
|
| 116 | #define IO_COMPL_BATCH 32
|
| 117 | #define IO_REQ_ALLOC_BATCH 8
|
| 118 |
|
| 119 |
|
| 120 | #define IO_DISARM_MASK (REQ_F_ARM_LTIMEOUT | REQ_F_LINK_TIMEOUT | REQ_F_FAIL)
|
| 121 |
|
| 122 | static void io_queue_sqe(struct io_kiocb *req, unsigned int extra_flags);
|
| 123 | static void __io_req_caches_free(struct io_ring_ctx *ctx);
|
| 124 |
|
| 125 | static __read_mostly DEFINE_STATIC_KEY_DEFERRED_FALSE(io_key_has_sqarray, HZ);
|
| 126 |
|
| 127 | struct kmem_cache *req_cachep;
|
| 128 | static struct workqueue_struct *iou_wq __ro_after_init;
|
| 129 |
|
| 130 | static int __read_mostly sysctl_io_uring_disabled;
|
| 131 | static int __read_mostly sysctl_io_uring_group = -1;
|
| 132 |
|
| 133 | #ifdef CONFIG_SYSCTL
|
| 134 | static const struct ctl_table kernel_io_uring_disabled_table[] = {
|
| 135 | {
|
| 136 | .procname = "io_uring_disabled",
|
| 137 | .data = &sysctl_io_uring_disabled,
|
| 138 | .maxlen = sizeof(sysctl_io_uring_disabled),
|
| 139 | .mode = 0644,
|
| 140 | .proc_handler = proc_dointvec_minmax,
|
| 141 | .extra1 = SYSCTL_ZERO,
|
| 142 | .extra2 = SYSCTL_TWO,
|
| 143 | },
|
| 144 | {
|
| 145 | .procname = "io_uring_group",
|
| 146 | .data = &sysctl_io_uring_group,
|
| 147 | .maxlen = sizeof(gid_t),
|
| 148 | .mode = 0644,
|
| 149 | .proc_handler = proc_dointvec,
|
| 150 | },
|
| 151 | };
|
| 152 | #endif
|
| 153 |
|
| 154 | static void io_poison_cached_req(struct io_kiocb *req)
|
| 155 | {
|
| 156 | req->ctx = IO_URING_PTR_POISON;
|
| 157 | req->tctx = IO_URING_PTR_POISON;
|
| 158 | req->file = IO_URING_PTR_POISON;
|
| 159 | req->creds = IO_URING_PTR_POISON;
|
| 160 | req->io_task_work.func = IO_URING_PTR_POISON;
|
| 161 | req->apoll = IO_URING_PTR_POISON;
|
| 162 | }
|
| 163 |
|
| 164 | void io_poison_req(struct io_kiocb *req)
|
| 165 | {
|
| 166 | io_poison_cached_req(req);
|
| 167 | req->async_data = IO_URING_PTR_POISON;
|
| 168 | req->kbuf = IO_URING_PTR_POISON;
|
| 169 | req->comp_list.next = IO_URING_PTR_POISON;
|
| 170 | req->file_node = IO_URING_PTR_POISON;
|
| 171 | req->link = IO_URING_PTR_POISON;
|
| 172 | }
|
| 173 |
|
| 174 | static inline void req_fail_link_node(struct io_kiocb *req, int res)
|
| 175 | {
|
| 176 | req_set_fail(req);
|
| 177 | io_req_set_res(req, res, 0);
|
| 178 | }
|
| 179 |
|
| 180 | static inline void io_req_add_to_cache(struct io_kiocb *req, struct io_ring_ctx *ctx)
|
| 181 | {
|
| 182 | if (IS_ENABLED(CONFIG_KASAN))
|
| 183 | io_poison_cached_req(req);
|
| 184 | wq_stack_add_head(&req->comp_list, &ctx->submit_state.free_list);
|
| 185 | }
|
| 186 |
|
| 187 | static __cold void io_ring_ctx_ref_free(struct percpu_ref *ref)
|
| 188 | {
|
| 189 | struct io_ring_ctx *ctx = container_of(ref, struct io_ring_ctx, refs);
|
| 190 |
|
| 191 | complete(&ctx->ref_comp);
|
| 192 | }
|
| 193 |
|
| 194 | static int io_alloc_hash_table(struct io_hash_table *table, unsigned bits)
|
| 195 | {
|
| 196 | unsigned int hash_buckets;
|
| 197 | int i;
|
| 198 |
|
| 199 | do {
|
| 200 | hash_buckets = 1U << bits;
|
| 201 | table->hbs = kvmalloc_objs(table->hbs[0], hash_buckets,
|
| 202 | GFP_KERNEL_ACCOUNT);
|
| 203 | if (table->hbs)
|
| 204 | break;
|
| 205 | if (bits == 1)
|
| 206 | return -ENOMEM;
|
| 207 | bits--;
|
| 208 | } while (1);
|
| 209 |
|
| 210 | table->hash_bits = bits;
|
| 211 | for (i = 0; i < hash_buckets; i++)
|
| 212 | INIT_HLIST_HEAD(&table->hbs[i].list);
|
| 213 | return 0;
|
| 214 | }
|
| 215 |
|
| 216 | static void io_free_alloc_caches(struct io_ring_ctx *ctx)
|
| 217 | {
|
| 218 | io_alloc_cache_free(&ctx->apoll_cache, kfree);
|
| 219 | io_alloc_cache_free(&ctx->netmsg_cache, io_netmsg_cache_free);
|
| 220 | io_alloc_cache_free(&ctx->rw_cache, io_rw_cache_free);
|
| 221 | io_alloc_cache_free(&ctx->cmd_cache, io_cmd_cache_free);
|
| 222 | io_futex_cache_free(ctx);
|
| 223 | io_rsrc_cache_free(ctx);
|
| 224 | }
|
| 225 |
|
| 226 | static __cold struct io_ring_ctx *io_ring_ctx_alloc(struct io_uring_params *p)
|
| 227 | {
|
| 228 | struct io_ring_ctx *ctx;
|
| 229 | int hash_bits;
|
| 230 | bool ret;
|
| 231 |
|
| 232 | ctx = kzalloc_obj(*ctx);
|
| 233 | if (!ctx)
|
| 234 | return NULL;
|
| 235 |
|
| 236 | xa_init(&ctx->io_bl_xa);
|
| 237 | xa_init(&ctx->hpage_acct);
|
| 238 |
|
| 239 | |
| 240 | * Use 5 bits less than the max cq entries, that should give us around
|
| 241 | * 32 entries per hash list if totally full and uniformly spread, but
|
| 242 | * don't keep too many buckets to not overconsume memory.
|
| 243 |
|
| 244 | hash_bits = ilog2(p->cq_entries) - 5;
|
| 245 | hash_bits = clamp(hash_bits, 1, 8);
|
| 246 | if (io_alloc_hash_table(&ctx->cancel_table, hash_bits))
|
| 247 | goto err;
|
| 248 | if (percpu_ref_init(&ctx->refs, io_ring_ctx_ref_free,
|
| 249 | 0, GFP_KERNEL))
|
| 250 | goto err;
|
| 251 |
|
| 252 | ctx->flags = p->flags;
|
| 253 | ctx->hybrid_poll_time = LLONG_MAX;
|
| 254 | atomic_set(&ctx->cq_wait_nr, IO_CQ_WAKE_INIT);
|
| 255 | init_waitqueue_head(&ctx->sqo_sq_wait);
|
| 256 | INIT_LIST_HEAD(&ctx->sqd_list);
|
| 257 | INIT_LIST_HEAD(&ctx->cq_overflow_list);
|
| 258 | ret = io_alloc_cache_init(&ctx->apoll_cache, IO_POLL_ALLOC_CACHE_MAX,
|
| 259 | sizeof(struct async_poll), 0);
|
| 260 | ret |= io_alloc_cache_init(&ctx->netmsg_cache, IO_ALLOC_CACHE_MAX,
|
| 261 | sizeof(struct io_async_msghdr),
|
| 262 | offsetof(struct io_async_msghdr, clear));
|
| 263 | ret |= io_alloc_cache_init(&ctx->rw_cache, IO_ALLOC_CACHE_MAX,
|
| 264 | sizeof(struct io_async_rw),
|
| 265 | offsetof(struct io_async_rw, clear));
|
| 266 | ret |= io_alloc_cache_init(&ctx->cmd_cache, IO_ALLOC_CACHE_MAX,
|
| 267 | sizeof(struct io_async_cmd),
|
| 268 | sizeof(struct io_async_cmd));
|
| 269 | ret |= io_futex_cache_init(ctx);
|
| 270 | ret |= io_rsrc_cache_init(ctx);
|
| 271 | if (ret)
|
| 272 | goto free_ref;
|
| 273 | init_completion(&ctx->ref_comp);
|
| 274 | xa_init_flags(&ctx->personalities, XA_FLAGS_ALLOC1);
|
| 275 | mutex_init(&ctx->uring_lock);
|
| 276 | init_waitqueue_head(&ctx->cq_wait);
|
| 277 | init_waitqueue_head(&ctx->poll_wq);
|
| 278 | spin_lock_init(&ctx->completion_lock);
|
| 279 | raw_spin_lock_init(&ctx->timeout_lock);
|
| 280 | INIT_LIST_HEAD(&ctx->iopoll_list);
|
| 281 | INIT_LIST_HEAD(&ctx->defer_list);
|
| 282 | INIT_LIST_HEAD(&ctx->timeout_list);
|
| 283 | INIT_LIST_HEAD(&ctx->ltimeout_list);
|
| 284 | mpscq_init(&ctx->work_list, &ctx->work_head);
|
| 285 | INIT_LIST_HEAD(&ctx->tctx_list);
|
| 286 | mutex_init(&ctx->tctx_lock);
|
| 287 | ctx->submit_state.free_list.next = NULL;
|
| 288 | INIT_HLIST_HEAD(&ctx->waitid_list);
|
| 289 | xa_init_flags(&ctx->zcrx_ctxs, XA_FLAGS_ALLOC);
|
| 290 | #ifdef CONFIG_FUTEX
|
| 291 | INIT_HLIST_HEAD(&ctx->futex_list);
|
| 292 | #endif
|
| 293 | INIT_WQ_LIST(&ctx->submit_state.compl_reqs);
|
| 294 | INIT_HLIST_HEAD(&ctx->cancelable_uring_cmd);
|
| 295 | io_napi_init(ctx);
|
| 296 | mutex_init(&ctx->mmap_lock);
|
| 297 | ctx->kcov_handle = kcov_common_handle();
|
| 298 |
|
| 299 | return ctx;
|
| 300 |
|
| 301 | free_ref:
|
| 302 | percpu_ref_exit(&ctx->refs);
|
| 303 | err:
|
| 304 | io_free_alloc_caches(ctx);
|
| 305 | kvfree(ctx->cancel_table.hbs);
|
| 306 | xa_destroy(&ctx->io_bl_xa);
|
| 307 | xa_destroy(&ctx->hpage_acct);
|
| 308 | kfree(ctx);
|
| 309 | return NULL;
|
| 310 | }
|
| 311 |
|
| 312 | static void io_clean_op(struct io_kiocb *req)
|
| 313 | {
|
| 314 | if (unlikely(req->flags & REQ_F_BUFFER_SELECTED))
|
| 315 | io_kbuf_drop_legacy(req);
|
| 316 |
|
| 317 | if (req->flags & REQ_F_NEED_CLEANUP) {
|
| 318 | const struct io_cold_def *def = &io_cold_defs[req->opcode];
|
| 319 |
|
| 320 | if (def->cleanup)
|
| 321 | def->cleanup(req);
|
| 322 | }
|
| 323 | if (req->flags & REQ_F_INFLIGHT)
|
| 324 | atomic_dec(&req->tctx->inflight_tracked);
|
| 325 | if (req->flags & REQ_F_CREDS)
|
| 326 | put_cred(req->creds);
|
| 327 | if (req->flags & REQ_F_ASYNC_DATA) {
|
| 328 | kfree(req->async_data);
|
| 329 | req->async_data = NULL;
|
| 330 | }
|
| 331 | req->flags &= ~IO_REQ_CLEAN_FLAGS;
|
| 332 | }
|
| 333 |
|
| 334 | |
| 335 | * Mark the request as inflight, so that file cancelation will find it.
|
| 336 | * Can be used if the file is an io_uring instance, or if the request itself
|
| 337 | * relies on ->mm being alive for the duration of the request.
|
| 338 |
|
| 339 | inline void io_req_track_inflight(struct io_kiocb *req)
|
| 340 | {
|
| 341 | if (!(req->flags & REQ_F_INFLIGHT)) {
|
| 342 | req->flags |= REQ_F_INFLIGHT;
|
| 343 | atomic_inc(&req->tctx->inflight_tracked);
|
| 344 | }
|
| 345 | }
|
| 346 |
|
| 347 | static struct io_kiocb *__io_prep_linked_timeout(struct io_kiocb *req)
|
| 348 | {
|
| 349 | if (WARN_ON_ONCE(!req->link))
|
| 350 | return NULL;
|
| 351 |
|
| 352 | req->flags &= ~REQ_F_ARM_LTIMEOUT;
|
| 353 | req->flags |= REQ_F_LINK_TIMEOUT;
|
| 354 |
|
| 355 |
|
| 356 | io_req_set_refcount(req);
|
| 357 | __io_req_set_refcount(req->link, 2);
|
| 358 | return req->link;
|
| 359 | }
|
| 360 |
|
| 361 | static void io_prep_async_work(struct io_kiocb *req)
|
| 362 | {
|
| 363 | const struct io_issue_def *def = &io_issue_defs[req->opcode];
|
| 364 |
|
| 365 | if (!(req->flags & REQ_F_CREDS)) {
|
| 366 | req->flags |= REQ_F_CREDS;
|
| 367 | req->creds = get_current_cred();
|
| 368 | }
|
| 369 |
|
| 370 | req->work.list.next = NULL;
|
| 371 | atomic_set(&req->work.flags, 0);
|
| 372 | if (req->flags & REQ_F_FORCE_ASYNC)
|
| 373 | atomic_or(IO_WQ_WORK_CONCURRENT, &req->work.flags);
|
| 374 |
|
| 375 | if (req->file && !(req->flags & REQ_F_FIXED_FILE))
|
| 376 | req->flags |= io_file_get_flags(req->file);
|
| 377 |
|
| 378 | if (req->file && (req->flags & REQ_F_ISREG)) {
|
| 379 | bool should_hash = def->hash_reg_file;
|
| 380 |
|
| 381 |
|
| 382 | if (should_hash && (req->file->f_flags & O_DIRECT) &&
|
| 383 | (req->file->f_op->fop_flags & FOP_DIO_PARALLEL_WRITE))
|
| 384 | should_hash = false;
|
| 385 | if (should_hash || (req->flags & REQ_F_IOPOLL))
|
| 386 | io_wq_hash_work(&req->work, file_inode(req->file));
|
| 387 | } else if (!req->file || !S_ISBLK(file_inode(req->file)->i_mode)) {
|
| 388 | if (def->unbound_nonreg_file)
|
| 389 | atomic_or(IO_WQ_WORK_UNBOUND, &req->work.flags);
|
| 390 | }
|
| 391 | }
|
| 392 |
|
| 393 | static void io_prep_async_link(struct io_kiocb *req)
|
| 394 | {
|
| 395 | struct io_kiocb *cur;
|
| 396 |
|
| 397 | if (req->flags & REQ_F_LINK_TIMEOUT) {
|
| 398 | struct io_ring_ctx *ctx = req->ctx;
|
| 399 |
|
| 400 | raw_spin_lock_irq(&ctx->timeout_lock);
|
| 401 | io_for_each_link(cur, req)
|
| 402 | io_prep_async_work(cur);
|
| 403 | raw_spin_unlock_irq(&ctx->timeout_lock);
|
| 404 | } else {
|
| 405 | io_for_each_link(cur, req)
|
| 406 | io_prep_async_work(cur);
|
| 407 | }
|
| 408 | }
|
| 409 |
|
| 410 | void io_queue_iowq(struct io_kiocb *req)
|
| 411 | {
|
| 412 | struct io_uring_task *tctx = req->tctx;
|
| 413 |
|
| 414 | BUG_ON(!tctx);
|
| 415 |
|
| 416 | if ((current->flags & PF_KTHREAD) || !tctx->io_wq) {
|
| 417 | io_req_task_queue_fail(req, -ECANCELED);
|
| 418 | return;
|
| 419 | }
|
| 420 |
|
| 421 |
|
| 422 | io_prep_async_link(req);
|
| 423 |
|
| 424 | |
| 425 | * Not expected to happen, but if we do have a bug where this _can_
|
| 426 | * happen, catch it here and ensure the request is marked as
|
| 427 | * canceled. That will make io-wq go through the usual work cancel
|
| 428 | * procedure rather than attempt to run this request (or create a new
|
| 429 | * worker for it).
|
| 430 |
|
| 431 | if (WARN_ON_ONCE(!same_thread_group(tctx->task, current)))
|
| 432 | atomic_or(IO_WQ_WORK_CANCEL, &req->work.flags);
|
| 433 |
|
| 434 | trace_io_uring_queue_async_work(req, io_wq_is_hashed(&req->work));
|
| 435 | io_wq_enqueue(tctx->io_wq, &req->work);
|
| 436 | }
|
| 437 |
|
| 438 | unsigned io_linked_nr(struct io_kiocb *req)
|
| 439 | {
|
| 440 | struct io_kiocb *tmp;
|
| 441 | unsigned nr = 0;
|
| 442 |
|
| 443 | io_for_each_link(tmp, req)
|
| 444 | nr++;
|
| 445 | return nr;
|
| 446 | }
|
| 447 |
|
| 448 | static __cold noinline void io_queue_deferred(struct io_ring_ctx *ctx)
|
| 449 | {
|
| 450 | bool drain_seen = false, first = true;
|
| 451 |
|
| 452 | lockdep_assert_held(&ctx->uring_lock);
|
| 453 | __io_req_caches_free(ctx);
|
| 454 |
|
| 455 | while (!list_empty(&ctx->defer_list)) {
|
| 456 | struct io_defer_entry *de = list_first_entry(&ctx->defer_list,
|
| 457 | struct io_defer_entry, list);
|
| 458 |
|
| 459 | drain_seen |= de->req->flags & REQ_F_IO_DRAIN;
|
| 460 | if ((drain_seen || first) && ctx->nr_req_allocated != ctx->nr_drained)
|
| 461 | return;
|
| 462 |
|
| 463 | list_del_init(&de->list);
|
| 464 | ctx->nr_drained -= io_linked_nr(de->req);
|
| 465 | io_req_task_queue(de->req);
|
| 466 | kfree(de);
|
| 467 | first = false;
|
| 468 | }
|
| 469 | }
|
| 470 |
|
| 471 | void __io_commit_cqring_flush(struct io_ring_ctx *ctx)
|
| 472 | {
|
| 473 | if (ctx->int_flags & IO_RING_F_POLL_ACTIVATED)
|
| 474 | io_poll_wq_wake(ctx);
|
| 475 | if (ctx->int_flags & IO_RING_F_OFF_TIMEOUT_USED)
|
| 476 | io_flush_timeouts(ctx);
|
| 477 | if (ctx->int_flags & IO_RING_F_HAS_EVFD)
|
| 478 | io_eventfd_signal(ctx, true, false);
|
| 479 | }
|
| 480 |
|
| 481 | static inline void __io_cq_lock(struct io_ring_ctx *ctx)
|
| 482 | {
|
| 483 | if (!(ctx->int_flags & IO_RING_F_LOCKLESS_CQ))
|
| 484 | spin_lock(&ctx->completion_lock);
|
| 485 | }
|
| 486 |
|
| 487 | static inline void io_cq_lock(struct io_ring_ctx *ctx)
|
| 488 | __acquires(ctx->completion_lock)
|
| 489 | {
|
| 490 | spin_lock(&ctx->completion_lock);
|
| 491 | }
|
| 492 |
|
| 493 | static inline void __io_cq_unlock_post(struct io_ring_ctx *ctx)
|
| 494 | {
|
| 495 | io_commit_cqring(ctx);
|
| 496 | if (!(ctx->int_flags & IO_RING_F_TASK_COMPLETE)) {
|
| 497 | if (!(ctx->int_flags & IO_RING_F_LOCKLESS_CQ))
|
| 498 | spin_unlock(&ctx->completion_lock);
|
| 499 |
|
| 500 | if (!(ctx->int_flags & IO_RING_F_SYSCALL_IOPOLL))
|
| 501 | io_cqring_wake(ctx);
|
| 502 | }
|
| 503 | io_commit_cqring_flush(ctx);
|
| 504 | }
|
| 505 |
|
| 506 | static void io_cq_unlock_post(struct io_ring_ctx *ctx)
|
| 507 | __releases(ctx->completion_lock)
|
| 508 | {
|
| 509 | io_commit_cqring(ctx);
|
| 510 | spin_unlock(&ctx->completion_lock);
|
| 511 | io_cqring_wake(ctx);
|
| 512 | io_commit_cqring_flush(ctx);
|
| 513 | }
|
| 514 |
|
| 515 | static void __io_cqring_overflow_flush(struct io_ring_ctx *ctx, bool dying)
|
| 516 | {
|
| 517 | lockdep_assert_held(&ctx->uring_lock);
|
| 518 |
|
| 519 |
|
| 520 | if (!dying && __io_cqring_events(ctx) == ctx->cq_entries)
|
| 521 | return;
|
| 522 |
|
| 523 | io_cq_lock(ctx);
|
| 524 | while (!list_empty(&ctx->cq_overflow_list)) {
|
| 525 | size_t cqe_size = sizeof(struct io_uring_cqe);
|
| 526 | struct io_uring_cqe *cqe;
|
| 527 | struct io_overflow_cqe *ocqe;
|
| 528 | bool is_cqe32 = false;
|
| 529 |
|
| 530 | ocqe = list_first_entry(&ctx->cq_overflow_list,
|
| 531 | struct io_overflow_cqe, list);
|
| 532 | if (ocqe->cqe.flags & IORING_CQE_F_32 ||
|
| 533 | ctx->flags & IORING_SETUP_CQE32) {
|
| 534 | is_cqe32 = true;
|
| 535 | cqe_size <<= 1;
|
| 536 | }
|
| 537 | if (ctx->flags & IORING_SETUP_CQE32)
|
| 538 | is_cqe32 = false;
|
| 539 |
|
| 540 | if (!dying) {
|
| 541 | if (!io_get_cqe_overflow(ctx, &cqe, true, is_cqe32))
|
| 542 | break;
|
| 543 | memcpy(cqe, &ocqe->cqe, cqe_size);
|
| 544 | }
|
| 545 | list_del(&ocqe->list);
|
| 546 | kfree(ocqe);
|
| 547 |
|
| 548 | |
| 549 | * For silly syzbot cases that deliberately overflow by huge
|
| 550 | * amounts, check if we need to resched and drop and
|
| 551 | * reacquire the locks if so. Nothing real would ever hit this.
|
| 552 | * Ideally we'd have a non-posting unlock for this, but hard
|
| 553 | * to care for a non-real case.
|
| 554 |
|
| 555 | if (need_resched()) {
|
| 556 | ctx->cqe_sentinel = ctx->cqe_cached;
|
| 557 | io_cq_unlock_post(ctx);
|
| 558 | mutex_unlock(&ctx->uring_lock);
|
| 559 | cond_resched();
|
| 560 | mutex_lock(&ctx->uring_lock);
|
| 561 | io_cq_lock(ctx);
|
| 562 | }
|
| 563 | }
|
| 564 |
|
| 565 | if (list_empty(&ctx->cq_overflow_list)) {
|
| 566 | clear_bit(IO_CHECK_CQ_OVERFLOW_BIT, &ctx->check_cq);
|
| 567 | atomic_andnot(IORING_SQ_CQ_OVERFLOW, &ctx->rings->sq_flags);
|
| 568 | }
|
| 569 | io_cq_unlock_post(ctx);
|
| 570 | }
|
| 571 |
|
| 572 | static void io_cqring_overflow_kill(struct io_ring_ctx *ctx)
|
| 573 | {
|
| 574 | if (ctx->rings)
|
| 575 | __io_cqring_overflow_flush(ctx, true);
|
| 576 | }
|
| 577 |
|
| 578 | void io_cqring_do_overflow_flush(struct io_ring_ctx *ctx)
|
| 579 | {
|
| 580 | mutex_lock(&ctx->uring_lock);
|
| 581 | __io_cqring_overflow_flush(ctx, false);
|
| 582 | mutex_unlock(&ctx->uring_lock);
|
| 583 | }
|
| 584 |
|
| 585 | void io_cqring_overflow_flush_locked(struct io_ring_ctx *ctx)
|
| 586 | {
|
| 587 | __io_cqring_overflow_flush(ctx, false);
|
| 588 | }
|
| 589 |
|
| 590 |
|
| 591 | static inline void io_put_task(struct io_kiocb *req)
|
| 592 | {
|
| 593 | struct io_uring_task *tctx = req->tctx;
|
| 594 |
|
| 595 | if (likely(tctx->task == current)) {
|
| 596 | tctx->cached_refs++;
|
| 597 | } else {
|
| 598 | percpu_counter_sub(&tctx->inflight, 1);
|
| 599 | if (unlikely(atomic_read(&tctx->in_cancel)))
|
| 600 | wake_up(&tctx->wait);
|
| 601 | put_task_struct(tctx->task);
|
| 602 | }
|
| 603 | }
|
| 604 |
|
| 605 | void io_task_refs_refill(struct io_uring_task *tctx)
|
| 606 | {
|
| 607 | unsigned int refill = -tctx->cached_refs + IO_TCTX_REFS_CACHE_NR;
|
| 608 |
|
| 609 | percpu_counter_add(&tctx->inflight, refill);
|
| 610 | refcount_add(refill, ¤t->usage);
|
| 611 | tctx->cached_refs += refill;
|
| 612 | }
|
| 613 |
|
| 614 | __cold void io_uring_drop_tctx_refs(struct task_struct *task)
|
| 615 | {
|
| 616 | struct io_uring_task *tctx = task->io_uring;
|
| 617 | unsigned int refs = tctx->cached_refs;
|
| 618 |
|
| 619 | if (refs) {
|
| 620 | tctx->cached_refs = 0;
|
| 621 | percpu_counter_sub(&tctx->inflight, refs);
|
| 622 | put_task_struct_many(task, refs);
|
| 623 | }
|
| 624 | }
|
| 625 |
|
| 626 | static __cold bool io_cqring_add_overflow(struct io_ring_ctx *ctx,
|
| 627 | struct io_overflow_cqe *ocqe)
|
| 628 | {
|
| 629 | lockdep_assert_held(&ctx->completion_lock);
|
| 630 |
|
| 631 | if (!ocqe) {
|
| 632 | struct io_rings *r = ctx->rings;
|
| 633 |
|
| 634 | |
| 635 | * If we're in ring overflow flush mode, or in task cancel mode,
|
| 636 | * or cannot allocate an overflow entry, then we need to drop it
|
| 637 | * on the floor.
|
| 638 |
|
| 639 | WRITE_ONCE(r->cq_overflow, READ_ONCE(r->cq_overflow) + 1);
|
| 640 | set_bit(IO_CHECK_CQ_DROPPED_BIT, &ctx->check_cq);
|
| 641 | return false;
|
| 642 | }
|
| 643 | if (list_empty(&ctx->cq_overflow_list)) {
|
| 644 | set_bit(IO_CHECK_CQ_OVERFLOW_BIT, &ctx->check_cq);
|
| 645 | atomic_or(IORING_SQ_CQ_OVERFLOW, &ctx->rings->sq_flags);
|
| 646 |
|
| 647 | }
|
| 648 | list_add_tail(&ocqe->list, &ctx->cq_overflow_list);
|
| 649 | return true;
|
| 650 | }
|
| 651 |
|
| 652 | static struct io_overflow_cqe *io_alloc_ocqe(struct io_ring_ctx *ctx,
|
| 653 | struct io_cqe *cqe,
|
| 654 | struct io_big_cqe *big_cqe, gfp_t gfp)
|
| 655 | {
|
| 656 | struct io_overflow_cqe *ocqe;
|
| 657 | size_t ocq_size = sizeof(struct io_overflow_cqe);
|
| 658 | bool is_cqe32 = false;
|
| 659 |
|
| 660 | if (cqe->flags & IORING_CQE_F_32 || ctx->flags & IORING_SETUP_CQE32) {
|
| 661 | is_cqe32 = true;
|
| 662 | ocq_size += sizeof(struct io_uring_cqe);
|
| 663 | }
|
| 664 |
|
| 665 | ocqe = kzalloc(ocq_size, gfp | __GFP_ACCOUNT);
|
| 666 | trace_io_uring_cqe_overflow(ctx, cqe->user_data, cqe->res, cqe->flags, ocqe);
|
| 667 | if (ocqe) {
|
| 668 | ocqe->cqe.user_data = cqe->user_data;
|
| 669 | ocqe->cqe.res = cqe->res;
|
| 670 | ocqe->cqe.flags = cqe->flags;
|
| 671 | if (is_cqe32 && big_cqe) {
|
| 672 | ocqe->cqe.big_cqe[0] = big_cqe->extra1;
|
| 673 | ocqe->cqe.big_cqe[1] = big_cqe->extra2;
|
| 674 | }
|
| 675 | }
|
| 676 | if (big_cqe)
|
| 677 | big_cqe->extra1 = big_cqe->extra2 = 0;
|
| 678 | return ocqe;
|
| 679 | }
|
| 680 |
|
| 681 | |
| 682 | * Compute queued CQEs for free-space calculation, clamped to cq_entries.
|
| 683 |
|
| 684 | static unsigned int io_cqring_queued(struct io_ring_ctx *ctx)
|
| 685 | {
|
| 686 | struct io_rings *rings = io_get_rings(ctx);
|
| 687 | int diff;
|
| 688 |
|
| 689 | diff = (int)(ctx->cached_cq_tail - READ_ONCE(rings->cq.head));
|
| 690 | if (diff >= 0)
|
| 691 | return min((unsigned int)diff, ctx->cq_entries);
|
| 692 | return 0;
|
| 693 | }
|
| 694 |
|
| 695 | |
| 696 | * Fill an empty dummy CQE, in case alignment is off for posting a 32b CQE
|
| 697 | * because the ring is a single 16b entry away from wrapping.
|
| 698 |
|
| 699 | static bool io_fill_nop_cqe(struct io_ring_ctx *ctx, unsigned int off)
|
| 700 | {
|
| 701 | if (io_cqring_queued(ctx) < ctx->cq_entries) {
|
| 702 | struct io_uring_cqe *cqe = &ctx->rings->cqes[off];
|
| 703 |
|
| 704 | cqe->user_data = 0;
|
| 705 | cqe->res = 0;
|
| 706 | cqe->flags = IORING_CQE_F_SKIP;
|
| 707 | ctx->cached_cq_tail++;
|
| 708 | return true;
|
| 709 | }
|
| 710 | return false;
|
| 711 | }
|
| 712 |
|
| 713 | |
| 714 | * writes to the cq entry need to come after reading head; the
|
| 715 | * control dependency is enough as we're using WRITE_ONCE to
|
| 716 | * fill the cq entry
|
| 717 |
|
| 718 | bool io_cqe_cache_refill(struct io_ring_ctx *ctx, bool overflow, bool cqe32)
|
| 719 | {
|
| 720 | struct io_rings *rings = ctx->rings;
|
| 721 | unsigned int off = ctx->cached_cq_tail & (ctx->cq_entries - 1);
|
| 722 | unsigned int free, len;
|
| 723 |
|
| 724 | |
| 725 | * Posting into the CQ when there are pending overflowed CQEs may break
|
| 726 | * ordering guarantees, which will affect links, F_MORE users and more.
|
| 727 | * Force overflow the completion.
|
| 728 |
|
| 729 | if (!overflow && (ctx->check_cq & BIT(IO_CHECK_CQ_OVERFLOW_BIT)))
|
| 730 | return false;
|
| 731 |
|
| 732 | |
| 733 | * Post dummy CQE if a 32b CQE is needed and there's only room for a
|
| 734 | * 16b CQE before the ring wraps.
|
| 735 |
|
| 736 | if (cqe32 && (ctx->flags & IORING_SETUP_CQE_MIXED) &&
|
| 737 | off + 1 == ctx->cq_entries) {
|
| 738 | if (!io_fill_nop_cqe(ctx, off))
|
| 739 | return false;
|
| 740 | off = 0;
|
| 741 | }
|
| 742 |
|
| 743 | free = ctx->cq_entries - io_cqring_queued(ctx);
|
| 744 |
|
| 745 | len = min(free, ctx->cq_entries - off);
|
| 746 | if (ctx->flags & IORING_SETUP_CQE32) {
|
| 747 | if (!len)
|
| 748 | return false;
|
| 749 | off <<= 1;
|
| 750 | len <<= 1;
|
| 751 | } else if (len < (cqe32 + 1)) {
|
| 752 | return false;
|
| 753 | }
|
| 754 |
|
| 755 | ctx->cqe_cached = &rings->cqes[off];
|
| 756 | ctx->cqe_sentinel = ctx->cqe_cached + len;
|
| 757 | return true;
|
| 758 | }
|
| 759 |
|
| 760 | static bool io_fill_cqe_aux32(struct io_ring_ctx *ctx,
|
| 761 | struct io_uring_cqe src_cqe[2])
|
| 762 | {
|
| 763 | struct io_uring_cqe *cqe;
|
| 764 |
|
| 765 | if (WARN_ON_ONCE(!(ctx->flags & (IORING_SETUP_CQE32|IORING_SETUP_CQE_MIXED))))
|
| 766 | return false;
|
| 767 | if (unlikely(!io_get_cqe(ctx, &cqe, true)))
|
| 768 | return false;
|
| 769 |
|
| 770 | memcpy(cqe, src_cqe, 2 * sizeof(*cqe));
|
| 771 | trace_io_uring_complete(ctx, NULL, cqe);
|
| 772 | return true;
|
| 773 | }
|
| 774 |
|
| 775 | static bool io_fill_cqe_aux(struct io_ring_ctx *ctx, u64 user_data, s32 res,
|
| 776 | u32 cflags)
|
| 777 | {
|
| 778 | bool cqe32 = cflags & IORING_CQE_F_32;
|
| 779 | struct io_uring_cqe *cqe;
|
| 780 |
|
| 781 | if (likely(io_get_cqe(ctx, &cqe, cqe32))) {
|
| 782 | WRITE_ONCE(cqe->user_data, user_data);
|
| 783 | WRITE_ONCE(cqe->res, res);
|
| 784 | WRITE_ONCE(cqe->flags, cflags);
|
| 785 |
|
| 786 | if (cqe32 || (ctx->flags & IORING_SETUP_CQE32)) {
|
| 787 | WRITE_ONCE(cqe->big_cqe[0], 0);
|
| 788 | WRITE_ONCE(cqe->big_cqe[1], 0);
|
| 789 | }
|
| 790 |
|
| 791 | trace_io_uring_complete(ctx, NULL, cqe);
|
| 792 | return true;
|
| 793 | }
|
| 794 | return false;
|
| 795 | }
|
| 796 |
|
| 797 | static inline struct io_cqe io_init_cqe(u64 user_data, s32 res, u32 cflags)
|
| 798 | {
|
| 799 | return (struct io_cqe) { .user_data = user_data, .res = res, .flags = cflags };
|
| 800 | }
|
| 801 |
|
| 802 | static __cold void io_cqe_overflow(struct io_ring_ctx *ctx, struct io_cqe *cqe,
|
| 803 | struct io_big_cqe *big_cqe)
|
| 804 | {
|
| 805 | struct io_overflow_cqe *ocqe;
|
| 806 |
|
| 807 | ocqe = io_alloc_ocqe(ctx, cqe, big_cqe, GFP_KERNEL);
|
| 808 | spin_lock(&ctx->completion_lock);
|
| 809 | io_cqring_add_overflow(ctx, ocqe);
|
| 810 | spin_unlock(&ctx->completion_lock);
|
| 811 | }
|
| 812 |
|
| 813 | static __cold bool io_cqe_overflow_locked(struct io_ring_ctx *ctx,
|
| 814 | struct io_cqe *cqe,
|
| 815 | struct io_big_cqe *big_cqe)
|
| 816 | {
|
| 817 | struct io_overflow_cqe *ocqe;
|
| 818 |
|
| 819 | ocqe = io_alloc_ocqe(ctx, cqe, big_cqe, GFP_NOWAIT);
|
| 820 | return io_cqring_add_overflow(ctx, ocqe);
|
| 821 | }
|
| 822 |
|
| 823 | bool io_post_aux_cqe(struct io_ring_ctx *ctx, u64 user_data, s32 res, u32 cflags)
|
| 824 | {
|
| 825 | bool filled;
|
| 826 |
|
| 827 | io_cq_lock(ctx);
|
| 828 | filled = io_fill_cqe_aux(ctx, user_data, res, cflags);
|
| 829 | if (unlikely(!filled)) {
|
| 830 | struct io_cqe cqe = io_init_cqe(user_data, res, cflags);
|
| 831 |
|
| 832 | filled = io_cqe_overflow_locked(ctx, &cqe, NULL);
|
| 833 | }
|
| 834 | io_cq_unlock_post(ctx);
|
| 835 | return filled;
|
| 836 | }
|
| 837 |
|
| 838 | |
| 839 | * Must be called from inline task_work so we know a flush will happen later,
|
| 840 | * and obviously with ctx->uring_lock held (tw always has that).
|
| 841 |
|
| 842 | void io_add_aux_cqe(struct io_ring_ctx *ctx, u64 user_data, s32 res, u32 cflags)
|
| 843 | {
|
| 844 | lockdep_assert_held(&ctx->uring_lock);
|
| 845 | lockdep_assert(ctx->int_flags & IO_RING_F_LOCKLESS_CQ);
|
| 846 |
|
| 847 | if (!io_fill_cqe_aux(ctx, user_data, res, cflags)) {
|
| 848 | struct io_cqe cqe = io_init_cqe(user_data, res, cflags);
|
| 849 |
|
| 850 | io_cqe_overflow(ctx, &cqe, NULL);
|
| 851 | }
|
| 852 | ctx->submit_state.cq_flush = true;
|
| 853 | }
|
| 854 |
|
| 855 | |
| 856 | * A helper for multishot requests posting additional CQEs.
|
| 857 | * Should only be used from a task_work including IO_URING_F_MULTISHOT.
|
| 858 |
|
| 859 | bool io_req_post_cqe(struct io_kiocb *req, s32 res, u32 cflags)
|
| 860 | {
|
| 861 | struct io_ring_ctx *ctx = req->ctx;
|
| 862 | bool posted;
|
| 863 |
|
| 864 | |
| 865 | * If multishot has already posted deferred completions, ensure that
|
| 866 | * those are flushed first before posting this one. If not, CQEs
|
| 867 | * could get reordered.
|
| 868 |
|
| 869 | if (!wq_list_empty(&ctx->submit_state.compl_reqs))
|
| 870 | __io_submit_flush_completions(ctx);
|
| 871 |
|
| 872 | lockdep_assert(!io_wq_current_is_worker());
|
| 873 | lockdep_assert_held(&ctx->uring_lock);
|
| 874 |
|
| 875 | if (!(ctx->int_flags & IO_RING_F_LOCKLESS_CQ)) {
|
| 876 | spin_lock(&ctx->completion_lock);
|
| 877 | posted = io_fill_cqe_aux(ctx, req->cqe.user_data, res, cflags);
|
| 878 | spin_unlock(&ctx->completion_lock);
|
| 879 | } else {
|
| 880 | posted = io_fill_cqe_aux(ctx, req->cqe.user_data, res, cflags);
|
| 881 | }
|
| 882 |
|
| 883 | ctx->submit_state.cq_flush = true;
|
| 884 | return posted;
|
| 885 | }
|
| 886 |
|
| 887 | |
| 888 | * A helper for multishot requests posting additional CQEs.
|
| 889 | * Should only be used from a task_work including IO_URING_F_MULTISHOT.
|
| 890 |
|
| 891 | bool io_req_post_cqe32(struct io_kiocb *req, struct io_uring_cqe cqe[2])
|
| 892 | {
|
| 893 | struct io_ring_ctx *ctx = req->ctx;
|
| 894 | bool posted;
|
| 895 |
|
| 896 | lockdep_assert(!io_wq_current_is_worker());
|
| 897 | lockdep_assert_held(&ctx->uring_lock);
|
| 898 |
|
| 899 | cqe[0].user_data = req->cqe.user_data;
|
| 900 | if (!(ctx->int_flags & IO_RING_F_LOCKLESS_CQ)) {
|
| 901 | spin_lock(&ctx->completion_lock);
|
| 902 | posted = io_fill_cqe_aux32(ctx, cqe);
|
| 903 | spin_unlock(&ctx->completion_lock);
|
| 904 | } else {
|
| 905 | posted = io_fill_cqe_aux32(ctx, cqe);
|
| 906 | }
|
| 907 |
|
| 908 | ctx->submit_state.cq_flush = true;
|
| 909 | return posted;
|
| 910 | }
|
| 911 |
|
| 912 | static void io_req_complete_post(struct io_kiocb *req, unsigned issue_flags)
|
| 913 | {
|
| 914 | struct io_ring_ctx *ctx = req->ctx;
|
| 915 | bool completed = true;
|
| 916 |
|
| 917 | |
| 918 | * All execution paths but io-wq use the deferred completions by
|
| 919 | * passing IO_URING_F_COMPLETE_DEFER and thus should not end up here.
|
| 920 |
|
| 921 | if (WARN_ON_ONCE(!(issue_flags & IO_URING_F_IOWQ)))
|
| 922 | return;
|
| 923 |
|
| 924 | |
| 925 | * Handle special CQ sync cases via task_work. DEFER_TASKRUN requires
|
| 926 | * the submitter task context, IOPOLL protects with uring_lock.
|
| 927 |
|
| 928 | if ((ctx->int_flags & IO_RING_F_LOCKLESS_CQ) || (req->flags & REQ_F_REISSUE)) {
|
| 929 | defer_complete:
|
| 930 | req->io_task_work.func = io_req_task_complete;
|
| 931 | io_req_task_work_add(req);
|
| 932 | return;
|
| 933 | }
|
| 934 |
|
| 935 | io_cq_lock(ctx);
|
| 936 | if (!(req->flags & REQ_F_CQE_SKIP))
|
| 937 | completed = io_fill_cqe_req(ctx, req);
|
| 938 | io_cq_unlock_post(ctx);
|
| 939 |
|
| 940 | if (!completed)
|
| 941 | goto defer_complete;
|
| 942 |
|
| 943 | |
| 944 | * We don't free the request here because we know it's called from
|
| 945 | * io-wq only, which holds a reference, so it cannot be the last put.
|
| 946 |
|
| 947 | req_ref_put(req);
|
| 948 | }
|
| 949 |
|
| 950 | void io_req_defer_failed(struct io_kiocb *req, s32 res)
|
| 951 | __must_hold(&ctx->uring_lock)
|
| 952 | {
|
| 953 | const struct io_cold_def *def = &io_cold_defs[req->opcode];
|
| 954 |
|
| 955 | lockdep_assert_held(&req->ctx->uring_lock);
|
| 956 |
|
| 957 | req_set_fail(req);
|
| 958 | io_req_set_res(req, res, io_put_kbuf(req, res, NULL));
|
| 959 | if (def->fail)
|
| 960 | def->fail(req);
|
| 961 | io_req_complete_defer(req);
|
| 962 | }
|
| 963 |
|
| 964 | |
| 965 | * A request might get retired back into the request caches even before opcode
|
| 966 | * handlers and io_issue_sqe() are done with it, e.g. inline completion path.
|
| 967 | * Because of that, io_alloc_req() should be called only under ->uring_lock
|
| 968 | * and with extra caution to not get a request that is still worked on.
|
| 969 |
|
| 970 | __cold bool __io_alloc_req_refill(struct io_ring_ctx *ctx)
|
| 971 | __must_hold(&ctx->uring_lock)
|
| 972 | {
|
| 973 | gfp_t gfp = GFP_KERNEL | __GFP_NOWARN | __GFP_ZERO;
|
| 974 | void *reqs[IO_REQ_ALLOC_BATCH];
|
| 975 | int nr_reqs = ARRAY_SIZE(reqs);
|
| 976 |
|
| 977 | |
| 978 | * Bulk alloc is all-or-nothing. If we fail to get a batch, retry a
|
| 979 | * single allocation to be on the safe side.
|
| 980 |
|
| 981 | if (!kmem_cache_alloc_bulk(req_cachep, gfp, nr_reqs, reqs)) {
|
| 982 | reqs[0] = kmem_cache_alloc(req_cachep, gfp);
|
| 983 | if (!reqs[0])
|
| 984 | return false;
|
| 985 | nr_reqs = 1;
|
| 986 | }
|
| 987 |
|
| 988 | percpu_ref_get_many(&ctx->refs, nr_reqs);
|
| 989 | ctx->nr_req_allocated += nr_reqs;
|
| 990 |
|
| 991 | while (nr_reqs--)
|
| 992 | io_req_add_to_cache(reqs[nr_reqs], ctx);
|
| 993 | return true;
|
| 994 | }
|
| 995 |
|
| 996 | __cold void io_free_req(struct io_kiocb *req)
|
| 997 | {
|
| 998 |
|
| 999 | req->flags &= ~REQ_F_REFCOUNT;
|
| 1000 |
|
| 1001 | req->flags |= REQ_F_CQE_SKIP;
|
| 1002 | req->io_task_work.func = io_req_task_complete;
|
| 1003 | io_req_task_work_add(req);
|
| 1004 | }
|
| 1005 |
|
| 1006 | static void __io_req_find_next_prep(struct io_kiocb *req)
|
| 1007 | {
|
| 1008 | struct io_ring_ctx *ctx = req->ctx;
|
| 1009 |
|
| 1010 | spin_lock(&ctx->completion_lock);
|
| 1011 | io_disarm_next(req);
|
| 1012 | spin_unlock(&ctx->completion_lock);
|
| 1013 | }
|
| 1014 |
|
| 1015 | static inline struct io_kiocb *io_req_find_next(struct io_kiocb *req)
|
| 1016 | {
|
| 1017 | struct io_kiocb *nxt;
|
| 1018 |
|
| 1019 | |
| 1020 | * If LINK is set, we have dependent requests in this chain. If we
|
| 1021 | * didn't fail this request, queue the first one up, moving any other
|
| 1022 | * dependencies to the next request. In case of failure, fail the rest
|
| 1023 | * of the chain.
|
| 1024 |
|
| 1025 | if (unlikely(req->flags & IO_DISARM_MASK))
|
| 1026 | __io_req_find_next_prep(req);
|
| 1027 | nxt = req->link;
|
| 1028 | req->link = NULL;
|
| 1029 | return nxt;
|
| 1030 | }
|
| 1031 |
|
| 1032 | static void io_req_task_cancel(struct io_tw_req tw_req, io_tw_token_t tw)
|
| 1033 | {
|
| 1034 | struct io_kiocb *req = tw_req.req;
|
| 1035 |
|
| 1036 | io_tw_lock(req->ctx, tw);
|
| 1037 | io_req_defer_failed(req, req->cqe.res);
|
| 1038 | }
|
| 1039 |
|
| 1040 | void io_req_task_submit(struct io_tw_req tw_req, io_tw_token_t tw)
|
| 1041 | {
|
| 1042 | struct io_kiocb *req = tw_req.req;
|
| 1043 | struct io_ring_ctx *ctx = req->ctx;
|
| 1044 |
|
| 1045 | io_tw_lock(ctx, tw);
|
| 1046 | if (unlikely(tw.cancel))
|
| 1047 | io_req_defer_failed(req, -EFAULT);
|
| 1048 | else if (req->flags & REQ_F_FORCE_ASYNC)
|
| 1049 | io_queue_iowq(req);
|
| 1050 | else
|
| 1051 | io_queue_sqe(req, 0);
|
| 1052 | }
|
| 1053 |
|
| 1054 | void io_req_task_queue_fail(struct io_kiocb *req, int ret)
|
| 1055 | {
|
| 1056 | io_req_set_res(req, ret, 0);
|
| 1057 | req->io_task_work.func = io_req_task_cancel;
|
| 1058 | io_req_task_work_add(req);
|
| 1059 | }
|
| 1060 |
|
| 1061 | void io_req_task_queue(struct io_kiocb *req)
|
| 1062 | {
|
| 1063 | req->io_task_work.func = io_req_task_submit;
|
| 1064 | io_req_task_work_add(req);
|
| 1065 | }
|
| 1066 |
|
| 1067 | void io_queue_next(struct io_kiocb *req)
|
| 1068 | {
|
| 1069 | struct io_kiocb *nxt = io_req_find_next(req);
|
| 1070 |
|
| 1071 | if (nxt)
|
| 1072 | io_req_task_queue(nxt);
|
| 1073 | }
|
| 1074 |
|
| 1075 | static inline void io_req_put_rsrc_nodes(struct io_kiocb *req)
|
| 1076 | {
|
| 1077 | struct io_ring_ctx *ctx = req->ctx;
|
| 1078 |
|
| 1079 | if (req->file_node) {
|
| 1080 | io_put_rsrc_node(ctx, req->file_node);
|
| 1081 | req->file_node = NULL;
|
| 1082 | }
|
| 1083 | if (req->flags & REQ_F_BUF_NODE)
|
| 1084 | io_put_rsrc_node(ctx, req->buf_node);
|
| 1085 | }
|
| 1086 |
|
| 1087 | static void io_free_batch_list(struct io_ring_ctx *ctx,
|
| 1088 | struct io_wq_work_node *node)
|
| 1089 | __must_hold(&ctx->uring_lock)
|
| 1090 | {
|
| 1091 | do {
|
| 1092 | struct io_kiocb *req = container_of(node, struct io_kiocb,
|
| 1093 | comp_list);
|
| 1094 |
|
| 1095 | if (unlikely(req->flags & IO_REQ_CLEAN_SLOW_FLAGS)) {
|
| 1096 | if (req->flags & REQ_F_REISSUE) {
|
| 1097 | node = req->comp_list.next;
|
| 1098 | req->flags &= ~REQ_F_REISSUE;
|
| 1099 | io_queue_iowq(req);
|
| 1100 | continue;
|
| 1101 | }
|
| 1102 | if (req->flags & REQ_F_REFCOUNT) {
|
| 1103 | node = req->comp_list.next;
|
| 1104 | if (!req_ref_put_and_test(req))
|
| 1105 | continue;
|
| 1106 | }
|
| 1107 | if ((req->flags & REQ_F_POLLED) && req->apoll) {
|
| 1108 | struct async_poll *apoll = req->apoll;
|
| 1109 |
|
| 1110 | if (apoll->double_poll)
|
| 1111 | kfree(apoll->double_poll);
|
| 1112 | io_cache_free(&ctx->apoll_cache, apoll);
|
| 1113 | req->flags &= ~REQ_F_POLLED;
|
| 1114 | }
|
| 1115 | if (req->flags & IO_REQ_LINK_FLAGS)
|
| 1116 | io_queue_next(req);
|
| 1117 | if (unlikely(req->flags & IO_REQ_CLEAN_FLAGS))
|
| 1118 | io_clean_op(req);
|
| 1119 | }
|
| 1120 | io_put_file(req);
|
| 1121 | io_req_put_rsrc_nodes(req);
|
| 1122 | io_put_task(req);
|
| 1123 |
|
| 1124 | node = req->comp_list.next;
|
| 1125 | io_req_add_to_cache(req, ctx);
|
| 1126 | } while (node);
|
| 1127 | }
|
| 1128 |
|
| 1129 | void __io_submit_flush_completions(struct io_ring_ctx *ctx)
|
| 1130 | __must_hold(&ctx->uring_lock)
|
| 1131 | {
|
| 1132 | struct io_submit_state *state = &ctx->submit_state;
|
| 1133 | struct io_wq_work_node *node;
|
| 1134 |
|
| 1135 | __io_cq_lock(ctx);
|
| 1136 | __wq_list_for_each(node, &state->compl_reqs) {
|
| 1137 | struct io_kiocb *req = container_of(node, struct io_kiocb,
|
| 1138 | comp_list);
|
| 1139 |
|
| 1140 | |
| 1141 | * Requests marked with REQUEUE should not post a CQE, they
|
| 1142 | * will go through the io-wq retry machinery and post one
|
| 1143 | * later.
|
| 1144 |
|
| 1145 | if (!(req->flags & (REQ_F_CQE_SKIP | REQ_F_REISSUE)) &&
|
| 1146 | unlikely(!io_fill_cqe_req(ctx, req))) {
|
| 1147 | if (ctx->int_flags & IO_RING_F_LOCKLESS_CQ)
|
| 1148 | io_cqe_overflow(ctx, &req->cqe, &req->big_cqe);
|
| 1149 | else
|
| 1150 | io_cqe_overflow_locked(ctx, &req->cqe, &req->big_cqe);
|
| 1151 | }
|
| 1152 | }
|
| 1153 | __io_cq_unlock_post(ctx);
|
| 1154 |
|
| 1155 | if (!wq_list_empty(&state->compl_reqs)) {
|
| 1156 | io_free_batch_list(ctx, state->compl_reqs.first);
|
| 1157 | INIT_WQ_LIST(&state->compl_reqs);
|
| 1158 | }
|
| 1159 |
|
| 1160 | if (unlikely(ctx->int_flags & IO_RING_F_DRAIN_ACTIVE))
|
| 1161 | io_queue_deferred(ctx);
|
| 1162 |
|
| 1163 | ctx->submit_state.cq_flush = false;
|
| 1164 | }
|
| 1165 |
|
| 1166 | |
| 1167 | * We can't just wait for polled events to come to us, we have to actively
|
| 1168 | * find and complete them.
|
| 1169 |
|
| 1170 | __cold void io_iopoll_try_reap_events(struct io_ring_ctx *ctx)
|
| 1171 | {
|
| 1172 | if (!(ctx->flags & IORING_SETUP_IOPOLL))
|
| 1173 | return;
|
| 1174 |
|
| 1175 | mutex_lock(&ctx->uring_lock);
|
| 1176 | while (!list_empty(&ctx->iopoll_list)) {
|
| 1177 |
|
| 1178 | if (io_do_iopoll(ctx, true) == 0)
|
| 1179 | break;
|
| 1180 | |
| 1181 | * Ensure we allow local-to-the-cpu processing to take place,
|
| 1182 | * in this case we need to ensure that we reap all events.
|
| 1183 | * Also let task_work, etc. to progress by releasing the mutex
|
| 1184 |
|
| 1185 | if (need_resched()) {
|
| 1186 | mutex_unlock(&ctx->uring_lock);
|
| 1187 | cond_resched();
|
| 1188 | mutex_lock(&ctx->uring_lock);
|
| 1189 | }
|
| 1190 | }
|
| 1191 | mutex_unlock(&ctx->uring_lock);
|
| 1192 |
|
| 1193 | if (ctx->flags & IORING_SETUP_DEFER_TASKRUN)
|
| 1194 | io_cancel_local_task_work(ctx);
|
| 1195 | }
|
| 1196 |
|
| 1197 | static int io_iopoll_check(struct io_ring_ctx *ctx, unsigned int min_events)
|
| 1198 | {
|
| 1199 | unsigned long check_cq;
|
| 1200 |
|
| 1201 | min_events = min(min_events, ctx->cq_entries);
|
| 1202 |
|
| 1203 | lockdep_assert_held(&ctx->uring_lock);
|
| 1204 |
|
| 1205 | if (!io_allowed_run_tw(ctx))
|
| 1206 | return -EEXIST;
|
| 1207 |
|
| 1208 | check_cq = READ_ONCE(ctx->check_cq);
|
| 1209 | if (unlikely(check_cq)) {
|
| 1210 | if (check_cq & BIT(IO_CHECK_CQ_OVERFLOW_BIT))
|
| 1211 | __io_cqring_overflow_flush(ctx, false);
|
| 1212 | |
| 1213 | * Similarly do not spin if we have not informed the user of any
|
| 1214 | * dropped CQE.
|
| 1215 |
|
| 1216 | if (check_cq & BIT(IO_CHECK_CQ_DROPPED_BIT))
|
| 1217 | return -EBADR;
|
| 1218 | }
|
| 1219 | |
| 1220 | * Don't enter poll loop if we already have events pending.
|
| 1221 | * If we do, we can potentially be spinning for commands that
|
| 1222 | * already triggered a CQE (eg in error).
|
| 1223 |
|
| 1224 | if (io_cqring_events(ctx))
|
| 1225 | return 0;
|
| 1226 |
|
| 1227 | do {
|
| 1228 | int ret = 0;
|
| 1229 |
|
| 1230 | |
| 1231 | * If a submit got punted to a workqueue, we can have the
|
| 1232 | * application entering polling for a command before it gets
|
| 1233 | * issued. That app will hold the uring_lock for the duration
|
| 1234 | * of the poll right here, so we need to take a breather every
|
| 1235 | * now and then to ensure that the issue has a chance to add
|
| 1236 | * the poll to the issued list. Otherwise we can spin here
|
| 1237 | * forever, while the workqueue is stuck trying to acquire the
|
| 1238 | * very same mutex.
|
| 1239 |
|
| 1240 | if (list_empty(&ctx->iopoll_list) || io_task_work_pending(ctx)) {
|
| 1241 | (void) io_run_local_work_locked(ctx, min_events);
|
| 1242 |
|
| 1243 | if (task_work_pending(current) || list_empty(&ctx->iopoll_list)) {
|
| 1244 | mutex_unlock(&ctx->uring_lock);
|
| 1245 | io_run_task_work();
|
| 1246 | mutex_lock(&ctx->uring_lock);
|
| 1247 | }
|
| 1248 |
|
| 1249 | if (list_empty(&ctx->iopoll_list))
|
| 1250 | break;
|
| 1251 | }
|
| 1252 | ret = io_do_iopoll(ctx, !min_events);
|
| 1253 | if (unlikely(ret < 0))
|
| 1254 | return ret;
|
| 1255 |
|
| 1256 | if (task_sigpending(current))
|
| 1257 | return -EINTR;
|
| 1258 | if (need_resched())
|
| 1259 | break;
|
| 1260 | } while (io_cqring_events(ctx) < min_events);
|
| 1261 |
|
| 1262 | return 0;
|
| 1263 | }
|
| 1264 |
|
| 1265 | void io_req_task_complete(struct io_tw_req tw_req, io_tw_token_t tw)
|
| 1266 | {
|
| 1267 | io_req_complete_defer(tw_req.req);
|
| 1268 | }
|
| 1269 |
|
| 1270 | |
| 1271 | * After the iocb has been issued, it's safe to be found on the poll list.
|
| 1272 | * Adding the kiocb to the list AFTER submission ensures that we don't
|
| 1273 | * find it from a io_do_iopoll() thread before the issuer is done
|
| 1274 | * accessing the kiocb cookie.
|
| 1275 |
|
| 1276 | static void io_iopoll_req_issued(struct io_kiocb *req, unsigned int issue_flags)
|
| 1277 | {
|
| 1278 | struct io_ring_ctx *ctx = req->ctx;
|
| 1279 | const bool needs_lock = issue_flags & IO_URING_F_UNLOCKED;
|
| 1280 |
|
| 1281 |
|
| 1282 | if (unlikely(needs_lock))
|
| 1283 | mutex_lock(&ctx->uring_lock);
|
| 1284 |
|
| 1285 | |
| 1286 | * Track whether we have multiple files in our lists. This will impact
|
| 1287 | * how we do polling eventually, not spinning if we're on potentially
|
| 1288 | * different devices.
|
| 1289 |
|
| 1290 | if (list_empty(&ctx->iopoll_list)) {
|
| 1291 | ctx->poll_multi_queue = false;
|
| 1292 | } else if (!ctx->poll_multi_queue) {
|
| 1293 | struct io_kiocb *list_req;
|
| 1294 |
|
| 1295 | list_req = list_first_entry(&ctx->iopoll_list, struct io_kiocb, iopoll_node);
|
| 1296 | if (list_req->file != req->file)
|
| 1297 | ctx->poll_multi_queue = true;
|
| 1298 | }
|
| 1299 |
|
| 1300 | list_add_tail(&req->iopoll_node, &ctx->iopoll_list);
|
| 1301 |
|
| 1302 | if (unlikely(needs_lock)) {
|
| 1303 | |
| 1304 | * If IORING_SETUP_SQPOLL is enabled, sqes are either handle
|
| 1305 | * in sq thread task context or in io worker task context. If
|
| 1306 | * current task context is sq thread, we don't need to check
|
| 1307 | * whether should wake up sq thread.
|
| 1308 |
|
| 1309 | if ((ctx->flags & IORING_SETUP_SQPOLL) &&
|
| 1310 | wq_has_sleeper(&ctx->sq_data->wait))
|
| 1311 | wake_up(&ctx->sq_data->wait);
|
| 1312 |
|
| 1313 | mutex_unlock(&ctx->uring_lock);
|
| 1314 | }
|
| 1315 | }
|
| 1316 |
|
| 1317 | io_req_flags_t io_file_get_flags(struct file *file)
|
| 1318 | {
|
| 1319 | io_req_flags_t res = 0;
|
| 1320 |
|
| 1321 | BUILD_BUG_ON(REQ_F_ISREG_BIT != REQ_F_SUPPORT_NOWAIT_BIT + 1);
|
| 1322 |
|
| 1323 | if (S_ISREG(file_inode(file)->i_mode))
|
| 1324 | res |= REQ_F_ISREG;
|
| 1325 | if ((file->f_flags & O_NONBLOCK) || (file->f_mode & FMODE_NOWAIT))
|
| 1326 | res |= REQ_F_SUPPORT_NOWAIT;
|
| 1327 | return res;
|
| 1328 | }
|
| 1329 |
|
| 1330 | static __cold void io_drain_req(struct io_kiocb *req)
|
| 1331 | __must_hold(&ctx->uring_lock)
|
| 1332 | {
|
| 1333 | struct io_ring_ctx *ctx = req->ctx;
|
| 1334 | bool drain = req->flags & IOSQE_IO_DRAIN;
|
| 1335 | struct io_defer_entry *de;
|
| 1336 |
|
| 1337 | de = kmalloc_obj(*de, GFP_KERNEL_ACCOUNT);
|
| 1338 | if (!de) {
|
| 1339 | io_req_defer_failed(req, -ENOMEM);
|
| 1340 | return;
|
| 1341 | }
|
| 1342 |
|
| 1343 | io_prep_async_link(req);
|
| 1344 | trace_io_uring_defer(req);
|
| 1345 | de->req = req;
|
| 1346 |
|
| 1347 | ctx->nr_drained += io_linked_nr(req);
|
| 1348 | list_add_tail(&de->list, &ctx->defer_list);
|
| 1349 | io_queue_deferred(ctx);
|
| 1350 | if (!drain && list_empty(&ctx->defer_list))
|
| 1351 | ctx->int_flags &= ~IO_RING_F_DRAIN_ACTIVE;
|
| 1352 | }
|
| 1353 |
|
| 1354 | static bool io_assign_file(struct io_kiocb *req, const struct io_issue_def *def,
|
| 1355 | unsigned int issue_flags)
|
| 1356 | {
|
| 1357 | if (req->file || !def->needs_file)
|
| 1358 | return true;
|
| 1359 |
|
| 1360 | if (req->flags & REQ_F_FIXED_FILE)
|
| 1361 | req->file = io_file_get_fixed(req, req->cqe.fd, issue_flags);
|
| 1362 | else
|
| 1363 | req->file = io_file_get_normal(req, req->cqe.fd);
|
| 1364 |
|
| 1365 | return !!req->file;
|
| 1366 | }
|
| 1367 |
|
| 1368 | #define REQ_ISSUE_SLOW_FLAGS (REQ_F_CREDS | REQ_F_ARM_LTIMEOUT)
|
| 1369 |
|
| 1370 | static inline int __io_issue_sqe(struct io_kiocb *req,
|
| 1371 | unsigned int issue_flags,
|
| 1372 | const struct io_issue_def *def)
|
| 1373 | {
|
| 1374 | const struct cred *creds = NULL;
|
| 1375 | struct io_kiocb *link = NULL;
|
| 1376 | int ret;
|
| 1377 |
|
| 1378 | if (unlikely(req->flags & REQ_ISSUE_SLOW_FLAGS)) {
|
| 1379 | if ((req->flags & REQ_F_CREDS) && req->creds != current_cred())
|
| 1380 | creds = override_creds(req->creds);
|
| 1381 | if (req->flags & REQ_F_ARM_LTIMEOUT)
|
| 1382 | link = __io_prep_linked_timeout(req);
|
| 1383 | }
|
| 1384 |
|
| 1385 | if (!def->audit_skip)
|
| 1386 | audit_uring_entry(req->opcode);
|
| 1387 |
|
| 1388 | ret = def->issue(req, issue_flags);
|
| 1389 |
|
| 1390 | if (!def->audit_skip)
|
| 1391 | audit_uring_exit(!ret, ret);
|
| 1392 |
|
| 1393 | if (unlikely(creds || link)) {
|
| 1394 | if (creds)
|
| 1395 | revert_creds(creds);
|
| 1396 | if (link)
|
| 1397 | io_queue_linked_timeout(link);
|
| 1398 | }
|
| 1399 |
|
| 1400 | return ret;
|
| 1401 | }
|
| 1402 |
|
| 1403 | static int io_issue_sqe(struct io_kiocb *req, unsigned int issue_flags)
|
| 1404 | {
|
| 1405 | const struct io_issue_def *def = &io_issue_defs[req->opcode];
|
| 1406 | int ret;
|
| 1407 |
|
| 1408 | if (unlikely(!io_assign_file(req, def, issue_flags)))
|
| 1409 | return -EBADF;
|
| 1410 |
|
| 1411 | ret = __io_issue_sqe(req, issue_flags, def);
|
| 1412 |
|
| 1413 | if (ret == IOU_COMPLETE) {
|
| 1414 | if (issue_flags & IO_URING_F_COMPLETE_DEFER)
|
| 1415 | io_req_complete_defer(req);
|
| 1416 | else
|
| 1417 | io_req_complete_post(req, issue_flags);
|
| 1418 |
|
| 1419 | return 0;
|
| 1420 | }
|
| 1421 |
|
| 1422 | if (ret == IOU_ISSUE_SKIP_COMPLETE) {
|
| 1423 | ret = 0;
|
| 1424 |
|
| 1425 | if (req->flags & REQ_F_IOPOLL)
|
| 1426 | io_iopoll_req_issued(req, issue_flags);
|
| 1427 | }
|
| 1428 | return ret;
|
| 1429 | }
|
| 1430 |
|
| 1431 | int io_poll_issue(struct io_kiocb *req, io_tw_token_t tw)
|
| 1432 | {
|
| 1433 | const unsigned int issue_flags = IO_URING_F_NONBLOCK |
|
| 1434 | IO_URING_F_MULTISHOT |
|
| 1435 | IO_URING_F_COMPLETE_DEFER;
|
| 1436 | int ret;
|
| 1437 |
|
| 1438 | io_tw_lock(req->ctx, tw);
|
| 1439 |
|
| 1440 | WARN_ON_ONCE(!req->file);
|
| 1441 | if (WARN_ON_ONCE(req->flags & REQ_F_IOPOLL))
|
| 1442 | return -EFAULT;
|
| 1443 |
|
| 1444 | ret = __io_issue_sqe(req, issue_flags, &io_issue_defs[req->opcode]);
|
| 1445 |
|
| 1446 | WARN_ON_ONCE(ret == IOU_ISSUE_SKIP_COMPLETE);
|
| 1447 | return ret;
|
| 1448 | }
|
| 1449 |
|
| 1450 | struct io_wq_work *io_wq_free_work(struct io_wq_work *work)
|
| 1451 | {
|
| 1452 | struct io_kiocb *req = container_of(work, struct io_kiocb, work);
|
| 1453 | struct io_kiocb *nxt = NULL;
|
| 1454 |
|
| 1455 | if (req_ref_put_and_test_atomic(req)) {
|
| 1456 | if (req->flags & IO_REQ_LINK_FLAGS) {
|
| 1457 | struct io_ring_ctx *ctx = req->ctx;
|
| 1458 |
|
| 1459 | mutex_lock(&ctx->uring_lock);
|
| 1460 | nxt = io_req_find_next(req);
|
| 1461 | mutex_unlock(&ctx->uring_lock);
|
| 1462 | }
|
| 1463 | io_free_req(req);
|
| 1464 | }
|
| 1465 | return nxt ? &nxt->work : NULL;
|
| 1466 | }
|
| 1467 |
|
| 1468 | void io_wq_submit_work(struct io_wq_work *work)
|
| 1469 | {
|
| 1470 | struct io_kiocb *req = container_of(work, struct io_kiocb, work);
|
| 1471 | const struct io_issue_def *def = &io_issue_defs[req->opcode];
|
| 1472 | unsigned int issue_flags = IO_URING_F_UNLOCKED | IO_URING_F_IOWQ;
|
| 1473 | bool needs_poll = false;
|
| 1474 | int ret = 0, err = -ECANCELED;
|
| 1475 |
|
| 1476 |
|
| 1477 | if (!(req->flags & REQ_F_REFCOUNT))
|
| 1478 | __io_req_set_refcount(req, 2);
|
| 1479 | else
|
| 1480 | req_ref_get(req);
|
| 1481 |
|
| 1482 |
|
| 1483 | if (atomic_read(&work->flags) & IO_WQ_WORK_CANCEL) {
|
| 1484 | fail:
|
| 1485 | io_req_task_queue_fail(req, err);
|
| 1486 | return;
|
| 1487 | }
|
| 1488 | if (!io_assign_file(req, def, issue_flags)) {
|
| 1489 | err = -EBADF;
|
| 1490 | atomic_or(IO_WQ_WORK_CANCEL, &work->flags);
|
| 1491 | goto fail;
|
| 1492 | }
|
| 1493 |
|
| 1494 | |
| 1495 | * If DEFER_TASKRUN is set, it's only allowed to post CQEs from the
|
| 1496 | * submitter task context. Final request completions are handed to the
|
| 1497 | * right context, however this is not the case of auxiliary CQEs,
|
| 1498 | * which is the main mean of operation for multishot requests.
|
| 1499 | * Don't allow any multishot execution from io-wq. It's more restrictive
|
| 1500 | * than necessary and also cleaner.
|
| 1501 |
|
| 1502 | if (req->flags & (REQ_F_MULTISHOT|REQ_F_APOLL_MULTISHOT)) {
|
| 1503 | err = -EBADFD;
|
| 1504 | if (!io_file_can_poll(req))
|
| 1505 | goto fail;
|
| 1506 | if (req->file->f_flags & O_NONBLOCK ||
|
| 1507 | req->file->f_mode & FMODE_NOWAIT) {
|
| 1508 | err = -ECANCELED;
|
| 1509 | if (io_arm_poll_handler(req, issue_flags) != IO_APOLL_OK)
|
| 1510 | goto fail;
|
| 1511 | return;
|
| 1512 | } else {
|
| 1513 | req->flags &= ~(REQ_F_APOLL_MULTISHOT|REQ_F_MULTISHOT);
|
| 1514 | }
|
| 1515 | }
|
| 1516 |
|
| 1517 | if (req->flags & REQ_F_FORCE_ASYNC) {
|
| 1518 | bool opcode_poll = def->pollin || def->pollout;
|
| 1519 |
|
| 1520 | if (opcode_poll && io_file_can_poll(req)) {
|
| 1521 | needs_poll = true;
|
| 1522 | issue_flags |= IO_URING_F_NONBLOCK;
|
| 1523 | }
|
| 1524 | }
|
| 1525 |
|
| 1526 | do {
|
| 1527 | ret = io_issue_sqe(req, issue_flags);
|
| 1528 | if (ret != -EAGAIN)
|
| 1529 | break;
|
| 1530 |
|
| 1531 | |
| 1532 | * If REQ_F_NOWAIT is set, then don't wait or retry with
|
| 1533 | * poll. -EAGAIN is final for that case.
|
| 1534 |
|
| 1535 | if (req->flags & REQ_F_NOWAIT)
|
| 1536 | break;
|
| 1537 |
|
| 1538 | |
| 1539 | * We can get EAGAIN for iopolled IO even though we're
|
| 1540 | * forcing a sync submission from here, since we can't
|
| 1541 | * wait for request slots on the block side.
|
| 1542 |
|
| 1543 | if (!needs_poll) {
|
| 1544 | if (!(req->flags & REQ_F_IOPOLL))
|
| 1545 | break;
|
| 1546 | if (io_wq_worker_stopped())
|
| 1547 | break;
|
| 1548 | cond_resched();
|
| 1549 | continue;
|
| 1550 | }
|
| 1551 |
|
| 1552 | if (io_arm_poll_handler(req, issue_flags) == IO_APOLL_OK)
|
| 1553 | return;
|
| 1554 |
|
| 1555 | needs_poll = false;
|
| 1556 | issue_flags &= ~IO_URING_F_NONBLOCK;
|
| 1557 | } while (1);
|
| 1558 |
|
| 1559 |
|
| 1560 | if (ret)
|
| 1561 | io_req_task_queue_fail(req, ret);
|
| 1562 | }
|
| 1563 |
|
| 1564 | inline struct file *io_file_get_fixed(struct io_kiocb *req, int fd,
|
| 1565 | unsigned int issue_flags)
|
| 1566 | {
|
| 1567 | struct io_ring_ctx *ctx = req->ctx;
|
| 1568 | struct io_rsrc_node *node;
|
| 1569 | struct file *file = NULL;
|
| 1570 |
|
| 1571 | io_ring_submit_lock(ctx, issue_flags);
|
| 1572 | node = io_rsrc_node_lookup(&ctx->file_table.data, fd);
|
| 1573 | if (node) {
|
| 1574 | node->refs++;
|
| 1575 | req->file_node = node;
|
| 1576 | req->flags |= io_slot_flags(node);
|
| 1577 | file = io_slot_file(node);
|
| 1578 | }
|
| 1579 | io_ring_submit_unlock(ctx, issue_flags);
|
| 1580 | return file;
|
| 1581 | }
|
| 1582 |
|
| 1583 | struct file *io_file_get_normal(struct io_kiocb *req, int fd)
|
| 1584 | {
|
| 1585 | struct file *file = fget(fd);
|
| 1586 |
|
| 1587 | trace_io_uring_file_get(req, fd);
|
| 1588 |
|
| 1589 |
|
| 1590 | if (file && io_is_uring_fops(file))
|
| 1591 | io_req_track_inflight(req);
|
| 1592 | return file;
|
| 1593 | }
|
| 1594 |
|
| 1595 | static int io_req_sqe_copy(struct io_kiocb *req, unsigned int issue_flags)
|
| 1596 | {
|
| 1597 | const struct io_cold_def *def = &io_cold_defs[req->opcode];
|
| 1598 |
|
| 1599 | if (req->flags & REQ_F_SQE_COPIED)
|
| 1600 | return 0;
|
| 1601 | req->flags |= REQ_F_SQE_COPIED;
|
| 1602 | if (!def->sqe_copy)
|
| 1603 | return 0;
|
| 1604 | if (WARN_ON_ONCE(!(issue_flags & IO_URING_F_INLINE)))
|
| 1605 | return -EFAULT;
|
| 1606 | def->sqe_copy(req);
|
| 1607 | return 0;
|
| 1608 | }
|
| 1609 |
|
| 1610 | static void io_queue_async(struct io_kiocb *req, unsigned int issue_flags, int ret)
|
| 1611 | __must_hold(&req->ctx->uring_lock)
|
| 1612 | {
|
| 1613 | if (ret != -EAGAIN || (req->flags & REQ_F_NOWAIT)) {
|
| 1614 | fail:
|
| 1615 | io_req_defer_failed(req, ret);
|
| 1616 | return;
|
| 1617 | }
|
| 1618 |
|
| 1619 | ret = io_req_sqe_copy(req, issue_flags);
|
| 1620 | if (unlikely(ret))
|
| 1621 | goto fail;
|
| 1622 |
|
| 1623 | switch (io_arm_poll_handler(req, 0)) {
|
| 1624 | case IO_APOLL_READY:
|
| 1625 | io_req_task_queue(req);
|
| 1626 | break;
|
| 1627 | case IO_APOLL_ABORTED:
|
| 1628 | io_queue_iowq(req);
|
| 1629 | break;
|
| 1630 | case IO_APOLL_OK:
|
| 1631 | break;
|
| 1632 | }
|
| 1633 | }
|
| 1634 |
|
| 1635 | static inline void io_queue_sqe(struct io_kiocb *req, unsigned int extra_flags)
|
| 1636 | __must_hold(&req->ctx->uring_lock)
|
| 1637 | {
|
| 1638 | unsigned int issue_flags = IO_URING_F_NONBLOCK |
|
| 1639 | IO_URING_F_COMPLETE_DEFER | extra_flags;
|
| 1640 | int ret;
|
| 1641 |
|
| 1642 | ret = io_issue_sqe(req, issue_flags);
|
| 1643 |
|
| 1644 | |
| 1645 | * We async punt it if the file wasn't marked NOWAIT, or if the file
|
| 1646 | * doesn't support non-blocking read/write attempts
|
| 1647 |
|
| 1648 | if (unlikely(ret))
|
| 1649 | io_queue_async(req, issue_flags, ret);
|
| 1650 | }
|
| 1651 |
|
| 1652 | static void io_queue_sqe_fallback(struct io_kiocb *req)
|
| 1653 | __must_hold(&req->ctx->uring_lock)
|
| 1654 | {
|
| 1655 | if (unlikely(req->flags & REQ_F_FAIL)) {
|
| 1656 | |
| 1657 | * We don't submit, fail them all, for that replace hardlinks
|
| 1658 | * with normal links. Extra REQ_F_LINK is tolerated.
|
| 1659 |
|
| 1660 | req->flags &= ~REQ_F_HARDLINK;
|
| 1661 | req->flags |= REQ_F_LINK;
|
| 1662 | io_req_defer_failed(req, req->cqe.res);
|
| 1663 | } else {
|
| 1664 |
|
| 1665 | io_req_sqe_copy(req, IO_URING_F_INLINE);
|
| 1666 | if (unlikely(req->ctx->int_flags & IO_RING_F_DRAIN_ACTIVE))
|
| 1667 | io_drain_req(req);
|
| 1668 | else
|
| 1669 | io_queue_iowq(req);
|
| 1670 | }
|
| 1671 | }
|
| 1672 |
|
| 1673 | |
| 1674 | * Check SQE restrictions (opcode and flags).
|
| 1675 | *
|
| 1676 | * Returns 'true' if SQE is allowed, 'false' otherwise.
|
| 1677 |
|
| 1678 | static inline bool io_check_restriction(struct io_ring_ctx *ctx,
|
| 1679 | struct io_kiocb *req,
|
| 1680 | unsigned int sqe_flags)
|
| 1681 | {
|
| 1682 | if (!(ctx->int_flags & IO_RING_F_OP_RESTRICTED))
|
| 1683 | return true;
|
| 1684 | if (!test_bit(req->opcode, ctx->restrictions.sqe_op))
|
| 1685 | return false;
|
| 1686 |
|
| 1687 | if ((sqe_flags & ctx->restrictions.sqe_flags_required) !=
|
| 1688 | ctx->restrictions.sqe_flags_required)
|
| 1689 | return false;
|
| 1690 |
|
| 1691 | if (sqe_flags & ~(ctx->restrictions.sqe_flags_allowed |
|
| 1692 | ctx->restrictions.sqe_flags_required))
|
| 1693 | return false;
|
| 1694 |
|
| 1695 | return true;
|
| 1696 | }
|
| 1697 |
|
| 1698 | static void io_init_drain(struct io_ring_ctx *ctx)
|
| 1699 | {
|
| 1700 | struct io_kiocb *head = ctx->submit_state.link.head;
|
| 1701 |
|
| 1702 | ctx->int_flags |= IO_RING_F_DRAIN_ACTIVE;
|
| 1703 | if (head) {
|
| 1704 | |
| 1705 | * If we need to drain a request in the middle of a link, drain
|
| 1706 | * the head request and the next request/link after the current
|
| 1707 | * link. Considering sequential execution of links,
|
| 1708 | * REQ_F_IO_DRAIN will be maintained for every request of our
|
| 1709 | * link.
|
| 1710 |
|
| 1711 | head->flags |= REQ_F_IO_DRAIN | REQ_F_FORCE_ASYNC;
|
| 1712 | ctx->int_flags |= IO_RING_F_DRAIN_NEXT;
|
| 1713 | }
|
| 1714 | }
|
| 1715 |
|
| 1716 | static __cold int io_init_fail_req(struct io_kiocb *req, int err)
|
| 1717 | {
|
| 1718 |
|
| 1719 | memset(&req->cmd.data, 0, sizeof(req->cmd.data));
|
| 1720 | return err;
|
| 1721 | }
|
| 1722 |
|
| 1723 | static int io_init_req(struct io_ring_ctx *ctx, struct io_kiocb *req,
|
| 1724 | const struct io_uring_sqe *sqe, unsigned int *left)
|
| 1725 | __must_hold(&ctx->uring_lock)
|
| 1726 | {
|
| 1727 | const struct io_issue_def *def;
|
| 1728 | unsigned int sqe_flags;
|
| 1729 | int personality;
|
| 1730 |
|
| 1731 | req->ctx = ctx;
|
| 1732 | req->opcode = READ_ONCE(sqe->opcode);
|
| 1733 |
|
| 1734 | sqe_flags = READ_ONCE(sqe->flags);
|
| 1735 | req->flags = (__force io_req_flags_t) sqe_flags;
|
| 1736 | req->cqe.user_data = READ_ONCE(sqe->user_data);
|
| 1737 | req->file = NULL;
|
| 1738 | req->tctx = current->io_uring;
|
| 1739 | req->cancel_seq_set = false;
|
| 1740 | req->async_data = NULL;
|
| 1741 |
|
| 1742 | if (unlikely(req->opcode >= IORING_OP_LAST)) {
|
| 1743 | req->opcode = 0;
|
| 1744 | return io_init_fail_req(req, -EINVAL);
|
| 1745 | }
|
| 1746 | req->opcode = array_index_nospec(req->opcode, IORING_OP_LAST);
|
| 1747 |
|
| 1748 | def = &io_issue_defs[req->opcode];
|
| 1749 | if (def->is_128 && !(ctx->flags & IORING_SETUP_SQE128)) {
|
| 1750 | |
| 1751 | * A 128b op on a non-128b SQ requires mixed SQE support as
|
| 1752 | * well as 2 contiguous entries.
|
| 1753 |
|
| 1754 | if (!(ctx->flags & IORING_SETUP_SQE_MIXED) || *left < 2 ||
|
| 1755 | (unsigned)(sqe - ctx->sq_sqes) >= ctx->sq_entries - 1)
|
| 1756 | return io_init_fail_req(req, -EINVAL);
|
| 1757 | |
| 1758 | * A 128b operation on a mixed SQ uses two entries, so we have
|
| 1759 | * to increment the head and cached refs, and decrement what's
|
| 1760 | * left.
|
| 1761 |
|
| 1762 | current->io_uring->cached_refs++;
|
| 1763 | ctx->cached_sq_head++;
|
| 1764 | (*left)--;
|
| 1765 | }
|
| 1766 |
|
| 1767 | if (unlikely(sqe_flags & ~SQE_COMMON_FLAGS)) {
|
| 1768 |
|
| 1769 | if (sqe_flags & ~SQE_VALID_FLAGS)
|
| 1770 | return io_init_fail_req(req, -EINVAL);
|
| 1771 | if (sqe_flags & IOSQE_BUFFER_SELECT) {
|
| 1772 | if (!def->buffer_select)
|
| 1773 | return io_init_fail_req(req, -EOPNOTSUPP);
|
| 1774 | req->buf_index = READ_ONCE(sqe->buf_group);
|
| 1775 | }
|
| 1776 | if (sqe_flags & IOSQE_CQE_SKIP_SUCCESS)
|
| 1777 | ctx->int_flags |= IO_RING_F_DRAIN_DISABLED;
|
| 1778 | if (sqe_flags & IOSQE_IO_DRAIN) {
|
| 1779 | if (ctx->int_flags & IO_RING_F_DRAIN_DISABLED)
|
| 1780 | return io_init_fail_req(req, -EOPNOTSUPP);
|
| 1781 | io_init_drain(ctx);
|
| 1782 | }
|
| 1783 | }
|
| 1784 | if (unlikely(ctx->int_flags & (IO_RING_F_OP_RESTRICTED | IO_RING_F_DRAIN_ACTIVE | IO_RING_F_DRAIN_NEXT))) {
|
| 1785 | if (!io_check_restriction(ctx, req, sqe_flags))
|
| 1786 | return io_init_fail_req(req, -EACCES);
|
| 1787 |
|
| 1788 | if (ctx->int_flags & IO_RING_F_DRAIN_ACTIVE)
|
| 1789 | req->flags |= REQ_F_FORCE_ASYNC;
|
| 1790 |
|
| 1791 | if (unlikely(ctx->int_flags & IO_RING_F_DRAIN_NEXT) && !ctx->submit_state.link.head) {
|
| 1792 | ctx->int_flags &= ~IO_RING_F_DRAIN_NEXT;
|
| 1793 | ctx->int_flags |= IO_RING_F_DRAIN_ACTIVE;
|
| 1794 | req->flags |= REQ_F_IO_DRAIN | REQ_F_FORCE_ASYNC;
|
| 1795 | }
|
| 1796 | }
|
| 1797 |
|
| 1798 | if (!def->ioprio && sqe->ioprio)
|
| 1799 | return io_init_fail_req(req, -EINVAL);
|
| 1800 | if (!def->iopoll && (ctx->flags & IORING_SETUP_IOPOLL))
|
| 1801 | return io_init_fail_req(req, -EINVAL);
|
| 1802 |
|
| 1803 | if (def->needs_file) {
|
| 1804 | struct io_submit_state *state = &ctx->submit_state;
|
| 1805 |
|
| 1806 | req->cqe.fd = READ_ONCE(sqe->fd);
|
| 1807 |
|
| 1808 | |
| 1809 | * Plug now if we have more than 2 IO left after this, and the
|
| 1810 | * target is potentially a read/write to block based storage.
|
| 1811 |
|
| 1812 | if (state->need_plug && def->plug) {
|
| 1813 | state->plug_started = true;
|
| 1814 | state->need_plug = false;
|
| 1815 | blk_start_plug_nr_ios(&state->plug, state->submit_nr);
|
| 1816 | }
|
| 1817 | }
|
| 1818 |
|
| 1819 | personality = READ_ONCE(sqe->personality);
|
| 1820 | if (personality) {
|
| 1821 | int ret;
|
| 1822 |
|
| 1823 | req->creds = xa_load(&ctx->personalities, personality);
|
| 1824 | if (!req->creds)
|
| 1825 | return io_init_fail_req(req, -EINVAL);
|
| 1826 | get_cred(req->creds);
|
| 1827 | ret = security_uring_override_creds(req->creds);
|
| 1828 | if (ret) {
|
| 1829 | put_cred(req->creds);
|
| 1830 | return io_init_fail_req(req, ret);
|
| 1831 | }
|
| 1832 | req->flags |= REQ_F_CREDS;
|
| 1833 | }
|
| 1834 |
|
| 1835 | return def->prep(req, sqe);
|
| 1836 | }
|
| 1837 |
|
| 1838 | static __cold int io_submit_fail_init(const struct io_uring_sqe *sqe,
|
| 1839 | struct io_kiocb *req, int ret)
|
| 1840 | {
|
| 1841 | struct io_ring_ctx *ctx = req->ctx;
|
| 1842 | struct io_submit_link *link = &ctx->submit_state.link;
|
| 1843 | struct io_kiocb *head = link->head;
|
| 1844 |
|
| 1845 | trace_io_uring_req_failed(sqe, req, ret);
|
| 1846 |
|
| 1847 | |
| 1848 | * Avoid breaking links in the middle as it renders links with SQPOLL
|
| 1849 | * unusable. Instead of failing eagerly, continue assembling the link if
|
| 1850 | * applicable and mark the head with REQ_F_FAIL. The link flushing code
|
| 1851 | * should find the flag and handle the rest.
|
| 1852 |
|
| 1853 | req_fail_link_node(req, ret);
|
| 1854 | if (head && !(head->flags & REQ_F_FAIL))
|
| 1855 | req_fail_link_node(head, -ECANCELED);
|
| 1856 |
|
| 1857 | if (!(req->flags & IO_REQ_LINK_FLAGS)) {
|
| 1858 | if (head) {
|
| 1859 | link->last->link = req;
|
| 1860 | link->head = NULL;
|
| 1861 | req = head;
|
| 1862 | }
|
| 1863 | io_queue_sqe_fallback(req);
|
| 1864 | return ret;
|
| 1865 | }
|
| 1866 |
|
| 1867 | if (head)
|
| 1868 | link->last->link = req;
|
| 1869 | else
|
| 1870 | link->head = req;
|
| 1871 | link->last = req;
|
| 1872 | return 0;
|
| 1873 | }
|
| 1874 |
|
| 1875 | static inline int io_submit_sqe(struct io_ring_ctx *ctx, struct io_kiocb *req,
|
| 1876 | const struct io_uring_sqe *sqe, unsigned int *left)
|
| 1877 | __must_hold(&ctx->uring_lock)
|
| 1878 | {
|
| 1879 | struct io_submit_link *link = &ctx->submit_state.link;
|
| 1880 | int ret;
|
| 1881 |
|
| 1882 | ret = io_init_req(ctx, req, sqe, left);
|
| 1883 | if (unlikely(ret))
|
| 1884 | return io_submit_fail_init(sqe, req, ret);
|
| 1885 |
|
| 1886 | if (unlikely(ctx->bpf_filters)) {
|
| 1887 | ret = io_uring_run_bpf_filters(ctx->bpf_filters, req);
|
| 1888 | if (ret)
|
| 1889 | return io_submit_fail_init(sqe, req, ret);
|
| 1890 | }
|
| 1891 |
|
| 1892 | trace_io_uring_submit_req(req);
|
| 1893 |
|
| 1894 | |
| 1895 | * If we already have a head request, queue this one for async
|
| 1896 | * submittal once the head completes. If we don't have a head but
|
| 1897 | * IOSQE_IO_LINK is set in the sqe, start a new head. This one will be
|
| 1898 | * submitted sync once the chain is complete. If none of those
|
| 1899 | * conditions are true (normal request), then just queue it.
|
| 1900 |
|
| 1901 | if (unlikely(link->head)) {
|
| 1902 | trace_io_uring_link(req, link->last);
|
| 1903 | io_req_sqe_copy(req, IO_URING_F_INLINE);
|
| 1904 | link->last->link = req;
|
| 1905 | link->last = req;
|
| 1906 |
|
| 1907 | if (req->flags & IO_REQ_LINK_FLAGS)
|
| 1908 | return 0;
|
| 1909 |
|
| 1910 | req = link->head;
|
| 1911 | link->head = NULL;
|
| 1912 | if (req->flags & (REQ_F_FORCE_ASYNC | REQ_F_FAIL))
|
| 1913 | goto fallback;
|
| 1914 |
|
| 1915 | } else if (unlikely(req->flags & (IO_REQ_LINK_FLAGS |
|
| 1916 | REQ_F_FORCE_ASYNC | REQ_F_FAIL))) {
|
| 1917 | if (req->flags & IO_REQ_LINK_FLAGS) {
|
| 1918 | link->head = req;
|
| 1919 | link->last = req;
|
| 1920 | } else {
|
| 1921 | fallback:
|
| 1922 | io_queue_sqe_fallback(req);
|
| 1923 | }
|
| 1924 | return 0;
|
| 1925 | }
|
| 1926 |
|
| 1927 | io_queue_sqe(req, IO_URING_F_INLINE);
|
| 1928 | return 0;
|
| 1929 | }
|
| 1930 |
|
| 1931 | |
| 1932 | * Batched submission is done, ensure local IO is flushed out.
|
| 1933 |
|
| 1934 | static void io_submit_state_end(struct io_ring_ctx *ctx)
|
| 1935 | {
|
| 1936 | struct io_submit_state *state = &ctx->submit_state;
|
| 1937 |
|
| 1938 | if (unlikely(state->link.head))
|
| 1939 | io_queue_sqe_fallback(state->link.head);
|
| 1940 |
|
| 1941 | io_submit_flush_completions(ctx);
|
| 1942 | if (state->plug_started)
|
| 1943 | blk_finish_plug(&state->plug);
|
| 1944 | }
|
| 1945 |
|
| 1946 | |
| 1947 | * Start submission side cache.
|
| 1948 |
|
| 1949 | static void io_submit_state_start(struct io_submit_state *state,
|
| 1950 | unsigned int max_ios)
|
| 1951 | {
|
| 1952 | state->plug_started = false;
|
| 1953 | state->need_plug = max_ios > 2;
|
| 1954 | state->submit_nr = max_ios;
|
| 1955 |
|
| 1956 | state->link.head = NULL;
|
| 1957 | }
|
| 1958 |
|
| 1959 | static void io_commit_sqring(struct io_ring_ctx *ctx)
|
| 1960 | {
|
| 1961 | struct io_rings *rings = ctx->rings;
|
| 1962 |
|
| 1963 | if (ctx->flags & IORING_SETUP_SQ_REWIND) {
|
| 1964 | ctx->cached_sq_head = 0;
|
| 1965 | } else {
|
| 1966 | |
| 1967 | * Ensure any loads from the SQEs are done at this point,
|
| 1968 | * since once we write the new head, the application could
|
| 1969 | * write new data to them.
|
| 1970 |
|
| 1971 | smp_store_release(&rings->sq.head, ctx->cached_sq_head);
|
| 1972 | }
|
| 1973 | }
|
| 1974 |
|
| 1975 | |
| 1976 | * Fetch an sqe, if one is available. Note this returns a pointer to memory
|
| 1977 | * that is mapped by userspace. This means that care needs to be taken to
|
| 1978 | * ensure that reads are stable, as we cannot rely on userspace always
|
| 1979 | * being a good citizen. If members of the sqe are validated and then later
|
| 1980 | * used, it's important that those reads are done through READ_ONCE() to
|
| 1981 | * prevent a re-load down the line.
|
| 1982 |
|
| 1983 | static bool io_get_sqe(struct io_ring_ctx *ctx, const struct io_uring_sqe **sqe)
|
| 1984 | {
|
| 1985 | unsigned mask = ctx->sq_entries - 1;
|
| 1986 | unsigned head = ctx->cached_sq_head++ & mask;
|
| 1987 |
|
| 1988 | if (static_branch_unlikely(&io_key_has_sqarray.key) &&
|
| 1989 | (!(ctx->flags & IORING_SETUP_NO_SQARRAY))) {
|
| 1990 | head = READ_ONCE(ctx->sq_array[head]);
|
| 1991 | if (unlikely(head >= ctx->sq_entries)) {
|
| 1992 | WRITE_ONCE(ctx->rings->sq_dropped,
|
| 1993 | READ_ONCE(ctx->rings->sq_dropped) + 1);
|
| 1994 | return false;
|
| 1995 | }
|
| 1996 | head = array_index_nospec(head, ctx->sq_entries);
|
| 1997 | }
|
| 1998 |
|
| 1999 | |
| 2000 | * The cached sq head (or cq tail) serves two purposes:
|
| 2001 | *
|
| 2002 | * 1) allows us to batch the cost of updating the user visible
|
| 2003 | * head updates.
|
| 2004 | * 2) allows the kernel side to track the head on its own, even
|
| 2005 | * though the application is the one updating it.
|
| 2006 |
|
| 2007 |
|
| 2008 |
|
| 2009 | if (ctx->flags & IORING_SETUP_SQE128)
|
| 2010 | head <<= 1;
|
| 2011 | *sqe = &ctx->sq_sqes[head];
|
| 2012 | return true;
|
| 2013 | }
|
| 2014 |
|
| 2015 | int io_submit_sqes(struct io_ring_ctx *ctx, unsigned int nr)
|
| 2016 | __must_hold(&ctx->uring_lock)
|
| 2017 | {
|
| 2018 | unsigned int entries;
|
| 2019 | unsigned int left;
|
| 2020 | int ret;
|
| 2021 |
|
| 2022 | if (ctx->flags & IORING_SETUP_SQ_REWIND)
|
| 2023 | entries = ctx->sq_entries;
|
| 2024 | else
|
| 2025 | entries = __io_sqring_entries(ctx);
|
| 2026 |
|
| 2027 | entries = min(nr, entries);
|
| 2028 | if (unlikely(!entries))
|
| 2029 | return 0;
|
| 2030 |
|
| 2031 | ret = left = entries;
|
| 2032 | io_get_task_refs(left);
|
| 2033 | io_submit_state_start(&ctx->submit_state, left);
|
| 2034 |
|
| 2035 | do {
|
| 2036 | const struct io_uring_sqe *sqe;
|
| 2037 | struct io_kiocb *req;
|
| 2038 |
|
| 2039 | if (unlikely(!io_alloc_req(ctx, &req)))
|
| 2040 | break;
|
| 2041 | if (unlikely(!io_get_sqe(ctx, &sqe))) {
|
| 2042 | io_req_add_to_cache(req, ctx);
|
| 2043 | break;
|
| 2044 | }
|
| 2045 |
|
| 2046 | |
| 2047 | * Continue submitting even for sqe failure if the
|
| 2048 | * ring was setup with IORING_SETUP_SUBMIT_ALL
|
| 2049 |
|
| 2050 | if (unlikely(io_submit_sqe(ctx, req, sqe, &left)) &&
|
| 2051 | !(ctx->flags & IORING_SETUP_SUBMIT_ALL)) {
|
| 2052 | left--;
|
| 2053 | break;
|
| 2054 | }
|
| 2055 | } while (--left);
|
| 2056 |
|
| 2057 | if (unlikely(left)) {
|
| 2058 | ret -= left;
|
| 2059 |
|
| 2060 | if (!ret && io_req_cache_empty(ctx))
|
| 2061 | ret = -EAGAIN;
|
| 2062 | current->io_uring->cached_refs += left;
|
| 2063 | }
|
| 2064 |
|
| 2065 | io_submit_state_end(ctx);
|
| 2066 |
|
| 2067 | io_commit_sqring(ctx);
|
| 2068 | return ret;
|
| 2069 | }
|
| 2070 |
|
| 2071 | static void io_rings_free(struct io_ring_ctx *ctx)
|
| 2072 | {
|
| 2073 |
|
| 2074 | io_free_region(NULL, &ctx->sq_region);
|
| 2075 | io_free_region(NULL, &ctx->ring_region);
|
| 2076 | ctx->rings = NULL;
|
| 2077 | RCU_INIT_POINTER(ctx->rings_rcu, NULL);
|
| 2078 | ctx->sq_sqes = NULL;
|
| 2079 | }
|
| 2080 |
|
| 2081 | static int rings_size(unsigned int flags, unsigned int sq_entries,
|
| 2082 | unsigned int cq_entries, struct io_rings_layout *rl)
|
| 2083 | {
|
| 2084 | struct io_rings *rings;
|
| 2085 | size_t sqe_size;
|
| 2086 | size_t off;
|
| 2087 |
|
| 2088 | if (flags & IORING_SETUP_CQE_MIXED) {
|
| 2089 | if (cq_entries < 2)
|
| 2090 | return -EOVERFLOW;
|
| 2091 | }
|
| 2092 | if (flags & IORING_SETUP_SQE_MIXED) {
|
| 2093 | if (sq_entries < 2)
|
| 2094 | return -EOVERFLOW;
|
| 2095 | }
|
| 2096 |
|
| 2097 | rl->sq_array_offset = SIZE_MAX;
|
| 2098 |
|
| 2099 | sqe_size = sizeof(struct io_uring_sqe);
|
| 2100 | if (flags & IORING_SETUP_SQE128)
|
| 2101 | sqe_size *= 2;
|
| 2102 |
|
| 2103 | rl->sq_size = array_size(sqe_size, sq_entries);
|
| 2104 | if (rl->sq_size == SIZE_MAX)
|
| 2105 | return -EOVERFLOW;
|
| 2106 |
|
| 2107 | off = struct_size(rings, cqes, cq_entries);
|
| 2108 | if (flags & IORING_SETUP_CQE32)
|
| 2109 | off = size_mul(off, 2);
|
| 2110 | if (off == SIZE_MAX)
|
| 2111 | return -EOVERFLOW;
|
| 2112 |
|
| 2113 | #ifdef CONFIG_SMP
|
| 2114 | off = ALIGN(off, SMP_CACHE_BYTES);
|
| 2115 | if (off == 0)
|
| 2116 | return -EOVERFLOW;
|
| 2117 | #endif
|
| 2118 |
|
| 2119 | if (!(flags & IORING_SETUP_NO_SQARRAY)) {
|
| 2120 | size_t sq_array_size;
|
| 2121 |
|
| 2122 | rl->sq_array_offset = off;
|
| 2123 |
|
| 2124 | sq_array_size = array_size(sizeof(u32), sq_entries);
|
| 2125 | off = size_add(off, sq_array_size);
|
| 2126 | if (off == SIZE_MAX)
|
| 2127 | return -EOVERFLOW;
|
| 2128 | }
|
| 2129 |
|
| 2130 | rl->rings_size = off;
|
| 2131 | return 0;
|
| 2132 | }
|
| 2133 |
|
| 2134 | static __cold void __io_req_caches_free(struct io_ring_ctx *ctx)
|
| 2135 | {
|
| 2136 | struct io_kiocb *req;
|
| 2137 | int nr = 0;
|
| 2138 |
|
| 2139 | while (!io_req_cache_empty(ctx)) {
|
| 2140 | req = io_extract_req(ctx);
|
| 2141 | io_poison_req(req);
|
| 2142 | kmem_cache_free(req_cachep, req);
|
| 2143 | nr++;
|
| 2144 | }
|
| 2145 | if (nr) {
|
| 2146 | ctx->nr_req_allocated -= nr;
|
| 2147 | percpu_ref_put_many(&ctx->refs, nr);
|
| 2148 | }
|
| 2149 | }
|
| 2150 |
|
| 2151 | static __cold void io_req_caches_free(struct io_ring_ctx *ctx)
|
| 2152 | {
|
| 2153 | guard(mutex)(&ctx->uring_lock);
|
| 2154 | __io_req_caches_free(ctx);
|
| 2155 | }
|
| 2156 |
|
| 2157 | static __cold void io_ring_ctx_free(struct io_ring_ctx *ctx)
|
| 2158 | {
|
| 2159 | io_unregister_bpf_ops(ctx);
|
| 2160 | io_sq_thread_finish(ctx);
|
| 2161 |
|
| 2162 | mutex_lock(&ctx->uring_lock);
|
| 2163 | io_sqe_buffers_unregister(ctx);
|
| 2164 | io_sqe_files_unregister(ctx);
|
| 2165 | io_unregister_zcrx(ctx);
|
| 2166 | io_cqring_overflow_kill(ctx);
|
| 2167 | io_eventfd_unregister(ctx);
|
| 2168 | io_free_alloc_caches(ctx);
|
| 2169 | io_destroy_buffers(ctx);
|
| 2170 | io_free_region(ctx->user, &ctx->param_region);
|
| 2171 | mutex_unlock(&ctx->uring_lock);
|
| 2172 | if (ctx->sq_creds)
|
| 2173 | put_cred(ctx->sq_creds);
|
| 2174 | if (ctx->submitter_task)
|
| 2175 | put_task_struct(ctx->submitter_task);
|
| 2176 |
|
| 2177 | WARN_ON_ONCE(!list_empty(&ctx->ltimeout_list));
|
| 2178 |
|
| 2179 | if (ctx->mm_account) {
|
| 2180 | mmdrop(ctx->mm_account);
|
| 2181 | ctx->mm_account = NULL;
|
| 2182 | }
|
| 2183 | io_rings_free(ctx);
|
| 2184 |
|
| 2185 | if (!(ctx->flags & IORING_SETUP_NO_SQARRAY))
|
| 2186 | static_branch_slow_dec_deferred(&io_key_has_sqarray);
|
| 2187 |
|
| 2188 | percpu_ref_exit(&ctx->refs);
|
| 2189 | free_uid(ctx->user);
|
| 2190 | io_req_caches_free(ctx);
|
| 2191 |
|
| 2192 | if (ctx->restrictions.bpf_filters) {
|
| 2193 | WARN_ON_ONCE(ctx->bpf_filters !=
|
| 2194 | ctx->restrictions.bpf_filters->filters);
|
| 2195 | } else {
|
| 2196 | WARN_ON_ONCE(ctx->bpf_filters);
|
| 2197 | }
|
| 2198 | io_put_bpf_filters(&ctx->restrictions);
|
| 2199 |
|
| 2200 | WARN_ON_ONCE(ctx->nr_req_allocated);
|
| 2201 |
|
| 2202 | if (ctx->hash_map)
|
| 2203 | io_wq_put_hash(ctx->hash_map);
|
| 2204 | io_napi_free(ctx);
|
| 2205 | kvfree(ctx->cancel_table.hbs);
|
| 2206 | xa_destroy(&ctx->io_bl_xa);
|
| 2207 | xa_destroy(&ctx->hpage_acct);
|
| 2208 | kfree(ctx);
|
| 2209 | }
|
| 2210 |
|
| 2211 | static __cold void io_activate_pollwq_cb(struct callback_head *cb)
|
| 2212 | {
|
| 2213 | struct io_ring_ctx *ctx = container_of(cb, struct io_ring_ctx,
|
| 2214 | poll_wq_task_work);
|
| 2215 |
|
| 2216 | mutex_lock(&ctx->uring_lock);
|
| 2217 | ctx->int_flags |= IO_RING_F_POLL_ACTIVATED;
|
| 2218 | mutex_unlock(&ctx->uring_lock);
|
| 2219 |
|
| 2220 | |
| 2221 | * Wake ups for some events between start of polling and activation
|
| 2222 | * might've been lost due to loose synchronisation.
|
| 2223 |
|
| 2224 | wake_up_all(&ctx->poll_wq);
|
| 2225 | percpu_ref_put(&ctx->refs);
|
| 2226 | }
|
| 2227 |
|
| 2228 | __cold void io_activate_pollwq(struct io_ring_ctx *ctx)
|
| 2229 | {
|
| 2230 | spin_lock(&ctx->completion_lock);
|
| 2231 |
|
| 2232 | if ((ctx->int_flags & IO_RING_F_POLL_ACTIVATED) || ctx->poll_wq_task_work.func)
|
| 2233 | goto out;
|
| 2234 | if (WARN_ON_ONCE(!(ctx->int_flags & IO_RING_F_TASK_COMPLETE)))
|
| 2235 | goto out;
|
| 2236 | if (!ctx->submitter_task)
|
| 2237 | goto out;
|
| 2238 | |
| 2239 | * with ->submitter_task only the submitter task completes requests, we
|
| 2240 | * only need to sync with it, which is done by injecting a tw
|
| 2241 |
|
| 2242 | init_task_work(&ctx->poll_wq_task_work, io_activate_pollwq_cb);
|
| 2243 | percpu_ref_get(&ctx->refs);
|
| 2244 | if (task_work_add(ctx->submitter_task, &ctx->poll_wq_task_work, TWA_SIGNAL))
|
| 2245 | percpu_ref_put(&ctx->refs);
|
| 2246 | out:
|
| 2247 | spin_unlock(&ctx->completion_lock);
|
| 2248 | }
|
| 2249 |
|
| 2250 | static __poll_t io_uring_poll(struct file *file, poll_table *wait)
|
| 2251 | {
|
| 2252 | struct io_ring_ctx *ctx = file->private_data;
|
| 2253 | __poll_t mask = 0;
|
| 2254 |
|
| 2255 | if (unlikely(!(data_race(ctx->int_flags) & IO_RING_F_POLL_ACTIVATED)))
|
| 2256 | io_activate_pollwq(ctx);
|
| 2257 | |
| 2258 | * provides mb() which pairs with barrier from wq_has_sleeper
|
| 2259 | * call in io_commit_cqring
|
| 2260 |
|
| 2261 | poll_wait(file, &ctx->poll_wq, wait);
|
| 2262 |
|
| 2263 | rcu_read_lock();
|
| 2264 |
|
| 2265 | if (!__io_sqring_full(ctx))
|
| 2266 | mask |= EPOLLOUT | EPOLLWRNORM;
|
| 2267 |
|
| 2268 | |
| 2269 | * Don't flush cqring overflow list here, just do a simple check.
|
| 2270 | * Otherwise there could possible be ABBA deadlock:
|
| 2271 | * CPU0 CPU1
|
| 2272 | * ---- ----
|
| 2273 | * lock(&ctx->uring_lock);
|
| 2274 | * lock(&ep->mtx);
|
| 2275 | * lock(&ctx->uring_lock);
|
| 2276 | * lock(&ep->mtx);
|
| 2277 | *
|
| 2278 | * Users may get EPOLLIN meanwhile seeing nothing in cqring, this
|
| 2279 | * pushes them to do the flush.
|
| 2280 |
|
| 2281 |
|
| 2282 | if (__io_cqring_events_user(ctx) || io_has_work(ctx))
|
| 2283 | mask |= EPOLLIN | EPOLLRDNORM;
|
| 2284 |
|
| 2285 | rcu_read_unlock();
|
| 2286 | return mask;
|
| 2287 | }
|
| 2288 |
|
| 2289 | struct io_tctx_exit {
|
| 2290 | struct callback_head task_work;
|
| 2291 | struct completion completion;
|
| 2292 | struct io_ring_ctx *ctx;
|
| 2293 | };
|
| 2294 |
|
| 2295 | static __cold void io_tctx_exit_cb(struct callback_head *cb)
|
| 2296 | {
|
| 2297 | struct io_uring_task *tctx = current->io_uring;
|
| 2298 | struct io_tctx_exit *work;
|
| 2299 |
|
| 2300 | work = container_of(cb, struct io_tctx_exit, task_work);
|
| 2301 | |
| 2302 | * When @in_cancel, we're in cancellation and it's racy to remove the
|
| 2303 | * node. It'll be removed by the end of cancellation, just ignore it.
|
| 2304 | * tctx can be NULL if the queueing of this task_work raced with
|
| 2305 | * work cancelation off the exec path.
|
| 2306 |
|
| 2307 | if (tctx && !atomic_read(&tctx->in_cancel))
|
| 2308 | io_uring_del_tctx_node((unsigned long)work->ctx);
|
| 2309 | complete(&work->completion);
|
| 2310 | }
|
| 2311 |
|
| 2312 | static __cold void io_ring_exit_work(struct work_struct *work)
|
| 2313 | {
|
| 2314 | struct io_ring_ctx *ctx = container_of(work, struct io_ring_ctx, exit_work);
|
| 2315 | unsigned long timeout = jiffies + IO_URING_EXIT_WAIT_MAX;
|
| 2316 | unsigned long interval = HZ / 20;
|
| 2317 | struct io_tctx_exit exit;
|
| 2318 | struct io_tctx_node *node;
|
| 2319 | int ret;
|
| 2320 |
|
| 2321 | mutex_lock(&ctx->uring_lock);
|
| 2322 | io_terminate_zcrx(ctx);
|
| 2323 | mutex_unlock(&ctx->uring_lock);
|
| 2324 |
|
| 2325 | |
| 2326 | * If we're doing polled IO and end up having requests being
|
| 2327 | * submitted async (out-of-line), then completions can come in while
|
| 2328 | * we're waiting for refs to drop. We need to reap these manually,
|
| 2329 | * as nobody else will be looking for them.
|
| 2330 |
|
| 2331 | do {
|
| 2332 | if (test_bit(IO_CHECK_CQ_OVERFLOW_BIT, &ctx->check_cq)) {
|
| 2333 | mutex_lock(&ctx->uring_lock);
|
| 2334 | io_cqring_overflow_kill(ctx);
|
| 2335 | mutex_unlock(&ctx->uring_lock);
|
| 2336 | }
|
| 2337 |
|
| 2338 |
|
| 2339 | do {
|
| 2340 | if (ctx->flags & IORING_SETUP_DEFER_TASKRUN)
|
| 2341 | io_cancel_local_task_work(ctx);
|
| 2342 | cond_resched();
|
| 2343 | } while (io_uring_try_cancel_requests(ctx, NULL, true, false));
|
| 2344 |
|
| 2345 | if (ctx->sq_data) {
|
| 2346 | struct io_sq_data *sqd = ctx->sq_data;
|
| 2347 | struct task_struct *tsk;
|
| 2348 |
|
| 2349 | io_sq_thread_park(sqd);
|
| 2350 | tsk = sqpoll_task_locked(sqd);
|
| 2351 | if (tsk && tsk->io_uring && tsk->io_uring->io_wq)
|
| 2352 | io_wq_cancel_cb(tsk->io_uring->io_wq,
|
| 2353 | io_cancel_ctx_cb, ctx, true);
|
| 2354 | io_sq_thread_unpark(sqd);
|
| 2355 | }
|
| 2356 |
|
| 2357 | io_req_caches_free(ctx);
|
| 2358 |
|
| 2359 | if (WARN_ON_ONCE(time_after(jiffies, timeout))) {
|
| 2360 |
|
| 2361 | interval = HZ * 60;
|
| 2362 | }
|
| 2363 | |
| 2364 | * This is really an uninterruptible wait, as it has to be
|
| 2365 | * complete. But it's also run from a kworker, which doesn't
|
| 2366 | * take signals, so it's fine to make it interruptible. This
|
| 2367 | * avoids scenarios where we knowingly can wait much longer
|
| 2368 | * on completions, for example if someone does a SIGSTOP on
|
| 2369 | * a task that needs to finish task_work to make this loop
|
| 2370 | * complete. That's a synthetic situation that should not
|
| 2371 | * cause a stuck task backtrace, and hence a potential panic
|
| 2372 | * on stuck tasks if that is enabled.
|
| 2373 |
|
| 2374 | } while (!wait_for_completion_interruptible_timeout(&ctx->ref_comp, interval));
|
| 2375 |
|
| 2376 | init_completion(&exit.completion);
|
| 2377 | init_task_work(&exit.task_work, io_tctx_exit_cb);
|
| 2378 | exit.ctx = ctx;
|
| 2379 |
|
| 2380 | mutex_lock(&ctx->uring_lock);
|
| 2381 | mutex_lock(&ctx->tctx_lock);
|
| 2382 | while (!list_empty(&ctx->tctx_list)) {
|
| 2383 | WARN_ON_ONCE(time_after(jiffies, timeout));
|
| 2384 |
|
| 2385 | node = list_first_entry(&ctx->tctx_list, struct io_tctx_node,
|
| 2386 | ctx_node);
|
| 2387 |
|
| 2388 | list_rotate_left(&ctx->tctx_list);
|
| 2389 | ret = task_work_add(node->task, &exit.task_work, TWA_SIGNAL);
|
| 2390 | if (WARN_ON_ONCE(ret))
|
| 2391 | continue;
|
| 2392 |
|
| 2393 | mutex_unlock(&ctx->tctx_lock);
|
| 2394 | mutex_unlock(&ctx->uring_lock);
|
| 2395 | |
| 2396 | * See comment above for
|
| 2397 | * wait_for_completion_interruptible_timeout() on why this
|
| 2398 | * wait is marked as interruptible.
|
| 2399 |
|
| 2400 | wait_for_completion_interruptible(&exit.completion);
|
| 2401 | mutex_lock(&ctx->uring_lock);
|
| 2402 | mutex_lock(&ctx->tctx_lock);
|
| 2403 | }
|
| 2404 | mutex_unlock(&ctx->tctx_lock);
|
| 2405 | mutex_unlock(&ctx->uring_lock);
|
| 2406 | spin_lock(&ctx->completion_lock);
|
| 2407 | spin_unlock(&ctx->completion_lock);
|
| 2408 |
|
| 2409 |
|
| 2410 | if (ctx->flags & (IORING_SETUP_DEFER_TASKRUN | IORING_SETUP_SQPOLL))
|
| 2411 | synchronize_rcu();
|
| 2412 |
|
| 2413 | io_ring_ctx_free(ctx);
|
| 2414 | }
|
| 2415 |
|
| 2416 | static __cold void io_ring_ctx_wait_and_kill(struct io_ring_ctx *ctx)
|
| 2417 | {
|
| 2418 | unsigned long index;
|
| 2419 | struct cred *creds;
|
| 2420 |
|
| 2421 | mutex_lock(&ctx->uring_lock);
|
| 2422 | percpu_ref_kill(&ctx->refs);
|
| 2423 | xa_for_each(&ctx->personalities, index, creds)
|
| 2424 | io_unregister_personality(ctx, index);
|
| 2425 | mutex_unlock(&ctx->uring_lock);
|
| 2426 |
|
| 2427 | INIT_WORK(&ctx->exit_work, io_ring_exit_work);
|
| 2428 | |
| 2429 | * Use system_dfl_wq to avoid spawning tons of event kworkers
|
| 2430 | * if we're exiting a ton of rings at the same time. It just adds
|
| 2431 | * noise and overhead, there's no discernable change in runtime
|
| 2432 | * over using system_percpu_wq.
|
| 2433 |
|
| 2434 | queue_work(iou_wq, &ctx->exit_work);
|
| 2435 | }
|
| 2436 |
|
| 2437 | static int io_uring_release(struct inode *inode, struct file *file)
|
| 2438 | {
|
| 2439 | struct io_ring_ctx *ctx = file->private_data;
|
| 2440 |
|
| 2441 | file->private_data = NULL;
|
| 2442 | io_ring_ctx_wait_and_kill(ctx);
|
| 2443 | return 0;
|
| 2444 | }
|
| 2445 |
|
| 2446 | static struct io_uring_reg_wait *io_get_ext_arg_reg(struct io_ring_ctx *ctx,
|
| 2447 | const struct io_uring_getevents_arg __user *uarg)
|
| 2448 | {
|
| 2449 | unsigned long size = sizeof(struct io_uring_reg_wait);
|
| 2450 | unsigned long offset = (uintptr_t)uarg;
|
| 2451 | unsigned long end;
|
| 2452 |
|
| 2453 | if (unlikely(offset % sizeof(long)))
|
| 2454 | return ERR_PTR(-EFAULT);
|
| 2455 |
|
| 2456 |
|
| 2457 | if (unlikely(check_add_overflow(offset, size, &end) ||
|
| 2458 | end > ctx->cq_wait_size))
|
| 2459 | return ERR_PTR(-EFAULT);
|
| 2460 |
|
| 2461 | offset = array_index_nospec(offset, ctx->cq_wait_size - size);
|
| 2462 | return ctx->cq_wait_arg + offset;
|
| 2463 | }
|
| 2464 |
|
| 2465 | static int io_validate_ext_arg(struct io_ring_ctx *ctx, unsigned flags,
|
| 2466 | const void __user *argp, size_t argsz)
|
| 2467 | {
|
| 2468 | struct io_uring_getevents_arg arg;
|
| 2469 |
|
| 2470 | if (!(flags & IORING_ENTER_EXT_ARG))
|
| 2471 | return 0;
|
| 2472 | if (flags & IORING_ENTER_EXT_ARG_REG)
|
| 2473 | return -EINVAL;
|
| 2474 | if (argsz != sizeof(arg))
|
| 2475 | return -EINVAL;
|
| 2476 | if (copy_from_user(&arg, argp, sizeof(arg)))
|
| 2477 | return -EFAULT;
|
| 2478 | return 0;
|
| 2479 | }
|
| 2480 |
|
| 2481 | static int io_get_ext_arg(struct io_ring_ctx *ctx, unsigned flags,
|
| 2482 | const void __user *argp, struct ext_arg *ext_arg)
|
| 2483 | {
|
| 2484 | const struct io_uring_getevents_arg __user *uarg = argp;
|
| 2485 | struct io_uring_getevents_arg arg;
|
| 2486 |
|
| 2487 | ext_arg->iowait = !(flags & IORING_ENTER_NO_IOWAIT);
|
| 2488 |
|
| 2489 | |
| 2490 | * If EXT_ARG isn't set, then we have no timespec and the argp pointer
|
| 2491 | * is just a pointer to the sigset_t.
|
| 2492 |
|
| 2493 | if (!(flags & IORING_ENTER_EXT_ARG)) {
|
| 2494 | ext_arg->sig = (const sigset_t __user *) argp;
|
| 2495 | return 0;
|
| 2496 | }
|
| 2497 |
|
| 2498 | if (flags & IORING_ENTER_EXT_ARG_REG) {
|
| 2499 | struct io_uring_reg_wait *w;
|
| 2500 |
|
| 2501 | if (ext_arg->argsz != sizeof(struct io_uring_reg_wait))
|
| 2502 | return -EINVAL;
|
| 2503 | w = io_get_ext_arg_reg(ctx, argp);
|
| 2504 | if (IS_ERR(w))
|
| 2505 | return PTR_ERR(w);
|
| 2506 |
|
| 2507 | if (w->flags & ~IORING_REG_WAIT_TS)
|
| 2508 | return -EINVAL;
|
| 2509 | ext_arg->min_time = READ_ONCE(w->min_wait_usec) * NSEC_PER_USEC;
|
| 2510 | ext_arg->sig = u64_to_user_ptr(READ_ONCE(w->sigmask));
|
| 2511 | ext_arg->argsz = READ_ONCE(w->sigmask_sz);
|
| 2512 | if (w->flags & IORING_REG_WAIT_TS) {
|
| 2513 | ext_arg->ts.tv_sec = READ_ONCE(w->ts.tv_sec);
|
| 2514 | ext_arg->ts.tv_nsec = READ_ONCE(w->ts.tv_nsec);
|
| 2515 | ext_arg->ts_set = true;
|
| 2516 | }
|
| 2517 | return 0;
|
| 2518 | }
|
| 2519 |
|
| 2520 | |
| 2521 | * EXT_ARG is set - ensure we agree on the size of it and copy in our
|
| 2522 | * timespec and sigset_t pointers if good.
|
| 2523 |
|
| 2524 | if (ext_arg->argsz != sizeof(arg))
|
| 2525 | return -EINVAL;
|
| 2526 | #ifdef CONFIG_64BIT
|
| 2527 | if (!user_access_begin(uarg, sizeof(*uarg)))
|
| 2528 | return -EFAULT;
|
| 2529 | unsafe_get_user(arg.sigmask, &uarg->sigmask, uaccess_end);
|
| 2530 | unsafe_get_user(arg.sigmask_sz, &uarg->sigmask_sz, uaccess_end);
|
| 2531 | unsafe_get_user(arg.min_wait_usec, &uarg->min_wait_usec, uaccess_end);
|
| 2532 | unsafe_get_user(arg.ts, &uarg->ts, uaccess_end);
|
| 2533 | user_access_end();
|
| 2534 | #else
|
| 2535 | if (copy_from_user(&arg, uarg, sizeof(arg)))
|
| 2536 | return -EFAULT;
|
| 2537 | #endif
|
| 2538 | ext_arg->min_time = arg.min_wait_usec * NSEC_PER_USEC;
|
| 2539 | ext_arg->sig = u64_to_user_ptr(arg.sigmask);
|
| 2540 | ext_arg->argsz = arg.sigmask_sz;
|
| 2541 | if (arg.ts) {
|
| 2542 | if (get_timespec64(&ext_arg->ts, u64_to_user_ptr(arg.ts)))
|
| 2543 | return -EFAULT;
|
| 2544 | ext_arg->ts_set = true;
|
| 2545 | }
|
| 2546 | return 0;
|
| 2547 | #ifdef CONFIG_64BIT
|
| 2548 | uaccess_end:
|
| 2549 | user_access_end();
|
| 2550 | return -EFAULT;
|
| 2551 | #endif
|
| 2552 | }
|
| 2553 |
|
| 2554 | |
| 2555 | * Given an 'fd' value, return the ctx associated with if. If 'registered' is
|
| 2556 | * true, then the registered index is used. Otherwise, the normal fd table.
|
| 2557 | * Caller must call fput() on the returned file if it isn't a registered file,
|
| 2558 | * unless it's an ERR_PTR.
|
| 2559 |
|
| 2560 | struct file *io_uring_ctx_get_file(unsigned int fd, bool registered)
|
| 2561 | {
|
| 2562 | struct file *file;
|
| 2563 |
|
| 2564 | if (registered) {
|
| 2565 | |
| 2566 | * Ring fd has been registered via IORING_REGISTER_RING_FDS, we
|
| 2567 | * need only dereference our task private array to find it.
|
| 2568 |
|
| 2569 | struct io_uring_task *tctx = current->io_uring;
|
| 2570 |
|
| 2571 | if (unlikely(!tctx || fd >= IO_RINGFD_REG_MAX))
|
| 2572 | return ERR_PTR(-EINVAL);
|
| 2573 | fd = array_index_nospec(fd, IO_RINGFD_REG_MAX);
|
| 2574 | file = tctx->registered_rings[fd];
|
| 2575 | } else {
|
| 2576 | file = fget(fd);
|
| 2577 | }
|
| 2578 |
|
| 2579 | if (unlikely(!file))
|
| 2580 | return ERR_PTR(-EBADF);
|
| 2581 | if (io_is_uring_fops(file))
|
| 2582 | return file;
|
| 2583 | if (!registered)
|
| 2584 | fput(file);
|
| 2585 | return ERR_PTR(-EOPNOTSUPP);
|
| 2586 | }
|
| 2587 |
|
| 2588 |
|
| 2589 | SYSCALL_DEFINE6(io_uring_enter, unsigned int, fd, u32, to_submit,
|
| 2590 | u32, min_complete, u32, flags, const void __user *, argp,
|
| 2591 | size_t, argsz)
|
| 2592 | {
|
| 2593 | struct io_ring_ctx *ctx;
|
| 2594 | struct file *file;
|
| 2595 | long ret;
|
| 2596 |
|
| 2597 | if (unlikely(flags & ~IORING_ENTER_FLAGS))
|
| 2598 | return -EINVAL;
|
| 2599 |
|
| 2600 | file = io_uring_ctx_get_file(fd, flags & IORING_ENTER_REGISTERED_RING);
|
| 2601 | if (IS_ERR(file))
|
| 2602 | return PTR_ERR(file);
|
| 2603 | ctx = file->private_data;
|
| 2604 | ret = -EBADFD;
|
| 2605 | |
| 2606 | * Keep IORING_SETUP_R_DISABLED check before submitter_task load
|
| 2607 | * in io_uring_add_tctx_node() -> __io_uring_add_tctx_node_from_submit()
|
| 2608 |
|
| 2609 | if (unlikely(smp_load_acquire(&ctx->flags) & IORING_SETUP_R_DISABLED))
|
| 2610 | goto out;
|
| 2611 |
|
| 2612 | if (io_has_loop_ops(ctx)) {
|
| 2613 | ret = io_run_loop(ctx);
|
| 2614 | goto out;
|
| 2615 | }
|
| 2616 |
|
| 2617 | |
| 2618 | * For SQ polling, the thread will do all submissions and completions.
|
| 2619 | * Just return the requested submit count, and wake the thread if
|
| 2620 | * we were asked to.
|
| 2621 |
|
| 2622 | ret = 0;
|
| 2623 | if (ctx->flags & IORING_SETUP_SQPOLL) {
|
| 2624 | if (unlikely(ctx->sq_data->thread == NULL)) {
|
| 2625 | ret = -EOWNERDEAD;
|
| 2626 | goto out;
|
| 2627 | }
|
| 2628 | if (flags & IORING_ENTER_SQ_WAKEUP)
|
| 2629 | wake_up(&ctx->sq_data->wait);
|
| 2630 | if (flags & IORING_ENTER_SQ_WAIT)
|
| 2631 | io_sqpoll_wait_sq(ctx);
|
| 2632 |
|
| 2633 | ret = to_submit;
|
| 2634 | } else if (to_submit) {
|
| 2635 | ret = io_uring_add_tctx_node(ctx);
|
| 2636 | if (unlikely(ret))
|
| 2637 | goto out;
|
| 2638 |
|
| 2639 | mutex_lock(&ctx->uring_lock);
|
| 2640 | ret = io_submit_sqes(ctx, to_submit);
|
| 2641 | if (ret != to_submit) {
|
| 2642 | mutex_unlock(&ctx->uring_lock);
|
| 2643 | goto out;
|
| 2644 | }
|
| 2645 | if (flags & IORING_ENTER_GETEVENTS) {
|
| 2646 | if (ctx->int_flags & IO_RING_F_SYSCALL_IOPOLL)
|
| 2647 | goto iopoll_locked;
|
| 2648 | |
| 2649 | * Ignore errors, we'll soon call io_cqring_wait() and
|
| 2650 | * it should handle ownership problems if any.
|
| 2651 |
|
| 2652 | if (ctx->flags & IORING_SETUP_DEFER_TASKRUN)
|
| 2653 | (void)io_run_local_work_locked(ctx, min_complete);
|
| 2654 | }
|
| 2655 | mutex_unlock(&ctx->uring_lock);
|
| 2656 | }
|
| 2657 |
|
| 2658 | if (flags & IORING_ENTER_GETEVENTS) {
|
| 2659 | int ret2;
|
| 2660 |
|
| 2661 | if (ctx->int_flags & IO_RING_F_SYSCALL_IOPOLL) {
|
| 2662 | |
| 2663 | * We disallow the app entering submit/complete with
|
| 2664 | * polling, but we still need to lock the ring to
|
| 2665 | * prevent racing with polled issue that got punted to
|
| 2666 | * a workqueue.
|
| 2667 |
|
| 2668 | mutex_lock(&ctx->uring_lock);
|
| 2669 | iopoll_locked:
|
| 2670 | ret2 = io_validate_ext_arg(ctx, flags, argp, argsz);
|
| 2671 | if (likely(!ret2))
|
| 2672 | ret2 = io_iopoll_check(ctx, min_complete);
|
| 2673 | mutex_unlock(&ctx->uring_lock);
|
| 2674 | } else {
|
| 2675 | struct ext_arg ext_arg = { .argsz = argsz };
|
| 2676 |
|
| 2677 | ret2 = io_get_ext_arg(ctx, flags, argp, &ext_arg);
|
| 2678 | if (likely(!ret2))
|
| 2679 | ret2 = io_cqring_wait(ctx, min_complete, flags,
|
| 2680 | &ext_arg);
|
| 2681 | }
|
| 2682 |
|
| 2683 | if (!ret) {
|
| 2684 | ret = ret2;
|
| 2685 |
|
| 2686 | |
| 2687 | * EBADR indicates that one or more CQE were dropped.
|
| 2688 | * Once the user has been informed we can clear the bit
|
| 2689 | * as they are obviously ok with those drops.
|
| 2690 |
|
| 2691 | if (unlikely(ret2 == -EBADR))
|
| 2692 | clear_bit(IO_CHECK_CQ_DROPPED_BIT,
|
| 2693 | &ctx->check_cq);
|
| 2694 | }
|
| 2695 | }
|
| 2696 | out:
|
| 2697 | if (!(flags & IORING_ENTER_REGISTERED_RING))
|
| 2698 | fput(file);
|
| 2699 | return ret;
|
| 2700 | }
|
| 2701 |
|
| 2702 | static const struct file_operations io_uring_fops = {
|
| 2703 | .release = io_uring_release,
|
| 2704 | .mmap = io_uring_mmap,
|
| 2705 | .get_unmapped_area = io_uring_get_unmapped_area,
|
| 2706 | #ifndef CONFIG_MMU
|
| 2707 | .mmap_capabilities = io_uring_nommu_mmap_capabilities,
|
| 2708 | #endif
|
| 2709 | .poll = io_uring_poll,
|
| 2710 | #ifdef CONFIG_PROC_FS
|
| 2711 | .show_fdinfo = io_uring_show_fdinfo,
|
| 2712 | #endif
|
| 2713 | };
|
| 2714 |
|
| 2715 | bool io_is_uring_fops(struct file *file)
|
| 2716 | {
|
| 2717 | return file->f_op == &io_uring_fops;
|
| 2718 | }
|
| 2719 |
|
| 2720 | static __cold int io_allocate_scq_urings(struct io_ring_ctx *ctx,
|
| 2721 | struct io_ctx_config *config)
|
| 2722 | {
|
| 2723 | struct io_uring_params *p = &config->p;
|
| 2724 | struct io_rings_layout *rl = &config->layout;
|
| 2725 | struct io_uring_region_desc rd;
|
| 2726 | struct io_rings *rings;
|
| 2727 | int ret;
|
| 2728 |
|
| 2729 |
|
| 2730 | ctx->sq_entries = p->sq_entries;
|
| 2731 | ctx->cq_entries = p->cq_entries;
|
| 2732 |
|
| 2733 | memset(&rd, 0, sizeof(rd));
|
| 2734 | rd.size = PAGE_ALIGN(rl->rings_size);
|
| 2735 | if (ctx->flags & IORING_SETUP_NO_MMAP) {
|
| 2736 | rd.user_addr = p->cq_off.user_addr;
|
| 2737 | rd.flags |= IORING_MEM_REGION_TYPE_USER;
|
| 2738 | }
|
| 2739 |
|
| 2740 | ret = io_create_region(NULL, &ctx->ring_region, &rd, IORING_OFF_CQ_RING);
|
| 2741 | if (ret)
|
| 2742 | return ret;
|
| 2743 | ctx->rings = rings = io_region_get_ptr(&ctx->ring_region);
|
| 2744 | rcu_assign_pointer(ctx->rings_rcu, rings);
|
| 2745 | if (!(ctx->flags & IORING_SETUP_NO_SQARRAY))
|
| 2746 | ctx->sq_array = (u32 *)((char *)rings + rl->sq_array_offset);
|
| 2747 |
|
| 2748 | memset(&rd, 0, sizeof(rd));
|
| 2749 | rd.size = PAGE_ALIGN(rl->sq_size);
|
| 2750 | if (ctx->flags & IORING_SETUP_NO_MMAP) {
|
| 2751 | rd.user_addr = p->sq_off.user_addr;
|
| 2752 | rd.flags |= IORING_MEM_REGION_TYPE_USER;
|
| 2753 | }
|
| 2754 | ret = io_create_region(NULL, &ctx->sq_region, &rd, IORING_OFF_SQES);
|
| 2755 | if (ret) {
|
| 2756 | io_rings_free(ctx);
|
| 2757 | return ret;
|
| 2758 | }
|
| 2759 | ctx->sq_sqes = io_region_get_ptr(&ctx->sq_region);
|
| 2760 |
|
| 2761 | memset(rings, 0, sizeof(*rings));
|
| 2762 | WRITE_ONCE(rings->sq_ring_mask, ctx->sq_entries - 1);
|
| 2763 | WRITE_ONCE(rings->cq_ring_mask, ctx->cq_entries - 1);
|
| 2764 | WRITE_ONCE(rings->sq_ring_entries, ctx->sq_entries);
|
| 2765 | WRITE_ONCE(rings->cq_ring_entries, ctx->cq_entries);
|
| 2766 | return 0;
|
| 2767 | }
|
| 2768 |
|
| 2769 | static int io_uring_install_fd(struct file *file)
|
| 2770 | {
|
| 2771 | int fd;
|
| 2772 |
|
| 2773 | fd = get_unused_fd_flags(O_RDWR | O_CLOEXEC);
|
| 2774 | if (fd < 0)
|
| 2775 | return fd;
|
| 2776 | fd_install(fd, file);
|
| 2777 | return fd;
|
| 2778 | }
|
| 2779 |
|
| 2780 | |
| 2781 | * Allocate an anonymous fd, this is what constitutes the application
|
| 2782 | * visible backing of an io_uring instance. The application mmaps this
|
| 2783 | * fd to gain access to the SQ/CQ ring details.
|
| 2784 |
|
| 2785 | static struct file *io_uring_get_file(struct io_ring_ctx *ctx)
|
| 2786 | {
|
| 2787 |
|
| 2788 | return anon_inode_create_getfile("[io_uring]", &io_uring_fops, ctx,
|
| 2789 | O_RDWR | O_CLOEXEC, NULL);
|
| 2790 | }
|
| 2791 |
|
| 2792 | static int io_uring_sanitise_params(struct io_uring_params *p)
|
| 2793 | {
|
| 2794 | unsigned flags = p->flags;
|
| 2795 |
|
| 2796 | if (flags & ~IORING_SETUP_FLAGS)
|
| 2797 | return -EINVAL;
|
| 2798 |
|
| 2799 | if (flags & IORING_SETUP_SQ_REWIND) {
|
| 2800 | if ((flags & IORING_SETUP_SQPOLL) ||
|
| 2801 | !(flags & IORING_SETUP_NO_SQARRAY))
|
| 2802 | return -EINVAL;
|
| 2803 | }
|
| 2804 |
|
| 2805 |
|
| 2806 | if ((flags & IORING_SETUP_REGISTERED_FD_ONLY) &&
|
| 2807 | !(flags & IORING_SETUP_NO_MMAP))
|
| 2808 | return -EINVAL;
|
| 2809 |
|
| 2810 | if (flags & IORING_SETUP_SQPOLL) {
|
| 2811 |
|
| 2812 | if (flags & (IORING_SETUP_COOP_TASKRUN |
|
| 2813 | IORING_SETUP_TASKRUN_FLAG |
|
| 2814 | IORING_SETUP_DEFER_TASKRUN))
|
| 2815 | return -EINVAL;
|
| 2816 | }
|
| 2817 |
|
| 2818 | if (flags & IORING_SETUP_TASKRUN_FLAG) {
|
| 2819 | if (!(flags & (IORING_SETUP_COOP_TASKRUN |
|
| 2820 | IORING_SETUP_DEFER_TASKRUN)))
|
| 2821 | return -EINVAL;
|
| 2822 | }
|
| 2823 |
|
| 2824 |
|
| 2825 | if ((flags & IORING_SETUP_HYBRID_IOPOLL) && !(flags & IORING_SETUP_IOPOLL))
|
| 2826 | return -EINVAL;
|
| 2827 |
|
| 2828 | |
| 2829 | * For DEFER_TASKRUN we require the completion task to be the same as
|
| 2830 | * the submission task. This implies that there is only one submitter.
|
| 2831 |
|
| 2832 | if ((flags & IORING_SETUP_DEFER_TASKRUN) &&
|
| 2833 | !(flags & IORING_SETUP_SINGLE_ISSUER))
|
| 2834 | return -EINVAL;
|
| 2835 |
|
| 2836 | |
| 2837 | * Nonsensical to ask for CQE32 and mixed CQE support, it's not
|
| 2838 | * supported to post 16b CQEs on a ring setup with CQE32.
|
| 2839 |
|
| 2840 | if ((flags & (IORING_SETUP_CQE32|IORING_SETUP_CQE_MIXED)) ==
|
| 2841 | (IORING_SETUP_CQE32|IORING_SETUP_CQE_MIXED))
|
| 2842 | return -EINVAL;
|
| 2843 | |
| 2844 | * Nonsensical to ask for SQE128 and mixed SQE support, it's not
|
| 2845 | * supported to post 64b SQEs on a ring setup with SQE128.
|
| 2846 |
|
| 2847 | if ((flags & (IORING_SETUP_SQE128|IORING_SETUP_SQE_MIXED)) ==
|
| 2848 | (IORING_SETUP_SQE128|IORING_SETUP_SQE_MIXED))
|
| 2849 | return -EINVAL;
|
| 2850 |
|
| 2851 | return 0;
|
| 2852 | }
|
| 2853 |
|
| 2854 | static int io_uring_fill_params(struct io_uring_params *p)
|
| 2855 | {
|
| 2856 | unsigned entries = p->sq_entries;
|
| 2857 |
|
| 2858 | if (!entries)
|
| 2859 | return -EINVAL;
|
| 2860 | if (entries > IORING_MAX_ENTRIES) {
|
| 2861 | if (!(p->flags & IORING_SETUP_CLAMP))
|
| 2862 | return -EINVAL;
|
| 2863 | entries = IORING_MAX_ENTRIES;
|
| 2864 | }
|
| 2865 |
|
| 2866 | |
| 2867 | * Use twice as many entries for the CQ ring. It's possible for the
|
| 2868 | * application to drive a higher depth than the size of the SQ ring,
|
| 2869 | * since the sqes are only used at submission time. This allows for
|
| 2870 | * some flexibility in overcommitting a bit. If the application has
|
| 2871 | * set IORING_SETUP_CQSIZE, it will have passed in the desired number
|
| 2872 | * of CQ ring entries manually.
|
| 2873 |
|
| 2874 | p->sq_entries = roundup_pow_of_two(entries);
|
| 2875 | if (p->flags & IORING_SETUP_CQSIZE) {
|
| 2876 | |
| 2877 | * If IORING_SETUP_CQSIZE is set, we do the same roundup
|
| 2878 | * to a power-of-two, if it isn't already. We do NOT impose
|
| 2879 | * any cq vs sq ring sizing.
|
| 2880 |
|
| 2881 | if (!p->cq_entries)
|
| 2882 | return -EINVAL;
|
| 2883 | if (p->cq_entries > IORING_MAX_CQ_ENTRIES) {
|
| 2884 | if (!(p->flags & IORING_SETUP_CLAMP))
|
| 2885 | return -EINVAL;
|
| 2886 | p->cq_entries = IORING_MAX_CQ_ENTRIES;
|
| 2887 | }
|
| 2888 | p->cq_entries = roundup_pow_of_two(p->cq_entries);
|
| 2889 | if (p->cq_entries < p->sq_entries)
|
| 2890 | return -EINVAL;
|
| 2891 | } else {
|
| 2892 | p->cq_entries = 2 * p->sq_entries;
|
| 2893 | }
|
| 2894 |
|
| 2895 | return 0;
|
| 2896 | }
|
| 2897 |
|
| 2898 | int io_prepare_config(struct io_ctx_config *config)
|
| 2899 | {
|
| 2900 | struct io_uring_params *p = &config->p;
|
| 2901 | int ret;
|
| 2902 |
|
| 2903 | ret = io_uring_sanitise_params(p);
|
| 2904 | if (ret)
|
| 2905 | return ret;
|
| 2906 |
|
| 2907 | ret = io_uring_fill_params(p);
|
| 2908 | if (ret)
|
| 2909 | return ret;
|
| 2910 |
|
| 2911 | ret = rings_size(p->flags, p->sq_entries, p->cq_entries,
|
| 2912 | &config->layout);
|
| 2913 | if (ret)
|
| 2914 | return ret;
|
| 2915 |
|
| 2916 | p->sq_off.head = offsetof(struct io_rings, sq.head);
|
| 2917 | p->sq_off.tail = offsetof(struct io_rings, sq.tail);
|
| 2918 | p->sq_off.ring_mask = offsetof(struct io_rings, sq_ring_mask);
|
| 2919 | p->sq_off.ring_entries = offsetof(struct io_rings, sq_ring_entries);
|
| 2920 | p->sq_off.flags = offsetof(struct io_rings, sq_flags);
|
| 2921 | p->sq_off.dropped = offsetof(struct io_rings, sq_dropped);
|
| 2922 | p->sq_off.resv1 = 0;
|
| 2923 | if (!(p->flags & IORING_SETUP_NO_MMAP))
|
| 2924 | p->sq_off.user_addr = 0;
|
| 2925 |
|
| 2926 | p->cq_off.head = offsetof(struct io_rings, cq.head);
|
| 2927 | p->cq_off.tail = offsetof(struct io_rings, cq.tail);
|
| 2928 | p->cq_off.ring_mask = offsetof(struct io_rings, cq_ring_mask);
|
| 2929 | p->cq_off.ring_entries = offsetof(struct io_rings, cq_ring_entries);
|
| 2930 | p->cq_off.overflow = offsetof(struct io_rings, cq_overflow);
|
| 2931 | p->cq_off.cqes = offsetof(struct io_rings, cqes);
|
| 2932 | p->cq_off.flags = offsetof(struct io_rings, cq_flags);
|
| 2933 | p->cq_off.resv1 = 0;
|
| 2934 | if (!(p->flags & IORING_SETUP_NO_MMAP))
|
| 2935 | p->cq_off.user_addr = 0;
|
| 2936 | if (!(p->flags & IORING_SETUP_NO_SQARRAY))
|
| 2937 | p->sq_off.array = config->layout.sq_array_offset;
|
| 2938 |
|
| 2939 | return 0;
|
| 2940 | }
|
| 2941 |
|
| 2942 | void io_restriction_clone(struct io_restriction *dst, struct io_restriction *src)
|
| 2943 | {
|
| 2944 | memcpy(&dst->register_op, &src->register_op, sizeof(dst->register_op));
|
| 2945 | memcpy(&dst->sqe_op, &src->sqe_op, sizeof(dst->sqe_op));
|
| 2946 | dst->sqe_flags_allowed = src->sqe_flags_allowed;
|
| 2947 | dst->sqe_flags_required = src->sqe_flags_required;
|
| 2948 | dst->op_registered = src->op_registered;
|
| 2949 | dst->reg_registered = src->reg_registered;
|
| 2950 |
|
| 2951 | io_bpf_filter_clone(dst, src);
|
| 2952 | }
|
| 2953 |
|
| 2954 | static void io_ctx_restriction_clone(struct io_ring_ctx *ctx,
|
| 2955 | struct io_restriction *src)
|
| 2956 | {
|
| 2957 | struct io_restriction *dst = &ctx->restrictions;
|
| 2958 |
|
| 2959 | io_restriction_clone(dst, src);
|
| 2960 | if (dst->bpf_filters)
|
| 2961 | WRITE_ONCE(ctx->bpf_filters, dst->bpf_filters->filters);
|
| 2962 | if (dst->op_registered)
|
| 2963 | ctx->int_flags |= IO_RING_F_OP_RESTRICTED;
|
| 2964 | if (dst->reg_registered)
|
| 2965 | ctx->int_flags |= IO_RING_F_REG_RESTRICTED |
|
| 2966 | IO_RING_F_RESTRICT_INHERITED;
|
| 2967 | }
|
| 2968 |
|
| 2969 | static __cold int io_uring_create(struct io_ctx_config *config)
|
| 2970 | {
|
| 2971 | struct io_uring_params *p = &config->p;
|
| 2972 | struct io_ring_ctx *ctx;
|
| 2973 | struct io_uring_task *tctx;
|
| 2974 | struct file *file;
|
| 2975 | int ret;
|
| 2976 |
|
| 2977 | ret = io_prepare_config(config);
|
| 2978 | if (ret)
|
| 2979 | return ret;
|
| 2980 |
|
| 2981 | ctx = io_ring_ctx_alloc(p);
|
| 2982 | if (!ctx)
|
| 2983 | return -ENOMEM;
|
| 2984 |
|
| 2985 | ctx->clockid = CLOCK_MONOTONIC;
|
| 2986 | ctx->clock_offset = 0;
|
| 2987 |
|
| 2988 | if (!(ctx->flags & IORING_SETUP_NO_SQARRAY))
|
| 2989 | static_branch_deferred_inc(&io_key_has_sqarray);
|
| 2990 |
|
| 2991 | if ((ctx->flags & IORING_SETUP_DEFER_TASKRUN) &&
|
| 2992 | !(ctx->flags & IORING_SETUP_IOPOLL))
|
| 2993 | ctx->int_flags |= IO_RING_F_TASK_COMPLETE;
|
| 2994 |
|
| 2995 | if ((ctx->int_flags & IO_RING_F_TASK_COMPLETE) ||
|
| 2996 | (ctx->flags & IORING_SETUP_IOPOLL))
|
| 2997 | ctx->int_flags |= IO_RING_F_LOCKLESS_CQ;
|
| 2998 |
|
| 2999 | |
| 3000 | * lazy poll_wq activation relies on ->task_complete for synchronisation
|
| 3001 | * purposes, see io_activate_pollwq()
|
| 3002 |
|
| 3003 | if (!(ctx->int_flags & IO_RING_F_TASK_COMPLETE))
|
| 3004 | ctx->int_flags |= IO_RING_F_POLL_ACTIVATED;
|
| 3005 |
|
| 3006 | |
| 3007 | * When SETUP_IOPOLL and SETUP_SQPOLL are both enabled, user
|
| 3008 | * space applications don't need to do io completion events
|
| 3009 | * polling again, they can rely on io_sq_thread to do polling
|
| 3010 | * work, which can reduce cpu usage and uring_lock contention.
|
| 3011 |
|
| 3012 | if (ctx->flags & IORING_SETUP_IOPOLL &&
|
| 3013 | !(ctx->flags & IORING_SETUP_SQPOLL))
|
| 3014 | ctx->int_flags |= IO_RING_F_SYSCALL_IOPOLL;
|
| 3015 |
|
| 3016 | if (in_compat_syscall())
|
| 3017 | ctx->int_flags |= IO_RING_F_COMPAT;
|
| 3018 | if (!ns_capable_noaudit(&init_user_ns, CAP_IPC_LOCK))
|
| 3019 | ctx->user = get_uid(current_user());
|
| 3020 |
|
| 3021 | |
| 3022 | * For SQPOLL, we just need a wakeup, always. For !SQPOLL, if
|
| 3023 | * COOP_TASKRUN is set, then IPIs are never needed by the app.
|
| 3024 |
|
| 3025 | if (ctx->flags & (IORING_SETUP_SQPOLL|IORING_SETUP_COOP_TASKRUN))
|
| 3026 | ctx->notify_method = TWA_SIGNAL_NO_IPI;
|
| 3027 | else
|
| 3028 | ctx->notify_method = TWA_SIGNAL;
|
| 3029 |
|
| 3030 | |
| 3031 | * If the current task has restrictions enabled, then copy them to
|
| 3032 | * our newly created ring and mark it as registered.
|
| 3033 |
|
| 3034 | if (current->io_uring_restrict)
|
| 3035 | io_ctx_restriction_clone(ctx, current->io_uring_restrict);
|
| 3036 |
|
| 3037 | |
| 3038 | * This is just grabbed for accounting purposes. When a process exits,
|
| 3039 | * the mm is exited and dropped before the files, hence we need to hang
|
| 3040 | * on to this mm purely for the purposes of being able to unaccount
|
| 3041 | * memory (locked/pinned vm). It's not used for anything else.
|
| 3042 |
|
| 3043 | mmgrab(current->mm);
|
| 3044 | ctx->mm_account = current->mm;
|
| 3045 |
|
| 3046 | ret = io_allocate_scq_urings(ctx, config);
|
| 3047 | if (ret)
|
| 3048 | goto err;
|
| 3049 |
|
| 3050 | ret = io_sq_offload_create(ctx, p);
|
| 3051 | if (ret)
|
| 3052 | goto err;
|
| 3053 |
|
| 3054 | p->features = IORING_FEAT_FLAGS;
|
| 3055 |
|
| 3056 | if (copy_to_user(config->uptr, p, sizeof(*p))) {
|
| 3057 | ret = -EFAULT;
|
| 3058 | goto err;
|
| 3059 | }
|
| 3060 |
|
| 3061 | if (ctx->flags & IORING_SETUP_SINGLE_ISSUER
|
| 3062 | && !(ctx->flags & IORING_SETUP_R_DISABLED))
|
| 3063 | ctx->submitter_task = get_task_struct(current);
|
| 3064 |
|
| 3065 | file = io_uring_get_file(ctx);
|
| 3066 | if (IS_ERR(file)) {
|
| 3067 | ret = PTR_ERR(file);
|
| 3068 | goto err;
|
| 3069 | }
|
| 3070 |
|
| 3071 | ret = __io_uring_add_tctx_node(ctx);
|
| 3072 | if (ret)
|
| 3073 | goto err_fput;
|
| 3074 | tctx = current->io_uring;
|
| 3075 |
|
| 3076 | |
| 3077 | * Install ring fd as the very last thing, so we don't risk someone
|
| 3078 | * having closed it before we finish setup
|
| 3079 |
|
| 3080 | if (p->flags & IORING_SETUP_REGISTERED_FD_ONLY)
|
| 3081 | ret = io_ring_add_registered_file(tctx, file, 0, IO_RINGFD_REG_MAX);
|
| 3082 | else
|
| 3083 | ret = io_uring_install_fd(file);
|
| 3084 | if (ret < 0)
|
| 3085 | goto err_fput;
|
| 3086 |
|
| 3087 | trace_io_uring_create(ret, ctx, p->sq_entries, p->cq_entries, p->flags);
|
| 3088 | return ret;
|
| 3089 | err:
|
| 3090 | io_ring_ctx_wait_and_kill(ctx);
|
| 3091 | return ret;
|
| 3092 | err_fput:
|
| 3093 | fput(file);
|
| 3094 | return ret;
|
| 3095 | }
|
| 3096 |
|
| 3097 | |
| 3098 | * Sets up an aio uring context, and returns the fd. Applications asks for a
|
| 3099 | * ring size, we return the actual sq/cq ring sizes (among other things) in the
|
| 3100 | * params structure passed in.
|
| 3101 |
|
| 3102 | static long io_uring_setup(u32 entries, struct io_uring_params __user *params)
|
| 3103 | {
|
| 3104 | struct io_ctx_config config;
|
| 3105 |
|
| 3106 | memset(&config, 0, sizeof(config));
|
| 3107 |
|
| 3108 | if (copy_from_user(&config.p, params, sizeof(config.p)))
|
| 3109 | return -EFAULT;
|
| 3110 |
|
| 3111 | if (!mem_is_zero(&config.p.resv, sizeof(config.p.resv)))
|
| 3112 | return -EINVAL;
|
| 3113 |
|
| 3114 | config.p.sq_entries = entries;
|
| 3115 | config.uptr = params;
|
| 3116 | return io_uring_create(&config);
|
| 3117 | }
|
| 3118 |
|
| 3119 | static inline int io_uring_allowed(void)
|
| 3120 | {
|
| 3121 | int disabled = READ_ONCE(sysctl_io_uring_disabled);
|
| 3122 | kgid_t io_uring_group;
|
| 3123 |
|
| 3124 | if (disabled == 2)
|
| 3125 | return -EPERM;
|
| 3126 |
|
| 3127 | if (disabled == 0 || capable(CAP_SYS_ADMIN))
|
| 3128 | goto allowed_lsm;
|
| 3129 |
|
| 3130 | io_uring_group = make_kgid(&init_user_ns, sysctl_io_uring_group);
|
| 3131 | if (!gid_valid(io_uring_group))
|
| 3132 | return -EPERM;
|
| 3133 |
|
| 3134 | if (!in_group_p(io_uring_group))
|
| 3135 | return -EPERM;
|
| 3136 |
|
| 3137 | allowed_lsm:
|
| 3138 | return security_uring_allowed();
|
| 3139 | }
|
| 3140 |
|
| 3141 | SYSCALL_DEFINE2(io_uring_setup, u32, entries,
|
| 3142 | struct io_uring_params __user *, params)
|
| 3143 | {
|
| 3144 | int ret;
|
| 3145 |
|
| 3146 | ret = io_uring_allowed();
|
| 3147 | if (ret)
|
| 3148 | return ret;
|
| 3149 |
|
| 3150 | return io_uring_setup(entries, params);
|
| 3151 | }
|
| 3152 |
|
| 3153 | static int __init io_uring_init(void)
|
| 3154 | {
|
| 3155 | struct kmem_cache_args kmem_args = {
|
| 3156 | .useroffset = offsetof(struct io_kiocb, cmd.data),
|
| 3157 | .usersize = sizeof_field(struct io_kiocb, cmd.data),
|
| 3158 | .freeptr_offset = offsetof(struct io_kiocb, work),
|
| 3159 | .use_freeptr_offset = true,
|
| 3160 | };
|
| 3161 |
|
| 3162 | #define __BUILD_BUG_VERIFY_OFFSET_SIZE(stype, eoffset, esize, ename) do { \
|
| 3163 | BUILD_BUG_ON(offsetof(stype, ename) != eoffset); \
|
| 3164 | BUILD_BUG_ON(sizeof_field(stype, ename) != esize); \
|
| 3165 | } while (0)
|
| 3166 |
|
| 3167 | #define BUILD_BUG_SQE_ELEM(eoffset, etype, ename) \
|
| 3168 | __BUILD_BUG_VERIFY_OFFSET_SIZE(struct io_uring_sqe, eoffset, sizeof(etype), ename)
|
| 3169 | #define BUILD_BUG_SQE_ELEM_SIZE(eoffset, esize, ename) \
|
| 3170 | __BUILD_BUG_VERIFY_OFFSET_SIZE(struct io_uring_sqe, eoffset, esize, ename)
|
| 3171 | BUILD_BUG_ON(sizeof(struct io_uring_sqe) != 64);
|
| 3172 | BUILD_BUG_SQE_ELEM(0, __u8, opcode);
|
| 3173 | BUILD_BUG_SQE_ELEM(1, __u8, flags);
|
| 3174 | BUILD_BUG_SQE_ELEM(2, __u16, ioprio);
|
| 3175 | BUILD_BUG_SQE_ELEM(4, __s32, fd);
|
| 3176 | BUILD_BUG_SQE_ELEM(8, __u64, off);
|
| 3177 | BUILD_BUG_SQE_ELEM(8, __u64, addr2);
|
| 3178 | BUILD_BUG_SQE_ELEM(8, __u32, cmd_op);
|
| 3179 | BUILD_BUG_SQE_ELEM(12, __u32, __pad1);
|
| 3180 | BUILD_BUG_SQE_ELEM(16, __u64, addr);
|
| 3181 | BUILD_BUG_SQE_ELEM(16, __u64, splice_off_in);
|
| 3182 | BUILD_BUG_SQE_ELEM(24, __u32, len);
|
| 3183 | BUILD_BUG_SQE_ELEM(28, __kernel_rwf_t, rw_flags);
|
| 3184 | BUILD_BUG_SQE_ELEM(28, int, rw_flags);
|
| 3185 | BUILD_BUG_SQE_ELEM(28, __u32, rw_flags);
|
| 3186 | BUILD_BUG_SQE_ELEM(28, __u32, fsync_flags);
|
| 3187 | BUILD_BUG_SQE_ELEM(28, __u16, poll_events);
|
| 3188 | BUILD_BUG_SQE_ELEM(28, __u32, poll32_events);
|
| 3189 | BUILD_BUG_SQE_ELEM(28, __u32, sync_range_flags);
|
| 3190 | BUILD_BUG_SQE_ELEM(28, __u32, msg_flags);
|
| 3191 | BUILD_BUG_SQE_ELEM(28, __u32, timeout_flags);
|
| 3192 | BUILD_BUG_SQE_ELEM(28, __u32, accept_flags);
|
| 3193 | BUILD_BUG_SQE_ELEM(28, __u32, cancel_flags);
|
| 3194 | BUILD_BUG_SQE_ELEM(28, __u32, open_flags);
|
| 3195 | BUILD_BUG_SQE_ELEM(28, __u32, statx_flags);
|
| 3196 | BUILD_BUG_SQE_ELEM(28, __u32, fadvise_advice);
|
| 3197 | BUILD_BUG_SQE_ELEM(28, __u32, splice_flags);
|
| 3198 | BUILD_BUG_SQE_ELEM(28, __u32, rename_flags);
|
| 3199 | BUILD_BUG_SQE_ELEM(28, __u32, unlink_flags);
|
| 3200 | BUILD_BUG_SQE_ELEM(28, __u32, hardlink_flags);
|
| 3201 | BUILD_BUG_SQE_ELEM(28, __u32, xattr_flags);
|
| 3202 | BUILD_BUG_SQE_ELEM(28, __u32, msg_ring_flags);
|
| 3203 | BUILD_BUG_SQE_ELEM(32, __u64, user_data);
|
| 3204 | BUILD_BUG_SQE_ELEM(40, __u16, buf_index);
|
| 3205 | BUILD_BUG_SQE_ELEM(40, __u16, buf_group);
|
| 3206 | BUILD_BUG_SQE_ELEM(42, __u16, personality);
|
| 3207 | BUILD_BUG_SQE_ELEM(44, __s32, splice_fd_in);
|
| 3208 | BUILD_BUG_SQE_ELEM(44, __u32, file_index);
|
| 3209 | BUILD_BUG_SQE_ELEM(44, __u16, addr_len);
|
| 3210 | BUILD_BUG_SQE_ELEM(44, __u8, write_stream);
|
| 3211 | BUILD_BUG_SQE_ELEM(45, __u8, __pad4[0]);
|
| 3212 | BUILD_BUG_SQE_ELEM(46, __u16, __pad3[0]);
|
| 3213 | BUILD_BUG_SQE_ELEM(48, __u64, addr3);
|
| 3214 | BUILD_BUG_SQE_ELEM_SIZE(48, 0, cmd);
|
| 3215 | BUILD_BUG_SQE_ELEM(48, __u64, attr_ptr);
|
| 3216 | BUILD_BUG_SQE_ELEM(56, __u64, attr_type_mask);
|
| 3217 | BUILD_BUG_SQE_ELEM(56, __u64, __pad2);
|
| 3218 |
|
| 3219 | BUILD_BUG_ON(sizeof(struct io_uring_files_update) !=
|
| 3220 | sizeof(struct io_uring_rsrc_update));
|
| 3221 | BUILD_BUG_ON(sizeof(struct io_uring_rsrc_update) >
|
| 3222 | sizeof(struct io_uring_rsrc_update2));
|
| 3223 |
|
| 3224 |
|
| 3225 | BUILD_BUG_ON(offsetof(struct io_uring_buf_ring, bufs) != 0);
|
| 3226 | BUILD_BUG_ON(offsetof(struct io_uring_buf, resv) !=
|
| 3227 | offsetof(struct io_uring_buf_ring, tail));
|
| 3228 |
|
| 3229 |
|
| 3230 | BUILD_BUG_ON(SQE_VALID_FLAGS >= (1 << 8));
|
| 3231 | BUILD_BUG_ON(SQE_COMMON_FLAGS >= (1 << 8));
|
| 3232 | BUILD_BUG_ON((SQE_VALID_FLAGS | SQE_COMMON_FLAGS) != SQE_VALID_FLAGS);
|
| 3233 |
|
| 3234 | BUILD_BUG_ON(__REQ_F_LAST_BIT > 8 * sizeof_field(struct io_kiocb, flags));
|
| 3235 |
|
| 3236 | BUILD_BUG_ON(sizeof(atomic_t) != sizeof(u32));
|
| 3237 |
|
| 3238 |
|
| 3239 | BUILD_BUG_ON((IORING_URING_CMD_MASK & 0xff000000) != 0);
|
| 3240 |
|
| 3241 | io_uring_optable_init();
|
| 3242 |
|
| 3243 |
|
| 3244 | BUILD_BUG_ON((IO_BUF_DEST | IO_BUF_SOURCE) > U8_MAX);
|
| 3245 |
|
| 3246 | |
| 3247 | * Allow user copy in the per-command field, which starts after the
|
| 3248 | * file in io_kiocb and until the opcode field. The openat2 handling
|
| 3249 | * requires copying in user memory into the io_kiocb object in that
|
| 3250 | * range, and HARDENED_USERCOPY will complain if we haven't
|
| 3251 | * correctly annotated this range.
|
| 3252 |
|
| 3253 | req_cachep = kmem_cache_create("io_kiocb", sizeof(struct io_kiocb), &kmem_args,
|
| 3254 | SLAB_HWCACHE_ALIGN | SLAB_PANIC | SLAB_ACCOUNT |
|
| 3255 | SLAB_TYPESAFE_BY_RCU);
|
| 3256 |
|
| 3257 | iou_wq = alloc_workqueue("iou_exit", WQ_UNBOUND, 64);
|
| 3258 | BUG_ON(!iou_wq);
|
| 3259 |
|
| 3260 | #ifdef CONFIG_SYSCTL
|
| 3261 | register_sysctl_init("kernel", kernel_io_uring_disabled_table);
|
| 3262 | #endif
|
| 3263 |
|
| 3264 | return 0;
|
| 3265 | };
|
| 3266 | __initcall(io_uring_init);
|