| 1 | |
| 2 | #include <linux/kernel.h>
|
| 3 | #include <linux/errno.h>
|
| 4 | #include <linux/fs.h>
|
| 5 | #include <linux/file.h>
|
| 6 | #include <linux/mm.h>
|
| 7 | #include <linux/slab.h>
|
| 8 | #include <linux/nospec.h>
|
| 9 | #include <linux/hugetlb.h>
|
| 10 | #include <linux/compat.h>
|
| 11 | #include <linux/io_uring.h>
|
| 12 | #include <linux/io_uring/cmd.h>
|
| 13 |
|
| 14 | #include <uapi/linux/io_uring.h>
|
| 15 |
|
| 16 | #include "filetable.h"
|
| 17 | #include "io_uring.h"
|
| 18 | #include "openclose.h"
|
| 19 | #include "rsrc.h"
|
| 20 | #include "memmap.h"
|
| 21 | #include "register.h"
|
| 22 |
|
| 23 | struct io_rsrc_update {
|
| 24 | struct file *file;
|
| 25 | u64 arg;
|
| 26 | u32 nr_args;
|
| 27 | u32 offset;
|
| 28 | };
|
| 29 |
|
| 30 | static struct io_rsrc_node *io_sqe_buffer_register(struct io_ring_ctx *ctx,
|
| 31 | struct iovec *iov);
|
| 32 |
|
| 33 | static int hpage_acct_ref(struct io_ring_ctx *ctx, struct page *hpage,
|
| 34 | bool *acct_new)
|
| 35 | {
|
| 36 | unsigned long key = (unsigned long) hpage;
|
| 37 | unsigned long count;
|
| 38 | void *entry;
|
| 39 | int ret;
|
| 40 |
|
| 41 | lockdep_assert_held(&ctx->uring_lock);
|
| 42 |
|
| 43 | entry = xa_load(&ctx->hpage_acct, key);
|
| 44 | if (entry) {
|
| 45 | *acct_new = false;
|
| 46 | count = xa_to_value(entry) + 1;
|
| 47 | } else {
|
| 48 | ret = xa_reserve(&ctx->hpage_acct, key, GFP_KERNEL_ACCOUNT);
|
| 49 | if (ret)
|
| 50 | return ret;
|
| 51 | *acct_new = true;
|
| 52 | count = 1;
|
| 53 | }
|
| 54 | xa_store(&ctx->hpage_acct, key, xa_mk_value(count), GFP_KERNEL_ACCOUNT);
|
| 55 | return 0;
|
| 56 | }
|
| 57 |
|
| 58 | static bool hpage_acct_unref(struct io_ring_ctx *ctx, struct page *hpage)
|
| 59 | {
|
| 60 | unsigned long key = (unsigned long) hpage;
|
| 61 | unsigned long count;
|
| 62 | void *entry;
|
| 63 |
|
| 64 | lockdep_assert_held(&ctx->uring_lock);
|
| 65 |
|
| 66 | entry = xa_load(&ctx->hpage_acct, key);
|
| 67 | if (WARN_ON_ONCE(!entry))
|
| 68 | return false;
|
| 69 | count = xa_to_value(entry);
|
| 70 | if (count == 1) {
|
| 71 | xa_erase(&ctx->hpage_acct, key);
|
| 72 | return true;
|
| 73 | }
|
| 74 | xa_store(&ctx->hpage_acct, key, xa_mk_value(count - 1), GFP_KERNEL_ACCOUNT);
|
| 75 | return false;
|
| 76 | }
|
| 77 |
|
| 78 |
|
| 79 | #define IORING_MAX_FIXED_FILES (1U << 20)
|
| 80 | #define IORING_MAX_REG_BUFFERS (1U << 14)
|
| 81 |
|
| 82 | #define IO_CACHED_BVECS_SEGS 32
|
| 83 |
|
| 84 | int __io_account_mem(struct user_struct *user, unsigned long nr_pages)
|
| 85 | {
|
| 86 | unsigned long page_limit, cur_pages, new_pages;
|
| 87 |
|
| 88 | if (!nr_pages)
|
| 89 | return 0;
|
| 90 |
|
| 91 |
|
| 92 | page_limit = rlimit(RLIMIT_MEMLOCK) >> PAGE_SHIFT;
|
| 93 |
|
| 94 | cur_pages = atomic_long_read(&user->locked_vm);
|
| 95 | do {
|
| 96 | new_pages = cur_pages + nr_pages;
|
| 97 | if (new_pages > page_limit)
|
| 98 | return -ENOMEM;
|
| 99 | } while (!atomic_long_try_cmpxchg(&user->locked_vm,
|
| 100 | &cur_pages, new_pages));
|
| 101 | return 0;
|
| 102 | }
|
| 103 |
|
| 104 | void io_unaccount_mem(struct user_struct *user, struct mm_struct *mm_account,
|
| 105 | unsigned long nr_pages)
|
| 106 | {
|
| 107 | if (user)
|
| 108 | __io_unaccount_mem(user, nr_pages);
|
| 109 |
|
| 110 | if (mm_account)
|
| 111 | atomic64_sub(nr_pages, &mm_account->pinned_vm);
|
| 112 | }
|
| 113 |
|
| 114 | int io_account_mem(struct user_struct *user, struct mm_struct *mm_account,
|
| 115 | unsigned long nr_pages)
|
| 116 | {
|
| 117 | int ret;
|
| 118 |
|
| 119 | if (user) {
|
| 120 | ret = __io_account_mem(user, nr_pages);
|
| 121 | if (ret)
|
| 122 | return ret;
|
| 123 | }
|
| 124 |
|
| 125 | if (mm_account)
|
| 126 | atomic64_add(nr_pages, &mm_account->pinned_vm);
|
| 127 |
|
| 128 | return 0;
|
| 129 | }
|
| 130 |
|
| 131 | int io_validate_user_buf_range(u64 uaddr, u64 ulen)
|
| 132 | {
|
| 133 | unsigned long tmp, base = (unsigned long)uaddr;
|
| 134 | unsigned long acct_len = (unsigned long)PAGE_ALIGN(ulen);
|
| 135 |
|
| 136 | if (!ulen)
|
| 137 | return -EFAULT;
|
| 138 |
|
| 139 | if (ulen > ULONG_MAX || uaddr > ULONG_MAX)
|
| 140 | return -EFAULT;
|
| 141 |
|
| 142 | if (ulen > SZ_1T)
|
| 143 | return -EINVAL;
|
| 144 | if (check_add_overflow(base, acct_len, &tmp))
|
| 145 | return -EOVERFLOW;
|
| 146 | return 0;
|
| 147 | }
|
| 148 |
|
| 149 | static void io_release_ubuf(void *priv)
|
| 150 | {
|
| 151 | struct io_mapped_ubuf *imu = priv;
|
| 152 | unsigned int i;
|
| 153 |
|
| 154 | for (i = 0; i < imu->nr_bvecs; i++) {
|
| 155 | struct folio *folio = bvec_folio(&imu->bvec[i]);
|
| 156 |
|
| 157 | unpin_user_folio(folio, 1);
|
| 158 | }
|
| 159 | }
|
| 160 |
|
| 161 | static struct io_mapped_ubuf *io_alloc_imu(struct io_ring_ctx *ctx,
|
| 162 | int nr_bvecs)
|
| 163 | {
|
| 164 | if (nr_bvecs <= IO_CACHED_BVECS_SEGS)
|
| 165 | return io_cache_alloc(&ctx->imu_cache, GFP_KERNEL);
|
| 166 | return kvmalloc_flex(struct io_mapped_ubuf, bvec, nr_bvecs);
|
| 167 | }
|
| 168 |
|
| 169 | static void io_free_imu(struct io_ring_ctx *ctx, struct io_mapped_ubuf *imu)
|
| 170 | {
|
| 171 | if (imu->nr_bvecs <= IO_CACHED_BVECS_SEGS)
|
| 172 | io_cache_free(&ctx->imu_cache, imu);
|
| 173 | else
|
| 174 | kvfree(imu);
|
| 175 | }
|
| 176 |
|
| 177 | static unsigned long io_buffer_unaccount_pages(struct io_ring_ctx *ctx,
|
| 178 | struct io_mapped_ubuf *imu)
|
| 179 | {
|
| 180 | struct page *seen = NULL;
|
| 181 | unsigned long acct = 0;
|
| 182 | int i;
|
| 183 |
|
| 184 | if (imu->flags & IO_REGBUF_F_KBUF || !ctx->user)
|
| 185 | return 0;
|
| 186 |
|
| 187 | for (i = 0; i < imu->nr_bvecs; i++) {
|
| 188 | struct page *page = imu->bvec[i].bv_page;
|
| 189 | struct page *hpage;
|
| 190 |
|
| 191 | if (!PageCompound(page)) {
|
| 192 | acct++;
|
| 193 | continue;
|
| 194 | }
|
| 195 |
|
| 196 | hpage = compound_head(page);
|
| 197 | if (hpage == seen)
|
| 198 | continue;
|
| 199 | seen = hpage;
|
| 200 |
|
| 201 |
|
| 202 | if (hpage_acct_unref(ctx, hpage))
|
| 203 | acct += page_size(hpage) >> PAGE_SHIFT;
|
| 204 | cond_resched();
|
| 205 | }
|
| 206 |
|
| 207 | return acct;
|
| 208 | }
|
| 209 |
|
| 210 | static void io_buffer_unmap(struct io_ring_ctx *ctx, struct io_mapped_ubuf *imu)
|
| 211 | {
|
| 212 | unsigned long acct_pages = 0;
|
| 213 |
|
| 214 |
|
| 215 | acct_pages = io_buffer_unaccount_pages(ctx, imu);
|
| 216 |
|
| 217 | if (unlikely(refcount_read(&imu->refs) > 1)) {
|
| 218 | if (!refcount_dec_and_test(&imu->refs))
|
| 219 | return;
|
| 220 | }
|
| 221 |
|
| 222 | if (acct_pages)
|
| 223 | io_unaccount_mem(ctx->user, ctx->mm_account, acct_pages);
|
| 224 | imu->release(imu->priv);
|
| 225 | io_free_imu(ctx, imu);
|
| 226 | }
|
| 227 |
|
| 228 | struct io_rsrc_node *io_rsrc_node_alloc(struct io_ring_ctx *ctx, int type)
|
| 229 | {
|
| 230 | struct io_rsrc_node *node;
|
| 231 |
|
| 232 | node = io_cache_alloc(&ctx->node_cache, GFP_KERNEL);
|
| 233 | if (node) {
|
| 234 | node->type = type;
|
| 235 | node->refs = 1;
|
| 236 | node->tag = 0;
|
| 237 | node->file_ptr = 0;
|
| 238 | }
|
| 239 | return node;
|
| 240 | }
|
| 241 |
|
| 242 | bool io_rsrc_cache_init(struct io_ring_ctx *ctx)
|
| 243 | {
|
| 244 | const int imu_cache_size = struct_size_t(struct io_mapped_ubuf, bvec,
|
| 245 | IO_CACHED_BVECS_SEGS);
|
| 246 | const int node_size = sizeof(struct io_rsrc_node);
|
| 247 | bool ret;
|
| 248 |
|
| 249 | ret = io_alloc_cache_init(&ctx->node_cache, IO_ALLOC_CACHE_MAX,
|
| 250 | node_size, 0);
|
| 251 | ret |= io_alloc_cache_init(&ctx->imu_cache, IO_ALLOC_CACHE_MAX,
|
| 252 | imu_cache_size, 0);
|
| 253 | return ret;
|
| 254 | }
|
| 255 |
|
| 256 | void io_rsrc_cache_free(struct io_ring_ctx *ctx)
|
| 257 | {
|
| 258 | io_alloc_cache_free(&ctx->node_cache, kfree);
|
| 259 | io_alloc_cache_free(&ctx->imu_cache, kvfree);
|
| 260 | }
|
| 261 |
|
| 262 | static void io_clear_table_tags(struct io_rsrc_data *data)
|
| 263 | {
|
| 264 | int i;
|
| 265 |
|
| 266 | for (i = 0; i < data->nr; i++) {
|
| 267 | struct io_rsrc_node *node = data->nodes[i];
|
| 268 |
|
| 269 | if (node)
|
| 270 | node->tag = 0;
|
| 271 | }
|
| 272 | }
|
| 273 |
|
| 274 | __cold void io_rsrc_data_free(struct io_ring_ctx *ctx,
|
| 275 | struct io_rsrc_data *data)
|
| 276 | {
|
| 277 | if (!data->nr)
|
| 278 | return;
|
| 279 | while (data->nr--) {
|
| 280 | if (data->nodes[data->nr])
|
| 281 | io_put_rsrc_node(ctx, data->nodes[data->nr]);
|
| 282 | }
|
| 283 | kvfree(data->nodes);
|
| 284 | data->nodes = NULL;
|
| 285 | data->nr = 0;
|
| 286 | }
|
| 287 |
|
| 288 | __cold int io_rsrc_data_alloc(struct io_rsrc_data *data, unsigned nr)
|
| 289 | {
|
| 290 | data->nodes = kvmalloc_objs(struct io_rsrc_node *, nr,
|
| 291 | GFP_KERNEL_ACCOUNT | __GFP_ZERO);
|
| 292 | if (data->nodes) {
|
| 293 | data->nr = nr;
|
| 294 | return 0;
|
| 295 | }
|
| 296 | return -ENOMEM;
|
| 297 | }
|
| 298 |
|
| 299 | static int __io_sqe_files_update(struct io_ring_ctx *ctx,
|
| 300 | struct io_uring_rsrc_update2 *up,
|
| 301 | unsigned nr_args)
|
| 302 | {
|
| 303 | u64 __user *tags = u64_to_user_ptr(up->tags);
|
| 304 | __s32 __user *fds = u64_to_user_ptr(up->data);
|
| 305 | int fd, i, err = 0;
|
| 306 | unsigned int done;
|
| 307 |
|
| 308 | if (!ctx->file_table.data.nr)
|
| 309 | return -ENXIO;
|
| 310 | if (up->offset + nr_args > ctx->file_table.data.nr)
|
| 311 | return -EINVAL;
|
| 312 |
|
| 313 | for (done = 0; done < nr_args; done++) {
|
| 314 | u64 tag = 0;
|
| 315 |
|
| 316 | if ((tags && copy_from_user(&tag, &tags[done], sizeof(tag))) ||
|
| 317 | copy_from_user(&fd, &fds[done], sizeof(fd))) {
|
| 318 | err = -EFAULT;
|
| 319 | break;
|
| 320 | }
|
| 321 | if ((fd == IORING_REGISTER_FILES_SKIP || fd == -1) && tag) {
|
| 322 | err = -EINVAL;
|
| 323 | break;
|
| 324 | }
|
| 325 | if (fd == IORING_REGISTER_FILES_SKIP)
|
| 326 | continue;
|
| 327 |
|
| 328 | i = up->offset + done;
|
| 329 | if (i >= ctx->file_table.data.nr)
|
| 330 | break;
|
| 331 | i = array_index_nospec(i, ctx->file_table.data.nr);
|
| 332 | if (io_reset_rsrc_node(ctx, &ctx->file_table.data, i))
|
| 333 | io_file_bitmap_clear(&ctx->file_table, i);
|
| 334 |
|
| 335 | if (fd != -1) {
|
| 336 | struct file *file = fget(fd);
|
| 337 | struct io_rsrc_node *node;
|
| 338 |
|
| 339 | if (!file) {
|
| 340 | err = -EBADF;
|
| 341 | break;
|
| 342 | }
|
| 343 | |
| 344 | * Don't allow io_uring instances to be registered.
|
| 345 |
|
| 346 | if (io_is_uring_fops(file)) {
|
| 347 | fput(file);
|
| 348 | err = -EBADF;
|
| 349 | break;
|
| 350 | }
|
| 351 | node = io_rsrc_node_alloc(ctx, IORING_RSRC_FILE);
|
| 352 | if (!node) {
|
| 353 | err = -ENOMEM;
|
| 354 | fput(file);
|
| 355 | break;
|
| 356 | }
|
| 357 | ctx->file_table.data.nodes[i] = node;
|
| 358 | if (tag)
|
| 359 | node->tag = tag;
|
| 360 | io_fixed_file_set(node, file);
|
| 361 | io_file_bitmap_set(&ctx->file_table, i);
|
| 362 | }
|
| 363 | }
|
| 364 | return done ? done : err;
|
| 365 | }
|
| 366 |
|
| 367 | static int __io_sqe_buffers_update(struct io_ring_ctx *ctx,
|
| 368 | struct io_uring_rsrc_update2 *up,
|
| 369 | unsigned int nr_args)
|
| 370 | {
|
| 371 | u64 __user *tags = u64_to_user_ptr(up->tags);
|
| 372 | struct iovec fast_iov, *iov;
|
| 373 | struct iovec __user *uvec;
|
| 374 | u64 user_data = up->data;
|
| 375 | __u32 done;
|
| 376 | int i, err;
|
| 377 |
|
| 378 | if (!ctx->buf_table.nr)
|
| 379 | return -ENXIO;
|
| 380 | if (up->offset + nr_args > ctx->buf_table.nr)
|
| 381 | return -EINVAL;
|
| 382 |
|
| 383 | for (done = 0; done < nr_args; done++) {
|
| 384 | struct io_rsrc_node *node;
|
| 385 | u64 tag = 0;
|
| 386 |
|
| 387 | uvec = u64_to_user_ptr(user_data);
|
| 388 | iov = iovec_from_user(uvec, 1, 1, &fast_iov, io_is_compat(ctx));
|
| 389 | if (IS_ERR(iov)) {
|
| 390 | err = PTR_ERR(iov);
|
| 391 | break;
|
| 392 | }
|
| 393 | if (tags && copy_from_user(&tag, &tags[done], sizeof(tag))) {
|
| 394 | err = -EFAULT;
|
| 395 | break;
|
| 396 | }
|
| 397 | node = io_sqe_buffer_register(ctx, iov);
|
| 398 | if (IS_ERR(node)) {
|
| 399 | err = PTR_ERR(node);
|
| 400 | break;
|
| 401 | }
|
| 402 | if (tag) {
|
| 403 | if (!node) {
|
| 404 | err = -EINVAL;
|
| 405 | break;
|
| 406 | }
|
| 407 | node->tag = tag;
|
| 408 | }
|
| 409 | i = array_index_nospec(up->offset + done, ctx->buf_table.nr);
|
| 410 | io_reset_rsrc_node(ctx, &ctx->buf_table, i);
|
| 411 | ctx->buf_table.nodes[i] = node;
|
| 412 | if (io_is_compat(ctx))
|
| 413 | user_data += sizeof(struct compat_iovec);
|
| 414 | else
|
| 415 | user_data += sizeof(struct iovec);
|
| 416 | }
|
| 417 | return done ? done : err;
|
| 418 | }
|
| 419 |
|
| 420 | static int __io_register_rsrc_update(struct io_ring_ctx *ctx, unsigned type,
|
| 421 | struct io_uring_rsrc_update2 *up,
|
| 422 | unsigned nr_args)
|
| 423 | {
|
| 424 | __u32 tmp;
|
| 425 |
|
| 426 | lockdep_assert_held(&ctx->uring_lock);
|
| 427 |
|
| 428 | if (check_add_overflow(up->offset, nr_args, &tmp))
|
| 429 | return -EOVERFLOW;
|
| 430 |
|
| 431 | switch (type) {
|
| 432 | case IORING_RSRC_FILE:
|
| 433 | return __io_sqe_files_update(ctx, up, nr_args);
|
| 434 | case IORING_RSRC_BUFFER:
|
| 435 | return __io_sqe_buffers_update(ctx, up, nr_args);
|
| 436 | }
|
| 437 | return -EINVAL;
|
| 438 | }
|
| 439 |
|
| 440 | int io_register_files_update(struct io_ring_ctx *ctx, void __user *arg,
|
| 441 | unsigned nr_args)
|
| 442 | {
|
| 443 | struct io_uring_rsrc_update2 up;
|
| 444 |
|
| 445 | if (!nr_args)
|
| 446 | return -EINVAL;
|
| 447 | memset(&up, 0, sizeof(up));
|
| 448 | if (copy_from_user(&up, arg, sizeof(struct io_uring_rsrc_update)))
|
| 449 | return -EFAULT;
|
| 450 | if (up.resv || up.resv2)
|
| 451 | return -EINVAL;
|
| 452 | return __io_register_rsrc_update(ctx, IORING_RSRC_FILE, &up, nr_args);
|
| 453 | }
|
| 454 |
|
| 455 | int io_register_rsrc_update(struct io_ring_ctx *ctx, void __user *arg,
|
| 456 | unsigned size, unsigned type)
|
| 457 | {
|
| 458 | struct io_uring_rsrc_update2 up;
|
| 459 |
|
| 460 | if (size != sizeof(up))
|
| 461 | return -EINVAL;
|
| 462 | if (copy_from_user(&up, arg, sizeof(up)))
|
| 463 | return -EFAULT;
|
| 464 | if (!up.nr || up.resv || up.resv2)
|
| 465 | return -EINVAL;
|
| 466 | return __io_register_rsrc_update(ctx, type, &up, up.nr);
|
| 467 | }
|
| 468 |
|
| 469 | __cold int io_register_rsrc(struct io_ring_ctx *ctx, void __user *arg,
|
| 470 | unsigned int size, unsigned int type)
|
| 471 | {
|
| 472 | struct io_uring_rsrc_register rr;
|
| 473 |
|
| 474 |
|
| 475 | if (size != sizeof(rr))
|
| 476 | return -EINVAL;
|
| 477 |
|
| 478 | memset(&rr, 0, sizeof(rr));
|
| 479 | if (copy_from_user(&rr, arg, size))
|
| 480 | return -EFAULT;
|
| 481 | if (!rr.nr || rr.resv2)
|
| 482 | return -EINVAL;
|
| 483 | if (rr.flags & ~IORING_RSRC_REGISTER_SPARSE)
|
| 484 | return -EINVAL;
|
| 485 |
|
| 486 | switch (type) {
|
| 487 | case IORING_RSRC_FILE:
|
| 488 | if (rr.flags & IORING_RSRC_REGISTER_SPARSE && rr.data)
|
| 489 | break;
|
| 490 | return io_sqe_files_register(ctx, u64_to_user_ptr(rr.data),
|
| 491 | rr.nr, u64_to_user_ptr(rr.tags));
|
| 492 | case IORING_RSRC_BUFFER:
|
| 493 | if (rr.flags & IORING_RSRC_REGISTER_SPARSE && rr.data)
|
| 494 | break;
|
| 495 | return io_sqe_buffers_register(ctx, u64_to_user_ptr(rr.data),
|
| 496 | rr.nr, u64_to_user_ptr(rr.tags));
|
| 497 | }
|
| 498 | return -EINVAL;
|
| 499 | }
|
| 500 |
|
| 501 | int io_files_update_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
|
| 502 | {
|
| 503 | struct io_rsrc_update *up = io_kiocb_to_cmd(req, struct io_rsrc_update);
|
| 504 |
|
| 505 | if (unlikely(req->flags & (REQ_F_FIXED_FILE | REQ_F_BUFFER_SELECT)))
|
| 506 | return -EINVAL;
|
| 507 | if (sqe->rw_flags || sqe->splice_fd_in)
|
| 508 | return -EINVAL;
|
| 509 |
|
| 510 | up->offset = READ_ONCE(sqe->off);
|
| 511 | up->nr_args = READ_ONCE(sqe->len);
|
| 512 | if (!up->nr_args)
|
| 513 | return -EINVAL;
|
| 514 | up->arg = READ_ONCE(sqe->addr);
|
| 515 | return 0;
|
| 516 | }
|
| 517 |
|
| 518 | static int io_files_update_with_index_alloc(struct io_kiocb *req,
|
| 519 | unsigned int issue_flags)
|
| 520 | {
|
| 521 | struct io_rsrc_update *up = io_kiocb_to_cmd(req, struct io_rsrc_update);
|
| 522 | __s32 __user *fds = u64_to_user_ptr(up->arg);
|
| 523 | unsigned int done;
|
| 524 | struct file *file;
|
| 525 | int ret, fd;
|
| 526 |
|
| 527 | if (!req->ctx->file_table.data.nr)
|
| 528 | return -ENXIO;
|
| 529 |
|
| 530 | for (done = 0; done < up->nr_args; done++) {
|
| 531 | if (get_user(fd, &fds[done])) {
|
| 532 | ret = -EFAULT;
|
| 533 | break;
|
| 534 | }
|
| 535 |
|
| 536 | file = fget(fd);
|
| 537 | if (!file) {
|
| 538 | ret = -EBADF;
|
| 539 | break;
|
| 540 | }
|
| 541 | ret = io_fixed_fd_install(req, issue_flags, file,
|
| 542 | IORING_FILE_INDEX_ALLOC);
|
| 543 | if (ret < 0)
|
| 544 | break;
|
| 545 | if (put_user(ret, &fds[done])) {
|
| 546 | __io_close_fixed(req->ctx, issue_flags, ret);
|
| 547 | ret = -EFAULT;
|
| 548 | break;
|
| 549 | }
|
| 550 | }
|
| 551 |
|
| 552 | if (done)
|
| 553 | return done;
|
| 554 | return ret;
|
| 555 | }
|
| 556 |
|
| 557 | int io_files_update(struct io_kiocb *req, unsigned int issue_flags)
|
| 558 | {
|
| 559 | struct io_rsrc_update *up = io_kiocb_to_cmd(req, struct io_rsrc_update);
|
| 560 | struct io_ring_ctx *ctx = req->ctx;
|
| 561 | struct io_uring_rsrc_update2 up2;
|
| 562 | int ret;
|
| 563 |
|
| 564 | up2.offset = up->offset;
|
| 565 | up2.data = up->arg;
|
| 566 | up2.nr = 0;
|
| 567 | up2.tags = 0;
|
| 568 | up2.resv = 0;
|
| 569 | up2.resv2 = 0;
|
| 570 |
|
| 571 | if (up->offset == IORING_FILE_INDEX_ALLOC) {
|
| 572 | ret = io_files_update_with_index_alloc(req, issue_flags);
|
| 573 | } else {
|
| 574 | io_ring_submit_lock(ctx, issue_flags);
|
| 575 | ret = __io_register_rsrc_update(ctx, IORING_RSRC_FILE,
|
| 576 | &up2, up->nr_args);
|
| 577 | io_ring_submit_unlock(ctx, issue_flags);
|
| 578 | }
|
| 579 |
|
| 580 | if (ret < 0)
|
| 581 | req_set_fail(req);
|
| 582 | io_req_set_res(req, ret, 0);
|
| 583 | return IOU_COMPLETE;
|
| 584 | }
|
| 585 |
|
| 586 | void io_free_rsrc_node(struct io_ring_ctx *ctx, struct io_rsrc_node *node)
|
| 587 | {
|
| 588 | if (node->tag)
|
| 589 | io_post_aux_cqe(ctx, node->tag, 0, 0);
|
| 590 |
|
| 591 | switch (node->type) {
|
| 592 | case IORING_RSRC_FILE:
|
| 593 | fput(io_slot_file(node));
|
| 594 | break;
|
| 595 | case IORING_RSRC_BUFFER:
|
| 596 | io_buffer_unmap(ctx, node->buf);
|
| 597 | break;
|
| 598 | default:
|
| 599 | WARN_ON_ONCE(1);
|
| 600 | break;
|
| 601 | }
|
| 602 |
|
| 603 | io_cache_free(&ctx->node_cache, node);
|
| 604 | }
|
| 605 |
|
| 606 | int io_sqe_files_unregister(struct io_ring_ctx *ctx)
|
| 607 | {
|
| 608 | if (!ctx->file_table.data.nr)
|
| 609 | return -ENXIO;
|
| 610 |
|
| 611 | io_free_file_tables(ctx, &ctx->file_table);
|
| 612 | io_file_table_set_alloc_range(ctx, 0, 0);
|
| 613 | return 0;
|
| 614 | }
|
| 615 |
|
| 616 | int io_sqe_files_register(struct io_ring_ctx *ctx, void __user *arg,
|
| 617 | unsigned nr_args, u64 __user *tags)
|
| 618 | {
|
| 619 | __s32 __user *fds = (__s32 __user *) arg;
|
| 620 | struct file *file;
|
| 621 | int fd, ret;
|
| 622 | unsigned i;
|
| 623 |
|
| 624 | if (ctx->file_table.data.nr)
|
| 625 | return -EBUSY;
|
| 626 | if (!nr_args)
|
| 627 | return -EINVAL;
|
| 628 | if (nr_args > IORING_MAX_FIXED_FILES)
|
| 629 | return -EMFILE;
|
| 630 | if (nr_args > rlimit(RLIMIT_NOFILE))
|
| 631 | return -EMFILE;
|
| 632 | if (!io_alloc_file_tables(ctx, &ctx->file_table, nr_args))
|
| 633 | return -ENOMEM;
|
| 634 |
|
| 635 | for (i = 0; i < nr_args; i++) {
|
| 636 | struct io_rsrc_node *node;
|
| 637 | u64 tag = 0;
|
| 638 |
|
| 639 | ret = -EFAULT;
|
| 640 | if (tags && copy_from_user(&tag, &tags[i], sizeof(tag)))
|
| 641 | goto fail;
|
| 642 | if (fds && copy_from_user(&fd, &fds[i], sizeof(fd)))
|
| 643 | goto fail;
|
| 644 |
|
| 645 | if (!fds || fd == -1) {
|
| 646 | ret = -EINVAL;
|
| 647 | if (tag)
|
| 648 | goto fail;
|
| 649 | continue;
|
| 650 | }
|
| 651 |
|
| 652 | file = fget(fd);
|
| 653 | ret = -EBADF;
|
| 654 | if (unlikely(!file))
|
| 655 | goto fail;
|
| 656 |
|
| 657 | |
| 658 | * Don't allow io_uring instances to be registered.
|
| 659 |
|
| 660 | if (io_is_uring_fops(file)) {
|
| 661 | fput(file);
|
| 662 | goto fail;
|
| 663 | }
|
| 664 | ret = -ENOMEM;
|
| 665 | node = io_rsrc_node_alloc(ctx, IORING_RSRC_FILE);
|
| 666 | if (!node) {
|
| 667 | fput(file);
|
| 668 | goto fail;
|
| 669 | }
|
| 670 | if (tag)
|
| 671 | node->tag = tag;
|
| 672 | ctx->file_table.data.nodes[i] = node;
|
| 673 | io_fixed_file_set(node, file);
|
| 674 | io_file_bitmap_set(&ctx->file_table, i);
|
| 675 | }
|
| 676 |
|
| 677 |
|
| 678 | io_file_table_set_alloc_range(ctx, 0, ctx->file_table.data.nr);
|
| 679 | return 0;
|
| 680 | fail:
|
| 681 | io_clear_table_tags(&ctx->file_table.data);
|
| 682 | io_sqe_files_unregister(ctx);
|
| 683 | return ret;
|
| 684 | }
|
| 685 |
|
| 686 | int io_sqe_buffers_unregister(struct io_ring_ctx *ctx)
|
| 687 | {
|
| 688 | if (!ctx->buf_table.nr)
|
| 689 | return -ENXIO;
|
| 690 | io_rsrc_data_free(ctx, &ctx->buf_table);
|
| 691 | return 0;
|
| 692 | }
|
| 693 |
|
| 694 | |
| 695 | * Undo hpage_acct_ref() calls made during io_buffer_account_pin() on failure.
|
| 696 | * This operates on the pages array since imu->bvec isn't populated yet.
|
| 697 |
|
| 698 | static void io_buffer_unaccount_hpages(struct io_ring_ctx *ctx,
|
| 699 | struct page **pages, int nr_pages)
|
| 700 | {
|
| 701 | struct page *seen = NULL;
|
| 702 | int i;
|
| 703 |
|
| 704 | if (!ctx->user)
|
| 705 | return;
|
| 706 |
|
| 707 | for (i = 0; i < nr_pages; i++) {
|
| 708 | struct page *hpage;
|
| 709 |
|
| 710 | if (!PageCompound(pages[i]))
|
| 711 | continue;
|
| 712 |
|
| 713 | hpage = compound_head(pages[i]);
|
| 714 | if (hpage == seen)
|
| 715 | continue;
|
| 716 | seen = hpage;
|
| 717 |
|
| 718 | hpage_acct_unref(ctx, hpage);
|
| 719 | cond_resched();
|
| 720 | }
|
| 721 | }
|
| 722 |
|
| 723 | static int io_buffer_account_pin(struct io_ring_ctx *ctx, struct page **pages,
|
| 724 | int nr_pages)
|
| 725 | {
|
| 726 | unsigned long acct_pages = 0;
|
| 727 | struct page *seen = NULL;
|
| 728 | int i, ret;
|
| 729 |
|
| 730 | if (!ctx->user)
|
| 731 | return 0;
|
| 732 |
|
| 733 | for (i = 0; i < nr_pages; i++) {
|
| 734 | struct page *hpage;
|
| 735 | bool acct_new;
|
| 736 |
|
| 737 | if (!PageCompound(pages[i])) {
|
| 738 | acct_pages++;
|
| 739 | continue;
|
| 740 | }
|
| 741 |
|
| 742 | hpage = compound_head(pages[i]);
|
| 743 | if (hpage == seen)
|
| 744 | continue;
|
| 745 | seen = hpage;
|
| 746 |
|
| 747 | ret = hpage_acct_ref(ctx, hpage, &acct_new);
|
| 748 | if (ret) {
|
| 749 | io_buffer_unaccount_hpages(ctx, pages, i);
|
| 750 | return ret;
|
| 751 | }
|
| 752 | if (acct_new)
|
| 753 | acct_pages += page_size(hpage) >> PAGE_SHIFT;
|
| 754 | cond_resched();
|
| 755 | }
|
| 756 |
|
| 757 |
|
| 758 | if (acct_pages) {
|
| 759 | ret = io_account_mem(ctx->user, ctx->mm_account, acct_pages);
|
| 760 | if (ret) {
|
| 761 |
|
| 762 | io_buffer_unaccount_hpages(ctx, pages, nr_pages);
|
| 763 | return ret;
|
| 764 | }
|
| 765 | }
|
| 766 |
|
| 767 | return 0;
|
| 768 | }
|
| 769 |
|
| 770 | static bool io_coalesce_buffer(struct page ***pages, int *nr_pages,
|
| 771 | struct io_imu_folio_data *data)
|
| 772 | {
|
| 773 | struct page **page_array = *pages, **new_array = NULL;
|
| 774 | unsigned nr_pages_left = *nr_pages;
|
| 775 | unsigned nr_folios = data->nr_folios;
|
| 776 | unsigned i, j;
|
| 777 |
|
| 778 |
|
| 779 | new_array = kvmalloc_objs(struct page *, nr_folios);
|
| 780 | if (!new_array)
|
| 781 | return false;
|
| 782 |
|
| 783 | for (i = 0, j = 0; i < nr_folios; i++) {
|
| 784 | struct page *p = compound_head(page_array[j]);
|
| 785 | struct folio *folio = page_folio(p);
|
| 786 | unsigned int nr;
|
| 787 |
|
| 788 | WARN_ON_ONCE(i > 0 && p != page_array[j]);
|
| 789 |
|
| 790 | nr = i ? data->nr_pages_mid : data->nr_pages_head;
|
| 791 | nr = min(nr, nr_pages_left);
|
| 792 |
|
| 793 | if (nr > 1)
|
| 794 | unpin_user_folio(folio, nr - 1);
|
| 795 | j += nr;
|
| 796 | nr_pages_left -= nr;
|
| 797 | new_array[i] = p;
|
| 798 | }
|
| 799 |
|
| 800 | WARN_ON_ONCE(j != *nr_pages);
|
| 801 |
|
| 802 | kvfree(page_array);
|
| 803 | *pages = new_array;
|
| 804 | *nr_pages = nr_folios;
|
| 805 | return true;
|
| 806 | }
|
| 807 |
|
| 808 | bool io_check_coalesce_buffer(struct page **page_array, int nr_pages,
|
| 809 | struct io_imu_folio_data *data)
|
| 810 | {
|
| 811 | struct folio *folio = page_folio(page_array[0]);
|
| 812 | unsigned int count = 1, nr_folios = 1;
|
| 813 | int i;
|
| 814 |
|
| 815 | data->nr_pages_mid = folio_nr_pages(folio);
|
| 816 | data->folio_shift = folio_shift(folio);
|
| 817 | data->first_folio_page_idx = folio_page_idx(folio, page_array[0]);
|
| 818 |
|
| 819 | |
| 820 | * Check if pages are contiguous inside a folio, and all folios have
|
| 821 | * the same page count except for the head and tail.
|
| 822 |
|
| 823 | for (i = 1; i < nr_pages; i++) {
|
| 824 | if (page_folio(page_array[i]) == folio &&
|
| 825 | page_array[i] == page_array[i-1] + 1) {
|
| 826 | count++;
|
| 827 | continue;
|
| 828 | }
|
| 829 |
|
| 830 | if (nr_folios == 1) {
|
| 831 | if (folio_page_idx(folio, page_array[i-1]) !=
|
| 832 | data->nr_pages_mid - 1)
|
| 833 | return false;
|
| 834 |
|
| 835 | data->nr_pages_head = count;
|
| 836 | } else if (count != data->nr_pages_mid) {
|
| 837 | return false;
|
| 838 | }
|
| 839 |
|
| 840 | folio = page_folio(page_array[i]);
|
| 841 | if (folio_size(folio) != (1UL << data->folio_shift) ||
|
| 842 | folio_page_idx(folio, page_array[i]) != 0)
|
| 843 | return false;
|
| 844 |
|
| 845 | count = 1;
|
| 846 | nr_folios++;
|
| 847 | }
|
| 848 | if (nr_folios == 1)
|
| 849 | data->nr_pages_head = count;
|
| 850 |
|
| 851 | data->nr_folios = nr_folios;
|
| 852 | return true;
|
| 853 | }
|
| 854 |
|
| 855 | static struct io_rsrc_node *io_sqe_buffer_register(struct io_ring_ctx *ctx,
|
| 856 | struct iovec *iov)
|
| 857 | {
|
| 858 | struct io_mapped_ubuf *imu = NULL;
|
| 859 | struct page **pages = NULL;
|
| 860 | struct io_rsrc_node *node;
|
| 861 | unsigned long off;
|
| 862 | size_t size;
|
| 863 | int ret, nr_pages, i;
|
| 864 | struct io_imu_folio_data data;
|
| 865 | bool coalesced = false;
|
| 866 |
|
| 867 | if (!iov->iov_base) {
|
| 868 | if (iov->iov_len)
|
| 869 | return ERR_PTR(-EFAULT);
|
| 870 |
|
| 871 | return NULL;
|
| 872 | }
|
| 873 |
|
| 874 | ret = io_validate_user_buf_range((unsigned long)iov->iov_base,
|
| 875 | iov->iov_len);
|
| 876 | if (ret)
|
| 877 | return ERR_PTR(ret);
|
| 878 |
|
| 879 | node = io_rsrc_node_alloc(ctx, IORING_RSRC_BUFFER);
|
| 880 | if (!node)
|
| 881 | return ERR_PTR(-ENOMEM);
|
| 882 |
|
| 883 | ret = -ENOMEM;
|
| 884 | pages = io_pin_pages((unsigned long) iov->iov_base, iov->iov_len,
|
| 885 | &nr_pages);
|
| 886 | if (IS_ERR(pages)) {
|
| 887 | ret = PTR_ERR(pages);
|
| 888 | pages = NULL;
|
| 889 | goto done;
|
| 890 | }
|
| 891 |
|
| 892 |
|
| 893 | if (nr_pages > 1 && io_check_coalesce_buffer(pages, nr_pages, &data)) {
|
| 894 | if (data.nr_pages_mid != 1)
|
| 895 | coalesced = io_coalesce_buffer(&pages, &nr_pages, &data);
|
| 896 | }
|
| 897 |
|
| 898 | imu = io_alloc_imu(ctx, nr_pages);
|
| 899 | if (!imu)
|
| 900 | goto done;
|
| 901 |
|
| 902 | imu->nr_bvecs = nr_pages;
|
| 903 | ret = io_buffer_account_pin(ctx, pages, nr_pages);
|
| 904 | if (ret)
|
| 905 | goto done;
|
| 906 |
|
| 907 | size = iov->iov_len;
|
| 908 |
|
| 909 | imu->ubuf = (unsigned long) iov->iov_base;
|
| 910 | imu->len = iov->iov_len;
|
| 911 | imu->folio_shift = PAGE_SHIFT;
|
| 912 | imu->release = io_release_ubuf;
|
| 913 | imu->priv = imu;
|
| 914 | imu->flags = 0;
|
| 915 | imu->dir = IO_BUF_DEST | IO_BUF_SOURCE;
|
| 916 | if (coalesced)
|
| 917 | imu->folio_shift = data.folio_shift;
|
| 918 | refcount_set(&imu->refs, 1);
|
| 919 |
|
| 920 | off = (unsigned long)iov->iov_base & ~PAGE_MASK;
|
| 921 | if (coalesced)
|
| 922 | off += data.first_folio_page_idx << PAGE_SHIFT;
|
| 923 |
|
| 924 | node->buf = imu;
|
| 925 | ret = 0;
|
| 926 |
|
| 927 | for (i = 0; i < nr_pages; i++) {
|
| 928 | size_t vec_len;
|
| 929 |
|
| 930 | vec_len = min_t(size_t, size, (1UL << imu->folio_shift) - off);
|
| 931 | bvec_set_page(&imu->bvec[i], pages[i], vec_len, off);
|
| 932 | off = 0;
|
| 933 | size -= vec_len;
|
| 934 | }
|
| 935 | done:
|
| 936 | if (ret) {
|
| 937 | if (imu)
|
| 938 | io_free_imu(ctx, imu);
|
| 939 | if (pages) {
|
| 940 | for (i = 0; i < nr_pages; i++)
|
| 941 | unpin_user_folio(page_folio(pages[i]), 1);
|
| 942 | }
|
| 943 | io_cache_free(&ctx->node_cache, node);
|
| 944 | node = ERR_PTR(ret);
|
| 945 | }
|
| 946 | kvfree(pages);
|
| 947 | return node;
|
| 948 | }
|
| 949 |
|
| 950 | int io_sqe_buffers_register(struct io_ring_ctx *ctx, void __user *arg,
|
| 951 | unsigned int nr_args, u64 __user *tags)
|
| 952 | {
|
| 953 | struct io_rsrc_data data;
|
| 954 | struct iovec fast_iov, *iov = &fast_iov;
|
| 955 | const struct iovec __user *uvec;
|
| 956 | int i, ret;
|
| 957 |
|
| 958 | BUILD_BUG_ON(IORING_MAX_REG_BUFFERS >= (1u << 16));
|
| 959 |
|
| 960 | if (ctx->buf_table.nr)
|
| 961 | return -EBUSY;
|
| 962 | if (!nr_args || nr_args > IORING_MAX_REG_BUFFERS)
|
| 963 | return -EINVAL;
|
| 964 | ret = io_rsrc_data_alloc(&data, nr_args);
|
| 965 | if (ret)
|
| 966 | return ret;
|
| 967 |
|
| 968 | if (!arg)
|
| 969 | memset(iov, 0, sizeof(*iov));
|
| 970 |
|
| 971 | for (i = 0; i < nr_args; i++) {
|
| 972 | struct io_rsrc_node *node;
|
| 973 | u64 tag = 0;
|
| 974 |
|
| 975 | if (arg) {
|
| 976 | uvec = (struct iovec __user *) arg;
|
| 977 | iov = iovec_from_user(uvec, 1, 1, &fast_iov, io_is_compat(ctx));
|
| 978 | if (IS_ERR(iov)) {
|
| 979 | ret = PTR_ERR(iov);
|
| 980 | break;
|
| 981 | }
|
| 982 | if (io_is_compat(ctx))
|
| 983 | arg += sizeof(struct compat_iovec);
|
| 984 | else
|
| 985 | arg += sizeof(struct iovec);
|
| 986 | }
|
| 987 |
|
| 988 | if (tags) {
|
| 989 | if (copy_from_user(&tag, &tags[i], sizeof(tag))) {
|
| 990 | ret = -EFAULT;
|
| 991 | break;
|
| 992 | }
|
| 993 | }
|
| 994 |
|
| 995 | node = io_sqe_buffer_register(ctx, iov);
|
| 996 | if (IS_ERR(node)) {
|
| 997 | ret = PTR_ERR(node);
|
| 998 | break;
|
| 999 | }
|
| 1000 | if (tag) {
|
| 1001 | if (!node) {
|
| 1002 | ret = -EINVAL;
|
| 1003 | break;
|
| 1004 | }
|
| 1005 | node->tag = tag;
|
| 1006 | }
|
| 1007 | data.nodes[i] = node;
|
| 1008 | }
|
| 1009 |
|
| 1010 | ctx->buf_table = data;
|
| 1011 | if (ret) {
|
| 1012 | io_clear_table_tags(&ctx->buf_table);
|
| 1013 | io_sqe_buffers_unregister(ctx);
|
| 1014 | }
|
| 1015 | return ret;
|
| 1016 | }
|
| 1017 |
|
| 1018 | static struct io_mapped_ubuf *io_kernel_buffer_init(struct io_ring_ctx *ctx,
|
| 1019 | unsigned int nr_bvecs,
|
| 1020 | unsigned int total_bytes,
|
| 1021 | u8 dir,
|
| 1022 | void (*release)(void *),
|
| 1023 | void *priv,
|
| 1024 | unsigned int index)
|
| 1025 | {
|
| 1026 | struct io_rsrc_data *data = &ctx->buf_table;
|
| 1027 | struct io_mapped_ubuf *imu;
|
| 1028 | struct io_rsrc_node *node;
|
| 1029 |
|
| 1030 | if (index >= data->nr)
|
| 1031 | return ERR_PTR(-EINVAL);
|
| 1032 | index = array_index_nospec(index, data->nr);
|
| 1033 |
|
| 1034 | if (data->nodes[index])
|
| 1035 | return ERR_PTR(-EBUSY);
|
| 1036 |
|
| 1037 | node = io_rsrc_node_alloc(ctx, IORING_RSRC_BUFFER);
|
| 1038 | if (!node)
|
| 1039 | return ERR_PTR(-ENOMEM);
|
| 1040 |
|
| 1041 | imu = io_alloc_imu(ctx, nr_bvecs);
|
| 1042 | if (!imu) {
|
| 1043 | io_cache_free(&ctx->node_cache, node);
|
| 1044 | return ERR_PTR(-ENOMEM);
|
| 1045 | }
|
| 1046 |
|
| 1047 | imu->ubuf = 0;
|
| 1048 | imu->len = total_bytes;
|
| 1049 | imu->folio_shift = PAGE_SHIFT;
|
| 1050 | imu->nr_bvecs = nr_bvecs;
|
| 1051 | refcount_set(&imu->refs, 1);
|
| 1052 | imu->release = release;
|
| 1053 | imu->priv = priv;
|
| 1054 | imu->dir = dir;
|
| 1055 | imu->flags = IO_REGBUF_F_KBUF;
|
| 1056 |
|
| 1057 | node->buf = imu;
|
| 1058 | data->nodes[index] = node;
|
| 1059 |
|
| 1060 | return imu;
|
| 1061 | }
|
| 1062 |
|
| 1063 | int io_buffer_register_request(struct io_uring_cmd *cmd, struct request *rq,
|
| 1064 | void (*release)(void *), unsigned int index,
|
| 1065 | unsigned int issue_flags)
|
| 1066 | {
|
| 1067 | struct io_ring_ctx *ctx = cmd_to_io_kiocb(cmd)->ctx;
|
| 1068 | struct req_iterator rq_iter;
|
| 1069 | struct io_mapped_ubuf *imu;
|
| 1070 | struct bio_vec bv;
|
| 1071 | |
| 1072 | * blk_rq_nr_phys_segments() may overestimate the number of bvecs
|
| 1073 | * but avoids needing to iterate over the bvecs
|
| 1074 |
|
| 1075 | unsigned int nr_bvecs = blk_rq_nr_phys_segments(rq);
|
| 1076 | unsigned int total_bytes = blk_rq_bytes(rq);
|
| 1077 | int ret = 0;
|
| 1078 |
|
| 1079 | io_ring_submit_lock(ctx, issue_flags);
|
| 1080 |
|
| 1081 | imu = io_kernel_buffer_init(ctx, nr_bvecs, total_bytes,
|
| 1082 | 1 << rq_data_dir(rq), release, rq, index);
|
| 1083 | if (IS_ERR(imu)) {
|
| 1084 | ret = PTR_ERR(imu);
|
| 1085 | goto unlock;
|
| 1086 | }
|
| 1087 |
|
| 1088 | nr_bvecs = 0;
|
| 1089 | rq_for_each_bvec(bv, rq, rq_iter)
|
| 1090 | imu->bvec[nr_bvecs++] = bv;
|
| 1091 | imu->nr_bvecs = nr_bvecs;
|
| 1092 |
|
| 1093 | unlock:
|
| 1094 | io_ring_submit_unlock(ctx, issue_flags);
|
| 1095 | return ret;
|
| 1096 | }
|
| 1097 | EXPORT_SYMBOL_GPL(io_buffer_register_request);
|
| 1098 |
|
| 1099 | |
| 1100 | * bvs is copied internally. caller may free it on return.
|
| 1101 |
|
| 1102 | int io_buffer_register_bvec(struct io_uring_cmd *cmd, const struct bio_vec *bvs,
|
| 1103 | unsigned int nr_bvecs, void (*release)(void *),
|
| 1104 | void *priv, u8 dir, unsigned int index,
|
| 1105 | unsigned int issue_flags)
|
| 1106 | {
|
| 1107 | struct io_ring_ctx *ctx = cmd_to_io_kiocb(cmd)->ctx;
|
| 1108 | struct io_mapped_ubuf *imu;
|
| 1109 | struct bio_vec *bvec;
|
| 1110 | unsigned int i, total_bytes = 0;
|
| 1111 | int ret = 0;
|
| 1112 |
|
| 1113 | for (i = 0; i < nr_bvecs; i++)
|
| 1114 | total_bytes += bvs[i].bv_len;
|
| 1115 |
|
| 1116 | io_ring_submit_lock(ctx, issue_flags);
|
| 1117 | imu = io_kernel_buffer_init(ctx, nr_bvecs, total_bytes, dir, release,
|
| 1118 | priv, index);
|
| 1119 | if (IS_ERR(imu)) {
|
| 1120 | ret = PTR_ERR(imu);
|
| 1121 | goto unlock;
|
| 1122 | }
|
| 1123 |
|
| 1124 | bvec = imu->bvec;
|
| 1125 | for (i = 0; i < nr_bvecs; i++)
|
| 1126 | bvec[i] = bvs[i];
|
| 1127 |
|
| 1128 | unlock:
|
| 1129 | io_ring_submit_unlock(ctx, issue_flags);
|
| 1130 | return ret;
|
| 1131 | }
|
| 1132 | EXPORT_SYMBOL_GPL(io_buffer_register_bvec);
|
| 1133 |
|
| 1134 | int io_buffer_unregister(struct io_uring_cmd *cmd, unsigned int index,
|
| 1135 | unsigned int issue_flags)
|
| 1136 | {
|
| 1137 | struct io_ring_ctx *ctx = cmd_to_io_kiocb(cmd)->ctx;
|
| 1138 | struct io_rsrc_data *data = &ctx->buf_table;
|
| 1139 | struct io_rsrc_node *node;
|
| 1140 | int ret = 0;
|
| 1141 |
|
| 1142 | io_ring_submit_lock(ctx, issue_flags);
|
| 1143 | if (index >= data->nr) {
|
| 1144 | ret = -EINVAL;
|
| 1145 | goto unlock;
|
| 1146 | }
|
| 1147 | index = array_index_nospec(index, data->nr);
|
| 1148 |
|
| 1149 | node = data->nodes[index];
|
| 1150 | if (!node) {
|
| 1151 | ret = -EINVAL;
|
| 1152 | goto unlock;
|
| 1153 | }
|
| 1154 | if (!(node->buf->flags & IO_REGBUF_F_KBUF)) {
|
| 1155 | ret = -EBUSY;
|
| 1156 | goto unlock;
|
| 1157 | }
|
| 1158 |
|
| 1159 | io_put_rsrc_node(ctx, node);
|
| 1160 | data->nodes[index] = NULL;
|
| 1161 | unlock:
|
| 1162 | io_ring_submit_unlock(ctx, issue_flags);
|
| 1163 | return ret;
|
| 1164 | }
|
| 1165 | EXPORT_SYMBOL_GPL(io_buffer_unregister);
|
| 1166 |
|
| 1167 | static int validate_fixed_range(u64 buf_addr, size_t len,
|
| 1168 | const struct io_mapped_ubuf *imu)
|
| 1169 | {
|
| 1170 | u64 buf_end;
|
| 1171 |
|
| 1172 | if (unlikely(check_add_overflow(buf_addr, (u64)len, &buf_end)))
|
| 1173 | return -EFAULT;
|
| 1174 |
|
| 1175 | if (unlikely(buf_addr < imu->ubuf || buf_end > (imu->ubuf + imu->len)))
|
| 1176 | return -EFAULT;
|
| 1177 | if (unlikely(len > MAX_RW_COUNT))
|
| 1178 | return -EFAULT;
|
| 1179 | return 0;
|
| 1180 | }
|
| 1181 |
|
| 1182 | static int io_import_kbuf(int ddir, struct iov_iter *iter,
|
| 1183 | struct io_mapped_ubuf *imu, size_t len, size_t offset)
|
| 1184 | {
|
| 1185 | size_t count = len + offset;
|
| 1186 |
|
| 1187 | iov_iter_bvec(iter, ddir, imu->bvec, imu->nr_bvecs, count);
|
| 1188 | iov_iter_advance(iter, offset);
|
| 1189 | return 0;
|
| 1190 | }
|
| 1191 |
|
| 1192 | static int io_import_fixed(int ddir, struct iov_iter *iter,
|
| 1193 | struct io_mapped_ubuf *imu,
|
| 1194 | u64 buf_addr, size_t len)
|
| 1195 | {
|
| 1196 | const struct bio_vec *bvec;
|
| 1197 | size_t folio_mask;
|
| 1198 | unsigned nr_segs;
|
| 1199 | size_t offset;
|
| 1200 | int ret;
|
| 1201 |
|
| 1202 | ret = validate_fixed_range(buf_addr, len, imu);
|
| 1203 | if (unlikely(ret))
|
| 1204 | return ret;
|
| 1205 | if (!(imu->dir & (1 << ddir)))
|
| 1206 | return -EFAULT;
|
| 1207 | if (unlikely(!len)) {
|
| 1208 | iov_iter_bvec(iter, ddir, NULL, 0, 0);
|
| 1209 | return 0;
|
| 1210 | }
|
| 1211 |
|
| 1212 | offset = buf_addr - imu->ubuf;
|
| 1213 |
|
| 1214 | if (imu->flags & IO_REGBUF_F_KBUF)
|
| 1215 | return io_import_kbuf(ddir, iter, imu, len, offset);
|
| 1216 |
|
| 1217 | |
| 1218 | * Don't use iov_iter_advance() here, as it's really slow for
|
| 1219 | * using the latter parts of a big fixed buffer - it iterates
|
| 1220 | * over each segment manually. We can cheat a bit here for user
|
| 1221 | * registered nodes, because we know that:
|
| 1222 | *
|
| 1223 | * 1) it's a BVEC iter, we set it up
|
| 1224 | * 2) all bvecs are the same in size, except potentially the
|
| 1225 | * first and last bvec
|
| 1226 |
|
| 1227 | folio_mask = (1UL << imu->folio_shift) - 1;
|
| 1228 | bvec = imu->bvec;
|
| 1229 | if (offset >= bvec->bv_len) {
|
| 1230 | unsigned long seg_skip;
|
| 1231 |
|
| 1232 |
|
| 1233 | offset -= bvec->bv_len;
|
| 1234 | seg_skip = 1 + (offset >> imu->folio_shift);
|
| 1235 | bvec += seg_skip;
|
| 1236 | offset &= folio_mask;
|
| 1237 | }
|
| 1238 | nr_segs = (offset + len + bvec->bv_offset + folio_mask) >> imu->folio_shift;
|
| 1239 | iov_iter_bvec(iter, ddir, bvec, nr_segs, len);
|
| 1240 | iter->iov_offset = offset;
|
| 1241 | return 0;
|
| 1242 | }
|
| 1243 |
|
| 1244 | inline struct io_rsrc_node *io_find_buf_node(struct io_kiocb *req,
|
| 1245 | unsigned issue_flags)
|
| 1246 | {
|
| 1247 | struct io_ring_ctx *ctx = req->ctx;
|
| 1248 | struct io_rsrc_node *node;
|
| 1249 |
|
| 1250 | if (req->flags & REQ_F_BUF_NODE)
|
| 1251 | return req->buf_node;
|
| 1252 | req->flags |= REQ_F_BUF_NODE;
|
| 1253 |
|
| 1254 | io_ring_submit_lock(ctx, issue_flags);
|
| 1255 | node = io_rsrc_node_lookup(&ctx->buf_table, req->buf_index);
|
| 1256 | if (node) {
|
| 1257 | node->refs++;
|
| 1258 | req->buf_node = node;
|
| 1259 | io_ring_submit_unlock(ctx, issue_flags);
|
| 1260 | return node;
|
| 1261 | }
|
| 1262 | req->flags &= ~REQ_F_BUF_NODE;
|
| 1263 | io_ring_submit_unlock(ctx, issue_flags);
|
| 1264 | return NULL;
|
| 1265 | }
|
| 1266 |
|
| 1267 | int io_import_reg_buf(struct io_kiocb *req, struct iov_iter *iter,
|
| 1268 | u64 buf_addr, size_t len, int ddir,
|
| 1269 | unsigned issue_flags)
|
| 1270 | {
|
| 1271 | struct io_rsrc_node *node;
|
| 1272 |
|
| 1273 | node = io_find_buf_node(req, issue_flags);
|
| 1274 | if (!node)
|
| 1275 | return -EFAULT;
|
| 1276 | return io_import_fixed(ddir, iter, node->buf, buf_addr, len);
|
| 1277 | }
|
| 1278 |
|
| 1279 | static int io_buffer_acct_cloned_hpages(struct io_ring_ctx *ctx,
|
| 1280 | struct io_mapped_ubuf *imu)
|
| 1281 | {
|
| 1282 | struct page *seen = NULL;
|
| 1283 | int i, ret = 0;
|
| 1284 |
|
| 1285 | if (imu->flags & IO_REGBUF_F_KBUF || !ctx->user)
|
| 1286 | return 0;
|
| 1287 |
|
| 1288 | for (i = 0; i < imu->nr_bvecs; i++) {
|
| 1289 | struct page *page = imu->bvec[i].bv_page;
|
| 1290 | struct page *hpage;
|
| 1291 | bool acct_new;
|
| 1292 |
|
| 1293 | if (!PageCompound(page))
|
| 1294 | continue;
|
| 1295 |
|
| 1296 | hpage = compound_head(page);
|
| 1297 | if (hpage == seen)
|
| 1298 | continue;
|
| 1299 | seen = hpage;
|
| 1300 |
|
| 1301 |
|
| 1302 | ret = hpage_acct_ref(ctx, hpage, &acct_new);
|
| 1303 | if (ret)
|
| 1304 | break;
|
| 1305 |
|
| 1306 | cond_resched();
|
| 1307 | }
|
| 1308 |
|
| 1309 | if (!ret)
|
| 1310 | return 0;
|
| 1311 |
|
| 1312 |
|
| 1313 | seen = NULL;
|
| 1314 | for (int j = 0; j < i; j++) {
|
| 1315 | struct page *p = imu->bvec[j].bv_page;
|
| 1316 | struct page *hp;
|
| 1317 |
|
| 1318 | if (!PageCompound(p))
|
| 1319 | continue;
|
| 1320 | hp = compound_head(p);
|
| 1321 | if (hp == seen)
|
| 1322 | continue;
|
| 1323 | seen = hp;
|
| 1324 | hpage_acct_unref(ctx, hp);
|
| 1325 | }
|
| 1326 | return ret;
|
| 1327 | }
|
| 1328 |
|
| 1329 |
|
| 1330 | static void lock_two_rings(struct io_ring_ctx *ctx1, struct io_ring_ctx *ctx2)
|
| 1331 | {
|
| 1332 | if (ctx1 > ctx2)
|
| 1333 | swap(ctx1, ctx2);
|
| 1334 | mutex_lock(&ctx1->uring_lock);
|
| 1335 | mutex_lock_nested(&ctx2->uring_lock, SINGLE_DEPTH_NESTING);
|
| 1336 | }
|
| 1337 |
|
| 1338 |
|
| 1339 | static int io_clone_buffers(struct io_ring_ctx *ctx, struct io_ring_ctx *src_ctx,
|
| 1340 | struct io_uring_clone_buffers *arg)
|
| 1341 | {
|
| 1342 | struct io_rsrc_data data;
|
| 1343 | int i, ret, off, nr;
|
| 1344 | unsigned int nbufs;
|
| 1345 |
|
| 1346 | lockdep_assert_held(&ctx->uring_lock);
|
| 1347 | lockdep_assert_held(&src_ctx->uring_lock);
|
| 1348 |
|
| 1349 | |
| 1350 | * Accounting state is shared between the two rings; that only works if
|
| 1351 | * both rings are accounted towards the same counters.
|
| 1352 |
|
| 1353 | if (ctx->user != src_ctx->user || ctx->mm_account != src_ctx->mm_account)
|
| 1354 | return -EINVAL;
|
| 1355 |
|
| 1356 |
|
| 1357 | if (!arg->nr && (arg->dst_off || arg->src_off))
|
| 1358 | return -EINVAL;
|
| 1359 |
|
| 1360 | if (ctx->buf_table.nr && !(arg->flags & IORING_REGISTER_DST_REPLACE))
|
| 1361 | return -EBUSY;
|
| 1362 |
|
| 1363 | nbufs = src_ctx->buf_table.nr;
|
| 1364 | if (!nbufs)
|
| 1365 | return -ENXIO;
|
| 1366 | if (!arg->nr)
|
| 1367 | arg->nr = nbufs;
|
| 1368 | else if (arg->nr > nbufs)
|
| 1369 | return -EINVAL;
|
| 1370 | else if (arg->nr > IORING_MAX_REG_BUFFERS)
|
| 1371 | return -EINVAL;
|
| 1372 | if (check_add_overflow(arg->nr, arg->src_off, &off) || off > nbufs)
|
| 1373 | return -EOVERFLOW;
|
| 1374 | if (check_add_overflow(arg->nr, arg->dst_off, &nbufs))
|
| 1375 | return -EOVERFLOW;
|
| 1376 | if (nbufs > IORING_MAX_REG_BUFFERS)
|
| 1377 | return -EINVAL;
|
| 1378 |
|
| 1379 | ret = io_rsrc_data_alloc(&data, max(nbufs, ctx->buf_table.nr));
|
| 1380 | if (ret)
|
| 1381 | return ret;
|
| 1382 |
|
| 1383 |
|
| 1384 | for (i = 0; i < min(arg->dst_off, ctx->buf_table.nr); i++) {
|
| 1385 | struct io_rsrc_node *node = ctx->buf_table.nodes[i];
|
| 1386 |
|
| 1387 | if (node) {
|
| 1388 | data.nodes[i] = node;
|
| 1389 | node->refs++;
|
| 1390 | }
|
| 1391 | }
|
| 1392 |
|
| 1393 | off = arg->dst_off;
|
| 1394 | i = arg->src_off;
|
| 1395 | nr = arg->nr;
|
| 1396 | while (nr--) {
|
| 1397 | struct io_rsrc_node *dst_node, *src_node;
|
| 1398 |
|
| 1399 | src_node = io_rsrc_node_lookup(&src_ctx->buf_table, i);
|
| 1400 | if (!src_node) {
|
| 1401 | dst_node = NULL;
|
| 1402 | } else {
|
| 1403 | dst_node = io_rsrc_node_alloc(ctx, IORING_RSRC_BUFFER);
|
| 1404 | if (!dst_node) {
|
| 1405 | io_rsrc_data_free(ctx, &data);
|
| 1406 | return -ENOMEM;
|
| 1407 | }
|
| 1408 |
|
| 1409 | refcount_inc(&src_node->buf->refs);
|
| 1410 | dst_node->buf = src_node->buf;
|
| 1411 |
|
| 1412 | ret = io_buffer_acct_cloned_hpages(ctx, src_node->buf);
|
| 1413 | if (ret) {
|
| 1414 | refcount_dec(&src_node->buf->refs);
|
| 1415 | io_cache_free(&ctx->node_cache, dst_node);
|
| 1416 | io_rsrc_data_free(ctx, &data);
|
| 1417 | return ret;
|
| 1418 | }
|
| 1419 | }
|
| 1420 | data.nodes[off++] = dst_node;
|
| 1421 | i++;
|
| 1422 | }
|
| 1423 |
|
| 1424 |
|
| 1425 | for (i = nbufs; i < ctx->buf_table.nr; i++) {
|
| 1426 | struct io_rsrc_node *node = ctx->buf_table.nodes[i];
|
| 1427 |
|
| 1428 | if (node) {
|
| 1429 | data.nodes[i] = node;
|
| 1430 | node->refs++;
|
| 1431 | }
|
| 1432 | }
|
| 1433 |
|
| 1434 | |
| 1435 | * If asked for replace, put the old table. data->nodes[] holds both
|
| 1436 | * old and new nodes at this point.
|
| 1437 |
|
| 1438 | if (arg->flags & IORING_REGISTER_DST_REPLACE)
|
| 1439 | io_rsrc_data_free(ctx, &ctx->buf_table);
|
| 1440 |
|
| 1441 | |
| 1442 | * ctx->buf_table must be empty now - either the contents are being
|
| 1443 | * replaced and we just freed the table, or the contents are being
|
| 1444 | * copied to a ring that does not have buffers yet (checked at function
|
| 1445 | * entry).
|
| 1446 |
|
| 1447 | WARN_ON_ONCE(ctx->buf_table.nr);
|
| 1448 | ctx->buf_table = data;
|
| 1449 | return 0;
|
| 1450 | }
|
| 1451 |
|
| 1452 | |
| 1453 | * Copy the registered buffers from the source ring whose file descriptor
|
| 1454 | * is given in the src_fd to the current ring. This is identical to registering
|
| 1455 | * the buffers with ctx, except faster as mappings already exist.
|
| 1456 | *
|
| 1457 | * Since the memory is already accounted once, don't account it again.
|
| 1458 |
|
| 1459 | int io_register_clone_buffers(struct io_ring_ctx *ctx, void __user *arg)
|
| 1460 | {
|
| 1461 | struct io_uring_clone_buffers buf;
|
| 1462 | struct io_ring_ctx *src_ctx;
|
| 1463 | bool registered_src;
|
| 1464 | struct file *file;
|
| 1465 | int ret;
|
| 1466 |
|
| 1467 | if (copy_from_user(&buf, arg, sizeof(buf)))
|
| 1468 | return -EFAULT;
|
| 1469 | if (buf.flags & ~(IORING_REGISTER_SRC_REGISTERED|IORING_REGISTER_DST_REPLACE))
|
| 1470 | return -EINVAL;
|
| 1471 | if (!(buf.flags & IORING_REGISTER_DST_REPLACE) && ctx->buf_table.nr)
|
| 1472 | return -EBUSY;
|
| 1473 | if (memchr_inv(buf.pad, 0, sizeof(buf.pad)))
|
| 1474 | return -EINVAL;
|
| 1475 |
|
| 1476 | registered_src = (buf.flags & IORING_REGISTER_SRC_REGISTERED) != 0;
|
| 1477 | file = io_uring_ctx_get_file(buf.src_fd, registered_src);
|
| 1478 | if (IS_ERR(file))
|
| 1479 | return PTR_ERR(file);
|
| 1480 |
|
| 1481 | src_ctx = file->private_data;
|
| 1482 | if (src_ctx != ctx) {
|
| 1483 | mutex_unlock(&ctx->uring_lock);
|
| 1484 | lock_two_rings(ctx, src_ctx);
|
| 1485 |
|
| 1486 | if (src_ctx->submitter_task &&
|
| 1487 | src_ctx->submitter_task != current) {
|
| 1488 | ret = -EEXIST;
|
| 1489 | goto out;
|
| 1490 | }
|
| 1491 | }
|
| 1492 |
|
| 1493 | ret = io_clone_buffers(ctx, src_ctx, &buf);
|
| 1494 |
|
| 1495 | out:
|
| 1496 | if (src_ctx != ctx)
|
| 1497 | mutex_unlock(&src_ctx->uring_lock);
|
| 1498 |
|
| 1499 | if (!registered_src)
|
| 1500 | fput(file);
|
| 1501 | return ret;
|
| 1502 | }
|
| 1503 |
|
| 1504 | void io_vec_free(struct iou_vec *iv)
|
| 1505 | {
|
| 1506 | if (!iv->iovec)
|
| 1507 | return;
|
| 1508 | kfree(iv->iovec);
|
| 1509 | iv->iovec = NULL;
|
| 1510 | iv->nr = 0;
|
| 1511 | }
|
| 1512 |
|
| 1513 | int io_vec_realloc(struct iou_vec *iv, unsigned nr_entries)
|
| 1514 | {
|
| 1515 | gfp_t gfp = GFP_KERNEL_ACCOUNT | __GFP_NOWARN;
|
| 1516 | struct iovec *iov;
|
| 1517 |
|
| 1518 | iov = kmalloc_objs(iov[0], nr_entries, gfp);
|
| 1519 | if (!iov)
|
| 1520 | return -ENOMEM;
|
| 1521 |
|
| 1522 | io_vec_free(iv);
|
| 1523 | iv->iovec = iov;
|
| 1524 | iv->nr = nr_entries;
|
| 1525 | return 0;
|
| 1526 | }
|
| 1527 |
|
| 1528 | static int io_vec_fill_bvec(int ddir, struct iov_iter *iter,
|
| 1529 | struct io_mapped_ubuf *imu,
|
| 1530 | struct iovec *iovec, unsigned nr_iovs,
|
| 1531 | struct iou_vec *vec)
|
| 1532 | {
|
| 1533 | unsigned long folio_size = 1UL << imu->folio_shift;
|
| 1534 | unsigned long folio_mask = folio_size - 1;
|
| 1535 | struct bio_vec *res_bvec = vec->bvec;
|
| 1536 | size_t total_len = 0;
|
| 1537 | unsigned bvec_idx = 0;
|
| 1538 | unsigned iov_idx;
|
| 1539 |
|
| 1540 | for (iov_idx = 0; iov_idx < nr_iovs; iov_idx++) {
|
| 1541 | size_t iov_len = iovec[iov_idx].iov_len;
|
| 1542 | u64 buf_addr = (u64)(uintptr_t)iovec[iov_idx].iov_base;
|
| 1543 | struct bio_vec *src_bvec;
|
| 1544 | size_t offset;
|
| 1545 | int ret;
|
| 1546 |
|
| 1547 | ret = validate_fixed_range(buf_addr, iov_len, imu);
|
| 1548 | if (unlikely(ret))
|
| 1549 | return ret;
|
| 1550 |
|
| 1551 | if (unlikely(!iov_len))
|
| 1552 | return -EFAULT;
|
| 1553 | if (unlikely(check_add_overflow(total_len, iov_len, &total_len)))
|
| 1554 | return -EOVERFLOW;
|
| 1555 |
|
| 1556 | offset = buf_addr - imu->ubuf;
|
| 1557 | |
| 1558 | * Only the first bvec can have non zero bv_offset, account it
|
| 1559 | * here and work with full folios below.
|
| 1560 |
|
| 1561 | offset += imu->bvec[0].bv_offset;
|
| 1562 |
|
| 1563 | src_bvec = imu->bvec + (offset >> imu->folio_shift);
|
| 1564 | offset &= folio_mask;
|
| 1565 |
|
| 1566 | for (; iov_len; offset = 0, bvec_idx++, src_bvec++) {
|
| 1567 | size_t seg_size = min_t(size_t, iov_len,
|
| 1568 | folio_size - offset);
|
| 1569 |
|
| 1570 | bvec_set_page(&res_bvec[bvec_idx],
|
| 1571 | src_bvec->bv_page, seg_size, offset);
|
| 1572 | iov_len -= seg_size;
|
| 1573 | }
|
| 1574 | }
|
| 1575 | if (total_len > MAX_RW_COUNT)
|
| 1576 | return -EINVAL;
|
| 1577 |
|
| 1578 | iov_iter_bvec(iter, ddir, res_bvec, bvec_idx, total_len);
|
| 1579 | return 0;
|
| 1580 | }
|
| 1581 |
|
| 1582 | static int io_estimate_bvec_size(struct iovec *iov, unsigned nr_iovs,
|
| 1583 | struct io_mapped_ubuf *imu)
|
| 1584 | {
|
| 1585 | unsigned shift = imu->folio_shift;
|
| 1586 | size_t max_segs = 0;
|
| 1587 | unsigned i;
|
| 1588 |
|
| 1589 | for (i = 0; i < nr_iovs; i++) {
|
| 1590 | max_segs += (iov[i].iov_len >> shift) + 2;
|
| 1591 | if (max_segs > INT_MAX)
|
| 1592 | return -EOVERFLOW;
|
| 1593 | }
|
| 1594 | return max_segs;
|
| 1595 | }
|
| 1596 |
|
| 1597 | static int io_vec_fill_kern_bvec(int ddir, struct iov_iter *iter,
|
| 1598 | struct io_mapped_ubuf *imu,
|
| 1599 | struct iovec *iovec, unsigned nr_iovs,
|
| 1600 | struct iou_vec *vec)
|
| 1601 | {
|
| 1602 | const struct bio_vec *src_bvec = imu->bvec;
|
| 1603 | struct bio_vec *res_bvec = vec->bvec;
|
| 1604 | unsigned res_idx = 0;
|
| 1605 | size_t total_len = 0;
|
| 1606 | unsigned iov_idx;
|
| 1607 |
|
| 1608 | for (iov_idx = 0; iov_idx < nr_iovs; iov_idx++) {
|
| 1609 | size_t offset = (size_t)(uintptr_t)iovec[iov_idx].iov_base;
|
| 1610 | size_t iov_len = iovec[iov_idx].iov_len;
|
| 1611 | struct bvec_iter bi = {
|
| 1612 | .bi_size = offset + iov_len,
|
| 1613 | };
|
| 1614 | struct bio_vec bv;
|
| 1615 |
|
| 1616 | bvec_iter_advance(src_bvec, &bi, offset);
|
| 1617 | for_each_mp_bvec(bv, src_bvec, bi, bi)
|
| 1618 | res_bvec[res_idx++] = bv;
|
| 1619 | total_len += iov_len;
|
| 1620 | }
|
| 1621 | iov_iter_bvec(iter, ddir, res_bvec, res_idx, total_len);
|
| 1622 | return 0;
|
| 1623 | }
|
| 1624 |
|
| 1625 | static int iov_kern_bvec_size(const struct iovec *iov,
|
| 1626 | const struct io_mapped_ubuf *imu,
|
| 1627 | unsigned int *nr_seg)
|
| 1628 | {
|
| 1629 | size_t offset = (size_t)(uintptr_t)iov->iov_base;
|
| 1630 | const struct bio_vec *bvec = imu->bvec;
|
| 1631 | int start = 0, i = 0;
|
| 1632 | size_t off = 0;
|
| 1633 | int ret;
|
| 1634 |
|
| 1635 | ret = validate_fixed_range(offset, iov->iov_len, imu);
|
| 1636 | if (unlikely(ret))
|
| 1637 | return ret;
|
| 1638 |
|
| 1639 | for (i = 0; off < offset + iov->iov_len && i < imu->nr_bvecs;
|
| 1640 | off += bvec[i].bv_len, i++) {
|
| 1641 | if (offset >= off && offset < off + bvec[i].bv_len)
|
| 1642 | start = i;
|
| 1643 | }
|
| 1644 | *nr_seg = i - start;
|
| 1645 | return 0;
|
| 1646 | }
|
| 1647 |
|
| 1648 | static int io_kern_bvec_size(struct iovec *iov, unsigned nr_iovs,
|
| 1649 | struct io_mapped_ubuf *imu, unsigned *nr_segs)
|
| 1650 | {
|
| 1651 | unsigned max_segs = 0;
|
| 1652 | size_t total_len = 0;
|
| 1653 | unsigned i;
|
| 1654 | int ret;
|
| 1655 |
|
| 1656 | *nr_segs = 0;
|
| 1657 | for (i = 0; i < nr_iovs; i++) {
|
| 1658 | if (unlikely(!iov[i].iov_len))
|
| 1659 | return -EFAULT;
|
| 1660 | if (unlikely(check_add_overflow(total_len, iov[i].iov_len,
|
| 1661 | &total_len)))
|
| 1662 | return -EOVERFLOW;
|
| 1663 | ret = iov_kern_bvec_size(&iov[i], imu, &max_segs);
|
| 1664 | if (unlikely(ret))
|
| 1665 | return ret;
|
| 1666 | *nr_segs += max_segs;
|
| 1667 | }
|
| 1668 | if (total_len > MAX_RW_COUNT)
|
| 1669 | return -EINVAL;
|
| 1670 | return 0;
|
| 1671 | }
|
| 1672 |
|
| 1673 | int io_import_reg_vec(int ddir, struct iov_iter *iter,
|
| 1674 | struct io_kiocb *req, struct iou_vec *vec,
|
| 1675 | unsigned nr_iovs, unsigned issue_flags)
|
| 1676 | {
|
| 1677 | struct io_rsrc_node *node;
|
| 1678 | struct io_mapped_ubuf *imu;
|
| 1679 | unsigned iovec_off;
|
| 1680 | struct iovec *iov;
|
| 1681 | unsigned nr_segs;
|
| 1682 |
|
| 1683 | node = io_find_buf_node(req, issue_flags);
|
| 1684 | if (!node)
|
| 1685 | return -EFAULT;
|
| 1686 | imu = node->buf;
|
| 1687 | if (!(imu->dir & (1 << ddir)))
|
| 1688 | return -EFAULT;
|
| 1689 |
|
| 1690 | iovec_off = vec->nr - nr_iovs;
|
| 1691 | iov = vec->iovec + iovec_off;
|
| 1692 |
|
| 1693 | if (imu->flags & IO_REGBUF_F_KBUF) {
|
| 1694 | int ret = io_kern_bvec_size(iov, nr_iovs, imu, &nr_segs);
|
| 1695 |
|
| 1696 | if (unlikely(ret))
|
| 1697 | return ret;
|
| 1698 | } else {
|
| 1699 | int ret = io_estimate_bvec_size(iov, nr_iovs, imu);
|
| 1700 |
|
| 1701 | if (ret < 0)
|
| 1702 | return ret;
|
| 1703 | nr_segs = ret;
|
| 1704 | }
|
| 1705 |
|
| 1706 | if (sizeof(struct bio_vec) > sizeof(struct iovec)) {
|
| 1707 | size_t bvec_bytes;
|
| 1708 |
|
| 1709 | if (check_mul_overflow((size_t)nr_segs, sizeof(struct bio_vec),
|
| 1710 | &bvec_bytes) ||
|
| 1711 | check_add_overflow(bvec_bytes, sizeof(*iov) - 1,
|
| 1712 | &bvec_bytes))
|
| 1713 | return -EOVERFLOW;
|
| 1714 | nr_segs = bvec_bytes / sizeof(*iov);
|
| 1715 | nr_segs += nr_iovs;
|
| 1716 | }
|
| 1717 |
|
| 1718 | if (nr_segs > vec->nr) {
|
| 1719 | struct iou_vec tmp_vec = {};
|
| 1720 | int ret;
|
| 1721 |
|
| 1722 | ret = io_vec_realloc(&tmp_vec, nr_segs);
|
| 1723 | if (ret)
|
| 1724 | return ret;
|
| 1725 |
|
| 1726 | iovec_off = tmp_vec.nr - nr_iovs;
|
| 1727 | memcpy(tmp_vec.iovec + iovec_off, iov, sizeof(*iov) * nr_iovs);
|
| 1728 | io_vec_free(vec);
|
| 1729 |
|
| 1730 | *vec = tmp_vec;
|
| 1731 | iov = vec->iovec + iovec_off;
|
| 1732 | req->flags |= REQ_F_NEED_CLEANUP;
|
| 1733 | }
|
| 1734 |
|
| 1735 | if (imu->flags & IO_REGBUF_F_KBUF)
|
| 1736 | return io_vec_fill_kern_bvec(ddir, iter, imu, iov, nr_iovs, vec);
|
| 1737 |
|
| 1738 | return io_vec_fill_bvec(ddir, iter, imu, iov, nr_iovs, vec);
|
| 1739 | }
|
| 1740 |
|
| 1741 | int io_prep_reg_iovec(struct io_kiocb *req, struct iou_vec *iv,
|
| 1742 | const struct iovec __user *uvec, size_t uvec_segs)
|
| 1743 | {
|
| 1744 | struct iovec *iov;
|
| 1745 | int iovec_off, ret;
|
| 1746 | void *res;
|
| 1747 |
|
| 1748 | if (uvec_segs > iv->nr) {
|
| 1749 | ret = io_vec_realloc(iv, uvec_segs);
|
| 1750 | if (ret)
|
| 1751 | return ret;
|
| 1752 | req->flags |= REQ_F_NEED_CLEANUP;
|
| 1753 | }
|
| 1754 |
|
| 1755 |
|
| 1756 | iovec_off = iv->nr - uvec_segs;
|
| 1757 | iov = iv->iovec + iovec_off;
|
| 1758 | res = iovec_from_user(uvec, uvec_segs, uvec_segs, iov,
|
| 1759 | io_is_compat(req->ctx));
|
| 1760 | if (IS_ERR(res))
|
| 1761 | return PTR_ERR(res);
|
| 1762 |
|
| 1763 | req->flags |= REQ_F_IMPORT_BUFFER;
|
| 1764 | return 0;
|
| 1765 | }
|