projects/linux·blob master · refs

448 lines · 10.5 KiB

1// SPDX-License-Identifier: GPL-2.0
2#include <linux/kernel.h>
3#include <linux/errno.h>
4#include <linux/fs.h>
5#include <linux/file.h>
6#include <linux/fdtable.h>
7#include <linux/fsnotify.h>
8#include <linux/namei.h>
9#include <linux/pipe_fs_i.h>
10#include <linux/watch_queue.h>
11#include <linux/io_uring.h>
12
13#include <uapi/linux/io_uring.h>
14
15#include "../fs/internal.h"
16
17#include "filetable.h"
18#include "io_uring.h"
19#include "rsrc.h"
20#include "openclose.h"
21
22struct io_open {
23 struct file *file;
24 int dfd;
25 u32 file_slot;
26 struct delayed_filename filename;
27 struct open_how how;
28 unsigned long nofile;
29};
30
31struct io_close {
32 struct file *file;
33 int fd;
34 u32 file_slot;
35};
36
37struct io_fixed_install {
38 struct file *file;
39 unsigned int o_flags;
40};
41
42static bool io_openat_force_async(struct io_open *open)
43{
44 /*
45 * Don't bother trying for O_TRUNC, O_CREAT, or O_TMPFILE open,
46 * it'll always -EAGAIN. Note that we test for __O_TMPFILE because
47 * O_TMPFILE includes O_DIRECTORY, which isn't a flag we need to force
48 * async for.
49 */
50 return open->how.flags & (O_TRUNC | O_CREAT | __O_TMPFILE);
51}
52
53static int __io_openat_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
54{
55 struct io_open *open = io_kiocb_to_cmd(req, struct io_open);
56 const char __user *fname;
57 int ret;
58
59 if (unlikely(sqe->buf_index))
60 return -EINVAL;
61 if (unlikely(req->flags & REQ_F_FIXED_FILE))
62 return -EBADF;
63
64 /* open.how should be already initialised */
65 if (!(open->how.flags & O_PATH) && force_o_largefile())
66 open->how.flags |= O_LARGEFILE;
67
68 open->dfd = READ_ONCE(sqe->fd);
69 fname = u64_to_user_ptr(READ_ONCE(sqe->addr));
70 ret = delayed_getname(&open->filename, fname);
71 if (unlikely(ret))
72 return ret;
73 req->flags |= REQ_F_NEED_CLEANUP;
74
75 open->file_slot = READ_ONCE(sqe->file_index);
76 if (open->file_slot && (open->how.flags & O_CLOEXEC))
77 return -EINVAL;
78
79 open->nofile = rlimit(RLIMIT_NOFILE);
80 if (io_openat_force_async(open))
81 req->flags |= REQ_F_FORCE_ASYNC;
82 return 0;
83}
84
85void io_openat_bpf_populate(struct io_uring_bpf_ctx *bctx, struct io_kiocb *req)
86{
87 struct io_open *open = io_kiocb_to_cmd(req, struct io_open);
88
89 bctx->open.flags = open->how.flags;
90 bctx->open.mode = open->how.mode;
91 bctx->open.resolve = open->how.resolve;
92}
93
94int io_openat_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
95{
96 struct io_open *open = io_kiocb_to_cmd(req, struct io_open);
97 u64 mode = READ_ONCE(sqe->len);
98 u64 flags = READ_ONCE(sqe->open_flags);
99
100 open->how = build_open_how(flags, mode);
101 return __io_openat_prep(req, sqe);
102}
103
104int io_openat2_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
105{
106 struct io_open *open = io_kiocb_to_cmd(req, struct io_open);
107 struct open_how __user *how;
108 size_t len;
109 int ret;
110
111 how = u64_to_user_ptr(READ_ONCE(sqe->addr2));
112 len = READ_ONCE(sqe->len);
113 if (len < OPEN_HOW_SIZE_VER0)
114 return -EINVAL;
115
116 ret = copy_struct_from_user(&open->how, sizeof(open->how), how, len);
117 if (ret)
118 return ret;
119
120 return __io_openat_prep(req, sqe);
121}
122
123int io_openat2(struct io_kiocb *req, unsigned int issue_flags)
124{
125 struct io_open *open = io_kiocb_to_cmd(req, struct io_open);
126 struct open_flags op;
127 struct file *file;
128 bool resolve_nonblock, nonblock_set;
129 bool fixed = !!open->file_slot;
130 CLASS(filename_complete_delayed, name)(&open->filename);
131 int ret;
132
133 ret = build_open_flags(&open->how, &op);
134 if (ret)
135 goto err;
136 nonblock_set = op.open_flag & O_NONBLOCK;
137 resolve_nonblock = open->how.resolve & RESOLVE_CACHED;
138 if (issue_flags & IO_URING_F_NONBLOCK) {
139 WARN_ON_ONCE(io_openat_force_async(open));
140 op.lookup_flags |= LOOKUP_CACHED;
141 op.open_flag |= O_NONBLOCK;
142 }
143
144 if (!fixed) {
145 ret = __get_unused_fd_flags(open->how.flags, open->nofile);
146 if (ret < 0)
147 goto err;
148 }
149
150 file = do_file_open(open->dfd, name, &op);
151 if (IS_ERR(file)) {
152 /*
153 * We could hang on to this 'fd' on retrying, but seems like
154 * marginal gain for something that is now known to be a slower
155 * path. So just put it, and we'll get a new one when we retry.
156 */
157 if (!fixed)
158 put_unused_fd(ret);
159
160 ret = PTR_ERR(file);
161 /* only retry if RESOLVE_CACHED wasn't already set by application */
162 if (ret == -EAGAIN && !resolve_nonblock &&
163 (issue_flags & IO_URING_F_NONBLOCK)) {
164 ret = putname_to_delayed(&open->filename,
165 no_free_ptr(name));
166 if (likely(!ret))
167 return -EAGAIN;
168 }
169 goto err;
170 }
171
172 if ((issue_flags & IO_URING_F_NONBLOCK) && !nonblock_set)
173 file->f_flags &= ~O_NONBLOCK;
174
175 if (!fixed)
176 fd_install(ret, file);
177 else
178 ret = io_fixed_fd_install(req, issue_flags, file,
179 open->file_slot);
180err:
181 req->flags &= ~REQ_F_NEED_CLEANUP;
182 if (ret < 0)
183 req_set_fail(req);
184 io_req_set_res(req, ret, 0);
185 return IOU_COMPLETE;
186}
187
188int io_openat(struct io_kiocb *req, unsigned int issue_flags)
189{
190 return io_openat2(req, issue_flags);
191}
192
193void io_open_cleanup(struct io_kiocb *req)
194{
195 struct io_open *open = io_kiocb_to_cmd(req, struct io_open);
196
197 dismiss_delayed_filename(&open->filename);
198}
199
200int __io_close_fixed(struct io_ring_ctx *ctx, unsigned int issue_flags,
201 unsigned int offset)
202{
203 int ret;
204
205 io_ring_submit_lock(ctx, issue_flags);
206 ret = io_fixed_fd_remove(ctx, offset);
207 io_ring_submit_unlock(ctx, issue_flags);
208
209 return ret;
210}
211
212static inline int io_close_fixed(struct io_kiocb *req, unsigned int issue_flags)
213{
214 struct io_close *close = io_kiocb_to_cmd(req, struct io_close);
215
216 return __io_close_fixed(req->ctx, issue_flags, close->file_slot - 1);
217}
218
219int io_close_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
220{
221 struct io_close *close = io_kiocb_to_cmd(req, struct io_close);
222
223 if (sqe->off || sqe->addr || sqe->len || sqe->rw_flags || sqe->buf_index)
224 return -EINVAL;
225 if (req->flags & REQ_F_FIXED_FILE)
226 return -EBADF;
227
228 close->fd = READ_ONCE(sqe->fd);
229 close->file_slot = READ_ONCE(sqe->file_index);
230 if (close->file_slot && close->fd)
231 return -EINVAL;
232
233 return 0;
234}
235
236int io_close(struct io_kiocb *req, unsigned int issue_flags)
237{
238 struct files_struct *files = current->files;
239 struct io_close *close = io_kiocb_to_cmd(req, struct io_close);
240 struct file *file;
241 int ret = -EBADF;
242
243 if (close->file_slot) {
244 ret = io_close_fixed(req, issue_flags);
245 goto err;
246 }
247
248 spin_lock(&files->file_lock);
249 file = files_lookup_fd_locked(files, close->fd);
250 if (!file || io_is_uring_fops(file)) {
251 spin_unlock(&files->file_lock);
252 goto err;
253 }
254
255 /* if the file has a flush method, be safe and punt to async */
256 if (file->f_op->flush && (issue_flags & IO_URING_F_NONBLOCK)) {
257 spin_unlock(&files->file_lock);
258 return -EAGAIN;
259 }
260
261 file = file_close_fd_locked(files, close->fd);
262 spin_unlock(&files->file_lock);
263 if (!file)
264 goto err;
265
266 /* No ->flush() or already async, safely close from here */
267 ret = filp_close(file, current->files);
268err:
269 if (ret < 0)
270 req_set_fail(req);
271 io_req_set_res(req, ret, 0);
272 return IOU_COMPLETE;
273}
274
275int io_install_fixed_fd_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
276{
277 struct io_fixed_install *ifi;
278 unsigned int flags;
279
280 if (sqe->off || sqe->addr || sqe->len || sqe->buf_index ||
281 sqe->splice_fd_in || sqe->addr3)
282 return -EINVAL;
283
284 /* must be a fixed file */
285 if (!(req->flags & REQ_F_FIXED_FILE))
286 return -EBADF;
287
288 flags = READ_ONCE(sqe->install_fd_flags);
289 if (flags & ~IORING_FIXED_FD_NO_CLOEXEC)
290 return -EINVAL;
291
292 /* ensure the task's creds are used when installing/receiving fds */
293 if (req->flags & REQ_F_CREDS)
294 return -EPERM;
295
296 /* default to O_CLOEXEC, disable if IORING_FIXED_FD_NO_CLOEXEC is set */
297 ifi = io_kiocb_to_cmd(req, struct io_fixed_install);
298 ifi->o_flags = O_CLOEXEC;
299 if (flags & IORING_FIXED_FD_NO_CLOEXEC)
300 ifi->o_flags = 0;
301
302 return 0;
303}
304
305int io_install_fixed_fd(struct io_kiocb *req, unsigned int issue_flags)
306{
307 struct io_fixed_install *ifi;
308 int ret;
309
310 ifi = io_kiocb_to_cmd(req, struct io_fixed_install);
311 ret = receive_fd(req->file, NULL, ifi->o_flags);
312 if (ret < 0)
313 req_set_fail(req);
314 io_req_set_res(req, ret, 0);
315 return IOU_COMPLETE;
316}
317
318struct io_pipe {
319 struct file *file;
320 int __user *fds;
321 int flags;
322 int file_slot;
323 unsigned long nofile;
324};
325
326int io_pipe_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
327{
328 struct io_pipe *p = io_kiocb_to_cmd(req, struct io_pipe);
329
330 if (sqe->fd || sqe->off || sqe->addr3)
331 return -EINVAL;
332
333 p->fds = u64_to_user_ptr(READ_ONCE(sqe->addr));
334 p->flags = READ_ONCE(sqe->pipe_flags);
335 if (p->flags & ~(O_CLOEXEC | O_NONBLOCK | O_DIRECT | O_NOTIFICATION_PIPE))
336 return -EINVAL;
337
338 p->file_slot = READ_ONCE(sqe->file_index);
339 p->nofile = rlimit(RLIMIT_NOFILE);
340 return 0;
341}
342
343static int io_pipe_fixed(struct io_kiocb *req, struct file **files,
344 unsigned int issue_flags)
345{
346 struct io_pipe *p = io_kiocb_to_cmd(req, struct io_pipe);
347 struct io_ring_ctx *ctx = req->ctx;
348 bool alloc_slot;
349 int ret, fds[2] = { -1, -1 };
350 int slot = p->file_slot;
351
352 if (p->flags & O_CLOEXEC)
353 return -EINVAL;
354
355 alloc_slot = slot == IORING_FILE_INDEX_ALLOC;
356
357 io_ring_submit_lock(ctx, issue_flags);
358
359 ret = __io_fixed_fd_install(ctx, files[0], slot);
360 if (ret < 0)
361 goto err;
362 fds[0] = alloc_slot ? ret : slot - 1;
363 files[0] = NULL;
364
365 /*
366 * If a specific slot is given, next one will be used for
367 * the write side.
368 */
369 if (!alloc_slot)
370 slot++;
371
372 ret = __io_fixed_fd_install(ctx, files[1], slot);
373 if (ret < 0)
374 goto err;
375 fds[1] = alloc_slot ? ret : slot - 1;
376 files[1] = NULL;
377
378 io_ring_submit_unlock(ctx, issue_flags);
379
380 if (!copy_to_user(p->fds, fds, sizeof(fds)))
381 return 0;
382
383 ret = -EFAULT;
384 io_ring_submit_lock(ctx, issue_flags);
385err:
386 if (fds[0] != -1)
387 io_fixed_fd_remove(ctx, fds[0]);
388 if (fds[1] != -1)
389 io_fixed_fd_remove(ctx, fds[1]);
390 io_ring_submit_unlock(ctx, issue_flags);
391 return ret;
392}
393
394static int io_pipe_fd(struct io_kiocb *req, struct file **files)
395{
396 struct io_pipe *p = io_kiocb_to_cmd(req, struct io_pipe);
397 int ret, fds[2] = { -1, -1 };
398
399 ret = __get_unused_fd_flags(p->flags, p->nofile);
400 if (ret < 0)
401 goto err;
402 fds[0] = ret;
403
404 ret = __get_unused_fd_flags(p->flags, p->nofile);
405 if (ret < 0)
406 goto err;
407 fds[1] = ret;
408
409 if (!copy_to_user(p->fds, fds, sizeof(fds))) {
410 fd_install(fds[0], files[0]);
411 fd_install(fds[1], files[1]);
412 return 0;
413 }
414 ret = -EFAULT;
415err:
416 if (fds[0] != -1)
417 put_unused_fd(fds[0]);
418 if (fds[1] != -1)
419 put_unused_fd(fds[1]);
420 return ret;
421}
422
423int io_pipe(struct io_kiocb *req, unsigned int issue_flags)
424{
425 struct io_pipe *p = io_kiocb_to_cmd(req, struct io_pipe);
426 struct file *files[2];
427 int ret;
428
429 ret = create_pipe_files(files, p->flags);
430 if (ret)
431 return ret;
432
433 if (!!p->file_slot)
434 ret = io_pipe_fixed(req, files, issue_flags);
435 else
436 ret = io_pipe_fd(req, files);
437
438 io_req_set_res(req, ret, 0);
439 if (!ret)
440 return IOU_COMPLETE;
441
442 req_set_fail(req);
443 if (files[0])
444 fput(files[0]);
445 if (files[1])
446 fput(files[1]);
447 return ret;
448}