projects/linux·blob master · refs

149 lines · 3.6 KiB

1// SPDX-License-Identifier: GPL-2.0
2#include <linux/kernel.h>
3#include <linux/errno.h>
4#include <linux/fs.h>
5#include <linux/file.h>
6#include <linux/mm.h>
7#include <linux/slab.h>
8#include <linux/namei.h>
9#include <linux/io_uring.h>
10#include <linux/splice.h>
11
12#include <uapi/linux/io_uring.h>
13
14#include "filetable.h"
15#include "io_uring.h"
16#include "splice.h"
17
18struct io_splice {
19 struct file *file_out;
20 loff_t off_out;
21 loff_t off_in;
22 u64 len;
23 int splice_fd_in;
24 unsigned int flags;
25 struct io_rsrc_node *rsrc_node;
26};
27
28static int __io_splice_prep(struct io_kiocb *req,
29 const struct io_uring_sqe *sqe)
30{
31 struct io_splice *sp = io_kiocb_to_cmd(req, struct io_splice);
32 unsigned int valid_flags = SPLICE_F_FD_IN_FIXED | SPLICE_F_ALL;
33
34 sp->len = READ_ONCE(sqe->len);
35 sp->flags = READ_ONCE(sqe->splice_flags);
36 if (unlikely(sp->flags & ~valid_flags))
37 return -EINVAL;
38 sp->splice_fd_in = READ_ONCE(sqe->splice_fd_in);
39 sp->rsrc_node = NULL;
40 req->flags |= REQ_F_FORCE_ASYNC;
41 return 0;
42}
43
44int io_tee_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
45{
46 if (READ_ONCE(sqe->splice_off_in) || READ_ONCE(sqe->off))
47 return -EINVAL;
48 return __io_splice_prep(req, sqe);
49}
50
51void io_splice_cleanup(struct io_kiocb *req)
52{
53 struct io_splice *sp = io_kiocb_to_cmd(req, struct io_splice);
54
55 if (sp->rsrc_node)
56 io_put_rsrc_node(req->ctx, sp->rsrc_node);
57}
58
59static struct file *io_splice_get_file(struct io_kiocb *req,
60 unsigned int issue_flags)
61{
62 struct io_splice *sp = io_kiocb_to_cmd(req, struct io_splice);
63 struct io_ring_ctx *ctx = req->ctx;
64 struct io_rsrc_node *node;
65 struct file *file = NULL;
66
67 if (!(sp->flags & SPLICE_F_FD_IN_FIXED))
68 return io_file_get_normal(req, sp->splice_fd_in);
69
70 io_ring_submit_lock(ctx, issue_flags);
71 node = io_rsrc_node_lookup(&ctx->file_table.data, sp->splice_fd_in);
72 if (node) {
73 node->refs++;
74 sp->rsrc_node = node;
75 file = io_slot_file(node);
76 req->flags |= REQ_F_NEED_CLEANUP;
77 }
78 io_ring_submit_unlock(ctx, issue_flags);
79 return file;
80}
81
82int io_tee(struct io_kiocb *req, unsigned int issue_flags)
83{
84 struct io_splice *sp = io_kiocb_to_cmd(req, struct io_splice);
85 struct file *out = sp->file_out;
86 unsigned int flags = sp->flags & ~SPLICE_F_FD_IN_FIXED;
87 struct file *in;
88 ssize_t ret = 0;
89
90 WARN_ON_ONCE(issue_flags & IO_URING_F_NONBLOCK);
91
92 in = io_splice_get_file(req, issue_flags);
93 if (!in) {
94 ret = -EBADF;
95 goto done;
96 }
97
98 if (sp->len)
99 ret = do_tee(in, out, sp->len, flags);
100
101 if (!(sp->flags & SPLICE_F_FD_IN_FIXED))
102 fput(in);
103done:
104 if (ret != sp->len)
105 req_set_fail(req);
106 io_req_set_res(req, ret, 0);
107 return IOU_COMPLETE;
108}
109
110int io_splice_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
111{
112 struct io_splice *sp = io_kiocb_to_cmd(req, struct io_splice);
113
114 sp->off_in = READ_ONCE(sqe->splice_off_in);
115 sp->off_out = READ_ONCE(sqe->off);
116 return __io_splice_prep(req, sqe);
117}
118
119int io_splice(struct io_kiocb *req, unsigned int issue_flags)
120{
121 struct io_splice *sp = io_kiocb_to_cmd(req, struct io_splice);
122 struct file *out = sp->file_out;
123 unsigned int flags = sp->flags & ~SPLICE_F_FD_IN_FIXED;
124 loff_t *poff_in, *poff_out;
125 struct file *in;
126 ssize_t ret = 0;
127
128 WARN_ON_ONCE(issue_flags & IO_URING_F_NONBLOCK);
129
130 in = io_splice_get_file(req, issue_flags);
131 if (!in) {
132 ret = -EBADF;
133 goto done;
134 }
135
136 poff_in = (sp->off_in == -1) ? NULL : &sp->off_in;
137 poff_out = (sp->off_out == -1) ? NULL : &sp->off_out;
138
139 if (sp->len)
140 ret = do_splice(in, poff_in, out, poff_out, sp->len, flags);
141
142 if (!(sp->flags & SPLICE_F_FD_IN_FIXED))
143 fput(in);
144done:
145 if (ret != sp->len)
146 req_set_fail(req);
147 io_req_set_res(req, ret, 0);
148 return IOU_COMPLETE;
149}