projects/linux·blob master · refs

351 lines · 8.3 KiB

1// SPDX-License-Identifier: GPL-2.0
2#include <linux/device.h>
3#include <linux/init.h>
4#include <linux/kernel.h>
5#include <linux/miscdevice.h>
6#include <linux/module.h>
7#include <linux/anon_inodes.h>
8#include <linux/ktime.h>
9#include <linux/hrtimer.h>
10#include <linux/poll.h>
11
12#include <linux/io_uring/cmd.h>
13#include <linux/io_uring_types.h>
14#include <uapi/linux/io_uring/mock_file.h>
15
16struct io_mock_iocb {
17 struct kiocb *iocb;
18 struct hrtimer timer;
19 int res;
20};
21
22struct io_mock_file {
23 size_t size;
24 u64 rw_delay_ns;
25 bool pollable;
26 struct wait_queue_head poll_wq;
27};
28
29#define IO_VALID_COPY_CMD_FLAGS IORING_MOCK_COPY_FROM
30
31static int io_copy_regbuf(struct iov_iter *reg_iter, void __user *ubuf)
32{
33 size_t ret, copied = 0;
34 size_t buflen = PAGE_SIZE;
35 void *tmp_buf;
36
37 tmp_buf = kzalloc(buflen, GFP_KERNEL);
38 if (!tmp_buf)
39 return -ENOMEM;
40
41 while (iov_iter_count(reg_iter)) {
42 size_t len = min(iov_iter_count(reg_iter), buflen);
43
44 if (iov_iter_rw(reg_iter) == ITER_SOURCE) {
45 ret = copy_from_iter(tmp_buf, len, reg_iter);
46 if (ret <= 0)
47 break;
48 if (copy_to_user(ubuf, tmp_buf, ret))
49 break;
50 } else {
51 if (copy_from_user(tmp_buf, ubuf, len))
52 break;
53 ret = copy_to_iter(tmp_buf, len, reg_iter);
54 if (ret <= 0)
55 break;
56 }
57 ubuf += ret;
58 copied += ret;
59 }
60
61 kfree(tmp_buf);
62 return copied;
63}
64
65static int io_cmd_copy_regbuf(struct io_uring_cmd *cmd, unsigned int issue_flags)
66{
67 const struct io_uring_sqe *sqe = cmd->sqe;
68 const struct iovec __user *iovec;
69 unsigned flags, iovec_len;
70 struct iov_iter iter;
71 void __user *ubuf;
72 int dir, ret;
73
74 ubuf = u64_to_user_ptr(READ_ONCE(sqe->addr3));
75 iovec = u64_to_user_ptr(READ_ONCE(sqe->addr));
76 iovec_len = READ_ONCE(sqe->len);
77 flags = READ_ONCE(sqe->file_index);
78
79 if (unlikely(sqe->ioprio || sqe->__pad1))
80 return -EINVAL;
81 if (flags & ~IO_VALID_COPY_CMD_FLAGS)
82 return -EINVAL;
83
84 dir = (flags & IORING_MOCK_COPY_FROM) ? ITER_SOURCE : ITER_DEST;
85 ret = io_uring_cmd_import_fixed_vec(cmd, iovec, iovec_len, dir, &iter,
86 issue_flags);
87 if (ret)
88 return ret;
89 ret = io_copy_regbuf(&iter, ubuf);
90 return ret ? ret : -EFAULT;
91}
92
93static int io_mock_cmd(struct io_uring_cmd *cmd, unsigned int issue_flags)
94{
95 switch (cmd->cmd_op) {
96 case IORING_MOCK_CMD_COPY_REGBUF:
97 return io_cmd_copy_regbuf(cmd, issue_flags);
98 }
99 return -ENOTSUPP;
100}
101
102static enum hrtimer_restart io_mock_rw_timer_expired(struct hrtimer *timer)
103{
104 struct io_mock_iocb *mio = container_of(timer, struct io_mock_iocb, timer);
105 struct kiocb *iocb = mio->iocb;
106
107 WRITE_ONCE(iocb->private, NULL);
108 iocb->ki_complete(iocb, mio->res);
109 kfree(mio);
110 return HRTIMER_NORESTART;
111}
112
113static ssize_t io_mock_delay_rw(struct kiocb *iocb, size_t len)
114{
115 struct io_mock_file *mf = iocb->ki_filp->private_data;
116 struct io_mock_iocb *mio;
117
118 mio = kzalloc_obj(*mio);
119 if (!mio)
120 return -ENOMEM;
121
122 mio->iocb = iocb;
123 mio->res = len;
124 hrtimer_setup(&mio->timer, io_mock_rw_timer_expired,
125 CLOCK_MONOTONIC, HRTIMER_MODE_REL);
126 hrtimer_start(&mio->timer, ns_to_ktime(mf->rw_delay_ns),
127 HRTIMER_MODE_REL);
128 return -EIOCBQUEUED;
129}
130
131static ssize_t io_mock_read_iter(struct kiocb *iocb, struct iov_iter *to)
132{
133 struct io_mock_file *mf = iocb->ki_filp->private_data;
134 size_t len = iov_iter_count(to);
135 size_t nr_zeroed;
136
137 if (iocb->ki_pos + len > mf->size)
138 return -EINVAL;
139 nr_zeroed = iov_iter_zero(len, to);
140 if (!mf->rw_delay_ns || nr_zeroed != len)
141 return nr_zeroed;
142
143 return io_mock_delay_rw(iocb, len);
144}
145
146static ssize_t io_mock_write_iter(struct kiocb *iocb, struct iov_iter *from)
147{
148 struct io_mock_file *mf = iocb->ki_filp->private_data;
149 size_t len = iov_iter_count(from);
150
151 if (iocb->ki_pos + len > mf->size)
152 return -EINVAL;
153 if (!mf->rw_delay_ns) {
154 iov_iter_advance(from, len);
155 return len;
156 }
157
158 return io_mock_delay_rw(iocb, len);
159}
160
161static loff_t io_mock_llseek(struct file *file, loff_t offset, int whence)
162{
163 struct io_mock_file *mf = file->private_data;
164
165 return fixed_size_llseek(file, offset, whence, mf->size);
166}
167
168static __poll_t io_mock_poll(struct file *file, struct poll_table_struct *pt)
169{
170 struct io_mock_file *mf = file->private_data;
171 __poll_t mask = 0;
172
173 poll_wait(file, &mf->poll_wq, pt);
174
175 mask |= EPOLLOUT | EPOLLWRNORM;
176 mask |= EPOLLIN | EPOLLRDNORM;
177 return mask;
178}
179
180static int io_mock_release(struct inode *inode, struct file *file)
181{
182 struct io_mock_file *mf = file->private_data;
183
184 kfree(mf);
185 return 0;
186}
187
188static const struct file_operations io_mock_fops = {
189 .owner = THIS_MODULE,
190 .release = io_mock_release,
191 .uring_cmd = io_mock_cmd,
192 .read_iter = io_mock_read_iter,
193 .write_iter = io_mock_write_iter,
194 .llseek = io_mock_llseek,
195};
196
197static const struct file_operations io_mock_poll_fops = {
198 .owner = THIS_MODULE,
199 .release = io_mock_release,
200 .uring_cmd = io_mock_cmd,
201 .read_iter = io_mock_read_iter,
202 .write_iter = io_mock_write_iter,
203 .llseek = io_mock_llseek,
204 .poll = io_mock_poll,
205};
206
207#define IO_VALID_CREATE_FLAGS (IORING_MOCK_CREATE_F_SUPPORT_NOWAIT | \
208 IORING_MOCK_CREATE_F_POLL)
209
210static int io_create_mock_file(struct io_uring_cmd *cmd, unsigned int issue_flags)
211{
212 const struct file_operations *fops = &io_mock_fops;
213 const struct io_uring_sqe *sqe = cmd->sqe;
214 struct io_uring_mock_create mc, __user *uarg;
215 struct file *file;
216 struct io_mock_file *mf __free(kfree) = NULL;
217 size_t uarg_size;
218
219 /*
220 * It's a testing only driver that allows exercising edge cases
221 * that wouldn't be possible to hit otherwise.
222 */
223 add_taint(TAINT_TEST, LOCKDEP_STILL_OK);
224
225 uarg = u64_to_user_ptr(READ_ONCE(sqe->addr));
226 uarg_size = READ_ONCE(sqe->len);
227
228 if (sqe->ioprio || sqe->__pad1 || sqe->addr3 || sqe->file_index)
229 return -EINVAL;
230 if (uarg_size != sizeof(mc))
231 return -EINVAL;
232
233 memset(&mc, 0, sizeof(mc));
234 if (copy_from_user(&mc, uarg, uarg_size))
235 return -EFAULT;
236 if (!mem_is_zero(mc.__resv, sizeof(mc.__resv)))
237 return -EINVAL;
238 if (mc.flags & ~IO_VALID_CREATE_FLAGS)
239 return -EINVAL;
240 if (mc.file_size > SZ_1G)
241 return -EINVAL;
242 if (mc.rw_delay_ns > NSEC_PER_SEC)
243 return -EINVAL;
244
245 mf = kzalloc_obj(*mf, GFP_KERNEL_ACCOUNT);
246 if (!mf)
247 return -ENOMEM;
248
249 init_waitqueue_head(&mf->poll_wq);
250 mf->size = mc.file_size;
251 mf->rw_delay_ns = mc.rw_delay_ns;
252 if (mc.flags & IORING_MOCK_CREATE_F_POLL) {
253 fops = &io_mock_poll_fops;
254 mf->pollable = true;
255 }
256
257 FD_PREPARE(fdf, O_RDWR | O_CLOEXEC,
258 anon_inode_create_getfile("[io_uring_mock]", fops, mf,
259 O_RDWR | O_CLOEXEC, NULL));
260 if (fdf->fd < 0)
261 return fdf->fd;
262
263 retain_and_null_ptr(mf);
264 file = fdf->file;
265 file->f_mode |= FMODE_READ | FMODE_CAN_READ | FMODE_WRITE |
266 FMODE_CAN_WRITE | FMODE_LSEEK;
267 if (mc.flags & IORING_MOCK_CREATE_F_SUPPORT_NOWAIT)
268 file->f_mode |= FMODE_NOWAIT;
269
270 mc.out_fd = fdf->fd;
271 if (copy_to_user(uarg, &mc, uarg_size))
272 return -EFAULT;
273
274 fd_publish(fdf);
275 return 0;
276}
277
278static int io_probe_mock(struct io_uring_cmd *cmd)
279{
280 const struct io_uring_sqe *sqe = cmd->sqe;
281 struct io_uring_mock_probe mp, __user *uarg;
282 size_t uarg_size;
283
284 uarg = u64_to_user_ptr(READ_ONCE(sqe->addr));
285 uarg_size = READ_ONCE(sqe->len);
286
287 if (sqe->ioprio || sqe->__pad1 || sqe->addr3 || sqe->file_index ||
288 uarg_size != sizeof(mp))
289 return -EINVAL;
290
291 memset(&mp, 0, sizeof(mp));
292 if (copy_from_user(&mp, uarg, uarg_size))
293 return -EFAULT;
294 if (!mem_is_zero(&mp, sizeof(mp)))
295 return -EINVAL;
296
297 mp.features = IORING_MOCK_FEAT_END;
298
299 if (copy_to_user(uarg, &mp, uarg_size))
300 return -EFAULT;
301 return 0;
302}
303
304static int iou_mock_mgr_cmd(struct io_uring_cmd *cmd, unsigned int issue_flags)
305{
306 if (!capable(CAP_SYS_ADMIN))
307 return -EPERM;
308
309 switch (cmd->cmd_op) {
310 case IORING_MOCK_MGR_CMD_PROBE:
311 return io_probe_mock(cmd);
312 case IORING_MOCK_MGR_CMD_CREATE:
313 return io_create_mock_file(cmd, issue_flags);
314 }
315 return -EOPNOTSUPP;
316}
317
318static const struct file_operations iou_mock_dev_fops = {
319 .owner = THIS_MODULE,
320 .uring_cmd = iou_mock_mgr_cmd,
321};
322
323static struct miscdevice iou_mock_miscdev = {
324 .minor = MISC_DYNAMIC_MINOR,
325 .name = "io_uring_mock",
326 .fops = &iou_mock_dev_fops,
327};
328
329static int __init io_mock_init(void)
330{
331 int ret;
332
333 ret = misc_register(&iou_mock_miscdev);
334 if (ret < 0) {
335 pr_err("Could not initialize io_uring mock device\n");
336 return ret;
337 }
338 return 0;
339}
340
341static void __exit io_mock_exit(void)
342{
343 misc_deregister(&iou_mock_miscdev);
344}
345
346module_init(io_mock_init)
347module_exit(io_mock_exit)
348
349MODULE_AUTHOR("Pavel Begunkov <[email protected]>");
350MODULE_DESCRIPTION("io_uring mock file");
351MODULE_LICENSE("GPL");