projects/linux·blob master · refs

356 lines · 8.9 KiB

1// SPDX-License-Identifier: GPL-2.0
2#include <linux/kernel.h>
3#include <linux/errno.h>
4#include <linux/fs.h>
5#include <linux/file.h>
6#include <linux/io_uring.h>
7
8#include <uapi/linux/io_uring.h>
9
10#include "../kernel/futex/futex.h"
11#include "io_uring.h"
12#include "alloc_cache.h"
13#include "futex.h"
14
15struct io_futex {
16 struct file *file;
17 void __user *uaddr;
18 unsigned long futex_val;
19 unsigned long futex_mask;
20 u32 futex_flags;
21 unsigned int futex_nr;
22 bool futexv_unqueued;
23};
24
25struct io_futex_data {
26 struct futex_q q;
27 struct io_kiocb *req;
28};
29
30struct io_futexv_data {
31 unsigned long owned;
32 struct futex_vector futexv[];
33};
34
35#define IO_FUTEX_ALLOC_CACHE_MAX 32
36
37bool io_futex_cache_init(struct io_ring_ctx *ctx)
38{
39 return io_alloc_cache_init(&ctx->futex_cache, IO_FUTEX_ALLOC_CACHE_MAX,
40 sizeof(struct io_futex_data), 0);
41}
42
43void io_futex_cache_free(struct io_ring_ctx *ctx)
44{
45 io_alloc_cache_free(&ctx->futex_cache, kfree);
46}
47
48static void __io_futex_complete(struct io_tw_req tw_req, io_tw_token_t tw)
49{
50 hlist_del_init(&tw_req.req->hash_node);
51 io_req_task_complete(tw_req, tw);
52}
53
54static void io_futex_complete(struct io_tw_req tw_req, io_tw_token_t tw)
55{
56 struct io_kiocb *req = tw_req.req;
57 struct io_ring_ctx *ctx = req->ctx;
58
59 io_tw_lock(ctx, tw);
60 io_cache_free(&ctx->futex_cache, req->async_data);
61 io_req_async_data_clear(req, 0);
62 __io_futex_complete(tw_req, tw);
63}
64
65static void io_futexv_complete(struct io_tw_req tw_req, io_tw_token_t tw)
66{
67 struct io_kiocb *req = tw_req.req;
68 struct io_futex *iof = io_kiocb_to_cmd(req, struct io_futex);
69 struct io_futexv_data *ifd = req->async_data;
70
71 io_tw_lock(req->ctx, tw);
72
73 if (!iof->futexv_unqueued) {
74 int res;
75
76 res = futex_unqueue_multiple(ifd->futexv, iof->futex_nr);
77 if (res != -1)
78 io_req_set_res(req, res, 0);
79 }
80
81 io_req_async_data_free(req);
82 __io_futex_complete(tw_req, tw);
83}
84
85static bool io_futexv_claim(struct io_futexv_data *ifd)
86{
87 if (test_bit(0, &ifd->owned) || test_and_set_bit_lock(0, &ifd->owned))
88 return false;
89 return true;
90}
91
92static bool __io_futex_cancel(struct io_kiocb *req)
93{
94 /* futex wake already done or in progress */
95 if (req->opcode == IORING_OP_FUTEX_WAIT) {
96 struct io_futex_data *ifd = req->async_data;
97
98 if (!futex_unqueue(&ifd->q))
99 return false;
100 req->io_task_work.func = io_futex_complete;
101 } else {
102 struct io_futexv_data *ifd = req->async_data;
103
104 if (!io_futexv_claim(ifd))
105 return false;
106 req->io_task_work.func = io_futexv_complete;
107 }
108
109 hlist_del_init(&req->hash_node);
110 io_req_set_res(req, -ECANCELED, 0);
111 io_req_task_work_add(req);
112 return true;
113}
114
115int io_futex_cancel(struct io_ring_ctx *ctx, struct io_cancel_data *cd,
116 unsigned int issue_flags)
117{
118 return io_cancel_remove(ctx, cd, issue_flags, &ctx->futex_list, __io_futex_cancel);
119}
120
121bool io_futex_remove_all(struct io_ring_ctx *ctx, struct io_uring_task *tctx,
122 bool cancel_all)
123{
124 return io_cancel_remove_all(ctx, tctx, &ctx->futex_list, cancel_all, __io_futex_cancel);
125}
126
127int io_futex_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
128{
129 struct io_futex *iof = io_kiocb_to_cmd(req, struct io_futex);
130 u32 flags;
131
132 if (unlikely(sqe->len || sqe->futex_flags || sqe->buf_index ||
133 sqe->file_index))
134 return -EINVAL;
135
136 iof->uaddr = u64_to_user_ptr(READ_ONCE(sqe->addr));
137 iof->futex_val = READ_ONCE(sqe->addr2);
138 iof->futex_mask = READ_ONCE(sqe->addr3);
139 flags = READ_ONCE(sqe->fd);
140
141 if (flags & ~FUTEX2_VALID_MASK)
142 return -EINVAL;
143
144 iof->futex_flags = futex2_to_flags(flags);
145 if (!futex_flags_valid(iof->futex_flags))
146 return -EINVAL;
147
148 if (!futex_validate_input(iof->futex_flags, iof->futex_val) ||
149 !futex_validate_input(iof->futex_flags, iof->futex_mask))
150 return -EINVAL;
151
152 return 0;
153}
154
155int io_futex_wait_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
156{
157 struct io_futex *iof = io_kiocb_to_cmd(req, struct io_futex);
158 int ret;
159
160 ret = io_futex_prep(req, sqe);
161 if (unlikely(ret))
162 return ret;
163
164 /* inflight tracking only needed for mm private hash */
165 if (!(iof->futex_flags & FLAGS_SHARED))
166 io_req_track_inflight(req);
167 return 0;
168}
169
170static void io_futex_wakev_fn(struct wake_q_head *wake_q, struct futex_q *q)
171{
172 struct io_kiocb *req = q->wake_data;
173 struct io_futexv_data *ifd = req->async_data;
174
175 if (!io_futexv_claim(ifd)) {
176 __futex_wake_mark(q);
177 return;
178 }
179 if (unlikely(!__futex_wake_mark(q)))
180 return;
181
182 io_req_set_res(req, 0, 0);
183 req->io_task_work.func = io_futexv_complete;
184 __io_req_task_work_add(req, IOU_F_TWQ_IN_WAKE);
185}
186
187int io_futexv_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
188{
189 struct io_futex *iof = io_kiocb_to_cmd(req, struct io_futex);
190 struct io_futexv_data *ifd;
191 unsigned int i;
192 int ret;
193
194 /* No flags or mask supported for waitv */
195 if (unlikely(sqe->fd || sqe->buf_index || sqe->file_index ||
196 sqe->addr2 || sqe->futex_flags || sqe->addr3))
197 return -EINVAL;
198
199 iof->uaddr = u64_to_user_ptr(READ_ONCE(sqe->addr));
200 iof->futex_nr = READ_ONCE(sqe->len);
201 if (!iof->futex_nr || iof->futex_nr > FUTEX_WAITV_MAX)
202 return -EINVAL;
203
204 ifd = kzalloc_flex(struct io_futexv_data, futexv, iof->futex_nr,
205 GFP_KERNEL_ACCOUNT);
206 if (!ifd)
207 return -ENOMEM;
208
209 ret = futex_parse_waitv(ifd->futexv, iof->uaddr, iof->futex_nr,
210 io_futex_wakev_fn, req);
211 if (ret) {
212 kfree(ifd);
213 return ret;
214 }
215
216 /* inflight tracking only needed for mm private hash */
217 for (i = 0; i < iof->futex_nr; i++) {
218 if (!(ifd->futexv[i].w.flags & FLAGS_SHARED)) {
219 io_req_track_inflight(req);
220 break;
221 }
222 }
223
224 iof->futexv_unqueued = 0;
225 req->flags |= REQ_F_ASYNC_DATA;
226 req->async_data = ifd;
227 return 0;
228}
229
230static void io_futex_wake_fn(struct wake_q_head *wake_q, struct futex_q *q)
231{
232 struct io_futex_data *ifd = container_of(q, struct io_futex_data, q);
233 struct io_kiocb *req = ifd->req;
234
235 if (unlikely(!__futex_wake_mark(q)))
236 return;
237
238 io_req_set_res(req, 0, 0);
239 req->io_task_work.func = io_futex_complete;
240 __io_req_task_work_add(req, IOU_F_TWQ_IN_WAKE);
241}
242
243int io_futexv_wait(struct io_kiocb *req, unsigned int issue_flags)
244{
245 struct io_futex *iof = io_kiocb_to_cmd(req, struct io_futex);
246 struct io_futexv_data *ifd = req->async_data;
247 struct io_ring_ctx *ctx = req->ctx;
248 int ret, woken = -1;
249
250 io_ring_submit_lock(ctx, issue_flags);
251
252 ret = futex_wait_multiple_setup(ifd->futexv, iof->futex_nr, &woken);
253
254 /*
255 * Error case, ret is < 0. Mark the request as failed.
256 */
257 if (unlikely(ret < 0)) {
258 io_ring_submit_unlock(ctx, issue_flags);
259 req_set_fail(req);
260 io_req_set_res(req, ret, 0);
261 io_req_async_data_free(req);
262 return IOU_COMPLETE;
263 }
264
265 /*
266 * 0 return means that we successfully setup the waiters, and that
267 * nobody triggered a wakeup while we were doing so. If the wakeup
268 * happened post setup, the task_work will be run post this issue and
269 * under the submission lock. 1 means We got woken while setting up,
270 * let that side do the completion. Note that
271 * futex_wait_multiple_setup() will have unqueued all the futexes in
272 * this case. Mark us as having done that already, since this is
273 * different from normal wakeup.
274 */
275 if (!ret) {
276 /*
277 * If futex_wait_multiple_setup() returns 0 for a
278 * successful setup, then the task state will not be
279 * runnable. This is fine for the sync syscall, as
280 * it'll be blocking unless we already got one of the
281 * futexes woken, but it obviously won't work for an
282 * async invocation. Mark us runnable again.
283 */
284 __set_current_state(TASK_RUNNING);
285 hlist_add_head(&req->hash_node, &ctx->futex_list);
286 } else {
287 iof->futexv_unqueued = 1;
288 if (woken != -1)
289 io_req_set_res(req, woken, 0);
290 }
291
292 io_ring_submit_unlock(ctx, issue_flags);
293 return IOU_ISSUE_SKIP_COMPLETE;
294}
295
296int io_futex_wait(struct io_kiocb *req, unsigned int issue_flags)
297{
298 struct io_futex *iof = io_kiocb_to_cmd(req, struct io_futex);
299 struct io_ring_ctx *ctx = req->ctx;
300 struct io_futex_data *ifd = NULL;
301 int ret;
302
303 if (!iof->futex_mask) {
304 ret = -EINVAL;
305 goto done;
306 }
307
308 io_ring_submit_lock(ctx, issue_flags);
309 ifd = io_cache_alloc(&ctx->futex_cache, GFP_NOWAIT);
310 if (!ifd) {
311 ret = -ENOMEM;
312 goto done_unlock;
313 }
314
315 req->flags |= REQ_F_ASYNC_DATA;
316 req->async_data = ifd;
317 ifd->q = futex_q_init;
318 ifd->q.bitset = iof->futex_mask;
319 ifd->q.wake = io_futex_wake_fn;
320 ifd->req = req;
321
322 ret = futex_wait_setup(iof->uaddr, iof->futex_val, iof->futex_flags,
323 &ifd->q, NULL, NULL);
324 if (!ret) {
325 hlist_add_head(&req->hash_node, &ctx->futex_list);
326 io_ring_submit_unlock(ctx, issue_flags);
327
328 return IOU_ISSUE_SKIP_COMPLETE;
329 }
330
331done_unlock:
332 io_ring_submit_unlock(ctx, issue_flags);
333done:
334 if (ret < 0)
335 req_set_fail(req);
336 io_req_set_res(req, ret, 0);
337 io_req_async_data_free(req);
338 return IOU_COMPLETE;
339}
340
341int io_futex_wake(struct io_kiocb *req, unsigned int issue_flags)
342{
343 struct io_futex *iof = io_kiocb_to_cmd(req, struct io_futex);
344 int ret;
345
346 /*
347 * Strict flags - ensure that waking 0 futexes yields a 0 result.
348 * See commit 43adf8449510 ("futex: FLAGS_STRICT") for details.
349 */
350 ret = futex_wake(iof->uaddr, FLAGS_STRICT | iof->futex_flags, NULL,
351 iof->futex_val, iof->futex_mask);
352 if (ret < 0)
353 req_set_fail(req);
354 io_req_set_res(req, ret, 0);
355 return IOU_COMPLETE;
356}