projects/linux·blob master · refs

1976 lines · 50.8 KiB

1// SPDX-License-Identifier: GPL-2.0
2#include <linux/kernel.h>
3#include <linux/errno.h>
4#include <linux/file.h>
5#include <linux/slab.h>
6#include <linux/net.h>
7#include <linux/un.h>
8#include <linux/compat.h>
9#include <net/compat.h>
10#include <net/ipv6.h>
11#include <linux/io_uring.h>
12
13#include <uapi/linux/io_uring.h>
14
15#include "filetable.h"
16#include "io_uring.h"
17#include "kbuf.h"
18#include "alloc_cache.h"
19#include "net.h"
20#include "notif.h"
21#include "rsrc.h"
22#include "zcrx.h"
23
24struct io_shutdown {
25 struct file *file;
26 int how;
27};
28
29struct io_accept {
30 struct file *file;
31 struct sockaddr __user *addr;
32 int __user *addr_len;
33 int flags;
34 int iou_flags;
35 u32 file_slot;
36 unsigned long nofile;
37};
38
39struct io_socket {
40 struct file *file;
41 int domain;
42 int type;
43 int protocol;
44 int flags;
45 u32 file_slot;
46 unsigned long nofile;
47};
48
49struct io_connect {
50 struct file *file;
51 struct sockaddr __user *addr;
52 int addr_len;
53 bool in_progress;
54 bool seen_econnaborted;
55};
56
57struct io_bind {
58 struct file *file;
59 int addr_len;
60};
61
62struct io_listen {
63 struct file *file;
64 int backlog;
65};
66
67struct io_sr_msg {
68 struct file *file;
69 union {
70 struct compat_msghdr __user *umsg_compat;
71 struct user_msghdr __user *umsg;
72 void __user *buf;
73 };
74 int len;
75 unsigned done_io;
76 unsigned msg_flags;
77 unsigned nr_multishot_loops;
78 u16 flags;
79 /* initialised and used only by !msg send variants */
80 u16 buf_group;
81 /* per-invocation mshot limit */
82 unsigned mshot_len;
83 /* overall mshot byte limit */
84 unsigned mshot_total_len;
85 void __user *msg_control;
86 /* used only for send zerocopy */
87 struct io_kiocb *notif;
88};
89
90/*
91 * The UAPI flags are the lower 8 bits, as that's all sqe->ioprio will hold
92 * anyway. Use the upper 8 bits for internal uses.
93 */
94enum sr_retry_flags {
95 IORING_RECV_RETRY = (1U << 15),
96 IORING_RECV_PARTIAL_MAP = (1U << 14),
97 IORING_RECV_MSHOT_CAP = (1U << 13),
98 IORING_RECV_MSHOT_LIM = (1U << 12),
99 IORING_RECV_MSHOT_DONE = (1U << 11),
100
101 IORING_RECV_RETRY_CLEAR = IORING_RECV_RETRY | IORING_RECV_PARTIAL_MAP,
102 IORING_RECV_NO_RETRY = IORING_RECV_RETRY | IORING_RECV_PARTIAL_MAP |
103 IORING_RECV_MSHOT_CAP | IORING_RECV_MSHOT_DONE,
104};
105
106/*
107 * Number of times we'll try and do receives if there's more data. If we
108 * exceed this limit, then add us to the back of the queue and retry from
109 * there. This helps fairness between flooding clients.
110 */
111#define MULTISHOT_MAX_RETRY 32
112
113struct io_recvzc {
114 struct file *file;
115 u16 flags;
116 u32 len;
117 struct io_zcrx_ifq *ifq;
118};
119
120static int io_sg_from_iter_iovec(struct sk_buff *skb,
121 struct iov_iter *from, size_t length);
122static int io_sg_from_iter(struct sk_buff *skb,
123 struct iov_iter *from, size_t length);
124
125int io_shutdown_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
126{
127 struct io_shutdown *shutdown = io_kiocb_to_cmd(req, struct io_shutdown);
128
129 if (unlikely(sqe->off || sqe->addr || sqe->rw_flags ||
130 sqe->buf_index || sqe->splice_fd_in))
131 return -EINVAL;
132
133 shutdown->how = READ_ONCE(sqe->len);
134 req->flags |= REQ_F_FORCE_ASYNC;
135 return 0;
136}
137
138int io_shutdown(struct io_kiocb *req, unsigned int issue_flags)
139{
140 struct io_shutdown *shutdown = io_kiocb_to_cmd(req, struct io_shutdown);
141 struct socket *sock;
142 int ret;
143
144 WARN_ON_ONCE(issue_flags & IO_URING_F_NONBLOCK);
145
146 sock = sock_from_file(req->file);
147 if (unlikely(!sock))
148 return -ENOTSOCK;
149
150 ret = __sys_shutdown_sock(sock, shutdown->how);
151 io_req_set_res(req, ret, 0);
152 return IOU_COMPLETE;
153}
154
155static bool io_net_retry(struct socket *sock, int flags)
156{
157 if (!(flags & MSG_WAITALL))
158 return false;
159 return sock->type == SOCK_STREAM || sock->type == SOCK_SEQPACKET;
160}
161
162static void io_netmsg_iovec_free(struct io_async_msghdr *kmsg)
163{
164 if (kmsg->vec.iovec)
165 io_vec_free(&kmsg->vec);
166}
167
168static void io_netmsg_recycle(struct io_kiocb *req, unsigned int issue_flags)
169{
170 struct io_async_msghdr *hdr = req->async_data;
171
172 /* can't recycle, ensure we free the iovec if we have one */
173 if (unlikely(issue_flags & IO_URING_F_UNLOCKED)) {
174 io_netmsg_iovec_free(hdr);
175 return;
176 }
177
178 /* Let normal cleanup path reap it if we fail adding to the cache */
179 io_alloc_cache_vec_kasan(&hdr->vec);
180 if (hdr->vec.nr > IO_VEC_CACHE_SOFT_CAP)
181 io_vec_free(&hdr->vec);
182
183 if (io_alloc_cache_put(&req->ctx->netmsg_cache, hdr))
184 io_req_async_data_clear(req, REQ_F_NEED_CLEANUP);
185}
186
187static struct io_async_msghdr *io_msg_alloc_async(struct io_kiocb *req)
188{
189 struct io_ring_ctx *ctx = req->ctx;
190 struct io_async_msghdr *hdr;
191
192 hdr = io_uring_alloc_async_data(&ctx->netmsg_cache, req);
193 if (!hdr)
194 return NULL;
195
196 /* If the async data was cached, we might have an iov cached inside. */
197 if (hdr->vec.iovec)
198 req->flags |= REQ_F_NEED_CLEANUP;
199 return hdr;
200}
201
202static inline void io_mshot_prep_retry(struct io_kiocb *req,
203 struct io_async_msghdr *kmsg)
204{
205 struct io_sr_msg *sr = io_kiocb_to_cmd(req, struct io_sr_msg);
206
207 req->flags &= ~REQ_F_BL_EMPTY;
208 sr->done_io = 0;
209 sr->flags &= ~IORING_RECV_RETRY_CLEAR;
210 sr->len = sr->mshot_len;
211}
212
213static int io_net_import_vec(struct io_kiocb *req, struct io_async_msghdr *iomsg,
214 const struct iovec __user *uiov, unsigned uvec_seg,
215 int ddir)
216{
217 struct iovec *iov;
218 int ret, nr_segs;
219
220 if (iomsg->vec.iovec) {
221 nr_segs = iomsg->vec.nr;
222 iov = iomsg->vec.iovec;
223 } else {
224 nr_segs = 1;
225 iov = &iomsg->fast_iov;
226 }
227
228 ret = __import_iovec(ddir, uiov, uvec_seg, nr_segs, &iov,
229 &iomsg->msg.msg_iter, io_is_compat(req->ctx));
230 if (unlikely(ret < 0))
231 return ret;
232
233 if (iov) {
234 req->flags |= REQ_F_NEED_CLEANUP;
235 io_vec_reset_iovec(&iomsg->vec, iov, iomsg->msg.msg_iter.nr_segs);
236 }
237 return 0;
238}
239
240static int io_copy_msghdr_from_user(struct user_msghdr *msg,
241 struct user_msghdr __user *umsg)
242{
243 if (!user_access_begin(umsg, sizeof(*umsg)))
244 return -EFAULT;
245 unsafe_get_user(msg->msg_name, &umsg->msg_name, ua_end);
246 unsafe_get_user(msg->msg_namelen, &umsg->msg_namelen, ua_end);
247 unsafe_get_user(msg->msg_iov, &umsg->msg_iov, ua_end);
248 unsafe_get_user(msg->msg_iovlen, &umsg->msg_iovlen, ua_end);
249 unsafe_get_user(msg->msg_control, &umsg->msg_control, ua_end);
250 unsafe_get_user(msg->msg_controllen, &umsg->msg_controllen, ua_end);
251 user_access_end();
252 return 0;
253ua_end:
254 user_access_end();
255 return -EFAULT;
256}
257
258static int io_msg_copy_hdr(struct io_kiocb *req, struct io_async_msghdr *iomsg,
259 struct user_msghdr *msg,
260 struct sockaddr __user **save_addr)
261{
262 struct io_sr_msg *sr = io_kiocb_to_cmd(req, struct io_sr_msg);
263 int ret;
264
265 iomsg->msg.msg_name = &iomsg->addr;
266 iomsg->msg.msg_iter.nr_segs = 0;
267
268 if (io_is_compat(req->ctx)) {
269 struct compat_msghdr cmsg;
270
271 if (copy_from_user(&cmsg, sr->umsg_compat, sizeof(cmsg)))
272 return -EFAULT;
273
274 ret = __get_compat_msghdr(&iomsg->msg, &cmsg, save_addr);
275 if (ret)
276 return ret;
277
278 memset(msg, 0, sizeof(*msg));
279 msg->msg_namelen = cmsg.msg_namelen;
280 msg->msg_controllen = cmsg.msg_controllen;
281 msg->msg_iov = compat_ptr(cmsg.msg_iov);
282 msg->msg_iovlen = cmsg.msg_iovlen;
283 } else {
284 ret = io_copy_msghdr_from_user(msg, sr->umsg);
285 if (unlikely(ret))
286 return ret;
287
288 msg->msg_flags = 0;
289
290 ret = __copy_msghdr(&iomsg->msg, msg, save_addr);
291 if (ret)
292 return ret;
293 }
294
295 if (req->flags & REQ_F_BUFFER_SELECT) {
296 if (msg->msg_iovlen == 0) {
297 sr->len = 0;
298 } else if (msg->msg_iovlen > 1) {
299 return -EINVAL;
300 } else {
301 struct iovec fast_iov, *iov;
302
303 iov = iovec_from_user(msg->msg_iov, 1, 1, &fast_iov,
304 io_is_compat(req->ctx));
305 if (IS_ERR(iov))
306 return PTR_ERR(iov);
307 sr->len = iov->iov_len;
308 }
309 }
310 return 0;
311}
312
313void io_sendmsg_recvmsg_cleanup(struct io_kiocb *req)
314{
315 struct io_async_msghdr *io = req->async_data;
316
317 io_netmsg_iovec_free(io);
318}
319
320static int io_send_setup(struct io_kiocb *req, const struct io_uring_sqe *sqe)
321{
322 struct io_sr_msg *sr = io_kiocb_to_cmd(req, struct io_sr_msg);
323 struct io_async_msghdr *kmsg = req->async_data;
324 void __user *addr;
325 u16 addr_len;
326 int ret;
327
328 sr->buf = u64_to_user_ptr(READ_ONCE(sqe->addr));
329
330 if (READ_ONCE(sqe->__pad3[0]))
331 return -EINVAL;
332
333 kmsg->msg.msg_name = NULL;
334 kmsg->msg.msg_namelen = 0;
335 kmsg->msg.msg_control = NULL;
336 kmsg->msg.msg_controllen = 0;
337 kmsg->msg.msg_ubuf = NULL;
338
339 addr = u64_to_user_ptr(READ_ONCE(sqe->addr2));
340 addr_len = READ_ONCE(sqe->addr_len);
341 if (addr) {
342 ret = move_addr_to_kernel(addr, addr_len, &kmsg->addr);
343 if (unlikely(ret < 0))
344 return ret;
345 kmsg->msg.msg_name = &kmsg->addr;
346 kmsg->msg.msg_namelen = addr_len;
347 }
348 if (sr->flags & IORING_RECVSEND_FIXED_BUF) {
349 if (!(sr->flags & IORING_SEND_VECTORIZED)) {
350 req->flags |= REQ_F_IMPORT_BUFFER;
351 return 0;
352 }
353
354 kmsg->msg.msg_iter.nr_segs = sr->len;
355 return io_prep_reg_iovec(req, &kmsg->vec, sr->buf, sr->len);
356 }
357 if (req->flags & REQ_F_BUFFER_SELECT)
358 return 0;
359
360 if (sr->flags & IORING_SEND_VECTORIZED)
361 return io_net_import_vec(req, kmsg, sr->buf, sr->len, ITER_SOURCE);
362
363 return import_ubuf(ITER_SOURCE, sr->buf, sr->len, &kmsg->msg.msg_iter);
364}
365
366static int io_sendmsg_setup(struct io_kiocb *req, const struct io_uring_sqe *sqe)
367{
368 struct io_sr_msg *sr = io_kiocb_to_cmd(req, struct io_sr_msg);
369 struct io_async_msghdr *kmsg = req->async_data;
370 struct user_msghdr msg;
371 int ret;
372
373 sr->flags |= IORING_SEND_VECTORIZED;
374 sr->umsg = u64_to_user_ptr(READ_ONCE(sqe->addr));
375 ret = io_msg_copy_hdr(req, kmsg, &msg, NULL);
376 if (unlikely(ret))
377 return ret;
378 /* save msg_control as sys_sendmsg() overwrites it */
379 sr->msg_control = kmsg->msg.msg_control_user;
380
381 if (sr->flags & IORING_RECVSEND_FIXED_BUF) {
382 kmsg->msg.msg_iter.nr_segs = msg.msg_iovlen;
383 return io_prep_reg_iovec(req, &kmsg->vec, msg.msg_iov,
384 msg.msg_iovlen);
385 }
386 if (req->flags & REQ_F_BUFFER_SELECT)
387 return 0;
388 return io_net_import_vec(req, kmsg, msg.msg_iov, msg.msg_iovlen, ITER_SOURCE);
389}
390
391#define SENDMSG_FLAGS (IORING_RECVSEND_POLL_FIRST | IORING_RECVSEND_BUNDLE | \
392 IORING_SEND_VECTORIZED | IORING_RECVSEND_FIXED_BUF)
393
394int io_sendmsg_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
395{
396 struct io_sr_msg *sr = io_kiocb_to_cmd(req, struct io_sr_msg);
397
398 sr->done_io = 0;
399 sr->len = READ_ONCE(sqe->len);
400 if (unlikely(sr->len < 0))
401 return -EINVAL;
402 sr->flags = READ_ONCE(sqe->ioprio);
403 if (sr->flags & ~SENDMSG_FLAGS)
404 return -EINVAL;
405 if (sr->flags & IORING_RECVSEND_FIXED_BUF) {
406 /* registered buffer send only supported for plain IORING_OP_SEND */
407 if (req->opcode != IORING_OP_SEND ||
408 (req->flags & REQ_F_BUFFER_SELECT) ||
409 (sr->flags & (IORING_RECVSEND_BUNDLE|IORING_SEND_VECTORIZED)))
410 return -EINVAL;
411 req->buf_index = READ_ONCE(sqe->buf_index);
412 }
413 sr->msg_flags = READ_ONCE(sqe->msg_flags) | MSG_NOSIGNAL;
414 if (sr->msg_flags & MSG_DONTWAIT)
415 req->flags |= REQ_F_NOWAIT;
416 if (req->flags & REQ_F_BUFFER_SELECT)
417 sr->buf_group = req->buf_index;
418 sr->mshot_total_len = sr->mshot_len = 0;
419 if (sr->flags & IORING_RECVSEND_BUNDLE) {
420 if (req->opcode == IORING_OP_SENDMSG)
421 return -EINVAL;
422 sr->msg_flags |= MSG_WAITALL;
423 req->flags |= REQ_F_MULTISHOT;
424 }
425
426 if (io_is_compat(req->ctx))
427 sr->msg_flags |= MSG_CMSG_COMPAT;
428
429 if (unlikely(!io_msg_alloc_async(req)))
430 return -ENOMEM;
431 if (req->opcode != IORING_OP_SENDMSG)
432 return io_send_setup(req, sqe);
433 if (unlikely(sqe->addr2 || sqe->file_index))
434 return -EINVAL;
435 return io_sendmsg_setup(req, sqe);
436}
437
438static void io_req_msg_cleanup(struct io_kiocb *req,
439 unsigned int issue_flags)
440{
441 io_netmsg_recycle(req, issue_flags);
442}
443
444/*
445 * For bundle completions, we need to figure out how many segments we consumed.
446 * A bundle could be using a single ITER_UBUF if that's all we mapped, or it
447 * could be using an ITER_IOVEC. If the latter, then if we consumed all of
448 * the segments, then it's a trivial questiont o answer. If we have residual
449 * data in the iter, then loop the segments to figure out how much we
450 * transferred.
451 */
452static int io_bundle_nbufs(struct io_async_msghdr *kmsg, int ret)
453{
454 struct iovec *iov;
455 int nbufs;
456
457 /* no data is always zero segments, and a ubuf is always 1 segment */
458 if (ret <= 0)
459 return 0;
460 if (iter_is_ubuf(&kmsg->msg.msg_iter))
461 return 1;
462
463 iov = kmsg->vec.iovec;
464 if (!iov)
465 iov = &kmsg->fast_iov;
466
467 /* if all data was transferred, it's basic pointer math */
468 if (!iov_iter_count(&kmsg->msg.msg_iter))
469 return iter_iov(&kmsg->msg.msg_iter) - iov;
470
471 /* short transfer, count segments */
472 nbufs = 0;
473 do {
474 int this_len = min_t(int, iov[nbufs].iov_len, ret);
475
476 nbufs++;
477 ret -= this_len;
478 } while (ret);
479
480 return nbufs;
481}
482
483static int io_net_kbuf_recyle(struct io_kiocb *req, struct io_buffer_list *bl,
484 struct io_async_msghdr *kmsg, int len)
485{
486 req->flags |= REQ_F_BL_NO_RECYCLE;
487 if (req->flags & REQ_F_BUFFERS_COMMIT)
488 io_kbuf_commit(req, bl, len, io_bundle_nbufs(kmsg, len));
489 return IOU_RETRY;
490}
491
492static inline bool io_send_finish(struct io_kiocb *req,
493 struct io_async_msghdr *kmsg,
494 struct io_br_sel *sel)
495{
496 struct io_sr_msg *sr = io_kiocb_to_cmd(req, struct io_sr_msg);
497 bool bundle_finished = sel->val <= 0;
498 unsigned int cflags;
499
500 if (!(sr->flags & IORING_RECVSEND_BUNDLE)) {
501 cflags = io_put_kbuf(req, sel->val, sel->buf_list);
502 goto finish;
503 }
504
505 cflags = io_put_kbufs(req, sel->val, sel->buf_list, io_bundle_nbufs(kmsg, sel->val));
506
507 /*
508 * Don't start new bundles if the buffer list is empty, or if the
509 * current operation needed to go through polling to complete.
510 */
511 if (bundle_finished || req->flags & (REQ_F_BL_EMPTY | REQ_F_POLLED))
512 goto finish;
513
514 /*
515 * Fill CQE for this receive and see if we should keep trying to
516 * receive from this socket.
517 */
518 if (io_req_post_cqe(req, sel->val, cflags | IORING_CQE_F_MORE)) {
519 io_mshot_prep_retry(req, kmsg);
520 return false;
521 }
522
523 /* Otherwise stop bundle and use the current result. */
524finish:
525 io_req_set_res(req, sel->val, cflags);
526 sel->val = IOU_COMPLETE;
527 return true;
528}
529
530int io_sendmsg(struct io_kiocb *req, unsigned int issue_flags)
531{
532 struct io_sr_msg *sr = io_kiocb_to_cmd(req, struct io_sr_msg);
533 struct io_async_msghdr *kmsg = req->async_data;
534 struct socket *sock;
535 unsigned flags;
536 int min_ret = 0;
537 int ret;
538
539 sock = sock_from_file(req->file);
540 if (unlikely(!sock))
541 return -ENOTSOCK;
542
543 if (!(req->flags & REQ_F_POLLED) &&
544 (sr->flags & IORING_RECVSEND_POLL_FIRST))
545 return -EAGAIN;
546
547 flags = sr->msg_flags;
548 if (issue_flags & IO_URING_F_NONBLOCK)
549 flags |= MSG_DONTWAIT;
550 if (flags & MSG_WAITALL)
551 min_ret = iov_iter_count(&kmsg->msg.msg_iter);
552
553 kmsg->msg.msg_control_user = sr->msg_control;
554
555 ret = __sys_sendmsg_sock(sock, &kmsg->msg, flags);
556
557 if (ret < min_ret) {
558 if (ret == -EAGAIN && (issue_flags & IO_URING_F_NONBLOCK))
559 return -EAGAIN;
560 if (ret > 0 && io_net_retry(sock, flags)) {
561 kmsg->msg.msg_controllen = 0;
562 kmsg->msg.msg_control = NULL;
563 sr->done_io += ret;
564 return -EAGAIN;
565 }
566 if (ret == -ERESTARTSYS)
567 ret = -EINTR;
568 req_set_fail(req);
569 }
570 io_req_msg_cleanup(req, issue_flags);
571 if (ret >= 0)
572 ret += sr->done_io;
573 else if (sr->done_io)
574 ret = sr->done_io;
575 io_req_set_res(req, ret, 0);
576 return IOU_COMPLETE;
577}
578
579static int io_send_select_buffer(struct io_kiocb *req, unsigned int issue_flags,
580 struct io_br_sel *sel, struct io_async_msghdr *kmsg)
581{
582 struct io_sr_msg *sr = io_kiocb_to_cmd(req, struct io_sr_msg);
583 struct buf_sel_arg arg = {
584 .iovs = &kmsg->fast_iov,
585 .max_len = min_not_zero(sr->len, INT_MAX),
586 .nr_iovs = 1,
587 .buf_group = sr->buf_group,
588 };
589 int ret;
590
591 if (kmsg->vec.iovec) {
592 arg.nr_iovs = kmsg->vec.nr;
593 arg.iovs = kmsg->vec.iovec;
594 arg.mode = KBUF_MODE_FREE;
595 }
596
597 if (!(sr->flags & IORING_RECVSEND_BUNDLE))
598 arg.nr_iovs = 1;
599 else
600 arg.mode |= KBUF_MODE_EXPAND;
601
602 ret = io_buffers_select(req, &arg, sel, issue_flags);
603 if (unlikely(ret < 0))
604 return ret;
605
606 if (arg.iovs != &kmsg->fast_iov && arg.iovs != kmsg->vec.iovec) {
607 kmsg->vec.nr = ret;
608 kmsg->vec.iovec = arg.iovs;
609 req->flags |= REQ_F_NEED_CLEANUP;
610 }
611 sr->len = arg.out_len;
612
613 if (ret == 1) {
614 sr->buf = arg.iovs[0].iov_base;
615 ret = import_ubuf(ITER_SOURCE, sr->buf, sr->len,
616 &kmsg->msg.msg_iter);
617 if (unlikely(ret))
618 return ret;
619 } else {
620 iov_iter_init(&kmsg->msg.msg_iter, ITER_SOURCE,
621 arg.iovs, ret, arg.out_len);
622 }
623
624 return 0;
625}
626
627int io_send(struct io_kiocb *req, unsigned int issue_flags)
628{
629 struct io_sr_msg *sr = io_kiocb_to_cmd(req, struct io_sr_msg);
630 struct io_async_msghdr *kmsg = req->async_data;
631 struct io_br_sel sel = { };
632 struct socket *sock;
633 unsigned flags;
634 int min_ret = 0;
635 int ret;
636
637 sock = sock_from_file(req->file);
638 if (unlikely(!sock))
639 return -ENOTSOCK;
640
641 if (!(req->flags & REQ_F_POLLED) &&
642 (sr->flags & IORING_RECVSEND_POLL_FIRST))
643 return -EAGAIN;
644
645 if (req->flags & REQ_F_IMPORT_BUFFER) {
646 ret = io_import_reg_buf(req, &kmsg->msg.msg_iter,
647 (u64)(uintptr_t)sr->buf, sr->len,
648 ITER_SOURCE, issue_flags);
649 if (unlikely(ret))
650 return ret;
651 req->flags &= ~REQ_F_IMPORT_BUFFER;
652 }
653
654 flags = sr->msg_flags;
655 if (issue_flags & IO_URING_F_NONBLOCK)
656 flags |= MSG_DONTWAIT;
657
658retry_bundle:
659 sel.buf_list = NULL;
660 if (io_do_buffer_select(req)) {
661 ret = io_send_select_buffer(req, issue_flags, &sel, kmsg);
662 if (ret)
663 return ret;
664 }
665
666 /*
667 * If MSG_WAITALL is set, or this is a bundle send, then we need
668 * the full amount. If just bundle is set, if we do a short send
669 * then we complete the bundle sequence rather than continue on.
670 */
671 if (flags & MSG_WAITALL || sr->flags & IORING_RECVSEND_BUNDLE)
672 min_ret = iov_iter_count(&kmsg->msg.msg_iter);
673
674 flags &= ~MSG_INTERNAL_SENDMSG_FLAGS;
675 kmsg->msg.msg_flags = flags;
676 ret = sock_sendmsg(sock, &kmsg->msg);
677 if (ret < min_ret) {
678 if (ret == -EAGAIN && (issue_flags & IO_URING_F_NONBLOCK))
679 return -EAGAIN;
680
681 if (ret > 0 && io_net_retry(sock, flags)) {
682 sr->len -= ret;
683 sr->buf += ret;
684 sr->done_io += ret;
685 return io_net_kbuf_recyle(req, sel.buf_list, kmsg, ret);
686 }
687 if (ret == -ERESTARTSYS)
688 ret = -EINTR;
689 req_set_fail(req);
690 }
691 if (ret >= 0)
692 ret += sr->done_io;
693 else if (sr->done_io)
694 ret = sr->done_io;
695
696 sel.val = ret;
697 if (!io_send_finish(req, kmsg, &sel))
698 goto retry_bundle;
699
700 io_req_msg_cleanup(req, issue_flags);
701 return sel.val;
702}
703
704static int io_recvmsg_mshot_prep(struct io_kiocb *req,
705 struct io_async_msghdr *iomsg,
706 int namelen, size_t controllen)
707{
708 if ((req->flags & (REQ_F_APOLL_MULTISHOT|REQ_F_BUFFER_SELECT)) ==
709 (REQ_F_APOLL_MULTISHOT|REQ_F_BUFFER_SELECT)) {
710 int hdr;
711
712 if (unlikely(namelen < 0))
713 return -EOVERFLOW;
714 if (check_add_overflow(sizeof(struct io_uring_recvmsg_out),
715 namelen, &hdr))
716 return -EOVERFLOW;
717 if (check_add_overflow(hdr, controllen, &hdr))
718 return -EOVERFLOW;
719
720 iomsg->namelen = namelen;
721 iomsg->controllen = controllen;
722 return 0;
723 }
724
725 return 0;
726}
727
728static int io_recvmsg_copy_hdr(struct io_kiocb *req,
729 struct io_async_msghdr *iomsg)
730{
731 struct user_msghdr msg;
732 int ret;
733
734 ret = io_msg_copy_hdr(req, iomsg, &msg, &iomsg->uaddr);
735 if (unlikely(ret))
736 return ret;
737
738 if (!(req->flags & REQ_F_BUFFER_SELECT)) {
739 ret = io_net_import_vec(req, iomsg, msg.msg_iov, msg.msg_iovlen,
740 ITER_DEST);
741 if (unlikely(ret))
742 return ret;
743 }
744 return io_recvmsg_mshot_prep(req, iomsg, msg.msg_namelen,
745 msg.msg_controllen);
746}
747
748static int io_recvmsg_prep_setup(struct io_kiocb *req)
749{
750 struct io_sr_msg *sr = io_kiocb_to_cmd(req, struct io_sr_msg);
751 struct io_async_msghdr *kmsg;
752
753 kmsg = io_msg_alloc_async(req);
754 if (unlikely(!kmsg))
755 return -ENOMEM;
756
757 if (req->opcode == IORING_OP_RECV) {
758 kmsg->msg.msg_name = NULL;
759 kmsg->msg.msg_namelen = 0;
760 kmsg->msg.msg_inq = 0;
761 kmsg->msg.msg_control = NULL;
762 kmsg->msg.msg_get_inq = 1;
763 kmsg->msg.msg_controllen = 0;
764 kmsg->msg.msg_ubuf = NULL;
765
766 if (req->flags & REQ_F_BUFFER_SELECT)
767 return 0;
768 if (sr->flags & IORING_RECVSEND_FIXED_BUF) {
769 req->flags |= REQ_F_IMPORT_BUFFER;
770 return 0;
771 }
772 return import_ubuf(ITER_DEST, sr->buf, sr->len,
773 &kmsg->msg.msg_iter);
774 }
775
776 return io_recvmsg_copy_hdr(req, kmsg);
777}
778
779#define RECVMSG_FLAGS (IORING_RECVSEND_POLL_FIRST | IORING_RECV_MULTISHOT | \
780 IORING_RECVSEND_BUNDLE | IORING_RECVSEND_FIXED_BUF)
781
782int io_recvmsg_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
783{
784 struct io_sr_msg *sr = io_kiocb_to_cmd(req, struct io_sr_msg);
785
786 sr->done_io = 0;
787
788 if (unlikely(sqe->addr2))
789 return -EINVAL;
790
791 sr->umsg = u64_to_user_ptr(READ_ONCE(sqe->addr));
792 sr->len = READ_ONCE(sqe->len);
793 if (unlikely(sr->len < 0))
794 return -EINVAL;
795 sr->flags = READ_ONCE(sqe->ioprio);
796 if (sr->flags & ~RECVMSG_FLAGS)
797 return -EINVAL;
798 if (sr->flags & IORING_RECVSEND_FIXED_BUF) {
799 /* registered buffer recv only for plain IORING_OP_RECV */
800 if (req->opcode != IORING_OP_RECV ||
801 (req->flags & REQ_F_BUFFER_SELECT) ||
802 (sr->flags & (IORING_RECV_MULTISHOT | IORING_RECVSEND_BUNDLE)))
803 return -EINVAL;
804 req->buf_index = READ_ONCE(sqe->buf_index);
805 }
806 sr->msg_flags = READ_ONCE(sqe->msg_flags);
807 if (sr->msg_flags & MSG_DONTWAIT)
808 req->flags |= REQ_F_NOWAIT;
809 if (sr->msg_flags & MSG_ERRQUEUE)
810 req->flags |= REQ_F_CLEAR_POLLIN;
811 if (req->flags & REQ_F_BUFFER_SELECT)
812 sr->buf_group = req->buf_index;
813 sr->mshot_total_len = sr->mshot_len = 0;
814 if (sr->flags & IORING_RECV_MULTISHOT) {
815 if (!(req->flags & REQ_F_BUFFER_SELECT))
816 return -EINVAL;
817 if (sr->msg_flags & MSG_WAITALL)
818 return -EINVAL;
819 if (req->opcode == IORING_OP_RECV) {
820 sr->mshot_len = sr->len;
821 sr->mshot_total_len = READ_ONCE(sqe->optlen);
822 if (sr->mshot_total_len)
823 sr->flags |= IORING_RECV_MSHOT_LIM;
824 } else if (sqe->optlen) {
825 return -EINVAL;
826 }
827 req->flags |= REQ_F_APOLL_MULTISHOT;
828 } else if (sqe->optlen) {
829 return -EINVAL;
830 }
831
832 if (sr->flags & IORING_RECVSEND_BUNDLE) {
833 if (req->opcode == IORING_OP_RECVMSG)
834 return -EINVAL;
835 }
836
837 if (io_is_compat(req->ctx))
838 sr->msg_flags |= MSG_CMSG_COMPAT;
839
840 sr->nr_multishot_loops = 0;
841 return io_recvmsg_prep_setup(req);
842}
843
844/* bits to clear in old and inherit in new cflags on bundle retry */
845#define CQE_F_MASK (IORING_CQE_F_SOCK_NONEMPTY|IORING_CQE_F_MORE|\
846 IORING_CQE_F_BUF_MORE)
847
848/*
849 * Finishes io_recv and io_recvmsg.
850 *
851 * Returns true if it is actually finished, or false if it should run
852 * again (for multishot).
853 */
854static inline bool io_recv_finish(struct io_kiocb *req,
855 struct io_async_msghdr *kmsg,
856 struct io_br_sel *sel, bool mshot_finished,
857 unsigned issue_flags, int consumed)
858{
859 struct io_sr_msg *sr = io_kiocb_to_cmd(req, struct io_sr_msg);
860 unsigned int cflags = 0;
861
862 if (kmsg->msg.msg_inq > 0)
863 cflags |= IORING_CQE_F_SOCK_NONEMPTY;
864
865 if (sel->val > 0 && sr->flags & IORING_RECV_MSHOT_LIM) {
866 /*
867 * If sr->len hits zero, the limit has been reached. Mark
868 * mshot as finished, and flag MSHOT_DONE as well to prevent
869 * a potential bundle from being retried.
870 */
871 sr->mshot_total_len -= min_t(int, sel->val, sr->mshot_total_len);
872 if (!sr->mshot_total_len) {
873 sr->flags |= IORING_RECV_MSHOT_DONE;
874 mshot_finished = true;
875 }
876 }
877
878 if (sr->flags & IORING_RECVSEND_BUNDLE) {
879 size_t this_ret = sel->val - sr->done_io;
880
881 cflags |= io_put_kbufs(req, consumed, sel->buf_list, io_bundle_nbufs(kmsg, consumed));
882 if (sr->flags & IORING_RECV_RETRY)
883 cflags = req->cqe.flags | (cflags & CQE_F_MASK);
884 if (sr->mshot_len && sel->val >= sr->mshot_len)
885 sr->flags |= IORING_RECV_MSHOT_CAP;
886 /* bundle with no more immediate buffers, we're done */
887 if (req->flags & REQ_F_BL_EMPTY)
888 goto finish;
889 /*
890 * If more is available AND it was a full transfer, retry and
891 * append to this one
892 */
893 if (!(sr->flags & IORING_RECV_NO_RETRY) &&
894 kmsg->msg.msg_inq > 1 && this_ret > 0 &&
895 !iov_iter_count(&kmsg->msg.msg_iter)) {
896 req->cqe.flags = cflags & ~CQE_F_MASK;
897 sr->len = kmsg->msg.msg_inq;
898 sr->done_io += this_ret;
899 sr->flags |= IORING_RECV_RETRY;
900 return false;
901 }
902 } else {
903 cflags |= io_put_kbuf(req, consumed, sel->buf_list);
904 }
905
906 /*
907 * Fill CQE for this receive and see if we should keep trying to
908 * receive from this socket.
909 */
910 if ((req->flags & REQ_F_APOLL_MULTISHOT) && !mshot_finished &&
911 io_req_post_cqe(req, sel->val, cflags | IORING_CQE_F_MORE)) {
912 sel->val = IOU_RETRY;
913 io_mshot_prep_retry(req, kmsg);
914 /* Known not-empty or unknown state, retry */
915 if (cflags & IORING_CQE_F_SOCK_NONEMPTY || kmsg->msg.msg_inq < 0) {
916 if (sr->nr_multishot_loops++ < MULTISHOT_MAX_RETRY &&
917 !(sr->flags & IORING_RECV_MSHOT_CAP)) {
918 return false;
919 }
920 /* mshot retries exceeded, force a requeue */
921 sr->nr_multishot_loops = 0;
922 sr->flags &= ~IORING_RECV_MSHOT_CAP;
923 if (issue_flags & IO_URING_F_MULTISHOT)
924 sel->val = IOU_REQUEUE;
925 }
926 return true;
927 }
928
929 /* Finish the request / stop multishot. */
930finish:
931 io_req_set_res(req, sel->val, cflags);
932 sel->val = IOU_COMPLETE;
933 io_req_msg_cleanup(req, issue_flags);
934 return true;
935}
936
937static int io_recvmsg_prep_multishot(struct io_async_msghdr *kmsg,
938 struct io_sr_msg *sr, void __user **buf,
939 size_t *len)
940{
941 unsigned long ubuf = (unsigned long) *buf;
942 unsigned long hdr;
943
944 hdr = sizeof(struct io_uring_recvmsg_out) + kmsg->namelen +
945 kmsg->controllen;
946 if (*len < hdr)
947 return -EFAULT;
948
949 if (kmsg->controllen) {
950 unsigned long control = ubuf + hdr - kmsg->controllen;
951
952 kmsg->msg.msg_control_user = (void __user *) control;
953 kmsg->msg.msg_controllen = kmsg->controllen;
954 }
955
956 sr->buf = *buf; /* stash for later copy */
957 *buf = (void __user *) (ubuf + hdr);
958 kmsg->payloadlen = *len = *len - hdr;
959 return 0;
960}
961
962struct io_recvmsg_multishot_hdr {
963 struct io_uring_recvmsg_out msg;
964 struct sockaddr_storage addr;
965};
966
967static int io_recvmsg_multishot(struct socket *sock, struct io_sr_msg *io,
968 struct io_async_msghdr *kmsg,
969 unsigned int flags, bool *finished)
970{
971 int err;
972 int copy_len;
973 struct io_recvmsg_multishot_hdr hdr;
974
975 if (kmsg->namelen)
976 kmsg->msg.msg_name = &hdr.addr;
977 kmsg->msg.msg_flags = flags & (MSG_CMSG_CLOEXEC|MSG_CMSG_COMPAT);
978 kmsg->msg.msg_namelen = 0;
979
980 if (sock->file->f_flags & O_NONBLOCK)
981 flags |= MSG_DONTWAIT;
982
983 err = sock_recvmsg(sock, &kmsg->msg, flags);
984 *finished = err <= 0;
985 if (err < 0)
986 return err;
987
988 hdr.msg = (struct io_uring_recvmsg_out) {
989 .controllen = kmsg->controllen - kmsg->msg.msg_controllen,
990 .flags = kmsg->msg.msg_flags & ~MSG_CMSG_COMPAT
991 };
992
993 hdr.msg.payloadlen = err;
994 if (err > kmsg->payloadlen)
995 err = kmsg->payloadlen;
996
997 copy_len = sizeof(struct io_uring_recvmsg_out);
998 if (kmsg->msg.msg_namelen > kmsg->namelen)
999 copy_len += kmsg->namelen;
1000 else
1001 copy_len += kmsg->msg.msg_namelen;
1002
1003 /*
1004 * "fromlen shall refer to the value before truncation.."
1005 * 1003.1g
1006 */
1007 hdr.msg.namelen = kmsg->msg.msg_namelen;
1008
1009 /* ensure that there is no gap between hdr and sockaddr_storage */
1010 BUILD_BUG_ON(offsetof(struct io_recvmsg_multishot_hdr, addr) !=
1011 sizeof(struct io_uring_recvmsg_out));
1012 if (copy_to_user(io->buf, &hdr, copy_len)) {
1013 *finished = true;
1014 return -EFAULT;
1015 }
1016
1017 return sizeof(struct io_uring_recvmsg_out) + kmsg->namelen +
1018 kmsg->controllen + err;
1019}
1020
1021int io_recvmsg(struct io_kiocb *req, unsigned int issue_flags)
1022{
1023 struct io_sr_msg *sr = io_kiocb_to_cmd(req, struct io_sr_msg);
1024 struct io_async_msghdr *kmsg = req->async_data;
1025 struct io_br_sel sel = { };
1026 struct socket *sock;
1027 unsigned flags;
1028 int ret, min_ret = 0;
1029 bool force_nonblock = issue_flags & IO_URING_F_NONBLOCK;
1030 bool mshot_finished = true;
1031 int consumed = 0;
1032 size_t len;
1033
1034 sock = sock_from_file(req->file);
1035 if (unlikely(!sock))
1036 return -ENOTSOCK;
1037
1038 if (!(req->flags & REQ_F_POLLED) &&
1039 (sr->flags & IORING_RECVSEND_POLL_FIRST))
1040 return -EAGAIN;
1041
1042 flags = sr->msg_flags;
1043 if (force_nonblock)
1044 flags |= MSG_DONTWAIT;
1045
1046retry_multishot:
1047 sel.buf_list = NULL;
1048 len = sr->len;
1049 if (io_do_buffer_select(req)) {
1050 sel = io_buffer_select(req, &len, sr->buf_group, issue_flags);
1051 if (!sel.addr)
1052 return -ENOBUFS;
1053
1054 if (req->flags & REQ_F_APOLL_MULTISHOT) {
1055 ret = io_recvmsg_prep_multishot(kmsg, sr, &sel.addr, &len);
1056 if (ret) {
1057 io_kbuf_recycle(req, sel.buf_list, issue_flags);
1058 return ret;
1059 }
1060 }
1061
1062 iov_iter_ubuf(&kmsg->msg.msg_iter, ITER_DEST, sel.addr, len);
1063 }
1064
1065 kmsg->msg.msg_get_inq = 1;
1066 kmsg->msg.msg_inq = -1;
1067 if (req->flags & REQ_F_APOLL_MULTISHOT) {
1068 ret = io_recvmsg_multishot(sock, sr, kmsg, flags,
1069 &mshot_finished);
1070 consumed = ret;
1071 } else {
1072 /* disable partial retry for recvmsg with cmsg attached */
1073 if (flags & MSG_WAITALL && !kmsg->msg.msg_controllen)
1074 min_ret = iov_iter_count(&kmsg->msg.msg_iter);
1075
1076 ret = __sys_recvmsg_sock(sock, &kmsg->msg, sr->umsg,
1077 kmsg->uaddr, flags);
1078 /*
1079 * With MSG_TRUNC, the net layer will return the full size of
1080 * the packet, even if we only filled part of it in the buffers.
1081 * Adjust the returned size to consume only the real part of the
1082 * buffer.
1083 */
1084 consumed = ret;
1085 if (ret > 0)
1086 consumed = min_t(size_t, ret, len);
1087 }
1088
1089 if (ret < min_ret) {
1090 if (ret == -EAGAIN && force_nonblock) {
1091 io_kbuf_recycle(req, sel.buf_list, issue_flags);
1092 return IOU_RETRY;
1093 }
1094 if (ret > 0 && io_net_retry(sock, flags)) {
1095 sr->done_io += ret;
1096 return io_net_kbuf_recyle(req, sel.buf_list, kmsg, ret);
1097 }
1098 if (ret == -ERESTARTSYS)
1099 ret = -EINTR;
1100 req_set_fail(req);
1101 } else if ((flags & MSG_WAITALL) && (kmsg->msg.msg_flags & (MSG_TRUNC | MSG_CTRUNC))) {
1102 req_set_fail(req);
1103 }
1104
1105 if (ret > 0)
1106 ret += sr->done_io;
1107 else if (sr->done_io)
1108 ret = sr->done_io;
1109 else
1110 io_kbuf_recycle(req, sel.buf_list, issue_flags);
1111
1112 sel.val = ret;
1113 if (!io_recv_finish(req, kmsg, &sel, mshot_finished, issue_flags, consumed))
1114 goto retry_multishot;
1115
1116 return sel.val;
1117}
1118
1119static int io_recv_buf_select(struct io_kiocb *req, struct io_async_msghdr *kmsg,
1120 struct io_br_sel *sel, unsigned int issue_flags)
1121{
1122 struct io_sr_msg *sr = io_kiocb_to_cmd(req, struct io_sr_msg);
1123 size_t len;
1124 int ret;
1125
1126 /*
1127 * If the ring isn't locked, then don't use the peek interface
1128 * to grab multiple buffers as we will lock/unlock between
1129 * this selection and posting the buffers.
1130 */
1131 if (!(issue_flags & IO_URING_F_UNLOCKED) &&
1132 sr->flags & IORING_RECVSEND_BUNDLE) {
1133 struct buf_sel_arg arg = {
1134 .iovs = &kmsg->fast_iov,
1135 .nr_iovs = 1,
1136 .mode = KBUF_MODE_EXPAND,
1137 .buf_group = sr->buf_group,
1138 };
1139
1140 if (kmsg->vec.iovec) {
1141 arg.nr_iovs = kmsg->vec.nr;
1142 arg.iovs = kmsg->vec.iovec;
1143 arg.mode |= KBUF_MODE_FREE;
1144 }
1145
1146 if (sel->val)
1147 arg.max_len = sel->val;
1148 else if (kmsg->msg.msg_inq > 1)
1149 arg.max_len = min_not_zero(sel->val, (ssize_t) kmsg->msg.msg_inq);
1150
1151 /* if mshot limited, ensure we don't go over */
1152 if (sr->flags & IORING_RECV_MSHOT_LIM)
1153 arg.max_len = min_not_zero(arg.max_len, sr->mshot_total_len);
1154 ret = io_buffers_peek(req, &arg, sel);
1155 if (unlikely(ret < 0))
1156 return ret;
1157
1158 if (arg.iovs != &kmsg->fast_iov && arg.iovs != kmsg->vec.iovec) {
1159 kmsg->vec.nr = ret;
1160 kmsg->vec.iovec = arg.iovs;
1161 req->flags |= REQ_F_NEED_CLEANUP;
1162 }
1163 if (arg.partial_map)
1164 sr->flags |= IORING_RECV_PARTIAL_MAP;
1165
1166 /* special case 1 vec, can be a fast path */
1167 if (ret == 1) {
1168 sr->buf = arg.iovs[0].iov_base;
1169 len = sr->len = arg.iovs[0].iov_len;
1170 goto map_ubuf;
1171 }
1172 iov_iter_init(&kmsg->msg.msg_iter, ITER_DEST, arg.iovs, ret,
1173 arg.out_len);
1174 len = arg.out_len;
1175 } else {
1176 len = sel->val;
1177
1178 *sel = io_buffer_select(req, &len, sr->buf_group, issue_flags);
1179 if (!sel->addr)
1180 return -ENOBUFS;
1181 sr->buf = sel->addr;
1182 sr->len = len;
1183map_ubuf:
1184 ret = import_ubuf(ITER_DEST, sr->buf, sr->len,
1185 &kmsg->msg.msg_iter);
1186 if (unlikely(ret))
1187 return ret;
1188 }
1189
1190 return len;
1191}
1192
1193int io_recv(struct io_kiocb *req, unsigned int issue_flags)
1194{
1195 struct io_sr_msg *sr = io_kiocb_to_cmd(req, struct io_sr_msg);
1196 struct io_async_msghdr *kmsg = req->async_data;
1197 struct io_br_sel sel;
1198 struct socket *sock;
1199 unsigned flags;
1200 int ret, min_ret = 0, consumed = 0;
1201 bool force_nonblock = issue_flags & IO_URING_F_NONBLOCK;
1202 bool mshot_finished;
1203 size_t len = 0;
1204
1205 sock = sock_from_file(req->file);
1206 if (unlikely(!sock))
1207 return -ENOTSOCK;
1208
1209 if (!(req->flags & REQ_F_POLLED) &&
1210 (sr->flags & IORING_RECVSEND_POLL_FIRST))
1211 return -EAGAIN;
1212
1213 flags = sr->msg_flags;
1214 if (force_nonblock)
1215 flags |= MSG_DONTWAIT;
1216
1217 if (req->flags & REQ_F_IMPORT_BUFFER) {
1218 ret = io_import_reg_buf(req, &kmsg->msg.msg_iter,
1219 (u64)(uintptr_t)sr->buf, sr->len,
1220 ITER_DEST, issue_flags);
1221 if (unlikely(ret)) {
1222 kmsg->msg.msg_inq = -1;
1223 sel.buf_list = NULL;
1224 goto out_free;
1225 }
1226 req->flags &= ~REQ_F_IMPORT_BUFFER;
1227 }
1228
1229retry_multishot:
1230 sel.buf_list = NULL;
1231 len = sr->len;
1232 if (io_do_buffer_select(req)) {
1233 sel.val = sr->len;
1234 ret = io_recv_buf_select(req, kmsg, &sel, issue_flags);
1235 if (unlikely(ret < 0)) {
1236 kmsg->msg.msg_inq = -1;
1237 goto out_free;
1238 }
1239 len = ret;
1240 sr->buf = NULL;
1241 }
1242
1243 kmsg->msg.msg_flags = 0;
1244 kmsg->msg.msg_inq = -1;
1245
1246 if (flags & MSG_WAITALL)
1247 min_ret = iov_iter_count(&kmsg->msg.msg_iter);
1248
1249 ret = sock_recvmsg(sock, &kmsg->msg, flags);
1250 if (ret < min_ret) {
1251 if (ret == -EAGAIN && force_nonblock) {
1252 io_kbuf_recycle(req, sel.buf_list, issue_flags);
1253 return IOU_RETRY;
1254 }
1255 if (ret > 0 && io_net_retry(sock, flags)) {
1256 sr->len -= ret;
1257 sr->buf += ret;
1258 sr->done_io += ret;
1259 return io_net_kbuf_recyle(req, sel.buf_list, kmsg, ret);
1260 }
1261 if (ret == -ERESTARTSYS)
1262 ret = -EINTR;
1263 req_set_fail(req);
1264 } else if ((flags & MSG_WAITALL) && (kmsg->msg.msg_flags & (MSG_TRUNC | MSG_CTRUNC))) {
1265out_free:
1266 req_set_fail(req);
1267 }
1268
1269 mshot_finished = ret <= 0;
1270
1271 /*
1272 * With MSG_TRUNC, the net layer will return the full size of
1273 * the packet, even if we only filled part of it in the buffers.
1274 * Adjust the returned size to consume only the real part of the
1275 * buffer.
1276 */
1277 consumed = ret;
1278 if (ret > 0)
1279 consumed = min_t(size_t, ret, len);
1280
1281 if (ret > 0)
1282 ret += sr->done_io;
1283 else if (sr->done_io)
1284 ret = sr->done_io;
1285 else
1286 io_kbuf_recycle(req, sel.buf_list, issue_flags);
1287
1288 sel.val = ret;
1289 if (!io_recv_finish(req, kmsg, &sel, mshot_finished, issue_flags, consumed))
1290 goto retry_multishot;
1291
1292 return sel.val;
1293}
1294
1295int io_recvzc_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
1296{
1297 struct io_recvzc *zc = io_kiocb_to_cmd(req, struct io_recvzc);
1298 unsigned ifq_idx;
1299
1300 if (unlikely(sqe->addr2 || sqe->addr || sqe->addr3))
1301 return -EINVAL;
1302
1303 ifq_idx = READ_ONCE(sqe->zcrx_ifq_idx);
1304 zc->ifq = xa_load(&req->ctx->zcrx_ctxs, ifq_idx);
1305 if (!zc->ifq)
1306 return -EINVAL;
1307
1308 zc->len = READ_ONCE(sqe->len);
1309 zc->flags = READ_ONCE(sqe->ioprio);
1310 if (READ_ONCE(sqe->msg_flags))
1311 return -EINVAL;
1312 if (zc->flags & ~(IORING_RECVSEND_POLL_FIRST | IORING_RECV_MULTISHOT))
1313 return -EINVAL;
1314 /* multishot required */
1315 if (!(zc->flags & IORING_RECV_MULTISHOT))
1316 return -EINVAL;
1317 /* All data completions are posted as aux CQEs. */
1318 req->flags |= REQ_F_APOLL_MULTISHOT;
1319
1320 return 0;
1321}
1322
1323int io_recvzc(struct io_kiocb *req, unsigned int issue_flags)
1324{
1325 struct io_recvzc *zc = io_kiocb_to_cmd(req, struct io_recvzc);
1326 struct socket *sock;
1327 unsigned int len;
1328 int ret;
1329
1330 sock = sock_from_file(req->file);
1331 if (unlikely(!sock))
1332 return -ENOTSOCK;
1333
1334 if (!(req->flags & REQ_F_POLLED) &&
1335 (zc->flags & IORING_RECVSEND_POLL_FIRST))
1336 return -EAGAIN;
1337
1338 len = zc->len;
1339 ret = io_zcrx_recv(req, zc->ifq, sock, 0, issue_flags, &zc->len);
1340 if (len && zc->len == 0) {
1341 io_req_set_res(req, 0, 0);
1342
1343 return IOU_COMPLETE;
1344 }
1345 if (unlikely(ret <= 0) && ret != -EAGAIN) {
1346 if (ret == -ERESTARTSYS)
1347 ret = -EINTR;
1348 if (ret == IOU_REQUEUE)
1349 return IOU_REQUEUE;
1350
1351 req_set_fail(req);
1352 io_req_set_res(req, ret, 0);
1353 return IOU_COMPLETE;
1354 }
1355 return IOU_RETRY;
1356}
1357
1358void io_send_zc_cleanup(struct io_kiocb *req)
1359{
1360 struct io_sr_msg *zc = io_kiocb_to_cmd(req, struct io_sr_msg);
1361 struct io_async_msghdr *io = req->async_data;
1362
1363 if (req_has_async_data(req))
1364 io_netmsg_iovec_free(io);
1365 if (zc->notif) {
1366 io_notif_flush(zc->notif);
1367 zc->notif = NULL;
1368 }
1369}
1370
1371#define IO_ZC_FLAGS_COMMON (IORING_RECVSEND_POLL_FIRST | IORING_RECVSEND_FIXED_BUF)
1372#define IO_ZC_FLAGS_VALID (IO_ZC_FLAGS_COMMON | IORING_SEND_ZC_REPORT_USAGE | \
1373 IORING_SEND_VECTORIZED)
1374
1375int io_send_zc_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
1376{
1377 struct io_sr_msg *zc = io_kiocb_to_cmd(req, struct io_sr_msg);
1378 struct io_ring_ctx *ctx = req->ctx;
1379 struct io_async_msghdr *iomsg;
1380 struct io_kiocb *notif;
1381 u64 user_data;
1382 int ret;
1383
1384 zc->done_io = 0;
1385
1386 if (unlikely(READ_ONCE(sqe->__pad2[0])))
1387 return -EINVAL;
1388 /* we don't support IOSQE_CQE_SKIP_SUCCESS just yet */
1389 if (req->flags & REQ_F_CQE_SKIP)
1390 return -EINVAL;
1391
1392 notif = zc->notif = io_alloc_notif(ctx);
1393 if (!notif)
1394 return -ENOMEM;
1395 user_data = READ_ONCE(sqe->addr3);
1396 if (!user_data)
1397 user_data = req->cqe.user_data;
1398
1399 notif->cqe.user_data = user_data;
1400 notif->cqe.res = 0;
1401 notif->cqe.flags = IORING_CQE_F_NOTIF;
1402 req->flags |= REQ_F_NEED_CLEANUP | REQ_F_POLL_NO_LAZY;
1403
1404 zc->flags = READ_ONCE(sqe->ioprio);
1405 if (unlikely(zc->flags & ~IO_ZC_FLAGS_COMMON)) {
1406 if (zc->flags & ~IO_ZC_FLAGS_VALID)
1407 return -EINVAL;
1408 if (zc->flags & IORING_SEND_ZC_REPORT_USAGE) {
1409 struct io_notif_data *nd = io_notif_to_data(notif);
1410
1411 nd->zc_report = true;
1412 nd->zc_used = false;
1413 nd->zc_copied = false;
1414 }
1415 }
1416
1417 zc->len = READ_ONCE(sqe->len);
1418 zc->msg_flags = READ_ONCE(sqe->msg_flags) | MSG_NOSIGNAL | MSG_ZEROCOPY;
1419 req->buf_index = READ_ONCE(sqe->buf_index);
1420 if (zc->msg_flags & MSG_DONTWAIT)
1421 req->flags |= REQ_F_NOWAIT;
1422
1423 if (io_is_compat(ctx))
1424 zc->msg_flags |= MSG_CMSG_COMPAT;
1425
1426 iomsg = io_msg_alloc_async(req);
1427 if (unlikely(!iomsg))
1428 return -ENOMEM;
1429
1430 if (req->opcode == IORING_OP_SEND_ZC) {
1431 ret = io_send_setup(req, sqe);
1432 } else {
1433 if (unlikely(sqe->addr2 || sqe->file_index))
1434 return -EINVAL;
1435 ret = io_sendmsg_setup(req, sqe);
1436 }
1437 if (unlikely(ret))
1438 return ret;
1439
1440 if (!(zc->flags & IORING_RECVSEND_FIXED_BUF)) {
1441 iomsg->msg.sg_from_iter = io_sg_from_iter_iovec;
1442 return io_notif_account_mem(zc->notif, iomsg->msg.msg_iter.count);
1443 }
1444 iomsg->msg.sg_from_iter = io_sg_from_iter;
1445 return 0;
1446}
1447
1448static int io_sg_from_iter_iovec(struct sk_buff *skb,
1449 struct iov_iter *from, size_t length)
1450{
1451 skb_zcopy_downgrade_managed(skb);
1452 return zerocopy_fill_skb_from_iter(skb, from, length);
1453}
1454
1455static int io_sg_from_iter(struct sk_buff *skb,
1456 struct iov_iter *from, size_t length)
1457{
1458 struct skb_shared_info *shinfo = skb_shinfo(skb);
1459 int frag = shinfo->nr_frags;
1460 int ret = 0;
1461 struct bvec_iter bi;
1462 ssize_t copied = 0;
1463 unsigned long truesize = 0;
1464
1465 if (!frag)
1466 shinfo->flags |= SKBFL_MANAGED_FRAG_REFS;
1467 else if (unlikely(!skb_zcopy_managed(skb)))
1468 return zerocopy_fill_skb_from_iter(skb, from, length);
1469
1470 bi.bi_size = min(from->count, length);
1471 bi.bi_offset = from->iov_offset;
1472 bi.bi_idx = 0;
1473
1474 while (bi.bi_size && frag < MAX_SKB_FRAGS) {
1475 struct bio_vec v = mp_bvec_iter_bvec(from->bvec, bi);
1476
1477 copied += v.bv_len;
1478 truesize += PAGE_ALIGN(v.bv_len + v.bv_offset);
1479 __skb_fill_page_desc_noacc(shinfo, frag++, v.bv_page,
1480 v.bv_offset, v.bv_len);
1481 bvec_iter_advance_single(from->bvec, &bi, v.bv_len);
1482 }
1483 if (bi.bi_size)
1484 ret = -EMSGSIZE;
1485
1486 shinfo->nr_frags = frag;
1487 from->bvec += bi.bi_idx;
1488 from->nr_segs -= bi.bi_idx;
1489 from->count -= copied;
1490 from->iov_offset = bi.bi_offset;
1491
1492 skb->data_len += copied;
1493 skb->len += copied;
1494 skb->truesize += truesize;
1495 return ret;
1496}
1497
1498static int io_send_zc_import(struct io_kiocb *req,
1499 struct io_async_msghdr *kmsg,
1500 unsigned int issue_flags)
1501{
1502 struct io_sr_msg *sr = io_kiocb_to_cmd(req, struct io_sr_msg);
1503 struct io_kiocb *notif = sr->notif;
1504 int ret;
1505
1506 WARN_ON_ONCE(!(sr->flags & IORING_RECVSEND_FIXED_BUF));
1507
1508 notif->buf_index = req->buf_index;
1509
1510 if (!(sr->flags & IORING_SEND_VECTORIZED)) {
1511 ret = io_import_reg_buf(notif, &kmsg->msg.msg_iter,
1512 (u64)(uintptr_t)sr->buf, sr->len,
1513 ITER_SOURCE, issue_flags);
1514 } else {
1515 unsigned uvec_segs = kmsg->msg.msg_iter.nr_segs;
1516
1517 ret = io_import_reg_vec(ITER_SOURCE, &kmsg->msg.msg_iter,
1518 notif, &kmsg->vec, uvec_segs,
1519 issue_flags);
1520 }
1521
1522 if (unlikely(ret))
1523 return ret;
1524 req->flags &= ~REQ_F_IMPORT_BUFFER;
1525 return 0;
1526}
1527
1528int io_sendmsg_zc(struct io_kiocb *req, unsigned int issue_flags)
1529{
1530 struct io_sr_msg *sr = io_kiocb_to_cmd(req, struct io_sr_msg);
1531 struct io_async_msghdr *kmsg = req->async_data;
1532 struct socket *sock;
1533 unsigned msg_flags;
1534 int ret, min_ret = 0;
1535
1536 sock = sock_from_file(req->file);
1537 if (unlikely(!sock))
1538 return -ENOTSOCK;
1539 if (!test_bit(SOCK_SUPPORT_ZC, &sock->flags))
1540 return -EOPNOTSUPP;
1541 if (!(req->flags & REQ_F_POLLED) &&
1542 (sr->flags & IORING_RECVSEND_POLL_FIRST))
1543 return -EAGAIN;
1544
1545 if (req->flags & REQ_F_IMPORT_BUFFER) {
1546 ret = io_send_zc_import(req, kmsg, issue_flags);
1547 if (unlikely(ret))
1548 return ret;
1549 }
1550
1551 msg_flags = sr->msg_flags;
1552 if (issue_flags & IO_URING_F_NONBLOCK)
1553 msg_flags |= MSG_DONTWAIT;
1554 if (msg_flags & MSG_WAITALL)
1555 min_ret = iov_iter_count(&kmsg->msg.msg_iter);
1556
1557 kmsg->msg.msg_ubuf = &io_notif_to_data(sr->notif)->uarg;
1558
1559 if (req->opcode == IORING_OP_SEND_ZC) {
1560 msg_flags &= ~MSG_INTERNAL_SENDMSG_FLAGS;
1561 kmsg->msg.msg_flags = msg_flags;
1562 ret = sock_sendmsg(sock, &kmsg->msg);
1563 } else {
1564 kmsg->msg.msg_control_user = sr->msg_control;
1565 ret = __sys_sendmsg_sock(sock, &kmsg->msg, msg_flags);
1566 }
1567
1568 if (unlikely(ret < min_ret)) {
1569 if (ret == -EAGAIN && (issue_flags & IO_URING_F_NONBLOCK))
1570 return -EAGAIN;
1571
1572 if (ret > 0 && io_net_retry(sock, sr->msg_flags)) {
1573 sr->done_io += ret;
1574 return -EAGAIN;
1575 }
1576 if (ret == -ERESTARTSYS)
1577 ret = -EINTR;
1578 req_set_fail(req);
1579 }
1580
1581 if (ret >= 0)
1582 ret += sr->done_io;
1583 else if (sr->done_io)
1584 ret = sr->done_io;
1585
1586 /*
1587 * If we're in io-wq we can't rely on tw ordering guarantees, defer
1588 * flushing notif to io_send_zc_cleanup()
1589 */
1590 if (!(issue_flags & IO_URING_F_UNLOCKED)) {
1591 io_notif_flush(sr->notif);
1592 sr->notif = NULL;
1593 io_req_msg_cleanup(req, 0);
1594 }
1595 io_req_set_res(req, ret, IORING_CQE_F_MORE);
1596 return IOU_COMPLETE;
1597}
1598
1599void io_sendrecv_fail(struct io_kiocb *req)
1600{
1601 struct io_sr_msg *sr = io_kiocb_to_cmd(req, struct io_sr_msg);
1602
1603 if (sr->done_io)
1604 req->cqe.res = sr->done_io;
1605
1606 if ((req->flags & REQ_F_NEED_CLEANUP) &&
1607 (req->opcode == IORING_OP_SEND_ZC || req->opcode == IORING_OP_SENDMSG_ZC))
1608 req->cqe.flags |= IORING_CQE_F_MORE;
1609}
1610
1611#define ACCEPT_FLAGS (IORING_ACCEPT_MULTISHOT | IORING_ACCEPT_DONTWAIT | \
1612 IORING_ACCEPT_POLL_FIRST)
1613
1614int io_accept_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
1615{
1616 struct io_accept *accept = io_kiocb_to_cmd(req, struct io_accept);
1617
1618 if (sqe->len || sqe->buf_index)
1619 return -EINVAL;
1620
1621 accept->addr = u64_to_user_ptr(READ_ONCE(sqe->addr));
1622 accept->addr_len = u64_to_user_ptr(READ_ONCE(sqe->addr2));
1623 accept->flags = READ_ONCE(sqe->accept_flags);
1624 accept->nofile = rlimit(RLIMIT_NOFILE);
1625 accept->iou_flags = READ_ONCE(sqe->ioprio);
1626 if (accept->iou_flags & ~ACCEPT_FLAGS)
1627 return -EINVAL;
1628
1629 accept->file_slot = READ_ONCE(sqe->file_index);
1630 if (accept->file_slot) {
1631 if (accept->flags & SOCK_CLOEXEC)
1632 return -EINVAL;
1633 if (accept->iou_flags & IORING_ACCEPT_MULTISHOT &&
1634 accept->file_slot != IORING_FILE_INDEX_ALLOC)
1635 return -EINVAL;
1636 }
1637 if (accept->flags & ~(SOCK_CLOEXEC | SOCK_NONBLOCK))
1638 return -EINVAL;
1639 if (SOCK_NONBLOCK != O_NONBLOCK && (accept->flags & SOCK_NONBLOCK))
1640 accept->flags = (accept->flags & ~SOCK_NONBLOCK) | O_NONBLOCK;
1641 if (accept->iou_flags & IORING_ACCEPT_MULTISHOT)
1642 req->flags |= REQ_F_APOLL_MULTISHOT;
1643 if (accept->iou_flags & IORING_ACCEPT_DONTWAIT)
1644 req->flags |= REQ_F_NOWAIT;
1645 return 0;
1646}
1647
1648int io_accept(struct io_kiocb *req, unsigned int issue_flags)
1649{
1650 struct io_accept *accept = io_kiocb_to_cmd(req, struct io_accept);
1651 bool force_nonblock = issue_flags & IO_URING_F_NONBLOCK;
1652 bool fixed = !!accept->file_slot;
1653 struct proto_accept_arg arg = {
1654 .flags = force_nonblock ? O_NONBLOCK : 0,
1655 };
1656 struct file *file;
1657 unsigned cflags;
1658 int ret, fd;
1659
1660 if (!(req->flags & REQ_F_POLLED) &&
1661 accept->iou_flags & IORING_ACCEPT_POLL_FIRST)
1662 return -EAGAIN;
1663
1664retry:
1665 if (!fixed) {
1666 fd = __get_unused_fd_flags(accept->flags, accept->nofile);
1667 if (unlikely(fd < 0))
1668 return fd;
1669 }
1670 arg.err = 0;
1671 arg.is_empty = -1;
1672 file = do_accept(req->file, &arg, accept->addr, accept->addr_len,
1673 accept->flags);
1674 if (IS_ERR(file)) {
1675 if (!fixed)
1676 put_unused_fd(fd);
1677 ret = PTR_ERR(file);
1678 if (ret == -EAGAIN && force_nonblock &&
1679 !(accept->iou_flags & IORING_ACCEPT_DONTWAIT))
1680 return IOU_RETRY;
1681
1682 if (ret == -ERESTARTSYS)
1683 ret = -EINTR;
1684 } else if (!fixed) {
1685 fd_install(fd, file);
1686 ret = fd;
1687 } else {
1688 ret = io_fixed_fd_install(req, issue_flags, file,
1689 accept->file_slot);
1690 }
1691
1692 cflags = 0;
1693 if (!arg.is_empty)
1694 cflags |= IORING_CQE_F_SOCK_NONEMPTY;
1695
1696 if (ret >= 0 && (req->flags & REQ_F_APOLL_MULTISHOT) &&
1697 io_req_post_cqe(req, ret, cflags | IORING_CQE_F_MORE)) {
1698 if (cflags & IORING_CQE_F_SOCK_NONEMPTY || arg.is_empty == -1)
1699 goto retry;
1700 return IOU_RETRY;
1701 }
1702
1703 io_req_set_res(req, ret, cflags);
1704 if (ret < 0)
1705 req_set_fail(req);
1706 return IOU_COMPLETE;
1707}
1708
1709void io_socket_bpf_populate(struct io_uring_bpf_ctx *bctx, struct io_kiocb *req)
1710{
1711 struct io_socket *sock = io_kiocb_to_cmd(req, struct io_socket);
1712
1713 bctx->socket.family = sock->domain;
1714 bctx->socket.type = sock->type;
1715 bctx->socket.protocol = sock->protocol;
1716}
1717
1718void io_connect_bpf_populate(struct io_uring_bpf_ctx *bctx, struct io_kiocb *req)
1719{
1720 struct io_connect *conn = io_kiocb_to_cmd(req, struct io_connect);
1721 struct sockaddr_storage *ss = req->async_data;
1722
1723 /*
1724 * move_addr_to_kernel() skips the copy for addr_len == 0, so
1725 * iomsg->addr may hold stale data from a prior CONNECT. Bail
1726 * unless addr_len covers the family discriminator.
1727 */
1728 if (conn->addr_len < (int)sizeof(sa_family_t))
1729 return;
1730
1731 bctx->connect.family = ss->ss_family;
1732 switch (ss->ss_family) {
1733 case AF_INET: {
1734 struct sockaddr_in *sin = (struct sockaddr_in *)ss;
1735
1736 if (conn->addr_len < (int)sizeof(*sin))
1737 break;
1738 bctx->connect.port = sin->sin_port;
1739 bctx->connect.v4_addr = sin->sin_addr.s_addr;
1740 break;
1741 }
1742 case AF_INET6: {
1743 struct sockaddr_in6 *sin6 = (struct sockaddr_in6 *)ss;
1744
1745 /* The IPv6 stack accepts SIN6_LEN_RFC2133; port and addr are in it. */
1746 if (conn->addr_len < SIN6_LEN_RFC2133)
1747 break;
1748 bctx->connect.port = sin6->sin6_port;
1749 memcpy(bctx->connect.v6_addr, &sin6->sin6_addr,
1750 sizeof(bctx->connect.v6_addr));
1751 break;
1752 }
1753 default:
1754 /* family is set; per-family fields stay zero - family-only filtering */
1755 break;
1756 }
1757}
1758
1759int io_socket_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
1760{
1761 struct io_socket *sock = io_kiocb_to_cmd(req, struct io_socket);
1762
1763 if (sqe->addr || sqe->rw_flags || sqe->buf_index)
1764 return -EINVAL;
1765
1766 sock->domain = READ_ONCE(sqe->fd);
1767 sock->type = READ_ONCE(sqe->off);
1768 sock->protocol = READ_ONCE(sqe->len);
1769 sock->file_slot = READ_ONCE(sqe->file_index);
1770 sock->nofile = rlimit(RLIMIT_NOFILE);
1771
1772 sock->flags = sock->type & ~SOCK_TYPE_MASK;
1773 if (sock->file_slot && (sock->flags & SOCK_CLOEXEC))
1774 return -EINVAL;
1775 if (sock->flags & ~(SOCK_CLOEXEC | SOCK_NONBLOCK))
1776 return -EINVAL;
1777 return 0;
1778}
1779
1780int io_socket(struct io_kiocb *req, unsigned int issue_flags)
1781{
1782 struct io_socket *sock = io_kiocb_to_cmd(req, struct io_socket);
1783 bool fixed = !!sock->file_slot;
1784 struct file *file;
1785 int ret, fd;
1786
1787 if (!fixed) {
1788 fd = __get_unused_fd_flags(sock->flags, sock->nofile);
1789 if (unlikely(fd < 0))
1790 return fd;
1791 }
1792 file = __sys_socket_file(sock->domain, sock->type, sock->protocol);
1793 if (IS_ERR(file)) {
1794 if (!fixed)
1795 put_unused_fd(fd);
1796 ret = PTR_ERR(file);
1797 if (ret == -EAGAIN && (issue_flags & IO_URING_F_NONBLOCK))
1798 return -EAGAIN;
1799 if (ret == -ERESTARTSYS)
1800 ret = -EINTR;
1801 req_set_fail(req);
1802 } else if (!fixed) {
1803 fd_install(fd, file);
1804 ret = fd;
1805 } else {
1806 ret = io_fixed_fd_install(req, issue_flags, file,
1807 sock->file_slot);
1808 }
1809 io_req_set_res(req, ret, 0);
1810 return IOU_COMPLETE;
1811}
1812
1813int io_connect_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
1814{
1815 struct io_connect *conn = io_kiocb_to_cmd(req, struct io_connect);
1816 struct sockaddr_storage *addr;
1817
1818 if (sqe->len || sqe->buf_index || sqe->rw_flags || sqe->splice_fd_in)
1819 return -EINVAL;
1820
1821 conn->addr = u64_to_user_ptr(READ_ONCE(sqe->addr));
1822 conn->addr_len = READ_ONCE(sqe->addr2);
1823 conn->in_progress = conn->seen_econnaborted = false;
1824
1825 addr = io_uring_alloc_async_data(NULL, req);
1826 if (unlikely(!addr))
1827 return -ENOMEM;
1828
1829 return move_addr_to_kernel(conn->addr, conn->addr_len, addr);
1830}
1831
1832int io_connect(struct io_kiocb *req, unsigned int issue_flags)
1833{
1834 struct io_connect *connect = io_kiocb_to_cmd(req, struct io_connect);
1835 struct sockaddr_storage *addr = req->async_data;
1836 unsigned file_flags;
1837 int ret;
1838 bool force_nonblock = issue_flags & IO_URING_F_NONBLOCK;
1839
1840 if (connect->in_progress) {
1841 struct poll_table_struct pt = { ._key = EPOLLERR };
1842
1843 if (vfs_poll(req->file, &pt) & EPOLLERR)
1844 goto get_sock_err;
1845 }
1846
1847 file_flags = force_nonblock ? O_NONBLOCK : 0;
1848
1849 ret = __sys_connect_file(req->file, addr, connect->addr_len, file_flags);
1850 if ((ret == -EAGAIN || ret == -EINPROGRESS || ret == -ECONNABORTED)
1851 && force_nonblock) {
1852 if (ret == -EINPROGRESS) {
1853 connect->in_progress = true;
1854 } else if (ret == -ECONNABORTED) {
1855 if (connect->seen_econnaborted)
1856 goto out;
1857 connect->seen_econnaborted = true;
1858 }
1859 return -EAGAIN;
1860 }
1861 if (connect->in_progress) {
1862 /*
1863 * At least bluetooth will return -EBADFD on a re-connect
1864 * attempt, and it's (supposedly) also valid to get -EISCONN
1865 * which means the previous result is good. For both of these,
1866 * grab the sock_error() and use that for the completion.
1867 */
1868 if (ret == -EBADFD || ret == -EISCONN) {
1869get_sock_err:
1870 ret = sock_error(sock_from_file(req->file)->sk);
1871 }
1872 }
1873 if (ret == -ERESTARTSYS)
1874 ret = -EINTR;
1875out:
1876 if (ret < 0)
1877 req_set_fail(req);
1878 io_req_set_res(req, ret, 0);
1879 return IOU_COMPLETE;
1880}
1881
1882/*
1883 * Check if bind request would potentially end up with filename_create(),
1884 * which in turn end up in mnt_want_write() which will grab the fs
1885 * percpu start write sem. This can trigger a lockdep warning.
1886 */
1887static int io_bind_file_create(const struct sockaddr_storage *addr, int addr_len)
1888{
1889 const struct sockaddr_un *sun;
1890
1891 if (addr->ss_family != AF_UNIX)
1892 return 0;
1893 if (addr_len <= offsetof(struct sockaddr_un, sun_path))
1894 return 0;
1895 sun = (const struct sockaddr_un *) addr;
1896 return sun->sun_path[0] != '\0';
1897}
1898
1899int io_bind_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
1900{
1901 struct io_bind *bind = io_kiocb_to_cmd(req, struct io_bind);
1902 struct sockaddr __user *uaddr;
1903 struct sockaddr_storage *addr;
1904 int ret;
1905
1906 if (sqe->len || sqe->buf_index || sqe->rw_flags || sqe->splice_fd_in)
1907 return -EINVAL;
1908
1909 uaddr = u64_to_user_ptr(READ_ONCE(sqe->addr));
1910 bind->addr_len = READ_ONCE(sqe->addr2);
1911
1912 addr = io_uring_alloc_async_data(NULL, req);
1913 if (unlikely(!addr))
1914 return -ENOMEM;
1915 ret = move_addr_to_kernel(uaddr, bind->addr_len, addr);
1916 if (unlikely(ret))
1917 return ret;
1918 if (io_bind_file_create(addr, bind->addr_len))
1919 req->flags |= REQ_F_FORCE_ASYNC;
1920 return 0;
1921}
1922
1923
1924int io_bind(struct io_kiocb *req, unsigned int issue_flags)
1925{
1926 struct io_bind *bind = io_kiocb_to_cmd(req, struct io_bind);
1927 struct sockaddr_storage *addr = req->async_data;
1928 struct socket *sock;
1929 int ret;
1930
1931 sock = sock_from_file(req->file);
1932 if (unlikely(!sock))
1933 return -ENOTSOCK;
1934
1935 ret = __sys_bind_socket(sock, addr, bind->addr_len);
1936 if (ret < 0)
1937 req_set_fail(req);
1938 io_req_set_res(req, ret, 0);
1939 return 0;
1940}
1941
1942int io_listen_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
1943{
1944 struct io_listen *listen = io_kiocb_to_cmd(req, struct io_listen);
1945
1946 if (sqe->addr || sqe->buf_index || sqe->rw_flags || sqe->splice_fd_in || sqe->addr2)
1947 return -EINVAL;
1948
1949 listen->backlog = READ_ONCE(sqe->len);
1950 return 0;
1951}
1952
1953int io_listen(struct io_kiocb *req, unsigned int issue_flags)
1954{
1955 struct io_listen *listen = io_kiocb_to_cmd(req, struct io_listen);
1956 struct socket *sock;
1957 int ret;
1958
1959 sock = sock_from_file(req->file);
1960 if (unlikely(!sock))
1961 return -ENOTSOCK;
1962
1963 ret = __sys_listen_socket(sock, listen->backlog);
1964 if (ret < 0)
1965 req_set_fail(req);
1966 io_req_set_res(req, ret, 0);
1967 return 0;
1968}
1969
1970void io_netmsg_cache_free(const void *entry)
1971{
1972 struct io_async_msghdr *kmsg = (struct io_async_msghdr *) entry;
1973
1974 io_vec_free(&kmsg->vec);
1975 kfree(kmsg);
1976}