projects/linux·commit f5437ff7

Merge tag 'for-7.3/io_uring-20260819' of git://git.kernel.org/pub/scm/linux/kernel/git/axboe/linux

f5437ff7 · Linus Torvalds <[email protected]> · 2026-08-20 13:47:56 -07:00

Pull io_uring update from Jens Axboe:
 "On top of the usual cleanups and fixes, the bigger items in here are:

   - zcrx work, most of it centered around adding dynamic area
     provisioning, plus a bunch of prep and cleanups leading up to it:
     scale refilling with large pages, coalesce same-niov RQEs on
     refill, separate the RQ head/tail cache lines and cache the RQ
     tail, and rework the area creation locking.

   - Fix the futex inflight accounting so that only private futex waits
     are marked inflight, and don't mark wake requests as inflight at
     all.

   - Drop the custom iov copy in the buffer select prep and msg header
     copy paths, using the generic helpers instead.

   - Fix a folio size overflow in io_vec_fill_bvec(), and account the
     pages a compound region really uses in the memmap path.

   - Fix an iovec leak in uring_cmd when the async cmd isn't recycled,
     skip the blocking task work for io_uring_cmd_issue_blocking(), and
     don't skip completion for a synchronous multishot cmd

   - Defer eventfd signaling when queued from a wakeup handler

   - Fix io-wq worker accounting when canceling creation callbacks

   - Annotate remote tasks for kcoverage"

* tag 'for-7.3/io_uring-20260819' of git://git.kernel.org/pub/scm/linux/kernel/git/axboe/linux: (30 commits)
  io_uring: Add missing include for ITER_SOURCE and ITER_DEST
  io_uring/uring_cmd: don't skip completion for a synchronous multishot cmd
  io_uring/memmap: account the pages a compound region really uses
  io_uring/zcrx: add dynamic area provisioning
  io_uring/zcrx: lock area creation with pp_lock
  io_uring/zcrx: keep array of areas
  io_uring/zcrx: move freelist lock to struct zcrx
  io_uring/zcrx: unmap under netdev lock
  io_uring/zcrx: split dmabuf unmap and release
  io_uring/zcrx: don't pass ifq_reg to area creation
  io_uring/zcrx: add helper for deriving area token
  io_uring/zcrx: don't reload skb_shinfo
  io_urint/zcrx: narrow var scope in io_zcrx_recv_skb()
  io_uring/zcrx: constify area_reg on import
  io_uring/zcrx: coalesce same-niov RQEs on refill
  io_uring/zcrx: cache RQ tail
  io_uring/zcrx: add RQ iterator
  io_uring/zcrx: move RQ head/tail to separate cache lines
  io_uring/zcrx: scale refilling with large pages
  io_uring/io-wq: fix worker accounting when canceling creation callbacks
  ...
Commit
f5437ff7299e47e76e52d37a2937a4b0f04e399f
Parents
c84d3e31 · 881dc9dd

Merge commit. Changes below are compared with the first parent: c84d3e31.

Changes

22 files changed · +448 −245

Only the first 600 diff lines are shown. Download the complete patch.

include/linux/io_uring_types.h

View file
diff --git a/include/linux/io_uring_types.h b/include/linux/io_uring_types.h
index 87151a5..94936ee 100644
--- a/include/linux/io_uring_types.h
+++ b/include/linux/io_uring_types.h
@@ -6,6 +6,7 @@
#include <linux/task_work.h>
#include <linux/bitmap.h>
#include <linux/llist.h>
+#include <linux/uio.h>
#include <uapi/linux/io_uring.h>
struct iou_loop_params;
@@ -20,6 +21,14 @@ enum {
* It's also ignored unless IORING_SETUP_DEFER_TASKRUN is set.
*/
IOU_F_TWQ_LAZY_WAKE = 1,
+
+ /*
+ * Set when task_work is queued from a waitqueue wakeup handler, where
+ * an arbitrary provider waitqueue lock is held. Signaling the CQ ring
+ * eventfd inline from there can recurse back into that lock through
+ * epoll, so the eventfd signal must be deferred.
+ */
+ IOU_F_TWQ_IN_WAKE = 2,
};
enum io_uring_cmd_flags {
@@ -534,6 +543,8 @@ struct io_ring_ctx {
struct io_mapped_region ring_region;
/* used for optimised request parameter and wait argument passing */
struct io_mapped_region param_region;
+
+ struct kcov_common_handle_id kcov_handle;
};
/*

include/uapi/linux/io_uring/zcrx.h

View file
diff --git a/include/uapi/linux/io_uring/zcrx.h b/include/uapi/linux/io_uring/zcrx.h
index e01bc0e..99b8636 100644
--- a/include/uapi/linux/io_uring/zcrx.h
+++ b/include/uapi/linux/io_uring/zcrx.h
@@ -116,6 +116,7 @@ enum zcrx_ctrl_op {
ZCRX_CTRL_FLUSH_RQ,
ZCRX_CTRL_EXPORT,
ZCRX_CTRL_ARM_EVENT,
+ ZCRX_CTRL_ADD_AREA,
__ZCRX_CTRL_LAST,
};
@@ -134,6 +135,11 @@ struct zcrx_ctrl_arm_event {
__u32 __resv[11];
};
+struct zcrx_ctrl_add_area {
+ __u64 area_ptr; /* pointer to struct io_uring_zcrx_area_reg */
+ __u64 __resv[5];
+};
+
struct zcrx_ctrl {
__u32 zcrx_id;
__u32 op; /* see enum zcrx_ctrl_op */
@@ -143,6 +149,7 @@ struct zcrx_ctrl {
struct zcrx_ctrl_export zc_export;
struct zcrx_ctrl_flush_rq zc_flush;
struct zcrx_ctrl_arm_event zc_arm_event;
+ struct zcrx_ctrl_add_area zc_area;
};
};

io_uring/eventfd.c

View file
diff --git a/io_uring/eventfd.c b/io_uring/eventfd.c
index d656cc2..63fe6e5 100644
--- a/io_uring/eventfd.c
+++ b/io_uring/eventfd.c
@@ -51,9 +51,9 @@ static void io_eventfd_do_signal(struct rcu_head *rcu)
/*
* Returns true if the caller should put the ev_fd reference, false if not.
*/
-static bool __io_eventfd_signal(struct io_ev_fd *ev_fd)
+static bool __io_eventfd_signal(struct io_ev_fd *ev_fd, bool defer)
{
- if (eventfd_signal_allowed()) {
+ if (!defer && eventfd_signal_allowed()) {
eventfd_signal_mask(ev_fd->cq_ev_fd, EPOLL_URING_WAKE);
return true;
}
@@ -73,7 +73,7 @@ static bool io_eventfd_trigger(struct io_ev_fd *ev_fd)
return !ev_fd->eventfd_async || io_wq_current_is_worker();
}
-void io_eventfd_signal(struct io_ring_ctx *ctx, bool cqe_event)
+void io_eventfd_signal(struct io_ring_ctx *ctx, bool cqe_event, bool defer)
{
bool skip = false;
struct io_ev_fd *ev_fd;
@@ -113,7 +113,7 @@ void io_eventfd_signal(struct io_ring_ctx *ctx, bool cqe_event)
spin_unlock(&ctx->completion_lock);
}
- if (skip || __io_eventfd_signal(ev_fd))
+ if (skip || __io_eventfd_signal(ev_fd, defer))
io_eventfd_put(ev_fd);
}

io_uring/eventfd.h

View file
diff --git a/io_uring/eventfd.h b/io_uring/eventfd.h
index 400eda4..e965d80 100644
--- a/io_uring/eventfd.h
+++ b/io_uring/eventfd.h
@@ -5,4 +5,4 @@ int io_eventfd_register(struct io_ring_ctx *ctx, void __user *arg,
unsigned int eventfd_async);
int io_eventfd_unregister(struct io_ring_ctx *ctx);
-void io_eventfd_signal(struct io_ring_ctx *ctx, bool cqe_event);
+void io_eventfd_signal(struct io_ring_ctx *ctx, bool cqe_event, bool defer);

io_uring/futex.c

View file
diff --git a/io_uring/futex.c b/io_uring/futex.c
index 906701b..eaee142 100644
--- a/io_uring/futex.c
+++ b/io_uring/futex.c
@@ -149,8 +149,21 @@ int io_futex_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
!futex_validate_input(iof->futex_flags, iof->futex_mask))
return -EINVAL;
- /* Mark as inflight, so file exit cancelation will find it */
- io_req_track_inflight(req);
+ return 0;
+}
+
+int io_futex_wait_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
+{
+ struct io_futex *iof = io_kiocb_to_cmd(req, struct io_futex);
+ int ret;
+
+ ret = io_futex_prep(req, sqe);
+ if (unlikely(ret))
+ return ret;
+
+ /* inflight tracking only needed for mm private hash */
+ if (!(iof->futex_flags & FLAGS_SHARED))
+ io_req_track_inflight(req);
return 0;
}
@@ -168,13 +181,14 @@ static void io_futex_wakev_fn(struct wake_q_head *wake_q, struct futex_q *q)
io_req_set_res(req, 0, 0);
req->io_task_work.func = io_futexv_complete;
- io_req_task_work_add(req);
+ __io_req_task_work_add(req, IOU_F_TWQ_IN_WAKE);
}
int io_futexv_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
{
struct io_futex *iof = io_kiocb_to_cmd(req, struct io_futex);
struct io_futexv_data *ifd;
+ unsigned int i;
int ret;
/* No flags or mask supported for waitv */
@@ -199,8 +213,14 @@ int io_futexv_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
return ret;
}
- /* Mark as inflight, so file exit cancelation will find it */
- io_req_track_inflight(req);
+ /* inflight tracking only needed for mm private hash */
+ for (i = 0; i < iof->futex_nr; i++) {
+ if (!(ifd->futexv[i].w.flags & FLAGS_SHARED)) {
+ io_req_track_inflight(req);
+ break;
+ }
+ }
+
iof->futexv_unqueued = 0;
req->flags |= REQ_F_ASYNC_DATA;
req->async_data = ifd;
@@ -217,7 +237,7 @@ static void io_futex_wake_fn(struct wake_q_head *wake_q, struct futex_q *q)
io_req_set_res(req, 0, 0);
req->io_task_work.func = io_futex_complete;
- io_req_task_work_add(req);
+ __io_req_task_work_add(req, IOU_F_TWQ_IN_WAKE);
}
int io_futexv_wait(struct io_kiocb *req, unsigned int issue_flags)

io_uring/futex.h

View file
diff --git a/io_uring/futex.h b/io_uring/futex.h
index d789fcf..987db3f 100644
--- a/io_uring/futex.h
+++ b/io_uring/futex.h
@@ -3,6 +3,7 @@
#include "cancel.h"
int io_futex_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe);
+int io_futex_wait_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe);
int io_futexv_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe);
int io_futex_wait(struct io_kiocb *req, unsigned int issue_flags);
int io_futexv_wait(struct io_kiocb *req, unsigned int issue_flags);

io_uring/io-wq.c

View file
diff --git a/io_uring/io-wq.c b/io_uring/io-wq.c
index 2e14880..2ca223e 100644
--- a/io_uring/io-wq.c
+++ b/io_uring/io-wq.c
@@ -19,6 +19,7 @@
#include <linux/mmu_context.h>
#include <linux/sched/sysctl.h>
#include <uapi/linux/io_uring.h>
+#include <linux/kcov.h>
#include "io-wq.h"
#include "slist.h"
@@ -211,9 +212,12 @@ static void io_worker_cancel_cb(struct io_worker *worker)
struct io_wq *wq = worker->wq;
atomic_dec(&acct->nr_running);
- raw_spin_lock(&acct->workers_lock);
- acct->nr_workers--;
- raw_spin_unlock(&acct->workers_lock);
+ /* create_worker_cb() has not reserved a worker slot yet. */
+ if (worker->create_work.func != create_worker_cb) {
+ raw_spin_lock(&acct->workers_lock);
+ acct->nr_workers--;
+ raw_spin_unlock(&acct->workers_lock);
+ }
io_worker_ref_put(wq);
clear_bit_unlock(0, &worker->create_state);
io_worker_release(worker);
@@ -643,13 +647,17 @@ static void io_worker_handle_work(struct io_wq_acct *acct,
unsigned int hash = __io_wq_is_hashed(work_flags)
? __io_get_work_hash(work_flags)
: -1U;
+ struct io_kiocb *req;
next_hashed = wq_next_work(work);
if (do_kill &&
(work_flags & IO_WQ_WORK_UNBOUND))
atomic_or(IO_WQ_WORK_CANCEL, &work->flags);
+ req = container_of(work, struct io_kiocb, work);
+ kcov_remote_start_common(req->ctx->kcov_handle);
io_wq_submit_work(work);
+ kcov_remote_stop();
io_assign_current_work(worker, NULL);
linked = io_wq_free_work(work);

io_uring/io_uring.c

View file
diff --git a/io_uring/io_uring.c b/io_uring/io_uring.c
index 1ea2fca..76f049e 100644
--- a/io_uring/io_uring.c
+++ b/io_uring/io_uring.c
@@ -59,6 +59,7 @@
#include <linux/audit.h>
#include <linux/security.h>
#include <linux/jump_label.h>
+#include <linux/kcov.h>
#define CREATE_TRACE_POINTS
#include <trace/events/io_uring.h>
@@ -293,6 +294,7 @@ static __cold struct io_ring_ctx *io_ring_ctx_alloc(struct io_uring_params *p)
INIT_HLIST_HEAD(&ctx->cancelable_uring_cmd);
io_napi_init(ctx);
mutex_init(&ctx->mmap_lock);
+ ctx->kcov_handle = kcov_common_handle();
return ctx;
@@ -405,7 +407,7 @@ static void io_prep_async_link(struct io_kiocb *req)
}
}
-static void io_queue_iowq(struct io_kiocb *req)
+void io_queue_iowq(struct io_kiocb *req)
{
struct io_uring_task *tctx = req->tctx;
@@ -433,17 +435,6 @@ static void io_queue_iowq(struct io_kiocb *req)
io_wq_enqueue(tctx->io_wq, &req->work);
}
-static void io_req_queue_iowq_tw(struct io_tw_req tw_req, io_tw_token_t tw)
-{
- io_queue_iowq(tw_req.req);
-}
-
-void io_req_queue_iowq(struct io_kiocb *req)
-{
- req->io_task_work.func = io_req_queue_iowq_tw;
- io_req_task_work_add(req);
-}
-
unsigned io_linked_nr(struct io_kiocb *req)
{
struct io_kiocb *tmp;
@@ -484,7 +475,7 @@ void __io_commit_cqring_flush(struct io_ring_ctx *ctx)
if (ctx->int_flags & IO_RING_F_OFF_TIMEOUT_USED)
io_flush_timeouts(ctx);
if (ctx->int_flags & IO_RING_F_HAS_EVFD)
- io_eventfd_signal(ctx, true);
+ io_eventfd_signal(ctx, true, false);
}
static inline void __io_cq_lock(struct io_ring_ctx *ctx)

io_uring/io_uring.h

View file
diff --git a/io_uring/io_uring.h b/io_uring/io_uring.h
index cb736b8..dfe26a9 100644
--- a/io_uring/io_uring.h
+++ b/io_uring/io_uring.h
@@ -195,7 +195,7 @@ __cold void io_uring_drop_tctx_refs(struct task_struct *task);
int io_ring_add_registered_file(struct io_uring_task *tctx, struct file *file,
int start, int end);
-void io_req_queue_iowq(struct io_kiocb *req);
+void io_queue_iowq(struct io_kiocb *req);
int io_poll_issue(struct io_kiocb *req, io_tw_token_t tw);
int io_submit_sqes(struct io_ring_ctx *ctx, unsigned int nr);

io_uring/memmap.c

View file
diff --git a/io_uring/memmap.c b/io_uring/memmap.c
index 23e8a85..48c0eb0 100644
--- a/io_uring/memmap.c
+++ b/io_uring/memmap.c
@@ -16,8 +16,10 @@
#include "zcrx.h"
static bool io_mem_alloc_compound(struct page **pages, int nr_pages,
- size_t size, gfp_t gfp)
+ size_t size, gfp_t gfp,
+ struct user_struct *user)
{
+ unsigned long nr_compound, extra;
struct page *page;
int i, order;
@@ -27,9 +29,22 @@ static bool io_mem_alloc_compound(struct page **pages, int nr_pages,
else if (order)
gfp |= __GFP_COMP;
+ /*
+ * get_order() rounds a non power of two size up, so the allocation
+ * can hold more pages than the region exposes. Account those too,
+ * and leave the compound allocation alone if they do not fit.
+ */
+ nr_compound = 1UL << order;
+ extra = nr_compound - nr_pages;
+ if (extra && user && __io_account_mem(user, extra))
+ return false;
+
page = alloc_pages(gfp, order);
- if (!page)
+ if (!page) {
+ if (extra && user)
+ __io_unaccount_mem(user, extra);
return false;
+ }
for (i = 0; i < nr_pages; i++)
pages[i] = page + i;
@@ -105,8 +120,15 @@ void io_free_region(struct user_struct *user, struct io_mapped_region *mr)
}
if ((mr->flags & IO_REGION_F_VMAP) && mr->ptr)
vunmap(mr->ptr);
- if (mr->nr_pages && user)
- __io_unaccount_mem(user, mr->nr_pages);
+ if (mr->nr_pages && user) {
+ unsigned long nr_accounted = mr->nr_pages;
+
+ /* a compound region was accounted for the whole allocation */
+ if (mr->flags & IO_REGION_F_SINGLE_REF)
+ nr_accounted = 1UL << get_order(io_region_size(mr));
+
+ __io_unaccount_mem(user, nr_accounted);
+ }
memset(mr, 0, sizeof(*mr));
}
@@ -151,7 +173,8 @@ static int io_region_pin_pages(struct io_mapped_region *mr,
static int io_region_allocate_pages(struct io_mapped_region *mr,
struct io_uring_region_desc *reg,
- unsigned long mmap_offset)
+ unsigned long mmap_offset,
+ struct user_struct *user)
{
gfp_t gfp = GFP_KERNEL_ACCOUNT | __GFP_ZERO | __GFP_NOWARN;
size_t size = io_region_size(mr);
@@ -162,7 +185,7 @@ static int io_region_allocate_pages(struct io_mapped_region *mr,
if (!pages)
return -ENOMEM;
- if (io_mem_alloc_compound(pages, mr->nr_pages, size, gfp)) {
+ if (io_mem_alloc_compound(pages, mr->nr_pages, size, gfp, user)) {
mr->flags |= IO_REGION_F_SINGLE_REF;
goto done;
}
@@ -217,7 +240,7 @@ int io_create_region(struct io_ring_ctx *ctx, struct io_mapped_region *mr,
if (reg->flags & IORING_MEM_REGION_TYPE_USER)
ret = io_region_pin_pages(mr, reg);
else
- ret = io_region_allocate_pages(mr, reg, mmap_offset);
+ ret = io_region_allocate_pages(mr, reg, mmap_offset, ctx->user);
if (ret)
goto out_free;

io_uring/net.c

View file
diff --git a/io_uring/net.c b/io_uring/net.c
index a74d15f..3f04aa1 100644
--- a/io_uring/net.c
+++ b/io_uring/net.c
@@ -236,39 +236,6 @@ static int io_net_import_vec(struct io_kiocb *req, struct io_async_msghdr *iomsg
return 0;
}
-static int io_compat_msg_copy_hdr(struct io_kiocb *req,
- struct io_async_msghdr *iomsg,
- struct compat_msghdr *msg, int ddir,
- struct sockaddr __user **save_addr)
-{
- struct io_sr_msg *sr = io_kiocb_to_cmd(req, struct io_sr_msg);
- struct compat_iovec __user *uiov;
- int ret;
-
- if (copy_from_user(msg, sr->umsg_compat, sizeof(*msg)))
- return -EFAULT;
-
- ret = __get_compat_msghdr(&iomsg->msg, msg, save_addr);
- if (ret)
- return ret;
-
- uiov = compat_ptr(msg->msg_iov);
- if (req->flags & REQ_F_BUFFER_SELECT) {
- if (msg->msg_iovlen == 0) {
- sr->len = 0;
- } else if (msg->msg_iovlen > 1) {
- return -EINVAL;
- } else {
- struct compat_iovec tmp_iov;
-
- if (copy_from_user(&tmp_iov, uiov, sizeof(tmp_iov)))
- return -EFAULT;
- sr->len = tmp_iov.iov_len;
- }
- }
- return 0;
-}
-
static int io_copy_msghdr_from_user(struct user_msghdr *msg,
struct user_msghdr __user *umsg)
{
@@ -288,11 +255,10 @@ ua_end:
}
static int io_msg_copy_hdr(struct io_kiocb *req, struct io_async_msghdr *iomsg,
- struct user_msghdr *msg, int ddir,
+ struct user_msghdr *msg,
struct sockaddr __user **save_addr)
{
struct io_sr_msg *sr = io_kiocb_to_cmd(req, struct io_sr_msg);
- struct user_msghdr __user *umsg = sr->umsg;
int ret;
iomsg->msg.msg_name = &iomsg->addr;
@@ -301,7 +267,10 @@ static int io_msg_copy_hdr(struct io_kiocb *req, struct io_async_msghdr *iomsg,
if (io_is_compat(req->ctx)) {
struct compat_msghdr cmsg;
- ret = io_compat_msg_copy_hdr(req, iomsg, &cmsg, ddir, save_addr);
+ if (copy_from_user(&cmsg, sr->umsg_compat, sizeof(cmsg)))
+ return -EFAULT;
+
+ ret = __get_compat_msghdr(&iomsg->msg, &cmsg, save_addr);
if (ret)
return ret;
@@ -310,18 +279,17 @@ static int io_msg_copy_hdr(struct io_kiocb *req, struct io_async_msghdr *iomsg,
msg->msg_controllen = cmsg.msg_controllen;
msg->msg_iov = compat_ptr(cmsg.msg_iov);
msg->msg_iovlen = cmsg.msg_iovlen;
- return 0;
- }
-
- ret = io_copy_msghdr_from_user(msg, umsg);
- if (unlikely(ret))
- return ret;
+ } else {
+ ret = io_copy_msghdr_from_user(msg, sr->umsg);
+ if (unlikely(ret))
+ return ret;
- msg->msg_flags = 0;
+ msg->msg_flags = 0;
- ret = __copy_msghdr(&iomsg->msg, msg, save_addr);
- if (ret)
- return ret;
+ ret = __copy_msghdr(&iomsg->msg, msg, save_addr);
+ if (ret)
+ return ret;
+ }
if (req->flags & REQ_F_BUFFER_SELECT) {
if (msg->msg_iovlen == 0) {
@@ -329,12 +297,13 @@ static int io_msg_copy_hdr(struct io_kiocb *req, struct io_async_msghdr *iomsg,
} else if (msg->msg_iovlen > 1) {
return -EINVAL;
} else {
- struct iovec __user *uiov = msg->msg_iov;
- struct iovec tmp_iov;
+ struct iovec fast_iov, *iov;
- if (copy_from_user(&tmp_iov, uiov, sizeof(tmp_iov)))
- return -EFAULT;
- sr->len = tmp_iov.iov_len;
+ iov = iovec_from_user(msg->msg_iov, 1, 1, &fast_iov,
+ io_is_compat(req->ctx));
+ if (IS_ERR(iov))
+ return PTR_ERR(iov);
+ sr->len = iov->iov_len;
}
}
return 0;
@@ -402,7 +371,7 @@ static int io_sendmsg_setup(struct io_kiocb *req, const struct io_uring_sqe *sqe
sr->flags |= IORING_SEND_VECTORIZED;
sr->umsg = u64_to_user_ptr(READ_ONCE(sqe->addr));
- ret = io_msg_copy_hdr(req, kmsg, &msg, ITER_SOURCE, NULL);
+ ret = io_msg_copy_hdr(req, kmsg, &msg, NULL);
if (unlikely(ret))
return ret;
/* save msg_control as sys_sendmsg() overwrites it */
@@ -761,7 +730,7 @@ static int io_recvmsg_copy_hdr(struct io_kiocb *req,
struct user_msghdr msg;
int ret;
- ret = io_msg_copy_hdr(req, iomsg, &msg, ITER_DEST, &iomsg->uaddr);
+ ret = io_msg_copy_hdr(req, iomsg, &msg, &iomsg->uaddr);
if (unlikely(ret))
return ret;

io_uring/opdef.c

View file
diff --git a/io_uring/opdef.c b/io_uring/opdef.c
index 4e58eb1..cf3aa22 100644
--- a/io_uring/opdef.c
+++ b/io_uring/opdef.c
@@ -467,7 +467,7 @@ const struct io_issue_def io_issue_defs[] = {
},
[IORING_OP_FUTEX_WAIT] = {
#if defined(CONFIG_FUTEX)
- .prep = io_futex_prep,
+ .prep = io_futex_wait_prep,
.issue = io_futex_wait,
#else
.prep = io_eopnotsupp_prep,

io_uring/poll.c

View file
diff --git a/io_uring/poll.c b/io_uring/poll.c
index 0204aff..5447a7c 100644
--- a/io_uring/poll.c
+++ b/io_uring/poll.c
@@ -208,9 +208,9 @@ enum {
IOU_POLL_REQUEUE = 4,
};
-static void __io_poll_execute(struct io_kiocb *req, int mask)
+static void __io_poll_execute(struct io_kiocb *req, int mask, unsigned tw_flags)
{
- unsigned flags = 0;
+ unsigned flags = tw_flags;
io_req_set_res(req, mask, 0);
req->io_task_work.func = io_poll_task_func;
@@ -218,14 +218,15 @@ static void __io_poll_execute(struct io_kiocb *req, int mask)
trace_io_uring_task_add(req, mask);
if (!(req->flags & REQ_F_POLL_NO_LAZY))
- flags = IOU_F_TWQ_LAZY_WAKE;
+ flags |= IOU_F_TWQ_LAZY_WAKE;
__io_req_task_work_add(req, flags);
}
-static inline void io_poll_execute(struct io_kiocb *req, int res)
+static inline void io_poll_execute(struct io_kiocb *req, int res,
+ unsigned tw_flags)
{
if (io_poll_get_ownership(req))
- __io_poll_execute(req, res);
+ __io_poll_execute(req, res, tw_flags);
}
/*
@@ -344,7 +345,7 @@ void io_poll_task_func(struct io_tw_req tw_req, io_tw_token_t tw)
if (ret == IOU_POLL_NO_ACTION) {
return;
} else if (ret == IOU_POLL_REQUEUE) {
- __io_poll_execute(req, 0);
+ __io_poll_execute(req, 0, 0);
return;
}
io_poll_remove_entries(req);
@@ -383,7 +384,7 @@ static void io_poll_cancel_req(struct io_kiocb *req)
{
io_poll_mark_cancelled(req);
/* kick tw, which should complete the request */
- io_poll_execute(req, 0);
+ io_poll_execute(req, 0, 0);
}
#define IO_ASYNC_POLL_COMMON (EPOLLONESHOT | EPOLLPRI)
@@ -392,7 +393,7 @@ static __cold int io_pollfree_wake(struct io_kiocb *req, struct io_poll *poll)
{
io_poll_mark_cancelled(req);
/* we have to kick tw in case it's not already */
- io_poll_execute(req, 0);

This diff is incomplete. Download the complete patch.

io_uring/query.c

View file

This diff is incomplete. Download the complete patch.

io_uring/rsrc.c

View file

This diff is incomplete. Download the complete patch.

io_uring/rw.c

View file

This diff is incomplete. Download the complete patch.

io_uring/sqpoll.c

View file

This diff is incomplete. Download the complete patch.

io_uring/tw.c

View file

This diff is incomplete. Download the complete patch.

io_uring/uring_cmd.c

View file

This diff is incomplete. Download the complete patch.

io_uring/waitid.c

View file

This diff is incomplete. Download the complete patch.

io_uring/zcrx.c

View file

This diff is incomplete. Download the complete patch.

io_uring/zcrx.h

View file

This diff is incomplete. Download the complete patch.