projects/linux·blob master · refs

256 lines · 5.5 KiB

1/* SPDX-License-Identifier: GPL-2.0
2 *
3 * FUSE: Filesystem in Userspace
4 * Copyright (c) 2023-2024 DataDirect Networks.
5 */
6
7#ifndef _FS_FUSE_DEV_URING_I_H
8#define _FS_FUSE_DEV_URING_I_H
9
10#include <linux/uio.h>
11
12#include "fuse_dev_i.h"
13
14#ifdef CONFIG_FUSE_IO_URING
15
16#define FUSE_URING_TEARDOWN_TIMEOUT (5 * HZ)
17#define FUSE_URING_TEARDOWN_INTERVAL (HZ/20)
18
19enum fuse_ring_req_state {
20 FRRS_INVALID = 0,
21
22 /* The ring entry received from userspace and it is being processed */
23 FRRS_COMMIT,
24
25 /* The ring entry is waiting for new fuse requests */
26 FRRS_AVAILABLE,
27
28 /* The ring entry got assigned a fuse req */
29 FRRS_FUSE_REQ,
30
31 /* The ring entry is in or on the way to user space */
32 FRRS_USERSPACE,
33
34 /* The ring entry is in teardown */
35 FRRS_TEARDOWN,
36
37 /* The ring entry is released, but not freed yet */
38 FRRS_RELEASED,
39};
40
41/* how a queue's payload buffers are provided */
42enum fuse_queue_payload_mode {
43 /* not yet committed (a bufpool may still be added) */
44 FUSE_PAYLOAD_UNSET = 0,
45 /* each entry registers its own payload buffer */
46 FUSE_PAYLOAD_PER_ENT,
47 /* each entry's payload buffer is assigned from a bufpool */
48 FUSE_PAYLOAD_BUFPOOL,
49};
50
51struct fuse_bufpool {
52 bool registered;
53
54 /*
55 * io_uring registered buffer table index for this pool, bound at
56 * ADD_BUFPOOL time. Only valid if the bufpool is registered
57 */
58 u16 registered_index;
59
60 /* starting uaddr of the bufpool */
61 uintptr_t base_uaddr;
62
63 /* size of each buffer in the pool */
64 size_t buf_size;
65
66 /* total number of buffers in the pool */
67 unsigned int nr_bufs;
68
69 /* bitmap tracking which buffers are free */
70 unsigned long free_map[];
71};
72
73/** A fuse ring entry, part of the ring queue */
74struct fuse_ring_ent {
75 /* userspace buffer */
76 struct fuse_uring_req_header __user *headers;
77 struct iovec payload;
78
79 /* buffer id in the pool, if bufpools are used. ignored otherwise */
80 unsigned int buf_id;
81
82 /* true if the request's pages are being zero-copied */
83 bool zero_copied;
84 unsigned int zero_copy_index;
85
86 /* the ring queue that owns the request */
87 struct fuse_ring_queue *queue;
88
89 /* fields below are protected by queue->lock */
90
91 struct io_uring_cmd *cmd;
92
93 struct list_head list;
94
95 enum fuse_ring_req_state state;
96
97 struct fuse_req *fuse_req;
98};
99
100struct fuse_ring_queue {
101 /*
102 * back pointer to the main fuse uring structure that holds this
103 * queue
104 */
105 struct fuse_ring *ring;
106
107 /* queue id, corresponds to the cpu core */
108 unsigned int qid;
109
110 /*
111 * queue lock, taken when any value in the queue changes _and_ also
112 * a ring entry state changes.
113 */
114 spinlock_t lock;
115
116 /* available ring entries (struct fuse_ring_ent) */
117 struct list_head ent_avail_queue;
118
119 /*
120 * entries in the process of being committed or in the process
121 * to be sent to userspace
122 */
123 struct list_head ent_w_req_queue;
124 struct list_head ent_commit_queue;
125
126 /* entries in userspace */
127 struct list_head ent_in_userspace;
128
129 /* entries that are released */
130 struct list_head ent_released;
131
132 /* fuse requests waiting for an entry slot */
133 struct list_head fuse_req_queue;
134
135 /* background fuse requests */
136 struct list_head fuse_req_bg_queue;
137
138 struct fuse_pqueue fpq;
139
140 unsigned int active_background;
141
142 bool stopped;
143
144 /* how this queue's payload buffers are provided */
145 enum fuse_queue_payload_mode payload_mode;
146
147 /* only allocated when payload_mode == FUSE_PAYLOAD_BUFPOOL */
148 struct fuse_bufpool *bufpool;
149
150 bool zero_copy;
151};
152
153/*
154 * Describes if uring is for communication and holds alls the data needed
155 * for uring communication
156 */
157struct fuse_ring {
158 /* back pointer */
159 struct fuse_chan *chan;
160
161 /* number of ring queues */
162 size_t nr_queues;
163
164 /* maximum payload/arg size */
165 size_t max_payload_sz;
166
167 struct fuse_ring_queue **queues;
168
169 /*
170 * Log ring entry states on stop when entries cannot be released
171 */
172 unsigned int stop_debug_log : 1;
173
174 wait_queue_head_t stop_waitq;
175
176 /* async tear down */
177 struct delayed_work async_teardown_work;
178
179 /* log */
180 unsigned long teardown_time;
181
182 atomic_t queue_refs;
183
184 bool ready;
185};
186
187void fuse_uring_conn_init(struct fuse_chan *fch);
188void fuse_uring_stop_queues(struct fuse_ring *ring);
189void fuse_uring_abort_end_requests(struct fuse_ring *ring);
190int fuse_uring_cmd(struct io_uring_cmd *cmd, unsigned int issue_flags);
191void fuse_uring_queue_fuse_req(struct fuse_iqueue *fiq, struct fuse_req *req);
192bool fuse_uring_queue_bq_req(struct fuse_req *req);
193bool fuse_uring_remove_pending_req(struct fuse_req *req);
194bool fuse_uring_request_expired(struct fuse_chan *fch);
195
196static inline void fuse_uring_abort(struct fuse_chan *fch)
197{
198 struct fuse_ring *ring = fch->ring;
199
200 if (ring == NULL)
201 return;
202
203 fuse_uring_abort_end_requests(ring);
204
205 if (atomic_read(&ring->queue_refs) > 0)
206 fuse_uring_stop_queues(ring);
207}
208
209static inline void fuse_uring_wait_stopped_queues(struct fuse_chan *fch)
210{
211 struct fuse_ring *ring = fch->ring;
212
213 if (ring)
214 wait_event(ring->stop_waitq,
215 atomic_read(&ring->queue_refs) == 0);
216}
217
218static inline bool fuse_uring_ready(struct fuse_chan *fch)
219{
220 struct fuse_ring *ring = READ_ONCE(fch->ring);
221
222 return ring && smp_load_acquire(&ring->ready);
223}
224
225#else /* CONFIG_FUSE_IO_URING */
226
227static inline void fuse_uring_conn_init(struct fuse_chan *fch)
228{
229}
230
231static inline void fuse_uring_abort(struct fuse_chan *fch)
232{
233}
234
235static inline void fuse_uring_wait_stopped_queues(struct fuse_chan *fch)
236{
237}
238
239static inline bool fuse_uring_ready(struct fuse_chan *fch)
240{
241 return false;
242}
243
244static inline bool fuse_uring_remove_pending_req(struct fuse_req *req)
245{
246 return false;
247}
248
249static inline bool fuse_uring_request_expired(struct fuse_chan *fch)
250{
251 return false;
252}
253
254#endif /* CONFIG_FUSE_IO_URING */
255
256#endif /* _FS_FUSE_DEV_URING_I_H */