projects/linux·blob master · refs

300 lines · 8.2 KiB

1// SPDX-License-Identifier: GPL-2.0
2/*
3 * I/O and data path helper functionality.
4 *
5 * Borrowed from NFS Copyright (c) 2016 Trond Myklebust
6 */
7
8#include <linux/kernel.h>
9#include <linux/netfs.h>
10#include "internal.h"
11
12struct netfs_wb_waiter {
13 struct list_head link; /* Link in ictx->wb_queue */
14 struct task_struct *waiter; /* Waiter task; cleared when lock granted */
15};
16
17/*
18 * inode_dio_wait_interruptible - wait for outstanding DIO requests to finish
19 * @inode: inode to wait for
20 *
21 * Waits for all pending direct I/O requests to finish so that we can
22 * proceed with a truncate or equivalent operation.
23 *
24 * Must be called under a lock that serializes taking new references
25 * to i_dio_count, usually by inode->i_mutex.
26 */
27static int netfs_inode_dio_wait_interruptible(struct inode *inode)
28{
29 if (inode_dio_finished(inode))
30 return 0;
31
32 inode_dio_wait_interruptible(inode);
33 return !inode_dio_finished(inode) ? -ERESTARTSYS : 0;
34}
35
36/* Call with exclusively locked inode->i_rwsem */
37static int netfs_block_o_direct(struct netfs_inode *ictx)
38{
39 if (!test_bit(NETFS_ICTX_ODIRECT, &ictx->flags))
40 return 0;
41 clear_bit(NETFS_ICTX_ODIRECT, &ictx->flags);
42 return netfs_inode_dio_wait_interruptible(&ictx->inode);
43}
44
45/**
46 * netfs_start_io_read - declare the file is being used for buffered reads
47 * @inode: file inode
48 *
49 * Declare that a buffered read operation is about to start, and ensure
50 * that we block all direct I/O.
51 * On exit, the function ensures that the NETFS_ICTX_ODIRECT flag is unset,
52 * and holds a shared lock on inode->i_rwsem to ensure that the flag
53 * cannot be changed.
54 * In practice, this means that buffered read operations are allowed to
55 * execute in parallel, thanks to the shared lock, whereas direct I/O
56 * operations need to wait to grab an exclusive lock in order to set
57 * NETFS_ICTX_ODIRECT.
58 * Note that buffered writes and truncates both take a write lock on
59 * inode->i_rwsem, meaning that those are serialised w.r.t. the reads.
60 */
61int netfs_start_io_read(struct inode *inode)
62 __acquires(inode->i_rwsem)
63{
64 struct netfs_inode *ictx = netfs_inode(inode);
65
66 /* Be an optimist! */
67 if (down_read_interruptible(&inode->i_rwsem) < 0)
68 return -ERESTARTSYS;
69 if (test_bit(NETFS_ICTX_ODIRECT, &ictx->flags) == 0)
70 return 0;
71 up_read(&inode->i_rwsem);
72
73 /* Slow path.... */
74 if (down_write_killable(&inode->i_rwsem) < 0)
75 return -ERESTARTSYS;
76 if (netfs_block_o_direct(ictx) < 0) {
77 up_write(&inode->i_rwsem);
78 return -ERESTARTSYS;
79 }
80 downgrade_write(&inode->i_rwsem);
81 return 0;
82}
83EXPORT_SYMBOL(netfs_start_io_read);
84
85/**
86 * netfs_end_io_read - declare that the buffered read operation is done
87 * @inode: file inode
88 *
89 * Declare that a buffered read operation is done, and release the shared
90 * lock on inode->i_rwsem.
91 */
92void netfs_end_io_read(struct inode *inode)
93 __releases(inode->i_rwsem)
94{
95 up_read(&inode->i_rwsem);
96}
97EXPORT_SYMBOL(netfs_end_io_read);
98
99/**
100 * netfs_start_io_write - declare the file is being used for buffered writes
101 * @inode: file inode
102 *
103 * Declare that a buffered read operation is about to start, and ensure
104 * that we block all direct I/O.
105 */
106int netfs_start_io_write(struct inode *inode)
107 __acquires(inode->i_rwsem)
108{
109 struct netfs_inode *ictx = netfs_inode(inode);
110
111 if (down_write_killable(&inode->i_rwsem) < 0)
112 return -ERESTARTSYS;
113 if (netfs_block_o_direct(ictx) < 0) {
114 up_write(&inode->i_rwsem);
115 return -ERESTARTSYS;
116 }
117 downgrade_write(&inode->i_rwsem);
118 return 0;
119}
120EXPORT_SYMBOL(netfs_start_io_write);
121
122/**
123 * netfs_end_io_write - declare that the buffered write operation is done
124 * @inode: file inode
125 *
126 * Declare that a buffered write operation is done, and release the
127 * lock on inode->i_rwsem.
128 */
129void netfs_end_io_write(struct inode *inode)
130 __releases(inode->i_rwsem)
131{
132 up_read(&inode->i_rwsem);
133}
134EXPORT_SYMBOL(netfs_end_io_write);
135
136/* Call with exclusively locked inode->i_rwsem */
137static int netfs_block_buffered(struct inode *inode)
138{
139 struct netfs_inode *ictx = netfs_inode(inode);
140 int ret;
141
142 if (!test_bit(NETFS_ICTX_ODIRECT, &ictx->flags)) {
143 set_bit(NETFS_ICTX_ODIRECT, &ictx->flags);
144 if (inode->i_mapping->nrpages != 0) {
145 unmap_mapping_range(inode->i_mapping, 0, 0, 0);
146 ret = filemap_fdatawait(inode->i_mapping);
147 if (ret < 0) {
148 clear_bit(NETFS_ICTX_ODIRECT, &ictx->flags);
149 return ret;
150 }
151 }
152 }
153 return 0;
154}
155
156/**
157 * netfs_start_io_direct - declare the file is being used for direct i/o
158 * @inode: file inode
159 *
160 * Declare that a direct I/O operation is about to start, and ensure
161 * that we block all buffered I/O.
162 * On exit, the function ensures that the NETFS_ICTX_ODIRECT flag is set,
163 * and holds a shared lock on inode->i_rwsem to ensure that the flag
164 * cannot be changed.
165 * In practice, this means that direct I/O operations are allowed to
166 * execute in parallel, thanks to the shared lock, whereas buffered I/O
167 * operations need to wait to grab an exclusive lock in order to clear
168 * NETFS_ICTX_ODIRECT.
169 * Note that buffered writes and truncates both take a write lock on
170 * inode->i_rwsem, meaning that those are serialised w.r.t. O_DIRECT.
171 */
172int netfs_start_io_direct(struct inode *inode)
173 __acquires(inode->i_rwsem)
174{
175 struct netfs_inode *ictx = netfs_inode(inode);
176 int ret;
177
178 /* Be an optimist! */
179 if (down_read_interruptible(&inode->i_rwsem) < 0)
180 return -ERESTARTSYS;
181 if (test_bit(NETFS_ICTX_ODIRECT, &ictx->flags) != 0)
182 return 0;
183 up_read(&inode->i_rwsem);
184
185 /* Slow path.... */
186 if (down_write_killable(&inode->i_rwsem) < 0)
187 return -ERESTARTSYS;
188 ret = netfs_block_buffered(inode);
189 if (ret < 0) {
190 up_write(&inode->i_rwsem);
191 return ret;
192 }
193 downgrade_write(&inode->i_rwsem);
194 return 0;
195}
196EXPORT_SYMBOL(netfs_start_io_direct);
197
198/**
199 * netfs_end_io_direct - declare that the direct i/o operation is done
200 * @inode: file inode
201 *
202 * Declare that a direct I/O operation is done, and release the shared
203 * lock on inode->i_rwsem.
204 */
205void netfs_end_io_direct(struct inode *inode)
206 __releases(inode->i_rwsem)
207{
208 up_read(&inode->i_rwsem);
209}
210EXPORT_SYMBOL(netfs_end_io_direct);
211
212/*
213 * Wait to have exclusive access to writeback.
214 */
215static bool netfs_wb_begin_wait(struct netfs_inode *ictx)
216{
217 struct netfs_wb_waiter waiter = {};
218 struct task_struct *tsk = current;
219 bool got = false;
220
221 spin_lock(&ictx->lock);
222
223 if (test_and_set_bit_lock(NETFS_ICTX_WB_LOCK, &ictx->flags)) {
224 get_task_struct(tsk);
225 waiter.waiter = tsk;
226 list_add_tail(&waiter.link, &ictx->wb_queue);
227 } else {
228 got = true;
229 }
230 spin_unlock(&ictx->lock);
231
232 if (!got) {
233 for (;;) {
234 set_current_state(TASK_UNINTERRUPTIBLE);
235 /* Read waiter before accessing inode state. */
236 if (smp_load_acquire(&waiter.waiter) == NULL)
237 break;
238 schedule();
239 }
240 }
241 __set_current_state(TASK_RUNNING);
242 return true;
243}
244
245/**
246 * netfs_wb_begin - Begin writeback, waiting if need be
247 * @ictx: The inode to get writeback access on
248 * @nowait: Return failure immediately rather than waiting if true
249 *
250 * Begin writeback to an inode, waiting for exclusive access if @nowait is
251 * false. This prevents collection from being done out of order with respect
252 * to the issuance of write subrequests.
253 *
254 * Note that writeback may be ended in a different process (e.g. the collection
255 * function on a workqueue) than started it.
256 *
257 * Return: True if can proceed, false if denied.
258 */
259bool netfs_wb_begin(struct netfs_inode *ictx, bool nowait)
260{
261 if (!test_and_set_bit_lock(NETFS_ICTX_WB_LOCK, &ictx->flags))
262 return true;
263 if (nowait) {
264 netfs_stat(&netfs_n_wb_lock_skip);
265 return false;
266 }
267 netfs_stat(&netfs_n_wb_lock_wait);
268 return netfs_wb_begin_wait(ictx);
269}
270EXPORT_SYMBOL(netfs_wb_begin);
271
272/* netfs_wb_end - End writeback
273 * @ictx: The inode we have writeback access to
274 *
275 * End writeback access on an inode, waking up the next writeback request.
276 */
277void netfs_wb_end(struct netfs_inode *ictx)
278{
279 struct netfs_wb_waiter *waiter;
280 struct task_struct *tsk;
281
282 WARN_ON_ONCE(!test_bit(NETFS_ICTX_WB_LOCK, &ictx->flags));
283
284 spin_lock(&ictx->lock);
285
286 waiter = list_first_entry_or_null(&ictx->wb_queue, struct netfs_wb_waiter, link);
287 if (waiter) {
288 list_del(&waiter->link);
289 tsk = waiter->waiter;
290 /* Write inode state before clearing waiter. */
291 smp_store_release(&waiter->waiter, NULL);
292 wake_up_process(tsk);
293 put_task_struct(tsk);
294 } else {
295 clear_bit_unlock(NETFS_ICTX_WB_LOCK, &ictx->flags);
296 }
297
298 spin_unlock(&ictx->lock);
299}
300EXPORT_SYMBOL(netfs_wb_end);