projects/linux·blob master · refs

683 lines · 18.8 KiB

1// SPDX-License-Identifier: GPL-2.0-only
2/* Miscellaneous routines.
3 *
4 * Copyright (C) 2023 Red Hat, Inc. All Rights Reserved.
5 * Written by David Howells ([email protected])
6 */
7
8#include <linux/swap.h>
9#include <linux/rmap.h>
10#include "internal.h"
11
12/**
13 * netfs_alloc_folioq_buffer - Allocate buffer space into a folio queue
14 * @mapping: Address space to set on the folio (or NULL).
15 * @_buffer: Pointer to the folio queue to add to (may point to a NULL; updated).
16 * @_cur_size: Current size of the buffer (updated).
17 * @size: Target size of the buffer.
18 * @gfp: The allocation constraints.
19 */
20int netfs_alloc_folioq_buffer(struct address_space *mapping,
21 struct folio_queue **_buffer,
22 size_t *_cur_size, ssize_t size, gfp_t gfp)
23{
24 struct folio_queue *tail = *_buffer, *p;
25
26 size = round_up(size, PAGE_SIZE);
27 if (*_cur_size >= size)
28 return 0;
29
30 if (tail)
31 while (tail->next)
32 tail = tail->next;
33
34 do {
35 struct folio *folio;
36 int order = 0, slot;
37
38 if (!tail || folioq_full(tail)) {
39 p = netfs_folioq_alloc(0, GFP_NOFS, netfs_trace_folioq_alloc_buffer);
40 if (!p)
41 return -ENOMEM;
42 if (tail) {
43 tail->next = p;
44 p->prev = tail;
45 } else {
46 *_buffer = p;
47 }
48 tail = p;
49 }
50
51 if (size - *_cur_size > PAGE_SIZE)
52 order = umin(ilog2(size - *_cur_size) - PAGE_SHIFT,
53 MAX_PAGECACHE_ORDER);
54
55 folio = folio_alloc(gfp, order);
56 if (!folio && order > 0)
57 folio = folio_alloc(gfp, 0);
58 if (!folio)
59 return -ENOMEM;
60
61 folio->mapping = mapping;
62 folio->index = *_cur_size / PAGE_SIZE;
63 trace_netfs_folio(folio, netfs_folio_trace_alloc_buffer);
64 slot = folioq_append_mark(tail, folio);
65 *_cur_size += folioq_folio_size(tail, slot);
66 } while (*_cur_size < size);
67
68 return 0;
69}
70EXPORT_SYMBOL(netfs_alloc_folioq_buffer);
71
72/**
73 * netfs_free_folioq_buffer - Free a folio queue.
74 * @fq: The start of the folio queue to free
75 *
76 * Free up a chain of folio_queues and, if marked, the marked folios they point
77 * to.
78 */
79void netfs_free_folioq_buffer(struct folio_queue *fq)
80{
81 struct folio_queue *next;
82 struct folio_batch fbatch;
83
84 folio_batch_init(&fbatch);
85
86 for (; fq; fq = next) {
87 for (int slot = 0; slot < folioq_count(fq); slot++) {
88 struct folio *folio = folioq_folio(fq, slot);
89
90 if (!folio ||
91 !folioq_is_marked(fq, slot))
92 continue;
93
94 trace_netfs_folio(folio, netfs_folio_trace_put);
95 if (folio_batch_add(&fbatch, folio))
96 folio_batch_release(&fbatch);
97 }
98
99 netfs_stat_d(&netfs_n_folioq);
100 next = fq->next;
101 kfree(fq);
102 }
103
104 folio_batch_release(&fbatch);
105}
106EXPORT_SYMBOL(netfs_free_folioq_buffer);
107
108/*
109 * Reset the subrequest iterator to refer just to the region remaining to be
110 * read. The iterator may or may not have been advanced by socket ops or
111 * extraction ops to an extent that may or may not match the amount actually
112 * read.
113 */
114void netfs_reset_iter(struct netfs_io_subrequest *subreq)
115{
116 struct iov_iter *io_iter = &subreq->io_iter;
117 size_t remain = subreq->len - subreq->transferred;
118
119 if (io_iter->count > remain)
120 iov_iter_advance(io_iter, io_iter->count - remain);
121 else if (io_iter->count < remain)
122 iov_iter_revert(io_iter, remain - io_iter->count);
123 iov_iter_truncate(&subreq->io_iter, remain);
124}
125
126/**
127 * netfs_dirty_folio - Mark folio dirty and pin a cache object for writeback
128 * @mapping: The mapping the folio belongs to.
129 * @folio: The folio being dirtied.
130 *
131 * Set the dirty flag on a folio and pin an in-use cache object in memory so
132 * that writeback can later write to it. This is intended to be called from
133 * the filesystem's ->dirty_folio() method.
134 *
135 * Return: true if the dirty flag was set on the folio, false otherwise.
136 */
137bool netfs_dirty_folio(struct address_space *mapping, struct folio *folio)
138{
139 struct inode *inode = mapping->host;
140 struct netfs_inode *ictx = netfs_inode(inode);
141 struct fscache_cookie *cookie = netfs_i_cookie(ictx);
142 bool need_use = false;
143
144 _enter("");
145
146 if (!filemap_dirty_folio(mapping, folio))
147 return false;
148 if (!fscache_cookie_valid(cookie))
149 return true;
150
151 if (!(inode_state_read_once(inode) & I_PINNING_NETFS_WB)) {
152 spin_lock(&inode->i_lock);
153 if (!(inode_state_read(inode) & I_PINNING_NETFS_WB)) {
154 inode_state_set(inode, I_PINNING_NETFS_WB);
155 need_use = true;
156 }
157 spin_unlock(&inode->i_lock);
158
159 if (need_use)
160 fscache_use_cookie(cookie, true);
161 }
162 return true;
163}
164EXPORT_SYMBOL(netfs_dirty_folio);
165
166/**
167 * netfs_unpin_writeback - Unpin writeback resources
168 * @inode: The inode on which the cookie resides
169 * @wbc: The writeback control
170 *
171 * Unpin the writeback resources pinned by netfs_dirty_folio(). This is
172 * intended to be called as/by the netfs's ->write_inode() method.
173 */
174int netfs_unpin_writeback(struct inode *inode, struct writeback_control *wbc)
175{
176 struct fscache_cookie *cookie = netfs_i_cookie(netfs_inode(inode));
177
178 if (wbc->unpinned_netfs_wb)
179 fscache_unuse_cookie(cookie, NULL, NULL);
180 return 0;
181}
182EXPORT_SYMBOL(netfs_unpin_writeback);
183
184/**
185 * netfs_clear_inode_writeback - Clear writeback resources pinned by an inode
186 * @inode: The inode to clean up
187 * @aux: Auxiliary data to apply to the inode
188 *
189 * Clear any writeback resources held by an inode when the inode is evicted.
190 * This must be called before clear_inode() is called.
191 */
192void netfs_clear_inode_writeback(struct inode *inode, const void *aux)
193{
194 struct fscache_cookie *cookie = netfs_i_cookie(netfs_inode(inode));
195
196 if (inode_state_read_once(inode) & I_PINNING_NETFS_WB) {
197 uoff_t i_size = i_size_read(inode);
198 fscache_unuse_cookie(cookie, aux, &i_size);
199 }
200}
201EXPORT_SYMBOL(netfs_clear_inode_writeback);
202
203/**
204 * netfs_invalidate_folio - Invalidate or partially invalidate a folio
205 * @folio: Folio proposed for release
206 * @offset: Offset of the invalidated region
207 * @length: Length of the invalidated region
208 *
209 * Invalidate part or all of a folio for a network filesystem. The folio will
210 * be removed afterwards if the invalidated region covers the entire folio.
211 */
212void netfs_invalidate_folio(struct folio *folio, size_t offset, size_t length)
213{
214 struct netfs_folio *finfo;
215 struct inode *inode = folio_inode(folio);
216 struct netfs_inode *ctx = netfs_inode(inode);
217 size_t flen = folio_size(folio);
218
219 _enter("{%lx},%zx,%zx", folio->index, offset, length);
220
221 if (offset == 0 && length == flen) {
222 uoff_t i_size, remote_i_size, zero_point;
223 uoff_t fpos = folio_pos(folio), end;
224
225 netfs_read_sizes(inode, &i_size, &remote_i_size, &zero_point);
226 end = umin(fpos + flen, i_size);
227 if (fpos < i_size && end > zero_point) {
228 spin_lock(&inode->i_lock);
229 end = umin(fpos + flen, inode->i_size);
230 if (fpos < i_size && end > ctx->_zero_point)
231 netfs_write_zero_point(inode, end);
232 spin_unlock(&inode->i_lock);
233 }
234 }
235
236 folio_wait_private_2(folio); /* [DEPRECATED] */
237
238 if (!folio_test_private(folio))
239 return;
240
241 finfo = netfs_folio_info(folio);
242
243 if (offset == 0 && length >= flen)
244 goto erase_completely;
245
246 if (finfo) {
247 /* We have a partially uptodate page from a streaming write. */
248 unsigned int fstart = finfo->dirty_offset;
249 unsigned int fend = fstart + finfo->dirty_len;
250 unsigned int iend = offset + length;
251
252 if (offset >= fend)
253 return;
254 if (iend <= fstart)
255 return;
256
257 /* The invalidation region overlaps the data. If the region
258 * covers the start of the data, we either move along the start
259 * or just erase the data entirely.
260 */
261 if (offset <= fstart) {
262 if (iend >= fend)
263 goto erase_completely;
264 /* Move the start of the data. */
265 finfo->dirty_len = fend - iend;
266 finfo->dirty_offset = iend;
267 trace_netfs_folio(folio, netfs_folio_trace_invalidate_front);
268 return;
269 }
270
271 /* Reduce the length of the data if the invalidation region
272 * covers the tail part.
273 */
274 if (iend >= fend) {
275 finfo->dirty_len = offset - fstart;
276 trace_netfs_folio(folio, netfs_folio_trace_invalidate_tail);
277 return;
278 }
279
280 /* A partial write was split. The caller has already zeroed
281 * it, so just absorb the hole.
282 */
283 trace_netfs_folio(folio, netfs_folio_trace_invalidate_middle);
284 }
285 return;
286
287erase_completely:
288 netfs_put_group(netfs_folio_group(folio));
289 folio_detach_private(folio);
290 folio_clear_uptodate(folio);
291 folio_cancel_dirty(folio);
292 kfree(finfo);
293 trace_netfs_folio(folio, netfs_folio_trace_invalidate_all);
294}
295EXPORT_SYMBOL(netfs_invalidate_folio);
296
297/**
298 * netfs_release_folio - Try to release a folio
299 * @folio: Folio proposed for release
300 * @gfp: Flags qualifying the release
301 *
302 * Request release of a folio and clean up its private state if it's not busy.
303 * Returns true if the folio can now be released, false if not
304 */
305bool netfs_release_folio(struct folio *folio, gfp_t gfp)
306{
307 struct inode *inode = folio_inode(folio);
308 struct netfs_inode *ctx = netfs_inode(inode);
309 uoff_t i_size, remote_i_size, zero_point, end;
310
311 if (folio_test_dirty(folio))
312 return false;
313
314 netfs_read_sizes(inode, &i_size, &remote_i_size, &zero_point);
315 end = folio_next_pos(folio);
316 if (end > zero_point) {
317 spin_lock(&inode->i_lock);
318 end = umin(end, ctx->_remote_i_size);
319 if (end > ctx->_zero_point)
320 netfs_write_zero_point(inode, end);
321 spin_unlock(&inode->i_lock);
322 }
323
324 if (folio_test_private(folio))
325 return false;
326 if (unlikely(folio_test_private_2(folio))) { /* [DEPRECATED] */
327 if (current_is_kswapd() || !(gfp & __GFP_FS))
328 return false;
329 folio_wait_private_2(folio);
330 }
331 fscache_note_page_release(netfs_i_cookie(ctx));
332 return true;
333}
334EXPORT_SYMBOL(netfs_release_folio);
335
336/*
337 * Wake the collection work item.
338 */
339void netfs_wake_collector(struct netfs_io_request *rreq)
340{
341 if (test_bit(NETFS_RREQ_OFFLOAD_COLLECTION, &rreq->flags) &&
342 !test_bit(NETFS_RREQ_RETRYING, &rreq->flags)) {
343 queue_work(system_dfl_wq, &rreq->work);
344 } else {
345 trace_netfs_rreq(rreq, netfs_rreq_trace_wake_queue);
346 wake_up(&rreq->waitq);
347 }
348}
349
350/*
351 * Mark a subrequest as no longer being in progress and, if need be, wake the
352 * collector.
353 */
354void netfs_subreq_clear_in_progress(struct netfs_io_subrequest *subreq)
355{
356 struct netfs_io_request *rreq = subreq->rreq;
357 struct netfs_io_stream *stream = &rreq->io_streams[subreq->stream_nr];
358
359 clear_bit_unlock(NETFS_SREQ_IN_PROGRESS, &subreq->flags);
360 smp_mb__after_atomic(); /* Clear IN_PROGRESS before task state */
361
362 /* If we are at the head of the queue, wake up the collector. */
363 if (list_is_first(&subreq->rreq_link, &stream->subrequests) ||
364 test_bit(NETFS_RREQ_RETRYING, &rreq->flags))
365 netfs_wake_collector(rreq);
366}
367
368/*
369 * Wait for all outstanding I/O in a stream to quiesce.
370 */
371void netfs_wait_for_in_progress_stream(struct netfs_io_request *rreq,
372 struct netfs_io_stream *stream)
373{
374 struct netfs_io_subrequest *subreq;
375 DEFINE_WAIT(myself);
376
377 list_for_each_entry(subreq, &stream->subrequests, rreq_link) {
378 smp_rmb(); /* Read ->next before IN_PROGRESS. */
379 if (!netfs_check_subreq_in_progress(subreq))
380 continue;
381
382 trace_netfs_rreq(rreq, netfs_rreq_trace_wait_quiesce);
383 for (;;) {
384 prepare_to_wait(&rreq->waitq, &myself, TASK_UNINTERRUPTIBLE);
385
386 if (!netfs_check_subreq_in_progress(subreq))
387 break;
388
389 trace_netfs_sreq(subreq, netfs_sreq_trace_wait_for);
390 schedule();
391 }
392 }
393
394 trace_netfs_rreq(rreq, netfs_rreq_trace_waited_quiesce);
395 finish_wait(&rreq->waitq, &myself);
396}
397
398/*
399 * Perform collection in app thread if not offloaded to workqueue.
400 */
401static int netfs_collect_in_app(struct netfs_io_request *rreq,
402 bool (*collector)(struct netfs_io_request *rreq))
403{
404 bool need_collect = false, inactive = true, done = true;
405
406 if (!netfs_check_rreq_in_progress(rreq)) {
407 trace_netfs_rreq(rreq, netfs_rreq_trace_recollect);
408 return 1; /* Done */
409 }
410
411 for (int i = 0; i < NR_IO_STREAMS; i++) {
412 struct netfs_io_subrequest *subreq;
413 struct netfs_io_stream *stream = &rreq->io_streams[i];
414
415 if (!stream->active)
416 continue;
417 inactive = false;
418 trace_netfs_collect_stream(rreq, stream);
419 subreq = list_first_entry_or_null(&stream->subrequests,
420 struct netfs_io_subrequest,
421 rreq_link);
422 if (subreq &&
423 (!netfs_check_subreq_in_progress(subreq) ||
424 test_bit(NETFS_SREQ_MADE_PROGRESS, &subreq->flags))) {
425 need_collect = true;
426 break;
427 }
428 if (subreq || !netfs_are_all_subreqs_queued(rreq))
429 done = false;
430 }
431
432 if (!need_collect && !inactive && !done)
433 return 0; /* Sleep */
434
435 __set_current_state(TASK_RUNNING);
436 if (collector(rreq)) {
437 /* Drop the ref from the NETFS_RREQ_IN_PROGRESS flag. */
438 netfs_put_request(rreq, netfs_rreq_trace_put_work_ip);
439 return 1; /* Done */
440 }
441
442 if (inactive) {
443 WARN(true, "Failed to collect inactive req R=%08x\n",
444 rreq->debug_id);
445 cond_resched();
446 }
447 return 2; /* Again */
448}
449
450/*
451 * Wait for a request to complete, successfully or otherwise.
452 */
453static ssize_t netfs_wait_for_in_progress(struct netfs_io_request *rreq,
454 bool (*collector)(struct netfs_io_request *rreq))
455{
456 DEFINE_WAIT(myself);
457 ssize_t ret;
458
459 for (;;) {
460 prepare_to_wait(&rreq->waitq, &myself, TASK_UNINTERRUPTIBLE);
461
462 if (!test_bit(NETFS_RREQ_OFFLOAD_COLLECTION, &rreq->flags)) {
463 switch (netfs_collect_in_app(rreq, collector)) {
464 case 0:
465 break;
466 case 1:
467 goto all_collected;
468 case 2:
469 if (!netfs_check_rreq_in_progress(rreq))
470 break;
471 cond_resched();
472 continue;
473 }
474 }
475
476 if (!netfs_check_rreq_in_progress(rreq))
477 break;
478
479 trace_netfs_rreq(rreq, netfs_rreq_trace_wait_ip);
480 schedule();
481 }
482
483all_collected:
484 trace_netfs_rreq(rreq, netfs_rreq_trace_waited_ip);
485 finish_wait(&rreq->waitq, &myself);
486
487 ret = rreq->error;
488 if (ret == 0) {
489 ret = rreq->transferred;
490 switch (rreq->origin) {
491 case NETFS_DIO_READ:
492 case NETFS_DIO_WRITE:
493 case NETFS_READ_SINGLE:
494 case NETFS_UNBUFFERED_READ:
495 case NETFS_UNBUFFERED_WRITE:
496 break;
497 default:
498 if (rreq->submitted < rreq->len) {
499 trace_netfs_failure(rreq, NULL, ret, netfs_fail_short_read);
500 ret = -EIO;
501 }
502 break;
503 }
504 }
505
506 return ret;
507}
508
509ssize_t netfs_wait_for_read(struct netfs_io_request *rreq)
510{
511 return netfs_wait_for_in_progress(rreq, netfs_read_collection);
512}
513
514ssize_t netfs_wait_for_write(struct netfs_io_request *rreq)
515{
516 return netfs_wait_for_in_progress(rreq, netfs_write_collection);
517}
518
519/*
520 * Wait for a paused operation to unpause or complete in some manner.
521 */
522static void netfs_wait_for_pause(struct netfs_io_request *rreq,
523 bool (*collector)(struct netfs_io_request *rreq))
524{
525 DEFINE_WAIT(myself);
526
527 for (;;) {
528 trace_netfs_rreq(rreq, netfs_rreq_trace_wait_pause);
529 prepare_to_wait(&rreq->waitq, &myself, TASK_UNINTERRUPTIBLE);
530
531 if (!test_bit(NETFS_RREQ_OFFLOAD_COLLECTION, &rreq->flags)) {
532 switch (netfs_collect_in_app(rreq, collector)) {
533 case 0:
534 break;
535 case 1:
536 goto all_collected;
537 case 2:
538 if (!netfs_check_rreq_in_progress(rreq) ||
539 !test_bit(NETFS_RREQ_PAUSE, &rreq->flags))
540 break;
541 cond_resched();
542 continue;
543 }
544 }
545
546 if (!netfs_check_rreq_in_progress(rreq) ||
547 !test_bit(NETFS_RREQ_PAUSE, &rreq->flags))
548 break;
549
550 schedule();
551 }
552
553all_collected:
554 trace_netfs_rreq(rreq, netfs_rreq_trace_waited_pause);
555 finish_wait(&rreq->waitq, &myself);
556}
557
558void netfs_wait_for_paused_read(struct netfs_io_request *rreq)
559{
560 return netfs_wait_for_pause(rreq, netfs_read_collection);
561}
562
563void netfs_wait_for_paused_write(struct netfs_io_request *rreq)
564{
565 return netfs_wait_for_pause(rreq, netfs_write_collection);
566}
567
568/*
569 * Wait for the readahead-acquired refs to be put.
570 */
571void netfs_wait_for_put_ra_refs(struct netfs_io_request *rreq)
572{
573 DEFINE_WAIT(myself);
574
575 for (;;) {
576 trace_netfs_rreq(rreq, netfs_rreq_trace_wait_put_ra_refs);
577 prepare_to_wait(&rreq->waitq, &myself, TASK_UNINTERRUPTIBLE);
578 if (!test_bit(NETFS_RREQ_NEED_PUT_RA_REFS, &rreq->flags))
579 break;
580 schedule();
581 }
582
583 trace_netfs_rreq(rreq, netfs_rreq_trace_waited_put_ra_refs);
584 finish_wait(&rreq->waitq, &myself);
585}
586
587/**
588 * netfs_clear_stale_isize - Clear stale pagecache in a to-be-created hole
589 * @inode: The inode to act upon.
590 * @from: The base of the hole to be made.
591 * @to: The top of the hole to be made.
592 * @nowait: True to return -EAGAIN rather than block.
593 * @exclusive: True if the caller holds i_rwsem exclusively for the resize.
594 *
595 * Zero any data left in the pagecache within the [@from, @to) hole by a
596 * write through an mmap so that it isn't exposed as file content once the
597 * file is extended. Only the uptodate folio straddling @from can hold such
598 * data as pages wholly beyond the EOF can't be faulted in, so the zeroing
599 * is limited to that folio. The folio is zeroed rather than dropped so
600 * that a concurrent extending write can't lose data.
601 *
602 * If @exclusive is false, @from is re-read from i_size and used to clamp
603 * the zeroed range, for callers that may race with another writer also
604 * extending the file (eg. multiple buffered writes extending the same file
605 * under a shared i_rwsem). If @exclusive is true, for callers that hold
606 * i_rwsem exclusively across the whole resize and have already updated
607 * i_size to @to, staleness is decided from the folio's dirty state instead:
608 * since no genuine concurrent buffered writer can be racing, a lockless
609 * stat() adopting a server-confirmed size mid-resize has no data behind it
610 * and never dirties the folio, so it can't fool this check into skipping
611 * the zeroing the way it could fool the @exclusive false clamp.
612 *
613 * pagecache_isize_extended() can't be reused here: it is keyed on a
614 * sub-page block size (a no-op when the block size is >= PAGE_SIZE, as on
615 * network filesystems), runs after i_size is updated, can't honour
616 * @nowait, and doesn't wait for writeback. Keep the two in sync if either
617 * is changed.
618 *
619 * Return: 0 on success, or -EAGAIN if @nowait is set and the folio is
620 * mapped or under writeback and so can't be cleaned without blocking.
621 */
622static int netfs_clear_stale_isize(struct inode *inode, uoff_t from,
623 uoff_t to, bool nowait, bool exclusive)
624{
625 struct address_space *mapping = inode->i_mapping;
626 fgf_t fgp = FGP_LOCK;
627 struct folio *folio;
628 int ret;
629
630 if (from >= to)
631 return 0;
632
633 if (nowait)
634 fgp |= FGP_NOWAIT;
635
636 folio = __filemap_get_folio(mapping, from >> PAGE_SHIFT, fgp, 0);
637 if (IS_ERR(folio))
638 return PTR_ERR(folio) == -EAGAIN ? -EAGAIN : 0;
639
640 ret = 0;
641 if (nowait && (folio_mapped(folio) || folio_test_writeback(folio))) {
642 ret = -EAGAIN;
643 goto out;
644 }
645
646 folio_wait_writeback(folio);
647
648 if (folio_mkclean(folio))
649 folio_mark_dirty(folio);
650
651 if (folio_test_uptodate(folio) &&
652 (!exclusive || folio_test_dirty(folio))) {
653 uoff_t fpos = folio_pos(folio);
654
655 if (!exclusive)
656 from = umax(from, i_size_read(inode));
657 if (from < to && from < fpos + folio_size(folio)) {
658 size_t end = umin(to - fpos, folio_size(folio));
659 size_t offset = from - fpos;
660
661 folio_zero_segment(folio, offset, end);
662 }
663 }
664out:
665 folio_unlock(folio);
666 folio_put(folio);
667 return ret;
668}
669
670/* Clear stale pagecache before an extending buffered/DIO write. */
671int netfs_clear_stale_pre_isize(struct inode *inode, uoff_t from,
672 uoff_t to, bool nowait)
673{
674 return netfs_clear_stale_isize(inode, from, to, nowait, false);
675}
676
677/* Clear stale pagecache when extending a file under an exclusive resize. */
678void netfs_clear_stale_post_isize(struct inode *inode, uoff_t from,
679 uoff_t to)
680{
681 netfs_clear_stale_isize(inode, from, to, false, true);
682}
683EXPORT_SYMBOL(netfs_clear_stale_post_isize);