projects/linux·blob master · refs

980 lines · 28.0 KiB

1// SPDX-License-Identifier: GPL-2.0-or-later
2/* Network filesystem high-level buffered read support.
3 *
4 * Copyright (C) 2021 Red Hat, Inc. All Rights Reserved.
5 * Written by David Howells ([email protected])
6 */
7
8#include <linux/export.h>
9#include <linux/task_io_accounting_ops.h>
10#include "internal.h"
11
12static void netfs_cache_expand_readahead(struct netfs_io_request *rreq,
13 uoff_t *_start,
14 uoff_t *_len,
15 uoff_t i_size)
16{
17 struct netfs_cache_resources *cres = &rreq->cache_resources;
18
19 if (cres->ops && cres->ops->expand_readahead)
20 cres->ops->expand_readahead(cres, _start, _len, i_size);
21}
22
23static void netfs_rreq_expand(struct netfs_io_request *rreq,
24 struct readahead_control *ractl)
25{
26 /* Give the cache a chance to change the request parameters. The
27 * resultant request must contain the original region.
28 */
29 netfs_cache_expand_readahead(rreq, &rreq->start, &rreq->len, rreq->i_size);
30
31 /* Give the netfs a chance to change the request parameters. The
32 * resultant request must contain the original region.
33 */
34 if (rreq->netfs_ops->expand_readahead)
35 rreq->netfs_ops->expand_readahead(rreq);
36
37 /* Expand the request if the cache wants it to start earlier. Note
38 * that the expansion may get further extended if the VM wishes to
39 * insert THPs and the preferred start and/or end wind up in the middle
40 * of THPs.
41 *
42 * If this is the case, however, the THP size should be an integer
43 * multiple of the cache granule size, so we get a whole number of
44 * granules to deal with.
45 */
46 if (rreq->start != readahead_pos(ractl) ||
47 rreq->len != readahead_length(ractl)) {
48 readahead_expand(ractl, rreq->start, rreq->len);
49 rreq->start = readahead_pos(ractl);
50 rreq->len = readahead_length(ractl);
51
52 trace_netfs_read(rreq, readahead_pos(ractl), readahead_length(ractl),
53 netfs_read_trace_expanded);
54 }
55}
56
57/*
58 * Drop the folio refs acquired from the readahead API.
59 */
60static void netfs_bulk_drop_ra_refs(struct netfs_io_request *rreq)
61{
62 struct folio_batch fbatch;
63 struct folio *folio;
64 pgoff_t nr_pages = DIV_ROUND_UP(rreq->len, PAGE_SIZE);
65 pgoff_t first = rreq->start / PAGE_SIZE;
66 XA_STATE(xas, &rreq->mapping->i_pages, first);
67
68 folio_batch_init(&fbatch);
69
70 rcu_read_lock();
71
72 xas_for_each(&xas, folio, first + nr_pages - 1) {
73 if (xas_retry(&xas, folio))
74 continue;
75
76 if (!folio_batch_add(&fbatch, folio))
77 folio_batch_release(&fbatch);
78 }
79
80 rcu_read_unlock();
81 folio_batch_release(&fbatch);
82 trace_netfs_rreq(rreq, netfs_rreq_trace_ra_put_ref);
83 clear_bit_unlock(NETFS_RREQ_NEED_PUT_RA_REFS, &rreq->flags);
84 wake_up(&rreq->waitq);
85}
86
87static void netfs_maybe_bulk_drop_ra_refs(struct netfs_io_request *rreq)
88{
89 if (test_bit(NETFS_RREQ_NEED_PUT_RA_REFS, &rreq->flags))
90 netfs_bulk_drop_ra_refs(rreq);
91}
92
93/*
94 * Begin an operation, and fetch the stored zero point value from the cookie if
95 * available.
96 */
97static int netfs_begin_cache_read(struct netfs_io_request *rreq, struct netfs_inode *ctx)
98{
99 return fscache_begin_read_operation(&rreq->cache_resources, netfs_i_cookie(ctx));
100}
101
102/*
103 * netfs_prepare_read_iterator - Prepare the subreq iterator for I/O
104 * @subreq: The subrequest to be set up
105 *
106 * Prepare the I/O iterator representing the read buffer on a subrequest for
107 * the filesystem to use for I/O (it can be passed directly to a socket). This
108 * is intended to be called from the ->issue_read() method once the filesystem
109 * has trimmed the request to the size it wants.
110 *
111 * Returns the limited size if successful and -ENOMEM if insufficient memory
112 * available.
113 */
114static ssize_t netfs_prepare_read_iterator(struct netfs_io_subrequest *subreq)
115{
116 struct netfs_io_request *rreq = subreq->rreq;
117 size_t rsize = subreq->len;
118
119 if (subreq->source == NETFS_DOWNLOAD_FROM_SERVER)
120 rsize = umin(rsize, rreq->io_streams[0].sreq_max_len);
121
122 subreq->len = rsize;
123 if (unlikely(rreq->io_streams[0].sreq_max_segs)) {
124 size_t limit = netfs_limit_iter(&rreq->buffer.iter, 0, rsize,
125 rreq->io_streams[0].sreq_max_segs);
126
127 if (limit < rsize) {
128 subreq->len = limit;
129 trace_netfs_sreq(subreq, netfs_sreq_trace_limited);
130 }
131 }
132
133 subreq->io_iter = rreq->buffer.iter;
134
135 iov_iter_truncate(&subreq->io_iter, subreq->len);
136 rolling_buffer_advance(&rreq->buffer, subreq->len);
137 return subreq->len;
138}
139
140/*
141 * Issue a read against the cache.
142 * - Eats the caller's ref on subreq.
143 */
144static void netfs_read_cache_to_pagecache(struct netfs_io_request *rreq,
145 struct netfs_io_subrequest *subreq)
146{
147 struct netfs_cache_resources *cres = &rreq->cache_resources;
148
149 netfs_stat(&netfs_n_rh_read);
150 cres->ops->read(cres, subreq->start, &subreq->io_iter, NETFS_READ_HOLE_IGNORE,
151 netfs_cache_read_terminated, subreq);
152}
153
154int netfs_read_query_cache(struct netfs_io_request *rreq, struct fscache_occupancy *occ)
155{
156 struct netfs_cache_resources *cres = &rreq->cache_resources;
157
158 occ->granularity = PAGE_SIZE;
159 if (occ->query_from >= occ->query_to)
160 return 0;
161 if (!cres->ops)
162 return 0;
163 occ->query_from = round_up(occ->query_from, occ->granularity);
164 return cres->ops->query_occupancy(cres, occ);
165}
166
167void netfs_queue_read(struct netfs_io_request *rreq,
168 struct netfs_io_subrequest *subreq)
169{
170 struct netfs_io_stream *stream = &rreq->io_streams[0];
171
172 __set_bit(NETFS_SREQ_IN_PROGRESS, &subreq->flags);
173
174 /* We add to the end of the list whilst the collector may be walking
175 * the list. The collector only goes nextwards and uses the lock to
176 * remove entries off of the front.
177 */
178 spin_lock(&rreq->lock);
179 /* Write IN_PROGRESS before pointer to new subreq */
180 list_add_tail_release(&subreq->rreq_link, &stream->subrequests);
181 if (list_is_first(&subreq->rreq_link, &stream->subrequests)) {
182 if (!stream->active) {
183 stream->collected_to = subreq->start;
184 /* Store list pointers before active flag */
185 smp_store_release(&stream->active, true);
186 }
187 }
188
189 spin_unlock(&rreq->lock);
190}
191
192static void netfs_issue_read(struct netfs_io_request *rreq,
193 struct netfs_io_subrequest *subreq)
194{
195 switch (subreq->source) {
196 case NETFS_DOWNLOAD_FROM_SERVER:
197 rreq->netfs_ops->issue_read(subreq);
198 break;
199 case NETFS_READ_FROM_CACHE:
200 netfs_read_cache_to_pagecache(rreq, subreq);
201 break;
202 default:
203 __set_bit(NETFS_SREQ_CLEAR_TAIL, &subreq->flags);
204 subreq->error = 0;
205 iov_iter_zero(subreq->len, &subreq->io_iter);
206 subreq->transferred = subreq->len;
207 netfs_read_subreq_terminated(subreq);
208 break;
209 }
210}
211
212/*
213 * Mark folios that we want to copy to the cache. For filesystems that use
214 * netfslib fully, we set folio->private to NETFS_FOLIO_COPY_TO_CACHE;
215 * otherwise we set the deprecated PG_private_2.
216 */
217static void netfs_mark_copy_to_cache(struct netfs_io_request *rreq,
218 struct folio_queue **fq,
219 unsigned int *offset,
220 int *slot,
221 size_t len,
222 bool copy)
223{
224 while (len > 0) {
225 struct folio *folio;
226 size_t fsize, overlap;
227
228 if (!*fq)
229 break;
230 if (*slot >= folioq_count(*fq)) {
231 *fq = (*fq)->next;
232 *slot = 0;
233 *offset = 0;
234 continue;
235 }
236
237 /* Determine how much the subreq overlaps the folio, if at all. */
238 fsize = folioq_folio_size(*fq, *slot);
239 overlap = min(len, fsize - *offset);
240
241 if (overlap > 0 && copy) {
242 folio = folioq_folio(*fq, *slot);
243 if (netfs_using_pgpriv2(rreq)) {
244 if (!folio_test_private_2(folio))
245 folio_start_private_2(folio);
246 } else {
247 if (!folio_get_private(folio))
248 folio_attach_private(folio, NETFS_FOLIO_COPY_TO_CACHE);
249 }
250 trace_netfs_folio(folio, netfs_folio_trace_mark_copy);
251 }
252
253 len -= overlap;
254 *offset += overlap;
255 if (*offset >= fsize) {
256 *slot += 1;
257 *offset = 0;
258 }
259 }
260}
261
262/*
263 * Perform a read to the pagecache from a series of sources of different types,
264 * slicing up the region to be read according to available cache blocks and
265 * network rsize.
266 */
267static void netfs_read_to_pagecache(struct netfs_io_request *rreq)
268{
269 struct fscache_occupancy _occ = {
270 .query_from = rreq->start,
271 .query_to = rreq->start + rreq->len,
272 .cached_from[0] = 0,
273 .cached_to[0] = 0,
274 .cached_from[1] = ULLONG_MAX,
275 .cached_to[1] = ULLONG_MAX,
276 };
277 struct fscache_occupancy *occ = &_occ;
278 struct folio_queue *fq = rreq->buffer.tail;
279 unsigned int offset = 0;
280 ssize_t size = rreq->len;
281 uoff_t start = rreq->start;
282 int ret = 0, slot = 0;
283
284 do {
285 int (*prepare_read)(struct netfs_io_subrequest *subreq) = NULL;
286 struct netfs_io_subrequest *subreq;
287 enum netfs_io_source source;
288 ssize_t slice;
289 uoff_t hole_to, cache_to;
290 size_t len = size;
291 bool copy = false;
292
293 /* If we don't have any, find out the next couple of data
294 * extents from the cache, containing of following the
295 * specified start offset. Holes have to be fetched from the
296 * server; data regions from the cache.
297 */
298 hole_to = occ->cached_from[0];
299 cache_to = occ->cached_to[0];
300 if (start >= cache_to) {
301 /* Extent exhausted; shuffle down. */
302 int i;
303
304 for (i = 0; i < ARRAY_SIZE(occ->cached_from) - 1; i++) {
305 occ->cached_from[i] = occ->cached_from[i + 1];
306 occ->cached_to[i] = occ->cached_to[i + 1];
307 occ->cached_type[i] = occ->cached_type[i + 1];
308 }
309 occ->cached_from[i] = ULLONG_MAX;
310 occ->cached_to[i] = ULLONG_MAX;
311
312 if (occ->cached_from[0] != ULLONG_MAX)
313 continue;
314
315 /* Get new extents */
316 ret = netfs_read_query_cache(rreq, occ);
317 if (ret < 0)
318 break;
319 continue;
320 }
321
322 uoff_t zero_point = netfs_read_zero_point(rreq->inode);
323 uoff_t zlimit = umin(zero_point, rreq->i_size);
324
325 _debug("rsub %llx %llx-%llx", start, hole_to, cache_to);
326
327 if (start >= hole_to && start < cache_to) {
328 /* Overlap with a cached region, where the cache may
329 * record a block of zeroes.
330 */
331 _debug("cached s=%llx c=%llx l=%zx", start, cache_to, size);
332 len = umin(cache_to - start, size);
333 len = round_up(len, occ->granularity);
334 if (occ->cached_type[0] == FSCACHE_EXTENT_ZERO) {
335 source = NETFS_FILL_WITH_ZEROES;
336 netfs_stat(&netfs_n_rh_zero);
337 } else {
338 source = NETFS_READ_FROM_CACHE;
339 prepare_read = rreq->cache_resources.ops->prepare_read;
340 }
341 } else if (start >= zlimit && size > 0) {
342 /* If this range lies beyond the zero-point, that part
343 * can just be cleared locally.
344 */
345 _debug("zero %llx-%llx", start, start + size);
346 len = size;
347 source = NETFS_FILL_WITH_ZEROES;
348 if (rreq->cache_resources.ops)
349 copy = true;
350 netfs_stat(&netfs_n_rh_zero);
351 } else {
352 /* Read a cache hole from the server. If any part of
353 * this range lies beyond the zero-point or the EOF,
354 * that part can just be cleared locally.
355 */
356 uoff_t limit = min3(zlimit, start + size, hole_to);
357
358 _debug("limit %llx %llx", rreq->i_size, zero_point);
359 _debug("download %llx-%llx", start, start + size);
360 len = umin(limit - start, ULONG_MAX);
361 source = NETFS_DOWNLOAD_FROM_SERVER;
362 prepare_read = rreq->netfs_ops->prepare_read;
363 if (rreq->cache_resources.ops)
364 copy = true;
365 netfs_stat(&netfs_n_rh_download);
366 }
367
368 if (len == 0) {
369 pr_err("ZERO-LEN READ: R=%08x l=%zx/%zx s=%llx z=%llx i=%llx",
370 rreq->debug_id, len, size,
371 start, zero_point, rreq->i_size);
372 break;
373 }
374
375 subreq = netfs_alloc_subrequest(rreq, source);
376 if (!subreq) {
377 ret = -ENOMEM;
378 break;
379 }
380
381 subreq->start = start;
382 subreq->len = len;
383 if (copy)
384 __set_bit(NETFS_SREQ_COPY_TO_CACHE, &subreq->flags);
385
386 netfs_queue_read(rreq, subreq);
387
388 rreq->io_streams[0].sreq_max_len = MAX_RW_COUNT;
389 rreq->io_streams[0].sreq_max_segs = INT_MAX;
390
391 if (prepare_read) {
392 ret = prepare_read(subreq);
393 if (ret < 0) {
394 netfs_cancel_read(subreq, ret);
395 break;
396 }
397 trace_netfs_sreq(subreq, netfs_sreq_trace_prepare);
398 }
399
400 slice = netfs_prepare_read_iterator(subreq);
401 if (slice < 0) {
402 ret = slice;
403 netfs_cancel_read(subreq, ret);
404 break;
405 }
406 start += slice;
407 size -= slice;
408 if (size <= 0)
409 netfs_all_subreqs_queued(rreq);
410
411 if (fq) {
412 /* See if the cache indicated this should be cached. */
413 copy = test_bit(NETFS_SREQ_COPY_TO_CACHE, &subreq->flags);
414 netfs_mark_copy_to_cache(rreq, &fq, &slot, &offset, slice, copy);
415 }
416
417 trace_netfs_sreq(subreq, netfs_sreq_trace_submit);
418 netfs_issue_read(rreq, subreq);
419 netfs_maybe_bulk_drop_ra_refs(rreq);
420
421 if (test_bit(NETFS_RREQ_PAUSE, &rreq->flags))
422 netfs_wait_for_paused_read(rreq);
423 if (test_bit(NETFS_RREQ_FAILED, &rreq->flags))
424 break;
425 cond_resched();
426 } while (size > 0);
427
428 if (unlikely(size > 0)) {
429 netfs_all_subreqs_queued(rreq);
430 netfs_wake_collector(rreq);
431 }
432
433 /* Defer error return as we may need to wait for outstanding I/O. */
434 cmpxchg(&rreq->error, 0, ret);
435}
436
437/**
438 * netfs_readahead - Helper to manage a read request
439 * @ractl: The description of the readahead request
440 *
441 * Fulfil a readahead request by drawing data from the cache if possible, or
442 * the netfs if not. Space beyond the EOF is zero-filled. Multiple I/O
443 * requests from different sources will get munged together. If necessary, the
444 * readahead window can be expanded in either direction to a more convenient
445 * alighment for RPC efficiency or to make storage in the cache feasible.
446 *
447 * The calling netfs must initialise a netfs context contiguous to the vfs
448 * inode before calling this.
449 *
450 * This is usable whether or not caching is enabled.
451 */
452void netfs_readahead(struct readahead_control *ractl)
453{
454 struct netfs_io_request *rreq;
455 struct netfs_inode *ictx = netfs_inode(ractl->mapping->host);
456 ssize_t added;
457 uoff_t start = readahead_pos(ractl);
458 size_t size = readahead_length(ractl);
459 int ret;
460
461 rreq = netfs_alloc_request(ractl->mapping, ractl->file, start, size,
462 NETFS_READAHEAD);
463 if (IS_ERR(rreq))
464 return;
465
466 __set_bit(NETFS_RREQ_OFFLOAD_COLLECTION, &rreq->flags);
467
468 ret = netfs_begin_cache_read(rreq, ictx);
469 if (ret == -ENOMEM || ret == -EINTR || ret == -ERESTARTSYS)
470 goto cleanup_free;
471
472 netfs_stat(&netfs_n_rh_readahead);
473 trace_netfs_read(rreq, readahead_pos(ractl), readahead_length(ractl),
474 netfs_read_trace_readahead);
475
476 netfs_rreq_expand(rreq, ractl);
477
478 /* Load the folios to be read into a bvecq chain. Note that this
479 * acquires a ref on each folio that we will need to release later -
480 * but we don't want to do that until after we've started the I/O.
481 */
482 added = rolling_buffer_bulk_load_from_ra(&rreq->buffer, ractl,
483 rreq->debug_id, rreq->gfp);
484 if (added < 0) {
485 ret = added;
486 goto cleanup_free;
487 }
488 __set_bit(NETFS_RREQ_NEED_PUT_RA_REFS, &rreq->flags);
489
490 rreq->submitted = rreq->start + added;
491 rreq->cleaned_to = rreq->start;
492 netfs_read_set_unlock_at(rreq);
493
494 netfs_read_to_pagecache(rreq);
495 netfs_maybe_bulk_drop_ra_refs(rreq);
496 return netfs_put_request(rreq, netfs_rreq_trace_put_return);
497
498cleanup_free:
499 return netfs_put_failed_request(rreq);
500}
501EXPORT_SYMBOL(netfs_readahead);
502
503/*
504 * Create a rolling buffer with a single occupying folio.
505 */
506static int netfs_create_singular_buffer(struct netfs_io_request *rreq, struct folio *folio,
507 unsigned int rollbuf_flags)
508{
509 ssize_t added;
510
511 if (rolling_buffer_init(&rreq->buffer, rreq->debug_id, ITER_DEST, rreq->gfp) < 0)
512 return -ENOMEM;
513
514 added = rolling_buffer_append(&rreq->buffer, folio, rollbuf_flags, rreq->gfp);
515 if (added < 0)
516 return added;
517 rreq->submitted = rreq->start + added;
518 rreq->progress_at = added;
519 return 0;
520}
521
522/*
523 * Read into gaps in a folio partially filled by a streaming write.
524 */
525static int netfs_read_gaps(struct file *file, struct folio *folio)
526{
527 struct netfs_io_request *rreq;
528 struct address_space *mapping = folio->mapping;
529 struct netfs_group *group = netfs_folio_group(folio);
530 struct netfs_folio *finfo = netfs_folio_info(folio);
531 struct netfs_inode *ctx = netfs_inode(mapping->host);
532 struct bio_vec *bvec = NULL;
533 unsigned int from = finfo->dirty_offset;
534 unsigned int to = from + finfo->dirty_len;
535 unsigned int off = 0;
536 size_t flen = folio_size(folio);
537 size_t nr_bvec = flen / PAGE_SIZE + 2;
538 size_t part;
539 int ret, i = 0, sink_from = -1, sink_to = -1;
540
541 _enter("%lx", folio->index);
542
543 rreq = netfs_alloc_request(mapping, file, folio_pos(folio), flen, NETFS_READ_GAPS);
544 if (IS_ERR(rreq)) {
545 ret = PTR_ERR(rreq);
546 goto alloc_error;
547 }
548
549 ret = netfs_begin_cache_read(rreq, ctx);
550 if (ret == -ENOMEM || ret == -EINTR || ret == -ERESTARTSYS)
551 goto discard;
552
553 netfs_stat(&netfs_n_rh_read_folio);
554 trace_netfs_read(rreq, rreq->start, rreq->len, netfs_read_trace_read_gaps);
555
556 /* Fiddle the buffer so that a gap at the beginning and/or a gap at the
557 * end get copied to, but the middle is discarded.
558 */
559 ret = -ENOMEM;
560 bvec = kmalloc_objs(*bvec, nr_bvec);
561 if (!bvec)
562 goto discard;
563
564 trace_netfs_folio(folio, netfs_folio_trace_read_gaps);
565
566 if (from > 0) {
567 bvec_set_folio(&bvec[i++], folio, from, 0);
568 off = from;
569 }
570 sink_from = i;
571 while (off < to) {
572 struct folio *sink = folio_alloc(GFP_KERNEL, 0);
573
574 if (!sink)
575 goto discard;
576 part = min_t(size_t, to - off, PAGE_SIZE);
577 bvec_set_folio(&bvec[i], sink, part, 0);
578 off += part;
579 sink_to = i;
580 i++;
581 }
582 if (to < flen)
583 bvec_set_folio(&bvec[i++], folio, flen - to, to);
584 iov_iter_bvec(&rreq->buffer.iter, ITER_DEST, bvec, i, rreq->len);
585 rreq->submitted = rreq->start + flen;
586
587 netfs_read_to_pagecache(rreq);
588
589 ret = netfs_wait_for_read(rreq);
590 if (ret >= 0) {
591 if (ret < flen) {
592 struct iov_iter iter;
593
594 iov_iter_bvec(&iter, ITER_DEST, bvec, i, flen);
595 iov_iter_advance(&iter, ret);
596 iov_iter_zero(flen - ret, &iter);
597 }
598 if (group)
599 folio_change_private(folio, group);
600 else
601 folio_detach_private(folio);
602 kfree(finfo);
603 trace_netfs_folio(folio, netfs_folio_trace_filled_gaps);
604 flush_dcache_folio(folio);
605 folio_mark_uptodate(folio);
606 }
607
608 if (sink_to >= 0)
609 for (; sink_from <= sink_to; sink_from++)
610 folio_put(bvec_folio(&bvec[sink_from]));
611 kfree(bvec);
612 folio_unlock(folio);
613 netfs_put_request(rreq, netfs_rreq_trace_put_return);
614 return ret < 0 ? ret : 0;
615
616discard:
617 netfs_put_failed_request(rreq);
618alloc_error:
619 folio_unlock(folio);
620 if (sink_to >= 0)
621 for (; sink_from <= sink_to; sink_from++)
622 folio_put(bvec_folio(&bvec[sink_from]));
623 kfree(bvec);
624 return ret;
625}
626
627/**
628 * netfs_read_folio - Helper to manage a read_folio request
629 * @file: The file to read from
630 * @folio: The folio to read
631 *
632 * Fulfil a read_folio request by drawing data from the cache if
633 * possible, or the netfs if not. Space beyond the EOF is zero-filled.
634 * Multiple I/O requests from different sources will get munged together.
635 *
636 * The calling netfs must initialise a netfs context contiguous to the vfs
637 * inode before calling this.
638 *
639 * This is usable whether or not caching is enabled.
640 */
641int netfs_read_folio(struct file *file, struct folio *folio)
642{
643 struct address_space *mapping = folio->mapping;
644 struct netfs_io_request *rreq;
645 struct netfs_inode *ctx = netfs_inode(mapping->host);
646 int ret;
647
648 folio_wait_writeback(folio);
649
650 if (folio_test_dirty(folio))
651 return netfs_read_gaps(file, folio);
652
653 _enter("%lx", folio->index);
654
655 rreq = netfs_alloc_request(mapping, file,
656 folio_pos(folio), folio_size(folio),
657 NETFS_READPAGE);
658 if (IS_ERR(rreq)) {
659 ret = PTR_ERR(rreq);
660 goto alloc_error;
661 }
662
663 ret = netfs_begin_cache_read(rreq, ctx);
664 if (ret == -ENOMEM || ret == -EINTR || ret == -ERESTARTSYS)
665 goto discard;
666
667 netfs_stat(&netfs_n_rh_read_folio);
668 trace_netfs_read(rreq, rreq->start, rreq->len, netfs_read_trace_readpage);
669
670 /* Set up the output buffer */
671 ret = netfs_create_singular_buffer(rreq, folio, 0);
672 if (ret < 0)
673 goto discard;
674
675 netfs_read_to_pagecache(rreq);
676 ret = netfs_wait_for_read(rreq);
677 netfs_put_request(rreq, netfs_rreq_trace_put_return);
678 return ret < 0 ? ret : 0;
679
680discard:
681 netfs_put_failed_request(rreq);
682alloc_error:
683 folio_unlock(folio);
684 return ret;
685}
686EXPORT_SYMBOL(netfs_read_folio);
687
688/*
689 * Prepare a folio for writing without reading first
690 * @folio: The folio being prepared
691 * @pos: starting position for the write
692 * @len: length of write
693 * @always_fill: T if the folio should always be completely filled/cleared
694 *
695 * In some cases, write_begin doesn't need to read at all:
696 * - full folio write
697 * - write that lies in a folio that is completely beyond EOF
698 * - write that covers the folio from start to EOF or beyond it
699 *
700 * If any of these criteria are met, then zero out the unwritten parts
701 * of the folio and return true. Otherwise, return false.
702 */
703static bool netfs_skip_folio_read(struct folio *folio, uoff_t pos, size_t len,
704 bool always_fill)
705{
706 struct inode *inode = folio_inode(folio);
707 uoff_t i_size = i_size_read(inode);
708 size_t offset = offset_in_folio(folio, pos);
709 size_t plen = folio_size(folio);
710
711 if (unlikely(always_fill)) {
712 if (pos - offset + len <= i_size)
713 return false; /* Page entirely before EOF */
714 folio_zero_segment(folio, 0, plen);
715 folio_mark_uptodate(folio);
716 return true;
717 }
718
719 /* Full folio write */
720 if (offset == 0 && len >= plen)
721 return true;
722
723 /* Page entirely beyond the end of the file */
724 if (pos - offset >= i_size)
725 goto zero_out;
726
727 /* Write that covers from the start of the folio to EOF or beyond */
728 if (offset == 0 && (pos + len) >= i_size)
729 goto zero_out;
730
731 return false;
732zero_out:
733 folio_zero_segments(folio, 0, offset, offset + len, plen);
734 return true;
735}
736
737/**
738 * netfs_write_begin - Helper to prepare for writing [DEPRECATED]
739 * @ctx: The netfs context
740 * @file: The file to read from
741 * @mapping: The mapping to read from
742 * @pos: File position at which the write will begin
743 * @len: The length of the write (may extend beyond the end of the folio chosen)
744 * @_folio: Where to put the resultant folio
745 * @_fsdata: Place for the netfs to store a cookie
746 *
747 * Pre-read data for a write-begin request by drawing data from the cache if
748 * possible, or the netfs if not. Space beyond the EOF is zero-filled.
749 * Multiple I/O requests from different sources will get munged together.
750 *
751 * The calling netfs must provide a table of operations, only one of which,
752 * issue_read, is mandatory.
753 *
754 * The check_write_begin() operation can be provided to check for and flush
755 * conflicting writes once the folio is grabbed and locked. It is passed a
756 * pointer to the fsdata cookie that gets returned to the VM to be passed to
757 * write_end. It is permitted to sleep. It should return 0 if the request
758 * should go ahead or it may return an error. It may also unlock and put the
759 * folio, provided it sets ``*foliop`` to NULL, in which case a return of 0
760 * will cause the folio to be re-got and the process to be retried.
761 *
762 * The calling netfs must initialise a netfs context contiguous to the vfs
763 * inode before calling this.
764 *
765 * This is usable whether or not caching is enabled.
766 *
767 * Note that this should be considered deprecated and netfs_perform_write()
768 * used instead.
769 */
770int netfs_write_begin(struct netfs_inode *ctx,
771 struct file *file, struct address_space *mapping,
772 uoff_t pos, unsigned int len, struct folio **_folio,
773 void **_fsdata)
774{
775 struct netfs_io_request *rreq;
776 struct folio *folio;
777 pgoff_t index = pos >> PAGE_SHIFT;
778 int ret;
779
780retry:
781 folio = __filemap_get_folio(mapping, index, FGP_WRITEBEGIN,
782 mapping_gfp_mask(mapping));
783 if (IS_ERR(folio))
784 return PTR_ERR(folio);
785
786 if (ctx->ops->check_write_begin) {
787 /* Allow the netfs (eg. ceph) to flush conflicts. */
788 ret = ctx->ops->check_write_begin(file, pos, len, &folio, _fsdata);
789 if (ret < 0) {
790 trace_netfs_failure(NULL, NULL, ret, netfs_fail_check_write_begin);
791 goto error;
792 }
793 if (!folio)
794 goto retry;
795 }
796
797 if (folio_test_uptodate(folio))
798 goto have_folio;
799
800 /* If the folio is beyond the EOF, we want to clear it - unless it's
801 * within the cache granule containing the EOF, in which case we need
802 * to preload the granule.
803 */
804 if (!netfs_is_cache_maybe_enabled(ctx) &&
805 netfs_skip_folio_read(folio, pos, len, false)) {
806 netfs_stat(&netfs_n_rh_write_zskip);
807 goto have_folio_no_wait;
808 }
809
810 rreq = netfs_alloc_request(mapping, file,
811 folio_pos(folio), folio_size(folio),
812 NETFS_READ_FOR_WRITE);
813 if (IS_ERR(rreq)) {
814 ret = PTR_ERR(rreq);
815 goto error;
816 }
817 rreq->no_unlock_folio = folio;
818 __set_bit(NETFS_RREQ_NO_UNLOCK_FOLIO, &rreq->flags);
819
820 ret = netfs_begin_cache_read(rreq, ctx);
821 if (ret == -ENOMEM || ret == -EINTR || ret == -ERESTARTSYS)
822 goto error_put;
823
824 netfs_stat(&netfs_n_rh_write_begin);
825 trace_netfs_read(rreq, pos, len, netfs_read_trace_write_begin);
826
827 /* Set up the output buffer */
828 ret = netfs_create_singular_buffer(rreq, folio, 0);
829 if (ret < 0)
830 goto error_put;
831
832 netfs_read_to_pagecache(rreq);
833 ret = netfs_wait_for_read(rreq);
834 netfs_put_request(rreq, netfs_rreq_trace_put_return);
835 if (ret < 0)
836 goto error;
837
838have_folio:
839 ret = folio_wait_private_2_killable(folio);
840 if (ret < 0)
841 goto error;
842have_folio_no_wait:
843 *_folio = folio;
844 _leave(" = 0");
845 return 0;
846
847error_put:
848 netfs_put_failed_request(rreq);
849error:
850 if (folio) {
851 folio_unlock(folio);
852 folio_put(folio);
853 }
854 _leave(" = %d", ret);
855 return ret;
856}
857EXPORT_SYMBOL(netfs_write_begin);
858
859/*
860 * Preload the data into a folio we're proposing to write into.
861 */
862int netfs_prefetch_for_write(struct file *file, struct folio *folio,
863 size_t offset, size_t len)
864{
865 struct netfs_io_request *rreq;
866 struct address_space *mapping = folio->mapping;
867 struct netfs_inode *ctx = netfs_inode(mapping->host);
868 uoff_t start = folio_pos(folio);
869 size_t flen = folio_size(folio);
870 int ret;
871
872 _enter("%zx @%llx", flen, start);
873
874 ret = -ENOMEM;
875
876 rreq = netfs_alloc_request(mapping, file, start, flen,
877 NETFS_READ_FOR_WRITE);
878 if (IS_ERR(rreq)) {
879 ret = PTR_ERR(rreq);
880 goto error;
881 }
882
883 rreq->no_unlock_folio = folio;
884 __set_bit(NETFS_RREQ_NO_UNLOCK_FOLIO, &rreq->flags);
885 ret = netfs_begin_cache_read(rreq, ctx);
886 if (ret == -ENOMEM || ret == -EINTR || ret == -ERESTARTSYS)
887 goto error_put;
888
889 netfs_stat(&netfs_n_rh_write_begin);
890 trace_netfs_read(rreq, start, flen, netfs_read_trace_prefetch_for_write);
891
892 /* Set up the output buffer */
893 ret = netfs_create_singular_buffer(rreq, folio, NETFS_ROLLBUF_PAGECACHE_MARK);
894 if (ret < 0)
895 goto error_put;
896
897 netfs_read_to_pagecache(rreq);
898 ret = netfs_wait_for_read(rreq);
899 netfs_put_request(rreq, netfs_rreq_trace_put_return);
900 return ret < 0 ? ret : 0;
901
902error_put:
903 netfs_put_failed_request(rreq);
904error:
905 _leave(" = %d", ret);
906 return ret;
907}
908
909/**
910 * netfs_buffered_read_iter - Filesystem buffered I/O read routine
911 * @iocb: kernel I/O control block
912 * @iter: destination for the data read
913 *
914 * This is the ->read_iter() routine for all filesystems that can use the page
915 * cache directly.
916 *
917 * The IOCB_NOWAIT flag in iocb->ki_flags indicates that -EAGAIN shall be
918 * returned when no data can be read without waiting for I/O requests to
919 * complete; it doesn't prevent readahead.
920 *
921 * The IOCB_NOIO flag in iocb->ki_flags indicates that no new I/O requests
922 * shall be made for the read or for readahead. When no data can be read,
923 * -EAGAIN shall be returned. When readahead would be triggered, a partial,
924 * possibly empty read shall be returned.
925 *
926 * Return:
927 * * number of bytes copied, even for partial reads
928 * * negative error code (or 0 if IOCB_NOIO) if nothing was read
929 */
930ssize_t netfs_buffered_read_iter(struct kiocb *iocb, struct iov_iter *iter)
931{
932 struct inode *inode = file_inode(iocb->ki_filp);
933 struct netfs_inode *ictx = netfs_inode(inode);
934 ssize_t ret;
935
936 if (WARN_ON_ONCE((iocb->ki_flags & IOCB_DIRECT) ||
937 test_bit(NETFS_ICTX_UNBUFFERED, &ictx->flags)))
938 return -EINVAL;
939
940 ret = netfs_start_io_read(inode);
941 if (ret == 0) {
942 ret = filemap_read(iocb, iter, 0);
943 netfs_end_io_read(inode);
944 }
945 return ret;
946}
947EXPORT_SYMBOL(netfs_buffered_read_iter);
948
949/**
950 * netfs_file_read_iter - Generic filesystem read routine
951 * @iocb: kernel I/O control block
952 * @iter: destination for the data read
953 *
954 * This is the ->read_iter() routine for all filesystems that can use the page
955 * cache directly.
956 *
957 * The IOCB_NOWAIT flag in iocb->ki_flags indicates that -EAGAIN shall be
958 * returned when no data can be read without waiting for I/O requests to
959 * complete; it doesn't prevent readahead.
960 *
961 * The IOCB_NOIO flag in iocb->ki_flags indicates that no new I/O requests
962 * shall be made for the read or for readahead. When no data can be read,
963 * -EAGAIN shall be returned. When readahead would be triggered, a partial,
964 * possibly empty read shall be returned.
965 *
966 * Return:
967 * * number of bytes copied, even for partial reads
968 * * negative error code (or 0 if IOCB_NOIO) if nothing was read
969 */
970ssize_t netfs_file_read_iter(struct kiocb *iocb, struct iov_iter *iter)
971{
972 struct netfs_inode *ictx = netfs_inode(iocb->ki_filp->f_mapping->host);
973
974 if ((iocb->ki_flags & IOCB_DIRECT) ||
975 test_bit(NETFS_ICTX_UNBUFFERED, &ictx->flags))
976 return netfs_unbuffered_read_iter(iocb, iter);
977
978 return netfs_buffered_read_iter(iocb, iter);
979}
980EXPORT_SYMBOL(netfs_file_read_iter);