projects/linux·blob master · refs

822 lines · 23.1 KiB

1// SPDX-License-Identifier: GPL-2.0-only
2/* Network filesystem high-level (buffered) writeback.
3 *
4 * Copyright (C) 2024 Red Hat, Inc. All Rights Reserved.
5 * Written by David Howells ([email protected])
6 *
7 *
8 * To support network filesystems with local caching, we manage a situation
9 * that can be envisioned like the following:
10 *
11 * +---+---+-----+-----+---+----------+
12 * Folios: | | | | | | |
13 * +---+---+-----+-----+---+----------+
14 *
15 * +------+------+ +----+----+
16 * Upload: | | |.....| | |
17 * (Stream 0) +------+------+ +----+----+
18 *
19 * +------+------+------+------+------+
20 * Cache: | | | | | |
21 * (Stream 1) +------+------+------+------+------+
22 *
23 * Where we have a sequence of folios of varying sizes that we need to overlay
24 * with multiple parallel streams of I/O requests, where the I/O requests in a
25 * stream may also be of various sizes (in cifs, for example, the sizes are
26 * negotiated with the server; in something like ceph, they may represent the
27 * sizes of storage objects).
28 *
29 * The sequence in each stream may contain gaps and noncontiguous subrequests
30 * may be glued together into single vectored write RPCs.
31 */
32
33#include <linux/export.h>
34#include <linux/fs.h>
35#include <linux/mm.h>
36#include <linux/pagemap.h>
37#include "internal.h"
38
39/*
40 * Kill all dirty folios in the event of an unrecoverable error, starting with
41 * a locked folio we've already obtained from writeback_iter().
42 */
43static void netfs_kill_dirty_pages(struct address_space *mapping,
44 struct writeback_control *wbc,
45 struct folio *folio)
46{
47 int error = 0;
48
49 do {
50 enum netfs_folio_trace why = netfs_folio_trace_kill;
51 struct netfs_group *group = NULL;
52 struct netfs_folio *finfo = NULL;
53 void *priv;
54
55 priv = folio_detach_private(folio);
56 if (priv) {
57 finfo = __netfs_folio_info(priv);
58 if (finfo) {
59 /* Kill folio from streaming write. */
60 group = finfo->netfs_group;
61 why = netfs_folio_trace_kill_s;
62 } else {
63 group = priv;
64 if (group == NETFS_FOLIO_COPY_TO_CACHE) {
65 /* Kill copy-to-cache folio */
66 why = netfs_folio_trace_kill_cc;
67 group = NULL;
68 } else {
69 /* Kill folio with group */
70 why = netfs_folio_trace_kill_g;
71 }
72 }
73 }
74
75 trace_netfs_folio(folio, why);
76
77 folio_start_writeback(folio);
78 folio_unlock(folio);
79 folio_end_writeback(folio);
80
81 netfs_put_group(group);
82 kfree(finfo);
83
84 } while ((folio = writeback_iter(mapping, wbc, folio, &error)));
85}
86
87/*
88 * Create a write request and set it up appropriately for the origin type.
89 */
90struct netfs_io_request *netfs_create_write_req(struct address_space *mapping,
91 struct file *file,
92 uoff_t start,
93 enum netfs_io_origin origin)
94{
95 struct netfs_io_request *wreq;
96 struct netfs_inode *ictx;
97 bool is_cacheable = (origin == NETFS_WRITEBACK ||
98 origin == NETFS_WRITEBACK_SINGLE ||
99 origin == NETFS_PGPRIV2_COPY_TO_CACHE);
100
101 wreq = netfs_alloc_request(mapping, file, start, 0, origin);
102 if (IS_ERR(wreq))
103 return wreq;
104
105 _enter("R=%x", wreq->debug_id);
106
107 ictx = netfs_inode(wreq->inode);
108 if (is_cacheable)
109 fscache_begin_write_operation(&wreq->cache_resources, netfs_i_cookie(ictx));
110 if (rolling_buffer_init(&wreq->buffer, wreq->debug_id, ITER_SOURCE, wreq->gfp) < 0)
111 goto nomem;
112
113 wreq->cleaned_to = wreq->start;
114 if (wreq->cache_resources.dio_size > 1)
115 wreq->cache_coll_to = round_down(wreq->start, wreq->cache_resources.dio_size);
116
117 wreq->io_streams[0].stream_nr = 0;
118 wreq->io_streams[0].source = NETFS_UPLOAD_TO_SERVER;
119 wreq->io_streams[0].prepare_write = ictx->ops->prepare_write;
120 wreq->io_streams[0].issue_write = ictx->ops->issue_write;
121 wreq->io_streams[0].collected_to = start;
122 wreq->io_streams[0].transferred = 0;
123
124 wreq->io_streams[1].stream_nr = 1;
125 wreq->io_streams[1].source = NETFS_WRITE_TO_CACHE;
126 wreq->io_streams[1].collected_to = start;
127 wreq->io_streams[1].transferred = 0;
128 if (fscache_resources_valid(&wreq->cache_resources)) {
129 wreq->io_streams[1].avail = true;
130 wreq->io_streams[1].active = true;
131 wreq->io_streams[1].prepare_write = wreq->cache_resources.ops->prepare_write_subreq;
132 wreq->io_streams[1].issue_write = wreq->cache_resources.ops->issue_write;
133 }
134
135 return wreq;
136nomem:
137 netfs_put_failed_request(wreq);
138 return ERR_PTR(-ENOMEM);
139}
140
141/**
142 * netfs_prepare_write_failed - Note write preparation failed
143 * @subreq: The subrequest to mark
144 *
145 * Mark a subrequest to note that preparation for write failed.
146 */
147void netfs_prepare_write_failed(struct netfs_io_subrequest *subreq)
148{
149 WARN_ON_ONCE(!subreq->error);
150 __set_bit(NETFS_SREQ_FAILED, &subreq->flags);
151 trace_netfs_sreq(subreq, netfs_sreq_trace_prep_failed);
152}
153EXPORT_SYMBOL(netfs_prepare_write_failed);
154
155/*
156 * Prepare a write subrequest. We need to allocate a new subrequest
157 * if we don't have one.
158 */
159void netfs_prepare_write(struct netfs_io_request *wreq,
160 struct netfs_io_stream *stream,
161 uoff_t start)
162{
163 struct netfs_io_subrequest *subreq;
164 struct iov_iter *wreq_iter = &wreq->buffer.iter;
165
166 /* Make sure we don't point the iterator at a used-up folio_queue
167 * struct being used as a placeholder to prevent the queue from
168 * collapsing. In such a case, extend the queue.
169 */
170 if (iov_iter_is_folioq(wreq_iter) &&
171 wreq_iter->folioq_slot >= folioq_nr_slots(wreq_iter->folioq))
172 rolling_buffer_make_space(&wreq->buffer, wreq->gfp);
173
174 subreq = netfs_alloc_subrequest(wreq, stream->source);
175 if (!subreq)
176 return;
177 subreq->start = start;
178 subreq->stream_nr = stream->stream_nr;
179 subreq->io_iter = *wreq_iter;
180
181 _enter("R=%x[%x]", wreq->debug_id, subreq->debug_index);
182
183 trace_netfs_sreq(subreq, netfs_sreq_trace_prepare);
184
185 stream->sreq_max_len = UINT_MAX;
186 stream->sreq_max_segs = INT_MAX;
187 switch (stream->source) {
188 case NETFS_UPLOAD_TO_SERVER:
189 netfs_stat(&netfs_n_wh_upload);
190 stream->sreq_max_len = wreq->wsize;
191 break;
192 case NETFS_WRITE_TO_CACHE:
193 netfs_stat(&netfs_n_wh_write);
194 break;
195 default:
196 WARN_ON_ONCE(1);
197 break;
198 }
199
200 if (stream->prepare_write)
201 stream->prepare_write(subreq);
202
203 __set_bit(NETFS_SREQ_IN_PROGRESS, &subreq->flags);
204
205 /* We add to the end of the list whilst the collector may be walking
206 * the list. The collector only goes nextwards and uses the lock to
207 * remove entries off of the front.
208 */
209 spin_lock(&wreq->lock);
210 /* Write IN_PROGRESS before pointer to new subreq */
211 list_add_tail_release(&subreq->rreq_link, &stream->subrequests);
212 if (list_is_first(&subreq->rreq_link, &stream->subrequests)) {
213 if (!stream->active) {
214 stream->collected_to = subreq->start;
215 /* Write list pointers before active flag */
216 smp_store_release(&stream->active, true);
217 }
218 }
219
220 spin_unlock(&wreq->lock);
221
222 stream->construct = subreq;
223}
224
225/*
226 * Set the I/O iterator for the filesystem/cache to use and dispatch the I/O
227 * operation. The operation may be asynchronous and should call
228 * netfs_write_subrequest_terminated() when complete.
229 */
230static void netfs_do_issue_write(struct netfs_io_stream *stream,
231 struct netfs_io_subrequest *subreq)
232{
233 struct netfs_io_request *wreq = subreq->rreq;
234
235 _enter("R=%x[%x],%zx", wreq->debug_id, subreq->debug_index, subreq->len);
236
237 if (stream->source == NETFS_WRITE_TO_CACHE &&
238 unlikely(test_bit(NETFS_RREQ_CACHE_STOP, &wreq->flags))) {
239 size_t dio_size = wreq->cache_resources.dio_size;
240 size_t len, disp;
241
242 disp = subreq->start & (dio_size - 1);
243 len = round_up(subreq->len + disp, dio_size);
244
245 subreq->start -= disp;
246 subreq->len = len;
247
248 __set_bit(NETFS_SREQ_CANCELLED, &subreq->flags);
249 return netfs_write_subrequest_terminated(subreq, subreq->len);
250 }
251
252 if (test_bit(NETFS_SREQ_FAILED, &subreq->flags))
253 return netfs_write_subrequest_terminated(subreq, subreq->error);
254
255 trace_netfs_sreq(subreq, netfs_sreq_trace_submit);
256 stream->issue_write(subreq);
257}
258
259void netfs_reissue_write(struct netfs_io_stream *stream,
260 struct netfs_io_subrequest *subreq,
261 struct iov_iter *source)
262{
263 size_t size = subreq->len - subreq->transferred;
264
265 // TODO: Use encrypted buffer
266 subreq->io_iter = *source;
267 iov_iter_advance(source, size);
268 iov_iter_truncate(&subreq->io_iter, size);
269
270 subreq->retry_count++;
271 subreq->error = 0;
272 __clear_bit(NETFS_SREQ_MADE_PROGRESS, &subreq->flags);
273 __set_bit(NETFS_SREQ_IN_PROGRESS, &subreq->flags);
274 netfs_stat(&netfs_n_wh_retry_write_subreq);
275 netfs_do_issue_write(stream, subreq);
276}
277
278void netfs_issue_write(struct netfs_io_request *wreq,
279 struct netfs_io_stream *stream)
280{
281 struct netfs_io_subrequest *subreq = stream->construct;
282
283 if (!subreq)
284 return;
285
286 stream->construct = NULL;
287 subreq->io_iter.count = subreq->len;
288 netfs_do_issue_write(stream, subreq);
289}
290
291/*
292 * Add data to the write subrequest, dispatching each as we fill it up or if it
293 * is discontiguous with the previous. We only fill one part at a time so that
294 * we can avoid overrunning the credits obtained (cifs) and try to parallelise
295 * content-crypto preparation with network writes.
296 */
297size_t netfs_advance_write(struct netfs_io_request *wreq,
298 struct netfs_io_stream *stream,
299 uoff_t start, size_t len, bool to_eof)
300{
301 struct netfs_io_subrequest *subreq = stream->construct;
302 size_t part;
303
304 if (!stream->avail) {
305 _leave("no write");
306 return len;
307 }
308
309 _enter("R=%x[%x]", wreq->debug_id, subreq ? subreq->debug_index : 0);
310
311 if (subreq && start != subreq->start + subreq->len) {
312 netfs_issue_write(wreq, stream);
313 subreq = NULL;
314 }
315
316 if (!stream->construct)
317 netfs_prepare_write(wreq, stream, start);
318 subreq = stream->construct;
319
320 part = umin(stream->sreq_max_len - subreq->len, len);
321 _debug("part %zx/%zx %zx/%zx", subreq->len, stream->sreq_max_len, part, len);
322 subreq->len += part;
323 subreq->nr_segs++;
324 stream->submit_extendable_to -= part;
325
326 if (subreq->len >= stream->sreq_max_len ||
327 subreq->nr_segs >= stream->sreq_max_segs ||
328 to_eof) {
329 netfs_issue_write(wreq, stream);
330 subreq = NULL;
331 }
332
333 return part;
334}
335
336/*
337 * Write some of a pending folio data back to the server.
338 */
339static int netfs_write_folio(struct netfs_io_request *wreq,
340 struct writeback_control *wbc,
341 struct folio *folio)
342{
343 struct netfs_io_stream *upload = &wreq->io_streams[0];
344 struct netfs_io_stream *cache = &wreq->io_streams[1];
345 struct netfs_io_stream *stream;
346 struct netfs_group *fgroup; /* TODO: Use this with ceph */
347 struct netfs_folio *finfo;
348 size_t iter_off = 0;
349 size_t fsize = folio_size(folio), flen = fsize, foff = 0;
350 uoff_t fpos = folio_pos(folio), i_size;
351 bool to_eof = false, streamw = false;
352 bool debug = false;
353
354 _enter("");
355
356 if (rolling_buffer_make_space(&wreq->buffer, wreq->gfp) < 0)
357 return -ENOMEM;
358
359 /* netfs_perform_write() may shift i_size around the page or from out
360 * of the page to beyond it, but cannot move i_size into or through the
361 * page since we have it locked.
362 */
363 i_size = i_size_read(wreq->inode);
364
365 if (fpos >= i_size) {
366 /* mmap beyond eof. */
367 _debug("beyond eof");
368 folio_start_writeback(folio);
369 folio_unlock(folio);
370 wreq->nr_group_rel += netfs_folio_written_back(folio);
371 netfs_put_group_many(wreq->group, wreq->nr_group_rel);
372 wreq->nr_group_rel = 0;
373 return 0;
374 }
375
376 if (fpos + fsize > wreq->i_size)
377 wreq->i_size = i_size;
378
379 fgroup = netfs_folio_group(folio);
380 finfo = netfs_folio_info(folio);
381 if (finfo) {
382 foff = finfo->dirty_offset;
383 flen = foff + finfo->dirty_len;
384 streamw = true;
385 }
386
387 if (flen > i_size - fpos) {
388 flen = i_size - fpos;
389 if (!streamw)
390 folio_zero_segment(folio, flen, fsize);
391 to_eof = true;
392 } else if (flen == i_size - fpos) {
393 to_eof = true;
394 }
395 flen -= foff;
396
397 _debug("folio %zx %zx %zx", foff, flen, fsize);
398
399 /* Deal with discontinuities in the stream of dirty pages. These can
400 * arise from a number of sources:
401 *
402 * (1) Intervening non-dirty pages from random-access writes, multiple
403 * flushers writing back different parts simultaneously and manual
404 * syncing.
405 *
406 * (2) Partially-written pages from write-streaming.
407 *
408 * (3) Pages that belong to a different write-back group (eg. Ceph
409 * snapshots).
410 *
411 * (4) Actually-clean pages that were marked for write to the cache
412 * when they were read. Note that these appear as a special
413 * write-back group.
414 */
415 if (fgroup == NETFS_FOLIO_COPY_TO_CACHE) {
416 netfs_issue_write(wreq, upload);
417 } else if (fgroup != wreq->group) {
418 /* We can't write this page to the server yet. */
419 kdebug("wrong group");
420 folio_redirty_for_writepage(wbc, folio);
421 folio_unlock(folio);
422 netfs_issue_write(wreq, upload);
423 netfs_issue_write(wreq, cache);
424 return 0;
425 }
426
427 if (foff > 0)
428 netfs_issue_write(wreq, upload);
429 if (streamw)
430 netfs_issue_write(wreq, cache);
431
432 folio_start_writeback(folio);
433 folio_unlock(folio);
434
435 if (fgroup == NETFS_FOLIO_COPY_TO_CACHE) {
436 if (!cache->avail) {
437 trace_netfs_folio(folio, netfs_folio_trace_cancel_copy);
438 netfs_issue_write(wreq, upload);
439 netfs_folio_written_back(folio);
440 return 0;
441 }
442 trace_netfs_folio(folio, netfs_folio_trace_store_copy);
443 } else if (!upload->avail && !cache->avail) {
444 trace_netfs_folio(folio, netfs_folio_trace_cancel_store);
445 netfs_folio_written_back(folio);
446 return 0;
447 } else if (!upload->construct) {
448 trace_netfs_folio(folio, netfs_folio_trace_store);
449 } else {
450 trace_netfs_folio(folio, netfs_folio_trace_store_plus);
451 }
452
453 /* Attach the folio to the rolling buffer. */
454 rolling_buffer_append(&wreq->buffer, folio, 0, wreq->gfp);
455
456 /* Move the submission point forward to allow for write-streaming data
457 * not starting at the front of the page. We don't do write-streaming
458 * with the cache as the cache requires DIO alignment.
459 *
460 * Also skip uploading for data that's been read and just needs copying
461 * to the cache.
462 */
463 for (int s = 0; s < NR_IO_STREAMS; s++) {
464 stream = &wreq->io_streams[s];
465 stream->submit_off = foff;
466 stream->submit_len = flen;
467 if (!stream->avail ||
468 (stream->source == NETFS_WRITE_TO_CACHE && streamw) ||
469 (stream->source == NETFS_UPLOAD_TO_SERVER &&
470 fgroup == NETFS_FOLIO_COPY_TO_CACHE)) {
471 stream->submit_off = UINT_MAX;
472 stream->submit_len = 0;
473 }
474 }
475
476 /* Attach the folio to one or more subrequests. For a big folio, we
477 * could end up with thousands of subrequests if the wsize is small -
478 * but we might need to wait during the creation of subrequests for
479 * network resources (eg. SMB credits).
480 */
481 for (;;) {
482 ssize_t part;
483 size_t lowest_off = ULONG_MAX;
484 int choose_s = -1;
485
486 /* Always add to the lowest-submitted stream first. */
487 for (int s = 0; s < NR_IO_STREAMS; s++) {
488 stream = &wreq->io_streams[s];
489 if (stream->submit_len > 0 &&
490 stream->submit_off < lowest_off) {
491 lowest_off = stream->submit_off;
492 choose_s = s;
493 }
494 }
495
496 if (choose_s < 0)
497 break;
498 stream = &wreq->io_streams[choose_s];
499
500 /* Advance the iterator(s). */
501 if (stream->submit_off > iter_off) {
502 rolling_buffer_advance(&wreq->buffer, stream->submit_off - iter_off);
503 iter_off = stream->submit_off;
504 }
505
506 atomic64_set(&wreq->issued_to, fpos + stream->submit_off);
507 stream->submit_extendable_to = fsize - stream->submit_off;
508 part = netfs_advance_write(wreq, stream, fpos + stream->submit_off,
509 stream->submit_len, to_eof);
510 stream->submit_off += part;
511 if (part > stream->submit_len)
512 stream->submit_len = 0;
513 else
514 stream->submit_len -= part;
515 if (part > 0)
516 debug = true;
517 }
518
519 if (fsize > iter_off)
520 rolling_buffer_advance(&wreq->buffer, fsize - iter_off);
521 atomic64_set(&wreq->issued_to, fpos + fsize);
522
523 if (!debug)
524 kdebug("R=%x: No submit", wreq->debug_id);
525
526 if (foff + flen < fsize)
527 for (int s = 0; s < NR_IO_STREAMS; s++)
528 netfs_issue_write(wreq, &wreq->io_streams[s]);
529
530 _leave(" = 0");
531 return 0;
532}
533
534/*
535 * End the issuing of writes, letting the collector know we're done.
536 */
537static void netfs_end_issue_write(struct netfs_io_request *wreq)
538{
539 bool needs_poke = true;
540
541 netfs_all_subreqs_queued(wreq);
542
543 for (int s = 0; s < NR_IO_STREAMS; s++) {
544 struct netfs_io_stream *stream = &wreq->io_streams[s];
545
546 if (!stream->active)
547 continue;
548 if (!list_empty(&stream->subrequests))
549 needs_poke = false;
550 netfs_issue_write(wreq, stream);
551 }
552
553 if (needs_poke)
554 netfs_wake_collector(wreq);
555}
556
557/*
558 * Write some of the pending data back to the server
559 */
560int netfs_writepages(struct address_space *mapping,
561 struct writeback_control *wbc)
562{
563 struct netfs_inode *ictx = netfs_inode(mapping->host);
564 struct netfs_io_request *wreq = NULL;
565 struct folio *folio;
566 int error = 0;
567
568 if (!netfs_wb_begin(ictx, wbc->sync_mode == WB_SYNC_NONE))
569 return 0;
570
571 /* Need the first folio to be able to set up the op. */
572 folio = writeback_iter(mapping, wbc, NULL, &error);
573 if (!folio)
574 goto out;
575
576 wreq = netfs_create_write_req(mapping, NULL, folio_pos(folio), NETFS_WRITEBACK);
577 if (IS_ERR(wreq)) {
578 error = PTR_ERR(wreq);
579 goto couldnt_start;
580 }
581
582 __set_bit(NETFS_RREQ_OFFLOAD_COLLECTION, &wreq->flags);
583 trace_netfs_write(wreq, netfs_write_trace_writeback);
584 netfs_stat(&netfs_n_wh_writepages);
585
586 do {
587 _debug("wbiter %lx %llx", folio->index, atomic64_read(&wreq->issued_to));
588
589 /* It appears we don't have to handle cyclic writeback wrapping. */
590 WARN_ON_ONCE(wreq && folio_pos(folio) < atomic64_read(&wreq->issued_to));
591
592 if (netfs_folio_group(folio) != NETFS_FOLIO_COPY_TO_CACHE &&
593 unlikely(!test_bit(NETFS_RREQ_UPLOAD_TO_SERVER, &wreq->flags))) {
594 set_bit(NETFS_RREQ_UPLOAD_TO_SERVER, &wreq->flags);
595 wreq->netfs_ops->begin_writeback(wreq);
596 }
597
598 error = netfs_write_folio(wreq, wbc, folio);
599 if (error == -ENOMEM) {
600 folio_redirty_for_writepage(wbc, folio);
601 folio_unlock(folio);
602 }
603 } while ((folio = writeback_iter(mapping, wbc, folio, &error)));
604
605 netfs_end_issue_write(wreq);
606 netfs_wake_collector(wreq);
607
608 netfs_put_request(wreq, netfs_rreq_trace_put_return);
609 _leave(" = %d", error);
610 return error;
611
612couldnt_start:
613 if (error == -ENOMEM) {
614 folio_redirty_for_writepage(wbc, folio);
615 folio_unlock(folio);
616 folio = writeback_iter(mapping, wbc, folio, &error);
617 WARN_ON_ONCE(folio != NULL);
618 } else {
619 netfs_kill_dirty_pages(mapping, wbc, folio);
620 }
621out:
622 netfs_wb_end(ictx);
623 _leave(" = %d", error);
624 return error;
625}
626EXPORT_SYMBOL(netfs_writepages);
627
628/*
629 * Write some of a pending folio data back to the server and/or the cache.
630 */
631static int netfs_write_folio_single(struct netfs_io_request *wreq,
632 struct folio *folio)
633{
634 struct netfs_io_stream *upload = &wreq->io_streams[0];
635 struct netfs_io_stream *cache = &wreq->io_streams[1];
636 struct netfs_io_stream *stream;
637 size_t iter_off = 0;
638 size_t fsize = folio_size(folio), flen;
639 uoff_t fpos = folio_pos(folio);
640 ssize_t ret;
641 bool to_eof = false;
642 bool no_debug = false;
643
644 _enter("");
645
646 flen = folio_size(folio);
647 if (flen > wreq->i_size - fpos) {
648 flen = wreq->i_size - fpos;
649 folio_zero_segment(folio, flen, fsize);
650 to_eof = true;
651 } else if (flen == wreq->i_size - fpos) {
652 to_eof = true;
653 }
654
655 _debug("folio %zx/%zx", flen, fsize);
656
657 if (!upload->avail && !cache->avail) {
658 trace_netfs_folio(folio, netfs_folio_trace_cancel_store);
659 return 0;
660 }
661
662 if (!upload->construct)
663 trace_netfs_folio(folio, netfs_folio_trace_store);
664 else
665 trace_netfs_folio(folio, netfs_folio_trace_store_plus);
666
667 /* Attach the folio to the rolling buffer. */
668 folio_get(folio);
669 ret = rolling_buffer_append(&wreq->buffer, folio, NETFS_ROLLBUF_PUT_MARK, wreq->gfp);
670 if (ret < 0) {
671 folio_put(folio);
672 return ret;
673 }
674
675 /* Move the submission point forward to allow for write-streaming data
676 * not starting at the front of the page. We don't do write-streaming
677 * with the cache as the cache requires DIO alignment.
678 *
679 * Also skip uploading for data that's been read and just needs copying
680 * to the cache.
681 */
682 for (int s = 0; s < NR_IO_STREAMS; s++) {
683 stream = &wreq->io_streams[s];
684 stream->submit_off = 0;
685 stream->submit_len = flen;
686 if (!stream->avail) {
687 stream->submit_off = UINT_MAX;
688 stream->submit_len = 0;
689 }
690 }
691
692 /* Attach the folio to one or more subrequests. For a big folio, we
693 * could end up with thousands of subrequests if the wsize is small -
694 * but we might need to wait during the creation of subrequests for
695 * network resources (eg. SMB credits).
696 */
697 for (;;) {
698 ssize_t part;
699 size_t lowest_off = ULONG_MAX;
700 int choose_s = -1;
701
702 /* Always add to the lowest-submitted stream first. */
703 for (int s = 0; s < NR_IO_STREAMS; s++) {
704 stream = &wreq->io_streams[s];
705 if (stream->submit_len > 0 &&
706 stream->submit_off < lowest_off) {
707 lowest_off = stream->submit_off;
708 choose_s = s;
709 }
710 }
711
712 if (choose_s < 0)
713 break;
714 stream = &wreq->io_streams[choose_s];
715
716 /* Advance the iterator(s). */
717 if (stream->submit_off > iter_off) {
718 rolling_buffer_advance(&wreq->buffer, stream->submit_off - iter_off);
719 iter_off = stream->submit_off;
720 }
721
722 atomic64_set(&wreq->issued_to, fpos + stream->submit_off);
723 stream->submit_extendable_to = fsize - stream->submit_off;
724 part = netfs_advance_write(wreq, stream, fpos + stream->submit_off,
725 stream->submit_len, to_eof);
726 stream->submit_off += part;
727 if (part > stream->submit_len)
728 stream->submit_len = 0;
729 else
730 stream->submit_len -= part;
731 if (part > 0)
732 no_debug = true;
733 }
734
735 wreq->buffer.iter.iov_offset = 0;
736 if (fsize > iter_off)
737 rolling_buffer_advance(&wreq->buffer, fsize - iter_off);
738 atomic64_set(&wreq->issued_to, fpos + fsize);
739
740 if (!no_debug)
741 kdebug("R=%x: No submit", wreq->debug_id);
742 _leave(" = 0");
743 return 0;
744}
745
746/**
747 * netfs_writeback_single - Write back a monolithic payload
748 * @mapping: The mapping to write from
749 * @wbc: Hints from the VM
750 * @iter: Data to write, must be ITER_FOLIOQ.
751 *
752 * Write a monolithic, non-pagecache object back to the server and/or
753 * the cache.
754 *
755 * Return: 0 if successful; 1 if skipped due to lock conflict and WB_SYNC_NONE;
756 * or a negative error code.
757 */
758int netfs_writeback_single(struct address_space *mapping,
759 struct writeback_control *wbc,
760 struct iov_iter *iter)
761{
762 struct netfs_io_request *wreq;
763 struct netfs_inode *ictx = netfs_inode(mapping->host);
764 struct folio_queue *fq;
765 size_t size = iov_iter_count(iter);
766 int ret;
767
768 if (WARN_ON_ONCE(!iov_iter_is_folioq(iter)))
769 return -EIO;
770
771 if (!netfs_wb_begin(ictx, wbc->sync_mode == WB_SYNC_NONE)) {
772 /* The VFS will have undirtied the inode. */
773 netfs_single_mark_inode_dirty(&ictx->inode);
774 return 1;
775 }
776
777 wreq = netfs_create_write_req(mapping, NULL, 0, NETFS_WRITEBACK_SINGLE);
778 if (IS_ERR(wreq)) {
779 ret = PTR_ERR(wreq);
780 goto couldnt_start;
781 }
782
783 __set_bit(NETFS_RREQ_OFFLOAD_COLLECTION, &wreq->flags);
784 trace_netfs_write(wreq, netfs_write_trace_writeback_single);
785 netfs_stat(&netfs_n_wh_writepages);
786
787 if (__test_and_set_bit(NETFS_RREQ_UPLOAD_TO_SERVER, &wreq->flags))
788 wreq->netfs_ops->begin_writeback(wreq);
789
790 for (fq = (struct folio_queue *)iter->folioq; fq; fq = fq->next) {
791 for (int slot = 0; slot < folioq_count(fq); slot++) {
792 struct folio *folio = folioq_folio(fq, slot);
793 size_t part = umin(folioq_folio_size(fq, slot), size);
794
795 _debug("wbiter %lx %llx", folio->index, atomic64_read(&wreq->issued_to));
796
797 ret = netfs_write_folio_single(wreq, folio);
798 if (ret < 0)
799 goto stop;
800 size -= part;
801 if (size <= 0)
802 goto stop;
803 }
804 }
805
806stop:
807 for (int s = 0; s < NR_IO_STREAMS; s++)
808 netfs_issue_write(wreq, &wreq->io_streams[s]);
809 netfs_all_subreqs_queued(wreq);
810
811 netfs_wake_collector(wreq);
812
813 netfs_put_request(wreq, netfs_rreq_trace_put_return);
814 _leave(" = %d", ret);
815 return ret;
816
817couldnt_start:
818 netfs_wb_end(ictx);
819 _leave(" = %d", ret);
820 return ret;
821}
822EXPORT_SYMBOL(netfs_writeback_single);