projects/linux·blob master · refs

702 lines · 17.6 KiB

1// SPDX-License-Identifier: GPL-2.0
2/*
3 * linux/mm/page_io.c
4 *
5 * Copyright (C) 1991, 1992, 1993, 1994 Linus Torvalds
6 *
7 * Swap reorganised 29.12.95,
8 * Asynchronous swapping added 30.12.95. Stephen Tweedie
9 * Removed race in async swapping. 14.4.1996. Bruno Haible
10 * Add swap of shared pages through the page cache. 20.2.1998. Stephen Tweedie
11 * Always use brw_page, life becomes simpler. 12 May 1998 Eric Biederman
12 */
13
14#include <linux/mm.h>
15#include <linux/kernel_stat.h>
16#include <linux/gfp.h>
17#include <linux/pagemap.h>
18#include <linux/swap.h>
19#include <linux/bio.h>
20#include <linux/swapops.h>
21#include <linux/writeback.h>
22#include <linux/blkdev.h>
23#include <linux/psi.h>
24#include <linux/uio.h>
25#include <linux/sched/task.h>
26#include <linux/delayacct.h>
27#include <linux/zswap.h>
28#include <linux/swap_ops.h>
29#include "swap.h"
30#include "swap_table.h"
31
32int generic_swapfile_activate(struct swap_info_struct *sis,
33 struct file *swap_file,
34 sector_t *span)
35{
36 struct address_space *mapping = swap_file->f_mapping;
37 struct inode *inode = mapping->host;
38 unsigned blocks_per_page;
39 unsigned long page_no;
40 unsigned blkbits;
41 sector_t probe_block;
42 sector_t last_block;
43 sector_t lowest_block = -1;
44 sector_t highest_block = 0;
45 int nr_extents = 0;
46 int ret;
47
48 blkbits = inode->i_blkbits;
49 blocks_per_page = PAGE_SIZE >> blkbits;
50
51 /*
52 * Map all the blocks into the extent tree. This code doesn't try
53 * to be very smart.
54 */
55 probe_block = 0;
56 page_no = 0;
57 last_block = i_size_read(inode) >> blkbits;
58 while ((probe_block + blocks_per_page) <= last_block &&
59 page_no < sis->max) {
60 unsigned block_in_page;
61 sector_t first_block;
62
63 cond_resched();
64
65 first_block = probe_block;
66 ret = bmap(inode, &first_block);
67 if (ret || !first_block)
68 goto bad_bmap;
69
70 /*
71 * It must be PAGE_SIZE aligned on-disk
72 */
73 if (first_block & (blocks_per_page - 1)) {
74 probe_block++;
75 goto reprobe;
76 }
77
78 for (block_in_page = 1; block_in_page < blocks_per_page;
79 block_in_page++) {
80 sector_t block;
81
82 block = probe_block + block_in_page;
83 ret = bmap(inode, &block);
84 if (ret || !block)
85 goto bad_bmap;
86
87 if (block != first_block + block_in_page) {
88 /* Discontiguity */
89 probe_block++;
90 goto reprobe;
91 }
92 }
93
94 first_block >>= (PAGE_SHIFT - blkbits);
95 if (page_no) { /* exclude the header page */
96 if (first_block < lowest_block)
97 lowest_block = first_block;
98 if (first_block > highest_block)
99 highest_block = first_block;
100 }
101
102 /*
103 * We found a PAGE_SIZE-length, PAGE_SIZE-aligned run of blocks
104 */
105 ret = add_swap_extent(sis, page_no, 1, first_block);
106 if (ret < 0)
107 goto out;
108 nr_extents += ret;
109 page_no++;
110 probe_block += blocks_per_page;
111reprobe:
112 continue;
113 }
114 ret = nr_extents;
115 *span = 1 + highest_block - lowest_block;
116 if (page_no == 0)
117 page_no = 1; /* force Empty message */
118 sis->max = page_no;
119 sis->pages = page_no - 1;
120out:
121 return ret;
122bad_bmap:
123 pr_err("swapon: swapfile has holes\n");
124 ret = -EINVAL;
125 goto out;
126}
127
128static bool is_folio_zero_filled(struct folio *folio)
129{
130 unsigned int pos, last_pos;
131 unsigned long *data;
132 unsigned int i;
133
134 last_pos = PAGE_SIZE / sizeof(*data) - 1;
135 for (i = 0; i < folio_nr_pages(folio); i++) {
136 data = kmap_local_folio(folio, i * PAGE_SIZE);
137 /*
138 * Check last word first, incase the page is zero-filled at
139 * the start and has non-zero data at the end, which is common
140 * in real-world workloads.
141 */
142 if (data[last_pos]) {
143 kunmap_local(data);
144 return false;
145 }
146 for (pos = 0; pos < last_pos; pos++) {
147 if (data[pos]) {
148 kunmap_local(data);
149 return false;
150 }
151 }
152 kunmap_local(data);
153 }
154
155 return true;
156}
157
158static void swap_zeromap_folio_set(struct folio *folio)
159{
160 struct obj_cgroup *objcg = get_obj_cgroup_from_folio(folio);
161 int nr_pages = folio_nr_pages(folio);
162 struct swap_cluster_info *ci;
163 swp_entry_t entry;
164 unsigned int i;
165
166 VM_WARN_ON_ONCE_FOLIO(!folio_test_swapcache(folio), folio);
167 VM_WARN_ON_ONCE_FOLIO(!folio_test_locked(folio), folio);
168
169 ci = swap_cluster_get_and_lock(folio);
170 for (i = 0; i < folio_nr_pages(folio); i++) {
171 entry = page_swap_entry(folio_page(folio, i));
172 __swap_table_set_zero(ci, swp_cluster_offset(entry));
173 }
174 swap_cluster_unlock(ci);
175
176 count_vm_events(SWPOUT_ZERO, nr_pages);
177 if (objcg) {
178 count_objcg_events(objcg, SWPOUT_ZERO, nr_pages);
179 obj_cgroup_put(objcg);
180 }
181}
182
183static void swap_zeromap_folio_clear(struct folio *folio)
184{
185 struct swap_cluster_info *ci;
186 swp_entry_t entry;
187 unsigned int i;
188
189 VM_WARN_ON_ONCE_FOLIO(!folio_test_swapcache(folio), folio);
190 VM_WARN_ON_ONCE_FOLIO(!folio_test_locked(folio), folio);
191
192 ci = swap_cluster_get_and_lock(folio);
193 for (i = 0; i < folio_nr_pages(folio); i++) {
194 entry = page_swap_entry(folio_page(folio, i));
195 __swap_table_clear_zero(ci, swp_cluster_offset(entry));
196 }
197 swap_cluster_unlock(ci);
198}
199
200/*
201 * We may have stale swap cache pages in memory: notice
202 * them here and get rid of the unnecessary final write.
203 */
204int swap_writeout(struct swap_io_ctx *ctx, struct folio *folio)
205{
206 int ret = 0;
207
208 if (folio_free_swap(folio))
209 goto out_unlock;
210
211 /*
212 * Arch code may have to preserve more data than just the page
213 * contents, e.g. memory tags.
214 */
215 ret = arch_prepare_to_swap(folio);
216 if (ret) {
217 folio_mark_dirty(folio);
218 goto out_unlock;
219 }
220
221 /*
222 * Use the swap table zero mark to avoid doing IO for zero-filled
223 * pages. The zero mark is protected by the cluster lock, which is
224 * acquired internally by swap_zeromap_folio_set/clear.
225 */
226 if (is_folio_zero_filled(folio)) {
227 swap_zeromap_folio_set(folio);
228 goto out_unlock;
229 }
230
231 /*
232 * Clear bits this folio occupies in the zeromap to prevent zero data
233 * being read in from any previous zero writes that occupied the same
234 * swap entries.
235 */
236 swap_zeromap_folio_clear(folio);
237
238 if (zswap_store(folio)) {
239 count_mthp_stat(folio_order(folio), MTHP_STAT_ZSWPOUT);
240 goto out_unlock;
241 }
242
243 rcu_read_lock();
244 if (!mem_cgroup_zswap_writeback_enabled(folio_memcg(folio))) {
245 rcu_read_unlock();
246 folio_mark_dirty(folio);
247 return AOP_WRITEPAGE_ACTIVATE;
248 }
249 rcu_read_unlock();
250
251 __swap_writepage(ctx, folio);
252 return 0;
253out_unlock:
254 folio_unlock(folio);
255 return ret;
256}
257
258#if defined(CONFIG_MEMCG) && defined(CONFIG_BLK_CGROUP)
259static struct cgroup_subsys_state *folio_memcg_blkg_css(struct folio *folio)
260{
261 return cgroup_e_css(folio_memcg(folio)->css.cgroup, &io_cgrp_subsys);
262}
263
264static bool folio_blkg_can_merge(struct folio *folio, struct folio *prev_folio)
265{
266 bool can_merge = true;
267
268 if (folio_memcg_charged(folio) != folio_memcg_charged(prev_folio))
269 return false;
270 if (folio_memcg_charged(folio)) {
271 rcu_read_lock();
272 if (folio_memcg_blkg_css(folio) !=
273 folio_memcg_blkg_css(prev_folio))
274 can_merge = false;
275 rcu_read_unlock();
276 }
277 return can_merge;
278}
279
280static void bio_associate_blkg_from_page(struct bio *bio, struct folio *folio)
281{
282 struct cgroup_subsys_state *css;
283
284 if (!folio_memcg_charged(folio))
285 return;
286 rcu_read_lock();
287 css = folio_memcg_blkg_css(folio);
288 if (css && !css_tryget(css))
289 css = NULL;
290 rcu_read_unlock();
291
292 bio_associate_blkg_from_css(bio, css);
293 if (css)
294 css_put(css);
295}
296#else
297static bool folio_blkg_can_merge(struct folio *folio, struct folio *prev_folio)
298{
299 return true;
300}
301#define bio_associate_blkg_from_page(bio, folio) do { } while (0)
302#endif /* CONFIG_MEMCG && CONFIG_BLK_CGROUP */
303
304static mempool_t *sio_pool;
305
306int sio_pool_init(void)
307{
308 if (!sio_pool) {
309 mempool_t *pool = mempool_create_kmalloc_pool(
310 SWAP_CLUSTER_MAX, sizeof(struct swap_iocb));
311 if (cmpxchg(&sio_pool, NULL, pool))
312 mempool_destroy(pool);
313 }
314 if (!sio_pool)
315 return -ENOMEM;
316 return 0;
317}
318
319static bool swap_can_merge(struct swap_io_ctx *ctx, struct folio *folio,
320 int rw)
321{
322 struct swap_info_struct *sis = __swap_entry_to_info(folio->swap);
323 struct bio_vec *last_bv = &ctx->sio->bvecs[ctx->sio->nr_bvecs - 1];
324 struct folio *prev_folio = bvec_folio(last_bv);
325 size_t prev_folio_size = folio_size(prev_folio);
326
327 if (ctx->sis != sis)
328 return false;
329 return sis->ops->can_merge(folio, prev_folio, prev_folio_size, rw);
330}
331
332static void swap_add_folio(struct swap_io_ctx *ctx, struct folio *folio, int rw)
333{
334 struct swap_info_struct *sis = __swap_entry_to_info(folio->swap);
335 struct swap_iocb *sio = ctx->sio;
336
337 if (sio && !swap_can_merge(ctx, folio, rw)) {
338 if (rw == WRITE)
339 swap_write_submit(ctx);
340 else
341 swap_read_submit(ctx);
342 sio = ctx->sio;
343 }
344
345 if (!sio) {
346 ctx->sis = sis;
347 ctx->sio = sio = mempool_alloc(sio_pool, GFP_NOIO);
348 sio->nr_bvecs = 0;
349 sio->len = 0;
350 }
351 bvec_set_folio(&sio->bvecs[sio->nr_bvecs], folio, folio_size(folio), 0);
352 sio->len += folio_size(folio);
353
354 /*
355 * Write out the iocb if we filled it, or if the device is synchronous.
356 *
357 * The latter is to work around expectations in the classic LRU code
358 * which make synchronous clearing of the folio writeback flag in the
359 * reclaim path beneficial.
360 */
361 if (++sio->nr_bvecs == ARRAY_SIZE(sio->bvecs) ||
362 (rw == WRITE && (sis->flags & SWP_SYNCHRONOUS_IO))) {
363 if (rw == WRITE)
364 swap_write_submit(ctx);
365 else
366 swap_read_submit(ctx);
367 }
368}
369
370void __swap_writepage(struct swap_io_ctx *ctx, struct folio *folio)
371{
372 VM_BUG_ON_FOLIO(!folio_test_swapcache(folio), folio);
373
374#ifdef CONFIG_TRANSPARENT_HUGEPAGE
375 if (unlikely(folio_test_pmd_mappable(folio))) {
376 count_memcg_folio_events(folio, THP_SWPOUT, 1);
377 count_vm_event(THP_SWPOUT);
378 }
379#endif
380 count_mthp_stat(folio_order(folio), MTHP_STAT_SWPOUT);
381 count_memcg_folio_events(folio, PSWPOUT, folio_nr_pages(folio));
382 count_vm_events(PSWPOUT, folio_nr_pages(folio));
383
384 folio_start_writeback(folio);
385 folio_unlock(folio);
386 swap_add_folio(ctx, folio, WRITE);
387}
388
389/*
390 * Return the count of contiguous swap entries that share the same
391 * zeromap status as the starting entry. If is_zerop is not NULL,
392 * it will return the zeromap status of the starting entry.
393 *
394 * Context: Caller must ensure the cluster containing the entries
395 * that will be checked won't be freed.
396 */
397static int swap_zeromap_batch(swp_entry_t entry, int max_nr,
398 bool *is_zerop)
399{
400 int i;
401 bool is_zero;
402 unsigned int ci_start = swp_cluster_offset(entry);
403 struct swap_cluster_info *ci = __swap_entry_to_cluster(entry);
404
405 VM_WARN_ON_ONCE(ci_start + max_nr > SWAPFILE_CLUSTER);
406
407 rcu_read_lock();
408 is_zero = __swap_table_test_zero(ci, ci_start);
409 for (i = 1; i < max_nr; i++)
410 if (is_zero != __swap_table_test_zero(ci, ci_start + i))
411 break;
412 rcu_read_unlock();
413 if (is_zerop)
414 *is_zerop = is_zero;
415
416 return i;
417}
418
419static bool swap_read_folio_zeromap(struct folio *folio)
420{
421 int nr_pages = folio_nr_pages(folio);
422 struct obj_cgroup *objcg;
423 bool is_zeromap;
424
425 VM_WARN_ON_ONCE_FOLIO(!folio_test_locked(folio), folio);
426
427 /*
428 * Swapping in a large folio that is partially in the zeromap is not
429 * currently handled. Return true without marking the folio uptodate so
430 * that an IO error is emitted (e.g. do_swap_page() will sigbus).
431 * Folio lock stabilizes the cluster and map, so the check is safe.
432 */
433 if (WARN_ON_ONCE(swap_zeromap_batch(folio->swap, nr_pages,
434 &is_zeromap) != nr_pages))
435 return true;
436
437 if (!is_zeromap)
438 return false;
439
440 objcg = get_obj_cgroup_from_folio(folio);
441 count_vm_events(SWPIN_ZERO, nr_pages);
442 if (objcg) {
443 count_objcg_events(objcg, SWPIN_ZERO, nr_pages);
444 obj_cgroup_put(objcg);
445 }
446
447 folio_zero_range(folio, 0, folio_size(folio));
448 folio_mark_uptodate(folio);
449 return true;
450}
451
452void swap_read_folio(struct swap_io_ctx *ctx, struct folio *folio)
453{
454 struct swap_info_struct *sis = __swap_entry_to_info(folio->swap);
455 bool synchronous = sis->flags & SWP_SYNCHRONOUS_IO;
456 bool workingset = folio_test_workingset(folio);
457 unsigned long pflags;
458 bool in_thrashing;
459
460 VM_BUG_ON_FOLIO(!folio_test_swapcache(folio) && !synchronous, folio);
461 VM_BUG_ON_FOLIO(!folio_test_locked(folio), folio);
462 VM_BUG_ON_FOLIO(folio_test_uptodate(folio), folio);
463
464 /*
465 * Count submission time as memory stall and delay. When the device
466 * is congested, or the submitting cgroup IO-throttled, submission
467 * can be a significant part of overall IO time.
468 */
469 if (workingset) {
470 delayacct_thrashing_start(&in_thrashing);
471 psi_memstall_enter(&pflags);
472 }
473 delayacct_swapin_start();
474
475 if (swap_read_folio_zeromap(folio)) {
476 folio_unlock(folio);
477 goto finish;
478 }
479
480 if (zswap_load(folio) != -ENOENT)
481 goto finish;
482
483 /* We have to read from slower devices. Increase zswap protection. */
484 zswap_folio_swapin(folio);
485 swap_add_folio(ctx, folio, READ);
486
487finish:
488 if (workingset) {
489 delayacct_thrashing_end(&in_thrashing);
490 psi_memstall_leave(&pflags);
491 }
492 delayacct_swapin_end();
493}
494
495static void swap_write_end(struct swap_iocb *sio, bool failed)
496{
497 int p;
498
499 for (p = 0; p < sio->nr_bvecs; p++) {
500 struct page *page = sio->bvecs[p].bv_page;
501
502 if (failed) {
503 set_page_dirty(page);
504 ClearPageReclaim(page);
505 }
506 end_page_writeback(page);
507 }
508 mempool_free(sio, sio_pool);
509}
510
511static void swap_fs_write_complete(struct kiocb *iocb, long ret)
512{
513 struct swap_iocb *sio = container_of(iocb, struct swap_iocb, iocb);
514 bool failed = ret != sio->len;
515
516 if (failed) {
517 struct page *page = sio->bvecs[0].bv_page;
518
519 /*
520 * In the case of swap-over-nfs, this can be a temporary failure
521 * if the system has limited memory for allocating transmit
522 * buffers. Mark the page dirty and avoid
523 * folio_rotate_reclaimable but rate-limit the messages.
524 */
525 pr_err_ratelimited("Write error %ld on dio swapfile (%llu)\n",
526 ret, swap_dev_pos(page_swap_entry(page)));
527 }
528
529 swap_write_end(sio, failed);
530}
531
532static void end_swap_bio_write(struct bio *bio)
533{
534 struct swap_iocb *sio = container_of(bio, struct swap_iocb, bio);
535 bool failed = !!bio->bi_status;
536
537 if (failed)
538 pr_alert_ratelimited("Write-error on swap-device (%u:%u:%llu)\n",
539 MAJOR(bio_dev(bio)), MINOR(bio_dev(bio)),
540 (unsigned long long)bio->bi_iter.bi_sector);
541 bio_uninit(bio);
542 swap_write_end(sio, failed);
543}
544
545static void swap_read_end(struct swap_iocb *sio, bool failed)
546{
547 int p;
548
549 for (p = 0; p < sio->nr_bvecs; p++) {
550 struct folio *folio = bvec_folio(&sio->bvecs[p]);
551
552 if (!failed) {
553 count_mthp_stat(folio_order(folio), MTHP_STAT_SWPIN);
554 count_memcg_folio_events(folio, PSWPIN,
555 folio_nr_pages(folio));
556 folio_mark_uptodate(folio);
557 }
558 folio_unlock(folio);
559 }
560
561 if (!failed)
562 count_vm_events(PSWPIN, sio->len >> PAGE_SHIFT);
563
564 mempool_free(sio, sio_pool);
565}
566
567static void swap_fs_read_complete(struct kiocb *iocb, long ret)
568{
569 struct swap_iocb *sio = container_of(iocb, struct swap_iocb, iocb);
570 bool failed = ret != sio->len;
571
572 if (failed)
573 pr_alert_ratelimited("Read-error on swap-device\n");
574 swap_read_end(sio, failed);
575}
576
577static void swap_bio_read_end_io(struct bio *bio)
578{
579 struct swap_iocb *sio = container_of(bio, struct swap_iocb, bio);
580 bool failed = !!bio->bi_status;
581
582 if (failed)
583 pr_alert_ratelimited("Read-error on swap-device (%u:%u:%llu)\n",
584 MAJOR(bio_dev(bio)), MINOR(bio_dev(bio)),
585 (unsigned long long)bio->bi_iter.bi_sector);
586 bio_uninit(bio);
587 swap_read_end(sio, failed);
588}
589
590static void swap_bdev_submit_write(struct swap_io_ctx *ctx)
591{
592 struct swap_iocb *sio = ctx->sio;
593 struct bio *bio = &sio->bio;
594
595 bio_init(bio, ctx->sis->bdev, sio->bvecs, ARRAY_SIZE(sio->bvecs),
596 REQ_OP_WRITE | REQ_SWAP);
597 bio->bi_iter.bi_size = sio->len;
598 bio->bi_iter.bi_sector = swap_folio_sector(bio_first_folio_all(bio));
599 bio_associate_blkg_from_page(bio, bio_first_folio_all(bio));
600
601 if (ctx->sis->flags & SWP_SYNCHRONOUS_IO) {
602 submit_bio_wait(bio);
603 end_swap_bio_write(bio);
604 } else {
605 bio->bi_end_io = end_swap_bio_write;
606 submit_bio(bio);
607 }
608}
609
610static void swap_bdev_submit_read(struct swap_io_ctx *ctx)
611{
612 struct swap_iocb *sio = ctx->sio;
613 struct bio *bio = &sio->bio;
614
615 bio_init(bio, ctx->sis->bdev, sio->bvecs, ARRAY_SIZE(sio->bvecs),
616 REQ_OP_READ);
617 bio->bi_iter.bi_size = sio->len;
618 bio->bi_iter.bi_sector = swap_folio_sector(bio_first_folio_all(bio));
619
620 if (ctx->sis->flags & SWP_SYNCHRONOUS_IO) {
621 /*
622 * Keep this task valid during swap readpage because the oom
623 * killer may attempt to access it in the page fault retry
624 * time check.
625 */
626 get_task_struct(current);
627 submit_bio_wait(bio);
628 swap_bio_read_end_io(bio);
629 put_task_struct(current);
630 } else {
631 bio->bi_end_io = swap_bio_read_end_io;
632 submit_bio(bio);
633 }
634}
635
636static bool swap_bdev_can_merge(struct folio *folio, struct folio *prev_folio,
637 size_t prev_folio_size, int rw)
638{
639 if (swap_folio_sector(folio) !=
640 swap_folio_sector(prev_folio) + (prev_folio_size >> SECTOR_SHIFT))
641 return false;
642 if (rw == WRITE && !folio_blkg_can_merge(folio, prev_folio))
643 return false;
644 return true;
645}
646
647const struct swap_ops swap_bdev_ops = {
648 .submit_write = swap_bdev_submit_write,
649 .submit_read = swap_bdev_submit_read,
650 .can_merge = swap_bdev_can_merge,
651};
652
653void swap_fs_prepare_rw(struct swap_io_ctx *ctx, int rw, struct iov_iter *iter)
654{
655 struct swap_iocb *sio = ctx->sio;
656
657 init_sync_kiocb(&sio->iocb, ctx->sis->swap_file);
658 sio->iocb.ki_pos = swap_dev_pos(bvec_folio(&sio->bvecs[0])->swap);
659 if (rw == WRITE)
660 sio->iocb.ki_complete = swap_fs_write_complete;
661 else
662 sio->iocb.ki_complete = swap_fs_read_complete;
663
664 iov_iter_bvec(iter, rw == WRITE ? ITER_SOURCE : ITER_DEST,
665 sio->bvecs, sio->nr_bvecs, sio->len);
666}
667EXPORT_SYMBOL_GPL(swap_fs_prepare_rw);
668
669bool swap_fs_can_merge(struct folio *folio, struct folio *prev_folio,
670 size_t prev_folio_size, int rw)
671{
672 return swap_dev_pos(folio->swap) ==
673 swap_dev_pos(prev_folio->swap) + prev_folio_size;
674}
675EXPORT_SYMBOL_GPL(swap_fs_can_merge);
676
677int swap_fs_activate(struct swap_info_struct *sis, const struct swap_ops *ops)
678{
679 sis->ops = ops;
680 return add_swap_extent(sis, 0, sis->max, 0);
681}
682EXPORT_SYMBOL_GPL(swap_fs_activate);
683
684void swap_write_submit(struct swap_io_ctx *ctx)
685{
686 if (!ctx->sio)
687 return;
688 count_vm_events(NRSWPOUT, 1);
689 ctx->sis->ops->submit_write(ctx);
690 ctx->sio = NULL;
691 ctx->sis = NULL;
692}
693
694void swap_read_submit(struct swap_io_ctx *ctx)
695{
696 if (!ctx->sio)
697 return;
698 count_vm_events(NRSWPIN, 1);
699 ctx->sis->ops->submit_read(ctx);
700 ctx->sio = NULL;
701 ctx->sis = NULL;
702}