projects/linux·blob master · refs

8131 lines · 226.7 KiB

1// SPDX-License-Identifier: GPL-2.0
2/*
3 * Copyright (C) 1991, 1992, 1993, 1994 Linus Torvalds
4 *
5 * Swap reorganised 29.12.95, Stephen Tweedie.
6 * kswapd added: 7.1.96 sct
7 * Removed kswapd_ctl limits, and swap out as many pages as needed
8 * to bring the system back to freepages.high: 2.4.97, Rik van Riel.
9 * Zone aware kswapd started 02/00, Kanoj Sarcar ([email protected]).
10 * Multiqueue VM started 5.8.00, Rik van Riel.
11 */
12
13#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
14
15#include <linux/mm.h>
16#include <linux/sched/mm.h>
17#include <linux/module.h>
18#include <linux/gfp.h>
19#include <linux/kernel_stat.h>
20#include <linux/swap.h>
21#include <linux/pagemap.h>
22#include <linux/init.h>
23#include <linux/highmem.h>
24#include <linux/vmpressure.h>
25#include <linux/vmstat.h>
26#include <linux/file.h>
27#include <linux/writeback.h>
28#include <linux/blk_plug.h>
29#include <linux/buffer_head.h> /* for buffer_heads_over_limit */
30#include <linux/mm_inline.h>
31#include <linux/backing-dev.h>
32#include <linux/rmap.h>
33#include <linux/topology.h>
34#include <linux/cpu.h>
35#include <linux/cpuset.h>
36#include <linux/compaction.h>
37#include <linux/notifier.h>
38#include <linux/delay.h>
39#include <linux/kthread.h>
40#include <linux/freezer.h>
41#include <linux/memcontrol.h>
42#include <linux/migrate.h>
43#include <linux/delayacct.h>
44#include <linux/sysctl.h>
45#include <linux/memory-tiers.h>
46#include <linux/oom.h>
47#include <linux/folio_batch.h>
48#include <linux/prefetch.h>
49#include <linux/printk.h>
50#include <linux/dax.h>
51#include <linux/psi.h>
52#include <linux/pagewalk.h>
53#include <linux/shmem_fs.h>
54#include <linux/ctype.h>
55#include <linux/debugfs.h>
56#include <linux/khugepaged.h>
57#include <linux/rculist_nulls.h>
58#include <linux/random.h>
59#include <linux/mmu_notifier.h>
60#include <linux/parser.h>
61#include <linux/swap_ops.h>
62
63#include <asm/tlbflush.h>
64#include <asm/div64.h>
65
66#include <linux/swapops.h>
67#include <linux/sched/sysctl.h>
68
69#include "internal.h"
70#include "page_alloc.h"
71#include "swap.h"
72
73#define CREATE_TRACE_POINTS
74#include <trace/events/vmscan.h>
75
76struct scan_control {
77 /* How many pages shrink_list() should reclaim */
78 unsigned long nr_to_reclaim;
79
80 /*
81 * Nodemask of nodes allowed by the caller. If NULL, all nodes
82 * are scanned.
83 */
84 const nodemask_t *nodemask;
85
86 /*
87 * The memory cgroup that hit its limit and as a result is the
88 * primary target of this reclaim invocation.
89 */
90 struct mem_cgroup *target_mem_cgroup;
91
92 /*
93 * Scan pressure balancing between anon and file LRUs
94 */
95 unsigned long anon_cost;
96 unsigned long file_cost;
97
98 /* Swappiness value for proactive reclaim. Always use sc_swappiness()! */
99 int *proactive_swappiness;
100
101 /* Can active folios be deactivated as part of reclaim? */
102#define DEACTIVATE_ANON 1
103#define DEACTIVATE_FILE 2
104 unsigned int may_deactivate:2;
105 unsigned int force_deactivate:1;
106 unsigned int skipped_deactivate:1;
107
108 /* zone_reclaim_mode, boost reclaim */
109 unsigned int may_writepage:1;
110
111 /* zone_reclaim_mode */
112 unsigned int may_unmap:1;
113
114 /* zone_reclaim_mode, boost reclaim, cgroup restrictions */
115 unsigned int may_swap:1;
116
117 /* Not allow cache_trim_mode to be turned on as part of reclaim? */
118 unsigned int no_cache_trim_mode:1;
119
120 /* Has cache_trim_mode failed at least once? */
121 unsigned int cache_trim_mode_failed:1;
122
123 /* Proactive reclaim invoked by userspace */
124 unsigned int proactive:1;
125
126 /*
127 * Cgroup memory below memory.low is protected as long as we
128 * don't threaten to OOM. If any cgroup is reclaimed at
129 * reduced force or passed over entirely due to its memory.low
130 * setting (memcg_low_skipped), and nothing is reclaimed as a
131 * result, then go back for one more cycle that reclaims the protected
132 * memory (memcg_low_reclaim) to avert OOM.
133 */
134 unsigned int memcg_low_reclaim:1;
135 unsigned int memcg_low_skipped:1;
136
137 /* Shared cgroup tree walk failed, rescan the whole tree */
138 unsigned int memcg_full_walk:1;
139
140 unsigned int hibernation_mode:1;
141
142 /* One of the zones is ready for compaction */
143 unsigned int compaction_ready:1;
144
145 /* There is easily reclaimable cold cache in the current node */
146 unsigned int cache_trim_mode:1;
147
148 /* The file folios on the current node are dangerously low */
149 unsigned int file_is_tiny:1;
150
151 /* Always discard instead of demoting to lower tier memory */
152 unsigned int no_demotion:1;
153
154 /* Allocation order */
155 s8 order;
156
157 /* Scan (total_size >> priority) pages at once */
158 s8 priority;
159
160 /* The highest zone to isolate folios for reclaim from */
161 s8 reclaim_idx;
162
163 /* This context's GFP mask */
164 gfp_t gfp_mask;
165
166 /* Incremented by the number of inactive pages that were scanned */
167 unsigned long nr_scanned;
168
169 /* Number of pages freed so far during a call to shrink_zones() */
170 unsigned long nr_reclaimed;
171
172 struct {
173 unsigned int dirty;
174 unsigned int congested;
175 unsigned int writeback;
176 unsigned int immediate;
177 unsigned int taken;
178 } nr;
179
180 /* for recording the reclaimed slab by now */
181 struct reclaim_state reclaim_state;
182};
183
184#ifdef ARCH_HAS_PREFETCHW
185#define prefetchw_prev_lru_folio(_folio, _base, _field) \
186 do { \
187 if ((_folio)->lru.prev != _base) { \
188 struct folio *prev; \
189 \
190 prev = lru_to_folio(&(_folio->lru)); \
191 prefetchw(&prev->_field); \
192 } \
193 } while (0)
194#else
195#define prefetchw_prev_lru_folio(_folio, _base, _field) do { } while (0)
196#endif
197
198/*
199 * From 0 .. MAX_SWAPPINESS. Higher means more swappy.
200 */
201int vm_swappiness = 60;
202
203static int sc_swappiness(struct scan_control *sc, struct mem_cgroup *memcg)
204{
205 if (sc->proactive && sc->proactive_swappiness)
206 return *sc->proactive_swappiness;
207 return mem_cgroup_swappiness(memcg);
208}
209
210#ifdef CONFIG_MEMCG
211
212/* Returns true for reclaim through cgroup limits or cgroup interfaces. */
213static bool cgroup_reclaim(struct scan_control *sc)
214{
215 return sc->target_mem_cgroup;
216}
217
218/*
219 * Returns true for reclaim on the root cgroup. This is true for direct
220 * allocator reclaim and reclaim through cgroup interfaces on the root cgroup.
221 */
222static bool root_reclaim(struct scan_control *sc)
223{
224 return !sc->target_mem_cgroup || mem_cgroup_is_root(sc->target_mem_cgroup);
225}
226
227/**
228 * writeback_throttling_sane - is the usual dirty throttling mechanism available?
229 * @sc: scan_control in question
230 *
231 * The normal page dirty throttling mechanism in balance_dirty_pages() is
232 * completely broken with the legacy memcg and direct stalling in
233 * shrink_folio_list() is used for throttling instead, which lacks all the
234 * niceties such as fairness, adaptive pausing, bandwidth proportional
235 * allocation and configurability.
236 *
237 * This function tests whether the vmscan currently in progress can assume
238 * that the normal dirty throttling mechanism is operational.
239 */
240static bool writeback_throttling_sane(struct scan_control *sc)
241{
242 if (!cgroup_reclaim(sc))
243 return true;
244#ifdef CONFIG_CGROUP_WRITEBACK
245 if (cgroup_subsys_on_dfl(memory_cgrp_subsys))
246 return true;
247#endif
248 return false;
249}
250#else
251static bool cgroup_reclaim(struct scan_control *sc)
252{
253 return false;
254}
255
256static bool root_reclaim(struct scan_control *sc)
257{
258 return true;
259}
260
261static bool writeback_throttling_sane(struct scan_control *sc)
262{
263 return true;
264}
265#endif
266
267static inline bool is_exec_file_folio(const struct folio *folio,
268 const vma_flags_t *vma_flags)
269{
270 return vma_flags_test(vma_flags, VMA_EXEC_BIT) && folio_is_file_lru(folio);
271}
272
273static void set_task_reclaim_state(struct task_struct *task,
274 struct reclaim_state *rs)
275{
276 /* Check for an overwrite */
277 WARN_ON_ONCE(rs && task->reclaim_state);
278
279 /* Check for the nulling of an already-nulled member */
280 WARN_ON_ONCE(!rs && !task->reclaim_state);
281
282 task->reclaim_state = rs;
283}
284
285/*
286 * flush_reclaim_state(): add pages reclaimed outside of LRU-based reclaim to
287 * scan_control->nr_reclaimed.
288 */
289static void flush_reclaim_state(struct scan_control *sc)
290{
291 /*
292 * Currently, reclaim_state->reclaimed includes three types of pages
293 * freed outside of vmscan:
294 * (1) Slab pages.
295 * (2) Clean file pages from pruned inodes (on highmem systems).
296 * (3) XFS freed buffer pages.
297 *
298 * For all of these cases, we cannot universally link the pages to a
299 * single memcg. For example, a memcg-aware shrinker can free one object
300 * charged to the target memcg, causing an entire page to be freed.
301 * If we count the entire page as reclaimed from the memcg, we end up
302 * overestimating the reclaimed amount (potentially under-reclaiming).
303 *
304 * Only count such pages for global reclaim to prevent under-reclaiming
305 * from the target memcg; preventing unnecessary retries during memcg
306 * charging and false positives from proactive reclaim.
307 *
308 * For uncommon cases where the freed pages were actually mostly
309 * charged to the target memcg, we end up underestimating the reclaimed
310 * amount. This should be fine. The freed pages will be uncharged
311 * anyway, even if they are not counted here properly, and we will be
312 * able to make forward progress in charging (which is usually in a
313 * retry loop).
314 *
315 * We can go one step further, and report the uncharged objcg pages in
316 * memcg reclaim, to make reporting more accurate and reduce
317 * underestimation, but it's probably not worth the complexity for now.
318 */
319 if (current->reclaim_state && root_reclaim(sc)) {
320 sc->nr_reclaimed += current->reclaim_state->reclaimed;
321 current->reclaim_state->reclaimed = 0;
322 }
323}
324
325static bool can_demote(int nid, struct scan_control *sc,
326 struct mem_cgroup *memcg)
327{
328 struct pglist_data *pgdat = NODE_DATA(nid);
329 nodemask_t allowed_mask;
330
331 if (!pgdat || !numa_demotion_enabled)
332 return false;
333 if (sc && sc->no_demotion)
334 return false;
335
336 node_get_allowed_targets(pgdat, &allowed_mask);
337 if (nodes_empty(allowed_mask))
338 return false;
339
340 /* Filter out nodes that are not in cgroup's mems_allowed. */
341 mem_cgroup_node_filter_allowed(memcg, &allowed_mask);
342 return !nodes_empty(allowed_mask);
343}
344
345static inline bool can_reclaim_anon_pages(struct mem_cgroup *memcg,
346 int nid,
347 struct scan_control *sc)
348{
349 if (memcg == NULL) {
350 /*
351 * For non-memcg reclaim, is there
352 * space in any swap device?
353 */
354 if (get_nr_swap_pages() > 0)
355 return true;
356 } else {
357 /* Is the memcg below its swap limit? */
358 if (mem_cgroup_get_nr_swap_pages(memcg) > 0)
359 return true;
360 }
361
362 /*
363 * The page can not be swapped.
364 *
365 * Can it be reclaimed from this node via demotion?
366 */
367 return can_demote(nid, sc, memcg);
368}
369
370/*
371 * This misses isolated folios which are not accounted for to save counters.
372 * As the data only determines if reclaim or compaction continues, it is
373 * not expected that isolated folios will be a dominating factor.
374 */
375unsigned long zone_reclaimable_pages(struct zone *zone)
376{
377 unsigned long nr;
378
379 nr = zone_page_state_snapshot(zone, NR_ZONE_INACTIVE_FILE) +
380 zone_page_state_snapshot(zone, NR_ZONE_ACTIVE_FILE);
381 if (can_reclaim_anon_pages(NULL, zone_to_nid(zone), NULL))
382 nr += zone_page_state_snapshot(zone, NR_ZONE_INACTIVE_ANON) +
383 zone_page_state_snapshot(zone, NR_ZONE_ACTIVE_ANON);
384
385 return nr;
386}
387
388/**
389 * lruvec_lru_size - Returns the number of pages on the given LRU list.
390 * @lruvec: lru vector
391 * @lru: lru to use
392 * @zone_idx: zones to consider (use MAX_NR_ZONES - 1 for the whole LRU list)
393 */
394unsigned long lruvec_lru_size(struct lruvec *lruvec, enum lru_list lru, int zone_idx)
395{
396 unsigned long size = 0;
397 int zid;
398 struct zone *zone;
399
400 for_each_managed_zone_pgdat(zone, lruvec_pgdat(lruvec), zid, zone_idx) {
401 if (!mem_cgroup_disabled())
402 size += mem_cgroup_get_zone_lru_size(lruvec, lru, zid);
403 else
404 size += zone_page_state(zone, NR_ZONE_LRU_BASE + lru);
405 }
406 return size;
407}
408
409static unsigned long drop_slab_node(int nid)
410{
411 unsigned long freed = 0;
412 struct mem_cgroup *memcg = NULL;
413
414 memcg = mem_cgroup_iter(NULL, NULL, NULL);
415 do {
416 freed += shrink_slab(GFP_KERNEL, nid, memcg, 0);
417 } while ((memcg = mem_cgroup_iter(NULL, memcg, NULL)) != NULL);
418
419 return freed;
420}
421
422void drop_slab(void)
423{
424 int nid;
425 int shift = 0;
426 unsigned long freed;
427
428 do {
429 freed = 0;
430 for_each_online_node(nid) {
431 if (fatal_signal_pending(current))
432 return;
433
434 freed += drop_slab_node(nid);
435 }
436 } while ((freed >> shift++) > 1);
437}
438
439#define CHECK_RECLAIMER_OFFSET(type) \
440 do { \
441 BUILD_BUG_ON(PGSTEAL_##type - PGSTEAL_KSWAPD != \
442 PGDEMOTE_##type - PGDEMOTE_KSWAPD); \
443 BUILD_BUG_ON(PGSTEAL_##type - PGSTEAL_KSWAPD != \
444 PGSCAN_##type - PGSCAN_KSWAPD); \
445 } while (0)
446
447static int reclaimer_offset(struct scan_control *sc)
448{
449 CHECK_RECLAIMER_OFFSET(DIRECT);
450 CHECK_RECLAIMER_OFFSET(KHUGEPAGED);
451 CHECK_RECLAIMER_OFFSET(PROACTIVE);
452
453 if (current_is_kswapd())
454 return 0;
455 if (current_is_khugepaged())
456 return PGSTEAL_KHUGEPAGED - PGSTEAL_KSWAPD;
457 if (sc->proactive)
458 return PGSTEAL_PROACTIVE - PGSTEAL_KSWAPD;
459 return PGSTEAL_DIRECT - PGSTEAL_KSWAPD;
460}
461
462/*
463 * We detected a synchronous write error writing a folio out. Probably
464 * -ENOSPC. We need to propagate that into the address_space for a subsequent
465 * fsync(), msync() or close().
466 *
467 * The tricky part is that after writepage we cannot touch the mapping: nothing
468 * prevents it from being freed up. But we have a ref on the folio and once
469 * that folio is locked, the mapping is pinned.
470 *
471 * We're allowed to run sleeping folio_lock() here because we know the caller has
472 * __GFP_FS.
473 */
474static void handle_write_error(struct address_space *mapping,
475 struct folio *folio, int error)
476{
477 folio_lock(folio);
478 if (folio_mapping(folio) == mapping)
479 mapping_set_error(mapping, error);
480 folio_unlock(folio);
481}
482
483static bool skip_throttle_noprogress(pg_data_t *pgdat)
484{
485 int reclaimable = 0, write_pending = 0;
486 int i;
487 struct zone *zone;
488 /*
489 * If kswapd is disabled, reschedule if necessary but do not
490 * throttle as the system is likely near OOM.
491 */
492 if (kswapd_test_hopeless(pgdat))
493 return true;
494
495 /*
496 * If there are a lot of dirty/writeback folios then do not
497 * throttle as throttling will occur when the folios cycle
498 * towards the end of the LRU if still under writeback.
499 */
500 for_each_managed_zone_pgdat(zone, pgdat, i, MAX_NR_ZONES - 1) {
501 reclaimable += zone_reclaimable_pages(zone);
502 write_pending += zone_page_state_snapshot(zone,
503 NR_ZONE_WRITE_PENDING);
504 }
505 if (2 * write_pending <= reclaimable)
506 return true;
507
508 return false;
509}
510
511void reclaim_throttle(pg_data_t *pgdat, enum vmscan_throttle_state reason)
512{
513 wait_queue_head_t *wqh = &pgdat->reclaim_wait[reason];
514 long timeout, ret;
515 DEFINE_WAIT(wait);
516
517 /*
518 * Do not throttle user workers, kthreads other than kswapd or
519 * workqueues. They may be required for reclaim to make
520 * forward progress (e.g. journalling workqueues or kthreads).
521 */
522 if (!current_is_kswapd() &&
523 current->flags & (PF_USER_WORKER|PF_KTHREAD)) {
524 cond_resched();
525 return;
526 }
527
528 /*
529 * These figures are pulled out of thin air.
530 * VMSCAN_THROTTLE_ISOLATED is a transient condition based on too many
531 * parallel reclaimers which is a short-lived event so the timeout is
532 * short. Failing to make progress or waiting on writeback are
533 * potentially long-lived events so use a longer timeout. This is shaky
534 * logic as a failure to make progress could be due to anything from
535 * writeback to a slow device to excessive referenced folios at the tail
536 * of the inactive LRU.
537 */
538 switch(reason) {
539 case VMSCAN_THROTTLE_WRITEBACK:
540 timeout = HZ/10;
541
542 if (atomic_inc_return(&pgdat->nr_writeback_throttled) == 1) {
543 WRITE_ONCE(pgdat->nr_reclaim_start,
544 node_page_state(pgdat, NR_THROTTLED_WRITTEN));
545 }
546
547 break;
548 case VMSCAN_THROTTLE_CONGESTED:
549 fallthrough;
550 case VMSCAN_THROTTLE_NOPROGRESS:
551 if (skip_throttle_noprogress(pgdat)) {
552 cond_resched();
553 return;
554 }
555
556 timeout = 1;
557
558 break;
559 case VMSCAN_THROTTLE_ISOLATED:
560 timeout = HZ/50;
561 break;
562 default:
563 WARN_ON_ONCE(1);
564 timeout = HZ;
565 break;
566 }
567
568 prepare_to_wait(wqh, &wait, TASK_UNINTERRUPTIBLE);
569 ret = schedule_timeout(timeout);
570 finish_wait(wqh, &wait);
571
572 if (reason == VMSCAN_THROTTLE_WRITEBACK)
573 atomic_dec(&pgdat->nr_writeback_throttled);
574
575 trace_mm_vmscan_throttled(pgdat->node_id, jiffies_to_usecs(timeout),
576 jiffies_to_usecs(timeout - ret),
577 reason);
578}
579
580/*
581 * Account for folios written if tasks are throttled waiting on dirty
582 * folios to clean. If enough folios have been cleaned since throttling
583 * started then wakeup the throttled tasks.
584 */
585void __acct_reclaim_writeback(pg_data_t *pgdat, struct folio *folio,
586 int nr_throttled)
587{
588 unsigned long nr_written;
589
590 node_stat_add_folio(folio, NR_THROTTLED_WRITTEN);
591
592 /*
593 * This is an inaccurate read as the per-cpu deltas may not
594 * be synchronised. However, given that the system is
595 * writeback throttled, it is not worth taking the penalty
596 * of getting an accurate count. At worst, the throttle
597 * timeout guarantees forward progress.
598 */
599 nr_written = node_page_state(pgdat, NR_THROTTLED_WRITTEN) -
600 READ_ONCE(pgdat->nr_reclaim_start);
601
602 if (nr_written > SWAP_CLUSTER_MAX * nr_throttled)
603 wake_up(&pgdat->reclaim_wait[VMSCAN_THROTTLE_WRITEBACK]);
604}
605
606/* possible outcome of pageout() */
607typedef enum {
608 /* failed to write folio out, folio is locked */
609 PAGE_KEEP,
610 /* move folio to the active list, folio is locked */
611 PAGE_ACTIVATE,
612 /* folio has been sent to the disk successfully, folio is unlocked */
613 PAGE_SUCCESS,
614 /* folio is clean and locked */
615 PAGE_CLEAN,
616} pageout_t;
617
618/*
619 * pageout is called by shrink_folio_list() for each dirty folio.
620 */
621static pageout_t pageout(struct swap_io_ctx *ctx, struct address_space *mapping,
622 struct folio *folio, struct list_head *folio_list)
623{
624 int res;
625
626 /*
627 * We no longer attempt to writeback filesystem folios here, other
628 * than tmpfs/shmem. That's taken care of in page-writeback.
629 * If we find a dirty filesystem folio at the end of the LRU list,
630 * typically that means the filesystem is saturating the storage
631 * with contiguous writes and telling it to write a folio here
632 * would only make the situation worse by injecting an element
633 * of random access.
634 *
635 * If the folio is swapcache, write it back even if that would
636 * block, for some throttling. This happens by accident, because
637 * swap_backing_dev_info is bust: it doesn't reflect the
638 * congestion state of the swapdevs. Easy to fix, if needed.
639 *
640 * A freeable shmem or swapcache folio is referenced only by the
641 * caller that isolated the folio and the page cache.
642 */
643 if (folio_ref_count(folio) != 1 + folio_nr_pages(folio) || !mapping)
644 return PAGE_KEEP;
645 if (!shmem_mapping(mapping) && !folio_test_anon(folio))
646 return PAGE_ACTIVATE;
647 if (!folio_clear_dirty_for_io(folio))
648 return PAGE_CLEAN;
649
650 folio_set_reclaim(folio);
651
652 /*
653 * The large shmem folio can be split if CONFIG_THP_SWAP is not enabled
654 * or we failed to allocate contiguous swap entries, in which case
655 * the split out folios get added back to folio_list.
656 */
657 if (shmem_mapping(mapping))
658 res = shmem_writeout(ctx, folio, folio_list);
659 else
660 res = swap_writeout(ctx, folio);
661
662 if (res < 0)
663 handle_write_error(mapping, folio, res);
664 if (res == AOP_WRITEPAGE_ACTIVATE) {
665 folio_clear_reclaim(folio);
666 return PAGE_ACTIVATE;
667 }
668
669 /* synchronous write? */
670 if (!folio_test_writeback(folio))
671 folio_clear_reclaim(folio);
672
673 trace_mm_vmscan_write_folio(folio);
674 lruvec_stat_mod_folio(folio, NR_VMSCAN_WRITE, folio_nr_pages(folio));
675 return PAGE_SUCCESS;
676}
677
678/*
679 * Same as remove_mapping, but if the folio is removed from the mapping, it
680 * gets returned with a refcount of 0.
681 */
682static int __remove_mapping(struct address_space *mapping, struct folio *folio,
683 bool reclaimed, struct mem_cgroup *target_memcg)
684{
685 int refcount;
686 void *shadow = NULL;
687 struct swap_cluster_info *ci;
688
689 BUG_ON(!folio_test_locked(folio));
690 BUG_ON(mapping != folio_mapping(folio));
691
692 if (folio_test_swapcache(folio)) {
693 ci = swap_cluster_get_and_lock_irq(folio);
694 } else {
695 spin_lock(&mapping->host->i_lock);
696 xa_lock_irq(&mapping->i_pages);
697 }
698
699 /*
700 * The non racy check for a busy folio.
701 *
702 * Must be careful with the order of the tests. When someone has
703 * a ref to the folio, it may be possible that they dirty it then
704 * drop the reference. So if the dirty flag is tested before the
705 * refcount here, then the following race may occur:
706 *
707 * get_user_pages(&page);
708 * [user mapping goes away]
709 * write_to(page);
710 * !folio_test_dirty(folio) [good]
711 * folio_set_dirty(folio);
712 * folio_put(folio);
713 * !refcount(folio) [good, discard it]
714 *
715 * [oops, our write_to data is lost]
716 *
717 * Reversing the order of the tests ensures such a situation cannot
718 * escape unnoticed. The smp_rmb is needed to ensure the folio->flags
719 * load is not satisfied before that of folio->_refcount.
720 *
721 * Note that if the dirty flag is always set via folio_mark_dirty,
722 * and thus under the i_pages lock, then this ordering is not required.
723 */
724 refcount = 1 + folio_nr_pages(folio);
725 if (!folio_ref_freeze(folio, refcount))
726 goto cannot_free;
727 /* note: atomic_cmpxchg in folio_ref_freeze provides the smp_rmb */
728 if (unlikely(folio_test_dirty(folio))) {
729 folio_ref_unfreeze(folio, refcount);
730 goto cannot_free;
731 }
732
733 if (folio_test_swapcache(folio)) {
734 swp_entry_t swap = folio->swap;
735
736 if (reclaimed && !mapping_exiting(mapping))
737 shadow = workingset_eviction(folio, target_memcg);
738 __memcg1_swapout(folio, ci);
739 __swap_cache_del_folio(ci, folio, swap, shadow);
740 swap_cluster_unlock_irq(ci);
741 } else {
742 void (*free_folio)(struct folio *);
743
744 free_folio = mapping->a_ops->free_folio;
745 /*
746 * Remember a shadow entry for reclaimed file cache in
747 * order to detect refaults, thus thrashing, later on.
748 *
749 * But don't store shadows in an address space that is
750 * already exiting. This is not just an optimization,
751 * inode reclaim needs to empty out the radix tree or
752 * the nodes are lost. Don't plant shadows behind its
753 * back.
754 *
755 * We also don't store shadows for DAX mappings because the
756 * only page cache folios found in these are zero pages
757 * covering holes, and because we don't want to mix DAX
758 * exceptional entries and shadow exceptional entries in the
759 * same address_space.
760 */
761 if (reclaimed && folio_is_file_lru(folio) &&
762 !mapping_exiting(mapping) && !dax_mapping(mapping))
763 shadow = workingset_eviction(folio, target_memcg);
764 __filemap_remove_folio(folio, shadow);
765 xa_unlock_irq(&mapping->i_pages);
766 if (mapping_shrinkable(mapping))
767 inode_lru_list_add(mapping->host);
768 spin_unlock(&mapping->host->i_lock);
769
770 if (free_folio)
771 free_folio(folio);
772 }
773
774 return 1;
775
776cannot_free:
777 if (folio_test_swapcache(folio)) {
778 swap_cluster_unlock_irq(ci);
779 } else {
780 xa_unlock_irq(&mapping->i_pages);
781 spin_unlock(&mapping->host->i_lock);
782 }
783 return 0;
784}
785
786/**
787 * remove_mapping() - Attempt to remove a folio from its mapping.
788 * @mapping: The address space.
789 * @folio: The folio to remove.
790 *
791 * If the folio is dirty, under writeback or if someone else has a ref
792 * on it, removal will fail.
793 * Return: The number of pages removed from the mapping. 0 if the folio
794 * could not be removed.
795 * Context: The caller should have a single refcount on the folio and
796 * hold its lock.
797 */
798long remove_mapping(struct address_space *mapping, struct folio *folio)
799{
800 if (__remove_mapping(mapping, folio, false, NULL)) {
801 /*
802 * Unfreezing the refcount with 1 effectively
803 * drops the pagecache ref for us without requiring another
804 * atomic operation.
805 */
806 folio_ref_unfreeze(folio, 1);
807 return folio_nr_pages(folio);
808 }
809 return 0;
810}
811
812/**
813 * folio_putback_lru - Put previously isolated folio onto appropriate LRU list.
814 * @folio: Folio to be returned to an LRU list.
815 *
816 * Add previously isolated @folio to appropriate LRU list.
817 * The folio may still be unevictable for other reasons.
818 *
819 * Context: lru_lock must not be held, interrupts must be enabled.
820 */
821void folio_putback_lru(struct folio *folio)
822{
823 folio_add_lru(folio);
824 folio_put(folio); /* drop ref from isolate */
825}
826
827enum folio_references {
828 FOLIOREF_RECLAIM,
829 FOLIOREF_KEEP,
830 FOLIOREF_ACTIVATE,
831};
832
833#ifdef CONFIG_LRU_GEN
834/*
835 * Only used on a mapped folio in the eviction (rmap walk) path, where promotion
836 * needs to be done by taking the folio off the LRU list and then adding it back
837 * with PG_active set. In contrast, the aging (page table walk) path uses
838 * folio_update_gen().
839 */
840static bool lru_gen_set_refs(struct folio *folio, const vma_flags_t *vma_flags)
841{
842 /* see the comment on LRU_REFS_FLAGS */
843 if (!folio_test_referenced(folio) && !folio_test_workingset(folio)) {
844 /* Activate file-backed executable folios after first usage. */
845 if (is_exec_file_folio(folio, vma_flags)) {
846 set_mask_bits(&folio->flags.f, LRU_REFS_FLAGS, BIT(PG_workingset));
847 return true;
848 }
849
850 set_mask_bits(&folio->flags.f, LRU_REFS_MASK, BIT(PG_referenced));
851 return false;
852 }
853
854 /* Promote on second access */
855 if (folio_lru_refs(folio) > 1)
856 set_mask_bits(&folio->flags.f, LRU_REFS_FLAGS, BIT(PG_workingset));
857 else
858 folio_mark_accessed(folio);
859 return true;
860}
861#else
862static bool lru_gen_set_refs(struct folio *folio, const vma_flags_t *vma_flags)
863{
864 return false;
865}
866#endif /* CONFIG_LRU_GEN */
867
868static enum folio_references folio_check_references(struct folio *folio,
869 struct scan_control *sc)
870{
871 int referenced_ptes, referenced_folio;
872 vma_flags_t vma_flags;
873
874 referenced_ptes = folio_referenced(folio, 1, sc->target_mem_cgroup,
875 &vma_flags);
876
877 /*
878 * The supposedly reclaimable folio was found to be in a VM_LOCKED vma.
879 * Let the folio, now marked Mlocked, be moved to the unevictable list.
880 */
881 if (vma_flags_test(&vma_flags, VMA_LOCKED_BIT))
882 return FOLIOREF_ACTIVATE;
883
884 /*
885 * There are two cases to consider.
886 * 1) Rmap lock contention: rotate.
887 * 2) Skip the non-shared swapbacked folio mapped solely by
888 * the exiting or OOM-reaped process.
889 */
890 if (referenced_ptes == -1)
891 return FOLIOREF_KEEP;
892
893 if (lru_gen_enabled() && !lru_gen_switching()) {
894 if (!referenced_ptes)
895 return FOLIOREF_RECLAIM;
896
897 return lru_gen_set_refs(folio, &vma_flags) ? FOLIOREF_ACTIVATE : FOLIOREF_KEEP;
898 }
899
900 referenced_folio = folio_test_clear_referenced(folio);
901
902 if (referenced_ptes) {
903 /*
904 * All mapped folios start out with page table
905 * references from the instantiating fault, so we need
906 * to look twice if a mapped file/anon folio is used more
907 * than once.
908 *
909 * Mark it and spare it for another trip around the
910 * inactive list. Another page table reference will
911 * lead to its activation.
912 *
913 * Note: the mark is set for activated folios as well
914 * so that recently deactivated but used folios are
915 * quickly recovered.
916 */
917 folio_set_referenced(folio);
918
919 if (referenced_folio || referenced_ptes > 1)
920 return FOLIOREF_ACTIVATE;
921
922 /*
923 * Activate file-backed executable folios after first usage.
924 */
925 if (is_exec_file_folio(folio, &vma_flags))
926 return FOLIOREF_ACTIVATE;
927
928 return FOLIOREF_KEEP;
929 }
930
931 return FOLIOREF_RECLAIM;
932}
933
934/* Check if a folio is dirty or under writeback */
935static void folio_check_dirty_writeback(struct folio *folio,
936 bool *dirty, bool *writeback)
937{
938 struct address_space *mapping;
939
940 /*
941 * Anonymous folios are not handled by flushers and must be written
942 * from reclaim context. Do not stall reclaim based on them.
943 * MADV_FREE anonymous folios are put into inactive file list too.
944 * They could be mistakenly treated as file lru. So further anon
945 * test is needed.
946 */
947 if (!folio_is_file_lru(folio) || folio_test_lazyfree(folio)) {
948 *dirty = false;
949 *writeback = false;
950 return;
951 }
952
953 /* By default assume that the folio flags are accurate */
954 *dirty = folio_test_dirty(folio);
955 *writeback = folio_test_writeback(folio);
956
957 /* Verify dirty/writeback state if the filesystem supports it */
958 if (!folio_test_private(folio))
959 return;
960
961 mapping = folio_mapping(folio);
962 if (mapping && mapping->a_ops->is_dirty_writeback)
963 mapping->a_ops->is_dirty_writeback(folio, dirty, writeback);
964}
965
966static struct folio *alloc_demote_folio(struct folio *src,
967 unsigned long private)
968{
969 struct migration_target_control *mtc, target_nid_mtc;
970 struct folio *dst;
971
972 mtc = (struct migration_target_control *)private;
973
974 /*
975 * make sure we allocate from the target node first also trying to
976 * demote or reclaim pages from the target node via kswapd if we are
977 * low on free memory on target node. If we don't do this and if
978 * we have free memory on the slower(lower) memtier, we would start
979 * allocating pages from slower(lower) memory tiers without even forcing
980 * a demotion of cold pages from the target memtier. This can result
981 * in the kernel placing hot pages in slower(lower) memory tiers.
982 */
983 target_nid_mtc = *mtc;
984 target_nid_mtc.nmask = NULL;
985 target_nid_mtc.gfp_mask |= __GFP_THISNODE;
986 dst = alloc_migration_target(src, (unsigned long)&target_nid_mtc);
987 if (dst)
988 return dst;
989
990 return alloc_migration_target(src, (unsigned long)mtc);
991}
992
993/*
994 * Take folios on @demote_folios and attempt to demote them to another node.
995 * Folios which are not demoted are left on @demote_folios.
996 */
997static unsigned int demote_folio_list(struct list_head *demote_folios,
998 struct pglist_data *pgdat,
999 struct mem_cgroup *memcg)
1000{
1001 int target_nid;
1002 unsigned int nr_succeeded;
1003 nodemask_t allowed_mask;
1004
1005 struct migration_target_control mtc = {
1006 /*
1007 * Allocate from 'node', or fail quickly and quietly.
1008 * When this happens, 'page' will likely just be discarded
1009 * instead of migrated.
1010 */
1011 .gfp_mask = (GFP_HIGHUSER_MOVABLE & ~__GFP_RECLAIM) |
1012 __GFP_NOMEMALLOC | GFP_NOWAIT,
1013 .nmask = &allowed_mask,
1014 .reason = MR_DEMOTION,
1015 };
1016
1017 if (list_empty(demote_folios))
1018 return 0;
1019
1020 node_get_allowed_targets(pgdat, &allowed_mask);
1021 mem_cgroup_node_filter_allowed(memcg, &allowed_mask);
1022 if (nodes_empty(allowed_mask))
1023 return 0;
1024
1025 target_nid = next_demotion_node(pgdat->node_id, &allowed_mask);
1026 if (target_nid == NUMA_NO_NODE)
1027 /* No lower-tier nodes or nodes were hot-unplugged. */
1028 return 0;
1029
1030 mtc.nid = target_nid;
1031
1032 /* Demotion ignores all cpuset and mempolicy settings */
1033 migrate_pages(demote_folios, alloc_demote_folio, NULL,
1034 (unsigned long)&mtc, MIGRATE_ASYNC, MR_DEMOTION,
1035 &nr_succeeded);
1036
1037 return nr_succeeded;
1038}
1039
1040static bool may_enter_fs(struct folio *folio, gfp_t gfp_mask)
1041{
1042 if (gfp_mask & __GFP_FS)
1043 return true;
1044 /*
1045 * We can "enter_fs" for swap-cache with only __GFP_IO unless backed by
1046 * a swapfile that requires GFP_NOFS I/O.
1047 */
1048 if (folio_test_swapcache(folio) && (gfp_mask & __GFP_IO) &&
1049 !(__swap_entry_to_info(folio->swap)->ops->flags &
1050 SWAP_OPS_F_REQUIRE_NOFS))
1051 return true;
1052 return false;
1053}
1054
1055/*
1056 * shrink_folio_list() returns the number of reclaimed pages
1057 */
1058static unsigned int shrink_folio_list(struct list_head *folio_list,
1059 struct pglist_data *pgdat, struct scan_control *sc,
1060 struct reclaim_stat *stat, bool ignore_references,
1061 struct mem_cgroup *memcg)
1062{
1063 struct folio_batch free_folios;
1064 LIST_HEAD(ret_folios);
1065 LIST_HEAD(demote_folios);
1066 unsigned int nr_reclaimed = 0, nr_demoted = 0;
1067 unsigned int pgactivate = 0;
1068 bool do_demote_pass;
1069 struct swap_io_ctx ctx = {};
1070
1071 folio_batch_init(&free_folios);
1072 memset(stat, 0, sizeof(*stat));
1073 cond_resched();
1074 do_demote_pass = can_demote(pgdat->node_id, sc, memcg);
1075
1076retry:
1077 while (!list_empty(folio_list)) {
1078 struct address_space *mapping;
1079 struct folio *folio;
1080 enum folio_references references = FOLIOREF_RECLAIM;
1081 bool dirty, writeback;
1082 unsigned int nr_pages;
1083
1084 cond_resched();
1085
1086 folio = lru_to_folio(folio_list);
1087 list_del(&folio->lru);
1088
1089 if (!folio_trylock(folio))
1090 goto keep;
1091
1092 if (folio_contain_hwpoisoned_page(folio)) {
1093 /*
1094 * unmap_poisoned_folio() can't handle large
1095 * folio, just skip it. memory_failure() will
1096 * handle it if the UCE is triggered again.
1097 */
1098 if (folio_test_large(folio))
1099 goto keep_locked;
1100
1101 unmap_poisoned_folio(folio, folio_pfn(folio), false);
1102 folio_unlock(folio);
1103 folio_put(folio);
1104 continue;
1105 }
1106
1107 VM_BUG_ON_FOLIO(folio_test_active(folio), folio);
1108
1109 nr_pages = folio_nr_pages(folio);
1110
1111 /* Account the number of base pages */
1112 sc->nr_scanned += nr_pages;
1113
1114 if (unlikely(!folio_evictable(folio)))
1115 goto activate_locked;
1116
1117 if (!sc->may_unmap && folio_mapped(folio))
1118 goto keep_locked;
1119
1120 /*
1121 * The number of dirty pages determines if a node is marked
1122 * reclaim_congested. kswapd will stall and start writing
1123 * folios if the tail of the LRU is all dirty unqueued folios.
1124 */
1125 folio_check_dirty_writeback(folio, &dirty, &writeback);
1126 if (dirty || writeback)
1127 stat->nr_dirty += nr_pages;
1128
1129 if (dirty && !writeback)
1130 stat->nr_unqueued_dirty += nr_pages;
1131
1132 /*
1133 * Treat this folio as congested if folios are cycling
1134 * through the LRU so quickly that the folios marked
1135 * for immediate reclaim are making it to the end of
1136 * the LRU a second time.
1137 */
1138 if (writeback && folio_test_reclaim(folio))
1139 stat->nr_congested += nr_pages;
1140
1141 /*
1142 * If a folio at the tail of the LRU is under writeback, there
1143 * are three cases to consider.
1144 *
1145 * 1) If reclaim is encountering an excessive number
1146 * of folios under writeback and this folio has both
1147 * the writeback and reclaim flags set, then it
1148 * indicates that folios are being queued for I/O but
1149 * are being recycled through the LRU before the I/O
1150 * can complete. Waiting on the folio itself risks an
1151 * indefinite stall if it is impossible to writeback
1152 * the folio due to I/O error or disconnected storage
1153 * so instead note that the LRU is being scanned too
1154 * quickly and the caller can stall after the folio
1155 * list has been processed.
1156 *
1157 * 2) Global or new memcg reclaim encounters a folio that is
1158 * not marked for immediate reclaim, or the caller does not
1159 * have __GFP_FS (or __GFP_IO if it's simply going to swap,
1160 * not to fs), or the folio belongs to a mapping where
1161 * waiting on writeback during reclaim may lead to a deadlock.
1162 * In this case mark the folio for immediate reclaim and
1163 * continue scanning.
1164 *
1165 * Require may_enter_fs() because we would wait on fs, which
1166 * may not have submitted I/O yet. And the loop driver might
1167 * enter reclaim, and deadlock if it waits on a folio for
1168 * which it is needed to do the write (loop masks off
1169 * __GFP_IO|__GFP_FS for this reason); but more thought
1170 * would probably show more reasons.
1171 *
1172 * 3) Legacy memcg encounters a folio that already has the
1173 * reclaim flag set. memcg does not have any dirty folio
1174 * throttling so we could easily OOM just because too many
1175 * folios are in writeback and there is nothing else to
1176 * reclaim. Wait for the writeback to complete.
1177 *
1178 * In cases 1) and 2) we activate the folios to get them out of
1179 * the way while we continue scanning for clean folios on the
1180 * inactive list and refilling from the active list. The
1181 * observation here is that waiting for disk writes is more
1182 * expensive than potentially causing reloads down the line.
1183 * Since they're marked for immediate reclaim, they won't put
1184 * memory pressure on the cache working set any longer than it
1185 * takes to write them to disk.
1186 */
1187 if (folio_test_writeback(folio)) {
1188 mapping = folio_mapping(folio);
1189
1190 /* Case 1 above */
1191 if (current_is_kswapd() &&
1192 folio_test_reclaim(folio) &&
1193 test_bit(PGDAT_WRITEBACK, &pgdat->flags)) {
1194 stat->nr_immediate += nr_pages;
1195 goto activate_locked;
1196
1197 /* Case 2 above */
1198 } else if (writeback_throttling_sane(sc) ||
1199 !folio_test_reclaim(folio) ||
1200 !may_enter_fs(folio, sc->gfp_mask) ||
1201 (mapping &&
1202 mapping_writeback_may_deadlock_on_reclaim(mapping))) {
1203 /*
1204 * This is slightly racy -
1205 * folio_end_writeback() might have
1206 * just cleared the reclaim flag, then
1207 * setting the reclaim flag here ends up
1208 * interpreted as the readahead flag - but
1209 * that does not matter enough to care.
1210 * What we do want is for this folio to
1211 * have the reclaim flag set next time
1212 * memcg reclaim reaches the tests above,
1213 * so it will then wait for writeback to
1214 * avoid OOM; and it's also appropriate
1215 * in global reclaim.
1216 */
1217 folio_set_reclaim(folio);
1218 stat->nr_writeback += nr_pages;
1219 goto activate_locked;
1220
1221 /* Case 3 above */
1222 } else {
1223 folio_unlock(folio);
1224 folio_wait_writeback(folio);
1225 /* then go back and try same folio again */
1226 list_add_tail(&folio->lru, folio_list);
1227 continue;
1228 }
1229 }
1230
1231 if (!ignore_references)
1232 references = folio_check_references(folio, sc);
1233
1234 switch (references) {
1235 case FOLIOREF_ACTIVATE:
1236 goto activate_locked;
1237 case FOLIOREF_KEEP:
1238 stat->nr_ref_keep += nr_pages;
1239 goto keep_locked;
1240 case FOLIOREF_RECLAIM:
1241 ; /* try to reclaim the folio below */
1242 }
1243
1244 /*
1245 * Before reclaiming the folio, try to relocate
1246 * its contents to another node.
1247 */
1248 if (do_demote_pass &&
1249 (thp_migration_supported() || !folio_test_large(folio))) {
1250 list_add(&folio->lru, &demote_folios);
1251 folio_unlock(folio);
1252 continue;
1253 }
1254
1255 /*
1256 * Anonymous process memory has backing store?
1257 * Try to allocate it some swap space here.
1258 * Lazyfree folio could be freed directly
1259 */
1260 if (folio_test_anon(folio) && folio_test_swapbacked(folio) &&
1261 !folio_test_swapcache(folio)) {
1262 if (!(sc->gfp_mask & __GFP_IO))
1263 goto keep_locked;
1264 if (folio_maybe_dma_pinned(folio))
1265 goto keep_locked;
1266 if (folio_test_large(folio)) {
1267 /* cannot split folio, skip it */
1268 if (folio_expected_ref_count(folio) !=
1269 folio_ref_count(folio) - 1)
1270 goto activate_locked;
1271 /*
1272 * Split partially mapped folios right away.
1273 * We can free the unmapped pages without IO.
1274 */
1275 if (data_race(!list_empty(&folio->_deferred_list) &&
1276 folio_test_partially_mapped(folio)) &&
1277 split_folio_to_list(folio, folio_list))
1278 goto activate_locked;
1279 }
1280 if (folio_alloc_swap(folio)) {
1281 int __maybe_unused order = folio_order(folio);
1282
1283 if (!folio_test_large(folio))
1284 goto activate_locked_split;
1285 /* Fallback to swap normal pages */
1286 if (split_folio_to_list(folio, folio_list))
1287 goto activate_locked;
1288#ifdef CONFIG_TRANSPARENT_HUGEPAGE
1289 if (nr_pages >= HPAGE_PMD_NR) {
1290 count_memcg_folio_events(folio,
1291 THP_SWPOUT_FALLBACK, 1);
1292 count_vm_event(THP_SWPOUT_FALLBACK);
1293 }
1294#endif
1295 count_mthp_stat(order, MTHP_STAT_SWPOUT_FALLBACK);
1296 if (folio_alloc_swap(folio))
1297 goto activate_locked_split;
1298 }
1299 /*
1300 * Normally the folio will be dirtied in unmap because
1301 * its pte should be dirty. A special case is MADV_FREE
1302 * page. The page's pte could have dirty bit cleared but
1303 * the folio's SwapBacked flag is still set because
1304 * clearing the dirty bit and SwapBacked flag has no
1305 * lock protected. For such folio, unmap will not set
1306 * dirty bit for it, so folio reclaim will not write the
1307 * folio out. This can cause data corruption when the
1308 * folio is swapped in later. Always setting the dirty
1309 * flag for the folio solves the problem.
1310 */
1311 folio_mark_dirty(folio);
1312 }
1313
1314 /*
1315 * If the folio was split above, the tail pages will make
1316 * their own pass through this function and be accounted
1317 * then.
1318 */
1319 if ((nr_pages > 1) && !folio_test_large(folio)) {
1320 sc->nr_scanned -= (nr_pages - 1);
1321 nr_pages = 1;
1322 }
1323
1324 /*
1325 * The folio is mapped into the page tables of one or more
1326 * processes. Try to unmap it here.
1327 */
1328 if (folio_mapped(folio)) {
1329 enum ttu_flags flags = TTU_BATCH_FLUSH;
1330 bool was_swapbacked = folio_test_swapbacked(folio);
1331
1332 if (folio_test_pmd_mappable(folio))
1333 flags |= TTU_SPLIT_HUGE_PMD;
1334 /*
1335 * Without TTU_SYNC, try_to_unmap will only begin to
1336 * hold PTL from the first present PTE within a large
1337 * folio. Some initial PTEs might be skipped due to
1338 * races with parallel PTE writes in which PTEs can be
1339 * cleared temporarily before being written new present
1340 * values. This will lead to a large folio is still
1341 * mapped while some subpages have been partially
1342 * unmapped after try_to_unmap; TTU_SYNC helps
1343 * try_to_unmap acquire PTL from the first PTE,
1344 * eliminating the influence of temporary PTE values.
1345 */
1346 if (folio_test_large(folio))
1347 flags |= TTU_SYNC;
1348
1349 try_to_unmap(folio, flags);
1350 if (folio_mapped(folio)) {
1351 stat->nr_unmap_fail += nr_pages;
1352 if (!was_swapbacked &&
1353 folio_test_swapbacked(folio))
1354 stat->nr_lazyfree_fail += nr_pages;
1355 goto activate_locked;
1356 }
1357 }
1358
1359 /*
1360 * Folio is unmapped now so it cannot be newly pinned anymore.
1361 * No point in trying to reclaim folio if it is pinned.
1362 * Furthermore we don't want to reclaim underlying fs metadata
1363 * if the folio is pinned and thus potentially modified by the
1364 * pinning process as that may upset the filesystem.
1365 */
1366 if (folio_maybe_dma_pinned(folio))
1367 goto activate_locked;
1368
1369 mapping = folio_mapping(folio);
1370 if (folio_test_dirty(folio)) {
1371 if (folio_is_file_lru(folio)) {
1372 /*
1373 * Immediately reclaim when written back.
1374 * Similar in principle to folio_deactivate()
1375 * except we already have the folio isolated
1376 * and know it's dirty
1377 */
1378 node_stat_mod_folio(folio, NR_VMSCAN_IMMEDIATE,
1379 nr_pages);
1380 if (!folio_test_reclaim(folio))
1381 folio_set_reclaim(folio);
1382
1383 goto activate_locked;
1384 }
1385
1386 if (!may_enter_fs(folio, sc->gfp_mask))
1387 goto keep_locked;
1388 if (!sc->may_writepage)
1389 goto keep_locked;
1390
1391 /*
1392 * Folio is dirty. Flush the TLB if a writable entry
1393 * potentially exists to avoid CPU writes after I/O
1394 * starts and then write it out here.
1395 */
1396 try_to_unmap_flush_dirty();
1397 switch (pageout(&ctx, mapping, folio, folio_list)) {
1398 case PAGE_KEEP:
1399 goto keep_locked;
1400 case PAGE_ACTIVATE:
1401 /*
1402 * If shmem folio is split when writeback to swap,
1403 * the tail pages will make their own pass through
1404 * this function and be accounted then.
1405 */
1406 if (nr_pages > 1 && !folio_test_large(folio)) {
1407 sc->nr_scanned -= (nr_pages - 1);
1408 nr_pages = 1;
1409 }
1410 goto activate_locked;
1411 case PAGE_SUCCESS:
1412 if (nr_pages > 1 && !folio_test_large(folio)) {
1413 sc->nr_scanned -= (nr_pages - 1);
1414 nr_pages = 1;
1415 }
1416 if (folio_test_writeback(folio))
1417 goto keep;
1418 if (folio_test_dirty(folio))
1419 goto keep;
1420
1421 /*
1422 * A synchronous write - probably a ramdisk. Go
1423 * ahead and try to reclaim the folio.
1424 */
1425 if (!folio_trylock(folio))
1426 goto keep;
1427 if (folio_test_dirty(folio) ||
1428 folio_test_writeback(folio))
1429 goto keep_locked;
1430 mapping = folio_mapping(folio);
1431 fallthrough;
1432 case PAGE_CLEAN:
1433 ; /* try to free the folio below */
1434 }
1435 }
1436
1437 /*
1438 * If the folio has buffers, try to free the buffer
1439 * mappings associated with this folio. If we succeed
1440 * we try to free the folio as well.
1441 *
1442 * We do this even if the folio is dirty.
1443 * filemap_release_folio() does not perform I/O, but it
1444 * is possible for a folio to have the dirty flag set,
1445 * but it is actually clean (all its buffers are clean).
1446 * This happens if the buffers were written out directly,
1447 * with bh_submit(). ext3 will do this, as well as
1448 * the blockdev mapping. filemap_release_folio() will
1449 * discover that cleanness and will drop the buffers
1450 * and mark the folio clean - it can be freed.
1451 *
1452 * Rarely, folios can have buffers and no ->mapping.
1453 * These are the folios which were not successfully
1454 * invalidated in truncate_cleanup_folio(). We try to
1455 * drop those buffers here and if that worked, and the
1456 * folio is no longer mapped into process address space
1457 * (refcount == 1) it can be freed. Otherwise, leave
1458 * the folio on the LRU so it is swappable.
1459 */
1460 if (folio_needs_release(folio)) {
1461 if (!filemap_release_folio(folio, sc->gfp_mask))
1462 goto activate_locked;
1463 if (!mapping && folio_ref_count(folio) == 1) {
1464 folio_unlock(folio);
1465 if (folio_put_testzero(folio))
1466 goto free_it;
1467 else {
1468 /*
1469 * rare race with speculative reference.
1470 * the speculative reference will free
1471 * this folio shortly, so we may
1472 * increment nr_reclaimed here (and
1473 * leave it off the LRU).
1474 */
1475 nr_reclaimed += nr_pages;
1476 continue;
1477 }
1478 }
1479 }
1480
1481 if (folio_test_lazyfree(folio)) {
1482 /* follow __remove_mapping for reference */
1483 if (!folio_ref_freeze(folio, 1))
1484 goto keep_locked;
1485 /*
1486 * The folio has only one reference left, which is
1487 * from the isolation. After the caller puts the
1488 * folio back on the lru and drops the reference, the
1489 * folio will be freed anyway. It doesn't matter
1490 * which lru it goes on. So we don't bother checking
1491 * the dirty flag here.
1492 */
1493 count_vm_events(PGLAZYFREED, nr_pages);
1494 count_memcg_folio_events(folio, PGLAZYFREED, nr_pages);
1495 } else if (!mapping || !__remove_mapping(mapping, folio, true,
1496 sc->target_mem_cgroup))
1497 goto keep_locked;
1498
1499 folio_unlock(folio);
1500free_it:
1501 /*
1502 * Folio may get swapped out as a whole, need to account
1503 * all pages in it.
1504 */
1505 nr_reclaimed += nr_pages;
1506
1507 folio_unqueue_deferred_split(folio);
1508 if (folio_batch_add(&free_folios, folio) == 0) {
1509 mem_cgroup_uncharge_folios(&free_folios);
1510 try_to_unmap_flush();
1511 free_unref_folios(&free_folios);
1512 }
1513 continue;
1514
1515activate_locked_split:
1516 /*
1517 * The tail pages that are failed to add into swap cache
1518 * reach here. Fixup nr_scanned and nr_pages.
1519 */
1520 if (nr_pages > 1) {
1521 sc->nr_scanned -= (nr_pages - 1);
1522 nr_pages = 1;
1523 }
1524activate_locked:
1525 /* Not a candidate for swapping, so reclaim swap space. */
1526 if (folio_test_swapcache(folio) &&
1527 (mem_cgroup_swap_full(folio) || folio_test_mlocked(folio)))
1528 folio_free_swap(folio);
1529 VM_BUG_ON_FOLIO(folio_test_active(folio), folio);
1530 if (!folio_test_mlocked(folio)) {
1531 int type = folio_is_file_lru(folio);
1532 folio_set_active(folio);
1533 stat->nr_activate[type] += nr_pages;
1534 count_memcg_folio_events(folio, PGACTIVATE, nr_pages);
1535 }
1536keep_locked:
1537 folio_unlock(folio);
1538keep:
1539 list_add(&folio->lru, &ret_folios);
1540 VM_BUG_ON_FOLIO(folio_test_lru(folio) ||
1541 folio_test_unevictable(folio), folio);
1542 }
1543 /* 'folio_list' is always empty here */
1544
1545 /* Migrate folios selected for demotion */
1546 nr_demoted = demote_folio_list(&demote_folios, pgdat, memcg);
1547 nr_reclaimed += nr_demoted;
1548 stat->nr_demoted += nr_demoted;
1549 /* Folios that could not be demoted are still in @demote_folios */
1550 if (!list_empty(&demote_folios)) {
1551 /* Folios which weren't demoted go back on @folio_list */
1552 list_splice_init(&demote_folios, folio_list);
1553
1554 /*
1555 * goto retry to reclaim the undemoted folios in folio_list if
1556 * desired.
1557 *
1558 * Reclaiming directly from top tier nodes is not often desired
1559 * due to it breaking the LRU ordering: in general memory
1560 * should be reclaimed from lower tier nodes and demoted from
1561 * top tier nodes.
1562 *
1563 * However, disabling reclaim from top tier nodes entirely
1564 * would cause ooms in edge scenarios where lower tier memory
1565 * is unreclaimable for whatever reason, eg memory being
1566 * mlocked or too hot to reclaim. We can disable reclaim
1567 * from top tier nodes in proactive reclaim though as that is
1568 * not real memory pressure.
1569 */
1570 if (!sc->proactive) {
1571 do_demote_pass = false;
1572 goto retry;
1573 }
1574 }
1575
1576 pgactivate = stat->nr_activate[0] + stat->nr_activate[1];
1577
1578 mem_cgroup_uncharge_folios(&free_folios);
1579 try_to_unmap_flush();
1580 free_unref_folios(&free_folios);
1581
1582 list_splice(&ret_folios, folio_list);
1583 count_vm_events(PGACTIVATE, pgactivate);
1584
1585 swap_write_submit(&ctx);
1586 return nr_reclaimed;
1587}
1588
1589unsigned int reclaim_clean_pages_from_list(struct zone *zone,
1590 struct list_head *folio_list)
1591{
1592 struct scan_control sc = {
1593 .gfp_mask = GFP_KERNEL,
1594 .may_unmap = 1,
1595 };
1596 struct reclaim_stat stat;
1597 unsigned int nr_reclaimed;
1598 struct folio *folio, *next;
1599 LIST_HEAD(clean_folios);
1600 unsigned int noreclaim_flag;
1601
1602 list_for_each_entry_safe(folio, next, folio_list, lru) {
1603 /* TODO: these pages should not even appear in this list. */
1604 if (page_has_movable_ops(&folio->page))
1605 continue;
1606 if (!folio_test_hugetlb(folio) && folio_is_file_lru(folio) &&
1607 !folio_test_dirty(folio) && !folio_test_unevictable(folio)) {
1608 folio_clear_active(folio);
1609 list_move(&folio->lru, &clean_folios);
1610 }
1611 }
1612
1613 /*
1614 * We should be safe here since we are only dealing with file pages and
1615 * we are not kswapd and therefore cannot write dirty file pages. But
1616 * call memalloc_noreclaim_save() anyway, just in case these conditions
1617 * change in the future.
1618 */
1619 noreclaim_flag = memalloc_noreclaim_save();
1620 nr_reclaimed = shrink_folio_list(&clean_folios, zone->zone_pgdat, &sc,
1621 &stat, true, NULL);
1622 memalloc_noreclaim_restore(noreclaim_flag);
1623
1624 list_splice(&clean_folios, folio_list);
1625 mod_node_page_state(zone->zone_pgdat, NR_ISOLATED_FILE,
1626 -(long)nr_reclaimed);
1627 /*
1628 * Since lazyfree pages are isolated from file LRU from the beginning,
1629 * they will rotate back to anonymous LRU in the end if it failed to
1630 * discard so isolated count will be mismatched.
1631 * Compensate the isolated count for both LRU lists.
1632 */
1633 mod_node_page_state(zone->zone_pgdat, NR_ISOLATED_ANON,
1634 stat.nr_lazyfree_fail);
1635 mod_node_page_state(zone->zone_pgdat, NR_ISOLATED_FILE,
1636 -(long)stat.nr_lazyfree_fail);
1637 return nr_reclaimed;
1638}
1639
1640/*
1641 * Update LRU sizes after isolating pages. The LRU size updates must
1642 * be complete before mem_cgroup_update_lru_size due to a sanity check.
1643 */
1644static __always_inline void update_lru_sizes(struct lruvec *lruvec,
1645 enum lru_list lru, unsigned long *nr_zone_taken)
1646{
1647 int zid;
1648
1649 for (zid = 0; zid < MAX_NR_ZONES; zid++) {
1650 if (!nr_zone_taken[zid])
1651 continue;
1652
1653 update_lru_size(lruvec, lru, zid, -nr_zone_taken[zid]);
1654 }
1655
1656}
1657
1658/*
1659 * Isolating page from the lruvec to fill in @dst list by nr_to_scan times.
1660 *
1661 * lruvec->lru_lock is heavily contended. Some of the functions that
1662 * shrink the lists perform better by taking out a batch of pages
1663 * and working on them outside the LRU lock.
1664 *
1665 * For pagecache intensive workloads, this function is the hottest
1666 * spot in the kernel (apart from copy_*_user functions).
1667 *
1668 * Lru_lock must be held before calling this function.
1669 *
1670 * @nr_to_scan: The number of eligible pages to look through on the list.
1671 * @lruvec: The LRU vector to pull pages from.
1672 * @dst: The temp list to put pages on to.
1673 * @nr_scanned: The number of pages that were scanned.
1674 * @sc: The scan_control struct for this reclaim session
1675 * @lru: LRU list id for isolating
1676 *
1677 * returns how many pages were moved onto *@dst.
1678 */
1679static unsigned long isolate_lru_folios(unsigned long nr_to_scan,
1680 struct lruvec *lruvec, struct list_head *dst,
1681 unsigned long *nr_scanned, struct scan_control *sc,
1682 enum lru_list lru)
1683{
1684 struct list_head *src = &lruvec->lists[lru];
1685 unsigned long nr_taken = 0;
1686 unsigned long nr_zone_taken[MAX_NR_ZONES] = { 0 };
1687 unsigned long nr_skipped[MAX_NR_ZONES] = { 0, };
1688 unsigned long skipped = 0, total_scan = 0, scan = 0;
1689 unsigned long nr_pages;
1690 unsigned long max_nr_skipped = 0;
1691 LIST_HEAD(folios_skipped);
1692
1693 while (scan < nr_to_scan && !list_empty(src)) {
1694 struct list_head *move_to = src;
1695 struct folio *folio;
1696
1697 folio = lru_to_folio(src);
1698 prefetchw_prev_lru_folio(folio, src, flags);
1699
1700 nr_pages = folio_nr_pages(folio);
1701 total_scan += nr_pages;
1702
1703 /* Using max_nr_skipped to prevent hard LOCKUP*/
1704 if (max_nr_skipped < SWAP_CLUSTER_MAX_SKIPPED &&
1705 (folio_zonenum(folio) > sc->reclaim_idx)) {
1706 nr_skipped[folio_zonenum(folio)] += nr_pages;
1707 move_to = &folios_skipped;
1708 max_nr_skipped++;
1709 goto move;
1710 }
1711
1712 /*
1713 * Do not count skipped folios because that makes the function
1714 * return with no isolated folios if the LRU mostly contains
1715 * ineligible folios. This causes the VM to not reclaim any
1716 * folios, triggering a premature OOM.
1717 * Account all pages in a folio.
1718 */
1719 scan += nr_pages;
1720
1721 if (!folio_test_lru(folio))
1722 goto move;
1723 if (!sc->may_unmap && folio_mapped(folio))
1724 goto move;
1725
1726 /*
1727 * Be careful not to clear the lru flag until after we're
1728 * sure the folio is not being freed elsewhere -- the
1729 * folio release code relies on it.
1730 */
1731 if (unlikely(!folio_try_get(folio)))
1732 goto move;
1733
1734 if (!folio_test_clear_lru(folio)) {
1735 /* Another thread is already isolating this folio */
1736 folio_put(folio);
1737 goto move;
1738 }
1739
1740 nr_taken += nr_pages;
1741 nr_zone_taken[folio_zonenum(folio)] += nr_pages;
1742 move_to = dst;
1743move:
1744 list_move(&folio->lru, move_to);
1745 }
1746
1747 /*
1748 * Splice any skipped folios to the start of the LRU list. Note that
1749 * this disrupts the LRU order when reclaiming for lower zones but
1750 * we cannot splice to the tail. If we did then the SWAP_CLUSTER_MAX
1751 * scanning would soon rescan the same folios to skip and waste lots
1752 * of cpu cycles.
1753 */
1754 if (!list_empty(&folios_skipped)) {
1755 int zid;
1756
1757 list_splice(&folios_skipped, src);
1758 for (zid = 0; zid < MAX_NR_ZONES; zid++) {
1759 if (!nr_skipped[zid])
1760 continue;
1761
1762 __count_zid_vm_events(PGSCAN_SKIP, zid, nr_skipped[zid]);
1763 skipped += nr_skipped[zid];
1764 }
1765 }
1766 *nr_scanned = total_scan;
1767 trace_mm_vmscan_lru_isolate(sc->reclaim_idx, sc->order, nr_to_scan,
1768 total_scan, skipped, nr_taken, lru);
1769 update_lru_sizes(lruvec, lru, nr_zone_taken);
1770 return nr_taken;
1771}
1772
1773/**
1774 * folio_isolate_lru() - Try to isolate a folio from its LRU list.
1775 * @folio: Folio to isolate from its LRU list.
1776 *
1777 * Isolate a @folio from an LRU list and adjust the vmstat statistic
1778 * corresponding to whatever LRU list the folio was on.
1779 *
1780 * The folio will have its LRU flag cleared. If it was found on the
1781 * active list, it will have the Active flag set. If it was found on the
1782 * unevictable list, it will have the Unevictable flag set. These flags
1783 * may need to be cleared by the caller before letting the page go.
1784 *
1785 * Context:
1786 *
1787 * (1) Must be called with an elevated refcount on the folio. This is a
1788 * fundamental difference from isolate_lru_folios() (which is called
1789 * without a stable reference).
1790 * (2) The lru_lock must not be held.
1791 * (3) Interrupts must be enabled.
1792 *
1793 * Return: true if the folio was removed from an LRU list.
1794 * false if the folio was not on an LRU list.
1795 */
1796bool folio_isolate_lru(struct folio *folio)
1797{
1798 bool ret = false;
1799
1800 VM_BUG_ON_FOLIO(!folio_ref_count(folio), folio);
1801
1802 if (folio_test_clear_lru(folio)) {
1803 struct lruvec *lruvec;
1804
1805 folio_get(folio);
1806 lruvec = folio_lruvec_lock_irq(folio);
1807 lruvec_del_folio(lruvec, folio);
1808 lruvec_unlock_irq(lruvec);
1809 ret = true;
1810 }
1811
1812 return ret;
1813}
1814
1815/*
1816 * A direct reclaimer may isolate SWAP_CLUSTER_MAX pages from the LRU list and
1817 * then get rescheduled. When there are massive number of tasks doing page
1818 * allocation, such sleeping direct reclaimers may keep piling up on each CPU,
1819 * the LRU list will go small and be scanned faster than necessary, leading to
1820 * unnecessary swapping, thrashing and OOM.
1821 */
1822static bool too_many_isolated(struct pglist_data *pgdat, int file,
1823 struct scan_control *sc)
1824{
1825 unsigned long inactive, isolated;
1826 bool too_many;
1827
1828 if (current_is_kswapd())
1829 return false;
1830
1831 if (!writeback_throttling_sane(sc))
1832 return false;
1833
1834 if (file) {
1835 inactive = node_page_state(pgdat, NR_INACTIVE_FILE);
1836 isolated = node_page_state(pgdat, NR_ISOLATED_FILE);
1837 } else {
1838 inactive = node_page_state(pgdat, NR_INACTIVE_ANON);
1839 isolated = node_page_state(pgdat, NR_ISOLATED_ANON);
1840 }
1841
1842 /*
1843 * GFP_NOIO/GFP_NOFS callers are allowed to isolate more pages, so they
1844 * won't get blocked by normal direct-reclaimers, forming a circular
1845 * deadlock.
1846 */
1847 if (gfp_has_io_fs(sc->gfp_mask))
1848 inactive >>= 3;
1849
1850 too_many = isolated > inactive;
1851
1852 /* Wake up tasks throttled due to too_many_isolated. */
1853 if (!too_many)
1854 wake_throttle_isolated(pgdat);
1855
1856 return too_many;
1857}
1858
1859/*
1860 * move_folios_to_lru() moves folios from private @list to appropriate LRU list.
1861 *
1862 * Returns the number of pages moved to the appropriate lruvec.
1863 *
1864 * Note: The caller must not hold any lruvec lock.
1865 */
1866static unsigned int move_folios_to_lru(struct list_head *list)
1867{
1868 int nr_pages, nr_moved = 0;
1869 struct lruvec *lruvec = NULL;
1870 struct folio_batch free_folios;
1871
1872 folio_batch_init(&free_folios);
1873 while (!list_empty(list)) {
1874 struct folio *folio = lru_to_folio(list);
1875
1876 lruvec = folio_lruvec_relock_irq(folio, lruvec);
1877 VM_BUG_ON_FOLIO(folio_test_lru(folio), folio);
1878 list_del(&folio->lru);
1879 if (unlikely(!folio_evictable(folio))) {
1880 lruvec_unlock_irq(lruvec);
1881 folio_putback_lru(folio);
1882 lruvec = NULL;
1883 continue;
1884 }
1885
1886 /*
1887 * The folio_set_lru needs to be kept here for list integrity.
1888 * Otherwise:
1889 * #0 move_folios_to_lru #1 release_pages
1890 * if (!folio_put_testzero())
1891 * if (folio_put_testzero())
1892 * !lru //skip lru_lock
1893 * folio_set_lru()
1894 * list_add(&folio->lru,)
1895 * list_add(&folio->lru,)
1896 */
1897 folio_set_lru(folio);
1898
1899 if (unlikely(folio_put_testzero(folio))) {
1900 __folio_clear_lru_flags(folio);
1901
1902 folio_unqueue_deferred_split(folio);
1903 if (folio_batch_add(&free_folios, folio) == 0) {
1904 lruvec_unlock_irq(lruvec);
1905 mem_cgroup_uncharge_folios(&free_folios);
1906 free_unref_folios(&free_folios);
1907 lruvec = NULL;
1908 }
1909
1910 continue;
1911 }
1912
1913 lruvec_add_folio(lruvec, folio);
1914 nr_pages = folio_nr_pages(folio);
1915 nr_moved += nr_pages;
1916 if (folio_test_active(folio))
1917 workingset_age_nonresident(lruvec, nr_pages);
1918 }
1919
1920 if (lruvec)
1921 lruvec_unlock_irq(lruvec);
1922
1923 if (free_folios.nr) {
1924 mem_cgroup_uncharge_folios(&free_folios);
1925 free_unref_folios(&free_folios);
1926 }
1927
1928 return nr_moved;
1929}
1930
1931/*
1932 * If a kernel thread (such as nfsd for loop-back mounts) services a backing
1933 * device by writing to the page cache it sets PF_LOCAL_THROTTLE. In this case
1934 * we should not throttle. Otherwise it is safe to do so.
1935 */
1936static int current_may_throttle(void)
1937{
1938 return !(current->flags & PF_LOCAL_THROTTLE);
1939}
1940
1941static void handle_reclaim_writeback(unsigned long nr_taken,
1942 struct pglist_data *pgdat,
1943 struct scan_control *sc,
1944 struct reclaim_stat *stat)
1945{
1946 /*
1947 * If dirty folios are scanned that are not queued for IO, it
1948 * implies that flushers are not doing their job. This can
1949 * happen when memory pressure pushes dirty folios to the end of
1950 * the LRU before the dirty limits are breached and the dirty
1951 * data has expired. It can also happen when the proportion of
1952 * dirty folios grows not through writes but through memory
1953 * pressure reclaiming all the clean cache. And in some cases,
1954 * the flushers simply cannot keep up with the allocation
1955 * rate. Nudge the flusher threads in case they are asleep.
1956 */
1957 if (stat->nr_unqueued_dirty == nr_taken) {
1958 wakeup_flusher_threads(WB_REASON_VMSCAN);
1959 /*
1960 * For cgroupv1 dirty throttling is achieved by waking up
1961 * the kernel flusher here and later waiting on folios
1962 * which are in writeback to finish (see shrink_folio_list()).
1963 *
1964 * Flusher may not be able to issue writeback quickly
1965 * enough for cgroupv1 writeback throttling to work
1966 * on a large system.
1967 */
1968 if (!writeback_throttling_sane(sc))
1969 reclaim_throttle(pgdat, VMSCAN_THROTTLE_WRITEBACK);
1970 }
1971
1972 sc->nr.dirty += stat->nr_dirty;
1973 sc->nr.congested += stat->nr_congested;
1974 sc->nr.writeback += stat->nr_writeback;
1975 sc->nr.immediate += stat->nr_immediate;
1976 sc->nr.taken += nr_taken;
1977}
1978
1979/*
1980 * shrink_inactive_list() is a helper for shrink_node(). It returns the number
1981 * of reclaimed pages
1982 */
1983static unsigned long shrink_inactive_list(unsigned long nr_to_scan,
1984 struct lruvec *lruvec, struct scan_control *sc,
1985 enum lru_list lru)
1986{
1987 LIST_HEAD(folio_list);
1988 unsigned long nr_scanned;
1989 unsigned int nr_reclaimed = 0;
1990 unsigned long nr_taken;
1991 struct reclaim_stat stat;
1992 bool file = is_file_lru(lru);
1993 enum node_stat_item item;
1994 struct pglist_data *pgdat = lruvec_pgdat(lruvec);
1995 bool stalled = false;
1996
1997 while (unlikely(too_many_isolated(pgdat, file, sc))) {
1998 if (stalled)
1999 return 0;
2000
2001 /* wait a bit for the reclaimer. */
2002 stalled = true;
2003 reclaim_throttle(pgdat, VMSCAN_THROTTLE_ISOLATED);
2004
2005 /* We are about to die and free our memory. Return now. */
2006 if (fatal_signal_pending(current))
2007 return SWAP_CLUSTER_MAX;
2008 }
2009
2010 lru_add_drain();
2011
2012 lruvec_lock_irq(lruvec);
2013
2014 nr_taken = isolate_lru_folios(nr_to_scan, lruvec, &folio_list,
2015 &nr_scanned, sc, lru);
2016
2017 __mod_node_page_state(pgdat, NR_ISOLATED_ANON + file, nr_taken);
2018 item = PGSCAN_KSWAPD + reclaimer_offset(sc);
2019 mod_lruvec_state(lruvec, item, nr_scanned);
2020 mod_lruvec_state(lruvec, PGSCAN_ANON + file, nr_scanned);
2021
2022 lruvec_unlock_irq(lruvec);
2023
2024 if (nr_taken == 0)
2025 return 0;
2026
2027 nr_reclaimed = shrink_folio_list(&folio_list, pgdat, sc, &stat, false,
2028 lruvec_memcg(lruvec));
2029
2030 move_folios_to_lru(&folio_list);
2031
2032 mod_lruvec_state(lruvec, PGDEMOTE_KSWAPD + reclaimer_offset(sc),
2033 stat.nr_demoted);
2034 mod_node_page_state(pgdat, NR_ISOLATED_ANON + file, -nr_taken);
2035 item = PGSTEAL_KSWAPD + reclaimer_offset(sc);
2036 mod_lruvec_state(lruvec, item, nr_reclaimed);
2037 mod_lruvec_state(lruvec, PGSTEAL_ANON + file, nr_reclaimed);
2038 if (nr_scanned > nr_reclaimed)
2039 mod_lruvec_state(lruvec, PGROTATE_ANON + file,
2040 nr_scanned - nr_reclaimed);
2041
2042 handle_reclaim_writeback(nr_taken, pgdat, sc, &stat);
2043 trace_mm_vmscan_lru_shrink_inactive(pgdat->node_id,
2044 nr_scanned, nr_reclaimed, &stat, sc->priority, file);
2045 return nr_reclaimed;
2046}
2047
2048/*
2049 * shrink_active_list() moves folios from the active LRU to the inactive LRU.
2050 *
2051 * We move them the other way if the folio is referenced by one or more
2052 * processes.
2053 *
2054 * If the folios are mostly unmapped, the processing is fast and it is
2055 * appropriate to hold lru_lock across the whole operation. But if
2056 * the folios are mapped, the processing is slow (folio_referenced()), so
2057 * we should drop lru_lock around each folio. It's impossible to balance
2058 * this, so instead we remove the folios from the LRU while processing them.
2059 * It is safe to rely on the active flag against the non-LRU folios in here
2060 * because nobody will play with that bit on a non-LRU folio.
2061 *
2062 * The downside is that we have to touch folio->_refcount against each folio.
2063 * But we had to alter folio->flags anyway.
2064 */
2065static void shrink_active_list(unsigned long nr_to_scan,
2066 struct lruvec *lruvec,
2067 struct scan_control *sc,
2068 enum lru_list lru)
2069{
2070 unsigned long nr_taken;
2071 unsigned long nr_scanned;
2072 vma_flags_t vma_flags;
2073 LIST_HEAD(l_hold); /* The folios which were snipped off */
2074 LIST_HEAD(l_active);
2075 LIST_HEAD(l_inactive);
2076 unsigned nr_deactivate, nr_activate;
2077 unsigned nr_rotated = 0;
2078 bool file = is_file_lru(lru);
2079 struct pglist_data *pgdat = lruvec_pgdat(lruvec);
2080
2081 lru_add_drain();
2082
2083 lruvec_lock_irq(lruvec);
2084
2085 nr_taken = isolate_lru_folios(nr_to_scan, lruvec, &l_hold,
2086 &nr_scanned, sc, lru);
2087
2088 __mod_node_page_state(pgdat, NR_ISOLATED_ANON + file, nr_taken);
2089
2090 mod_lruvec_state(lruvec, PGREFILL, nr_scanned);
2091
2092 lruvec_unlock_irq(lruvec);
2093
2094 while (!list_empty(&l_hold)) {
2095 struct folio *folio;
2096
2097 cond_resched();
2098 folio = lru_to_folio(&l_hold);
2099 list_del(&folio->lru);
2100
2101 if (unlikely(!folio_evictable(folio))) {
2102 folio_putback_lru(folio);
2103 continue;
2104 }
2105
2106 if (unlikely(buffer_heads_over_limit)) {
2107 if (folio_needs_release(folio) &&
2108 folio_trylock(folio)) {
2109 filemap_release_folio(folio, 0);
2110 folio_unlock(folio);
2111 }
2112 }
2113
2114 /* Referenced or rmap lock contention: rotate */
2115 if (folio_referenced(folio, 0, sc->target_mem_cgroup,
2116 &vma_flags) != 0) {
2117 /*
2118 * Identify referenced, file-backed active folios and
2119 * give them one more trip around the active list. So
2120 * that executable code get better chances to stay in
2121 * memory under moderate memory pressure. Anon folios
2122 * are not likely to be evicted by use-once streaming
2123 * IO, plus JVM can create lots of anon VM_EXEC folios,
2124 * so we ignore them here.
2125 */
2126 if (is_exec_file_folio(folio, &vma_flags)) {
2127 nr_rotated += folio_nr_pages(folio);
2128 list_add(&folio->lru, &l_active);
2129 continue;
2130 }
2131 }
2132
2133 folio_clear_active(folio); /* we are de-activating */
2134 folio_set_workingset(folio);
2135 list_add(&folio->lru, &l_inactive);
2136 }
2137
2138 /*
2139 * Move folios back to the lru list.
2140 */
2141 nr_activate = move_folios_to_lru(&l_active);
2142 nr_deactivate = move_folios_to_lru(&l_inactive);
2143
2144 count_vm_events(PGDEACTIVATE, nr_deactivate);
2145 count_memcg_events(lruvec_memcg(lruvec), PGDEACTIVATE, nr_deactivate);
2146 mod_node_page_state(pgdat, NR_ISOLATED_ANON + file, -nr_taken);
2147 if (nr_rotated)
2148 mod_lruvec_state(lruvec, PGROTATE_ANON + file, nr_rotated);
2149
2150 trace_mm_vmscan_lru_shrink_active(pgdat->node_id, nr_taken, nr_activate,
2151 nr_deactivate, nr_rotated, sc->priority, file);
2152}
2153
2154static unsigned int reclaim_folio_list(struct list_head *folio_list,
2155 struct pglist_data *pgdat)
2156{
2157 struct reclaim_stat stat;
2158 unsigned int nr_reclaimed;
2159 struct folio *folio;
2160 struct scan_control sc = {
2161 .gfp_mask = GFP_KERNEL,
2162 .may_writepage = 1,
2163 .may_unmap = 1,
2164 .may_swap = 1,
2165 .no_demotion = 1,
2166 };
2167
2168 nr_reclaimed = shrink_folio_list(folio_list, pgdat, &sc, &stat, true, NULL);
2169 while (!list_empty(folio_list)) {
2170 folio = lru_to_folio(folio_list);
2171 list_del(&folio->lru);
2172 folio_putback_lru(folio);
2173 }
2174 trace_mm_vmscan_reclaim_pages(pgdat->node_id, sc.nr_scanned, nr_reclaimed, &stat);
2175
2176 return nr_reclaimed;
2177}
2178
2179unsigned long reclaim_pages(struct list_head *folio_list)
2180{
2181 int nid;
2182 unsigned int nr_reclaimed = 0;
2183 LIST_HEAD(node_folio_list);
2184 unsigned int noreclaim_flag;
2185
2186 if (list_empty(folio_list))
2187 return nr_reclaimed;
2188
2189 noreclaim_flag = memalloc_noreclaim_save();
2190
2191 nid = folio_nid(lru_to_folio(folio_list));
2192 do {
2193 struct folio *folio = lru_to_folio(folio_list);
2194
2195 if (nid == folio_nid(folio)) {
2196 folio_clear_active(folio);
2197 list_move(&folio->lru, &node_folio_list);
2198 continue;
2199 }
2200
2201 nr_reclaimed += reclaim_folio_list(&node_folio_list, NODE_DATA(nid));
2202 nid = folio_nid(lru_to_folio(folio_list));
2203 } while (!list_empty(folio_list));
2204
2205 nr_reclaimed += reclaim_folio_list(&node_folio_list, NODE_DATA(nid));
2206
2207 memalloc_noreclaim_restore(noreclaim_flag);
2208
2209 return nr_reclaimed;
2210}
2211
2212static unsigned long shrink_list(enum lru_list lru, unsigned long nr_to_scan,
2213 struct lruvec *lruvec, struct scan_control *sc)
2214{
2215 if (is_active_lru(lru)) {
2216 if (sc->may_deactivate & (1 << is_file_lru(lru)))
2217 shrink_active_list(nr_to_scan, lruvec, sc, lru);
2218 else
2219 sc->skipped_deactivate = 1;
2220 return 0;
2221 }
2222
2223 return shrink_inactive_list(nr_to_scan, lruvec, sc, lru);
2224}
2225
2226/*
2227 * The inactive anon list should be small enough that the VM never has
2228 * to do too much work.
2229 *
2230 * The inactive file list should be small enough to leave most memory
2231 * to the established workingset on the scan-resistant active list,
2232 * but large enough to avoid thrashing the aggregate readahead window.
2233 *
2234 * Both inactive lists should also be large enough that each inactive
2235 * folio has a chance to be referenced again before it is reclaimed.
2236 *
2237 * If that fails and refaulting is observed, the inactive list grows.
2238 *
2239 * The inactive_ratio is the target ratio of ACTIVE to INACTIVE folios
2240 * on this LRU, maintained by the pageout code. An inactive_ratio
2241 * of 3 means 3:1 or 25% of the folios are kept on the inactive list.
2242 *
2243 * total target max
2244 * memory ratio inactive
2245 * -------------------------------------
2246 * 10MB 1 5MB
2247 * 100MB 1 50MB
2248 * 1GB 3 250MB
2249 * 10GB 10 0.9GB
2250 * 100GB 31 3GB
2251 * 1TB 101 10GB
2252 * 10TB 320 32GB
2253 */
2254static bool inactive_is_low(struct lruvec *lruvec, enum lru_list inactive_lru)
2255{
2256 enum lru_list active_lru = inactive_lru + LRU_ACTIVE;
2257 unsigned long inactive, active;
2258 unsigned long inactive_ratio;
2259 unsigned long gb;
2260
2261 inactive = lruvec_page_state(lruvec, NR_LRU_BASE + inactive_lru);
2262 active = lruvec_page_state(lruvec, NR_LRU_BASE + active_lru);
2263
2264 gb = (inactive + active) >> (30 - PAGE_SHIFT);
2265 if (gb)
2266 inactive_ratio = int_sqrt(10 * gb);
2267 else
2268 inactive_ratio = 1;
2269
2270 return inactive * inactive_ratio < active;
2271}
2272
2273enum scan_balance {
2274 SCAN_EQUAL,
2275 SCAN_FRACT,
2276 SCAN_ANON,
2277 SCAN_FILE,
2278};
2279
2280static void prepare_scan_control(pg_data_t *pgdat, struct scan_control *sc)
2281{
2282 struct lru_cost *anon_cost, *file_cost;
2283 struct lruvec *target_lruvec;
2284 unsigned long lrusize;
2285 unsigned long file;
2286
2287 if (lru_gen_enabled() && !lru_gen_switching())
2288 return;
2289
2290 target_lruvec = mem_cgroup_lruvec(sc->target_mem_cgroup, pgdat);
2291
2292 /*
2293 * Flush the memory cgroup stats in rate-limited way as we don't need
2294 * most accurate stats here. We may switch to regular stats flushing
2295 * in the future once it is cheap enough.
2296 */
2297 mem_cgroup_flush_stats_ratelimited(sc->target_mem_cgroup);
2298
2299 /*
2300 * Determine the scan balance between anon and file LRUs.
2301 *
2302 * The cost model is based on rotations, refaults and
2303 * reclaim-driven writes (anon only) on each side.
2304 *
2305 * These event counters are monotonic, so each reclaim cycle
2306 * the delta since the last scan is extracted and incorporated
2307 * into a decaying average. This ensures currency, as workloads
2308 * change over time, and avoids overflow in the calculations.
2309 *
2310 * Use lruvec_page_state_monotonic() so unsigned subtraction
2311 * yields the correct delta across a signed-long wraparound of
2312 * the underlying counter (a real hazard on 32-bit that the
2313 * clamp in lruvec_page_state() would otherwise turn into a huge
2314 * spurious delta).
2315 */
2316 spin_lock(&target_lruvec->cost_lock);
2317
2318 for (int f = 0; f <= 1; f++) {
2319 struct lru_cost *cost = &target_lruvec->cost[f];
2320 unsigned long rotated, io, nr_rotated, nr_io;
2321
2322 rotated = lruvec_page_state_monotonic(target_lruvec,
2323 PGROTATE_ANON + f);
2324 io = lruvec_page_state_monotonic(target_lruvec,
2325 WORKINGSET_RESTORE_BASE + f);
2326 if (f == WORKINGSET_ANON)
2327 io += lruvec_page_state_monotonic(target_lruvec,
2328 NR_VMSCAN_WRITE);
2329
2330 nr_rotated = rotated - cost->last_rotated;
2331 nr_io = io - cost->last_io;
2332
2333 /*
2334 * Reflect the relative cost of incurring IO and spending
2335 * CPU time on rotations. This doesn't attempt to make a
2336 * precise comparison, it just says: if reloads are about
2337 * comparable between the LRU lists, or rotations are
2338 * overwhelmingly different between them, adjust scan
2339 * balance for CPU work.
2340 */
2341 cost->count += nr_io * SWAP_CLUSTER_MAX + nr_rotated;
2342
2343 cost->last_rotated = rotated;
2344 cost->last_io = io;
2345 }
2346
2347 anon_cost = &target_lruvec->cost[WORKINGSET_ANON];
2348 file_cost = &target_lruvec->cost[WORKINGSET_FILE];
2349
2350 lrusize = lruvec_page_state(target_lruvec, NR_INACTIVE_ANON) +
2351 lruvec_page_state(target_lruvec, NR_ACTIVE_ANON) +
2352 lruvec_page_state(target_lruvec, NR_INACTIVE_FILE) +
2353 lruvec_page_state(target_lruvec, NR_ACTIVE_FILE);
2354
2355 while (anon_cost->count + file_cost->count > lrusize / 4) {
2356 anon_cost->count /= 2;
2357 file_cost->count /= 2;
2358 }
2359
2360 sc->anon_cost = anon_cost->count;
2361 sc->file_cost = file_cost->count;
2362
2363 spin_unlock(&target_lruvec->cost_lock);
2364
2365 /*
2366 * Target desirable inactive:active list ratios for the anon
2367 * and file LRU lists.
2368 */
2369 if (!sc->force_deactivate) {
2370 unsigned long refaults;
2371
2372 /*
2373 * When refaults are being observed, it means a new
2374 * workingset is being established. Deactivate to get
2375 * rid of any stale active pages quickly.
2376 */
2377 refaults = lruvec_page_state(target_lruvec,
2378 WORKINGSET_ACTIVATE_ANON);
2379 if (refaults != target_lruvec->refaults[WORKINGSET_ANON] ||
2380 inactive_is_low(target_lruvec, LRU_INACTIVE_ANON))
2381 sc->may_deactivate |= DEACTIVATE_ANON;
2382 else
2383 sc->may_deactivate &= ~DEACTIVATE_ANON;
2384
2385 refaults = lruvec_page_state(target_lruvec,
2386 WORKINGSET_ACTIVATE_FILE);
2387 if (refaults != target_lruvec->refaults[WORKINGSET_FILE] ||
2388 inactive_is_low(target_lruvec, LRU_INACTIVE_FILE))
2389 sc->may_deactivate |= DEACTIVATE_FILE;
2390 else
2391 sc->may_deactivate &= ~DEACTIVATE_FILE;
2392 } else
2393 sc->may_deactivate = DEACTIVATE_ANON | DEACTIVATE_FILE;
2394
2395 /*
2396 * If we have plenty of inactive file pages that aren't
2397 * thrashing, try to reclaim those first before touching
2398 * anonymous pages.
2399 */
2400 file = lruvec_page_state(target_lruvec, NR_INACTIVE_FILE);
2401 if (file >> sc->priority && !(sc->may_deactivate & DEACTIVATE_FILE) &&
2402 !sc->no_cache_trim_mode)
2403 sc->cache_trim_mode = 1;
2404 else
2405 sc->cache_trim_mode = 0;
2406
2407 /*
2408 * Prevent the reclaimer from falling into the cache trap: as
2409 * cache pages start out inactive, every cache fault will tip
2410 * the scan balance towards the file LRU. And as the file LRU
2411 * shrinks, so does the window for rotation from references.
2412 * This means we have a runaway feedback loop where a tiny
2413 * thrashing file LRU becomes infinitely more attractive than
2414 * anon pages. Try to detect this based on file LRU size.
2415 */
2416 if (!cgroup_reclaim(sc)) {
2417 unsigned long total_high_wmark = 0;
2418 unsigned long free, anon;
2419 int z;
2420 struct zone *zone;
2421
2422 free = sum_zone_node_page_state(pgdat->node_id, NR_FREE_PAGES);
2423 file = node_page_state(pgdat, NR_ACTIVE_FILE) +
2424 node_page_state(pgdat, NR_INACTIVE_FILE);
2425
2426 for_each_managed_zone_pgdat(zone, pgdat, z, MAX_NR_ZONES - 1) {
2427 total_high_wmark += high_wmark_pages(zone);
2428 }
2429
2430 /*
2431 * Consider anon: if that's low too, this isn't a
2432 * runaway file reclaim problem, but rather just
2433 * extreme pressure. Reclaim as per usual then.
2434 */
2435 anon = node_page_state(pgdat, NR_INACTIVE_ANON);
2436
2437 sc->file_is_tiny =
2438 file + free <= total_high_wmark &&
2439 !(sc->may_deactivate & DEACTIVATE_ANON) &&
2440 anon >> sc->priority;
2441 }
2442}
2443
2444static inline void calculate_pressure_balance(struct scan_control *sc,
2445 int swappiness, u64 *fraction, u64 *denominator)
2446{
2447 unsigned long anon_cost, file_cost, total_cost;
2448 unsigned long ap, fp;
2449
2450 /*
2451 * Calculate the pressure balance between anon and file pages.
2452 *
2453 * The amount of pressure we put on each LRU is inversely
2454 * proportional to the cost of reclaiming each list, as
2455 * determined by the share of pages that are refaulting, times
2456 * the relative IO cost of bringing back a swapped out
2457 * anonymous page vs reloading a filesystem page (swappiness).
2458 *
2459 * Although we limit that influence to ensure no list gets
2460 * left behind completely: at least a third of the pressure is
2461 * applied, before swappiness.
2462 *
2463 * With swappiness at 100, anon and file have equal IO cost.
2464 */
2465 total_cost = sc->anon_cost + sc->file_cost;
2466 anon_cost = total_cost + sc->anon_cost;
2467 file_cost = total_cost + sc->file_cost;
2468 total_cost = anon_cost + file_cost;
2469
2470 ap = swappiness * (total_cost + 1);
2471 ap /= anon_cost + 1;
2472
2473 fp = (MAX_SWAPPINESS - swappiness) * (total_cost + 1);
2474 fp /= file_cost + 1;
2475
2476 fraction[WORKINGSET_ANON] = ap;
2477 fraction[WORKINGSET_FILE] = fp;
2478 *denominator = ap + fp;
2479}
2480
2481static unsigned long apply_proportional_protection(struct mem_cgroup *memcg,
2482 struct scan_control *sc, unsigned long scan)
2483{
2484 unsigned long min, low, usage;
2485
2486 mem_cgroup_protection(sc->target_mem_cgroup, memcg, &min, &low, &usage);
2487
2488 if (min || low) {
2489 /*
2490 * Scale a cgroup's reclaim pressure by proportioning
2491 * its current usage to its memory.low or memory.min
2492 * setting.
2493 *
2494 * This is important, as otherwise scanning aggression
2495 * becomes extremely binary -- from nothing as we
2496 * approach the memory protection threshold, to totally
2497 * nominal as we exceed it. This results in requiring
2498 * setting extremely liberal protection thresholds. It
2499 * also means we simply get no protection at all if we
2500 * set it too low, which is not ideal.
2501 *
2502 * If there is any protection in place, we reduce scan
2503 * pressure by how much of the total memory used is
2504 * within protection thresholds.
2505 *
2506 * There is one special case: in the first reclaim pass,
2507 * we skip over all groups that are within their low
2508 * protection. If that fails to reclaim enough pages to
2509 * satisfy the reclaim goal, we come back and override
2510 * the best-effort low protection. However, we still
2511 * ideally want to honor how well-behaved groups are in
2512 * that case instead of simply punishing them all
2513 * equally. As such, we reclaim them based on how much
2514 * memory they are using, reducing the scan pressure
2515 * again by how much of the total memory used is under
2516 * hard protection.
2517 */
2518 unsigned long protection;
2519
2520 /* memory.low scaling, make sure we retry before OOM */
2521 if (!sc->memcg_low_reclaim && low > min) {
2522 protection = low;
2523 sc->memcg_low_skipped = 1;
2524 } else {
2525 protection = min;
2526 }
2527
2528 /* Avoid TOCTOU with earlier protection check */
2529 usage = max(usage, protection);
2530
2531 scan -= scan * protection / (usage + 1);
2532
2533 /*
2534 * Minimally target SWAP_CLUSTER_MAX pages to keep
2535 * reclaim moving forwards, avoiding decrementing
2536 * sc->priority further than desirable.
2537 */
2538 scan = max(scan, SWAP_CLUSTER_MAX);
2539 }
2540 return scan;
2541}
2542
2543/*
2544 * Determine how aggressively the anon and file LRU lists should be
2545 * scanned.
2546 *
2547 * nr[0] = anon inactive folios to scan; nr[1] = anon active folios to scan
2548 * nr[2] = file inactive folios to scan; nr[3] = file active folios to scan
2549 */
2550static void get_scan_count(struct lruvec *lruvec, struct scan_control *sc,
2551 unsigned long *nr)
2552{
2553 struct pglist_data *pgdat = lruvec_pgdat(lruvec);
2554 struct mem_cgroup *memcg = lruvec_memcg(lruvec);
2555 int swappiness = sc_swappiness(sc, memcg);
2556 u64 fraction[ANON_AND_FILE];
2557 u64 denominator = 0; /* gcc */
2558 enum scan_balance scan_balance;
2559 enum lru_list lru;
2560
2561 /*
2562 * Proactive reclaim initiated by userspace for anonymous memory only.
2563 * SWAPPINESS_ANON_ONLY is set only on the proactive reclaim path, so
2564 * warn if it shows up elsewhere. When anon cannot be reclaimed (e.g.
2565 * no swap), bail out instead of falling back to evicting file pages,
2566 * which would violate the anon-only semantics.
2567 */
2568 if (swappiness == SWAPPINESS_ANON_ONLY) {
2569 WARN_ON_ONCE(!sc->proactive);
2570 if (!can_reclaim_anon_pages(memcg, pgdat->node_id, sc)) {
2571 memset(nr, 0, sizeof(*nr) * NR_LRU_LISTS);
2572 return;
2573 }
2574 scan_balance = SCAN_ANON;
2575 goto out;
2576 }
2577
2578 /* If we have no swap space, do not bother scanning anon folios. */
2579 if (!sc->may_swap || !can_reclaim_anon_pages(memcg, pgdat->node_id, sc)) {
2580 scan_balance = SCAN_FILE;
2581 goto out;
2582 }
2583
2584 /*
2585 * Global reclaim will swap to prevent OOM even with no
2586 * swappiness, but memcg users want to use this knob to
2587 * disable swapping for individual groups completely when
2588 * using the memory controller's swap limit feature would be
2589 * too expensive.
2590 */
2591 if (cgroup_reclaim(sc) && !swappiness) {
2592 scan_balance = SCAN_FILE;
2593 goto out;
2594 }
2595
2596 /*
2597 * Do not apply any pressure balancing cleverness when the
2598 * system is close to OOM, scan both anon and file equally
2599 * (unless the swappiness setting disagrees with swapping).
2600 */
2601 if (!sc->priority && swappiness) {
2602 scan_balance = SCAN_EQUAL;
2603 goto out;
2604 }
2605
2606 /*
2607 * If the system is almost out of file pages, force-scan anon.
2608 */
2609 if (sc->file_is_tiny) {
2610 scan_balance = SCAN_ANON;
2611 goto out;
2612 }
2613
2614 /*
2615 * If there is enough inactive page cache, we do not reclaim
2616 * anything from the anonymous working right now to make sure
2617 * a streaming file access pattern doesn't cause swapping.
2618 */
2619 if (sc->cache_trim_mode) {
2620 scan_balance = SCAN_FILE;
2621 goto out;
2622 }
2623
2624 scan_balance = SCAN_FRACT;
2625 calculate_pressure_balance(sc, swappiness, fraction, &denominator);
2626
2627out:
2628 for_each_evictable_lru(lru) {
2629 bool file = is_file_lru(lru);
2630 unsigned long lruvec_size;
2631 unsigned long scan;
2632
2633 lruvec_size = lruvec_lru_size(lruvec, lru, sc->reclaim_idx);
2634 scan = apply_proportional_protection(memcg, sc, lruvec_size);
2635 scan >>= sc->priority;
2636
2637 /*
2638 * If the cgroup's already been deleted, make sure to
2639 * scrape out the remaining cache.
2640 */
2641 if (!scan && !mem_cgroup_online(memcg))
2642 scan = min(lruvec_size, SWAP_CLUSTER_MAX);
2643
2644 switch (scan_balance) {
2645 case SCAN_EQUAL:
2646 /* Scan lists relative to size */
2647 break;
2648 case SCAN_FRACT:
2649 /*
2650 * Scan types proportional to swappiness and
2651 * their relative recent reclaim efficiency.
2652 * Make sure we don't miss the last page on
2653 * the offlined memory cgroups because of a
2654 * round-off error.
2655 */
2656 scan = mem_cgroup_online(memcg) ?
2657 div64_u64(scan * fraction[file], denominator) :
2658 DIV64_U64_ROUND_UP(scan * fraction[file],
2659 denominator);
2660 break;
2661 case SCAN_FILE:
2662 case SCAN_ANON:
2663 /* Scan one type exclusively */
2664 if ((scan_balance == SCAN_FILE) != file)
2665 scan = 0;
2666 break;
2667 default:
2668 /* Look ma, no brain */
2669 BUG();
2670 }
2671
2672 nr[lru] = scan;
2673 }
2674}
2675
2676/*
2677 * Anonymous LRU management is a waste if there is
2678 * ultimately no way to reclaim the memory.
2679 */
2680static bool can_age_anon_pages(struct lruvec *lruvec,
2681 struct scan_control *sc)
2682{
2683 /* Aging the anon LRU is valuable if swap is present: */
2684 if (total_swap_pages > 0)
2685 return true;
2686
2687 /* Also valuable if anon pages can be demoted: */
2688 return can_demote(lruvec_pgdat(lruvec)->node_id, sc,
2689 lruvec_memcg(lruvec));
2690}
2691
2692#ifdef CONFIG_LRU_GEN
2693
2694DEFINE_STATIC_KEY_FALSE(lru_switch);
2695#ifdef CONFIG_LRU_GEN_ENABLED
2696DEFINE_STATIC_KEY_ARRAY_TRUE(lru_gen_caps, NR_LRU_GEN_CAPS);
2697#define get_cap(cap) static_branch_likely(&lru_gen_caps[cap])
2698#else
2699DEFINE_STATIC_KEY_ARRAY_FALSE(lru_gen_caps, NR_LRU_GEN_CAPS);
2700#define get_cap(cap) static_branch_unlikely(&lru_gen_caps[cap])
2701#endif
2702
2703static bool should_walk_mmu(void)
2704{
2705 return arch_has_hw_pte_young() && get_cap(LRU_GEN_MM_WALK);
2706}
2707
2708static bool should_clear_pmd_young(void)
2709{
2710 return arch_has_hw_nonleaf_pmd_young() && get_cap(LRU_GEN_NONLEAF_YOUNG);
2711}
2712
2713/******************************************************************************
2714 * shorthand helpers
2715 ******************************************************************************/
2716
2717#define DEFINE_MAX_SEQ(lruvec) \
2718 unsigned long max_seq = READ_ONCE((lruvec)->lrugen.max_seq)
2719
2720#define DEFINE_MIN_SEQ(lruvec) \
2721 unsigned long min_seq[ANON_AND_FILE] = { \
2722 READ_ONCE((lruvec)->lrugen.min_seq[LRU_GEN_ANON]), \
2723 READ_ONCE((lruvec)->lrugen.min_seq[LRU_GEN_FILE]), \
2724 }
2725
2726/* Get the min/max evictable type based on swappiness */
2727#define min_type(swappiness) (!(swappiness))
2728#define max_type(swappiness) ((swappiness) < SWAPPINESS_ANON_ONLY)
2729
2730#define evictable_min_seq(min_seq, swappiness) \
2731 min((min_seq)[min_type(swappiness)], (min_seq)[max_type(swappiness)])
2732
2733#define for_each_gen_type_zone(gen, type, zone) \
2734 for ((gen) = 0; (gen) < MAX_NR_GENS; (gen)++) \
2735 for ((type) = 0; (type) < ANON_AND_FILE; (type)++) \
2736 for ((zone) = 0; (zone) < MAX_NR_ZONES; (zone)++)
2737
2738#define for_each_evictable_type(type, swappiness) \
2739 for ((type) = min_type(swappiness); (type) <= max_type(swappiness); (type)++)
2740
2741#define get_memcg_gen(seq) ((seq) % MEMCG_NR_GENS)
2742#define get_memcg_bin(bin) ((bin) % MEMCG_NR_BINS)
2743
2744static struct lruvec *get_lruvec(struct mem_cgroup *memcg, int nid)
2745{
2746 struct pglist_data *pgdat = NODE_DATA(nid);
2747
2748#ifdef CONFIG_MEMCG
2749 if (memcg) {
2750 struct lruvec *lruvec = &memcg->nodeinfo[nid]->lruvec;
2751
2752 /* see the comment in mem_cgroup_lruvec() */
2753 if (!lruvec->pgdat)
2754 lruvec->pgdat = pgdat;
2755
2756 return lruvec;
2757 }
2758#endif
2759 VM_WARN_ON_ONCE(!mem_cgroup_disabled());
2760
2761 return &pgdat->__lruvec;
2762}
2763
2764static int get_swappiness(struct lruvec *lruvec, struct scan_control *sc)
2765{
2766 struct mem_cgroup *memcg = lruvec_memcg(lruvec);
2767 struct pglist_data *pgdat = lruvec_pgdat(lruvec);
2768 int swappiness = sc_swappiness(sc, memcg);
2769
2770 if (swappiness == SWAPPINESS_ANON_ONLY)
2771 return swappiness;
2772
2773 if (!sc->may_swap)
2774 return 0;
2775
2776 if (!can_demote(pgdat->node_id, sc, memcg) &&
2777 mem_cgroup_get_nr_swap_pages(memcg) < MIN_LRU_BATCH)
2778 return 0;
2779
2780 return swappiness;
2781}
2782
2783static int get_nr_gens(struct lruvec *lruvec, int type)
2784{
2785 return lruvec->lrugen.max_seq - lruvec->lrugen.min_seq[type] + 1;
2786}
2787
2788static bool __maybe_unused seq_is_valid(struct lruvec *lruvec)
2789{
2790 int type;
2791
2792 for (type = 0; type < ANON_AND_FILE; type++) {
2793 int n = get_nr_gens(lruvec, type);
2794
2795 if (n < MIN_NR_GENS || n > MAX_NR_GENS)
2796 return false;
2797 }
2798
2799 return true;
2800}
2801
2802/******************************************************************************
2803 * Bloom filters
2804 ******************************************************************************/
2805
2806/*
2807 * Bloom filters with m=1<<15, k=2 and the false positive rates of ~1/5 when
2808 * n=10,000 and ~1/2 when n=20,000, where, conventionally, m is the number of
2809 * bits in a bitmap, k is the number of hash functions and n is the number of
2810 * inserted items.
2811 *
2812 * Page table walkers use one of the two filters to reduce their search space.
2813 * To get rid of non-leaf entries that no longer have enough leaf entries, the
2814 * aging uses the double-buffering technique to flip to the other filter each
2815 * time it produces a new generation. For non-leaf entries that have enough
2816 * leaf entries, the aging carries them over to the next generation in
2817 * walk_pmd_range(); the eviction also report them when walking the rmap
2818 * in lru_gen_look_around().
2819 *
2820 * For future optimizations:
2821 * 1. It's not necessary to keep both filters all the time. The spare one can be
2822 * freed after the RCU grace period and reallocated if needed again.
2823 * 2. And when reallocating, it's worth scaling its size according to the number
2824 * of inserted entries in the other filter, to reduce the memory overhead on
2825 * small systems and false positives on large systems.
2826 * 3. Jenkins' hash function is an alternative to Knuth's.
2827 */
2828#define BLOOM_FILTER_SHIFT 15
2829
2830static inline int filter_gen_from_seq(unsigned long seq)
2831{
2832 return seq % NR_BLOOM_FILTERS;
2833}
2834
2835static void get_item_key(void *item, int *key)
2836{
2837 u32 hash = hash_ptr(item, BLOOM_FILTER_SHIFT * 2);
2838
2839 BUILD_BUG_ON(BLOOM_FILTER_SHIFT * 2 > BITS_PER_TYPE(u32));
2840
2841 key[0] = hash & (BIT(BLOOM_FILTER_SHIFT) - 1);
2842 key[1] = hash >> BLOOM_FILTER_SHIFT;
2843}
2844
2845static bool test_bloom_filter(struct lru_gen_mm_state *mm_state, unsigned long seq,
2846 void *item)
2847{
2848 int key[2];
2849 unsigned long *filter;
2850 int gen = filter_gen_from_seq(seq);
2851
2852 filter = READ_ONCE(mm_state->filters[gen]);
2853 if (!filter)
2854 return true;
2855
2856 get_item_key(item, key);
2857
2858 return test_bit(key[0], filter) && test_bit(key[1], filter);
2859}
2860
2861static void update_bloom_filter(struct lru_gen_mm_state *mm_state, unsigned long seq,
2862 void *item)
2863{
2864 int key[2];
2865 unsigned long *filter;
2866 int gen = filter_gen_from_seq(seq);
2867
2868 filter = READ_ONCE(mm_state->filters[gen]);
2869 if (!filter)
2870 return;
2871
2872 get_item_key(item, key);
2873
2874 if (!test_bit(key[0], filter))
2875 set_bit(key[0], filter);
2876 if (!test_bit(key[1], filter))
2877 set_bit(key[1], filter);
2878}
2879
2880static void reset_bloom_filter(struct lru_gen_mm_state *mm_state, unsigned long seq)
2881{
2882 unsigned long *filter;
2883 int gen = filter_gen_from_seq(seq);
2884
2885 filter = mm_state->filters[gen];
2886 if (filter) {
2887 bitmap_clear(filter, 0, BIT(BLOOM_FILTER_SHIFT));
2888 return;
2889 }
2890
2891 filter = bitmap_zalloc(BIT(BLOOM_FILTER_SHIFT),
2892 __GFP_HIGH | __GFP_NOMEMALLOC | __GFP_NOWARN);
2893 WRITE_ONCE(mm_state->filters[gen], filter);
2894}
2895
2896/******************************************************************************
2897 * mm_struct list
2898 ******************************************************************************/
2899
2900#ifdef CONFIG_LRU_GEN_WALKS_MMU
2901
2902static struct lru_gen_mm_list *get_mm_list(struct mem_cgroup *memcg)
2903{
2904 static struct lru_gen_mm_list mm_list = {
2905 .fifo = LIST_HEAD_INIT(mm_list.fifo),
2906 .lock = __SPIN_LOCK_UNLOCKED(mm_list.lock),
2907 };
2908
2909#ifdef CONFIG_MEMCG
2910 if (memcg)
2911 return &memcg->mm_list;
2912#endif
2913 VM_WARN_ON_ONCE(!mem_cgroup_disabled());
2914
2915 return &mm_list;
2916}
2917
2918static struct lru_gen_mm_state *get_mm_state(struct lruvec *lruvec)
2919{
2920 return &lruvec->mm_state;
2921}
2922
2923static struct mm_struct *get_next_mm(struct lru_gen_mm_walk *walk)
2924{
2925 int key;
2926 struct mm_struct *mm;
2927 struct pglist_data *pgdat = lruvec_pgdat(walk->lruvec);
2928 struct lru_gen_mm_state *mm_state = get_mm_state(walk->lruvec);
2929
2930 mm = list_entry(mm_state->head, struct mm_struct, lru_gen.list);
2931 key = pgdat->node_id % BITS_PER_TYPE(mm->lru_gen.bitmap);
2932
2933 if (!walk->force_scan && !test_bit(key, &mm->lru_gen.bitmap))
2934 return NULL;
2935
2936 clear_bit(key, &mm->lru_gen.bitmap);
2937 mmgrab(mm);
2938
2939 return mm;
2940}
2941
2942void lru_gen_add_mm(struct mm_struct *mm)
2943{
2944 int nid;
2945 struct mem_cgroup *memcg = get_mem_cgroup_from_mm(mm);
2946 struct lru_gen_mm_list *mm_list = get_mm_list(memcg);
2947
2948 VM_WARN_ON_ONCE(!list_empty(&mm->lru_gen.list));
2949#ifdef CONFIG_MEMCG
2950 VM_WARN_ON_ONCE(mm->lru_gen.memcg);
2951 mm->lru_gen.memcg = memcg;
2952#endif
2953 spin_lock(&mm_list->lock);
2954
2955 for_each_node_state(nid, N_MEMORY) {
2956 struct lruvec *lruvec = get_lruvec(memcg, nid);
2957 struct lru_gen_mm_state *mm_state = get_mm_state(lruvec);
2958
2959 /* the first addition since the last iteration */
2960 if (mm_state->tail == &mm_list->fifo)
2961 mm_state->tail = &mm->lru_gen.list;
2962 }
2963
2964 list_add_tail(&mm->lru_gen.list, &mm_list->fifo);
2965
2966 spin_unlock(&mm_list->lock);
2967}
2968
2969void lru_gen_del_mm(struct mm_struct *mm)
2970{
2971 int nid;
2972 struct lru_gen_mm_list *mm_list;
2973 struct mem_cgroup *memcg = NULL;
2974
2975 if (list_empty(&mm->lru_gen.list))
2976 return;
2977
2978#ifdef CONFIG_MEMCG
2979 memcg = mm->lru_gen.memcg;
2980#endif
2981 mm_list = get_mm_list(memcg);
2982
2983 spin_lock(&mm_list->lock);
2984
2985 for_each_node(nid) {
2986 struct lruvec *lruvec = get_lruvec(memcg, nid);
2987 struct lru_gen_mm_state *mm_state = get_mm_state(lruvec);
2988
2989 /* where the current iteration continues after */
2990 if (mm_state->head == &mm->lru_gen.list)
2991 mm_state->head = mm_state->head->prev;
2992
2993 /* where the last iteration ended before */
2994 if (mm_state->tail == &mm->lru_gen.list)
2995 mm_state->tail = mm_state->tail->next;
2996 }
2997
2998 list_del_init(&mm->lru_gen.list);
2999
3000 spin_unlock(&mm_list->lock);
3001
3002#ifdef CONFIG_MEMCG
3003 mem_cgroup_put(mm->lru_gen.memcg);
3004 mm->lru_gen.memcg = NULL;
3005#endif
3006}
3007
3008#ifdef CONFIG_MEMCG
3009void lru_gen_migrate_mm(struct mm_struct *mm)
3010{
3011 struct mem_cgroup *memcg;
3012 struct task_struct *task = rcu_dereference_protected(mm->owner, true);
3013
3014 VM_WARN_ON_ONCE(task->mm != mm);
3015 lockdep_assert_held(&task->alloc_lock);
3016
3017 /* for mm_update_next_owner() */
3018 if (mem_cgroup_disabled())
3019 return;
3020
3021 /* migration can happen before addition */
3022 if (!mm->lru_gen.memcg)
3023 return;
3024
3025 rcu_read_lock();
3026 memcg = mem_cgroup_from_task(task);
3027 rcu_read_unlock();
3028 if (memcg == mm->lru_gen.memcg)
3029 return;
3030
3031 VM_WARN_ON_ONCE(list_empty(&mm->lru_gen.list));
3032
3033 lru_gen_del_mm(mm);
3034 lru_gen_add_mm(mm);
3035}
3036#endif
3037
3038#else /* !CONFIG_LRU_GEN_WALKS_MMU */
3039
3040static struct lru_gen_mm_list *get_mm_list(struct mem_cgroup *memcg)
3041{
3042 return NULL;
3043}
3044
3045static struct lru_gen_mm_state *get_mm_state(struct lruvec *lruvec)
3046{
3047 return NULL;
3048}
3049
3050static struct mm_struct *get_next_mm(struct lru_gen_mm_walk *walk)
3051{
3052 return NULL;
3053}
3054
3055#endif
3056
3057static void reset_mm_stats(struct lru_gen_mm_walk *walk, bool last)
3058{
3059 int i;
3060 int hist;
3061 struct lruvec *lruvec = walk->lruvec;
3062 struct lru_gen_mm_state *mm_state = get_mm_state(lruvec);
3063
3064 lockdep_assert_held(&get_mm_list(lruvec_memcg(lruvec))->lock);
3065
3066 hist = lru_hist_from_seq(walk->seq);
3067
3068 for (i = 0; i < NR_MM_STATS; i++) {
3069 WRITE_ONCE(mm_state->stats[hist][i],
3070 mm_state->stats[hist][i] + walk->mm_stats[i]);
3071 walk->mm_stats[i] = 0;
3072 }
3073
3074 if (NR_HIST_GENS > 1 && last) {
3075 hist = lru_hist_from_seq(walk->seq + 1);
3076
3077 for (i = 0; i < NR_MM_STATS; i++)
3078 WRITE_ONCE(mm_state->stats[hist][i], 0);
3079 }
3080}
3081
3082static bool iterate_mm_list(struct lru_gen_mm_walk *walk, struct mm_struct **iter)
3083{
3084 bool first = false;
3085 bool last = false;
3086 struct mm_struct *mm = NULL;
3087 struct lruvec *lruvec = walk->lruvec;
3088 struct mem_cgroup *memcg = lruvec_memcg(lruvec);
3089 struct lru_gen_mm_list *mm_list = get_mm_list(memcg);
3090 struct lru_gen_mm_state *mm_state = get_mm_state(lruvec);
3091
3092 /*
3093 * mm_state->seq is incremented after each iteration of mm_list. There
3094 * are three interesting cases for this page table walker:
3095 * 1. It tries to start a new iteration with a stale max_seq: there is
3096 * nothing left to do.
3097 * 2. It started the next iteration: it needs to reset the Bloom filter
3098 * so that a fresh set of PTE tables can be recorded.
3099 * 3. It ended the current iteration: it needs to reset the mm stats
3100 * counters and tell its caller to increment max_seq.
3101 */
3102 spin_lock(&mm_list->lock);
3103
3104 VM_WARN_ON_ONCE(mm_state->seq + 1 < walk->seq);
3105
3106 if (walk->seq <= mm_state->seq)
3107 goto done;
3108
3109 if (!mm_state->head)
3110 mm_state->head = &mm_list->fifo;
3111
3112 if (mm_state->head == &mm_list->fifo)
3113 first = true;
3114
3115 do {
3116 mm_state->head = mm_state->head->next;
3117 if (mm_state->head == &mm_list->fifo) {
3118 WRITE_ONCE(mm_state->seq, mm_state->seq + 1);
3119 last = true;
3120 break;
3121 }
3122
3123 /* force scan for those added after the last iteration */
3124 if (!mm_state->tail || mm_state->tail == mm_state->head) {
3125 mm_state->tail = mm_state->head->next;
3126 walk->force_scan = true;
3127 }
3128 } while (!(mm = get_next_mm(walk)));
3129done:
3130 if (*iter || last)
3131 reset_mm_stats(walk, last);
3132
3133 spin_unlock(&mm_list->lock);
3134
3135 if (mm && first)
3136 reset_bloom_filter(mm_state, walk->seq + 1);
3137
3138 if (*iter)
3139 mmdrop(*iter);
3140
3141 *iter = mm;
3142
3143 return last;
3144}
3145
3146static bool iterate_mm_list_nowalk(struct lruvec *lruvec, unsigned long seq)
3147{
3148 bool success = false;
3149 struct mem_cgroup *memcg = lruvec_memcg(lruvec);
3150 struct lru_gen_mm_list *mm_list = get_mm_list(memcg);
3151 struct lru_gen_mm_state *mm_state = get_mm_state(lruvec);
3152
3153 spin_lock(&mm_list->lock);
3154
3155 VM_WARN_ON_ONCE(mm_state->seq + 1 < seq);
3156
3157 if (seq > mm_state->seq) {
3158 mm_state->head = NULL;
3159 mm_state->tail = NULL;
3160 WRITE_ONCE(mm_state->seq, mm_state->seq + 1);
3161 success = true;
3162 }
3163
3164 spin_unlock(&mm_list->lock);
3165
3166 return success;
3167}
3168
3169/******************************************************************************
3170 * PID controller
3171 ******************************************************************************/
3172
3173/*
3174 * A feedback loop based on Proportional-Integral-Derivative (PID) controller.
3175 *
3176 * The P term is refaulted/(evicted+protected) from a tier in the generation
3177 * currently being evicted; the I term is the exponential moving average of the
3178 * P term over the generations previously evicted, using the smoothing factor
3179 * 1/2; the D term isn't supported.
3180 *
3181 * The setpoint (SP) is always the first tier of one type; the process variable
3182 * (PV) is either any tier of the other type or any other tier of the same
3183 * type.
3184 *
3185 * The error is the difference between the SP and the PV; the correction is to
3186 * turn off protection when SP>PV or turn on protection when SP<PV.
3187 *
3188 * For future optimizations:
3189 * 1. The D term may discount the other two terms over time so that long-lived
3190 * generations can resist stale information.
3191 */
3192struct ctrl_pos {
3193 unsigned long refaulted;
3194 unsigned long total;
3195 int gain;
3196};
3197
3198static void read_ctrl_pos(struct lruvec *lruvec, int type, int tier, int gain,
3199 struct ctrl_pos *pos)
3200{
3201 int i;
3202 struct lru_gen_folio *lrugen = &lruvec->lrugen;
3203 int hist = lru_hist_from_seq(lrugen->min_seq[type]);
3204
3205 pos->gain = gain;
3206 pos->refaulted = pos->total = 0;
3207
3208 for (i = tier % MAX_NR_TIERS; i <= min(tier, MAX_NR_TIERS - 1); i++) {
3209 pos->refaulted += lrugen->avg_refaulted[type][i] +
3210 atomic_long_read(&lrugen->refaulted[hist][type][i]);
3211 pos->total += lrugen->avg_total[type][i] +
3212 lrugen->protected[hist][type][i] +
3213 atomic_long_read(&lrugen->evicted[hist][type][i]);
3214 }
3215}
3216
3217static void reset_ctrl_pos(struct lruvec *lruvec, int type, bool carryover)
3218{
3219 int hist, tier;
3220 struct lru_gen_folio *lrugen = &lruvec->lrugen;
3221 bool clear = carryover ? NR_HIST_GENS == 1 : NR_HIST_GENS > 1;
3222 unsigned long seq = carryover ? lrugen->min_seq[type] : lrugen->max_seq + 1;
3223
3224 lockdep_assert_held(&lruvec->lru_lock);
3225
3226 if (!carryover && !clear)
3227 return;
3228
3229 hist = lru_hist_from_seq(seq);
3230
3231 for (tier = 0; tier < MAX_NR_TIERS; tier++) {
3232 if (carryover) {
3233 unsigned long sum;
3234
3235 sum = lrugen->avg_refaulted[type][tier] +
3236 atomic_long_read(&lrugen->refaulted[hist][type][tier]);
3237 WRITE_ONCE(lrugen->avg_refaulted[type][tier], sum / 2);
3238
3239 sum = lrugen->avg_total[type][tier] +
3240 lrugen->protected[hist][type][tier] +
3241 atomic_long_read(&lrugen->evicted[hist][type][tier]);
3242 WRITE_ONCE(lrugen->avg_total[type][tier], sum / 2);
3243 }
3244
3245 if (clear) {
3246 atomic_long_set(&lrugen->refaulted[hist][type][tier], 0);
3247 atomic_long_set(&lrugen->evicted[hist][type][tier], 0);
3248 WRITE_ONCE(lrugen->protected[hist][type][tier], 0);
3249 }
3250 }
3251}
3252
3253static bool positive_ctrl_err(struct ctrl_pos *sp, struct ctrl_pos *pv)
3254{
3255 /*
3256 * Return true if the PV has a limited number of refaults or a lower
3257 * refaulted/total than the SP.
3258 */
3259 return pv->refaulted < MIN_LRU_BATCH ||
3260 pv->refaulted * (sp->total + MIN_LRU_BATCH) * sp->gain <=
3261 (sp->refaulted + 1) * pv->total * pv->gain;
3262}
3263
3264/******************************************************************************
3265 * the aging
3266 ******************************************************************************/
3267
3268/* promote pages accessed through page tables */
3269static int folio_update_gen(struct folio *folio, int gen, const vma_flags_t *vma_flags)
3270{
3271 unsigned long new_flags, old_flags = READ_ONCE(folio->flags.f);
3272
3273 VM_WARN_ON_ONCE(gen >= MAX_NR_GENS);
3274
3275 /*
3276 * See the comment on LRU_REFS_FLAGS, and activate file-backed
3277 * executable folios after first usage to avoid typical IO
3278 * thrashing from reclaiming.
3279 */
3280 if (!folio_test_referenced(folio) && !folio_test_workingset(folio) &&
3281 !is_exec_file_folio(folio, vma_flags)) {
3282 set_mask_bits(&folio->flags.f, LRU_REFS_MASK, BIT(PG_referenced));
3283 return -1;
3284 }
3285
3286 do {
3287 /* lru_gen_del_folio() has isolated this page? */
3288 if (!(old_flags & LRU_GEN_MASK))
3289 return -1;
3290
3291 new_flags = old_flags & ~(LRU_GEN_MASK | LRU_REFS_FLAGS);
3292 new_flags |= ((gen + 1UL) << LRU_GEN_PGOFF) | BIT(PG_workingset);
3293 } while (!try_cmpxchg(&folio->flags.f, &old_flags, new_flags));
3294
3295 return ((old_flags & LRU_GEN_MASK) >> LRU_GEN_PGOFF) - 1;
3296}
3297
3298/* protect pages accessed multiple times through file descriptors */
3299static int folio_inc_gen(struct lruvec *lruvec, struct folio *folio)
3300{
3301 int type = folio_is_file_lru(folio);
3302 struct lru_gen_folio *lrugen = &lruvec->lrugen;
3303 int new_gen, old_gen = lru_gen_from_seq(lrugen->min_seq[type]);
3304 unsigned long new_flags, old_flags = READ_ONCE(folio->flags.f);
3305
3306 VM_WARN_ON_ONCE_FOLIO(!(old_flags & LRU_GEN_MASK), folio);
3307
3308 do {
3309 new_gen = ((old_flags & LRU_GEN_MASK) >> LRU_GEN_PGOFF) - 1;
3310 /* folio_update_gen() has promoted this page? */
3311 if (new_gen >= 0 && new_gen != old_gen)
3312 return new_gen;
3313
3314 new_gen = (old_gen + 1) % MAX_NR_GENS;
3315
3316 new_flags = old_flags & ~(LRU_GEN_MASK | LRU_REFS_FLAGS);
3317 new_flags |= (new_gen + 1UL) << LRU_GEN_PGOFF;
3318 } while (!try_cmpxchg(&folio->flags.f, &old_flags, new_flags));
3319
3320 lru_gen_update_size(lruvec, folio, old_gen, new_gen);
3321
3322 return new_gen;
3323}
3324
3325static void update_batch_size(struct lru_gen_mm_walk *walk, struct folio *folio,
3326 int old_gen, int new_gen)
3327{
3328 int type = folio_is_file_lru(folio);
3329 int zone = folio_zonenum(folio);
3330 int delta = folio_nr_pages(folio);
3331
3332 VM_WARN_ON_ONCE(old_gen >= MAX_NR_GENS);
3333 VM_WARN_ON_ONCE(new_gen >= MAX_NR_GENS);
3334
3335 walk->batched++;
3336
3337 walk->nr_pages[old_gen][type][zone] -= delta;
3338 walk->nr_pages[new_gen][type][zone] += delta;
3339}
3340
3341static void reset_batch_size(struct lru_gen_mm_walk *walk)
3342{
3343 int gen, type, zone;
3344 struct lruvec *lruvec = lruvec_live_lock_irq(walk->lruvec);
3345 struct lru_gen_folio *lrugen = &lruvec->lrugen;
3346
3347 walk->batched = 0;
3348
3349 for_each_gen_type_zone(gen, type, zone) {
3350 enum lru_list lru = type * LRU_INACTIVE_FILE;
3351 int delta = walk->nr_pages[gen][type][zone];
3352
3353 if (!delta)
3354 continue;
3355
3356 walk->nr_pages[gen][type][zone] = 0;
3357 WRITE_ONCE(lrugen->nr_pages[gen][type][zone],
3358 lrugen->nr_pages[gen][type][zone] + delta);
3359
3360 if (lru_gen_is_active(lruvec, gen))
3361 lru += LRU_ACTIVE;
3362 __update_lru_size(lruvec, lru, zone, delta);
3363 }
3364
3365 lruvec_unlock_irq(lruvec);
3366}
3367
3368static int should_skip_vma(unsigned long start, unsigned long end, struct mm_walk *args)
3369{
3370 struct address_space *mapping;
3371 struct vm_area_struct *vma = args->vma;
3372 struct lru_gen_mm_walk *walk = args->private;
3373
3374 if (!vma_is_accessible(vma))
3375 return true;
3376
3377 if (is_vm_hugetlb_page(vma))
3378 return true;
3379
3380 if (!vma_has_recency(vma))
3381 return true;
3382
3383 if (vma->vm_flags & (VM_LOCKED | VM_SPECIAL))
3384 return true;
3385
3386 if (vma == get_gate_vma(vma->vm_mm))
3387 return true;
3388
3389 if (vma_is_anonymous(vma))
3390 return !walk->swappiness;
3391
3392 if (WARN_ON_ONCE(!vma->vm_file || !vma->vm_file->f_mapping))
3393 return true;
3394
3395 mapping = vma->vm_file->f_mapping;
3396 if (mapping_unevictable(mapping))
3397 return true;
3398
3399 if (shmem_mapping(mapping))
3400 return !walk->swappiness;
3401
3402 if (walk->swappiness > MAX_SWAPPINESS)
3403 return true;
3404
3405 /* to exclude special mappings like dax, etc. */
3406 return !mapping->a_ops->read_folio;
3407}
3408
3409/*
3410 * Some userspace memory allocators map many single-page VMAs. Instead of
3411 * returning back to the PGD table for each of such VMAs, finish an entire PMD
3412 * table to reduce zigzags and improve cache performance.
3413 */
3414static bool get_next_vma(unsigned long mask, unsigned long size, struct mm_walk *args,
3415 unsigned long *vm_start, unsigned long *vm_end)
3416{
3417 unsigned long start = round_up(*vm_end, size);
3418 unsigned long end = (start | ~mask) + 1;
3419 VMA_ITERATOR(vmi, args->mm, start);
3420
3421 VM_WARN_ON_ONCE(mask & size);
3422 VM_WARN_ON_ONCE((start & mask) != (*vm_start & mask));
3423
3424 for_each_vma(vmi, args->vma) {
3425 if (end && end <= args->vma->vm_start)
3426 return false;
3427
3428 if (should_skip_vma(args->vma->vm_start, args->vma->vm_end, args))
3429 continue;
3430
3431 *vm_start = max(start, args->vma->vm_start);
3432 *vm_end = min(end - 1, args->vma->vm_end - 1) + 1;
3433
3434 return true;
3435 }
3436
3437 return false;
3438}
3439
3440static unsigned long get_pte_pfn(pte_t pte, struct vm_area_struct *vma, unsigned long addr,
3441 struct pglist_data *pgdat)
3442{
3443 unsigned long pfn = pte_pfn(pte);
3444
3445 VM_WARN_ON_ONCE(addr < vma->vm_start || addr >= vma->vm_end);
3446
3447 if (!pte_present(pte) || is_zero_pfn(pfn))
3448 return -1;
3449
3450 if (WARN_ON_ONCE(pte_special(pte)))
3451 return -1;
3452
3453 if (!pte_young(pte) && !mm_has_notifiers(vma->vm_mm))
3454 return -1;
3455
3456 if (WARN_ON_ONCE(!pfn_valid(pfn)))
3457 return -1;
3458
3459 if (pfn < pgdat->node_start_pfn || pfn >= pgdat_end_pfn(pgdat))
3460 return -1;
3461
3462 return pfn;
3463}
3464
3465static unsigned long get_pmd_pfn(pmd_t pmd, struct vm_area_struct *vma, unsigned long addr,
3466 struct pglist_data *pgdat)
3467{
3468 unsigned long pfn = pmd_pfn(pmd);
3469
3470 VM_WARN_ON_ONCE(addr < vma->vm_start || addr >= vma->vm_end);
3471
3472 if (!pmd_present(pmd) || is_huge_zero_pmd(pmd))
3473 return -1;
3474
3475 if (!pmd_young(pmd) && !mm_has_notifiers(vma->vm_mm))
3476 return -1;
3477
3478 if (WARN_ON_ONCE(!pfn_valid(pfn)))
3479 return -1;
3480
3481 if (pfn < pgdat->node_start_pfn || pfn >= pgdat_end_pfn(pgdat))
3482 return -1;
3483
3484 return pfn;
3485}
3486
3487static struct folio *get_pfn_folio(unsigned long pfn, struct mem_cgroup *memcg,
3488 struct pglist_data *pgdat)
3489{
3490 struct folio *folio = pfn_folio(pfn);
3491
3492 if (folio_lru_gen(folio) < 0)
3493 return NULL;
3494
3495 if (folio_nid(folio) != pgdat->node_id)
3496 return NULL;
3497
3498 rcu_read_lock();
3499 if (folio_memcg(folio) != memcg)
3500 folio = NULL;
3501 rcu_read_unlock();
3502
3503 return folio;
3504}
3505
3506static bool suitable_to_scan(int total, int young)
3507{
3508 int n = clamp_t(int, cache_line_size() / sizeof(pte_t), 2, 8);
3509
3510 /* suitable if the average number of young PTEs per cacheline is >=1 */
3511 return young * n >= total;
3512}
3513
3514static void walk_update_folio(struct lru_gen_mm_walk *walk, struct vm_area_struct *vma,
3515 struct folio *folio, int new_gen, bool dirty)
3516{
3517 int old_gen;
3518
3519 if (!folio)
3520 return;
3521
3522 if (dirty && !folio_test_dirty(folio) &&
3523 !(folio_test_anon(folio) && folio_test_swapbacked(folio) &&
3524 !folio_test_swapcache(folio)))
3525 folio_mark_dirty(folio);
3526
3527 if (walk) {
3528 old_gen = folio_update_gen(folio, new_gen, &vma->flags);
3529 if (old_gen >= 0 && old_gen != new_gen)
3530 update_batch_size(walk, folio, old_gen, new_gen);
3531 } else if (lru_gen_set_refs(folio, &vma->flags)) {
3532 old_gen = folio_lru_gen(folio);
3533 if (old_gen >= 0 && old_gen != new_gen)
3534 folio_activate(folio);
3535 }
3536}
3537
3538static bool walk_pte_range(pmd_t *pmd, unsigned long start, unsigned long end,
3539 struct mm_walk *args)
3540{
3541 int i;
3542 bool dirty;
3543 pte_t *pte;
3544 spinlock_t *ptl;
3545 unsigned long addr;
3546 int total = 0;
3547 int young = 0;
3548 struct folio *last = NULL;
3549 struct lru_gen_mm_walk *walk = args->private;
3550 struct mem_cgroup *memcg = lruvec_memcg(walk->lruvec);
3551 struct pglist_data *pgdat = lruvec_pgdat(walk->lruvec);
3552 DEFINE_MAX_SEQ(walk->lruvec);
3553 int gen = lru_gen_from_seq(max_seq);
3554 unsigned int nr;
3555 pmd_t pmdval;
3556
3557 pte = pte_offset_map_rw_nolock(args->mm, pmd, start & PMD_MASK, &pmdval, &ptl);
3558 if (!pte)
3559 return false;
3560
3561 if (!spin_trylock(ptl)) {
3562 pte_unmap(pte);
3563 return true;
3564 }
3565
3566 if (unlikely(!pmd_same(pmdval, pmdp_get_lockless(pmd)))) {
3567 pte_unmap_unlock(pte, ptl);
3568 return false;
3569 }
3570
3571 lazy_mmu_mode_enable();
3572restart:
3573 for (i = pte_index(start), addr = start; addr != end; i += nr, addr += nr * PAGE_SIZE) {
3574 unsigned long pfn;
3575 struct folio *folio;
3576 pte_t *cur_pte = pte + i;
3577 pte_t ptent = ptep_get(cur_pte);
3578
3579 nr = 1;
3580 total++;
3581 walk->mm_stats[MM_LEAF_TOTAL]++;
3582
3583 pfn = get_pte_pfn(ptent, args->vma, addr, pgdat);
3584 if (pfn == -1)
3585 continue;
3586
3587 folio = get_pfn_folio(pfn, memcg, pgdat);
3588 if (!folio)
3589 continue;
3590
3591 if (folio_test_large(folio)) {
3592 const unsigned int max_nr = (end - addr) >> PAGE_SHIFT;
3593
3594 nr = folio_pte_batch_flags(folio, NULL, cur_pte, &ptent,
3595 max_nr, FPB_MERGE_YOUNG_DIRTY);
3596 total += nr - 1;
3597 walk->mm_stats[MM_LEAF_TOTAL] += nr - 1;
3598 }
3599
3600 if (!test_and_clear_young_ptes_notify(args->vma, addr, cur_pte, nr))
3601 continue;
3602
3603 if (last != folio) {
3604 walk_update_folio(walk, args->vma, last, gen, dirty);
3605
3606 last = folio;
3607 dirty = false;
3608 }
3609
3610 if (pte_dirty(ptent))
3611 dirty = true;
3612
3613 young += nr;
3614 walk->mm_stats[MM_LEAF_YOUNG] += nr;
3615 }
3616
3617 walk_update_folio(walk, args->vma, last, gen, dirty);
3618 last = NULL;
3619
3620 if (i < PTRS_PER_PTE && get_next_vma(PMD_MASK, PAGE_SIZE, args, &start, &end))
3621 goto restart;
3622
3623 lazy_mmu_mode_disable();
3624 pte_unmap_unlock(pte, ptl);
3625
3626 return suitable_to_scan(total, young);
3627}
3628
3629static void walk_pmd_range_locked(pud_t *pud, unsigned long addr, struct vm_area_struct *vma,
3630 struct mm_walk *args, unsigned long *bitmap, unsigned long *first)
3631{
3632 int i;
3633 bool dirty;
3634 pmd_t *pmd;
3635 spinlock_t *ptl;
3636 struct folio *last = NULL;
3637 struct lru_gen_mm_walk *walk = args->private;
3638 struct mem_cgroup *memcg = lruvec_memcg(walk->lruvec);
3639 struct pglist_data *pgdat = lruvec_pgdat(walk->lruvec);
3640 DEFINE_MAX_SEQ(walk->lruvec);
3641 int gen = lru_gen_from_seq(max_seq);
3642
3643 VM_WARN_ON_ONCE(pud_leaf(*pud));
3644
3645 /* try to batch at most 1+MIN_LRU_BATCH+1 entries */
3646 if (*first == -1) {
3647 *first = addr;
3648 bitmap_zero(bitmap, MIN_LRU_BATCH);
3649 return;
3650 }
3651
3652 i = addr == -1 ? 0 : pmd_index(addr) - pmd_index(*first);
3653 if (i && i <= MIN_LRU_BATCH) {
3654 __set_bit(i - 1, bitmap);
3655 return;
3656 }
3657
3658 pmd = pmd_offset(pud, *first);
3659
3660 ptl = pmd_lockptr(args->mm, pmd);
3661 if (!spin_trylock(ptl))
3662 goto done;
3663
3664 lazy_mmu_mode_enable();
3665
3666 do {
3667 unsigned long pfn;
3668 struct folio *folio;
3669
3670 /* don't round down the first address */
3671 addr = i ? (*first & PMD_MASK) + i * PMD_SIZE : *first;
3672
3673 if (!pmd_present(pmd[i]))
3674 goto next;
3675
3676 if (!pmd_trans_huge(pmd[i])) {
3677 if (!walk->force_scan && should_clear_pmd_young() &&
3678 !mm_has_notifiers(args->mm))
3679 pmdp_test_and_clear_young(vma, addr, pmd + i);
3680 goto next;
3681 }
3682
3683 pfn = get_pmd_pfn(pmd[i], vma, addr, pgdat);
3684 if (pfn == -1)
3685 goto next;
3686
3687 folio = get_pfn_folio(pfn, memcg, pgdat);
3688 if (!folio)
3689 goto next;
3690
3691 if (!pmdp_test_and_clear_young_notify(vma, addr, pmd + i))
3692 goto next;
3693
3694 if (last != folio) {
3695 walk_update_folio(walk, vma, last, gen, dirty);
3696
3697 last = folio;
3698 dirty = false;
3699 }
3700
3701 if (pmd_dirty(pmd[i]))
3702 dirty = true;
3703
3704 walk->mm_stats[MM_LEAF_YOUNG]++;
3705next:
3706 i = i > MIN_LRU_BATCH ? 0 : find_next_bit(bitmap, MIN_LRU_BATCH, i) + 1;
3707 } while (i <= MIN_LRU_BATCH);
3708
3709 walk_update_folio(walk, vma, last, gen, dirty);
3710
3711 lazy_mmu_mode_disable();
3712 spin_unlock(ptl);
3713done:
3714 *first = -1;
3715}
3716
3717static void walk_pmd_range(pud_t *pud, unsigned long start, unsigned long end,
3718 struct mm_walk *args)
3719{
3720 int i;
3721 pmd_t *pmd;
3722 unsigned long next;
3723 unsigned long addr;
3724 struct vm_area_struct *vma;
3725 DECLARE_BITMAP(bitmap, MIN_LRU_BATCH);
3726 unsigned long first = -1;
3727 struct lru_gen_mm_walk *walk = args->private;
3728 struct lru_gen_mm_state *mm_state = get_mm_state(walk->lruvec);
3729
3730 VM_WARN_ON_ONCE(pud_leaf(*pud));
3731
3732 /*
3733 * Finish an entire PMD in two passes: the first only reaches to PTE
3734 * tables to avoid taking the PMD lock; the second, if necessary, takes
3735 * the PMD lock to clear the accessed bit in PMD entries.
3736 */
3737 pmd = pmd_offset(pud, start & PUD_MASK);
3738restart:
3739 /* walk_pte_range() may call get_next_vma() */
3740 vma = args->vma;
3741 for (i = pmd_index(start), addr = start; addr != end; i++, addr = next) {
3742 pmd_t val = pmdp_get_lockless(pmd + i);
3743
3744 next = pmd_addr_end(addr, end);
3745
3746 if (!pmd_present(val) || is_huge_zero_pmd(val)) {
3747 walk->mm_stats[MM_LEAF_TOTAL]++;
3748 continue;
3749 }
3750
3751 if (pmd_trans_huge(val)) {
3752 struct pglist_data *pgdat = lruvec_pgdat(walk->lruvec);
3753 unsigned long pfn = get_pmd_pfn(val, vma, addr, pgdat);
3754
3755 walk->mm_stats[MM_LEAF_TOTAL]++;
3756
3757 if (pfn != -1)
3758 walk_pmd_range_locked(pud, addr, vma, args, bitmap, &first);
3759 continue;
3760 }
3761
3762 if (!walk->force_scan && should_clear_pmd_young() &&
3763 !mm_has_notifiers(args->mm)) {
3764 if (!pmd_young(val))
3765 continue;
3766
3767 walk_pmd_range_locked(pud, addr, vma, args, bitmap, &first);
3768 }
3769
3770 if (!walk->force_scan && !test_bloom_filter(mm_state, walk->seq, pmd + i))
3771 continue;
3772
3773 walk->mm_stats[MM_NONLEAF_FOUND]++;
3774
3775 if (!walk_pte_range(&val, addr, next, args))
3776 continue;
3777
3778 walk->mm_stats[MM_NONLEAF_ADDED]++;
3779
3780 /* carry over to the next generation */
3781 update_bloom_filter(mm_state, walk->seq + 1, pmd + i);
3782 }
3783
3784 walk_pmd_range_locked(pud, -1, vma, args, bitmap, &first);
3785
3786 if (i < PTRS_PER_PMD && get_next_vma(PUD_MASK, PMD_SIZE, args, &start, &end))
3787 goto restart;
3788}
3789
3790static int walk_pud_range(p4d_t *p4d, unsigned long start, unsigned long end,
3791 struct mm_walk *args)
3792{
3793 int i;
3794 pud_t *pud;
3795 unsigned long addr;
3796 unsigned long next;
3797 struct lru_gen_mm_walk *walk = args->private;
3798
3799 VM_WARN_ON_ONCE(p4d_leaf(*p4d));
3800
3801 pud = pud_offset(p4d, start & P4D_MASK);
3802restart:
3803 for (i = pud_index(start), addr = start; addr != end; i++, addr = next) {
3804 pud_t val = pudp_get(pud + i);
3805
3806 next = pud_addr_end(addr, end);
3807
3808 if (!pud_present(val) || WARN_ON_ONCE(pud_leaf(val)))
3809 continue;
3810
3811 walk_pmd_range(&val, addr, next, args);
3812
3813 if (need_resched() || walk->batched >= MAX_LRU_BATCH) {
3814 end = (addr | ~PUD_MASK) + 1;
3815 goto done;
3816 }
3817 }
3818
3819 if (i < PTRS_PER_PUD && get_next_vma(P4D_MASK, PUD_SIZE, args, &start, &end))
3820 goto restart;
3821
3822 end = round_up(end, P4D_SIZE);
3823done:
3824 if (!end || !args->vma)
3825 return 1;
3826
3827 walk->next_addr = max(end, args->vma->vm_start);
3828
3829 return -EAGAIN;
3830}
3831
3832static void walk_mm(struct mm_struct *mm, struct lru_gen_mm_walk *walk)
3833{
3834 static const struct mm_walk_ops mm_walk_ops = {
3835 .test_walk = should_skip_vma,
3836 .p4d_entry = walk_pud_range,
3837 .walk_lock = PGWALK_RDLOCK,
3838 };
3839 int err;
3840 struct lruvec *lruvec = walk->lruvec;
3841
3842 walk->next_addr = FIRST_USER_ADDRESS;
3843
3844 do {
3845 DEFINE_MAX_SEQ(lruvec);
3846
3847 err = -EBUSY;
3848
3849 /* another thread might have called inc_max_seq() */
3850 if (walk->seq != max_seq)
3851 break;
3852
3853 /* the caller might be holding the lock for write */
3854 if (mmap_read_trylock(mm)) {
3855 err = walk_page_range(mm, walk->next_addr, ULONG_MAX, &mm_walk_ops, walk);
3856
3857 mmap_read_unlock(mm);
3858 }
3859
3860 if (walk->batched)
3861 reset_batch_size(walk);
3862
3863 cond_resched();
3864 } while (err == -EAGAIN);
3865}
3866
3867static struct lru_gen_mm_walk *set_mm_walk(struct pglist_data *pgdat, bool force_alloc)
3868{
3869 struct lru_gen_mm_walk *walk = current->reclaim_state->mm_walk;
3870
3871 if (pgdat && current_is_kswapd()) {
3872 VM_WARN_ON_ONCE(walk);
3873
3874 walk = &pgdat->mm_walk;
3875 } else if (!walk && force_alloc) {
3876 VM_WARN_ON_ONCE(current_is_kswapd());
3877
3878 walk = kzalloc_obj(*walk,
3879 __GFP_HIGH | __GFP_NOMEMALLOC | __GFP_NOWARN);
3880 }
3881
3882 current->reclaim_state->mm_walk = walk;
3883
3884 return walk;
3885}
3886
3887static void clear_mm_walk(void)
3888{
3889 struct lru_gen_mm_walk *walk = current->reclaim_state->mm_walk;
3890
3891 VM_WARN_ON_ONCE(walk && memchr_inv(walk->nr_pages, 0, sizeof(walk->nr_pages)));
3892 VM_WARN_ON_ONCE(walk && memchr_inv(walk->mm_stats, 0, sizeof(walk->mm_stats)));
3893
3894 current->reclaim_state->mm_walk = NULL;
3895
3896 if (!current_is_kswapd())
3897 kfree(walk);
3898}
3899
3900static bool inc_min_seq(struct lruvec *lruvec, int type, int swappiness)
3901{
3902 int zone;
3903 int remaining = MAX_LRU_BATCH;
3904 struct lru_gen_folio *lrugen = &lruvec->lrugen;
3905 int hist = lru_hist_from_seq(lrugen->min_seq[type]);
3906 int new_gen, old_gen = lru_gen_from_seq(lrugen->min_seq[type]);
3907
3908 /* For file type, skip the check if swappiness is anon only */
3909 if (type && (swappiness == SWAPPINESS_ANON_ONLY))
3910 goto done;
3911
3912 /* For anon type, skip the check if swappiness is zero (file only) */
3913 if (!type && !swappiness)
3914 goto done;
3915
3916 /* prevent cold/hot inversion if the type is evictable */
3917 for (zone = 0; zone < MAX_NR_ZONES; zone++) {
3918 struct list_head *head = &lrugen->folios[old_gen][type][zone];
3919
3920 while (!list_empty(head)) {
3921 struct folio *folio = lru_to_folio(head);
3922 int refs = folio_lru_refs(folio);
3923 bool workingset = folio_test_workingset(folio);
3924
3925 VM_WARN_ON_ONCE_FOLIO(folio_test_unevictable(folio), folio);
3926 VM_WARN_ON_ONCE_FOLIO(folio_test_active(folio), folio);
3927 VM_WARN_ON_ONCE_FOLIO(folio_is_file_lru(folio) != type, folio);
3928 VM_WARN_ON_ONCE_FOLIO(folio_zonenum(folio) != zone, folio);
3929
3930 new_gen = folio_inc_gen(lruvec, folio);
3931 list_move_tail(&folio->lru, &lrugen->folios[new_gen][type][zone]);
3932
3933 /* don't count the workingset being lazily promoted */
3934 if (refs + workingset != BIT(LRU_REFS_WIDTH) + 1) {
3935 int tier = lru_tier_from_refs(refs, workingset);
3936 int delta = folio_nr_pages(folio);
3937
3938 WRITE_ONCE(lrugen->protected[hist][type][tier],
3939 lrugen->protected[hist][type][tier] + delta);
3940 }
3941
3942 if (!--remaining)
3943 return false;
3944 }
3945 }
3946done:
3947 reset_ctrl_pos(lruvec, type, true);
3948 WRITE_ONCE(lrugen->min_seq[type], lrugen->min_seq[type] + 1);
3949
3950 return true;
3951}
3952
3953static void try_to_inc_min_seq(struct lruvec *lruvec, int swappiness)
3954{
3955 int gen, type, zone;
3956 bool seq_inc_flag = false;
3957 struct lru_gen_folio *lrugen = &lruvec->lrugen;
3958 DEFINE_MIN_SEQ(lruvec);
3959
3960 VM_WARN_ON_ONCE(!seq_is_valid(lruvec));
3961
3962 /* find the oldest populated generation */
3963 for_each_evictable_type(type, swappiness) {
3964 while (min_seq[type] + MIN_NR_GENS <= lrugen->max_seq) {
3965 gen = lru_gen_from_seq(min_seq[type]);
3966
3967 for (zone = 0; zone < MAX_NR_ZONES; zone++) {
3968 if (!list_empty(&lrugen->folios[gen][type][zone]))
3969 goto next;
3970 }
3971
3972 min_seq[type]++;
3973 seq_inc_flag = true;
3974 }
3975next:
3976 ;
3977 }
3978
3979 /*
3980 * If min_seq[type] of both anonymous and file is not increased,
3981 * return here to avoid unnecessary checking overhead later.
3982 */
3983 if (!seq_inc_flag)
3984 return;
3985
3986 /* see the comment on lru_gen_folio */
3987 if (swappiness && swappiness <= MAX_SWAPPINESS) {
3988 unsigned long seq = lrugen->max_seq - MIN_NR_GENS;
3989
3990 if (min_seq[LRU_GEN_ANON] > seq && min_seq[LRU_GEN_FILE] < seq)
3991 min_seq[LRU_GEN_ANON] = seq;
3992 else if (min_seq[LRU_GEN_FILE] > seq && min_seq[LRU_GEN_ANON] < seq)
3993 min_seq[LRU_GEN_FILE] = seq;
3994 }
3995
3996 for_each_evictable_type(type, swappiness) {
3997 if (min_seq[type] <= lrugen->min_seq[type])
3998 continue;
3999
4000 reset_ctrl_pos(lruvec, type, true);
4001 WRITE_ONCE(lrugen->min_seq[type], min_seq[type]);
4002 }
4003}
4004
4005static bool inc_max_seq(struct lruvec *lruvec, unsigned long seq, int swappiness)
4006{
4007 bool success;
4008 int prev, next;
4009 int type, zone;
4010 struct lru_gen_folio *lrugen = &lruvec->lrugen;
4011restart:
4012 if (seq < READ_ONCE(lrugen->max_seq))
4013 return false;
4014
4015 lruvec_lock_irq(lruvec);
4016
4017 VM_WARN_ON_ONCE(!seq_is_valid(lruvec));
4018
4019 success = seq == lrugen->max_seq;
4020 if (!success)
4021 goto unlock;
4022
4023 for (type = 0; type < ANON_AND_FILE; type++) {
4024 if (get_nr_gens(lruvec, type) != MAX_NR_GENS)
4025 continue;
4026
4027 if (inc_min_seq(lruvec, type, swappiness))
4028 continue;
4029
4030 lruvec_unlock_irq(lruvec);
4031 cond_resched();
4032 goto restart;
4033 }
4034
4035 /*
4036 * Update the active/inactive LRU sizes for compatibility. Both sides of
4037 * the current max_seq need to be covered, since max_seq+1 can overlap
4038 * with min_seq[LRU_GEN_ANON] if swapping is constrained. And if they do
4039 * overlap, cold/hot inversion happens.
4040 */
4041 prev = lru_gen_from_seq(lrugen->max_seq - 1);
4042 next = lru_gen_from_seq(lrugen->max_seq + 1);
4043
4044 for (type = 0; type < ANON_AND_FILE; type++) {
4045 for (zone = 0; zone < MAX_NR_ZONES; zone++) {
4046 enum lru_list lru = type * LRU_INACTIVE_FILE;
4047 long delta = lrugen->nr_pages[prev][type][zone] -
4048 lrugen->nr_pages[next][type][zone];
4049
4050 if (!delta)
4051 continue;
4052
4053 __update_lru_size(lruvec, lru, zone, delta);
4054 __update_lru_size(lruvec, lru + LRU_ACTIVE, zone, -delta);
4055 }
4056 }
4057
4058 for (type = 0; type < ANON_AND_FILE; type++)
4059 reset_ctrl_pos(lruvec, type, false);
4060
4061 WRITE_ONCE(lrugen->timestamps[next], jiffies);
4062 /* make sure preceding modifications appear */
4063 smp_store_release(&lrugen->max_seq, lrugen->max_seq + 1);
4064unlock:
4065 lruvec_unlock_irq(lruvec);
4066
4067 return success;
4068}
4069
4070static bool try_to_inc_max_seq(struct lruvec *lruvec, unsigned long seq,
4071 int swappiness, bool force_scan)
4072{
4073 bool success;
4074 struct lru_gen_mm_walk *walk;
4075 struct mm_struct *mm = NULL;
4076 struct lru_gen_folio *lrugen = &lruvec->lrugen;
4077 struct lru_gen_mm_state *mm_state = get_mm_state(lruvec);
4078
4079 VM_WARN_ON_ONCE(seq > READ_ONCE(lrugen->max_seq));
4080
4081 if (!mm_state)
4082 return inc_max_seq(lruvec, seq, swappiness);
4083
4084 /* see the comment in iterate_mm_list() */
4085 if (seq <= READ_ONCE(mm_state->seq))
4086 return false;
4087
4088 /*
4089 * If the hardware doesn't automatically set the accessed bit, fallback
4090 * to lru_gen_look_around(), which only clears the accessed bit in a
4091 * handful of PTEs. Spreading the work out over a period of time usually
4092 * is less efficient, but it avoids bursty page faults.
4093 */
4094 if (!should_walk_mmu()) {
4095 success = iterate_mm_list_nowalk(lruvec, seq);
4096 goto done;
4097 }
4098
4099 walk = set_mm_walk(NULL, true);
4100 if (!walk) {
4101 success = iterate_mm_list_nowalk(lruvec, seq);
4102 goto done;
4103 }
4104
4105 walk->lruvec = lruvec;
4106 walk->seq = seq;
4107 walk->swappiness = swappiness;
4108 walk->force_scan = force_scan;
4109
4110 do {
4111 success = iterate_mm_list(walk, &mm);
4112 if (mm)
4113 walk_mm(mm, walk);
4114 } while (mm);
4115done:
4116 if (success) {
4117 success = inc_max_seq(lruvec, seq, swappiness);
4118 WARN_ON_ONCE(!success);
4119 }
4120
4121 return success;
4122}
4123
4124/******************************************************************************
4125 * working set protection
4126 ******************************************************************************/
4127
4128static void set_initial_priority(struct pglist_data *pgdat, struct scan_control *sc)
4129{
4130 int priority;
4131 unsigned long reclaimable;
4132
4133 if (sc->priority != DEF_PRIORITY || sc->nr_to_reclaim < MIN_LRU_BATCH)
4134 return;
4135 /*
4136 * Determine the initial priority based on
4137 * (total >> priority) * reclaimed_to_scanned_ratio = nr_to_reclaim,
4138 * where reclaimed_to_scanned_ratio = inactive / total.
4139 */
4140 reclaimable = node_page_state(pgdat, NR_INACTIVE_FILE);
4141 if (can_reclaim_anon_pages(NULL, pgdat->node_id, sc))
4142 reclaimable += node_page_state(pgdat, NR_INACTIVE_ANON);
4143
4144 /* round down reclaimable and round up sc->nr_to_reclaim */
4145 priority = fls_long(reclaimable) - 1 - fls_long(sc->nr_to_reclaim - 1);
4146
4147 /*
4148 * The estimation is based on LRU pages only, so cap it to prevent
4149 * overshoots of shrinker objects by large margins.
4150 */
4151 sc->priority = clamp(priority, DEF_PRIORITY / 2, DEF_PRIORITY);
4152}
4153
4154static unsigned long lruvec_evictable_size(struct lruvec *lruvec, int swappiness)
4155{
4156 int gen, type, zone;
4157 unsigned long seq, total = 0;
4158 struct lru_gen_folio *lrugen = &lruvec->lrugen;
4159 DEFINE_MAX_SEQ(lruvec);
4160 DEFINE_MIN_SEQ(lruvec);
4161
4162 for_each_evictable_type(type, swappiness) {
4163 for (seq = min_seq[type]; seq <= max_seq; seq++) {
4164 gen = lru_gen_from_seq(seq);
4165 for (zone = 0; zone < MAX_NR_ZONES; zone++)
4166 total += max(READ_ONCE(lrugen->nr_pages[gen][type][zone]), 0L);
4167 }
4168 }
4169
4170 return total;
4171}
4172
4173static bool lruvec_is_sizable(struct lruvec *lruvec, struct scan_control *sc)
4174{
4175 unsigned long total;
4176 int swappiness = get_swappiness(lruvec, sc);
4177 struct mem_cgroup *memcg = lruvec_memcg(lruvec);
4178
4179 total = lruvec_evictable_size(lruvec, swappiness);
4180
4181 /* whether the size is big enough to be helpful */
4182 return mem_cgroup_online(memcg) ? (total >> sc->priority) : total;
4183}
4184
4185static bool lruvec_is_reclaimable(struct lruvec *lruvec, struct scan_control *sc,
4186 unsigned long min_ttl)
4187{
4188 int gen;
4189 unsigned long birth;
4190 int swappiness = get_swappiness(lruvec, sc);
4191 struct mem_cgroup *memcg = lruvec_memcg(lruvec);
4192 DEFINE_MIN_SEQ(lruvec);
4193
4194 if (mem_cgroup_below_min(NULL, memcg))
4195 return false;
4196
4197 if (!lruvec_is_sizable(lruvec, sc))
4198 return false;
4199
4200 gen = lru_gen_from_seq(evictable_min_seq(min_seq, swappiness));
4201 birth = READ_ONCE(lruvec->lrugen.timestamps[gen]);
4202
4203 return time_is_before_jiffies(birth + min_ttl);
4204}
4205
4206/* to protect the working set of the last N jiffies */
4207static unsigned long lru_gen_min_ttl __read_mostly;
4208
4209static void lru_gen_age_node(struct pglist_data *pgdat, struct scan_control *sc)
4210{
4211 struct mem_cgroup *memcg;
4212 unsigned long min_ttl = READ_ONCE(lru_gen_min_ttl);
4213 bool reclaimable = !min_ttl;
4214
4215 VM_WARN_ON_ONCE(!current_is_kswapd());
4216
4217 set_initial_priority(pgdat, sc);
4218
4219 memcg = mem_cgroup_iter(NULL, NULL, NULL);
4220 do {
4221 struct lruvec *lruvec = mem_cgroup_lruvec(memcg, pgdat);
4222
4223 mem_cgroup_calculate_protection(NULL, memcg);
4224
4225 if (!reclaimable)
4226 reclaimable = lruvec_is_reclaimable(lruvec, sc, min_ttl);
4227 } while ((memcg = mem_cgroup_iter(NULL, memcg, NULL)));
4228
4229 /*
4230 * The main goal is to OOM kill if every generation from all memcgs is
4231 * younger than min_ttl. However, another possibility is all memcgs are
4232 * either too small or below min.
4233 */
4234 if (!reclaimable && mutex_trylock(&oom_lock)) {
4235 struct oom_control oc = {
4236 .gfp_mask = sc->gfp_mask,
4237 };
4238
4239 out_of_memory(&oc);
4240
4241 mutex_unlock(&oom_lock);
4242 }
4243}
4244
4245/******************************************************************************
4246 * rmap/PT walk feedback
4247 ******************************************************************************/
4248
4249/*
4250 * This function exploits spatial locality when shrink_folio_list() walks the
4251 * rmap. It scans the adjacent PTEs of a young PTE and promotes hot pages. If
4252 * the scan was done cacheline efficiently, it adds the PMD entry pointing to
4253 * the PTE table to the Bloom filter. This forms a feedback loop between the
4254 * eviction and the aging.
4255 */
4256bool lru_gen_look_around(struct page_vma_mapped_walk *pvmw, unsigned int nr)
4257{
4258 int i;
4259 bool dirty;
4260 unsigned long start;
4261 unsigned long end;
4262 struct lru_gen_mm_walk *walk;
4263 struct folio *last = NULL;
4264 int young = nr;
4265 pte_t *pte = pvmw->pte;
4266 unsigned long addr = pvmw->address;
4267 struct vm_area_struct *vma = pvmw->vma;
4268 struct folio *folio = pfn_folio(pvmw->pfn);
4269 struct mem_cgroup *memcg;
4270 struct pglist_data *pgdat = folio_pgdat(folio);
4271 struct lruvec *lruvec;
4272 struct lru_gen_mm_state *mm_state;
4273 unsigned long max_seq;
4274 int gen;
4275
4276 lockdep_assert_held(pvmw->ptl);
4277 VM_WARN_ON_ONCE_FOLIO(folio_test_lru(folio), folio);
4278
4279 if (!test_and_clear_young_ptes_notify(vma, addr, pte, nr))
4280 return false;
4281
4282 if (spin_is_contended(pvmw->ptl))
4283 return true;
4284
4285 /* exclude special VMAs containing anon pages from COW */
4286 if (vma->vm_flags & VM_SPECIAL)
4287 return true;
4288
4289 /* avoid taking the LRU lock under the PTL when possible */
4290 walk = current->reclaim_state ? current->reclaim_state->mm_walk : NULL;
4291
4292 start = max(addr & PMD_MASK, vma->vm_start);
4293 end = min(addr | ~PMD_MASK, vma->vm_end - 1) + 1;
4294
4295 if (end - start == PAGE_SIZE)
4296 return true;
4297
4298 if (end - start > MIN_LRU_BATCH * PAGE_SIZE) {
4299 if (addr - start < MIN_LRU_BATCH * PAGE_SIZE / 2)
4300 end = start + MIN_LRU_BATCH * PAGE_SIZE;
4301 else if (end - addr < MIN_LRU_BATCH * PAGE_SIZE / 2)
4302 start = end - MIN_LRU_BATCH * PAGE_SIZE;
4303 else {
4304 start = addr - MIN_LRU_BATCH * PAGE_SIZE / 2;
4305 end = addr + MIN_LRU_BATCH * PAGE_SIZE / 2;
4306 }
4307 }
4308
4309 memcg = get_mem_cgroup_from_folio(folio);
4310 lruvec = mem_cgroup_lruvec(memcg, pgdat);
4311 max_seq = READ_ONCE((lruvec)->lrugen.max_seq);
4312 gen = lru_gen_from_seq(max_seq);
4313 mm_state = get_mm_state(lruvec);
4314
4315 lazy_mmu_mode_enable();
4316
4317 pte -= (addr - start) / PAGE_SIZE;
4318
4319 for (i = 0, addr = start; addr != end;
4320 i += nr, pte += nr, addr += nr * PAGE_SIZE) {
4321 unsigned long pfn;
4322 pte_t ptent = ptep_get(pte);
4323
4324 nr = 1;
4325 pfn = get_pte_pfn(ptent, vma, addr, pgdat);
4326 if (pfn == -1)
4327 continue;
4328
4329 folio = get_pfn_folio(pfn, memcg, pgdat);
4330 if (!folio)
4331 continue;
4332
4333 if (folio_test_large(folio)) {
4334 const unsigned int max_nr = (end - addr) >> PAGE_SHIFT;
4335
4336 nr = folio_pte_batch_flags(folio, NULL, pte, &ptent,
4337 max_nr, FPB_MERGE_YOUNG_DIRTY);
4338 }
4339
4340 if (!test_and_clear_young_ptes_notify(vma, addr, pte, nr))
4341 continue;
4342
4343 if (last != folio) {
4344 walk_update_folio(walk, vma, last, gen, dirty);
4345
4346 last = folio;
4347 dirty = false;
4348 }
4349
4350 if (pte_dirty(ptent))
4351 dirty = true;
4352
4353 young += nr;
4354 }
4355
4356 walk_update_folio(walk, vma, last, gen, dirty);
4357
4358 lazy_mmu_mode_disable();
4359
4360 /* feedback from rmap walkers to page table walkers */
4361 if (mm_state && suitable_to_scan(i, young))
4362 update_bloom_filter(mm_state, max_seq, pvmw->pmd);
4363
4364 mem_cgroup_put(memcg);
4365
4366 return true;
4367}
4368
4369/******************************************************************************
4370 * memcg LRU
4371 ******************************************************************************/
4372
4373/* see the comment on MEMCG_NR_GENS */
4374enum {
4375 MEMCG_LRU_NOP,
4376 MEMCG_LRU_HEAD,
4377 MEMCG_LRU_TAIL,
4378 MEMCG_LRU_OLD,
4379 MEMCG_LRU_YOUNG,
4380};
4381
4382static void lru_gen_rotate_memcg(struct lruvec *lruvec, int op)
4383{
4384 int seg;
4385 int old, new;
4386 unsigned long flags;
4387 int bin = get_random_u32_below(MEMCG_NR_BINS);
4388 struct pglist_data *pgdat = lruvec_pgdat(lruvec);
4389
4390 spin_lock_irqsave(&pgdat->memcg_lru.lock, flags);
4391
4392 VM_WARN_ON_ONCE(hlist_nulls_unhashed(&lruvec->lrugen.list));
4393
4394 seg = 0;
4395 new = old = lruvec->lrugen.gen;
4396
4397 /* see the comment on MEMCG_NR_GENS */
4398 if (op == MEMCG_LRU_HEAD)
4399 seg = MEMCG_LRU_HEAD;
4400 else if (op == MEMCG_LRU_TAIL)
4401 seg = MEMCG_LRU_TAIL;
4402 else if (op == MEMCG_LRU_OLD)
4403 new = get_memcg_gen(pgdat->memcg_lru.seq);
4404 else if (op == MEMCG_LRU_YOUNG)
4405 new = get_memcg_gen(pgdat->memcg_lru.seq + 1);
4406 else
4407 VM_WARN_ON_ONCE(true);
4408
4409 WRITE_ONCE(lruvec->lrugen.seg, seg);
4410 WRITE_ONCE(lruvec->lrugen.gen, new);
4411
4412 hlist_nulls_del_rcu(&lruvec->lrugen.list);
4413
4414 if (op == MEMCG_LRU_HEAD || op == MEMCG_LRU_OLD)
4415 hlist_nulls_add_head_rcu(&lruvec->lrugen.list, &pgdat->memcg_lru.fifo[new][bin]);
4416 else
4417 hlist_nulls_add_tail_rcu(&lruvec->lrugen.list, &pgdat->memcg_lru.fifo[new][bin]);
4418
4419 pgdat->memcg_lru.nr_memcgs[old]--;
4420 pgdat->memcg_lru.nr_memcgs[new]++;
4421
4422 if (!pgdat->memcg_lru.nr_memcgs[old] && old == get_memcg_gen(pgdat->memcg_lru.seq))
4423 WRITE_ONCE(pgdat->memcg_lru.seq, pgdat->memcg_lru.seq + 1);
4424
4425 spin_unlock_irqrestore(&pgdat->memcg_lru.lock, flags);
4426}
4427
4428#ifdef CONFIG_MEMCG
4429
4430void lru_gen_online_memcg(struct mem_cgroup *memcg)
4431{
4432 int gen;
4433 int nid;
4434 int bin = get_random_u32_below(MEMCG_NR_BINS);
4435
4436 for_each_node(nid) {
4437 struct pglist_data *pgdat = NODE_DATA(nid);
4438 struct lruvec *lruvec = get_lruvec(memcg, nid);
4439
4440 spin_lock_irq(&pgdat->memcg_lru.lock);
4441
4442 VM_WARN_ON_ONCE(!hlist_nulls_unhashed(&lruvec->lrugen.list));
4443
4444 gen = get_memcg_gen(pgdat->memcg_lru.seq);
4445
4446 lruvec->lrugen.gen = gen;
4447
4448 hlist_nulls_add_tail_rcu(&lruvec->lrugen.list, &pgdat->memcg_lru.fifo[gen][bin]);
4449 pgdat->memcg_lru.nr_memcgs[gen]++;
4450
4451 spin_unlock_irq(&pgdat->memcg_lru.lock);
4452 }
4453}
4454
4455void lru_gen_offline_memcg(struct mem_cgroup *memcg)
4456{
4457 int nid;
4458
4459 for_each_node(nid) {
4460 struct lruvec *lruvec = get_lruvec(memcg, nid);
4461
4462 lru_gen_rotate_memcg(lruvec, MEMCG_LRU_OLD);
4463 }
4464}
4465
4466void lru_gen_release_memcg(struct mem_cgroup *memcg)
4467{
4468 int gen;
4469 int nid;
4470
4471 for_each_node(nid) {
4472 struct pglist_data *pgdat = NODE_DATA(nid);
4473 struct lruvec *lruvec = get_lruvec(memcg, nid);
4474
4475 spin_lock_irq(&pgdat->memcg_lru.lock);
4476
4477 if (hlist_nulls_unhashed(&lruvec->lrugen.list))
4478 goto unlock;
4479
4480 gen = lruvec->lrugen.gen;
4481
4482 hlist_nulls_del_init_rcu(&lruvec->lrugen.list);
4483 pgdat->memcg_lru.nr_memcgs[gen]--;
4484
4485 if (!pgdat->memcg_lru.nr_memcgs[gen] && gen == get_memcg_gen(pgdat->memcg_lru.seq))
4486 WRITE_ONCE(pgdat->memcg_lru.seq, pgdat->memcg_lru.seq + 1);
4487unlock:
4488 spin_unlock_irq(&pgdat->memcg_lru.lock);
4489 }
4490}
4491
4492void lru_gen_soft_reclaim(struct mem_cgroup *memcg, int nid)
4493{
4494 struct lruvec *lruvec = get_lruvec(memcg, nid);
4495
4496 /* see the comment on MEMCG_NR_GENS */
4497 if (READ_ONCE(lruvec->lrugen.seg) != MEMCG_LRU_HEAD)
4498 lru_gen_rotate_memcg(lruvec, MEMCG_LRU_HEAD);
4499}
4500
4501bool recheck_lru_gen_max_memcg(struct mem_cgroup *memcg, int nid)
4502{
4503 struct lruvec *lruvec = get_lruvec(memcg, nid);
4504 int type;
4505
4506 for (type = 0; type < ANON_AND_FILE; type++) {
4507 if (get_nr_gens(lruvec, type) != MAX_NR_GENS)
4508 return false;
4509 }
4510
4511 return true;
4512}
4513
4514static void try_to_inc_max_seq_nowalk(struct mem_cgroup *memcg,
4515 struct lruvec *lruvec)
4516{
4517 struct lru_gen_mm_list *mm_list = get_mm_list(memcg);
4518 struct lru_gen_mm_state *mm_state = get_mm_state(lruvec);
4519 int swappiness = mem_cgroup_swappiness(memcg);
4520 DEFINE_MAX_SEQ(lruvec);
4521 bool success = false;
4522
4523 /*
4524 * We are not iterating the mm_list here, updating mm_state->seq is just
4525 * to make mm walkers work properly.
4526 */
4527 if (mm_state) {
4528 spin_lock(&mm_list->lock);
4529 VM_WARN_ON_ONCE(mm_state->seq + 1 < max_seq);
4530 if (max_seq > mm_state->seq) {
4531 WRITE_ONCE(mm_state->seq, mm_state->seq + 1);
4532 success = true;
4533 }
4534 spin_unlock(&mm_list->lock);
4535 } else {
4536 success = true;
4537 }
4538
4539 if (success)
4540 inc_max_seq(lruvec, max_seq, swappiness);
4541}
4542
4543/*
4544 * We need to ensure that the folios of child memcg can be reparented to the
4545 * same gen of the parent memcg, so the gens of the parent memcg needed be
4546 * incremented to the MAX_NR_GENS before reparenting.
4547 */
4548void max_lru_gen_memcg(struct mem_cgroup *memcg, int nid)
4549{
4550 struct lruvec *lruvec = get_lruvec(memcg, nid);
4551 int type;
4552
4553 for (type = 0; type < ANON_AND_FILE; type++) {
4554 while (get_nr_gens(lruvec, type) < MAX_NR_GENS) {
4555 try_to_inc_max_seq_nowalk(memcg, lruvec);
4556 cond_resched();
4557 }
4558 }
4559}
4560
4561/*
4562 * Compared to traditional LRU, MGLRU faces the following challenges:
4563 *
4564 * 1. Each lruvec has between MIN_NR_GENS and MAX_NR_GENS generations, the
4565 * number of generations of the parent and child memcg may be different,
4566 * so we cannot simply transfer MGLRU folios in the child memcg to the
4567 * parent memcg as we did for traditional LRU folios.
4568 * 2. The generation information is stored in folio->flags, but we cannot
4569 * traverse these folios while holding the lru lock, otherwise it may
4570 * cause softlockup.
4571 * 3. In walk_update_folio(), the gen of folio and corresponding lru size
4572 * may be updated, but the folio is not immediately moved to the
4573 * corresponding lru list. Therefore, there may be folios of different
4574 * generations on an LRU list.
4575 * 4. In lru_gen_del_folio(), the generation to which the folio belongs is
4576 * found based on the generation information in folio->flags, and the
4577 * corresponding LRU size will be updated. Therefore, we need to update
4578 * the lru size correctly during reparenting, otherwise the lru size may
4579 * be updated incorrectly in lru_gen_del_folio().
4580 *
4581 * Finally, we choose a compromise method, which is to splice the lru list in
4582 * the child memcg to the lru list of the same generation in the parent memcg
4583 * during reparenting.
4584 *
4585 * The same generation has different meanings in the parent and child memcg,
4586 * so this compromise method will cause the LRU inversion problem. But as the
4587 * system runs, this problem will be fixed automatically.
4588 */
4589static void __lru_gen_reparent_memcg(struct lruvec *child_lruvec, struct lruvec *parent_lruvec,
4590 int zone, int type)
4591{
4592 struct lru_gen_folio *child_lrugen, *parent_lrugen;
4593 enum lru_list lru = type * LRU_INACTIVE_FILE;
4594 int i;
4595
4596 child_lrugen = &child_lruvec->lrugen;
4597 parent_lrugen = &parent_lruvec->lrugen;
4598
4599 for (i = 0; i < get_nr_gens(child_lruvec, type); i++) {
4600 int gen = lru_gen_from_seq(child_lrugen->max_seq - i);
4601 long nr_pages = child_lrugen->nr_pages[gen][type][zone];
4602 int child_lru_active = lru_gen_is_active(child_lruvec, gen) ? LRU_ACTIVE : 0;
4603 int parent_lru_active = lru_gen_is_active(parent_lruvec, gen) ? LRU_ACTIVE : 0;
4604
4605 /* Assuming that child pages are colder than parent pages */
4606 list_splice_tail_init(&child_lrugen->folios[gen][type][zone],
4607 &parent_lrugen->folios[gen][type][zone]);
4608
4609 WRITE_ONCE(child_lrugen->nr_pages[gen][type][zone], 0);
4610 WRITE_ONCE(parent_lrugen->nr_pages[gen][type][zone],
4611 parent_lrugen->nr_pages[gen][type][zone] + nr_pages);
4612
4613 if (lru_gen_is_active(child_lruvec, gen) != lru_gen_is_active(parent_lruvec, gen)) {
4614 __update_lru_size(child_lruvec, lru + child_lru_active, zone, -nr_pages);
4615 __update_lru_size(parent_lruvec, lru + parent_lru_active, zone, nr_pages);
4616 }
4617 }
4618}
4619
4620void lru_gen_reparent_memcg(struct mem_cgroup *memcg, struct mem_cgroup *parent, int nid)
4621{
4622 struct lruvec *child_lruvec, *parent_lruvec;
4623 int type, zid;
4624 struct zone *zone;
4625 enum lru_list lru;
4626
4627 child_lruvec = get_lruvec(memcg, nid);
4628 parent_lruvec = get_lruvec(parent, nid);
4629
4630 for_each_managed_zone_pgdat(zone, NODE_DATA(nid), zid, MAX_NR_ZONES - 1)
4631 for (type = 0; type < ANON_AND_FILE; type++)
4632 __lru_gen_reparent_memcg(child_lruvec, parent_lruvec, zid, type);
4633
4634 for_each_lru(lru) {
4635 for_each_managed_zone_pgdat(zone, NODE_DATA(nid), zid, MAX_NR_ZONES - 1) {
4636 unsigned long size = mem_cgroup_get_zone_lru_size(child_lruvec, lru, zid);
4637
4638 if (!size)
4639 continue;
4640
4641 /* Move the accounting, do not duplicate it. */
4642 mem_cgroup_update_lru_size(parent_lruvec, lru, zid, size);
4643 mem_cgroup_update_lru_size(child_lruvec, lru, zid, -(long)size);
4644 }
4645 }
4646}
4647
4648#endif /* CONFIG_MEMCG */
4649
4650/******************************************************************************
4651 * the eviction
4652 ******************************************************************************/
4653
4654static bool sort_folio(struct lruvec *lruvec, struct folio *folio, struct scan_control *sc,
4655 int tier_idx)
4656{
4657 int gen = folio_lru_gen(folio);
4658 int type = folio_is_file_lru(folio);
4659 int zone = folio_zonenum(folio);
4660 int delta = folio_nr_pages(folio);
4661 int refs = folio_lru_refs(folio);
4662 bool workingset = folio_test_workingset(folio);
4663 int tier = lru_tier_from_refs(refs, workingset);
4664 struct lru_gen_folio *lrugen = &lruvec->lrugen;
4665
4666 VM_WARN_ON_ONCE_FOLIO(gen >= MAX_NR_GENS, folio);
4667
4668 /* unevictable: let it through and the generic path will cull it */
4669 if (!folio_evictable(folio))
4670 return false;
4671
4672 /* promoted */
4673 if (gen != lru_gen_from_seq(lrugen->min_seq[type])) {
4674 list_move(&folio->lru, &lrugen->folios[gen][type][zone]);
4675 return true;
4676 }
4677
4678 /* protected */
4679 if (tier > tier_idx || refs + workingset == BIT(LRU_REFS_WIDTH) + 1) {
4680 gen = folio_inc_gen(lruvec, folio);
4681 list_move(&folio->lru, &lrugen->folios[gen][type][zone]);
4682
4683 /* don't count the workingset being lazily promoted */
4684 if (refs + workingset != BIT(LRU_REFS_WIDTH) + 1) {
4685 int hist = lru_hist_from_seq(lrugen->min_seq[type]);
4686
4687 WRITE_ONCE(lrugen->protected[hist][type][tier],
4688 lrugen->protected[hist][type][tier] + delta);
4689 }
4690 return true;
4691 }
4692
4693 /* ineligible */
4694 if (zone > sc->reclaim_idx) {
4695 gen = folio_inc_gen(lruvec, folio);
4696 list_move_tail(&folio->lru, &lrugen->folios[gen][type][zone]);
4697 return true;
4698 }
4699
4700 return false;
4701}
4702
4703static bool isolate_folio(struct lruvec *lruvec, struct folio *folio, struct scan_control *sc)
4704{
4705 bool success;
4706
4707 /* raced with release_pages() */
4708 if (!folio_try_get(folio))
4709 return false;
4710
4711 /* raced with another isolation */
4712 if (!folio_test_clear_lru(folio)) {
4713 folio_put(folio);
4714 return false;
4715 }
4716
4717 /* see the comment on LRU_REFS_FLAGS */
4718 if (!folio_test_referenced(folio))
4719 set_mask_bits(&folio->flags.f, LRU_REFS_MASK, 0);
4720
4721 success = lru_gen_del_folio(lruvec, folio, true);
4722 VM_WARN_ON_ONCE_FOLIO(!success, folio);
4723
4724 return true;
4725}
4726
4727static int scan_folios(unsigned long nr_to_scan, struct lruvec *lruvec,
4728 struct scan_control *sc, int type, int tier,
4729 struct list_head *list, int *isolatedp)
4730{
4731 int i;
4732 int gen;
4733 enum node_stat_item item;
4734 int sorted = 0;
4735 int scanned = 0;
4736 int isolated = 0;
4737 int skipped = 0;
4738 unsigned long remaining = nr_to_scan;
4739 struct lru_gen_folio *lrugen = &lruvec->lrugen;
4740
4741 VM_WARN_ON_ONCE(nr_to_scan > MAX_LRU_BATCH);
4742 VM_WARN_ON_ONCE(!list_empty(list));
4743
4744 if (get_nr_gens(lruvec, type) == MIN_NR_GENS)
4745 return 0;
4746
4747 gen = lru_gen_from_seq(lrugen->min_seq[type]);
4748
4749 for (i = MAX_NR_ZONES; i > 0; i--) {
4750 LIST_HEAD(moved);
4751 int skipped_zone = 0;
4752 int zone = (sc->reclaim_idx + i) % MAX_NR_ZONES;
4753 struct list_head *head = &lrugen->folios[gen][type][zone];
4754
4755 while (!list_empty(head)) {
4756 struct folio *folio = lru_to_folio(head);
4757 int delta = folio_nr_pages(folio);
4758
4759 VM_WARN_ON_ONCE_FOLIO(folio_test_unevictable(folio), folio);
4760 VM_WARN_ON_ONCE_FOLIO(folio_test_active(folio), folio);
4761 VM_WARN_ON_ONCE_FOLIO(folio_is_file_lru(folio) != type, folio);
4762 VM_WARN_ON_ONCE_FOLIO(folio_zonenum(folio) != zone, folio);
4763
4764 scanned += delta;
4765
4766 if (sort_folio(lruvec, folio, sc, tier))
4767 sorted += delta;
4768 else if (isolate_folio(lruvec, folio, sc)) {
4769 list_add(&folio->lru, list);
4770 isolated += delta;
4771 } else {
4772 list_move(&folio->lru, &moved);
4773 skipped_zone += delta;
4774 }
4775
4776 if (!--remaining || max(isolated, skipped_zone) >= MIN_LRU_BATCH)
4777 break;
4778 }
4779
4780 if (skipped_zone) {
4781 list_splice(&moved, head);
4782 __count_zid_vm_events(PGSCAN_SKIP, zone, skipped_zone);
4783 skipped += skipped_zone;
4784 }
4785
4786 if (!remaining || isolated >= MIN_LRU_BATCH)
4787 break;
4788 }
4789
4790 item = PGSCAN_KSWAPD + reclaimer_offset(sc);
4791 mod_lruvec_state(lruvec, item, isolated);
4792 mod_lruvec_state(lruvec, PGREFILL, sorted);
4793 mod_lruvec_state(lruvec, PGSCAN_ANON + type, isolated);
4794 trace_mm_vmscan_lru_isolate(sc->reclaim_idx, sc->order, nr_to_scan,
4795 scanned, skipped, isolated,
4796 type ? LRU_INACTIVE_FILE : LRU_INACTIVE_ANON);
4797
4798 *isolatedp = isolated;
4799 return scanned;
4800}
4801
4802static int get_tier_idx(struct lruvec *lruvec, int type)
4803{
4804 int tier;
4805 struct ctrl_pos sp, pv = {};
4806
4807 /*
4808 * To leave a margin for fluctuations, use a larger gain factor (2:3).
4809 * This value is chosen because any other tier would have at least twice
4810 * as many refaults as the first tier.
4811 */
4812 read_ctrl_pos(lruvec, type, 0, 2, &sp);
4813 for (tier = 1; tier < MAX_NR_TIERS; tier++) {
4814 read_ctrl_pos(lruvec, type, tier, 3, &pv);
4815 if (!positive_ctrl_err(&sp, &pv))
4816 break;
4817 }
4818
4819 return tier - 1;
4820}
4821
4822static int get_type_to_scan(struct lruvec *lruvec, int swappiness)
4823{
4824 struct ctrl_pos sp, pv = {};
4825
4826 if (swappiness <= MIN_SWAPPINESS + 1)
4827 return LRU_GEN_FILE;
4828
4829 if (swappiness >= MAX_SWAPPINESS)
4830 return LRU_GEN_ANON;
4831 /*
4832 * Compare the sum of all tiers of anon with that of file to determine
4833 * which type to scan.
4834 */
4835 read_ctrl_pos(lruvec, LRU_GEN_ANON, MAX_NR_TIERS, swappiness, &sp);
4836 read_ctrl_pos(lruvec, LRU_GEN_FILE, MAX_NR_TIERS, MAX_SWAPPINESS - swappiness, &pv);
4837
4838 return positive_ctrl_err(&sp, &pv);
4839}
4840
4841static int isolate_folios(unsigned long nr_to_scan, struct lruvec *lruvec,
4842 struct scan_control *sc, int swappiness,
4843 struct list_head *list, int *isolated,
4844 int *isolate_type, int *isolate_scanned)
4845{
4846 int i;
4847 int total_scanned = 0;
4848 int type = get_type_to_scan(lruvec, swappiness);
4849
4850 for_each_evictable_type(i, swappiness) {
4851 int scanned;
4852 int tier = get_tier_idx(lruvec, type);
4853
4854 scanned = scan_folios(nr_to_scan, lruvec, sc,
4855 type, tier, list, isolated);
4856
4857 total_scanned += scanned;
4858 if (*isolated) {
4859 *isolate_type = type;
4860 *isolate_scanned = scanned;
4861 break;
4862 }
4863 /*
4864 * If scanned > 0 and isolated == 0, avoid falling back to the
4865 * other type, as this type remains sufficient. Falling back
4866 * too readily can disrupt the positive_ctrl_err() bias.
4867 */
4868 if (!scanned)
4869 type = !type;
4870 }
4871
4872 return total_scanned;
4873}
4874
4875static int evict_folios(unsigned long nr_to_scan, struct lruvec *lruvec,
4876 struct scan_control *sc, int swappiness)
4877{
4878 LIST_HEAD(list);
4879 LIST_HEAD(clean);
4880 struct folio *folio;
4881 struct folio *next;
4882 enum node_stat_item item;
4883 struct reclaim_stat stat;
4884 struct lru_gen_mm_walk *walk;
4885 int scanned, reclaimed;
4886 int isolated = 0, nr_isolated = 0, type, type_scanned;
4887 unsigned long total_reclaimed = 0;
4888 bool skip_retry = false;
4889 struct mem_cgroup *memcg = lruvec_memcg(lruvec);
4890 struct pglist_data *pgdat = lruvec_pgdat(lruvec);
4891
4892 lruvec_lock_irq(lruvec);
4893
4894 /* In case folio deletion left empty old gens, flush them */
4895 try_to_inc_min_seq(lruvec, swappiness);
4896
4897 scanned = isolate_folios(nr_to_scan, lruvec, sc, swappiness,
4898 &list, &isolated, &type, &type_scanned);
4899 nr_isolated = isolated;
4900
4901 /* Scanning may have emptied the oldest gen, flush it */
4902 if (scanned)
4903 try_to_inc_min_seq(lruvec, swappiness);
4904
4905 lruvec_unlock_irq(lruvec);
4906
4907 if (list_empty(&list))
4908 return scanned;
4909retry:
4910 reclaimed = shrink_folio_list(&list, pgdat, sc, &stat, false, memcg);
4911 sc->nr_reclaimed += reclaimed;
4912 total_reclaimed += reclaimed;
4913 /* Retry pass is only meant for clean folios without new isolation */
4914 if (isolated)
4915 handle_reclaim_writeback(isolated, pgdat, sc, &stat);
4916 trace_mm_vmscan_lru_shrink_inactive(pgdat->node_id,
4917 type_scanned, reclaimed, &stat, sc->priority,
4918 type ? LRU_INACTIVE_FILE : LRU_INACTIVE_ANON);
4919
4920 list_for_each_entry_safe_reverse(folio, next, &list, lru) {
4921 DEFINE_MIN_SEQ(lruvec);
4922
4923 /* move_folios_to_lru() culls unevictable folios via folio_putback_lru() */
4924 if (!folio_evictable(folio))
4925 continue;
4926
4927 /* retry folios that may have missed folio_rotate_reclaimable() */
4928 if (!skip_retry && !folio_test_active(folio) && !folio_mapped(folio) &&
4929 !folio_test_dirty(folio) && !folio_test_writeback(folio)) {
4930 list_move(&folio->lru, &clean);
4931 continue;
4932 }
4933
4934 /* don't add rejected folios to the oldest generation */
4935 if (lru_gen_folio_seq(lruvec, folio, false) == min_seq[type])
4936 set_mask_bits(&folio->flags.f, LRU_REFS_FLAGS, BIT(PG_active));
4937 }
4938
4939 move_folios_to_lru(&list);
4940
4941 walk = current->reclaim_state->mm_walk;
4942 if (walk && walk->batched) {
4943 walk->lruvec = lruvec;
4944 reset_batch_size(walk);
4945 }
4946
4947 mod_lruvec_state(lruvec, PGDEMOTE_KSWAPD + reclaimer_offset(sc),
4948 stat.nr_demoted);
4949
4950 item = PGSTEAL_KSWAPD + reclaimer_offset(sc);
4951 mod_lruvec_state(lruvec, item, reclaimed);
4952 mod_lruvec_state(lruvec, PGSTEAL_ANON + type, reclaimed);
4953
4954 list_splice_init(&clean, &list);
4955
4956 if (!list_empty(&list)) {
4957 skip_retry = true;
4958 isolated = 0;
4959 goto retry;
4960 }
4961
4962 if (nr_isolated > total_reclaimed)
4963 mod_lruvec_state(lruvec, PGROTATE_ANON + type,
4964 nr_isolated - total_reclaimed);
4965
4966 return scanned;
4967}
4968
4969static bool should_run_aging(struct lruvec *lruvec, unsigned long max_seq,
4970 struct scan_control *sc, int swappiness)
4971{
4972 DEFINE_MIN_SEQ(lruvec);
4973
4974 /* have to run aging, since eviction is not possible anymore */
4975 if (evictable_min_seq(min_seq, swappiness) + MIN_NR_GENS > max_seq)
4976 return true;
4977
4978 /* try to avoid aging, do gentle reclaim at the default priority */
4979 if (sc->priority == DEF_PRIORITY)
4980 return false;
4981
4982 /* better to run aging even though eviction is still possible */
4983 return evictable_min_seq(min_seq, swappiness) + MIN_NR_GENS == max_seq;
4984}
4985
4986static long get_nr_to_scan(struct lruvec *lruvec, struct scan_control *sc,
4987 struct mem_cgroup *memcg, int swappiness)
4988{
4989 unsigned long nr_to_scan, evictable;
4990 struct pglist_data *pgdat = lruvec_pgdat(lruvec);
4991
4992 /*
4993 * Proactive reclaim initiated by userspace for anonymous memory only.
4994 * SWAPPINESS_ANON_ONLY is set only on the proactive reclaim path, so
4995 * warn if it shows up elsewhere. When anon cannot be reclaimed (e.g.
4996 * no swap), return 0 to skip the scan entirely, avoiding useless scan
4997 * work when there is nothing eligible to reclaim.
4998 */
4999 if (swappiness == SWAPPINESS_ANON_ONLY) {
5000 WARN_ON_ONCE(!sc->proactive);
5001 if (!can_reclaim_anon_pages(memcg, pgdat->node_id, sc))
5002 return 0;
5003 }
5004
5005 evictable = lruvec_evictable_size(lruvec, swappiness);
5006
5007 /* try to scrape all its memory if this memcg was deleted */
5008 if (!mem_cgroup_online(memcg))
5009 return evictable;
5010
5011 nr_to_scan = apply_proportional_protection(memcg, sc, evictable);
5012 nr_to_scan >>= sc->priority;
5013
5014 return nr_to_scan;
5015}
5016
5017static bool should_abort_scan(struct lruvec *lruvec, struct scan_control *sc)
5018{
5019 int i;
5020 enum zone_watermarks mark;
5021
5022 if (unlikely(sc->proactive && signal_pending(current)))
5023 return true;
5024
5025 if (sc->nr_reclaimed >= max(sc->nr_to_reclaim, compact_gap(sc->order)))
5026 return true;
5027
5028 /* check the order to exclude compaction-induced reclaim */
5029 if (!current_is_kswapd() || sc->order)
5030 return false;
5031
5032 mark = sysctl_numa_balancing_mode & NUMA_BALANCING_MEMORY_TIERING ?
5033 WMARK_PROMO : WMARK_HIGH;
5034
5035 for (i = 0; i <= sc->reclaim_idx; i++) {
5036 struct zone *zone = lruvec_pgdat(lruvec)->node_zones + i;
5037 unsigned long size = wmark_pages(zone, mark) + MIN_LRU_BATCH;
5038
5039 if (managed_zone(zone) && !zone_watermark_ok(zone, 0, size, sc->reclaim_idx, 0))
5040 return false;
5041 }
5042
5043 /* kswapd should abort if all eligible zones are safe */
5044 return true;
5045}
5046
5047/*
5048 * For future optimizations:
5049 * 1. Defer try_to_inc_max_seq() to workqueues to reduce latency for memcg
5050 * reclaim.
5051 */
5052static bool try_to_shrink_lruvec(struct lruvec *lruvec, struct scan_control *sc)
5053{
5054 bool need_rotate = false, should_age = false;
5055 long nr_batch, nr_to_scan;
5056 int swappiness = get_swappiness(lruvec, sc);
5057 struct mem_cgroup *memcg = lruvec_memcg(lruvec);
5058
5059 nr_to_scan = get_nr_to_scan(lruvec, sc, memcg, swappiness);
5060 while (nr_to_scan > 0) {
5061 int delta;
5062 DEFINE_MAX_SEQ(lruvec);
5063
5064 if (mem_cgroup_below_min(sc->target_mem_cgroup, memcg)) {
5065 need_rotate = true;
5066 break;
5067 }
5068
5069 if (should_run_aging(lruvec, max_seq, sc, swappiness)) {
5070 if (try_to_inc_max_seq(lruvec, max_seq, swappiness, false))
5071 need_rotate = true;
5072 should_age = true;
5073 }
5074
5075 nr_batch = min(nr_to_scan, MIN_LRU_BATCH);
5076 delta = evict_folios(nr_batch, lruvec, sc, swappiness);
5077 if (!delta)
5078 break;
5079
5080 if (should_abort_scan(lruvec, sc))
5081 break;
5082
5083 /*
5084 * Root reclaim needs rotation when low on cold folio for better
5085 * fairness. Cgroup reclaim gets fairness from the iterator.
5086 */
5087 if (root_reclaim(sc) && should_age)
5088 break;
5089
5090 nr_to_scan -= delta;
5091 cond_resched();
5092 }
5093
5094 return need_rotate;
5095}
5096
5097static int shrink_one(struct lruvec *lruvec, struct scan_control *sc)
5098{
5099 bool need_rotate;
5100 unsigned long scanned = sc->nr_scanned;
5101 unsigned long reclaimed = sc->nr_reclaimed;
5102 struct mem_cgroup *memcg = lruvec_memcg(lruvec);
5103 struct pglist_data *pgdat = lruvec_pgdat(lruvec);
5104
5105 /* lru_gen_age_node() called mem_cgroup_calculate_protection() */
5106 if (mem_cgroup_below_min(NULL, memcg))
5107 return MEMCG_LRU_YOUNG;
5108
5109 if (mem_cgroup_below_low(NULL, memcg)) {
5110 /* see the comment on MEMCG_NR_GENS */
5111 if (READ_ONCE(lruvec->lrugen.seg) != MEMCG_LRU_TAIL)
5112 return MEMCG_LRU_TAIL;
5113
5114 memcg_memory_event(memcg, MEMCG_LOW);
5115 }
5116
5117 need_rotate = try_to_shrink_lruvec(lruvec, sc);
5118
5119 shrink_slab(sc->gfp_mask, pgdat->node_id, memcg, sc->priority);
5120
5121 if (!sc->proactive)
5122 vmpressure(sc->gfp_mask, sc->order, memcg, false,
5123 sc->nr_scanned - scanned, sc->nr_reclaimed - reclaimed);
5124
5125 flush_reclaim_state(sc);
5126
5127 if (need_rotate && mem_cgroup_online(memcg))
5128 return MEMCG_LRU_YOUNG;
5129
5130 if (!need_rotate && lruvec_is_sizable(lruvec, sc))
5131 return 0;
5132
5133 /* one retry if offlined or too small */
5134 return READ_ONCE(lruvec->lrugen.seg) != MEMCG_LRU_TAIL ?
5135 MEMCG_LRU_TAIL : MEMCG_LRU_YOUNG;
5136}
5137
5138static void shrink_many(struct pglist_data *pgdat, struct scan_control *sc)
5139{
5140 int op;
5141 int gen;
5142 int bin;
5143 int first_bin;
5144 struct lruvec *lruvec;
5145 struct lru_gen_folio *lrugen;
5146 struct mem_cgroup *memcg;
5147 struct hlist_nulls_node *pos;
5148
5149 gen = get_memcg_gen(READ_ONCE(pgdat->memcg_lru.seq));
5150 bin = first_bin = get_random_u32_below(MEMCG_NR_BINS);
5151restart:
5152 op = 0;
5153 memcg = NULL;
5154
5155 rcu_read_lock();
5156
5157 hlist_nulls_for_each_entry_rcu(lrugen, pos, &pgdat->memcg_lru.fifo[gen][bin], list) {
5158 if (op) {
5159 lru_gen_rotate_memcg(lruvec, op);
5160 op = 0;
5161 }
5162
5163 mem_cgroup_put(memcg);
5164 memcg = NULL;
5165
5166 if (gen != READ_ONCE(lrugen->gen))
5167 continue;
5168
5169 lruvec = container_of(lrugen, struct lruvec, lrugen);
5170 memcg = lruvec_memcg(lruvec);
5171
5172 if (!mem_cgroup_tryget(memcg)) {
5173 lru_gen_release_memcg(memcg);
5174 memcg = NULL;
5175 continue;
5176 }
5177
5178 rcu_read_unlock();
5179
5180 op = shrink_one(lruvec, sc);
5181
5182 rcu_read_lock();
5183
5184 if (should_abort_scan(lruvec, sc))
5185 break;
5186 }
5187
5188 rcu_read_unlock();
5189
5190 if (op)
5191 lru_gen_rotate_memcg(lruvec, op);
5192
5193 mem_cgroup_put(memcg);
5194
5195 if (!is_a_nulls(pos))
5196 return;
5197
5198 /* restart if raced with lru_gen_rotate_memcg() */
5199 if (gen != get_nulls_value(pos))
5200 goto restart;
5201
5202 /* try the rest of the bins of the current generation */
5203 bin = get_memcg_bin(bin + 1);
5204 if (bin != first_bin)
5205 goto restart;
5206}
5207
5208static void lru_gen_shrink_lruvec(struct lruvec *lruvec, struct scan_control *sc)
5209{
5210 struct blk_plug plug;
5211
5212 VM_WARN_ON_ONCE(root_reclaim(sc));
5213 VM_WARN_ON_ONCE(!sc->may_writepage || !sc->may_unmap);
5214
5215 lru_add_drain();
5216
5217 blk_start_plug(&plug);
5218
5219 set_mm_walk(NULL, sc->proactive);
5220
5221 if (try_to_shrink_lruvec(lruvec, sc))
5222 lru_gen_rotate_memcg(lruvec, MEMCG_LRU_YOUNG);
5223
5224 clear_mm_walk();
5225
5226 blk_finish_plug(&plug);
5227}
5228
5229static void lru_gen_shrink_node(struct pglist_data *pgdat, struct scan_control *sc)
5230{
5231 struct blk_plug plug;
5232 unsigned long reclaimed = sc->nr_reclaimed;
5233
5234 VM_WARN_ON_ONCE(!root_reclaim(sc));
5235
5236 /*
5237 * Unmapped clean folios are already prioritized. Scanning for more of
5238 * them is likely futile and can cause high reclaim latency when there
5239 * is a large number of memcgs.
5240 */
5241 if (!sc->may_writepage || !sc->may_unmap)
5242 goto done;
5243
5244 lru_add_drain();
5245
5246 blk_start_plug(&plug);
5247
5248 set_mm_walk(pgdat, sc->proactive);
5249
5250 set_initial_priority(pgdat, sc);
5251
5252 if (current_is_kswapd())
5253 sc->nr_reclaimed = 0;
5254
5255 if (mem_cgroup_disabled())
5256 shrink_one(&pgdat->__lruvec, sc);
5257 else
5258 shrink_many(pgdat, sc);
5259
5260 if (current_is_kswapd())
5261 sc->nr_reclaimed += reclaimed;
5262
5263 clear_mm_walk();
5264
5265 blk_finish_plug(&plug);
5266done:
5267 if (sc->nr_reclaimed > reclaimed)
5268 kswapd_try_clear_hopeless(pgdat, sc->order, sc->reclaim_idx);
5269}
5270
5271/******************************************************************************
5272 * state change
5273 ******************************************************************************/
5274
5275static bool __maybe_unused state_is_valid(struct lruvec *lruvec)
5276{
5277 struct lru_gen_folio *lrugen = &lruvec->lrugen;
5278
5279 if (lrugen->enabled) {
5280 enum lru_list lru;
5281
5282 for_each_evictable_lru(lru) {
5283 if (!list_empty(&lruvec->lists[lru]))
5284 return false;
5285 }
5286 } else {
5287 int gen, type, zone;
5288
5289 for_each_gen_type_zone(gen, type, zone) {
5290 if (!list_empty(&lrugen->folios[gen][type][zone]))
5291 return false;
5292 }
5293 }
5294
5295 return true;
5296}
5297
5298static bool fill_evictable(struct lruvec *lruvec)
5299{
5300 enum lru_list lru;
5301 int remaining = MAX_LRU_BATCH;
5302
5303 for_each_evictable_lru(lru) {
5304 int type = is_file_lru(lru);
5305 bool active = is_active_lru(lru);
5306 struct list_head *head = &lruvec->lists[lru];
5307
5308 while (!list_empty(head)) {
5309 bool success;
5310 struct folio *folio = lru_to_folio(head);
5311
5312 VM_WARN_ON_ONCE_FOLIO(folio_test_unevictable(folio), folio);
5313 VM_WARN_ON_ONCE_FOLIO(folio_test_active(folio) != active, folio);
5314 VM_WARN_ON_ONCE_FOLIO(folio_is_file_lru(folio) != type, folio);
5315 VM_WARN_ON_ONCE_FOLIO(folio_lru_gen(folio) != -1, folio);
5316
5317 lruvec_del_folio(lruvec, folio);
5318 success = lru_gen_add_folio(lruvec, folio, false);
5319 VM_WARN_ON_ONCE(!success);
5320
5321 if (!--remaining)
5322 return false;
5323 }
5324 }
5325
5326 return true;
5327}
5328
5329static bool drain_evictable(struct lruvec *lruvec)
5330{
5331 int gen, type, zone;
5332 int remaining = MAX_LRU_BATCH;
5333
5334 for_each_gen_type_zone(gen, type, zone) {
5335 struct list_head *head = &lruvec->lrugen.folios[gen][type][zone];
5336
5337 while (!list_empty(head)) {
5338 bool success;
5339 struct folio *folio = lru_to_folio(head);
5340
5341 VM_WARN_ON_ONCE_FOLIO(folio_test_unevictable(folio), folio);
5342 VM_WARN_ON_ONCE_FOLIO(folio_test_active(folio), folio);
5343 VM_WARN_ON_ONCE_FOLIO(folio_is_file_lru(folio) != type, folio);
5344 VM_WARN_ON_ONCE_FOLIO(folio_zonenum(folio) != zone, folio);
5345
5346 success = lru_gen_del_folio(lruvec, folio, false);
5347 VM_WARN_ON_ONCE(!success);
5348 lruvec_add_folio(lruvec, folio);
5349
5350 if (!--remaining)
5351 return false;
5352 }
5353 }
5354
5355 return true;
5356}
5357
5358static void lru_gen_change_state(bool enabled)
5359{
5360 static DEFINE_MUTEX(state_mutex);
5361
5362 struct mem_cgroup *memcg;
5363
5364 cgroup_lock();
5365 cpus_read_lock();
5366 get_online_mems();
5367 mutex_lock(&state_mutex);
5368
5369 if (enabled == lru_gen_enabled())
5370 goto unlock;
5371
5372 static_branch_enable_cpuslocked(&lru_switch);
5373
5374 if (enabled)
5375 static_branch_enable_cpuslocked(&lru_gen_caps[LRU_GEN_CORE]);
5376 else
5377 static_branch_disable_cpuslocked(&lru_gen_caps[LRU_GEN_CORE]);
5378
5379 memcg = mem_cgroup_iter(NULL, NULL, NULL);
5380 do {
5381 int nid;
5382
5383 for_each_node(nid) {
5384 struct lruvec *lruvec = get_lruvec(memcg, nid);
5385
5386 lruvec_lock_irq(lruvec);
5387
5388 VM_WARN_ON_ONCE(!seq_is_valid(lruvec));
5389 VM_WARN_ON_ONCE(!state_is_valid(lruvec));
5390
5391 lruvec->lrugen.enabled = enabled;
5392
5393 while (!(enabled ? fill_evictable(lruvec) : drain_evictable(lruvec))) {
5394 lruvec_unlock_irq(lruvec);
5395 cond_resched();
5396 lruvec_lock_irq(lruvec);
5397 }
5398
5399 lruvec_unlock_irq(lruvec);
5400 }
5401
5402 cond_resched();
5403 } while ((memcg = mem_cgroup_iter(NULL, memcg, NULL)));
5404
5405 static_branch_disable_cpuslocked(&lru_switch);
5406
5407unlock:
5408 mutex_unlock(&state_mutex);
5409 put_online_mems();
5410 cpus_read_unlock();
5411 cgroup_unlock();
5412}
5413
5414/******************************************************************************
5415 * sysfs interface
5416 ******************************************************************************/
5417
5418static ssize_t min_ttl_ms_show(struct kobject *kobj, struct kobj_attribute *attr, char *buf)
5419{
5420 return sysfs_emit(buf, "%u\n", jiffies_to_msecs(READ_ONCE(lru_gen_min_ttl)));
5421}
5422
5423/* see Documentation/admin-guide/mm/multigen_lru.rst for details */
5424static ssize_t min_ttl_ms_store(struct kobject *kobj, struct kobj_attribute *attr,
5425 const char *buf, size_t len)
5426{
5427 unsigned int msecs;
5428
5429 if (kstrtouint(buf, 0, &msecs))
5430 return -EINVAL;
5431
5432 WRITE_ONCE(lru_gen_min_ttl, msecs_to_jiffies(msecs));
5433
5434 return len;
5435}
5436
5437static struct kobj_attribute lru_gen_min_ttl_attr = __ATTR_RW(min_ttl_ms);
5438
5439static ssize_t enabled_show(struct kobject *kobj, struct kobj_attribute *attr, char *buf)
5440{
5441 unsigned int caps = 0;
5442
5443 if (get_cap(LRU_GEN_CORE))
5444 caps |= BIT(LRU_GEN_CORE);
5445
5446 if (should_walk_mmu())
5447 caps |= BIT(LRU_GEN_MM_WALK);
5448
5449 if (should_clear_pmd_young())
5450 caps |= BIT(LRU_GEN_NONLEAF_YOUNG);
5451
5452 return sysfs_emit(buf, "0x%04x\n", caps);
5453}
5454
5455/* see Documentation/admin-guide/mm/multigen_lru.rst for details */
5456static ssize_t enabled_store(struct kobject *kobj, struct kobj_attribute *attr,
5457 const char *buf, size_t len)
5458{
5459 int i;
5460 unsigned int caps;
5461
5462 if (tolower(*buf) == 'n')
5463 caps = 0;
5464 else if (tolower(*buf) == 'y')
5465 caps = -1;
5466 else if (kstrtouint(buf, 0, &caps))
5467 return -EINVAL;
5468
5469 for (i = 0; i < NR_LRU_GEN_CAPS; i++) {
5470 bool enabled = caps & BIT(i);
5471
5472 if (i == LRU_GEN_CORE)
5473 lru_gen_change_state(enabled);
5474 else if (enabled)
5475 static_branch_enable(&lru_gen_caps[i]);
5476 else
5477 static_branch_disable(&lru_gen_caps[i]);
5478 }
5479
5480 return len;
5481}
5482
5483static struct kobj_attribute lru_gen_enabled_attr = __ATTR_RW(enabled);
5484
5485static struct attribute *lru_gen_attrs[] = {
5486 &lru_gen_min_ttl_attr.attr,
5487 &lru_gen_enabled_attr.attr,
5488 NULL
5489};
5490
5491static const struct attribute_group lru_gen_attr_group = {
5492 .name = "lru_gen",
5493 .attrs = lru_gen_attrs,
5494};
5495
5496/******************************************************************************
5497 * debugfs interface
5498 ******************************************************************************/
5499
5500static void *lru_gen_seq_start(struct seq_file *m, loff_t *pos)
5501{
5502 struct mem_cgroup *memcg;
5503 loff_t nr_to_skip = *pos;
5504
5505 m->private = kvmalloc(PATH_MAX, GFP_KERNEL);
5506 if (!m->private)
5507 return ERR_PTR(-ENOMEM);
5508
5509 memcg = mem_cgroup_iter(NULL, NULL, NULL);
5510 do {
5511 int nid;
5512
5513 for_each_node_state(nid, N_MEMORY) {
5514 if (!nr_to_skip--)
5515 return get_lruvec(memcg, nid);
5516 }
5517 } while ((memcg = mem_cgroup_iter(NULL, memcg, NULL)));
5518
5519 return NULL;
5520}
5521
5522static void lru_gen_seq_stop(struct seq_file *m, void *v)
5523{
5524 if (!IS_ERR_OR_NULL(v))
5525 mem_cgroup_iter_break(NULL, lruvec_memcg(v));
5526
5527 kvfree(m->private);
5528 m->private = NULL;
5529}
5530
5531static void *lru_gen_seq_next(struct seq_file *m, void *v, loff_t *pos)
5532{
5533 int nid = lruvec_pgdat(v)->node_id;
5534 struct mem_cgroup *memcg = lruvec_memcg(v);
5535
5536 ++*pos;
5537
5538 nid = next_memory_node(nid);
5539 if (nid == MAX_NUMNODES) {
5540 memcg = mem_cgroup_iter(NULL, memcg, NULL);
5541 if (!memcg)
5542 return NULL;
5543
5544 nid = first_memory_node;
5545 }
5546
5547 return get_lruvec(memcg, nid);
5548}
5549
5550static void lru_gen_seq_show_full(struct seq_file *m, struct lruvec *lruvec,
5551 unsigned long max_seq, unsigned long *min_seq,
5552 unsigned long seq)
5553{
5554 int i;
5555 int type, tier;
5556 int hist = lru_hist_from_seq(seq);
5557 struct lru_gen_folio *lrugen = &lruvec->lrugen;
5558 struct lru_gen_mm_state *mm_state = get_mm_state(lruvec);
5559
5560 for (tier = 0; tier < MAX_NR_TIERS; tier++) {
5561 seq_printf(m, " %10d", tier);
5562 for (type = 0; type < ANON_AND_FILE; type++) {
5563 const char *s = "xxx";
5564 unsigned long n[3] = {};
5565
5566 if (seq == max_seq) {
5567 s = "RTx";
5568 n[0] = READ_ONCE(lrugen->avg_refaulted[type][tier]);
5569 n[1] = READ_ONCE(lrugen->avg_total[type][tier]);
5570 } else if (seq == min_seq[type] || NR_HIST_GENS > 1) {
5571 s = "rep";
5572 n[0] = atomic_long_read(&lrugen->refaulted[hist][type][tier]);
5573 n[1] = atomic_long_read(&lrugen->evicted[hist][type][tier]);
5574 n[2] = READ_ONCE(lrugen->protected[hist][type][tier]);
5575 }
5576
5577 for (i = 0; i < 3; i++)
5578 seq_printf(m, " %10lu%c", n[i], s[i]);
5579 }
5580 seq_putc(m, '\n');
5581 }
5582
5583 if (!mm_state)
5584 return;
5585
5586 seq_puts(m, " ");
5587 for (i = 0; i < NR_MM_STATS; i++) {
5588 const char *s = "xxxx";
5589 unsigned long n = 0;
5590
5591 if (seq == max_seq && NR_HIST_GENS == 1) {
5592 s = "TYFA";
5593 n = READ_ONCE(mm_state->stats[hist][i]);
5594 } else if (seq != max_seq && NR_HIST_GENS > 1) {
5595 s = "tyfa";
5596 n = READ_ONCE(mm_state->stats[hist][i]);
5597 }
5598
5599 seq_printf(m, " %10lu%c", n, s[i]);
5600 }
5601 seq_putc(m, '\n');
5602}
5603
5604/* see Documentation/admin-guide/mm/multigen_lru.rst for details */
5605static int lru_gen_seq_show(struct seq_file *m, void *v)
5606{
5607 unsigned long seq;
5608 bool full = debugfs_get_aux_num(m->file);
5609 struct lruvec *lruvec = v;
5610 struct lru_gen_folio *lrugen = &lruvec->lrugen;
5611 int nid = lruvec_pgdat(lruvec)->node_id;
5612 struct mem_cgroup *memcg = lruvec_memcg(lruvec);
5613 DEFINE_MAX_SEQ(lruvec);
5614 DEFINE_MIN_SEQ(lruvec);
5615
5616 if (nid == first_memory_node) {
5617 const char *path = memcg ? m->private : "";
5618
5619#ifdef CONFIG_MEMCG
5620 if (memcg)
5621 cgroup_path(memcg->css.cgroup, m->private, PATH_MAX);
5622#endif
5623 seq_printf(m, "memcg %llu %s\n", mem_cgroup_id(memcg), path);
5624 }
5625
5626 seq_printf(m, " node %5d\n", nid);
5627
5628 if (!full)
5629 seq = evictable_min_seq(min_seq, MAX_SWAPPINESS / 2);
5630 else if (max_seq >= MAX_NR_GENS)
5631 seq = max_seq - MAX_NR_GENS + 1;
5632 else
5633 seq = 0;
5634
5635 for (; seq <= max_seq; seq++) {
5636 int type, zone;
5637 int gen = lru_gen_from_seq(seq);
5638 unsigned long birth = READ_ONCE(lruvec->lrugen.timestamps[gen]);
5639
5640 seq_printf(m, " %10lu %10u", seq, jiffies_to_msecs(jiffies - birth));
5641
5642 for (type = 0; type < ANON_AND_FILE; type++) {
5643 unsigned long size = 0;
5644 char mark = full && seq < min_seq[type] ? 'x' : ' ';
5645
5646 for (zone = 0; zone < MAX_NR_ZONES; zone++)
5647 size += max(READ_ONCE(lrugen->nr_pages[gen][type][zone]), 0L);
5648
5649 seq_printf(m, " %10lu%c", size, mark);
5650 }
5651
5652 seq_putc(m, '\n');
5653
5654 if (full)
5655 lru_gen_seq_show_full(m, lruvec, max_seq, min_seq, seq);
5656 }
5657
5658 return 0;
5659}
5660
5661static const struct seq_operations lru_gen_seq_ops = {
5662 .start = lru_gen_seq_start,
5663 .stop = lru_gen_seq_stop,
5664 .next = lru_gen_seq_next,
5665 .show = lru_gen_seq_show,
5666};
5667
5668static int run_aging(struct lruvec *lruvec, unsigned long seq,
5669 int swappiness, bool force_scan)
5670{
5671 DEFINE_MAX_SEQ(lruvec);
5672
5673 if (seq > max_seq)
5674 return -EINVAL;
5675
5676 return try_to_inc_max_seq(lruvec, max_seq, swappiness, force_scan) ? 0 : -EEXIST;
5677}
5678
5679static int run_eviction(struct lruvec *lruvec, unsigned long seq, struct scan_control *sc,
5680 int swappiness, unsigned long nr_to_reclaim)
5681{
5682 int nr_batch;
5683 DEFINE_MAX_SEQ(lruvec);
5684
5685 if (seq + MIN_NR_GENS > max_seq)
5686 return -EINVAL;
5687
5688 sc->nr_reclaimed = 0;
5689
5690 while (!signal_pending(current)) {
5691 DEFINE_MIN_SEQ(lruvec);
5692
5693 if (seq < evictable_min_seq(min_seq, swappiness))
5694 return 0;
5695
5696 if (sc->nr_reclaimed >= nr_to_reclaim)
5697 return 0;
5698
5699 nr_batch = min(nr_to_reclaim - sc->nr_reclaimed, MAX_LRU_BATCH);
5700 if (!evict_folios(nr_batch, lruvec, sc, swappiness))
5701 return 0;
5702
5703 cond_resched();
5704 }
5705
5706 return -EINTR;
5707}
5708
5709static int run_cmd(char cmd, u64 memcg_id, int nid, unsigned long seq,
5710 struct scan_control *sc, int swappiness, unsigned long opt)
5711{
5712 struct lruvec *lruvec;
5713 int err = -EINVAL;
5714 struct mem_cgroup *memcg = NULL;
5715
5716 if (nid < 0 || nid >= MAX_NUMNODES || !node_state(nid, N_MEMORY))
5717 return -EINVAL;
5718
5719 if (!mem_cgroup_disabled()) {
5720 memcg = mem_cgroup_get_from_id(memcg_id);
5721 if (!memcg)
5722 return -EINVAL;
5723 }
5724
5725 if (memcg_id != mem_cgroup_id(memcg))
5726 goto done;
5727
5728 sc->target_mem_cgroup = memcg;
5729 lruvec = get_lruvec(memcg, nid);
5730
5731 if (swappiness < MIN_SWAPPINESS)
5732 swappiness = get_swappiness(lruvec, sc);
5733 else if (swappiness > SWAPPINESS_ANON_ONLY)
5734 goto done;
5735
5736 switch (cmd) {
5737 case '+':
5738 err = run_aging(lruvec, seq, swappiness, opt);
5739 break;
5740 case '-':
5741 err = run_eviction(lruvec, seq, sc, swappiness, opt);
5742 break;
5743 }
5744done:
5745 mem_cgroup_put(memcg);
5746
5747 return err;
5748}
5749
5750/* see Documentation/admin-guide/mm/multigen_lru.rst for details */
5751static ssize_t lru_gen_seq_write(struct file *file, const char __user *src,
5752 size_t len, loff_t *pos)
5753{
5754 void *buf;
5755 char *cur, *next;
5756 unsigned int flags;
5757 struct blk_plug plug;
5758 int err = -EINVAL;
5759 struct scan_control sc = {
5760 .may_writepage = true,
5761 .may_unmap = true,
5762 .may_swap = true,
5763 .reclaim_idx = MAX_NR_ZONES - 1,
5764 .gfp_mask = GFP_KERNEL,
5765 .proactive = true,
5766 };
5767
5768 buf = kvmalloc(len + 1, GFP_KERNEL);
5769 if (!buf)
5770 return -ENOMEM;
5771
5772 if (copy_from_user(buf, src, len)) {
5773 kvfree(buf);
5774 return -EFAULT;
5775 }
5776
5777 set_task_reclaim_state(current, &sc.reclaim_state);
5778 flags = memalloc_noreclaim_save();
5779 blk_start_plug(&plug);
5780 if (!set_mm_walk(NULL, true)) {
5781 err = -ENOMEM;
5782 goto done;
5783 }
5784
5785 next = buf;
5786 next[len] = '\0';
5787
5788 while ((cur = strsep(&next, ",;\n"))) {
5789 int n;
5790 int end;
5791 char cmd, swap_string[5];
5792 u64 memcg_id;
5793 unsigned int nid;
5794 unsigned long seq;
5795 unsigned int swappiness;
5796 unsigned long opt = -1;
5797
5798 cur = skip_spaces(cur);
5799 if (!*cur)
5800 continue;
5801
5802 n = sscanf(cur, "%c %llu %u %lu %n %4s %n %lu %n", &cmd, &memcg_id, &nid,
5803 &seq, &end, swap_string, &end, &opt, &end);
5804 if (n < 4 || cur[end]) {
5805 err = -EINVAL;
5806 break;
5807 }
5808
5809 if (n == 4) {
5810 swappiness = -1;
5811 } else if (!strcmp("max", swap_string)) {
5812 /* set by userspace for anonymous memory only */
5813 swappiness = SWAPPINESS_ANON_ONLY;
5814 } else {
5815 err = kstrtouint(swap_string, 0, &swappiness);
5816 if (err)
5817 break;
5818 }
5819
5820 err = run_cmd(cmd, memcg_id, nid, seq, &sc, swappiness, opt);
5821 if (err)
5822 break;
5823 }
5824done:
5825 clear_mm_walk();
5826 blk_finish_plug(&plug);
5827 memalloc_noreclaim_restore(flags);
5828 set_task_reclaim_state(current, NULL);
5829
5830 kvfree(buf);
5831
5832 return err ? : len;
5833}
5834
5835static int lru_gen_seq_open(struct inode *inode, struct file *file)
5836{
5837 return seq_open(file, &lru_gen_seq_ops);
5838}
5839
5840static const struct file_operations lru_gen_rw_fops = {
5841 .open = lru_gen_seq_open,
5842 .read = seq_read,
5843 .write = lru_gen_seq_write,
5844 .llseek = seq_lseek,
5845 .release = seq_release,
5846};
5847
5848static const struct file_operations lru_gen_ro_fops = {
5849 .open = lru_gen_seq_open,
5850 .read = seq_read,
5851 .llseek = seq_lseek,
5852 .release = seq_release,
5853};
5854
5855/******************************************************************************
5856 * initialization
5857 ******************************************************************************/
5858
5859void lru_gen_init_pgdat(struct pglist_data *pgdat)
5860{
5861 int i, j;
5862
5863 spin_lock_init(&pgdat->memcg_lru.lock);
5864
5865 for (i = 0; i < MEMCG_NR_GENS; i++) {
5866 for (j = 0; j < MEMCG_NR_BINS; j++)
5867 INIT_HLIST_NULLS_HEAD(&pgdat->memcg_lru.fifo[i][j], i);
5868 }
5869}
5870
5871void lru_gen_init_lruvec(struct lruvec *lruvec)
5872{
5873 int i;
5874 int gen, type, zone;
5875 struct lru_gen_folio *lrugen = &lruvec->lrugen;
5876 struct lru_gen_mm_state *mm_state = get_mm_state(lruvec);
5877
5878 lrugen->max_seq = MIN_NR_GENS + 1;
5879 lrugen->enabled = lru_gen_enabled();
5880
5881 for (i = 0; i <= MIN_NR_GENS + 1; i++)
5882 lrugen->timestamps[i] = jiffies;
5883
5884 for_each_gen_type_zone(gen, type, zone)
5885 INIT_LIST_HEAD(&lrugen->folios[gen][type][zone]);
5886
5887 if (mm_state)
5888 mm_state->seq = MIN_NR_GENS;
5889}
5890
5891#ifdef CONFIG_MEMCG
5892
5893void lru_gen_init_memcg(struct mem_cgroup *memcg)
5894{
5895 struct lru_gen_mm_list *mm_list = get_mm_list(memcg);
5896
5897 if (!mm_list)
5898 return;
5899
5900 INIT_LIST_HEAD(&mm_list->fifo);
5901 spin_lock_init(&mm_list->lock);
5902}
5903
5904void lru_gen_exit_memcg(struct mem_cgroup *memcg)
5905{
5906 int i;
5907 int nid;
5908 struct lru_gen_mm_list *mm_list = get_mm_list(memcg);
5909
5910 VM_WARN_ON_ONCE(mm_list && !list_empty(&mm_list->fifo));
5911
5912 for_each_node(nid) {
5913 struct lruvec *lruvec = get_lruvec(memcg, nid);
5914 struct lru_gen_mm_state *mm_state = get_mm_state(lruvec);
5915
5916 VM_WARN_ON_ONCE(memchr_inv(lruvec->lrugen.nr_pages, 0,
5917 sizeof(lruvec->lrugen.nr_pages)));
5918
5919 lruvec->lrugen.list.next = LIST_POISON1;
5920
5921 if (!mm_state)
5922 continue;
5923
5924 for (i = 0; i < NR_BLOOM_FILTERS; i++) {
5925 bitmap_free(mm_state->filters[i]);
5926 mm_state->filters[i] = NULL;
5927 }
5928 }
5929}
5930
5931#endif /* CONFIG_MEMCG */
5932
5933static int __init init_lru_gen(void)
5934{
5935 BUILD_BUG_ON(MIN_NR_GENS + 1 >= MAX_NR_GENS);
5936 BUILD_BUG_ON(BIT(LRU_GEN_WIDTH) <= MAX_NR_GENS);
5937
5938 if (sysfs_create_group(mm_kobj, &lru_gen_attr_group))
5939 pr_err("lru_gen: failed to create sysfs group\n");
5940
5941 debugfs_create_file_aux_num("lru_gen", 0644, NULL, NULL, false,
5942 &lru_gen_rw_fops);
5943 debugfs_create_file_aux_num("lru_gen_full", 0444, NULL, NULL, true,
5944 &lru_gen_ro_fops);
5945
5946 return 0;
5947};
5948late_initcall(init_lru_gen);
5949
5950#else /* !CONFIG_LRU_GEN */
5951
5952static void lru_gen_age_node(struct pglist_data *pgdat, struct scan_control *sc)
5953{
5954 BUILD_BUG();
5955}
5956
5957static void lru_gen_shrink_lruvec(struct lruvec *lruvec, struct scan_control *sc)
5958{
5959 BUILD_BUG();
5960}
5961
5962static void lru_gen_shrink_node(struct pglist_data *pgdat, struct scan_control *sc)
5963{
5964 BUILD_BUG();
5965}
5966
5967#endif /* CONFIG_LRU_GEN */
5968
5969static void shrink_lruvec(struct lruvec *lruvec, struct scan_control *sc)
5970{
5971 unsigned long nr[NR_LRU_LISTS];
5972 unsigned long targets[NR_LRU_LISTS];
5973 unsigned long nr_to_scan;
5974 enum lru_list lru;
5975 unsigned long nr_reclaimed = 0;
5976 unsigned long nr_to_reclaim = sc->nr_to_reclaim;
5977 bool proportional_reclaim;
5978 struct blk_plug plug;
5979
5980 if ((lru_gen_enabled() || lru_gen_switching()) && !root_reclaim(sc)) {
5981 lru_gen_shrink_lruvec(lruvec, sc);
5982
5983 if (!lru_gen_switching())
5984 return;
5985
5986 }
5987
5988 get_scan_count(lruvec, sc, nr);
5989
5990 /* Record the original scan target for proportional adjustments later */
5991 memcpy(targets, nr, sizeof(nr));
5992
5993 /*
5994 * Global reclaiming within direct reclaim at DEF_PRIORITY is a normal
5995 * event that can occur when there is little memory pressure e.g.
5996 * multiple streaming readers/writers. Hence, we do not abort scanning
5997 * when the requested number of pages are reclaimed when scanning at
5998 * DEF_PRIORITY on the assumption that the fact we are direct
5999 * reclaiming implies that kswapd is not keeping up and it is best to
6000 * do a batch of work at once. For memcg reclaim one check is made to
6001 * abort proportional reclaim if either the file or anon lru has already
6002 * dropped to zero at the first pass.
6003 */
6004 proportional_reclaim = (!cgroup_reclaim(sc) && !current_is_kswapd() &&
6005 sc->priority == DEF_PRIORITY);
6006
6007 blk_start_plug(&plug);
6008 while (nr[LRU_INACTIVE_ANON] || nr[LRU_ACTIVE_FILE] ||
6009 nr[LRU_INACTIVE_FILE]) {
6010 unsigned long nr_anon, nr_file, percentage;
6011 unsigned long nr_scanned;
6012
6013 for_each_evictable_lru(lru) {
6014 if (nr[lru]) {
6015 nr_to_scan = min(nr[lru], SWAP_CLUSTER_MAX);
6016 nr[lru] -= nr_to_scan;
6017
6018 nr_reclaimed += shrink_list(lru, nr_to_scan,
6019 lruvec, sc);
6020 }
6021 }
6022
6023 cond_resched_tasks_rcu_qs();
6024
6025 if (nr_reclaimed < nr_to_reclaim || proportional_reclaim)
6026 continue;
6027
6028 /*
6029 * For kswapd and memcg, reclaim at least the number of pages
6030 * requested. Ensure that the anon and file LRUs are scanned
6031 * proportionally what was requested by get_scan_count(). We
6032 * stop reclaiming one LRU and reduce the amount scanning
6033 * proportional to the original scan target.
6034 */
6035 nr_file = nr[LRU_INACTIVE_FILE] + nr[LRU_ACTIVE_FILE];
6036 nr_anon = nr[LRU_INACTIVE_ANON] + nr[LRU_ACTIVE_ANON];
6037
6038 /*
6039 * It's just vindictive to attack the larger once the smaller
6040 * has gone to zero. And given the way we stop scanning the
6041 * smaller below, this makes sure that we only make one nudge
6042 * towards proportionality once we've got nr_to_reclaim.
6043 */
6044 if (!nr_file || !nr_anon)
6045 break;
6046
6047 if (nr_file > nr_anon) {
6048 unsigned long scan_target = targets[LRU_INACTIVE_ANON] +
6049 targets[LRU_ACTIVE_ANON] + 1;
6050 lru = LRU_BASE;
6051 percentage = nr_anon * 100 / scan_target;
6052 } else {
6053 unsigned long scan_target = targets[LRU_INACTIVE_FILE] +
6054 targets[LRU_ACTIVE_FILE] + 1;
6055 lru = LRU_FILE;
6056 percentage = nr_file * 100 / scan_target;
6057 }
6058
6059 /* Stop scanning the smaller of the LRU */
6060 nr[lru] = 0;
6061 nr[lru + LRU_ACTIVE] = 0;
6062
6063 /*
6064 * Recalculate the other LRU scan count based on its original
6065 * scan target and the percentage scanning already complete
6066 */
6067 lru = (lru == LRU_FILE) ? LRU_BASE : LRU_FILE;
6068 nr_scanned = targets[lru] - nr[lru];
6069 nr[lru] = targets[lru] * (100 - percentage) / 100;
6070 nr[lru] -= min(nr[lru], nr_scanned);
6071
6072 lru += LRU_ACTIVE;
6073 nr_scanned = targets[lru] - nr[lru];
6074 nr[lru] = targets[lru] * (100 - percentage) / 100;
6075 nr[lru] -= min(nr[lru], nr_scanned);
6076 }
6077 blk_finish_plug(&plug);
6078 sc->nr_reclaimed += nr_reclaimed;
6079
6080 /*
6081 * Even if we did not try to evict anon pages at all, we want to
6082 * rebalance the anon lru active/inactive ratio.
6083 */
6084 if (can_age_anon_pages(lruvec, sc) &&
6085 inactive_is_low(lruvec, LRU_INACTIVE_ANON))
6086 shrink_active_list(SWAP_CLUSTER_MAX, lruvec,
6087 sc, LRU_ACTIVE_ANON);
6088}
6089
6090/* Use reclaim/compaction for costly allocs or under memory pressure */
6091static bool in_reclaim_compaction(struct scan_control *sc)
6092{
6093 if (gfp_compaction_allowed(sc->gfp_mask) && sc->order &&
6094 (sc->order > PAGE_ALLOC_COSTLY_ORDER ||
6095 sc->priority < DEF_PRIORITY - 2))
6096 return true;
6097
6098 return false;
6099}
6100
6101/*
6102 * Reclaim/compaction is used for high-order allocation requests. It reclaims
6103 * order-0 pages before compacting the zone. should_continue_reclaim() returns
6104 * true if more pages should be reclaimed such that when the page allocator
6105 * calls try_to_compact_pages() that it will have enough free pages to succeed.
6106 * It will give up earlier than that if there is difficulty reclaiming pages.
6107 */
6108static inline bool should_continue_reclaim(struct pglist_data *pgdat,
6109 unsigned long nr_reclaimed,
6110 struct scan_control *sc)
6111{
6112 unsigned long pages_for_compaction;
6113 unsigned long inactive_lru_pages;
6114 int z;
6115 struct zone *zone;
6116
6117 /* If not in reclaim/compaction mode, stop */
6118 if (!in_reclaim_compaction(sc))
6119 return false;
6120
6121 /*
6122 * Stop if we failed to reclaim any pages from the last SWAP_CLUSTER_MAX
6123 * number of pages that were scanned. This will return to the caller
6124 * with the risk reclaim/compaction and the resulting allocation attempt
6125 * fails. In the past we have tried harder for __GFP_RETRY_MAYFAIL
6126 * allocations through requiring that the full LRU list has been scanned
6127 * first, by assuming that zero delta of sc->nr_scanned means full LRU
6128 * scan, but that approximation was wrong, and there were corner cases
6129 * where always a non-zero amount of pages were scanned.
6130 */
6131 if (!nr_reclaimed)
6132 return false;
6133
6134 /* If compaction would go ahead or the allocation would succeed, stop */
6135 for_each_managed_zone_pgdat(zone, pgdat, z, sc->reclaim_idx) {
6136 unsigned long watermark = min_wmark_pages(zone);
6137
6138 /* Allocation can already succeed, nothing to do */
6139 if (zone_watermark_ok(zone, sc->order, watermark,
6140 sc->reclaim_idx, 0))
6141 return false;
6142
6143 if (compaction_suitable(zone, sc->order, watermark,
6144 sc->reclaim_idx))
6145 return false;
6146 }
6147
6148 /*
6149 * If we have not reclaimed enough pages for compaction and the
6150 * inactive lists are large enough, continue reclaiming
6151 */
6152 pages_for_compaction = compact_gap(sc->order);
6153 inactive_lru_pages = node_page_state(pgdat, NR_INACTIVE_FILE);
6154 if (can_reclaim_anon_pages(NULL, pgdat->node_id, sc))
6155 inactive_lru_pages += node_page_state(pgdat, NR_INACTIVE_ANON);
6156
6157 return inactive_lru_pages > pages_for_compaction;
6158}
6159
6160static void shrink_node_memcgs(pg_data_t *pgdat, struct scan_control *sc)
6161{
6162 struct mem_cgroup *target_memcg = sc->target_mem_cgroup;
6163 struct mem_cgroup_reclaim_cookie reclaim = {
6164 .pgdat = pgdat,
6165 };
6166 struct mem_cgroup_reclaim_cookie *partial = &reclaim;
6167 struct mem_cgroup *memcg;
6168
6169 /*
6170 * In most cases, direct reclaimers can do partial walks
6171 * through the cgroup tree, using an iterator state that
6172 * persists across invocations. This strikes a balance between
6173 * fairness and allocation latency.
6174 *
6175 * For kswapd, reliable forward progress is more important
6176 * than a quick return to idle. Always do full walks.
6177 */
6178 if (current_is_kswapd() || sc->memcg_full_walk)
6179 partial = NULL;
6180
6181 memcg = mem_cgroup_iter(target_memcg, NULL, partial);
6182 do {
6183 struct lruvec *lruvec = mem_cgroup_lruvec(memcg, pgdat);
6184 unsigned long reclaimed;
6185 unsigned long scanned;
6186
6187 /*
6188 * This loop can become CPU-bound when target memcgs
6189 * aren't eligible for reclaim - either because they
6190 * don't have any reclaimable pages, or because their
6191 * memory is explicitly protected. Avoid soft lockups.
6192 */
6193 cond_resched();
6194
6195 mem_cgroup_calculate_protection(target_memcg, memcg);
6196
6197 if (mem_cgroup_below_min(target_memcg, memcg)) {
6198 /*
6199 * Hard protection.
6200 * If there is no reclaimable memory, OOM.
6201 */
6202 continue;
6203 } else if (mem_cgroup_below_low(target_memcg, memcg)) {
6204 /*
6205 * Soft protection.
6206 * Respect the protection only as long as
6207 * there is an unprotected supply
6208 * of reclaimable memory from other cgroups.
6209 */
6210 if (!sc->memcg_low_reclaim) {
6211 sc->memcg_low_skipped = 1;
6212 continue;
6213 }
6214 memcg_memory_event(memcg, MEMCG_LOW);
6215 }
6216
6217 reclaimed = sc->nr_reclaimed;
6218 scanned = sc->nr_scanned;
6219
6220 shrink_lruvec(lruvec, sc);
6221
6222 shrink_slab(sc->gfp_mask, pgdat->node_id, memcg,
6223 sc->priority);
6224
6225 /* Record the group's reclaim efficiency */
6226 if (!sc->proactive)
6227 vmpressure(sc->gfp_mask, sc->order, memcg, false,
6228 sc->nr_scanned - scanned,
6229 sc->nr_reclaimed - reclaimed);
6230
6231 /* If partial walks are allowed, bail once goal is reached */
6232 if (partial && sc->nr_reclaimed >= sc->nr_to_reclaim) {
6233 mem_cgroup_iter_break(target_memcg, memcg);
6234 break;
6235 }
6236 } while ((memcg = mem_cgroup_iter(target_memcg, memcg, partial)));
6237}
6238
6239static void shrink_node(pg_data_t *pgdat, struct scan_control *sc)
6240{
6241 unsigned long nr_reclaimed, nr_scanned, nr_node_reclaimed;
6242 struct lruvec *target_lruvec;
6243 bool reclaimable = false;
6244
6245 if ((lru_gen_enabled() || lru_gen_switching()) && root_reclaim(sc)) {
6246 memset(&sc->nr, 0, sizeof(sc->nr));
6247 lru_gen_shrink_node(pgdat, sc);
6248
6249 if (!lru_gen_switching())
6250 return;
6251
6252 }
6253
6254 target_lruvec = mem_cgroup_lruvec(sc->target_mem_cgroup, pgdat);
6255
6256again:
6257 memset(&sc->nr, 0, sizeof(sc->nr));
6258
6259 nr_reclaimed = sc->nr_reclaimed;
6260 nr_scanned = sc->nr_scanned;
6261
6262 prepare_scan_control(pgdat, sc);
6263
6264 shrink_node_memcgs(pgdat, sc);
6265
6266 flush_reclaim_state(sc);
6267
6268 nr_node_reclaimed = sc->nr_reclaimed - nr_reclaimed;
6269
6270 /* Record the subtree's reclaim efficiency */
6271 if (!sc->proactive)
6272 vmpressure(sc->gfp_mask, sc->order, sc->target_mem_cgroup, true,
6273 sc->nr_scanned - nr_scanned, nr_node_reclaimed);
6274
6275 if (nr_node_reclaimed)
6276 reclaimable = true;
6277
6278 if (current_is_kswapd()) {
6279 /*
6280 * If reclaim is isolating dirty pages under writeback,
6281 * it implies that the long-lived page allocation rate
6282 * is exceeding the page laundering rate. Either the
6283 * global limits are not being effective at throttling
6284 * processes due to the page distribution throughout
6285 * zones or there is heavy usage of a slow backing
6286 * device. The only option is to throttle from reclaim
6287 * context which is not ideal as there is no guarantee
6288 * the dirtying process is throttled in the same way
6289 * balance_dirty_pages() manages.
6290 *
6291 * Once a node is flagged PGDAT_WRITEBACK, kswapd will
6292 * count the number of pages under pages flagged for
6293 * immediate reclaim and stall if any are encountered
6294 * in the nr_immediate check below.
6295 */
6296 if (sc->nr.writeback && sc->nr.writeback == sc->nr.taken)
6297 set_bit(PGDAT_WRITEBACK, &pgdat->flags);
6298
6299 /*
6300 * If kswapd scans pages marked for immediate
6301 * reclaim and under writeback (nr_immediate), it
6302 * implies that pages are cycling through the LRU
6303 * faster than they are written so forcibly stall
6304 * until some pages complete writeback.
6305 */
6306 if (sc->nr.immediate)
6307 reclaim_throttle(pgdat, VMSCAN_THROTTLE_WRITEBACK);
6308 }
6309
6310 /*
6311 * Tag a node/memcg as congested if all the dirty pages were marked
6312 * for writeback and immediate reclaim (counted in nr.congested).
6313 *
6314 * Legacy memcg will stall in page writeback so avoid forcibly
6315 * stalling in reclaim_throttle().
6316 */
6317 if (sc->nr.dirty && sc->nr.dirty == sc->nr.congested) {
6318 if (cgroup_reclaim(sc) && writeback_throttling_sane(sc))
6319 set_bit(LRUVEC_CGROUP_CONGESTED, &target_lruvec->flags);
6320
6321 if (current_is_kswapd())
6322 set_bit(LRUVEC_NODE_CONGESTED, &target_lruvec->flags);
6323 }
6324
6325 /*
6326 * Stall direct reclaim for IO completions if the lruvec is
6327 * node is congested. Allow kswapd to continue until it
6328 * starts encountering unqueued dirty pages or cycling through
6329 * the LRU too quickly.
6330 */
6331 if (!current_is_kswapd() && current_may_throttle() &&
6332 !sc->hibernation_mode &&
6333 (test_bit(LRUVEC_CGROUP_CONGESTED, &target_lruvec->flags) ||
6334 test_bit(LRUVEC_NODE_CONGESTED, &target_lruvec->flags)))
6335 reclaim_throttle(pgdat, VMSCAN_THROTTLE_CONGESTED);
6336
6337 if (should_continue_reclaim(pgdat, nr_node_reclaimed, sc))
6338 goto again;
6339
6340 /*
6341 * Kswapd gives up on balancing particular nodes after too
6342 * many failures to reclaim anything from them and goes to
6343 * sleep. On reclaim progress, reset the failure counter. A
6344 * successful direct reclaim run will revive a dormant kswapd.
6345 */
6346 if (reclaimable)
6347 kswapd_try_clear_hopeless(pgdat, sc->order, sc->reclaim_idx);
6348 else if (sc->cache_trim_mode)
6349 sc->cache_trim_mode_failed = 1;
6350}
6351
6352/*
6353 * Returns true if compaction should go ahead for a costly-order request, or
6354 * the allocation would already succeed without compaction. Return false if we
6355 * should reclaim first.
6356 */
6357static inline bool compaction_ready(struct zone *zone, struct scan_control *sc)
6358{
6359 unsigned long watermark;
6360
6361 if (!gfp_compaction_allowed(sc->gfp_mask))
6362 return false;
6363
6364 /* Allocation can already succeed, nothing to do */
6365 if (zone_watermark_ok(zone, sc->order, min_wmark_pages(zone),
6366 sc->reclaim_idx, 0))
6367 return true;
6368
6369 /*
6370 * Direct reclaim usually targets the min watermark, but compaction
6371 * takes time to run and there are potentially other callers using the
6372 * pages just freed. So target a higher buffer to give compaction a
6373 * reasonable chance of completing and allocating the pages.
6374 *
6375 * Note that we won't actually reclaim the whole buffer in one attempt
6376 * as the target watermark in should_continue_reclaim() is lower. But if
6377 * we are already above the high+gap watermark, don't reclaim at all.
6378 */
6379 watermark = high_wmark_pages(zone);
6380 if (compaction_suitable(zone, sc->order, watermark, sc->reclaim_idx))
6381 return true;
6382
6383 return false;
6384}
6385
6386static void consider_reclaim_throttle(pg_data_t *pgdat, struct scan_control *sc)
6387{
6388 /*
6389 * If reclaim is making progress greater than 12% efficiency then
6390 * wake all the NOPROGRESS throttled tasks.
6391 */
6392 if (sc->nr_reclaimed > (sc->nr_scanned >> 3)) {
6393 wait_queue_head_t *wqh;
6394
6395 wqh = &pgdat->reclaim_wait[VMSCAN_THROTTLE_NOPROGRESS];
6396 if (waitqueue_active(wqh))
6397 wake_up(wqh);
6398
6399 return;
6400 }
6401
6402 /*
6403 * Do not throttle kswapd or cgroup reclaim on NOPROGRESS as it will
6404 * throttle on VMSCAN_THROTTLE_WRITEBACK if there are too many pages
6405 * under writeback and marked for immediate reclaim at the tail of the
6406 * LRU.
6407 */
6408 if (current_is_kswapd() || cgroup_reclaim(sc))
6409 return;
6410
6411 /* Throttle if making no progress at high priorities. */
6412 if (sc->priority == 1 && !sc->nr_reclaimed)
6413 reclaim_throttle(pgdat, VMSCAN_THROTTLE_NOPROGRESS);
6414}
6415
6416/*
6417 * This is the direct reclaim path, for page-allocating processes. We only
6418 * try to reclaim pages from zones which will satisfy the caller's allocation
6419 * request.
6420 *
6421 * If a zone is deemed to be full of pinned pages then just give it a light
6422 * scan then give up on it.
6423 */
6424static void shrink_zones(struct zonelist *zonelist, struct scan_control *sc)
6425{
6426 struct zoneref *z;
6427 struct zone *zone;
6428 unsigned long nr_soft_reclaimed;
6429 unsigned long nr_soft_scanned;
6430 gfp_t orig_mask;
6431 pg_data_t *last_pgdat = NULL;
6432 pg_data_t *first_pgdat = NULL;
6433
6434 /*
6435 * If the number of buffer_heads in the machine exceeds the maximum
6436 * allowed level, force direct reclaim to scan the highmem zone as
6437 * highmem pages could be pinning lowmem pages storing buffer_heads
6438 */
6439 orig_mask = sc->gfp_mask;
6440 if (buffer_heads_over_limit) {
6441 sc->gfp_mask |= __GFP_HIGHMEM;
6442 sc->reclaim_idx = gfp_zone(sc->gfp_mask);
6443 }
6444
6445 for_each_zone_zonelist_nodemask(zone, z, zonelist,
6446 sc->reclaim_idx, sc->nodemask) {
6447 /*
6448 * Take care memory controller reclaiming has small influence
6449 * to global LRU.
6450 */
6451 if (!cgroup_reclaim(sc)) {
6452 if (!cpuset_zone_allowed(zone,
6453 GFP_KERNEL | __GFP_HARDWALL))
6454 continue;
6455
6456 /*
6457 * If we already have plenty of memory free for
6458 * compaction in this zone, don't free any more.
6459 * Even though compaction is invoked for any
6460 * non-zero order, only frequent costly order
6461 * reclamation is disruptive enough to become a
6462 * noticeable problem, like transparent huge
6463 * page allocations.
6464 */
6465 if (IS_ENABLED(CONFIG_COMPACTION) &&
6466 sc->order > PAGE_ALLOC_COSTLY_ORDER &&
6467 compaction_ready(zone, sc)) {
6468 sc->compaction_ready = true;
6469 continue;
6470 }
6471
6472 /*
6473 * Shrink each node in the zonelist once. If the
6474 * zonelist is ordered by zone (not the default) then a
6475 * node may be shrunk multiple times but in that case
6476 * the user prefers lower zones being preserved.
6477 */
6478 if (zone->zone_pgdat == last_pgdat)
6479 continue;
6480
6481 /*
6482 * This steals pages from memory cgroups over softlimit
6483 * and returns the number of reclaimed pages and
6484 * scanned pages. This works for global memory pressure
6485 * and balancing, not for a memcg's limit.
6486 */
6487 nr_soft_scanned = 0;
6488 nr_soft_reclaimed = memcg1_soft_limit_reclaim(zone->zone_pgdat,
6489 sc->order, sc->gfp_mask,
6490 &nr_soft_scanned);
6491 sc->nr_reclaimed += nr_soft_reclaimed;
6492 sc->nr_scanned += nr_soft_scanned;
6493 /* need some check for avoid more shrink_zone() */
6494 }
6495
6496 if (!first_pgdat)
6497 first_pgdat = zone->zone_pgdat;
6498
6499 /* See comment about same check for global reclaim above */
6500 if (zone->zone_pgdat == last_pgdat)
6501 continue;
6502 last_pgdat = zone->zone_pgdat;
6503 shrink_node(zone->zone_pgdat, sc);
6504 }
6505
6506 if (first_pgdat)
6507 consider_reclaim_throttle(first_pgdat, sc);
6508
6509 /*
6510 * Restore to original mask to avoid the impact on the caller if we
6511 * promoted it to __GFP_HIGHMEM.
6512 */
6513 sc->gfp_mask = orig_mask;
6514}
6515
6516static void snapshot_refaults(struct mem_cgroup *target_memcg, pg_data_t *pgdat)
6517{
6518 struct lruvec *target_lruvec;
6519 unsigned long refaults;
6520
6521 if (lru_gen_enabled() && !lru_gen_switching())
6522 return;
6523
6524 target_lruvec = mem_cgroup_lruvec(target_memcg, pgdat);
6525 refaults = lruvec_page_state(target_lruvec, WORKINGSET_ACTIVATE_ANON);
6526 target_lruvec->refaults[WORKINGSET_ANON] = refaults;
6527 refaults = lruvec_page_state(target_lruvec, WORKINGSET_ACTIVATE_FILE);
6528 target_lruvec->refaults[WORKINGSET_FILE] = refaults;
6529}
6530
6531/*
6532 * This is the main entry point to direct page reclaim.
6533 *
6534 * If a full scan of the inactive list fails to free enough memory then we
6535 * are "out of memory" and something needs to be killed.
6536 *
6537 * If the caller is !__GFP_FS then the probability of a failure is reasonably
6538 * high - the zone may be full of dirty or under-writeback pages, which this
6539 * caller can't do much about. We kick the writeback threads and take explicit
6540 * naps in the hope that some of these pages can be written. But if the
6541 * allocating task holds filesystem locks which prevent writeout this might not
6542 * work, and the allocation attempt will fail.
6543 *
6544 * returns: 0, if no pages reclaimed
6545 * else, the number of pages reclaimed
6546 */
6547static unsigned long do_try_to_free_pages(struct zonelist *zonelist,
6548 struct scan_control *sc)
6549{
6550 int initial_priority = sc->priority;
6551 pg_data_t *last_pgdat;
6552 struct zoneref *z;
6553 struct zone *zone;
6554retry:
6555 delayacct_freepages_start();
6556
6557 if (!cgroup_reclaim(sc))
6558 __count_zid_vm_events(ALLOCSTALL, sc->reclaim_idx, 1);
6559
6560 do {
6561 if (!sc->proactive)
6562 vmpressure_prio(sc->gfp_mask, sc->target_mem_cgroup,
6563 sc->priority);
6564 sc->nr_scanned = 0;
6565 shrink_zones(zonelist, sc);
6566
6567 if (sc->nr_reclaimed >= sc->nr_to_reclaim)
6568 break;
6569
6570 if (sc->compaction_ready)
6571 break;
6572 } while (--sc->priority >= 0);
6573
6574 last_pgdat = NULL;
6575 for_each_zone_zonelist_nodemask(zone, z, zonelist, sc->reclaim_idx,
6576 sc->nodemask) {
6577 if (zone->zone_pgdat == last_pgdat)
6578 continue;
6579 last_pgdat = zone->zone_pgdat;
6580
6581 snapshot_refaults(sc->target_mem_cgroup, zone->zone_pgdat);
6582
6583 if (cgroup_reclaim(sc)) {
6584 struct lruvec *lruvec;
6585
6586 lruvec = mem_cgroup_lruvec(sc->target_mem_cgroup,
6587 zone->zone_pgdat);
6588 clear_bit(LRUVEC_CGROUP_CONGESTED, &lruvec->flags);
6589 }
6590 }
6591
6592 delayacct_freepages_end();
6593
6594 if (sc->nr_reclaimed)
6595 return sc->nr_reclaimed;
6596
6597 /* Aborted reclaim to try compaction? don't OOM, then */
6598 if (sc->compaction_ready)
6599 return 1;
6600
6601 /*
6602 * In most cases, direct reclaimers can do partial walks
6603 * through the cgroup tree to meet the reclaim goal while
6604 * keeping latency low. Since the iterator state is shared
6605 * among all direct reclaim invocations (to retain fairness
6606 * among cgroups), though, high concurrency can result in
6607 * individual threads not seeing enough cgroups to make
6608 * meaningful forward progress. Avoid false OOMs in this case.
6609 */
6610 if (!sc->memcg_full_walk) {
6611 sc->priority = initial_priority;
6612 sc->memcg_full_walk = 1;
6613 goto retry;
6614 }
6615
6616 /*
6617 * We make inactive:active ratio decisions based on the node's
6618 * composition of memory, but a restrictive reclaim_idx or a
6619 * memory.low cgroup setting can exempt large amounts of
6620 * memory from reclaim. Neither of which are very common, so
6621 * instead of doing costly eligibility calculations of the
6622 * entire cgroup subtree up front, we assume the estimates are
6623 * good, and retry with forcible deactivation if that fails.
6624 */
6625 if (sc->skipped_deactivate) {
6626 sc->priority = initial_priority;
6627 sc->force_deactivate = 1;
6628 sc->skipped_deactivate = 0;
6629 goto retry;
6630 }
6631
6632 /* Untapped cgroup reserves? Don't OOM, retry. */
6633 if (sc->memcg_low_skipped) {
6634 sc->priority = initial_priority;
6635 sc->force_deactivate = 0;
6636 sc->memcg_low_reclaim = 1;
6637 sc->memcg_low_skipped = 0;
6638 goto retry;
6639 }
6640
6641 return 0;
6642}
6643
6644static bool allow_direct_reclaim(pg_data_t *pgdat)
6645{
6646 struct zone *zone;
6647 unsigned long pfmemalloc_reserve = 0;
6648 unsigned long free_pages = 0;
6649 int i;
6650 bool wmark_ok;
6651
6652 if (kswapd_test_hopeless(pgdat))
6653 return true;
6654
6655 for_each_managed_zone_pgdat(zone, pgdat, i, ZONE_NORMAL) {
6656 if (!zone_reclaimable_pages(zone) && zone_page_state_snapshot(zone, NR_FREE_PAGES))
6657 continue;
6658
6659 pfmemalloc_reserve += min_wmark_pages(zone);
6660 free_pages += zone_page_state_snapshot(zone, NR_FREE_PAGES);
6661 }
6662
6663 /* If there are no reserves (unexpected config) then do not throttle */
6664 if (!pfmemalloc_reserve)
6665 return true;
6666
6667 wmark_ok = free_pages > pfmemalloc_reserve / 2;
6668
6669 /* kswapd must be awake if processes are being throttled */
6670 if (!wmark_ok && waitqueue_active(&pgdat->kswapd_wait)) {
6671 if (READ_ONCE(pgdat->kswapd_highest_zoneidx) > ZONE_NORMAL)
6672 WRITE_ONCE(pgdat->kswapd_highest_zoneidx, ZONE_NORMAL);
6673
6674 wake_up_interruptible(&pgdat->kswapd_wait);
6675 }
6676
6677 return wmark_ok;
6678}
6679
6680/*
6681 * Throttle direct reclaimers if backing storage is backed by the network
6682 * and the PFMEMALLOC reserve for the preferred node is getting dangerously
6683 * depleted. kswapd will continue to make progress and wake the processes
6684 * when the low watermark is reached.
6685 *
6686 * Returns true if a fatal signal was delivered during throttling. If this
6687 * happens, the page allocator should not consider triggering the OOM killer.
6688 */
6689static bool throttle_direct_reclaim(gfp_t gfp_mask, struct zonelist *zonelist,
6690 const nodemask_t *nodemask)
6691{
6692 struct zoneref *z;
6693 struct zone *zone;
6694 pg_data_t *pgdat = NULL;
6695
6696 /*
6697 * Kernel threads should not be throttled as they may be indirectly
6698 * responsible for cleaning pages necessary for reclaim to make forward
6699 * progress. kjournald for example may enter direct reclaim while
6700 * committing a transaction where throttling it could forcing other
6701 * processes to block on log_wait_commit().
6702 */
6703 if (current->flags & PF_KTHREAD)
6704 goto out;
6705
6706 /*
6707 * If a fatal signal is pending, this process should not throttle.
6708 * It should return quickly so it can exit and free its memory
6709 */
6710 if (fatal_signal_pending(current))
6711 goto out;
6712
6713 /*
6714 * Check if the pfmemalloc reserves are ok by finding the first node
6715 * with a usable ZONE_NORMAL or lower zone. The expectation is that
6716 * GFP_KERNEL will be required for allocating network buffers when
6717 * swapping over the network so ZONE_HIGHMEM is unusable.
6718 *
6719 * Throttling is based on the first usable node and throttled processes
6720 * wait on a queue until kswapd makes progress and wakes them. There
6721 * is an affinity then between processes waking up and where reclaim
6722 * progress has been made assuming the process wakes on the same node.
6723 * More importantly, processes running on remote nodes will not compete
6724 * for remote pfmemalloc reserves and processes on different nodes
6725 * should make reasonable progress.
6726 */
6727 for_each_zone_zonelist_nodemask(zone, z, zonelist,
6728 gfp_zone(gfp_mask), nodemask) {
6729 if (zone_idx(zone) > ZONE_NORMAL)
6730 continue;
6731
6732 /* Throttle based on the first usable node */
6733 pgdat = zone->zone_pgdat;
6734 if (allow_direct_reclaim(pgdat))
6735 goto out;
6736 break;
6737 }
6738
6739 /* If no zone was usable by the allocation flags then do not throttle */
6740 if (!pgdat)
6741 goto out;
6742
6743 /* Account for the throttling */
6744 count_vm_event(PGSCAN_DIRECT_THROTTLE);
6745
6746 /*
6747 * If the caller cannot enter the filesystem, it's possible that it
6748 * is due to the caller holding an FS lock or performing a journal
6749 * transaction in the case of a filesystem like ext[3|4]. In this case,
6750 * it is not safe to block on pfmemalloc_wait as kswapd could be
6751 * blocked waiting on the same lock. Instead, throttle for up to a
6752 * second before continuing.
6753 */
6754 if (!(gfp_mask & __GFP_FS))
6755 wait_event_interruptible_timeout(pgdat->pfmemalloc_wait,
6756 allow_direct_reclaim(pgdat), HZ);
6757 else
6758 /* Throttle until kswapd wakes the process */
6759 wait_event_killable(zone->zone_pgdat->pfmemalloc_wait,
6760 allow_direct_reclaim(pgdat));
6761
6762 if (fatal_signal_pending(current))
6763 return true;
6764
6765out:
6766 return false;
6767}
6768
6769unsigned long try_to_free_pages(struct zonelist *zonelist, int order,
6770 gfp_t gfp_mask, const nodemask_t *nodemask)
6771{
6772 unsigned long nr_reclaimed;
6773 struct scan_control sc = {
6774 .nr_to_reclaim = SWAP_CLUSTER_MAX,
6775 .gfp_mask = current_gfp_context(gfp_mask),
6776 .reclaim_idx = gfp_zone(gfp_mask),
6777 .order = order,
6778 .nodemask = nodemask,
6779 .priority = DEF_PRIORITY,
6780 .may_writepage = 1,
6781 .may_unmap = 1,
6782 .may_swap = 1,
6783 };
6784
6785 /*
6786 * scan_control uses s8 fields for order, priority, and reclaim_idx.
6787 * Confirm they are large enough for max values.
6788 */
6789 BUILD_BUG_ON(MAX_PAGE_ORDER >= S8_MAX);
6790 BUILD_BUG_ON(DEF_PRIORITY > S8_MAX);
6791 BUILD_BUG_ON(MAX_NR_ZONES > S8_MAX);
6792
6793 /*
6794 * Do not enter reclaim if fatal signal was delivered while throttled.
6795 * 1 is returned so that the page allocator does not OOM kill at this
6796 * point.
6797 */
6798 if (throttle_direct_reclaim(sc.gfp_mask, zonelist, nodemask))
6799 return 1;
6800
6801 set_task_reclaim_state(current, &sc.reclaim_state);
6802 trace_mm_vmscan_direct_reclaim_begin(sc.gfp_mask, order, NULL);
6803
6804 nr_reclaimed = do_try_to_free_pages(zonelist, &sc);
6805
6806 trace_mm_vmscan_direct_reclaim_end(nr_reclaimed, NULL);
6807 set_task_reclaim_state(current, NULL);
6808
6809 return nr_reclaimed;
6810}
6811
6812#ifdef CONFIG_MEMCG
6813
6814/* Only used by soft limit reclaim. Do not reuse for anything else. */
6815unsigned long mem_cgroup_shrink_node(struct mem_cgroup *memcg,
6816 gfp_t gfp_mask, bool noswap,
6817 pg_data_t *pgdat,
6818 unsigned long *nr_scanned)
6819{
6820 struct lruvec *lruvec = mem_cgroup_lruvec(memcg, pgdat);
6821 struct scan_control sc = {
6822 .nr_to_reclaim = SWAP_CLUSTER_MAX,
6823 .target_mem_cgroup = memcg,
6824 .may_writepage = 1,
6825 .may_unmap = 1,
6826 .reclaim_idx = MAX_NR_ZONES - 1,
6827 .may_swap = !noswap,
6828 };
6829
6830 WARN_ON_ONCE(!current->reclaim_state);
6831
6832 sc.gfp_mask = (gfp_mask & GFP_RECLAIM_MASK) |
6833 (GFP_HIGHUSER_MOVABLE & ~GFP_RECLAIM_MASK);
6834
6835 trace_mm_vmscan_memcg_softlimit_reclaim_begin(sc.gfp_mask,
6836 sc.order,
6837 memcg);
6838
6839 /*
6840 * NOTE: Although we can get the priority field, using it
6841 * here is not a good idea, since it limits the pages we can scan.
6842 * if we don't reclaim here, the shrink_node from balance_pgdat
6843 * will pick up pages from other mem cgroup's as well. We hack
6844 * the priority and make it zero.
6845 */
6846 shrink_lruvec(lruvec, &sc);
6847
6848 trace_mm_vmscan_memcg_softlimit_reclaim_end(sc.nr_reclaimed, memcg);
6849
6850 *nr_scanned = sc.nr_scanned;
6851
6852 return sc.nr_reclaimed;
6853}
6854
6855unsigned long try_to_free_mem_cgroup_pages(struct mem_cgroup *memcg,
6856 unsigned long nr_pages,
6857 gfp_t gfp_mask,
6858 unsigned int reclaim_options,
6859 int *swappiness)
6860{
6861 unsigned long nr_reclaimed;
6862 unsigned int noreclaim_flag;
6863 struct scan_control sc = {
6864 .nr_to_reclaim = max(nr_pages, SWAP_CLUSTER_MAX),
6865 .proactive_swappiness = swappiness,
6866 .gfp_mask = (current_gfp_context(gfp_mask) & GFP_RECLAIM_MASK) |
6867 (GFP_HIGHUSER_MOVABLE & ~GFP_RECLAIM_MASK),
6868 .reclaim_idx = MAX_NR_ZONES - 1,
6869 .target_mem_cgroup = memcg,
6870 .priority = DEF_PRIORITY,
6871 .may_writepage = 1,
6872 .may_unmap = 1,
6873 .may_swap = !!(reclaim_options & MEMCG_RECLAIM_MAY_SWAP),
6874 .proactive = !!(reclaim_options & MEMCG_RECLAIM_PROACTIVE),
6875 };
6876 /*
6877 * Traverse the ZONELIST_FALLBACK zonelist of the current node to put
6878 * equal pressure on all the nodes. This is based on the assumption that
6879 * the reclaim does not bail out early.
6880 */
6881 struct zonelist *zonelist = node_zonelist(numa_node_id(), sc.gfp_mask);
6882
6883 set_task_reclaim_state(current, &sc.reclaim_state);
6884 trace_mm_vmscan_memcg_reclaim_begin(sc.gfp_mask, 0, memcg);
6885 noreclaim_flag = memalloc_noreclaim_save();
6886
6887 nr_reclaimed = do_try_to_free_pages(zonelist, &sc);
6888
6889 memalloc_noreclaim_restore(noreclaim_flag);
6890 trace_mm_vmscan_memcg_reclaim_end(nr_reclaimed, memcg);
6891 set_task_reclaim_state(current, NULL);
6892
6893 return nr_reclaimed;
6894}
6895#else
6896unsigned long try_to_free_mem_cgroup_pages(struct mem_cgroup *memcg,
6897 unsigned long nr_pages,
6898 gfp_t gfp_mask,
6899 unsigned int reclaim_options,
6900 int *swappiness)
6901{
6902 return 0;
6903}
6904#endif
6905
6906static void kswapd_age_node(struct pglist_data *pgdat, struct scan_control *sc)
6907{
6908 struct mem_cgroup *memcg;
6909 struct lruvec *lruvec;
6910
6911 if (lru_gen_enabled() || lru_gen_switching()) {
6912 lru_gen_age_node(pgdat, sc);
6913
6914 if (!lru_gen_switching())
6915 return;
6916
6917 }
6918
6919 lruvec = mem_cgroup_lruvec(NULL, pgdat);
6920 if (!can_age_anon_pages(lruvec, sc))
6921 return;
6922
6923 if (!inactive_is_low(lruvec, LRU_INACTIVE_ANON))
6924 return;
6925
6926 memcg = mem_cgroup_iter(NULL, NULL, NULL);
6927 do {
6928 lruvec = mem_cgroup_lruvec(memcg, pgdat);
6929 shrink_active_list(SWAP_CLUSTER_MAX, lruvec,
6930 sc, LRU_ACTIVE_ANON);
6931 memcg = mem_cgroup_iter(NULL, memcg, NULL);
6932 } while (memcg);
6933}
6934
6935static bool pgdat_watermark_boosted(pg_data_t *pgdat, int highest_zoneidx)
6936{
6937 int i;
6938 struct zone *zone;
6939
6940 /*
6941 * Check for watermark boosts top-down as the higher zones
6942 * are more likely to be boosted. Both watermarks and boosts
6943 * should not be checked at the same time as reclaim would
6944 * start prematurely when there is no boosting and a lower
6945 * zone is balanced.
6946 */
6947 for (i = highest_zoneidx; i >= 0; i--) {
6948 zone = pgdat->node_zones + i;
6949 if (!managed_zone(zone))
6950 continue;
6951
6952 if (zone->watermark_boost)
6953 return true;
6954 }
6955
6956 return false;
6957}
6958
6959/*
6960 * Returns true if there is an eligible zone balanced for the request order
6961 * and highest_zoneidx
6962 */
6963static bool pgdat_balanced(pg_data_t *pgdat, int order, int highest_zoneidx)
6964{
6965 int i;
6966 unsigned long mark = -1;
6967 struct zone *zone;
6968
6969 /*
6970 * Check watermarks bottom-up as lower zones are more likely to
6971 * meet watermarks.
6972 */
6973 for_each_managed_zone_pgdat(zone, pgdat, i, highest_zoneidx) {
6974 enum zone_stat_item item;
6975 unsigned long free_pages;
6976
6977 if (sysctl_numa_balancing_mode & NUMA_BALANCING_MEMORY_TIERING)
6978 mark = promo_wmark_pages(zone);
6979 else
6980 mark = high_wmark_pages(zone);
6981
6982 /*
6983 * In defrag_mode, watermarks must be met in whole
6984 * blocks to avoid polluting allocator fallbacks.
6985 *
6986 * However, kswapd usually cannot accomplish this on
6987 * its own and needs kcompactd support. Once it's
6988 * reclaimed a compaction gap, and kswapd_shrink_node
6989 * has dropped order, simply ensure there are enough
6990 * base pages for compaction, wake kcompactd & sleep.
6991 */
6992 if (defrag_mode && order)
6993 item = NR_FREE_PAGES_BLOCKS;
6994 else
6995 item = NR_FREE_PAGES;
6996
6997 /*
6998 * When there is a high number of CPUs in the system,
6999 * the cumulative error from the vmstat per-cpu cache
7000 * can blur the line between the watermarks. In that
7001 * case, be safe and get an accurate snapshot.
7002 *
7003 * TODO: NR_FREE_PAGES_BLOCKS moves in steps of
7004 * pageblock_nr_pages, while the vmstat pcp threshold
7005 * is limited to 125. On many configurations that
7006 * counter won't actually be per-cpu cached. But keep
7007 * things simple for now; revisit when somebody cares.
7008 */
7009 free_pages = zone_page_state(zone, item);
7010 if (zone->percpu_drift_mark && free_pages < zone->percpu_drift_mark)
7011 free_pages = zone_page_state_snapshot(zone, item);
7012
7013 if (__zone_watermark_ok(zone, order, mark, highest_zoneidx,
7014 0, free_pages))
7015 return true;
7016 }
7017
7018 /*
7019 * If a node has no managed zone within highest_zoneidx, it does not
7020 * need balancing by definition. This can happen if a zone-restricted
7021 * allocation tries to wake a remote kswapd.
7022 */
7023 if (mark == -1)
7024 return true;
7025
7026 return false;
7027}
7028
7029/* Clear pgdat state for congested, dirty or under writeback. */
7030static void clear_pgdat_congested(pg_data_t *pgdat)
7031{
7032 struct lruvec *lruvec = mem_cgroup_lruvec(NULL, pgdat);
7033
7034 clear_bit(LRUVEC_NODE_CONGESTED, &lruvec->flags);
7035 clear_bit(LRUVEC_CGROUP_CONGESTED, &lruvec->flags);
7036 clear_bit(PGDAT_WRITEBACK, &pgdat->flags);
7037}
7038
7039/*
7040 * Prepare kswapd for sleeping. This verifies that there are no processes
7041 * waiting in throttle_direct_reclaim() and that watermarks have been met.
7042 *
7043 * Returns true if kswapd is ready to sleep
7044 */
7045static bool prepare_kswapd_sleep(pg_data_t *pgdat, int order,
7046 int highest_zoneidx)
7047{
7048 /*
7049 * The throttled processes are normally woken up in balance_pgdat() as
7050 * soon as allow_direct_reclaim() is true. But there is a potential
7051 * race between when kswapd checks the watermarks and a process gets
7052 * throttled. There is also a potential race if processes get
7053 * throttled, kswapd wakes, a large process exits thereby balancing the
7054 * zones, which causes kswapd to exit balance_pgdat() before reaching
7055 * the wake up checks. If kswapd is going to sleep, no process should
7056 * be sleeping on pfmemalloc_wait, so wake them now if necessary. If
7057 * the wake up is premature, processes will wake kswapd and get
7058 * throttled again. The difference from wake ups in balance_pgdat() is
7059 * that here we are under prepare_to_wait().
7060 */
7061 if (waitqueue_active(&pgdat->pfmemalloc_wait))
7062 wake_up_all(&pgdat->pfmemalloc_wait);
7063
7064 /* Hopeless node, leave it to direct reclaim */
7065 if (kswapd_test_hopeless(pgdat))
7066 return true;
7067
7068 if (pgdat_balanced(pgdat, order, highest_zoneidx)) {
7069 clear_pgdat_congested(pgdat);
7070 return true;
7071 }
7072
7073 return false;
7074}
7075
7076/*
7077 * kswapd shrinks a node of pages that are at or below the highest usable
7078 * zone that is currently unbalanced.
7079 *
7080 * Returns true if kswapd scanned at least the requested number of pages to
7081 * reclaim or if the lack of progress was due to pages under writeback.
7082 * This is used to determine if the scanning priority needs to be raised.
7083 */
7084static bool kswapd_shrink_node(pg_data_t *pgdat,
7085 struct scan_control *sc)
7086{
7087 struct zone *zone;
7088 int z;
7089 unsigned long nr_reclaimed = sc->nr_reclaimed;
7090
7091 /* Reclaim a number of pages proportional to the number of zones */
7092 sc->nr_to_reclaim = 0;
7093 for_each_managed_zone_pgdat(zone, pgdat, z, sc->reclaim_idx) {
7094 sc->nr_to_reclaim += max(high_wmark_pages(zone), SWAP_CLUSTER_MAX);
7095 }
7096
7097 /*
7098 * Historically care was taken to put equal pressure on all zones but
7099 * now pressure is applied based on node LRU order.
7100 */
7101 shrink_node(pgdat, sc);
7102
7103 /*
7104 * Fragmentation may mean that the system cannot be rebalanced for
7105 * high-order allocations. If at least the compaction gap has been
7106 * reclaimed then recheck watermarks only at order-0 to prevent
7107 * excessive reclaim. Assume that a process requested a high-order
7108 * can direct reclaim/compact.
7109 */
7110 if (sc->order && sc->nr_reclaimed >= compact_gap(sc->order))
7111 sc->order = 0;
7112
7113 /* account for progress from mm_account_reclaimed_pages() */
7114 return max(sc->nr_scanned, sc->nr_reclaimed - nr_reclaimed) >= sc->nr_to_reclaim;
7115}
7116
7117/* Page allocator PCP high watermark is lowered if reclaim is active. */
7118static inline void
7119update_reclaim_active(pg_data_t *pgdat, int highest_zoneidx, bool active)
7120{
7121 int i;
7122 struct zone *zone;
7123
7124 for_each_managed_zone_pgdat(zone, pgdat, i, highest_zoneidx) {
7125 if (active)
7126 set_bit(ZONE_RECLAIM_ACTIVE, &zone->flags);
7127 else
7128 clear_bit(ZONE_RECLAIM_ACTIVE, &zone->flags);
7129 }
7130}
7131
7132static inline void
7133set_reclaim_active(pg_data_t *pgdat, int highest_zoneidx)
7134{
7135 update_reclaim_active(pgdat, highest_zoneidx, true);
7136}
7137
7138static inline void
7139clear_reclaim_active(pg_data_t *pgdat, int highest_zoneidx)
7140{
7141 update_reclaim_active(pgdat, highest_zoneidx, false);
7142}
7143
7144/*
7145 * For kswapd, balance_pgdat() will reclaim pages across a node from zones
7146 * that are eligible for use by the caller until at least one zone is
7147 * balanced.
7148 *
7149 * Returns the order kswapd finished reclaiming at.
7150 *
7151 * kswapd scans the zones in the highmem->normal->dma direction. It skips
7152 * zones which have free_pages > high_wmark_pages(zone), but once a zone is
7153 * found to have free_pages <= high_wmark_pages(zone), any page in that zone
7154 * or lower is eligible for reclaim until at least one usable zone is
7155 * balanced.
7156 */
7157static int balance_pgdat(pg_data_t *pgdat, int order, int highest_zoneidx)
7158{
7159 int i;
7160 unsigned long nr_soft_reclaimed;
7161 unsigned long nr_soft_scanned;
7162 unsigned long pflags;
7163 unsigned long nr_boost_reclaim;
7164 unsigned long zone_boosts[MAX_NR_ZONES] = { 0, };
7165 bool boosted;
7166 struct zone *zone;
7167 struct scan_control sc = {
7168 .gfp_mask = GFP_KERNEL,
7169 .order = order,
7170 .may_unmap = 1,
7171 };
7172
7173 trace_mm_vmscan_balance_pgdat_begin(pgdat->node_id, order,
7174 highest_zoneidx);
7175 set_task_reclaim_state(current, &sc.reclaim_state);
7176 psi_memstall_enter(&pflags);
7177 __fs_reclaim_acquire(_THIS_IP_);
7178
7179 count_vm_event(PAGEOUTRUN);
7180
7181 /*
7182 * Account for the reclaim boost. Note that the zone boost is left in
7183 * place so that parallel allocations that are near the watermark will
7184 * stall or direct reclaim until kswapd is finished.
7185 */
7186 nr_boost_reclaim = 0;
7187 for_each_managed_zone_pgdat(zone, pgdat, i, highest_zoneidx) {
7188 nr_boost_reclaim += zone->watermark_boost;
7189 zone_boosts[i] = zone->watermark_boost;
7190 }
7191 boosted = nr_boost_reclaim;
7192
7193restart:
7194 set_reclaim_active(pgdat, highest_zoneidx);
7195 sc.priority = DEF_PRIORITY;
7196 do {
7197 unsigned long nr_reclaimed = sc.nr_reclaimed;
7198 bool raise_priority = true;
7199 bool balanced;
7200 bool ret;
7201 bool was_frozen;
7202
7203 sc.reclaim_idx = highest_zoneidx;
7204
7205 /*
7206 * If the number of buffer_heads exceeds the maximum allowed
7207 * then consider reclaiming from all zones. This has a dual
7208 * purpose -- on 64-bit systems it is expected that
7209 * buffer_heads are stripped during active rotation. On 32-bit
7210 * systems, highmem pages can pin lowmem memory and shrinking
7211 * buffers can relieve lowmem pressure. Reclaim may still not
7212 * go ahead if all eligible zones for the original allocation
7213 * request are balanced to avoid excessive reclaim from kswapd.
7214 */
7215 if (buffer_heads_over_limit) {
7216 for (i = MAX_NR_ZONES - 1; i >= 0; i--) {
7217 zone = pgdat->node_zones + i;
7218 if (!managed_zone(zone))
7219 continue;
7220
7221 sc.reclaim_idx = i;
7222 break;
7223 }
7224 }
7225
7226 /*
7227 * If the pgdat is imbalanced then ignore boosting and preserve
7228 * the watermarks for a later time and restart. Note that the
7229 * zone watermarks will be still reset at the end of balancing
7230 * on the grounds that the normal reclaim should be enough to
7231 * re-evaluate if boosting is required when kswapd next wakes.
7232 */
7233 balanced = pgdat_balanced(pgdat, sc.order, highest_zoneidx);
7234 if (!balanced && nr_boost_reclaim) {
7235 nr_boost_reclaim = 0;
7236 goto restart;
7237 }
7238
7239 /*
7240 * If boosting is not active then only reclaim if there are no
7241 * eligible zones. Note that sc.reclaim_idx is not used as
7242 * buffer_heads_over_limit may have adjusted it.
7243 */
7244 if (!nr_boost_reclaim && balanced)
7245 goto out;
7246
7247 /* Limit the priority of boosting to avoid reclaim writeback */
7248 if (nr_boost_reclaim && sc.priority == DEF_PRIORITY - 2)
7249 raise_priority = false;
7250
7251 /*
7252 * Do not writeback or swap pages for boosted reclaim. The
7253 * intent is to relieve pressure not issue sub-optimal IO
7254 * from reclaim context. If no pages are reclaimed, the
7255 * reclaim will be aborted.
7256 */
7257 sc.may_writepage = !nr_boost_reclaim;
7258 sc.may_swap = !nr_boost_reclaim;
7259
7260 /*
7261 * Do some background aging, to give pages a chance to be
7262 * referenced before reclaiming. All pages are rotated
7263 * regardless of classzone as this is about consistent aging.
7264 */
7265 kswapd_age_node(pgdat, &sc);
7266
7267 /* Call soft limit reclaim before calling shrink_node. */
7268 sc.nr_scanned = 0;
7269 nr_soft_scanned = 0;
7270 nr_soft_reclaimed = memcg1_soft_limit_reclaim(pgdat, sc.order,
7271 sc.gfp_mask, &nr_soft_scanned);
7272 sc.nr_reclaimed += nr_soft_reclaimed;
7273
7274 /*
7275 * There should be no need to raise the scanning priority if
7276 * enough pages are already being scanned that the high
7277 * watermark would be met at 100% efficiency.
7278 */
7279 if (kswapd_shrink_node(pgdat, &sc))
7280 raise_priority = false;
7281
7282 /*
7283 * If the low watermark is met there is no need for processes
7284 * to be throttled on pfmemalloc_wait as they should now be
7285 * able to safely make forward progress. Wake them
7286 */
7287 if (waitqueue_active(&pgdat->pfmemalloc_wait) &&
7288 allow_direct_reclaim(pgdat))
7289 wake_up_all(&pgdat->pfmemalloc_wait);
7290
7291 /* Check if kswapd should be suspending */
7292 __fs_reclaim_release(_THIS_IP_);
7293 ret = kthread_freezable_should_stop(&was_frozen);
7294 __fs_reclaim_acquire(_THIS_IP_);
7295 if (was_frozen || ret)
7296 break;
7297
7298 /*
7299 * Raise priority if scanning rate is too low or there was no
7300 * progress in reclaiming pages
7301 */
7302 nr_reclaimed = sc.nr_reclaimed - nr_reclaimed;
7303 nr_boost_reclaim -= min(nr_boost_reclaim, nr_reclaimed);
7304
7305 /*
7306 * If reclaim made no progress for a boost, stop reclaim as
7307 * IO cannot be queued and it could be an infinite loop in
7308 * extreme circumstances.
7309 */
7310 if (nr_boost_reclaim && !nr_reclaimed)
7311 break;
7312
7313 if (raise_priority || !nr_reclaimed)
7314 sc.priority--;
7315 } while (sc.priority >= 1);
7316
7317 /*
7318 * Restart only if it went through the priority loop all the way,
7319 * but cache_trim_mode didn't work.
7320 */
7321 if (!sc.nr_reclaimed && sc.priority < 1 &&
7322 !sc.no_cache_trim_mode && sc.cache_trim_mode_failed) {
7323 sc.no_cache_trim_mode = 1;
7324 goto restart;
7325 }
7326
7327 /*
7328 * If the reclaim was boosted, we might still be far from the
7329 * watermark_high at this point. We need to avoid increasing the
7330 * failure count to prevent the kswapd thread from stopping.
7331 */
7332 if (!sc.nr_reclaimed && !boosted) {
7333 int fail_cnt = atomic_inc_return(&pgdat->kswapd_failures);
7334 /* kswapd context, low overhead to trace every failure */
7335 trace_mm_vmscan_kswapd_reclaim_fail(pgdat->node_id, fail_cnt);
7336 }
7337
7338out:
7339 clear_reclaim_active(pgdat, highest_zoneidx);
7340
7341 /* If reclaim was boosted, account for the reclaim done in this pass */
7342 if (boosted) {
7343 unsigned long flags;
7344
7345 for (i = 0; i <= highest_zoneidx; i++) {
7346 if (!zone_boosts[i])
7347 continue;
7348
7349 /* Increments are under the zone lock */
7350 zone = pgdat->node_zones + i;
7351 spin_lock_irqsave(&zone->lock, flags);
7352 zone->watermark_boost -= min(zone->watermark_boost, zone_boosts[i]);
7353 spin_unlock_irqrestore(&zone->lock, flags);
7354 }
7355
7356 /*
7357 * As there is now likely space, wakeup kcompact to defragment
7358 * pageblocks.
7359 */
7360 wakeup_kcompactd(pgdat, pageblock_order, highest_zoneidx);
7361 }
7362
7363 snapshot_refaults(NULL, pgdat);
7364 __fs_reclaim_release(_THIS_IP_);
7365 psi_memstall_leave(&pflags);
7366 set_task_reclaim_state(current, NULL);
7367
7368 trace_mm_vmscan_balance_pgdat_end(pgdat->node_id, sc.order,
7369 highest_zoneidx, sc.nr_reclaimed);
7370
7371 /*
7372 * Return the order kswapd stopped reclaiming at as
7373 * prepare_kswapd_sleep() takes it into account. If another caller
7374 * entered the allocator slow path while kswapd was awake, order will
7375 * remain at the higher level.
7376 */
7377 return sc.order;
7378}
7379
7380/*
7381 * The pgdat->kswapd_highest_zoneidx is used to pass the highest zone index to
7382 * be reclaimed by kswapd from the waker. If the value is MAX_NR_ZONES which is
7383 * not a valid index then either kswapd runs for first time or kswapd couldn't
7384 * sleep after previous reclaim attempt (node is still unbalanced). In that
7385 * case return the zone index of the previous kswapd reclaim cycle.
7386 */
7387static enum zone_type kswapd_highest_zoneidx(pg_data_t *pgdat,
7388 enum zone_type prev_highest_zoneidx)
7389{
7390 enum zone_type curr_idx = READ_ONCE(pgdat->kswapd_highest_zoneidx);
7391
7392 return curr_idx == MAX_NR_ZONES ? prev_highest_zoneidx : curr_idx;
7393}
7394
7395static void kswapd_try_to_sleep(pg_data_t *pgdat, int alloc_order, int reclaim_order,
7396 unsigned int highest_zoneidx)
7397{
7398 long remaining = 0;
7399 DEFINE_WAIT(wait);
7400
7401 if (freezing(current) || kthread_should_stop())
7402 return;
7403
7404 prepare_to_wait(&pgdat->kswapd_wait, &wait, TASK_INTERRUPTIBLE);
7405
7406 /*
7407 * Try to sleep for a short interval. Note that kcompactd will only be
7408 * woken if it is possible to sleep for a short interval. This is
7409 * deliberate on the assumption that if reclaim cannot keep an
7410 * eligible zone balanced that it's also unlikely that compaction will
7411 * succeed.
7412 */
7413 if (prepare_kswapd_sleep(pgdat, reclaim_order, highest_zoneidx)) {
7414 /*
7415 * Compaction records what page blocks it recently failed to
7416 * isolate pages from and skips them in the future scanning.
7417 * When kswapd is going to sleep, it is reasonable to assume
7418 * that pages and compaction may succeed so reset the cache.
7419 */
7420 reset_isolation_suitable(pgdat);
7421
7422 /*
7423 * We have freed the memory, now we should compact it to make
7424 * allocation of the requested order possible.
7425 */
7426 wakeup_kcompactd(pgdat, alloc_order, highest_zoneidx);
7427
7428 remaining = schedule_timeout(HZ/10);
7429
7430 /*
7431 * If woken prematurely then reset kswapd_highest_zoneidx and
7432 * order. The values will either be from a wakeup request or
7433 * the previous request that slept prematurely.
7434 */
7435 if (remaining) {
7436 WRITE_ONCE(pgdat->kswapd_highest_zoneidx,
7437 kswapd_highest_zoneidx(pgdat,
7438 highest_zoneidx));
7439
7440 if (READ_ONCE(pgdat->kswapd_order) < reclaim_order)
7441 WRITE_ONCE(pgdat->kswapd_order, reclaim_order);
7442 }
7443
7444 finish_wait(&pgdat->kswapd_wait, &wait);
7445 prepare_to_wait(&pgdat->kswapd_wait, &wait, TASK_INTERRUPTIBLE);
7446 }
7447
7448 /*
7449 * After a short sleep, check if it was a premature sleep. If not, then
7450 * go fully to sleep until explicitly woken up.
7451 */
7452 if (!remaining &&
7453 prepare_kswapd_sleep(pgdat, reclaim_order, highest_zoneidx)) {
7454 trace_mm_vmscan_kswapd_sleep(pgdat->node_id);
7455
7456 /*
7457 * vmstat counters are not perfectly accurate and the estimated
7458 * value for counters such as NR_FREE_PAGES can deviate from the
7459 * true value by nr_online_cpus * threshold. To avoid the zone
7460 * watermarks being breached while under pressure, we reduce the
7461 * per-cpu vmstat threshold while kswapd is awake and restore
7462 * them before going back to sleep.
7463 */
7464 set_pgdat_percpu_threshold(pgdat, calculate_normal_threshold);
7465
7466 if (!kthread_should_stop())
7467 schedule();
7468
7469 set_pgdat_percpu_threshold(pgdat, calculate_pressure_threshold);
7470 } else {
7471 if (remaining)
7472 count_vm_event(KSWAPD_LOW_WMARK_HIT_QUICKLY);
7473 else
7474 count_vm_event(KSWAPD_HIGH_WMARK_HIT_QUICKLY);
7475 }
7476 finish_wait(&pgdat->kswapd_wait, &wait);
7477}
7478
7479/*
7480 * The background pageout daemon, started as a kernel thread
7481 * from the init process.
7482 *
7483 * This basically trickles out pages so that we have _some_
7484 * free memory available even if there is no other activity
7485 * that frees anything up. This is needed for things like routing
7486 * etc, where we otherwise might have all activity going on in
7487 * asynchronous contexts that cannot page things out.
7488 *
7489 * If there are applications that are active memory-allocators
7490 * (most normal use), this basically shouldn't matter.
7491 */
7492static int kswapd(void *p)
7493{
7494 unsigned int alloc_order, reclaim_order;
7495 unsigned int highest_zoneidx = MAX_NR_ZONES - 1;
7496 pg_data_t *pgdat = (pg_data_t *)p;
7497 struct task_struct *tsk = current;
7498
7499 /*
7500 * Tell the memory management that we're a "memory allocator",
7501 * and that if we need more memory we should get access to it
7502 * regardless (see "__alloc_pages()"). "kswapd" should
7503 * never get caught in the normal page freeing logic.
7504 *
7505 * (Kswapd normally doesn't need memory anyway, but sometimes
7506 * you need a small amount of memory in order to be able to
7507 * page out something else, and this flag essentially protects
7508 * us from recursively trying to free more memory as we're
7509 * trying to free the first piece of memory in the first place).
7510 */
7511 tsk->flags |= PF_MEMALLOC | PF_KSWAPD;
7512 set_freezable();
7513
7514 WRITE_ONCE(pgdat->kswapd_order, 0);
7515 WRITE_ONCE(pgdat->kswapd_highest_zoneidx, MAX_NR_ZONES);
7516 atomic_set(&pgdat->nr_writeback_throttled, 0);
7517 for ( ; ; ) {
7518 bool was_frozen;
7519
7520 alloc_order = reclaim_order = READ_ONCE(pgdat->kswapd_order);
7521 highest_zoneidx = kswapd_highest_zoneidx(pgdat,
7522 highest_zoneidx);
7523
7524kswapd_try_sleep:
7525 kswapd_try_to_sleep(pgdat, alloc_order, reclaim_order,
7526 highest_zoneidx);
7527
7528 /* Read the new order and highest_zoneidx */
7529 alloc_order = READ_ONCE(pgdat->kswapd_order);
7530 highest_zoneidx = kswapd_highest_zoneidx(pgdat,
7531 highest_zoneidx);
7532 WRITE_ONCE(pgdat->kswapd_order, 0);
7533 WRITE_ONCE(pgdat->kswapd_highest_zoneidx, MAX_NR_ZONES);
7534
7535 if (kthread_freezable_should_stop(&was_frozen))
7536 break;
7537
7538 /*
7539 * We can speed up thawing tasks if we don't call balance_pgdat
7540 * after returning from the refrigerator
7541 */
7542 if (was_frozen)
7543 continue;
7544
7545 /*
7546 * Reclaim begins at the requested order but if a high-order
7547 * reclaim fails then kswapd falls back to reclaiming for
7548 * order-0. If that happens, kswapd will consider sleeping
7549 * for the order it finished reclaiming at (reclaim_order)
7550 * but kcompactd is woken to compact for the original
7551 * request (alloc_order).
7552 */
7553 trace_mm_vmscan_kswapd_wake(pgdat->node_id, highest_zoneidx,
7554 alloc_order);
7555 reclaim_order = balance_pgdat(pgdat, alloc_order,
7556 highest_zoneidx);
7557 if (reclaim_order < alloc_order)
7558 goto kswapd_try_sleep;
7559 }
7560
7561 tsk->flags &= ~(PF_MEMALLOC | PF_KSWAPD);
7562
7563 return 0;
7564}
7565
7566/*
7567 * A zone is low on free memory or too fragmented for high-order memory. If
7568 * kswapd should reclaim (direct reclaim is deferred), wake it up for the zone's
7569 * pgdat. It will wake up kcompactd after reclaiming memory. If kswapd reclaim
7570 * has failed or is not needed, still wake up kcompactd if only compaction is
7571 * needed.
7572 */
7573void wakeup_kswapd(struct zone *zone, gfp_t gfp_flags, int order,
7574 enum zone_type highest_zoneidx)
7575{
7576 pg_data_t *pgdat;
7577 enum zone_type curr_idx;
7578
7579 if (!managed_zone(zone))
7580 return;
7581
7582 if (!cpuset_zone_allowed(zone, gfp_flags))
7583 return;
7584
7585 pgdat = zone->zone_pgdat;
7586 curr_idx = READ_ONCE(pgdat->kswapd_highest_zoneidx);
7587
7588 if (curr_idx == MAX_NR_ZONES || curr_idx < highest_zoneidx)
7589 WRITE_ONCE(pgdat->kswapd_highest_zoneidx, highest_zoneidx);
7590
7591 if (READ_ONCE(pgdat->kswapd_order) < order)
7592 WRITE_ONCE(pgdat->kswapd_order, order);
7593
7594 if (!waitqueue_active(&pgdat->kswapd_wait))
7595 return;
7596
7597 /* Hopeless node, leave it to direct reclaim if possible */
7598 if (kswapd_test_hopeless(pgdat) ||
7599 (pgdat_balanced(pgdat, order, highest_zoneidx) &&
7600 !pgdat_watermark_boosted(pgdat, highest_zoneidx))) {
7601 /*
7602 * There may be plenty of free memory available, but it's too
7603 * fragmented for high-order allocations. Wake up kcompactd
7604 * and rely on compaction_suitable() to determine if it's
7605 * needed. If it fails, it will defer subsequent attempts to
7606 * ratelimit its work.
7607 */
7608 if (!(gfp_flags & __GFP_DIRECT_RECLAIM))
7609 wakeup_kcompactd(pgdat, order, highest_zoneidx);
7610 return;
7611 }
7612
7613 trace_mm_vmscan_wakeup_kswapd(pgdat->node_id, highest_zoneidx, order,
7614 gfp_flags);
7615 wake_up_interruptible(&pgdat->kswapd_wait);
7616}
7617
7618void kswapd_clear_hopeless(pg_data_t *pgdat, enum kswapd_clear_hopeless_reason reason)
7619{
7620 /* Only trace actual resets, not redundant zero-to-zero */
7621 if (atomic_xchg(&pgdat->kswapd_failures, 0))
7622 trace_mm_vmscan_kswapd_clear_hopeless(pgdat->node_id, reason);
7623}
7624
7625/*
7626 * Reset kswapd_failures only when the node is balanced. Without this
7627 * check, successful direct reclaim (e.g., from cgroup memory.high
7628 * throttling) can keep resetting kswapd_failures even when the node
7629 * cannot be balanced, causing kswapd to run endlessly.
7630 */
7631void kswapd_try_clear_hopeless(struct pglist_data *pgdat,
7632 unsigned int order, int highest_zoneidx)
7633{
7634 if (pgdat_balanced(pgdat, order, highest_zoneidx))
7635 kswapd_clear_hopeless(pgdat, current_is_kswapd() ?
7636 KSWAPD_CLEAR_HOPELESS_KSWAPD : KSWAPD_CLEAR_HOPELESS_DIRECT);
7637}
7638
7639bool kswapd_test_hopeless(pg_data_t *pgdat)
7640{
7641 return atomic_read(&pgdat->kswapd_failures) >= MAX_RECLAIM_RETRIES;
7642}
7643
7644#ifdef CONFIG_HIBERNATION
7645/*
7646 * Try to free `nr_to_reclaim' of memory, system-wide, and return the number of
7647 * freed pages.
7648 *
7649 * Rather than trying to age LRUs the aim is to preserve the overall
7650 * LRU order by reclaiming preferentially
7651 * inactive > active > active referenced > active mapped
7652 */
7653unsigned long shrink_all_memory(unsigned long nr_to_reclaim)
7654{
7655 struct scan_control sc = {
7656 .nr_to_reclaim = nr_to_reclaim,
7657 .gfp_mask = GFP_HIGHUSER_MOVABLE,
7658 .reclaim_idx = MAX_NR_ZONES - 1,
7659 .priority = DEF_PRIORITY,
7660 .may_writepage = 1,
7661 .may_unmap = 1,
7662 .may_swap = 1,
7663 .hibernation_mode = 1,
7664 };
7665 struct zonelist *zonelist = node_zonelist(numa_node_id(), sc.gfp_mask);
7666 unsigned long nr_reclaimed;
7667 unsigned int noreclaim_flag;
7668
7669 fs_reclaim_acquire(sc.gfp_mask);
7670 noreclaim_flag = memalloc_noreclaim_save();
7671 set_task_reclaim_state(current, &sc.reclaim_state);
7672
7673 nr_reclaimed = do_try_to_free_pages(zonelist, &sc);
7674
7675 set_task_reclaim_state(current, NULL);
7676 memalloc_noreclaim_restore(noreclaim_flag);
7677 fs_reclaim_release(sc.gfp_mask);
7678
7679 return nr_reclaimed;
7680}
7681#endif /* CONFIG_HIBERNATION */
7682
7683/*
7684 * This kswapd start function will be called by init and node-hot-add.
7685 */
7686void __meminit kswapd_run(int nid)
7687{
7688 pg_data_t *pgdat = NODE_DATA(nid);
7689
7690 pgdat_kswapd_lock(pgdat);
7691 if (!pgdat->kswapd) {
7692 pgdat->kswapd = kthread_create_on_node(kswapd, pgdat, nid, "kswapd%d", nid);
7693 if (IS_ERR(pgdat->kswapd)) {
7694 /* failure at boot is fatal */
7695 pr_err("Failed to start kswapd on node %d, ret=%pe\n",
7696 nid, pgdat->kswapd);
7697 BUG_ON(system_state < SYSTEM_RUNNING);
7698 pgdat->kswapd = NULL;
7699 } else {
7700 wake_up_process(pgdat->kswapd);
7701 }
7702 }
7703 pgdat_kswapd_unlock(pgdat);
7704}
7705
7706/*
7707 * Called by memory hotplug when all memory in a node is offlined. Caller must
7708 * be holding mem_hotplug_begin/done().
7709 */
7710void __meminit kswapd_stop(int nid)
7711{
7712 pg_data_t *pgdat = NODE_DATA(nid);
7713 struct task_struct *kswapd;
7714
7715 pgdat_kswapd_lock(pgdat);
7716 kswapd = pgdat->kswapd;
7717 if (kswapd) {
7718 kthread_stop(kswapd);
7719 pgdat->kswapd = NULL;
7720 }
7721 pgdat_kswapd_unlock(pgdat);
7722}
7723
7724static const struct ctl_table vmscan_sysctl_table[] = {
7725 {
7726 .procname = "swappiness",
7727 .data = &vm_swappiness,
7728 .maxlen = sizeof(vm_swappiness),
7729 .mode = 0644,
7730 .proc_handler = proc_dointvec_minmax,
7731 .extra1 = SYSCTL_ZERO,
7732 .extra2 = SYSCTL_TWO_HUNDRED,
7733 },
7734#ifdef CONFIG_NUMA
7735 {
7736 .procname = "zone_reclaim_mode",
7737 .data = &node_reclaim_mode,
7738 .maxlen = sizeof(node_reclaim_mode),
7739 .mode = 0644,
7740 .proc_handler = proc_dointvec_minmax,
7741 .extra1 = SYSCTL_ZERO,
7742 }
7743#endif
7744};
7745
7746static int __init kswapd_init(void)
7747{
7748 int nid;
7749
7750 for_each_node_state(nid, N_MEMORY)
7751 kswapd_run(nid);
7752 register_sysctl_init("vm", vmscan_sysctl_table);
7753 return 0;
7754}
7755
7756module_init(kswapd_init)
7757
7758#ifdef CONFIG_NUMA
7759/*
7760 * Node reclaim mode
7761 *
7762 * If non-zero call node_reclaim when the number of free pages falls below
7763 * the watermarks.
7764 */
7765int node_reclaim_mode __read_mostly;
7766
7767/*
7768 * Priority for NODE_RECLAIM. This determines the fraction of pages
7769 * of a node considered for each zone_reclaim. 4 scans 1/16th of
7770 * a zone.
7771 */
7772#define NODE_RECLAIM_PRIORITY 4
7773
7774/*
7775 * Percentage of pages in a zone that must be unmapped for node_reclaim to
7776 * occur.
7777 */
7778int sysctl_min_unmapped_ratio = 1;
7779
7780/*
7781 * If the number of slab pages in a zone grows beyond this percentage then
7782 * slab reclaim needs to occur.
7783 */
7784int sysctl_min_slab_ratio = 5;
7785
7786static inline unsigned long node_unmapped_file_pages(struct pglist_data *pgdat)
7787{
7788 unsigned long file_mapped = node_page_state(pgdat, NR_FILE_MAPPED);
7789 unsigned long file_lru = node_page_state(pgdat, NR_INACTIVE_FILE) +
7790 node_page_state(pgdat, NR_ACTIVE_FILE);
7791
7792 /*
7793 * It's possible for there to be more file mapped pages than
7794 * accounted for by the pages on the file LRU lists because
7795 * tmpfs pages accounted for as ANON can also be FILE_MAPPED
7796 */
7797 return (file_lru > file_mapped) ? (file_lru - file_mapped) : 0;
7798}
7799
7800/* Work out how many page cache pages we can reclaim in this reclaim_mode */
7801static unsigned long node_pagecache_reclaimable(struct pglist_data *pgdat)
7802{
7803 unsigned long nr_pagecache_reclaimable;
7804 unsigned long delta = 0;
7805
7806 /*
7807 * If RECLAIM_UNMAP is set, then all file pages are considered
7808 * potentially reclaimable. Otherwise, we have to worry about
7809 * pages like swapcache and node_unmapped_file_pages() provides
7810 * a better estimate
7811 */
7812 if (node_reclaim_mode & RECLAIM_UNMAP)
7813 nr_pagecache_reclaimable = node_page_state(pgdat, NR_FILE_PAGES);
7814 else
7815 nr_pagecache_reclaimable = node_unmapped_file_pages(pgdat);
7816
7817 /*
7818 * Since we can't clean folios through reclaim, remove dirty file
7819 * folios from consideration.
7820 */
7821 delta += node_page_state(pgdat, NR_FILE_DIRTY);
7822
7823 /* Watch for any possible underflows due to delta */
7824 if (unlikely(delta > nr_pagecache_reclaimable))
7825 delta = nr_pagecache_reclaimable;
7826
7827 return nr_pagecache_reclaimable - delta;
7828}
7829
7830/*
7831 * Try to free up some pages from this node through reclaim.
7832 */
7833static unsigned long __node_reclaim(struct pglist_data *pgdat,
7834 unsigned long nr_pages,
7835 struct scan_control *sc)
7836{
7837 struct task_struct *p = current;
7838 unsigned int noreclaim_flag;
7839 unsigned long pflags;
7840
7841 trace_mm_vmscan_node_reclaim_begin(pgdat->node_id, sc->order,
7842 sc->gfp_mask);
7843
7844 cond_resched();
7845 psi_memstall_enter(&pflags);
7846 delayacct_freepages_start();
7847 fs_reclaim_acquire(sc->gfp_mask);
7848 /*
7849 * We need to be able to allocate from the reserves for RECLAIM_UNMAP
7850 */
7851 noreclaim_flag = memalloc_noreclaim_save();
7852 set_task_reclaim_state(p, &sc->reclaim_state);
7853
7854 if (node_pagecache_reclaimable(pgdat) > pgdat->min_unmapped_pages ||
7855 node_page_state_pages(pgdat, NR_SLAB_RECLAIMABLE_B) > pgdat->min_slab_pages) {
7856 /*
7857 * Free memory by calling shrink node with increasing
7858 * priorities until we have enough memory freed.
7859 */
7860 do {
7861 shrink_node(pgdat, sc);
7862 } while (sc->nr_reclaimed < nr_pages && --sc->priority >= 0);
7863 }
7864
7865 set_task_reclaim_state(p, NULL);
7866 memalloc_noreclaim_restore(noreclaim_flag);
7867 fs_reclaim_release(sc->gfp_mask);
7868 delayacct_freepages_end();
7869 psi_memstall_leave(&pflags);
7870
7871 trace_mm_vmscan_node_reclaim_end(sc->nr_reclaimed, NULL);
7872
7873 return sc->nr_reclaimed;
7874}
7875
7876unsigned long node_reclaim(struct pglist_data *pgdat, gfp_t gfp_mask, unsigned int order)
7877{
7878 unsigned long ret;
7879 /* Minimum pages needed in order to stay on node */
7880 const unsigned long nr_pages = 1 << order;
7881 struct scan_control sc = {
7882 .nr_to_reclaim = max(nr_pages, SWAP_CLUSTER_MAX),
7883 .gfp_mask = current_gfp_context(gfp_mask),
7884 .order = order,
7885 .priority = NODE_RECLAIM_PRIORITY,
7886 .may_writepage = !!(node_reclaim_mode & RECLAIM_WRITE),
7887 .may_unmap = !!(node_reclaim_mode & RECLAIM_UNMAP),
7888 .may_swap = 1,
7889 .reclaim_idx = gfp_zone(gfp_mask),
7890 };
7891
7892 /*
7893 * Node reclaim reclaims unmapped file backed pages and
7894 * slab pages if we are over the defined limits.
7895 *
7896 * A small portion of unmapped file backed pages is needed for
7897 * file I/O otherwise pages read by file I/O will be immediately
7898 * thrown out if the node is overallocated. So we do not reclaim
7899 * if less than a specified percentage of the node is used by
7900 * unmapped file backed pages.
7901 */
7902 if (node_pagecache_reclaimable(pgdat) <= pgdat->min_unmapped_pages &&
7903 node_page_state_pages(pgdat, NR_SLAB_RECLAIMABLE_B) <=
7904 pgdat->min_slab_pages)
7905 return 0;
7906
7907 /*
7908 * Do not scan if the allocation should not be delayed.
7909 */
7910 if (!gfpflags_allow_blocking(gfp_mask) || (current->flags & PF_MEMALLOC))
7911 return 0;
7912
7913 /*
7914 * Only run node reclaim on the local node or on nodes that do not
7915 * have associated processors. This will favor the local processor
7916 * over remote processors and spread off node memory allocations
7917 * as wide as possible.
7918 */
7919 if (node_state(pgdat->node_id, N_CPU) && pgdat->node_id != numa_node_id())
7920 return 0;
7921
7922 if (test_and_set_bit_lock(PGDAT_RECLAIM_LOCKED, &pgdat->flags))
7923 return 0;
7924
7925 ret = __node_reclaim(pgdat, nr_pages, &sc);
7926 clear_bit_unlock(PGDAT_RECLAIM_LOCKED, &pgdat->flags);
7927
7928 if (ret >= nr_pages)
7929 count_vm_event(PGSCAN_ZONE_RECLAIM_SUCCESS);
7930 else
7931 count_vm_event(PGSCAN_ZONE_RECLAIM_FAILED);
7932
7933 return ret;
7934}
7935
7936#else
7937
7938static unsigned long __node_reclaim(struct pglist_data *pgdat,
7939 unsigned long nr_pages,
7940 struct scan_control *sc)
7941{
7942 return 0;
7943}
7944
7945#endif
7946
7947enum {
7948 MEMORY_RECLAIM_SWAPPINESS = 0,
7949 MEMORY_RECLAIM_SWAPPINESS_MAX,
7950 MEMORY_RECLAIM_NULL,
7951};
7952static const match_table_t tokens = {
7953 { MEMORY_RECLAIM_SWAPPINESS, "swappiness=%d"},
7954 { MEMORY_RECLAIM_SWAPPINESS_MAX, "swappiness=max"},
7955 { MEMORY_RECLAIM_NULL, NULL },
7956};
7957
7958int user_proactive_reclaim(char *buf,
7959 struct mem_cgroup *memcg, pg_data_t *pgdat)
7960{
7961 unsigned int nr_retries = MAX_RECLAIM_RETRIES;
7962 unsigned long nr_to_reclaim, nr_reclaimed = 0;
7963 int swappiness = -1;
7964 char *old_buf, *start;
7965 substring_t args[MAX_OPT_ARGS];
7966 gfp_t gfp_mask = GFP_KERNEL;
7967
7968 if (!buf || (!memcg && !pgdat) || (memcg && pgdat))
7969 return -EINVAL;
7970
7971 buf = strstrip(buf);
7972
7973 old_buf = buf;
7974 nr_to_reclaim = memparse(buf, &buf) / PAGE_SIZE;
7975 if (buf == old_buf)
7976 return -EINVAL;
7977
7978 buf = strstrip(buf);
7979
7980 while ((start = strsep(&buf, " ")) != NULL) {
7981 if (!strlen(start))
7982 continue;
7983 switch (match_token(start, tokens, args)) {
7984 case MEMORY_RECLAIM_SWAPPINESS:
7985 if (match_int(&args[0], &swappiness))
7986 return -EINVAL;
7987 if (swappiness < MIN_SWAPPINESS ||
7988 swappiness > MAX_SWAPPINESS)
7989 return -EINVAL;
7990 break;
7991 case MEMORY_RECLAIM_SWAPPINESS_MAX:
7992 swappiness = SWAPPINESS_ANON_ONLY;
7993 break;
7994 default:
7995 return -EINVAL;
7996 }
7997 }
7998
7999 while (nr_reclaimed < nr_to_reclaim) {
8000 /* Will converge on zero, but reclaim enforces a minimum */
8001 unsigned long batch_size = (nr_to_reclaim - nr_reclaimed) / 4;
8002 unsigned long reclaimed;
8003
8004 /*
8005 * Return -ERESTARTSYS to allow the freezer to interrupt the
8006 * task. The syscall will be transparently restarted upon
8007 * resume. For real signals, it either restarts the syscall
8008 * (if SA_RESTART is set) or is converted to -EINTR by the
8009 * signal layer.
8010 */
8011 if (signal_pending(current))
8012 return -ERESTARTSYS;
8013
8014 /* cgroup_rmdir() waits for us with cgroup_mutex held. */
8015 if (memcg && memcg_is_dying(memcg))
8016 return -EAGAIN;
8017
8018 /*
8019 * This is the final attempt, drain percpu lru caches in the
8020 * hope of introducing more evictable pages.
8021 */
8022 if (!nr_retries)
8023 lru_add_drain_all();
8024
8025 if (memcg) {
8026 unsigned int reclaim_options;
8027
8028 reclaim_options = MEMCG_RECLAIM_MAY_SWAP |
8029 MEMCG_RECLAIM_PROACTIVE;
8030 reclaimed = try_to_free_mem_cgroup_pages(memcg,
8031 batch_size, gfp_mask,
8032 reclaim_options,
8033 swappiness == -1 ? NULL : &swappiness);
8034 } else {
8035 struct scan_control sc = {
8036 .gfp_mask = current_gfp_context(gfp_mask),
8037 .reclaim_idx = gfp_zone(gfp_mask),
8038 .proactive_swappiness = swappiness == -1 ? NULL : &swappiness,
8039 .priority = DEF_PRIORITY,
8040 .may_writepage = 1,
8041 .nr_to_reclaim = max(batch_size, SWAP_CLUSTER_MAX),
8042 .may_unmap = 1,
8043 .may_swap = 1,
8044 .proactive = 1,
8045 };
8046
8047 if (test_and_set_bit_lock(PGDAT_RECLAIM_LOCKED,
8048 &pgdat->flags))
8049 return -EBUSY;
8050
8051 reclaimed = __node_reclaim(pgdat, batch_size, &sc);
8052 clear_bit_unlock(PGDAT_RECLAIM_LOCKED, &pgdat->flags);
8053 }
8054
8055 if (!reclaimed && !nr_retries--)
8056 return -EAGAIN;
8057
8058 nr_reclaimed += reclaimed;
8059 }
8060
8061 return 0;
8062}
8063
8064/**
8065 * check_move_unevictable_folios - Move evictable folios to appropriate zone
8066 * lru list
8067 * @fbatch: Batch of lru folios to check.
8068 *
8069 * Checks folios for evictability, if an evictable folio is in the unevictable
8070 * lru list, moves it to the appropriate evictable lru list. This function
8071 * should be only used for lru folios.
8072 */
8073void check_move_unevictable_folios(struct folio_batch *fbatch)
8074{
8075 struct lruvec *lruvec = NULL;
8076 int pgscanned = 0;
8077 int pgrescued = 0;
8078 int i;
8079
8080 for (i = 0; i < fbatch->nr; i++) {
8081 struct folio *folio = fbatch->folios[i];
8082 int nr_pages = folio_nr_pages(folio);
8083
8084 pgscanned += nr_pages;
8085
8086 /* block memcg migration while the folio moves between lrus */
8087 if (!folio_test_clear_lru(folio))
8088 continue;
8089
8090 lruvec = folio_lruvec_relock_irq(folio, lruvec);
8091 if (folio_evictable(folio) && folio_test_unevictable(folio)) {
8092 lruvec_del_folio(lruvec, folio);
8093 folio_clear_unevictable(folio);
8094 lruvec_add_folio(lruvec, folio);
8095 pgrescued += nr_pages;
8096 }
8097 folio_set_lru(folio);
8098 }
8099
8100 if (lruvec) {
8101 __count_vm_events(UNEVICTABLE_PGRESCUED, pgrescued);
8102 __count_vm_events(UNEVICTABLE_PGSCANNED, pgscanned);
8103 lruvec_unlock_irq(lruvec);
8104 } else if (pgscanned) {
8105 count_vm_events(UNEVICTABLE_PGSCANNED, pgscanned);
8106 }
8107}
8108EXPORT_SYMBOL_GPL(check_move_unevictable_folios);
8109
8110#if defined(CONFIG_SYSFS) && defined(CONFIG_NUMA)
8111static ssize_t reclaim_store(struct device *dev,
8112 struct device_attribute *attr,
8113 const char *buf, size_t count)
8114{
8115 int ret, nid = dev->id;
8116
8117 ret = user_proactive_reclaim((char *)buf, NULL, NODE_DATA(nid));
8118 return ret ? ret : count;
8119}
8120
8121static DEVICE_ATTR_WO(reclaim);
8122int reclaim_register_node(struct node *node)
8123{
8124 return device_create_file(&node->dev, &dev_attr_reclaim);
8125}
8126
8127void reclaim_unregister_node(struct node *node)
8128{
8129 return device_remove_file(&node->dev, &dev_attr_reclaim);
8130}
8131#endif