linux - mm/folio.c

1159 lines · 32.3 KiB

1	// SPDX-License-Identifier: GPL-2.0-only
2	/*
3	* linux/mm/folio.c
4	*
5	* Copyright (C) 1991, 1992, 1993, 1994 Linus Torvalds
6	*/
7
8	/*
9	* Folio LRU helpers: add/remove folios from LRU lists, batching,
10	* activation/deactivation, and page cache release paths.
11	*/
12
13	#include <linux/mm.h>
14	#include <linux/sched.h>
15	#include <linux/kernel_stat.h>
16	#include <linux/swap.h>
17	#include <linux/mman.h>
18	#include <linux/pagemap.h>
19	#include <linux/folio_batch.h>
20	#include <linux/init.h>
21	#include <linux/export.h>
22	#include <linux/mm_inline.h>
23	#include <linux/percpu_counter.h>
24	#include <linux/memremap.h>
25	#include <linux/percpu.h>
26	#include <linux/cpu.h>
27	#include <linux/notifier.h>
28	#include <linux/backing-dev.h>
29	#include <linux/memcontrol.h>
30	#include <linux/gfp.h>
31	#include <linux/uio.h>
32	#include <linux/hugetlb.h>
33	#include <linux/page_idle.h>
34	#include <linux/local_lock.h>
35	#include <linux/buffer_head.h>
36	#include <linux/kvm_types.h>
37
38	#include "internal.h"
39	#include "page_alloc.h"
40
41	#define CREATE_TRACE_POINTS
42	#include <trace/events/pagemap.h>
43
44	struct cpu_fbatches {
45	/*
46	* The following folio batches are grouped together because they are protected
47	* by disabling preemption (and interrupts remain enabled).
48	*/
49	local_lock_t lock;
50	struct folio_batch lru_add;
51	struct folio_batch lru_deactivate_file;
52	struct folio_batch lru_deactivate;
53	struct folio_batch lru_lazyfree;
54	#ifdef CONFIG_SMP
55	struct folio_batch lru_activate;
56	#endif
57	/* Protecting the following batches which require disabling interrupts */
58	local_lock_t lock_irq;
59	struct folio_batch lru_move_tail;
60	};
61
62	static DEFINE_PER_CPU(struct cpu_fbatches, cpu_fbatches) = {
63	.lock = INIT_LOCAL_LOCK(lock),
64	.lock_irq = INIT_LOCAL_LOCK(lock_irq),
65	};
66
67	static void __page_cache_release(struct folio folio, struct lruvec *lruvecp,
68	unsigned long *flagsp)
69	{
70	if (folio_test_lru(folio)) {
71	folio_lruvec_relock_irqsave(folio, lruvecp, flagsp);
72	lruvec_del_folio(*lruvecp, folio);
73	__folio_clear_lru_flags(folio);
74	}
75	}
76
77	/*
78	* This path almost never happens for VM activity - pages are normally freed
79	* in batches. But it gets used by networking - and for compound pages.
80	*/
81	static void page_cache_release(struct folio *folio)
82	{
83	struct lruvec *lruvec = NULL;
84	unsigned long flags;
85
86	__page_cache_release(folio, &lruvec, &flags);
87	if (lruvec)
88	lruvec_unlock_irqrestore(lruvec, flags);
89	}
90
91	void __folio_put(struct folio *folio)
92	{
93	if (unlikely(folio_is_zone_device(folio))) {
94	free_zone_device_folio(folio);
95	return;
96	}
97
98	if (folio_test_hugetlb(folio)) {
99	free_huge_folio(folio);
100	return;
101	}
102
103	page_cache_release(folio);
104	folio_unqueue_deferred_split(folio);
105	mem_cgroup_uncharge(folio);
106	free_frozen_pages(&folio->page, folio_order(folio));
107	}
108	EXPORT_SYMBOL(__folio_put);
109
110	typedef void (move_fn_t)(struct lruvec lruvec, struct folio *folio);
111
112	static void lru_add(struct lruvec lruvec, struct folio folio)
113	{
114	int was_unevictable = folio_test_clear_unevictable(folio);
115	long nr_pages = folio_nr_pages(folio);
116
117	VM_BUG_ON_FOLIO(folio_test_lru(folio), folio);
118
119	/*
120	* Is an smp_mb__after_atomic() still required here, before
121	* folio_evictable() tests the mlocked flag, to rule out the possibility
122	* of stranding an evictable folio on an unevictable LRU? I think
123	* not, because __munlock_folio() only clears the mlocked flag
124	* while the LRU lock is held.
125	*
126	* (That is not true of __page_cache_release(), and not necessarily
127	* true of folios_put(): but those only clear the mlocked flag after
128	* folio_put_testzero() has excluded any other users of the folio.)
129	*/
130	if (folio_evictable(folio)) {
131	if (was_unevictable)
132	__count_vm_events(UNEVICTABLE_PGRESCUED, nr_pages);
133	} else {
134	folio_clear_active(folio);
135	folio_set_unevictable(folio);
136	/*
137	* folio->mlock_count = !!folio_test_mlocked(folio)?
138	* But that leaves __mlock_folio() in doubt whether another
139	* actor has already counted the mlock or not. Err on the
140	* safe side, underestimate, let page reclaim fix it, rather
141	* than leaving a page on the unevictable LRU indefinitely.
142	*/
143	folio->mlock_count = 0;
144	if (!was_unevictable)
145	__count_vm_events(UNEVICTABLE_PGCULLED, nr_pages);
146	}
147
148	lruvec_add_folio(lruvec, folio);
149	trace_mm_lru_insertion(folio);
150	}
151
152	static void folio_batch_move_lru(struct folio_batch *fbatch, move_fn_t move_fn)
153	{
154	int i;
155	struct lruvec *lruvec = NULL;
156	unsigned long flags = 0;
157	struct folio_batch free_fbatch;
158	bool is_lru_add = (move_fn == lru_add);
159
160	/*
161	* If we're adding to the LRU, preemptively filter dead folios. Use
162	* this dedicated folio batch for temp storage and deferred cleanup.
163	*/
164	if (is_lru_add)
165	folio_batch_init(&free_fbatch);
166
167	for (i = 0; i < folio_batch_count(fbatch); i++) {
168	struct folio *folio = fbatch->folios[i];
169
170	/* block memcg migration while the folio moves between lru */
171	if (!is_lru_add && !folio_test_clear_lru(folio))
172	continue;
173
174	/*
175	* Filter dead folios by moving them from the add batch to the temp
176	* batch for freeing after this loop.
177	*
178	* We're bypassing normal cleanup. Clear flags that are not
179	* applicable to dead folios.
180	*
181	* Since the folio may be part of a huge page, unqueue from
182	* deferred split list to avoid a dangling list entry.
183	*/
184	if (is_lru_add && folio_ref_freeze(folio, 1)) {
185	__folio_clear_active(folio);
186	__folio_clear_unevictable(folio);
187	folio_unqueue_deferred_split(folio);
188	fbatch->folios[i] = NULL;
189	folio_batch_add(&free_fbatch, folio);
190	continue;
191	}
192
193	folio_lruvec_relock_irqsave(folio, &lruvec, &flags);
194	move_fn(lruvec, folio);
195
196	folio_set_lru(folio);
197	}
198
199	if (lruvec)
200	lruvec_unlock_irqrestore(lruvec, flags);
201
202	/* Cleanup filtered dead folios. */
203	if (is_lru_add) {
204	mem_cgroup_uncharge_folios(&free_fbatch);
205	free_unref_folios(&free_fbatch);
206	}
207
208	folios_put(fbatch);
209	}
210
211	static void __folio_batch_add_and_move(struct folio_batch __percpu *fbatch,
212	struct folio *folio, move_fn_t move_fn, bool disable_irq)
213	{
214	unsigned long flags;
215
216	folio_get(folio);
217
218	if (disable_irq)
219	local_lock_irqsave(&cpu_fbatches.lock_irq, flags);
220	else
221	local_lock(&cpu_fbatches.lock);
222
223	if (!folio_batch_add(this_cpu_ptr(fbatch), folio) \|\|
224	!folio_may_be_lru_cached(folio) \|\| lru_cache_disabled())
225	folio_batch_move_lru(this_cpu_ptr(fbatch), move_fn);
226
227	if (disable_irq)
228	local_unlock_irqrestore(&cpu_fbatches.lock_irq, flags);
229	else
230	local_unlock(&cpu_fbatches.lock);
231	}
232
233	#define folio_batch_add_and_move(folio, op) \
234	__folio_batch_add_and_move( \
235	&cpu_fbatches.op, \
236	folio, \
237	op, \
238	offsetof(struct cpu_fbatches, op) >= \
239	offsetof(struct cpu_fbatches, lock_irq) \
240	)
241
242	static void lru_move_tail(struct lruvec lruvec, struct folio folio)
243	{
244	if (folio_test_unevictable(folio))
245	return;
246
247	lruvec_del_folio(lruvec, folio);
248	folio_clear_active(folio);
249	lruvec_add_folio_tail(lruvec, folio);
250	__count_vm_events(PGROTATED, folio_nr_pages(folio));
251	}
252
253	/*
254	* Writeback is about to end against a folio which has been marked for
255	* immediate reclaim. If it still appears to be reclaimable, move it
256	* to the tail of the inactive list.
257	*
258	* folio_rotate_reclaimable() must disable IRQs, to prevent nasty races.
259	*/
260	void folio_rotate_reclaimable(struct folio *folio)
261	{
262	if (folio_test_locked(folio) \|\| folio_test_dirty(folio) \|\|
263	folio_test_unevictable(folio) \|\| !folio_test_lru(folio))
264	return;
265
266	folio_batch_add_and_move(folio, lru_move_tail);
267	}
268
269	static void lru_activate(struct lruvec lruvec, struct folio folio)
270	{
271	long nr_pages = folio_nr_pages(folio);
272
273	if (folio_test_active(folio) \|\| folio_test_unevictable(folio))
274	return;
275
276
277	lruvec_del_folio(lruvec, folio);
278	folio_set_active(folio);
279	lruvec_add_folio(lruvec, folio);
280	trace_mm_lru_activate(folio);
281
282	__count_vm_events(PGACTIVATE, nr_pages);
283	count_memcg_events(lruvec_memcg(lruvec), PGACTIVATE, nr_pages);
284	}
285
286	#ifdef CONFIG_SMP
287	static void folio_activate_drain(int cpu)
288	{
289	struct folio_batch *fbatch = &per_cpu(cpu_fbatches.lru_activate, cpu);
290
291	if (folio_batch_count(fbatch))
292	folio_batch_move_lru(fbatch, lru_activate);
293	}
294
295	void folio_activate(struct folio *folio)
296	{
297	if (folio_test_active(folio) \|\| folio_test_unevictable(folio) \|\|
298	!folio_test_lru(folio))
299	return;
300
301	folio_batch_add_and_move(folio, lru_activate);
302	}
303
304	#else
305	static inline void folio_activate_drain(int cpu)
306	{
307	}
308
309	void folio_activate(struct folio *folio)
310	{
311	struct lruvec *lruvec;
312
313	if (!folio_test_clear_lru(folio))
314	return;
315
316	lruvec = folio_lruvec_lock_irq(folio);
317	lru_activate(lruvec, folio);
318	lruvec_unlock_irq(lruvec);
319	folio_set_lru(folio);
320	}
321	#endif
322
323	static void __lru_cache_activate_folio(struct folio *folio)
324	{
325	struct folio_batch *fbatch;
326	int i;
327
328	local_lock(&cpu_fbatches.lock);
329	fbatch = this_cpu_ptr(&cpu_fbatches.lru_add);
330
331	/*
332	* Search backwards on the optimistic assumption that the folio being
333	* activated has just been added to this batch. Note that only
334	* the local batch is examined as a !LRU folio could be in the
335	* process of being released, reclaimed, migrated or on a remote
336	* batch that is currently being drained. Furthermore, marking
337	* a remote batch's folio active potentially hits a race where
338	* a folio is marked active just after it is added to the inactive
339	* list causing accounting errors and BUG_ON checks to trigger.
340	*/
341	for (i = folio_batch_count(fbatch) - 1; i >= 0; i--) {
342	struct folio *batch_folio = fbatch->folios[i];
343
344	if (batch_folio == folio) {
345	folio_set_active(folio);
346	break;
347	}
348	}
349
350	local_unlock(&cpu_fbatches.lock);
351	}
352
353	#ifdef CONFIG_LRU_GEN
354
355	static void lru_gen_inc_refs(struct folio *folio)
356	{
357	unsigned long new_flags, old_flags = READ_ONCE(folio->flags.f);
358
359	if (folio_test_unevictable(folio))
360	return;
361
362	/* see the comment on LRU_REFS_FLAGS */
363	if (!folio_test_referenced(folio)) {
364	set_mask_bits(&folio->flags.f, LRU_REFS_MASK, BIT(PG_referenced));
365	return;
366	}
367
368	do {
369	if ((old_flags & LRU_REFS_MASK) == LRU_REFS_MASK) {
370	if (!folio_test_workingset(folio))
371	folio_set_workingset(folio);
372	return;
373	}
374
375	new_flags = old_flags + BIT(LRU_REFS_PGOFF);
376	} while (!try_cmpxchg(&folio->flags.f, &old_flags, new_flags));
377	}
378
379	static bool lru_gen_clear_refs(struct folio *folio)
380	{
381	int gen = folio_lru_gen(folio);
382	int type = folio_is_file_lru(folio);
383	unsigned long seq;
384
385	if (gen < 0)
386	return true;
387
388	set_mask_bits(&folio->flags.f, LRU_REFS_FLAGS \| BIT(PG_workingset), 0);
389
390	rcu_read_lock();
391	seq = READ_ONCE(folio_lruvec(folio)->lrugen.min_seq[type]);
392	rcu_read_unlock();
393	/* whether can do without shuffling under the LRU lock */
394	return gen == lru_gen_from_seq(seq);
395	}
396
397	#else /* !CONFIG_LRU_GEN */
398
399	static void lru_gen_inc_refs(struct folio *folio)
400	{
401	}
402
403	static bool lru_gen_clear_refs(struct folio *folio)
404	{
405	return false;
406	}
407
408	#endif /* CONFIG_LRU_GEN */
409
410	/**
411	* folio_mark_accessed - Mark a folio as having seen activity.
412	* @folio: The folio to mark.
413	*
414	* This function will perform one of the following transitions:
415	*
416	* * inactive,unreferenced -> inactive,referenced
417	* * inactive,referenced -> active,unreferenced
418	* * active,unreferenced -> active,referenced
419	*
420	* When a newly allocated folio is not yet visible, so safe for non-atomic ops,
421	* __folio_set_referenced() may be substituted for folio_mark_accessed().
422	*/
423	void folio_mark_accessed(struct folio *folio)
424	{
425	if (folio_test_dropbehind(folio))
426	return;
427	if (lru_gen_enabled()) {
428	lru_gen_inc_refs(folio);
429	return;
430	}
431
432	if (!folio_test_referenced(folio)) {
433	folio_set_referenced(folio);
434	} else if (folio_test_unevictable(folio)) {
435	/*
436	* Unevictable pages are on the "LRU_UNEVICTABLE" list. But,
437	* this list is never rotated or maintained, so marking an
438	* unevictable page accessed has no effect.
439	*/
440	} else if (!folio_test_active(folio)) {
441	/*
442	* If the folio is on the LRU, queue it for activation via
443	* cpu_fbatches.lru_activate. Otherwise, assume the folio is in a
444	* folio_batch, mark it active and it'll be moved to the active
445	* LRU on the next drain.
446	*/
447	if (folio_test_lru(folio))
448	folio_activate(folio);
449	else
450	__lru_cache_activate_folio(folio);
451	folio_clear_referenced(folio);
452	workingset_activation(folio);
453	}
454	if (folio_test_idle(folio))
455	folio_clear_idle(folio);
456	}
457	EXPORT_SYMBOL(folio_mark_accessed);
458
459	/**
460	* folio_add_lru - Add a folio to an LRU list.
461	* @folio: The folio to be added to the LRU.
462	*
463	* Queue the folio for addition to the LRU. The decision on whether
464	* to add the page to the [in]active [file\|anon] list is deferred until the
465	* folio_batch is drained. This gives a chance for the caller of folio_add_lru()
466	* have the folio added to the active list using folio_mark_accessed().
467	*/
468	void folio_add_lru(struct folio *folio)
469	{
470	VM_BUG_ON_FOLIO(folio_test_active(folio) &&
471	folio_test_unevictable(folio), folio);
472	VM_BUG_ON_FOLIO(folio_test_lru(folio), folio);
473
474	/*
475	* For refaulted workingset folios, set PG_active so they
476	* can be added to active generations.
477	* For prefaulted file folios, folio_mark_accessed() sets
478	* PG_referenced so lru_gen_folio_seq() places them into
479	* the second oldest generation.
480	*/
481	if (lru_gen_enabled() && !folio_test_unevictable(folio) &&
482	lru_gen_in_fault() && !(current->flags & PF_MEMALLOC)) {
483	if (folio_test_workingset(folio))
484	folio_set_active(folio);
485	else if (!folio_test_referenced(folio))
486	folio_mark_accessed(folio);
487	}
488
489	folio_batch_add_and_move(folio, lru_add);
490	}
491	EXPORT_SYMBOL(folio_add_lru);
492
493	/**
494	* folio_add_lru_vma() - Add a folio to the appropriate LRU list for this VMA.
495	* @folio: The folio to be added to the LRU.
496	* @vma: VMA in which the folio is mapped.
497	*
498	* If the VMA is mlocked, @folio is added to the unevictable list.
499	* Otherwise, it is treated the same way as folio_add_lru().
500	*/
501	void folio_add_lru_vma(struct folio folio, struct vm_area_struct vma)
502	{
503	VM_BUG_ON_FOLIO(folio_test_lru(folio), folio);
504
505	if (unlikely((vma->vm_flags & (VM_LOCKED \| VM_SPECIAL)) == VM_LOCKED))
506	mlock_new_folio(folio);
507	else
508	folio_add_lru(folio);
509	}
510
511	/*
512	* If the folio cannot be invalidated, it is moved to the
513	* inactive list to speed up its reclaim. It is moved to the
514	* head of the list, rather than the tail, to give the flusher
515	* threads some time to write it out, as this is much more
516	* effective than the single-page writeout from reclaim.
517	*
518	* If the folio isn't mapped and dirty/writeback, the folio
519	* could be reclaimed asap using the reclaim flag.
520	*
521	* 1. active, mapped folio -> none
522	* 2. active, dirty/writeback folio -> inactive, head, reclaim
523	* 3. inactive, mapped folio -> none
524	* 4. inactive, dirty/writeback folio -> inactive, head, reclaim
525	* 5. inactive, clean -> inactive, tail
526	* 6. Others -> none
527	*
528	* In 4, it moves to the head of the inactive list so the folio is
529	* written out by flusher threads as this is much more efficient
530	* than the single-page writeout from reclaim.
531	*/
532	static void lru_deactivate_file(struct lruvec lruvec, struct folio folio)
533	{
534	bool active = folio_test_active(folio) \|\| lru_gen_enabled();
535	long nr_pages = folio_nr_pages(folio);
536
537	if (folio_test_unevictable(folio))
538	return;
539
540	/* Some processes are using the folio */
541	if (folio_mapped(folio))
542	return;
543
544	lruvec_del_folio(lruvec, folio);
545	folio_clear_active(folio);
546	folio_clear_referenced(folio);
547
548	if (folio_test_writeback(folio) \|\| folio_test_dirty(folio)) {
549	/*
550	* Setting the reclaim flag could race with
551	* folio_end_writeback() and confuse readahead. But the
552	* race window is _really_ small and it's not a critical
553	* problem.
554	*/
555	lruvec_add_folio(lruvec, folio);
556	folio_set_reclaim(folio);
557	} else {
558	/*
559	* The folio's writeback ended while it was in the batch.
560	* We move that folio to the tail of the inactive list.
561	*/
562	lruvec_add_folio_tail(lruvec, folio);
563	__count_vm_events(PGROTATED, nr_pages);
564	}
565
566	if (active) {
567	__count_vm_events(PGDEACTIVATE, nr_pages);
568	count_memcg_events(lruvec_memcg(lruvec), PGDEACTIVATE,
569	nr_pages);
570	}
571	}
572
573	static void lru_deactivate(struct lruvec lruvec, struct folio folio)
574	{
575	long nr_pages = folio_nr_pages(folio);
576
577	if (folio_test_unevictable(folio) \|\| !(folio_test_active(folio) \|\| lru_gen_enabled()))
578	return;
579
580	lruvec_del_folio(lruvec, folio);
581	folio_clear_active(folio);
582	folio_clear_referenced(folio);
583	lruvec_add_folio(lruvec, folio);
584
585	__count_vm_events(PGDEACTIVATE, nr_pages);
586	count_memcg_events(lruvec_memcg(lruvec), PGDEACTIVATE, nr_pages);
587	}
588
589	static void lru_lazyfree(struct lruvec lruvec, struct folio folio)
590	{
591	long nr_pages = folio_nr_pages(folio);
592
593	if (!folio_test_anon(folio) \|\| !folio_test_swapbacked(folio) \|\|
594	folio_test_swapcache(folio) \|\| folio_test_unevictable(folio))
595	return;
596
597	lruvec_del_folio(lruvec, folio);
598	folio_clear_active(folio);
599	if (lru_gen_enabled())
600	lru_gen_clear_refs(folio);
601	else
602	folio_clear_referenced(folio);
603	/*
604	* Lazyfree folios are clean anonymous folios. They have
605	* the swapbacked flag cleared, to distinguish them from normal
606	* anonymous folios
607	*/
608	folio_clear_swapbacked(folio);
609	lruvec_add_folio(lruvec, folio);
610
611	__count_vm_events(PGLAZYFREE, nr_pages);
612	count_memcg_events(lruvec_memcg(lruvec), PGLAZYFREE, nr_pages);
613	}
614
615	/*
616	* Drain pages out of the cpu's folio_batch.
617	* Either "cpu" is the current CPU, and preemption has already been
618	* disabled; or "cpu" is being hot-unplugged, and is already dead.
619	*/
620	void lru_add_drain_cpu(int cpu)
621	{
622	struct cpu_fbatches *fbatches = &per_cpu(cpu_fbatches, cpu);
623	struct folio_batch *fbatch = &fbatches->lru_add;
624	unsigned int nr_folios = folio_batch_count(fbatch);
625
626	if (nr_folios) {
627	folio_batch_move_lru(fbatch, lru_add);
628	trace_mm_lru_add_drain_tp(cpu, nr_folios);
629	}
630
631	fbatch = &fbatches->lru_move_tail;
632	/* Disabling interrupts below acts as a compiler barrier. */
633	if (data_race(folio_batch_count(fbatch))) {
634	unsigned long flags;
635
636	/* No harm done if a racing interrupt already did this */
637	local_lock_irqsave(&cpu_fbatches.lock_irq, flags);
638	folio_batch_move_lru(fbatch, lru_move_tail);
639	local_unlock_irqrestore(&cpu_fbatches.lock_irq, flags);
640	}
641
642	fbatch = &fbatches->lru_deactivate_file;
643	if (folio_batch_count(fbatch))
644	folio_batch_move_lru(fbatch, lru_deactivate_file);
645
646	fbatch = &fbatches->lru_deactivate;
647	if (folio_batch_count(fbatch))
648	folio_batch_move_lru(fbatch, lru_deactivate);
649
650	fbatch = &fbatches->lru_lazyfree;
651	if (folio_batch_count(fbatch))
652	folio_batch_move_lru(fbatch, lru_lazyfree);
653
654	folio_activate_drain(cpu);
655	}
656
657	/**
658	* deactivate_file_folio() - Deactivate a file folio.
659	* @folio: Folio to deactivate.
660	*
661	* This function hints to the VM that @folio is a good reclaim candidate,
662	* for example if its invalidation fails due to the folio being dirty
663	* or under writeback.
664	*
665	* Context: Caller holds a reference on the folio.
666	*/
667	void deactivate_file_folio(struct folio *folio)
668	{
669	/* Deactivating an unevictable folio will not accelerate reclaim */
670	if (folio_test_unevictable(folio) \|\| !folio_test_lru(folio))
671	return;
672
673	if (lru_gen_enabled() && lru_gen_clear_refs(folio))
674	return;
675
676	folio_batch_add_and_move(folio, lru_deactivate_file);
677	}
678
679	/*
680	* folio_deactivate - deactivate a folio
681	* @folio: folio to deactivate
682	*
683	* folio_deactivate() moves @folio to the inactive list if @folio was on the
684	* active list and was not unevictable. This is done to accelerate the
685	* reclaim of @folio.
686	*/
687	void folio_deactivate(struct folio *folio)
688	{
689	if (folio_test_unevictable(folio) \|\| !folio_test_lru(folio))
690	return;
691
692	if (lru_gen_enabled() ? lru_gen_clear_refs(folio) : !folio_test_active(folio))
693	return;
694
695	folio_batch_add_and_move(folio, lru_deactivate);
696	}
697
698	/**
699	* folio_mark_lazyfree - make an anon folio lazyfree
700	* @folio: folio to deactivate
701	*
702	* folio_mark_lazyfree() moves @folio to the inactive file list.
703	* This is done to accelerate the reclaim of @folio.
704	*/
705	void folio_mark_lazyfree(struct folio *folio)
706	{
707	if (!folio_test_anon(folio) \|\| !folio_test_swapbacked(folio) \|\|
708	!folio_test_lru(folio) \|\|
709	folio_test_swapcache(folio) \|\| folio_test_unevictable(folio))
710	return;
711
712	folio_batch_add_and_move(folio, lru_lazyfree);
713	}
714
715	void lru_add_drain(void)
716	{
717	local_lock(&cpu_fbatches.lock);
718	lru_add_drain_cpu(smp_processor_id());
719	local_unlock(&cpu_fbatches.lock);
720	mlock_drain_local();
721	}
722
723	/*
724	* It's called from per-cpu workqueue context in SMP case so
725	* lru_add_drain_cpu and invalidate_bh_lrus_cpu should run on
726	* the same cpu. It shouldn't be a problem in !SMP case since
727	* the core is only one and the locks will disable preemption.
728	*/
729	static void lru_add_and_bh_lrus_drain(void)
730	{
731	local_lock(&cpu_fbatches.lock);
732	lru_add_drain_cpu(smp_processor_id());
733	local_unlock(&cpu_fbatches.lock);
734	invalidate_bh_lrus_cpu();
735	mlock_drain_local();
736	}
737
738	void lru_add_drain_cpu_zone(struct zone *zone)
739	{
740	local_lock(&cpu_fbatches.lock);
741	lru_add_drain_cpu(smp_processor_id());
742	drain_local_pages(zone);
743	local_unlock(&cpu_fbatches.lock);
744	mlock_drain_local();
745	}
746
747	#ifdef CONFIG_SMP
748
749	static DEFINE_PER_CPU(struct work_struct, lru_add_drain_work);
750
751	static void lru_add_drain_per_cpu(struct work_struct *dummy)
752	{
753	lru_add_and_bh_lrus_drain();
754	}
755
756	static bool cpu_needs_drain(unsigned int cpu)
757	{
758	struct cpu_fbatches *fbatches = &per_cpu(cpu_fbatches, cpu);
759
760	/* Check these in order of likelihood that they're not zero */
761	return data_race(folio_batch_count(&fbatches->lru_add) \|\|
762	folio_batch_count(&fbatches->lru_move_tail) \|\|
763	folio_batch_count(&fbatches->lru_deactivate_file) \|\|
764	folio_batch_count(&fbatches->lru_deactivate) \|\|
765	folio_batch_count(&fbatches->lru_lazyfree) \|\|
766	folio_batch_count(&fbatches->lru_activate) \|\|
767	need_mlock_drain(cpu)) \|\|
768	has_bh_in_lru(cpu, NULL);
769	}
770
771	/*
772	* Doesn't need any cpu hotplug locking because we do rely on per-cpu
773	* kworkers being shut down before our page_alloc_cpu_dead callback is
774	* executed on the offlined cpu.
775	* Calling this function with cpu hotplug locks held can actually lead
776	* to obscure indirect dependencies via WQ context.
777	*/
778	static inline void __lru_add_drain_all(bool force_all_cpus)
779	{
780	/*
781	* lru_drain_gen - Global pages generation number
782	*
783	* (A) Definition: global lru_drain_gen = x implies that all generations
784	* 0 < n <= x are already scheduled for draining.
785	*
786	* This is an optimization for the highly-contended use case where a
787	* user space workload keeps constantly generating a flow of pages for
788	* each CPU.
789	*/
790	static unsigned int lru_drain_gen;
791	static struct cpumask has_work;
792	static DEFINE_MUTEX(lock);
793	unsigned cpu, this_gen;
794
795	/*
796	* Make sure nobody triggers this path before mm_percpu_wq is fully
797	* initialized.
798	*/
799	if (WARN_ON(!mm_percpu_wq))
800	return;
801
802	trace_mm_lru_add_drain_all_tp(force_all_cpus);
803
804	/*
805	* Guarantee folio_batch counter stores visible by this CPU
806	* are visible to other CPUs before loading the current drain
807	* generation.
808	*/
809	smp_mb();
810
811	/*
812	* (B) Locally cache global LRU draining generation number
813	*
814	* The read barrier ensures that the counter is loaded before the mutex
815	* is taken. It pairs with smp_mb() inside the mutex critical section
816	* at (D).
817	*/
818	this_gen = smp_load_acquire(&lru_drain_gen);
819
820	/* It helps everyone if we do our own local drain immediately. */
821	lru_add_drain();
822
823	mutex_lock(&lock);
824
825	/*
826	* (C) Exit the draining operation if a newer generation, from another
827	* lru_add_drain_all(), was already scheduled for draining. Check (A).
828	*/
829	if (unlikely(this_gen != lru_drain_gen && !force_all_cpus))
830	goto done;
831
832	/*
833	* (D) Increment global generation number
834	*
835	* Pairs with smp_load_acquire() at (B), outside of the critical
836	* section. Use a full memory barrier to guarantee that the
837	* new global drain generation number is stored before loading
838	* folio_batch counters.
839	*
840	* This pairing must be done here, before the for_each_online_cpu loop
841	* below which drains the page vectors.
842	*
843	* Let x, y, and z represent some system CPU numbers, where x < y < z.
844	* Assume CPU #z is in the middle of the for_each_online_cpu loop
845	* below and has already reached CPU #y's per-cpu data. CPU #x comes
846	* along, adds some pages to its per-cpu vectors, then calls
847	* lru_add_drain_all().
848	*
849	* If the paired barrier is done at any later step, e.g. after the
850	* loop, CPU #x will just exit at (C) and miss flushing out all of its
851	* added pages.
852	*/
853	WRITE_ONCE(lru_drain_gen, lru_drain_gen + 1);
854	smp_mb();
855
856	cpumask_clear(&has_work);
857	for_each_online_cpu(cpu) {
858	struct work_struct *work = &per_cpu(lru_add_drain_work, cpu);
859
860	if (cpu_needs_drain(cpu)) {
861	INIT_WORK(work, lru_add_drain_per_cpu);
862	queue_work_on(cpu, mm_percpu_wq, work);
863	__cpumask_set_cpu(cpu, &has_work);
864	}
865	}
866
867	for_each_cpu(cpu, &has_work)
868	flush_work(&per_cpu(lru_add_drain_work, cpu));
869
870	done:
871	mutex_unlock(&lock);
872	}
873
874	void lru_add_drain_all(void)
875	{
876	__lru_add_drain_all(false);
877	}
878	#else
879	void lru_add_drain_all(void)
880	{
881	lru_add_drain();
882	}
883	#endif /* CONFIG_SMP */
884
885	/**
886	* lru_cache_drain_for_folio() - drain LRU caches if the caches might hold
887	* folio references
888	* @folio: The folio.
889	* @extra_refs: Extra folio references held by the caller.
890	* @drained: Drain status for batch folio processing.
891	*
892	* Drain LRU caches if the caches might hold folio references. Start
893	* with a local LRU cache drain, to then drain LRU caches on all CPUs if
894	* local draining was insufficient.
895	*
896	* This function detects LRU cache references by comparing the folio refcount
897	* with the sum of the expected folio refcount + extra references held by the
898	* caller. Note that we cannot rely on PG_lru to reliably detect all LRU
899	* cache references, and there are rare scenarios (concurrent folio (un)mapping)
900	* where this function might miss detecting LRU cache references.
901	*
902	* If @drained is not NULL, the function will avoid re-draining LRU caches
903	* when processing multiple folios in a row. In that case, the variable
904	* @drained points at must be initialized to LRU_CACHE_NOT_DRAINED before
905	* the first invocation by the caller.
906	*/
907	void lru_cache_drain_for_folio(const struct folio *folio,
908	unsigned int extra_refs, enum lru_cache_drained *drained)
909	{
910	if (!folio_may_be_lru_cached(folio))
911	return;
912
913	if (!drained \|\| *drained == LRU_CACHE_NOT_DRAINED) {
914	if (folio_ref_count(folio) ==
915	folio_expected_ref_count(folio) + extra_refs)
916	return;
917	lru_add_drain();
918	if (drained)
919	*drained = LRU_CACHE_DRAINED;
920	}
921	if (!drained \|\| *drained == LRU_CACHE_DRAINED) {
922	if (folio_ref_count(folio) ==
923	folio_expected_ref_count(folio) + extra_refs)
924	return;
925	lru_add_drain_all();
926	if (drained)
927	*drained = LRU_CACHE_DRAINED_ALL;
928	}
929	}
930	EXPORT_SYMBOL_FOR_KVM(lru_cache_drain_for_folio);
931
932	atomic_t lru_disable_count = ATOMIC_INIT(0);
933
934	/*
935	* lru_cache_disable() needs to be called before we start compiling
936	* a list of folios to be migrated using folio_isolate_lru().
937	* It drains folios on LRU cache and then disable on all cpus until
938	* lru_cache_enable is called.
939	*
940	* Must be paired with a call to lru_cache_enable().
941	*/
942	void lru_cache_disable(void)
943	{
944	atomic_inc(&lru_disable_count);
945	/*
946	* Readers of lru_disable_count are protected by either disabling
947	* preemption or rcu_read_lock:
948	*
949	* preempt_disable, local_irq_disable [bh_lru_lock()]
950	* rcu_read_lock [rt_spin_lock CONFIG_PREEMPT_RT]
951	* preempt_disable [local_lock !CONFIG_PREEMPT_RT]
952	*
953	* Since v5.1 kernel, synchronize_rcu() is guaranteed to wait on
954	* preempt_disable() regions of code. So any CPU which sees
955	* lru_disable_count = 0 will have exited the critical
956	* section when synchronize_rcu() returns.
957	*/
958	synchronize_rcu_expedited();
959	#ifdef CONFIG_SMP
960	__lru_add_drain_all(true);
961	#else
962	lru_add_and_bh_lrus_drain();
963	#endif
964	}
965
966	/**
967	* folios_put_refs - Reduce the reference count on a batch of folios.
968	* @folios: The folios.
969	* @refs: The number of refs to subtract from each folio.
970	*
971	* Like folio_put(), but for a batch of folios. This is more efficient
972	* than writing the loop yourself as it will optimise the locks which need
973	* to be taken if the folios are freed. The folios batch is returned
974	* empty and ready to be reused for another batch; there is no need
975	* to reinitialise it. If @refs is NULL, we subtract one from each
976	* folio refcount.
977	*
978	* Context: May be called in process or interrupt context, but not in NMI
979	* context. May be called while holding a spinlock.
980	*/
981	void folios_put_refs(struct folio_batch folios, unsigned int refs)
982	{
983	int i, j;
984	struct lruvec *lruvec = NULL;
985	unsigned long flags = 0;
986
987	for (i = 0, j = 0; i < folios->nr; i++) {
988	struct folio *folio = folios->folios[i];
989	unsigned int nr_refs = refs ? refs[i] : 1;
990
991	/* Folio batch entry may have been preemptively removed during drain. */
992	if (!folio)
993	continue;
994
995	if (is_huge_zero_folio(folio))
996	continue;
997
998	if (folio_is_zone_device(folio)) {
999	if (lruvec) {
1000	lruvec_unlock_irqrestore(lruvec, flags);
1001	lruvec = NULL;
1002	}
1003	if (folio_ref_sub_and_test(folio, nr_refs))
1004	free_zone_device_folio(folio);
1005	continue;
1006	}
1007
1008	if (!folio_ref_sub_and_test(folio, nr_refs))
1009	continue;
1010
1011	/* hugetlb has its own memcg */
1012	if (folio_test_hugetlb(folio)) {
1013	if (lruvec) {
1014	lruvec_unlock_irqrestore(lruvec, flags);
1015	lruvec = NULL;
1016	}
1017	free_huge_folio(folio);
1018	continue;
1019	}
1020	folio_unqueue_deferred_split(folio);
1021	__page_cache_release(folio, &lruvec, &flags);
1022
1023	if (j != i)
1024	folios->folios[j] = folio;
1025	j++;
1026	}
1027	if (lruvec)
1028	lruvec_unlock_irqrestore(lruvec, flags);
1029	if (!j) {
1030	folio_batch_reinit(folios);
1031	return;
1032	}
1033
1034	folios->nr = j;
1035	mem_cgroup_uncharge_folios(folios);
1036	free_unref_folios(folios);
1037	}
1038	EXPORT_SYMBOL(folios_put_refs);
1039
1040	/**
1041	* release_pages - batched put_page()
1042	* @arg: array of pages to release
1043	* @nr: number of pages
1044	*
1045	* Decrement the reference count on all the pages in @arg. If it
1046	* fell to zero, remove the page from the LRU and free it.
1047	*
1048	* Note that the argument can be an array of pages, encoded pages,
1049	* or folio pointers. We ignore any encoded bits, and turn any of
1050	* them into just a folio that gets free'd.
1051	*/
1052	void release_pages(release_pages_arg arg, int nr)
1053	{
1054	struct folio_batch fbatch;
1055	int refs[FOLIO_BATCH_SIZE];
1056	struct encoded_page **encoded = arg.encoded_pages;
1057	int i;
1058
1059	folio_batch_init(&fbatch);
1060	for (i = 0; i < nr; i++) {
1061	/* Turn any of the argument types into a folio */
1062	struct folio *folio = page_folio(encoded_page_ptr(encoded[i]));
1063
1064	/* Is our next entry actually "nr_pages" -> "nr_refs" ? */
1065	refs[fbatch.nr] = 1;
1066	if (unlikely(encoded_page_flags(encoded[i]) &
1067	ENCODED_PAGE_BIT_NR_PAGES_NEXT))
1068	refs[fbatch.nr] = encoded_nr_pages(encoded[++i]);
1069
1070	if (folio_batch_add(&fbatch, folio) > 0)
1071	continue;
1072	folios_put_refs(&fbatch, refs);
1073	}
1074
1075	if (fbatch.nr)
1076	folios_put_refs(&fbatch, refs);
1077	}
1078	EXPORT_SYMBOL(release_pages);
1079
1080	/*
1081	* The folios which we're about to release may be in the deferred lru-addition
1082	* queues. That would prevent them from really being freed right now. That's
1083	* OK from a correctness point of view but is inefficient - those folios may be
1084	* cache-warm and we want to give them back to the page allocator ASAP.
1085	*
1086	* So __folio_batch_release() will drain those queues here.
1087	* folio_batch_move_lru() calls folios_put() directly to avoid
1088	* mutual recursion.
1089	*/
1090	void __folio_batch_release(struct folio_batch *fbatch)
1091	{
1092	if (!fbatch->percpu_pvec_drained) {
1093	lru_add_drain();
1094	fbatch->percpu_pvec_drained = true;
1095	}
1096	folios_put(fbatch);
1097	}
1098	EXPORT_SYMBOL(__folio_batch_release);
1099
1100	/**
1101	* folio_batch_remove_exceptionals() - Prune non-folios from a batch.
1102	* @fbatch: The batch to prune
1103	*
1104	* find_get_entries() fills a batch with both folios and shadow/swap/DAX
1105	* entries. This function prunes all the non-folio entries from @fbatch
1106	* without leaving holes, so that it can be passed on to folio-only batch
1107	* operations.
1108	*/
1109	void folio_batch_remove_exceptionals(struct folio_batch *fbatch)
1110	{
1111	unsigned int i, j;
1112
1113	for (i = 0, j = 0; i < folio_batch_count(fbatch); i++) {
1114	struct folio *folio = fbatch->folios[i];
1115	if (!xa_is_value(folio))
1116	fbatch->folios[j++] = folio;
1117	}
1118	fbatch->nr = j;
1119	}
1120
1121	#ifdef CONFIG_MEMCG
1122	static void lruvec_reparent_lru(struct lruvec *child_lruvec,
1123	struct lruvec *parent_lruvec,
1124	enum lru_list lru, int nid)
1125	{
1126	int zid;
1127	struct zone *zone;
1128
1129	if (lru != LRU_UNEVICTABLE)
1130	list_splice_tail_init(&child_lruvec->lists[lru], &parent_lruvec->lists[lru]);
1131
1132	for_each_managed_zone_pgdat(zone, NODE_DATA(nid), zid, MAX_NR_ZONES - 1) {
1133	unsigned long size = mem_cgroup_get_zone_lru_size(child_lruvec, lru, zid);
1134
1135	if (!size)
1136	continue;
1137
1138	/*
1139	* The folios are accounted to the parent from now on, so the
1140	* size has to be moved, not just copied. Leaving it behind
1141	* makes the dying child describe folios it no longer owns.
1142	*/
1143	mem_cgroup_update_lru_size(parent_lruvec, lru, zid, size);
1144	mem_cgroup_update_lru_size(child_lruvec, lru, zid, -(long)size);
1145	}
1146	}
1147
1148	void lru_reparent_memcg(struct mem_cgroup memcg, struct mem_cgroup parent, int nid)
1149	{
1150	enum lru_list lru;
1151	struct lruvec child_lruvec, parent_lruvec;
1152
1153	child_lruvec = mem_cgroup_lruvec(memcg, NODE_DATA(nid));
1154	parent_lruvec = mem_cgroup_lruvec(parent, NODE_DATA(nid));
1155
1156	for_each_lru(lru)
1157	lruvec_reparent_lru(child_lruvec, parent_lruvec, lru, nid);
1158	}
1159	#endif